Source code

001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.demos.video.videosift;
031
032import java.awt.event.KeyEvent;
033import java.awt.event.KeyListener;
034import java.util.ArrayList;
035import java.util.List;
036
037import javax.swing.SwingUtilities;
038
039import org.openimaj.demos.video.utils.PolygonDrawingListener;
040import org.openimaj.demos.video.utils.PolygonExtractionProcessor;
041import org.openimaj.feature.local.list.LocalFeatureList;
042import org.openimaj.feature.local.matcher.FastBasicKeypointMatcher;
043import org.openimaj.feature.local.matcher.MatchingUtilities;
044import org.openimaj.feature.local.matcher.consistent.ConsistentLocalFeatureMatcher2d;
045import org.openimaj.image.FImage;
046import org.openimaj.image.MBFImage;
047import org.openimaj.image.colour.RGBColour;
048import org.openimaj.image.colour.Transforms;
049import org.openimaj.image.feature.local.engine.DoGSIFTEngine;
050import org.openimaj.image.feature.local.keypoints.Keypoint;
051import org.openimaj.image.processing.resize.ResizeProcessor;
052import org.openimaj.image.processing.transform.ProjectionProcessor;
053import org.openimaj.image.renderer.MBFImageRenderer;
054import org.openimaj.math.geometry.point.Point2d;
055import org.openimaj.math.geometry.point.Point2dImpl;
056import org.openimaj.math.geometry.shape.Polygon;
057import org.openimaj.math.geometry.shape.Shape;
058import org.openimaj.math.geometry.transforms.HomographyModel;
059import org.openimaj.math.geometry.transforms.HomographyRefinement;
060import org.openimaj.math.geometry.transforms.MatrixTransformProvider;
061import org.openimaj.math.geometry.transforms.TransformUtilities;
062import org.openimaj.math.geometry.transforms.estimation.RobustHomographyEstimator;
063import org.openimaj.math.geometry.transforms.residuals.SingleImageTransferResidual2d;
064import org.openimaj.math.model.fit.RANSAC;
065import org.openimaj.util.pair.IndependentPair;
066import org.openimaj.video.VideoDisplay;
067import org.openimaj.video.VideoDisplayListener;
068import org.openimaj.video.capture.VideoCapture;
069import org.openimaj.video.tracking.klt.Feature;
070import org.openimaj.video.tracking.klt.FeatureList;
071import org.openimaj.video.tracking.klt.KLTTracker;
072import org.openimaj.video.tracking.klt.TrackingContext;
073
074import Jama.Matrix;
075
076public class VideoKLTSIFT implements KeyListener, VideoDisplayListener<MBFImage> {
077        enum Mode {
078                TRACKING, LOOKING, NONE, START_LOOKING
079        }
080
081        private VideoCapture capture;
082        private VideoDisplay<MBFImage> videoFrame;
083        private KLTTracker tracker;
084        private FeatureList fl;
085
086        private FImage oldFrame;
087        private int frameNumber = 0;
088        private int nFeatures = 50;
089        private int nOriginalFoundFeatures = -1;
090        private DoGSIFTEngine engine;
091        private PolygonDrawingListener polygonListener;
092        private Mode mode = Mode.NONE;
093        private FeatureList initialFeatures;
094        private Polygon initialShape;
095        private ConsistentLocalFeatureMatcher2d<Keypoint> siftMatcher;
096        private MBFImage modelImage;
097        private MBFImage overlayFrame = null;
098
099        public VideoKLTSIFT() throws Exception {
100                capture = new VideoCapture(640, 480);
101                polygonListener = new PolygonDrawingListener();
102                videoFrame = VideoDisplay.createVideoDisplay(capture);
103                videoFrame.getScreen().addMouseListener(polygonListener);
104                // videoFrame.getScreen().setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
105                videoFrame.addVideoListener(this);
106                SwingUtilities.getRoot(videoFrame.getScreen()).addKeyListener(this);
107
108                reinitTracker();
109        }
110
111        @Override
112        public void afterUpdate(VideoDisplay<MBFImage> arg0) {
113                // TODO Auto-generated method stub
114
115        }
116
117        @Override
118        public synchronized void beforeUpdate(MBFImage image) {
119                this.polygonListener.drawPoints(image);
120                if (videoFrame.isPaused())
121                {
122                        return;
123                }
124                FImage greyFrame = null;
125                greyFrame = Transforms.calculateIntensityNTSC(image);
126                // If we are in looking mode, Use matcher to find a likely position
127                // every 5th frame
128                if (this.mode == Mode.LOOKING) {
129
130                        final Shape shape = findObject(greyFrame);
131                        if (shape == null) {
132                                this.oldFrame = greyFrame;
133                                return;
134                        }
135                        System.out.println("Object FOUND, switcihg to tracking mode");
136                        // If we find a likely position, init the tracker, we are now
137                        // tracking
138                        initTracking(greyFrame, shape);
139                        this.mode = Mode.TRACKING;
140                }
141                // If we are tracking, attempt to track the points every frame
142                else if (this.mode == Mode.TRACKING) {
143
144                        continueTracking(greyFrame);
145                        // If we don't track enough points, look again.
146                        if (fl.countRemainingFeatures() == 0 || fl.countRemainingFeatures() < nOriginalFoundFeatures * 0.2)
147                        {
148                                System.out.println("Object LOST, switching to LOOKING mode");
149                                this.mode = Mode.LOOKING;
150                                reinitTracker();
151                        }
152                        // else if(fl.countRemainingFeatures() < nOriginalFoundFeatures *
153                        // 0.8){
154                        // initTracking(greyFrame,polygonToDraw);
155                        // }
156                }
157                else if (this.mode == Mode.START_LOOKING) {
158                        this.reinitTracker();
159                        final Polygon p = this.polygonListener.getPolygon().clone();
160                        this.polygonListener.reset();
161                        final MBFImage modelImage = capture.getCurrentFrame();
162                        this.initTracking(greyFrame, p);
163                        this.initObjectFinder(modelImage, p);
164                        this.oldFrame = greyFrame;
165                        mode = Mode.LOOKING;
166                        return;
167                }
168
169                if (overlayFrame != null) {
170                        drawOverlay(image);
171                }
172                else {
173                        drawDebug(image, greyFrame);
174                }
175
176                this.oldFrame = greyFrame;
177
178        }
179
180        private void drawOverlay(MBFImage image) {
181                final ProjectionProcessor<Float[], MBFImage> proc = new ProjectionProcessor<Float[], MBFImage>();
182                image.accumulateWith(proc);
183                final Matrix model = this.estimateModel();
184                if (model != null) {
185                        proc.setMatrix(model);
186                        this.overlayFrame.accumulateWith(proc);
187                }
188                image.internalAssign(proc.performProjection());
189        }
190
191        private void drawDebug(MBFImage image, FImage greyFrame) {
192                this.polygonListener.drawPoints(image);
193
194                final MBFImageRenderer renderer = image.createRenderer();
195
196                if (this.initialShape != null) {
197                        renderer.drawPolygon(initialShape, RGBColour.RED);
198                }
199                if (this.initialFeatures != null) {
200                        image.internalAssign(MatchingUtilities.drawMatches(image, this.findAllMatchedPairs(), RGBColour.WHITE));
201                        final Matrix esitmatedModel = this.estimateModel();
202                        if (esitmatedModel != null)
203                        {
204                                final Polygon newPolygon = initialShape.transform(esitmatedModel);
205                                renderer.drawPolygon(newPolygon, RGBColour.GREEN);
206                                if (fl.countRemainingFeatures() < nOriginalFoundFeatures * 0.5) {
207                                        reinitTracker();
208                                        initTracking(greyFrame, newPolygon);
209                                }
210                        }
211                        estimateMovement();
212                }
213        }
214
215        private Shape findObject(FImage capImg) {
216                final float scaleImage = .5f;
217
218                final ResizeProcessor resize = new ResizeProcessor(scaleImage);
219                capImg = capImg.process(resize);
220                Shape sh = null;
221                if (siftMatcher != null && !videoFrame.isPaused() && engine != null) {
222                        final LocalFeatureList<Keypoint> kpl = engine.findFeatures(capImg);
223                        if (siftMatcher.findMatches(kpl)) {
224                                Matrix shTransform = ((MatrixTransformProvider) siftMatcher.getModel()).getTransform().copy();
225                                if (shTransform != null)
226                                {
227                                        try {
228                                                shTransform = TransformUtilities.scaleMatrix(1f / scaleImage, 1f / scaleImage).times(
229                                                                shTransform.inverse());
230                                                sh = modelImage.getBounds().transform(shTransform);
231                                        } catch (final Exception e) {
232                                                e.printStackTrace();
233                                        }
234                                }
235
236                        }
237
238                }
239                return sh;
240
241        }
242
243        private void reinitTracker() {
244                final TrackingContext tc = new TrackingContext();
245                fl = new FeatureList(nFeatures);
246                tracker = new KLTTracker(tc, fl);
247                tracker.setVerbosity(0);
248
249                tc.setSequentialMode(true);
250                tc.setWriteInternalImages(false);
251                tc.setAffineConsistencyCheck(-1); /*
252                                                                                 * set this to 2 to turn on affine
253                                                                                 * consistency check
254                                                                                 */
255                this.initialFeatures = null;
256                this.initialShape = null;
257        }
258
259        public void initTracking(FImage greyFrame, Shape location) {
260                frameNumber = 0;
261                tracker.getTrackingContext().setTargetArea(location);
262                tracker.selectGoodFeatures(greyFrame);
263                nOriginalFoundFeatures = fl.countRemainingFeatures();
264                initialFeatures = fl.clone();
265                initialShape = location.asPolygon().clone();
266        }
267
268        public void continueTracking(FImage greyFrame) {
269                tracker.trackFeatures(oldFrame, greyFrame);
270                this.frameNumber++;
271        }
272
273        private Matrix estimateModel() {
274                if (this.initialFeatures == null) {
275                        return null;
276                }
277                final List<? extends IndependentPair<Point2d, Point2d>> pairs = findAllMatchedPairs();
278                final HomographyModel model = new HomographyModel();
279                // model.estimate(pairs);
280                final RANSAC<Point2d, Point2d, HomographyModel> fitter = new RANSAC<Point2d, Point2d, HomographyModel>(model,
281                                new SingleImageTransferResidual2d<HomographyModel>(),
282                                10.0, 1500,
283                                new RANSAC.PercentageInliersStoppingCondition(0.5), false);
284                if (!fitter.fitData(pairs))
285                        return null;
286
287                model.getTransform().print(5, 5);
288                return model.getTransform();
289        }
290
291        private List<IndependentPair<Point2d, Point2d>> findAllMatchedPairs() {
292                final List<IndependentPair<Point2d, Point2d>> pairs = new ArrayList<IndependentPair<Point2d, Point2d>>();
293                for (int i = 0; i < this.initialFeatures.features.length; i++) {
294                        final Feature oldFeature = this.initialFeatures.features[i].clone();
295                        final Feature newFeature = fl.features[i].clone();
296                        if (oldFeature.val >= 0 && newFeature.val >= 0) {
297                                pairs.add(new IndependentPair<Point2d, Point2d>(oldFeature, newFeature));
298                        }
299                }
300                return pairs;
301        }
302
303        public Point2dImpl estimateMovement() {
304                final Feature[] oldFeatures = this.initialFeatures.features;
305                float sumX = 0;
306                float sumY = 0;
307                float total = 0;
308                if (oldFeatures != null) {
309                        for (int i = 0; i < oldFeatures.length; i++) {
310                                final Feature oldFeature = oldFeatures[i];
311                                final Feature newFeature = fl.features[i];
312                                if (oldFeature.val >= 0 && newFeature.val >= 0) {
313                                        sumX += newFeature.x - oldFeature.x;
314                                        sumY += newFeature.y - oldFeature.y;
315                                        total += 1f;
316                                }
317                        }
318                        sumX /= total;
319                        sumY /= total;
320                        System.out.println("Average displacement: " + sumX + "," + sumY);
321                }
322                return new Point2dImpl(sumX, sumY);
323        }
324
325        @Override
326        public void keyTyped(KeyEvent e) {
327
328        }
329
330        @Override
331        public synchronized void keyPressed(KeyEvent key) {
332                if (key.getKeyCode() == KeyEvent.VK_SPACE) {
333                        this.videoFrame.togglePause();
334                } else if (key.getKeyChar() == 'c' && this.polygonListener.getPolygon().getVertices().size() > 2) {
335                        mode = Mode.START_LOOKING;
336                } else if (key.getKeyChar() == 'd' && this.polygonListener.getPolygon().getVertices().size() > 2) {
337                        final Polygon p = this.polygonListener.getPolygon().clone();
338                        this.polygonListener.reset();
339                        overlayFrame = this.capture.getCurrentFrame().process(
340                                        new PolygonExtractionProcessor<Float[], MBFImage>(p, RGBColour.BLACK));
341                }
342                else if (key.getKeyChar() == 'r') {
343                        this.mode = Mode.NONE;
344
345                }
346        }
347
348        private void initObjectFinder(MBFImage frame, Polygon p) {
349                modelImage = frame.process(new PolygonExtractionProcessor<Float[], MBFImage>(p, RGBColour.BLACK));
350
351                // configure the matcher
352                siftMatcher = new ConsistentLocalFeatureMatcher2d<Keypoint>(new FastBasicKeypointMatcher<Keypoint>(8));
353                siftMatcher.setFittingModel(new RobustHomographyEstimator(10.0, 1500,
354                                new RANSAC.PercentageInliersStoppingCondition(0.5), HomographyRefinement.NONE));
355
356                engine = new DoGSIFTEngine();
357                engine.getOptions().setDoubleInitialImage(true);
358
359                final FImage modelF = Transforms.calculateIntensityNTSC(modelImage);
360                siftMatcher.setModelFeatures(engine.findFeatures(modelF));
361        }
362
363        @Override
364        public void keyReleased(KeyEvent e) {
365                // TODO Auto-generated method stub
366
367        }
368
369        public static void main(String args[]) throws Exception {
370                new VideoKLTSIFT();
371        }
372
373}