001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.demos.video;
031
032import java.awt.event.KeyEvent;
033import java.awt.event.KeyListener;
034
035import javax.swing.SwingUtilities;
036
037import org.openimaj.demos.video.utils.PolygonDrawingListener;
038import org.openimaj.demos.video.utils.PolygonExtractionProcessor;
039import org.openimaj.feature.local.list.LocalFeatureList;
040import org.openimaj.feature.local.matcher.FastBasicKeypointMatcher;
041import org.openimaj.feature.local.matcher.consistent.ConsistentLocalFeatureMatcher2d;
042import org.openimaj.image.FImage;
043import org.openimaj.image.MBFImage;
044import org.openimaj.image.colour.RGBColour;
045import org.openimaj.image.colour.Transforms;
046import org.openimaj.image.feature.local.engine.DoGSIFTEngine;
047import org.openimaj.image.feature.local.keypoints.Keypoint;
048import org.openimaj.math.geometry.shape.Polygon;
049import org.openimaj.math.geometry.transforms.HomographyRefinement;
050import org.openimaj.math.geometry.transforms.MatrixTransformProvider;
051import org.openimaj.math.geometry.transforms.estimation.RobustHomographyEstimator;
052import org.openimaj.math.model.fit.RANSAC;
053import org.openimaj.video.VideoDisplay;
054import org.openimaj.video.VideoDisplayListener;
055
056public class CaptureVideoSIFT implements KeyListener, VideoDisplayListener<MBFImage> {
057
058        private VideoWithinVideo vwv;
059        private PolygonDrawingListener polygonListener;
060        private DoGSIFTEngine engine;
061        private VideoDisplay<MBFImage> videoFrame;
062        private MBFImage modelImage;
063        private ConsistentLocalFeatureMatcher2d<Keypoint> matcher;
064        private boolean ransacReader = false;
065
066        public CaptureVideoSIFT(VideoWithinVideo videoWithinVideo) {
067                this.vwv = videoWithinVideo;
068                polygonListener = new PolygonDrawingListener();
069                this.vwv.display.getScreen().addMouseListener(polygonListener);
070                SwingUtilities.getRoot(this.vwv.display.getScreen()).addKeyListener(this);
071                engine = new DoGSIFTEngine();
072                engine.getOptions().setDoubleInitialImage(false);
073
074                this.videoFrame = VideoDisplay.createOffscreenVideoDisplay(vwv.capture);
075                this.videoFrame.addVideoListener(this);
076        }
077
078        @Override
079        public void keyPressed(KeyEvent key) {
080                if (key.getKeyCode() == KeyEvent.VK_SPACE) {
081                        this.videoFrame.togglePause();
082                }
083                else if (key.getKeyChar() == 'r') {
084                        vwv.display.seek(0);
085                }
086                else if (key.getKeyChar() == 'c' && this.polygonListener.getPolygon().getVertices().size() > 2) {
087                        try {
088                                ransacReader = false;
089                                final Polygon p = this.polygonListener.getPolygon().clone();
090                                this.polygonListener.reset();
091                                modelImage = this.vwv.capture.getCurrentFrame().process(
092                                                new PolygonExtractionProcessor<Float[], MBFImage>(p, RGBColour.BLACK));
093
094                                // configure the matcher
095                                matcher = new ConsistentLocalFeatureMatcher2d<Keypoint>(new FastBasicKeypointMatcher<Keypoint>(8));
096                                matcher.setFittingModel(new RobustHomographyEstimator(3.0, 1500,
097                                                new RANSAC.PercentageInliersStoppingCondition(0.01), HomographyRefinement.NONE));
098
099                                final DoGSIFTEngine engine = new DoGSIFTEngine();
100                                engine.getOptions().setDoubleInitialImage(false);
101
102                                final FImage modelF = Transforms.calculateIntensityNTSC(modelImage);
103                                matcher.setModelFeatures(engine.findFeatures(modelF));
104                                vwv.display.seek(0);
105                                ransacReader = true;
106
107                        } catch (final Exception e) {
108                                e.printStackTrace();
109                        }
110                }
111        }
112
113        @Override
114        public void keyReleased(KeyEvent arg0) {
115        }
116
117        @Override
118        public void keyTyped(KeyEvent arg0) {
119        }
120
121        @Override
122        public void afterUpdate(VideoDisplay<MBFImage> display) {
123                if (ransacReader && matcher != null && !videoFrame.isPaused()) {
124                        final MBFImage capImg = videoFrame.getVideo().getCurrentFrame();
125                        final LocalFeatureList<Keypoint> kpl = engine.findFeatures(Transforms.calculateIntensityNTSC(capImg));
126                        if (matcher.findMatches(kpl)) {
127                                try {
128                                        final Polygon poly = modelImage.getBounds()
129                                                        .transform(((MatrixTransformProvider) matcher.getModel()).getTransform().inverse())
130                                                        .asPolygon();
131
132                                        this.vwv.targetArea = poly;
133                                } catch (final RuntimeException e) {
134                                }
135
136                        } else {
137                                this.vwv.targetArea = null;
138                        }
139                }
140        }
141
142        @Override
143        public void beforeUpdate(MBFImage frame) {
144                final MBFImage frameWrite = frame;
145                this.polygonListener.drawPoints(frameWrite);
146                this.vwv.copyToCaptureFrame(frameWrite);
147
148        }
149
150}