001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.demos.video.videosift;
031
032import java.awt.Point;
033import java.awt.event.KeyEvent;
034import java.awt.event.KeyListener;
035
036import javax.swing.JFrame;
037import javax.swing.SwingUtilities;
038
039import org.openimaj.demos.video.utils.FeatureClickListener;
040import org.openimaj.demos.video.utils.PolygonDrawingListener;
041import org.openimaj.demos.video.utils.PolygonExtractionProcessor;
042import org.openimaj.feature.local.list.LocalFeatureList;
043import org.openimaj.feature.local.matcher.FastBasicKeypointMatcher;
044import org.openimaj.feature.local.matcher.MatchingUtilities;
045import org.openimaj.feature.local.matcher.consistent.ConsistentLocalFeatureMatcher2d;
046import org.openimaj.image.DisplayUtilities;
047import org.openimaj.image.FImage;
048import org.openimaj.image.MBFImage;
049import org.openimaj.image.colour.RGBColour;
050import org.openimaj.image.colour.Transforms;
051import org.openimaj.image.feature.local.engine.ipd.FinderMode;
052import org.openimaj.image.feature.local.engine.ipd.IPDSIFTEngine;
053import org.openimaj.image.feature.local.interest.HarrisIPD;
054import org.openimaj.image.feature.local.interest.IPDSelectionMode;
055import org.openimaj.image.feature.local.interest.InterestPointData;
056import org.openimaj.image.feature.local.interest.InterestPointVisualiser;
057import org.openimaj.image.feature.local.keypoints.InterestPointKeypoint;
058import org.openimaj.image.feature.local.keypoints.KeypointVisualizer;
059import org.openimaj.image.renderer.MBFImageRenderer;
060import org.openimaj.math.geometry.shape.Polygon;
061import org.openimaj.math.geometry.shape.Shape;
062import org.openimaj.math.geometry.transforms.HomographyRefinement;
063import org.openimaj.math.geometry.transforms.MatrixTransformProvider;
064import org.openimaj.math.geometry.transforms.estimation.RobustHomographyEstimator;
065import org.openimaj.math.model.fit.RANSAC;
066import org.openimaj.video.VideoDisplay;
067import org.openimaj.video.VideoDisplayListener;
068import org.openimaj.video.capture.VideoCapture;
069
070/**
071 * OpenIMAJ Real-time (ish) SIFT tracking and matching demo
072 * 
073 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
074 * @author Sina Samangooei (ss@ecs.soton.ac.uk)
075 */
076public class VideoIPD implements KeyListener, VideoDisplayListener<MBFImage> {
077        private VideoCapture capture;
078        private VideoDisplay<MBFImage> videoFrame;
079        private JFrame modelFrame;
080        private JFrame matchFrame;
081        private MBFImage modelImage;
082
083        private ConsistentLocalFeatureMatcher2d<InterestPointKeypoint<InterestPointData>> matcher;
084        private IPDSIFTEngine engine;
085        private PolygonDrawingListener polygonListener;
086        private FeatureClickListener<Float[], MBFImage> featureClickListener;
087
088        public VideoIPD() throws Exception {
089
090                engine = getNewEngine();
091
092                capture = new VideoCapture(320, 240);
093                polygonListener = new PolygonDrawingListener();
094                videoFrame = VideoDisplay.createVideoDisplay(capture);
095                SwingUtilities.getRoot(videoFrame.getScreen()).addKeyListener(this);
096                videoFrame.getScreen().addMouseListener(polygonListener);
097                this.featureClickListener = new FeatureClickListener<Float[], MBFImage>();
098                videoFrame.getScreen().addMouseListener(featureClickListener);
099                // videoFrame.getScreen().setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
100                videoFrame.addVideoListener(this);
101
102        }
103
104        private IPDSIFTEngine getNewEngine() {
105                final int derScale = 1;
106                final int intScale = 3;
107                final HarrisIPD ipd = new HarrisIPD(derScale, intScale);
108                engine = new IPDSIFTEngine(ipd);
109                engine.setSelectionMode(new IPDSelectionMode.Threshold(10000f));
110                engine.setFinderMode(new FinderMode.Characteristic<InterestPointData>());
111                // engine.setSelectionMode(new IPDSelectionMode.Count(10));
112                engine.setAcrossScales(true);
113
114                return engine;
115        }
116
117        @Override
118        public void keyPressed(KeyEvent key) {
119                if (key.getKeyCode() == KeyEvent.VK_SPACE) {
120                        this.videoFrame.togglePause();
121                } else if (key.getKeyChar() == 'c' && this.polygonListener.getPolygon().getVertices().size() > 2) {
122                        try {
123                                final Polygon p = this.polygonListener.getPolygon().clone();
124                                this.polygonListener.reset();
125                                modelImage = capture.getCurrentFrame().process(
126                                                new PolygonExtractionProcessor<Float[], MBFImage>(p, RGBColour.BLACK));
127
128                                if (modelFrame == null) {
129                                        modelFrame = DisplayUtilities.display(modelImage, "model");
130                                        modelFrame.addKeyListener(this);
131
132                                        // move the frame
133                                        final Point pt = modelFrame.getLocation();
134                                        modelFrame.setLocation(pt.x + this.videoFrame.getScreen().getWidth(), pt.y);
135
136                                        // configure the matcher
137                                        matcher = new ConsistentLocalFeatureMatcher2d<InterestPointKeypoint<InterestPointData>>(
138                                                        new FastBasicKeypointMatcher<InterestPointKeypoint<InterestPointData>>(8));
139                                        matcher.setFittingModel(new RobustHomographyEstimator(10.0, 1500,
140                                                        new RANSAC.PercentageInliersStoppingCondition(0.5), HomographyRefinement.NONE));
141                                } else {
142                                        DisplayUtilities.display(modelImage, modelFrame);
143                                }
144
145                                final FImage modelF = Transforms.calculateIntensityNTSC(modelImage);
146                                matcher.setModelFeatures(getNewEngine().findFeatures(modelF));
147                        } catch (final Exception e) {
148                                e.printStackTrace();
149                        }
150                }
151        }
152
153        @Override
154        public void keyReleased(KeyEvent arg0) {
155        }
156
157        @Override
158        public void keyTyped(KeyEvent arg0) {
159        }
160
161        public static void main(String[] args) throws Exception {
162                new VideoIPD();
163        }
164
165        @Override
166        public void afterUpdate(VideoDisplay<MBFImage> display) {
167                if (matcher != null && !videoFrame.isPaused()) {
168                        final MBFImage capImg = videoFrame.getVideo().getCurrentFrame();
169                        final LocalFeatureList<InterestPointKeypoint<InterestPointData>> kpl = engine.findFeatures(Transforms
170                                        .calculateIntensityNTSC(capImg));
171
172                        final MBFImageRenderer renderer = capImg.createRenderer();
173                        renderer.drawPoints(kpl, RGBColour.MAGENTA, 3);
174
175                        MBFImage matches;
176                        if (matcher.findMatches(kpl)) {
177                                try {
178                                        final Shape sh = modelImage.getBounds().transform(
179                                                        ((MatrixTransformProvider) matcher.getModel()).getTransform().inverse());
180                                        renderer.drawShape(sh, 3, RGBColour.BLUE);
181                                } catch (final RuntimeException e) {
182                                }
183
184                                matches = MatchingUtilities.drawMatches(modelImage, capImg, matcher.getMatches(), RGBColour.RED);
185                        } else {
186                                matches = MatchingUtilities.drawMatches(modelImage, capImg, null, RGBColour.RED);
187                        }
188
189                        if (matchFrame == null) {
190                                matchFrame = DisplayUtilities.display(matches, "matches");
191                                matchFrame.addKeyListener(this);
192
193                                final Point pt = matchFrame.getLocation();
194                                matchFrame.setLocation(pt.x, pt.y + matchFrame.getHeight());
195                        } else {
196                                DisplayUtilities.display(matches, matchFrame);
197                        }
198                }
199        }
200
201        @Override
202        public void beforeUpdate(MBFImage frame) {
203                drawKeypoints(frame);
204        }
205
206        private void drawKeypoints(MBFImage frame) {
207                final MBFImage capImg = frame;
208                final LocalFeatureList<InterestPointKeypoint<InterestPointData>> kpl = engine.findFeatures(Transforms
209                                .calculateIntensityNTSC(capImg));
210                this.featureClickListener.setImage(kpl, frame.clone());
211                final KeypointVisualizer<Float[], MBFImage> kpv = new KeypointVisualizer<Float[], MBFImage>(capImg, kpl);
212                final InterestPointVisualiser<Float[], MBFImage> ipv = InterestPointVisualiser.visualiseKeypoints(
213                                kpv.drawPatches(null, RGBColour.GREEN), kpl);
214                frame.internalAssign(ipv.drawPatches(RGBColour.GREEN, RGBColour.BLUE));
215        }
216}