001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.demos.video.videosift; 031 032import java.awt.Point; 033import java.awt.event.KeyEvent; 034import java.awt.event.KeyListener; 035 036import javax.swing.JFrame; 037import javax.swing.SwingUtilities; 038 039import org.openimaj.demos.video.utils.FeatureClickListener; 040import org.openimaj.demos.video.utils.PolygonDrawingListener; 041import org.openimaj.demos.video.utils.PolygonExtractionProcessor; 042import org.openimaj.feature.local.list.LocalFeatureList; 043import org.openimaj.feature.local.matcher.FastBasicKeypointMatcher; 044import org.openimaj.feature.local.matcher.MatchingUtilities; 045import org.openimaj.feature.local.matcher.consistent.ConsistentLocalFeatureMatcher2d; 046import org.openimaj.image.DisplayUtilities; 047import org.openimaj.image.FImage; 048import org.openimaj.image.MBFImage; 049import org.openimaj.image.colour.RGBColour; 050import org.openimaj.image.colour.Transforms; 051import org.openimaj.image.feature.local.engine.ipd.FinderMode; 052import org.openimaj.image.feature.local.engine.ipd.IPDSIFTEngine; 053import org.openimaj.image.feature.local.interest.HarrisIPD; 054import org.openimaj.image.feature.local.interest.IPDSelectionMode; 055import org.openimaj.image.feature.local.interest.InterestPointData; 056import org.openimaj.image.feature.local.interest.InterestPointVisualiser; 057import org.openimaj.image.feature.local.keypoints.InterestPointKeypoint; 058import org.openimaj.image.feature.local.keypoints.KeypointVisualizer; 059import org.openimaj.image.renderer.MBFImageRenderer; 060import org.openimaj.math.geometry.shape.Polygon; 061import org.openimaj.math.geometry.shape.Shape; 062import org.openimaj.math.geometry.transforms.HomographyRefinement; 063import org.openimaj.math.geometry.transforms.MatrixTransformProvider; 064import org.openimaj.math.geometry.transforms.estimation.RobustHomographyEstimator; 065import org.openimaj.math.model.fit.RANSAC; 066import org.openimaj.video.VideoDisplay; 067import org.openimaj.video.VideoDisplayListener; 068import org.openimaj.video.capture.VideoCapture; 069 070/** 071 * OpenIMAJ Real-time (ish) SIFT tracking and matching demo 072 * 073 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk) 074 * @author Sina Samangooei (ss@ecs.soton.ac.uk) 075 */ 076public class VideoIPD implements KeyListener, VideoDisplayListener<MBFImage> { 077 private VideoCapture capture; 078 private VideoDisplay<MBFImage> videoFrame; 079 private JFrame modelFrame; 080 private JFrame matchFrame; 081 private MBFImage modelImage; 082 083 private ConsistentLocalFeatureMatcher2d<InterestPointKeypoint<InterestPointData>> matcher; 084 private IPDSIFTEngine engine; 085 private PolygonDrawingListener polygonListener; 086 private FeatureClickListener<Float[], MBFImage> featureClickListener; 087 088 public VideoIPD() throws Exception { 089 090 engine = getNewEngine(); 091 092 capture = new VideoCapture(320, 240); 093 polygonListener = new PolygonDrawingListener(); 094 videoFrame = VideoDisplay.createVideoDisplay(capture); 095 SwingUtilities.getRoot(videoFrame.getScreen()).addKeyListener(this); 096 videoFrame.getScreen().addMouseListener(polygonListener); 097 this.featureClickListener = new FeatureClickListener<Float[], MBFImage>(); 098 videoFrame.getScreen().addMouseListener(featureClickListener); 099 // videoFrame.getScreen().setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); 100 videoFrame.addVideoListener(this); 101 102 } 103 104 private IPDSIFTEngine getNewEngine() { 105 final int derScale = 1; 106 final int intScale = 3; 107 final HarrisIPD ipd = new HarrisIPD(derScale, intScale); 108 engine = new IPDSIFTEngine(ipd); 109 engine.setSelectionMode(new IPDSelectionMode.Threshold(10000f)); 110 engine.setFinderMode(new FinderMode.Characteristic<InterestPointData>()); 111 // engine.setSelectionMode(new IPDSelectionMode.Count(10)); 112 engine.setAcrossScales(true); 113 114 return engine; 115 } 116 117 @Override 118 public void keyPressed(KeyEvent key) { 119 if (key.getKeyCode() == KeyEvent.VK_SPACE) { 120 this.videoFrame.togglePause(); 121 } else if (key.getKeyChar() == 'c' && this.polygonListener.getPolygon().getVertices().size() > 2) { 122 try { 123 final Polygon p = this.polygonListener.getPolygon().clone(); 124 this.polygonListener.reset(); 125 modelImage = capture.getCurrentFrame().process( 126 new PolygonExtractionProcessor<Float[], MBFImage>(p, RGBColour.BLACK)); 127 128 if (modelFrame == null) { 129 modelFrame = DisplayUtilities.display(modelImage, "model"); 130 modelFrame.addKeyListener(this); 131 132 // move the frame 133 final Point pt = modelFrame.getLocation(); 134 modelFrame.setLocation(pt.x + this.videoFrame.getScreen().getWidth(), pt.y); 135 136 // configure the matcher 137 matcher = new ConsistentLocalFeatureMatcher2d<InterestPointKeypoint<InterestPointData>>( 138 new FastBasicKeypointMatcher<InterestPointKeypoint<InterestPointData>>(8)); 139 matcher.setFittingModel(new RobustHomographyEstimator(10.0, 1500, 140 new RANSAC.PercentageInliersStoppingCondition(0.5), HomographyRefinement.NONE)); 141 } else { 142 DisplayUtilities.display(modelImage, modelFrame); 143 } 144 145 final FImage modelF = Transforms.calculateIntensityNTSC(modelImage); 146 matcher.setModelFeatures(getNewEngine().findFeatures(modelF)); 147 } catch (final Exception e) { 148 e.printStackTrace(); 149 } 150 } 151 } 152 153 @Override 154 public void keyReleased(KeyEvent arg0) { 155 } 156 157 @Override 158 public void keyTyped(KeyEvent arg0) { 159 } 160 161 public static void main(String[] args) throws Exception { 162 new VideoIPD(); 163 } 164 165 @Override 166 public void afterUpdate(VideoDisplay<MBFImage> display) { 167 if (matcher != null && !videoFrame.isPaused()) { 168 final MBFImage capImg = videoFrame.getVideo().getCurrentFrame(); 169 final LocalFeatureList<InterestPointKeypoint<InterestPointData>> kpl = engine.findFeatures(Transforms 170 .calculateIntensityNTSC(capImg)); 171 172 final MBFImageRenderer renderer = capImg.createRenderer(); 173 renderer.drawPoints(kpl, RGBColour.MAGENTA, 3); 174 175 MBFImage matches; 176 if (matcher.findMatches(kpl)) { 177 try { 178 final Shape sh = modelImage.getBounds().transform( 179 ((MatrixTransformProvider) matcher.getModel()).getTransform().inverse()); 180 renderer.drawShape(sh, 3, RGBColour.BLUE); 181 } catch (final RuntimeException e) { 182 } 183 184 matches = MatchingUtilities.drawMatches(modelImage, capImg, matcher.getMatches(), RGBColour.RED); 185 } else { 186 matches = MatchingUtilities.drawMatches(modelImage, capImg, null, RGBColour.RED); 187 } 188 189 if (matchFrame == null) { 190 matchFrame = DisplayUtilities.display(matches, "matches"); 191 matchFrame.addKeyListener(this); 192 193 final Point pt = matchFrame.getLocation(); 194 matchFrame.setLocation(pt.x, pt.y + matchFrame.getHeight()); 195 } else { 196 DisplayUtilities.display(matches, matchFrame); 197 } 198 } 199 } 200 201 @Override 202 public void beforeUpdate(MBFImage frame) { 203 drawKeypoints(frame); 204 } 205 206 private void drawKeypoints(MBFImage frame) { 207 final MBFImage capImg = frame; 208 final LocalFeatureList<InterestPointKeypoint<InterestPointData>> kpl = engine.findFeatures(Transforms 209 .calculateIntensityNTSC(capImg)); 210 this.featureClickListener.setImage(kpl, frame.clone()); 211 final KeypointVisualizer<Float[], MBFImage> kpv = new KeypointVisualizer<Float[], MBFImage>(capImg, kpl); 212 final InterestPointVisualiser<Float[], MBFImage> ipv = InterestPointVisualiser.visualiseKeypoints( 213 kpv.drawPatches(null, RGBColour.GREEN), kpl); 214 frame.internalAssign(ipv.drawPatches(RGBColour.GREEN, RGBColour.BLUE)); 215 } 216}