001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.demos.video; 031 032import java.awt.Dimension; 033import java.awt.GridBagConstraints; 034import java.awt.GridBagLayout; 035import java.awt.Insets; 036import java.awt.event.KeyEvent; 037import java.awt.event.KeyListener; 038import java.io.IOException; 039 040import javax.swing.BorderFactory; 041import javax.swing.JComponent; 042import javax.swing.JFrame; 043import javax.swing.JLabel; 044import javax.swing.JPanel; 045import javax.swing.SwingUtilities; 046 047import org.openimaj.demos.Demo; 048import org.openimaj.demos.video.utils.PolygonDrawingListener; 049import org.openimaj.demos.video.utils.PolygonExtractionProcessor; 050import org.openimaj.feature.local.list.LocalFeatureList; 051import org.openimaj.feature.local.matcher.FastBasicKeypointMatcher; 052import org.openimaj.feature.local.matcher.MatchingUtilities; 053import org.openimaj.feature.local.matcher.consistent.ConsistentLocalFeatureMatcher2d; 054import org.openimaj.image.DisplayUtilities.ImageComponent; 055import org.openimaj.image.FImage; 056import org.openimaj.image.ImageUtilities; 057import org.openimaj.image.MBFImage; 058import org.openimaj.image.colour.RGBColour; 059import org.openimaj.image.colour.Transforms; 060import org.openimaj.image.feature.local.engine.DoGSIFTEngine; 061import org.openimaj.image.feature.local.keypoints.Keypoint; 062import org.openimaj.image.processing.transform.MBFProjectionProcessor; 063import org.openimaj.image.renderer.MBFImageRenderer; 064import org.openimaj.math.geometry.shape.Polygon; 065import org.openimaj.math.geometry.shape.Rectangle; 066import org.openimaj.math.geometry.transforms.HomographyModel; 067import org.openimaj.math.geometry.transforms.HomographyRefinement; 068import org.openimaj.math.geometry.transforms.MatrixTransformProvider; 069import org.openimaj.math.geometry.transforms.TransformUtilities; 070import org.openimaj.math.geometry.transforms.check.TransformMatrixConditionCheck; 071import org.openimaj.math.geometry.transforms.estimation.RobustHomographyEstimator; 072import org.openimaj.math.model.fit.RANSAC; 073import org.openimaj.video.VideoDisplay; 074import org.openimaj.video.VideoDisplayListener; 075import org.openimaj.video.capture.VideoCapture; 076import org.openimaj.video.xuggle.XuggleVideo; 077 078import Jama.Matrix; 079 080/** 081 * OpenIMAJ Real-time (ish) SIFT tracking and matching demo 082 * 083 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk) 084 * @author Sina Samangooei (ss@ecs.soton.ac.uk) 085 */ 086@Demo( 087 author = "Jonathon Hare and Sina Samangooei", 088 description = "Realtime-ish SIFT-based tracking demonstration." + 089 "Hold an object in front of the camera, and press space. Select" + 090 "the outline of the object by clicking points on the frozen video " + 091 "image, and press C when you're done. Press space to start the video " + 092 "again, and the object should be tracked. This demo uses a homography " + 093 "to constrain the matches.", 094 keywords = { "video", "sift", "object tracking" }, 095 title = "VideoSIFT") 096public class VideoSIFT implements KeyListener, VideoDisplayListener<MBFImage> { 097 enum RenderMode { 098 SQUARE { 099 @Override 100 public void render(final MBFImageRenderer renderer, final Matrix transform, final Rectangle rectangle) { 101 renderer.drawShape(rectangle.transform(transform), 3, RGBColour.BLUE); 102 } 103 }, 104 PICTURE { 105 MBFImage toRender = null; 106 private Matrix renderToBounds; 107 108 @Override 109 public void render(final MBFImageRenderer renderer, final Matrix transform, final Rectangle rectangle) { 110 if (this.toRender == null) { 111 try { 112 this.toRender = ImageUtilities.readMBF(VideoSIFT.class 113 .getResource("/org/openimaj/demos/OpenIMAJ.png")); 114 } catch (final IOException e) { 115 System.err.println("Can't load image to render"); 116 } 117 this.renderToBounds = TransformUtilities.makeTransform(this.toRender.getBounds(), rectangle); 118 } 119 120 final MBFProjectionProcessor mbfPP = new MBFProjectionProcessor(); 121 mbfPP.setMatrix(transform.times(this.renderToBounds)); 122 mbfPP.accumulate(this.toRender); 123 mbfPP.performProjection(0, 0, renderer.getImage()); 124 125 } 126 }, 127 VIDEO { 128 private XuggleVideo toRender; 129 private Matrix renderToBounds; 130 131 @Override 132 public void render(final MBFImageRenderer renderer, final Matrix transform, final Rectangle rectangle) { 133 if (this.toRender == null) { 134 this.toRender = new XuggleVideo( 135 VideoSIFT.class.getResource("/org/openimaj/demos/video/keyboardcat.flv"), true); 136 this.renderToBounds = TransformUtilities.makeTransform(new Rectangle(0, 0, this.toRender.getWidth(), 137 this.toRender.getHeight()), rectangle); 138 } 139 140 final MBFProjectionProcessor mbfPP = new MBFProjectionProcessor(); 141 mbfPP.setMatrix(transform.times(this.renderToBounds)); 142 mbfPP.accumulate(this.toRender.getNextFrame()); 143 mbfPP.performProjection(0, 0, renderer.getImage()); 144 } 145 }; 146 public abstract void render(MBFImageRenderer renderer, Matrix transform, Rectangle rectangle); 147 } 148 149 private final VideoCapture capture; 150 private final VideoDisplay<MBFImage> videoFrame; 151 private final ImageComponent modelFrame; 152 private final ImageComponent matchFrame; 153 154 private MBFImage modelImage; 155 156 private ConsistentLocalFeatureMatcher2d<Keypoint> matcher; 157 private final DoGSIFTEngine engine; 158 private final PolygonDrawingListener polygonListener; 159 private final JPanel vidPanel; 160 private final JPanel modelPanel; 161 private final JPanel matchPanel; 162 private RenderMode renderMode = RenderMode.SQUARE; 163 private MBFImage currentFrame; 164 165 /** 166 * Construct the demo 167 * 168 * @param window 169 * @throws Exception 170 */ 171 public VideoSIFT(final JComponent window) throws Exception { 172 this(window, new VideoCapture(320, 240)); 173 } 174 175 /** 176 * Construct the demo 177 * 178 * @param window 179 * @param capture 180 * @throws Exception 181 */ 182 public VideoSIFT(final JComponent window, final VideoCapture capture) throws Exception { 183 final int width = capture.getWidth(); 184 final int height = capture.getHeight(); 185 this.capture = capture; 186 this.polygonListener = new PolygonDrawingListener(); 187 188 GridBagConstraints gbc = new GridBagConstraints(); 189 190 final JLabel label = new JLabel( 191 "<html><body><p>Hold an object in front of the camera, and press space. Select<br/>" + 192 "the outline of the object by clicking points on the frozen video<br/>" + 193 "image, and press C when you're done. Press space to start the video<br/>" + 194 "again, and the object should be tracked.</p></body></html>"); 195 gbc.gridx = 0; 196 gbc.gridy = 0; 197 gbc.gridwidth = 2; 198 gbc.insets = new Insets(8, 8, 8, 8); 199 window.add(label, gbc); 200 201 this.vidPanel = new JPanel(new GridBagLayout()); 202 this.vidPanel.setBorder(BorderFactory.createTitledBorder("Live Video")); 203 this.videoFrame = VideoDisplay.createVideoDisplay(capture, this.vidPanel); 204 gbc = new GridBagConstraints(); 205 gbc.gridy = 1; 206 gbc.gridx = 0; 207 gbc.gridwidth = 1; 208 window.add(this.vidPanel, gbc); 209 210 this.modelPanel = new JPanel(new GridBagLayout()); 211 this.modelPanel.setBorder(BorderFactory.createTitledBorder("Model")); 212 this.modelFrame = new ImageComponent(true, false); 213 this.modelFrame.setShowPixelColours(false); 214 this.modelFrame.setShowXYPosition(false); 215 this.modelFrame.removeMouseListener(this.modelFrame); 216 this.modelFrame.removeMouseMotionListener(this.modelFrame); 217 this.modelFrame.setSize(width, height); 218 this.modelFrame.setPreferredSize(new Dimension(width, height)); 219 this.modelPanel.add(this.modelFrame); 220 gbc = new GridBagConstraints(); 221 gbc.anchor = GridBagConstraints.PAGE_START; 222 gbc.gridy = 1; 223 gbc.gridx = 1; 224 window.add(this.modelPanel, gbc); 225 226 this.matchPanel = new JPanel(new GridBagLayout()); 227 this.matchPanel.setBorder(BorderFactory.createTitledBorder("Matches")); 228 this.matchFrame = new ImageComponent(true, false); 229 this.matchFrame.setShowPixelColours(false); 230 this.matchFrame.setShowXYPosition(false); 231 this.matchFrame.removeMouseListener(this.matchFrame); 232 this.matchFrame.removeMouseMotionListener(this.matchFrame); 233 this.matchFrame.setSize(width * 2, height); 234 this.matchFrame.setPreferredSize(new Dimension(width * 2, height)); 235 this.matchPanel.add(this.matchFrame); 236 gbc = new GridBagConstraints(); 237 gbc.anchor = GridBagConstraints.PAGE_END; 238 gbc.gridy = 2; 239 gbc.gridx = 0; 240 gbc.gridwidth = 2; 241 window.add(this.matchPanel, gbc); 242 243 this.videoFrame.getScreen().addMouseListener(this.polygonListener); 244 245 this.videoFrame.addVideoListener(this); 246 this.engine = new DoGSIFTEngine(); 247 this.engine.getOptions().setDoubleInitialImage(true); 248 } 249 250 @Override 251 public synchronized void keyPressed(final KeyEvent key) { 252 if (key.getKeyCode() == KeyEvent.VK_SPACE) { 253 this.videoFrame.togglePause(); 254 } else if (key.getKeyChar() == 'c' && this.polygonListener.getPolygon().getVertices().size() > 2) { 255 try { 256 final Polygon p = this.polygonListener.getPolygon().clone(); 257 this.polygonListener.reset(); 258 this.modelImage = this.currentFrame.process( 259 new PolygonExtractionProcessor<Float[], MBFImage>(p, RGBColour.BLACK)); 260 261 if (this.matcher == null) { 262 final RobustHomographyEstimator ransac = new RobustHomographyEstimator(0.5, 1500, 263 new RANSAC.PercentageInliersStoppingCondition(0.6), HomographyRefinement.NONE, 264 new TransformMatrixConditionCheck<HomographyModel>(10000)); 265 266 this.matcher = new ConsistentLocalFeatureMatcher2d<Keypoint>( 267 new FastBasicKeypointMatcher<Keypoint>(8)); 268 this.matcher.setFittingModel(ransac); 269 270 this.modelPanel.setPreferredSize(this.modelPanel.getSize()); 271 } 272 273 this.modelFrame.setImage(ImageUtilities.createBufferedImageForDisplay(this.modelImage)); 274 275 final DoGSIFTEngine engine = new DoGSIFTEngine(); 276 engine.getOptions().setDoubleInitialImage(true); 277 278 // final ASIFTEngine engine = new ASIFTEngine(true); 279 280 final FImage modelF = Transforms.calculateIntensityNTSC(this.modelImage); 281 this.matcher.setModelFeatures(engine.findFeatures(modelF)); 282 } catch (final Exception e) { 283 e.printStackTrace(); 284 } 285 } else if (key.getKeyChar() == '1') { 286 this.renderMode = RenderMode.SQUARE; 287 } else if (key.getKeyChar() == '2') { 288 this.renderMode = RenderMode.PICTURE; 289 } else if (key.getKeyChar() == '3') { 290 this.renderMode = RenderMode.VIDEO; 291 } 292 } 293 294 @Override 295 public void keyReleased(final KeyEvent arg0) { 296 } 297 298 @Override 299 public void keyTyped(final KeyEvent arg0) { 300 } 301 302 @Override 303 public synchronized void afterUpdate(final VideoDisplay<MBFImage> display) { 304 if (this.matcher != null && !this.videoFrame.isPaused()) { 305 final MBFImage capImg = this.currentFrame; 306 final LocalFeatureList<Keypoint> kpl = this.engine.findFeatures(Transforms.calculateIntensityNTSC(capImg)); 307 308 final MBFImageRenderer renderer = capImg.createRenderer(); 309 renderer.drawPoints(kpl, RGBColour.MAGENTA, 3); 310 311 MBFImage matches; 312 if (this.matcher.findMatches(kpl) 313 && ((MatrixTransformProvider) this.matcher.getModel()).getTransform().cond() < 1e6) 314 { 315 try { 316 final Matrix boundsToPoly = ((MatrixTransformProvider) this.matcher.getModel()).getTransform() 317 .inverse(); 318 319 if (modelImage.getBounds().transform(boundsToPoly).isConvex()) { 320 this.renderMode.render(renderer, boundsToPoly, this.modelImage.getBounds()); 321 } 322 } catch (final RuntimeException e) { 323 } 324 325 matches = MatchingUtilities 326 .drawMatches(this.modelImage, capImg, this.matcher.getMatches(), RGBColour.RED); 327 } else { 328 matches = MatchingUtilities 329 .drawMatches(this.modelImage, capImg, this.matcher.getMatches(), RGBColour.RED); 330 } 331 332 this.matchPanel.setPreferredSize(this.matchPanel.getSize()); 333 this.matchFrame.setImage(ImageUtilities.createBufferedImageForDisplay(matches)); 334 } 335 } 336 337 @Override 338 public void beforeUpdate(final MBFImage frame) { 339 if (!this.videoFrame.isPaused()) 340 this.currentFrame = frame.clone(); 341 else { 342 frame.drawImage(currentFrame, 0, 0); 343 } 344 this.polygonListener.drawPoints(frame); 345 } 346 347 /** 348 * Stop capture 349 */ 350 public void stop() { 351 this.videoFrame.close(); 352 this.capture.stopCapture(); 353 } 354 355 /** 356 * Main method 357 * 358 * @param args 359 * ignored 360 * @throws Exception 361 */ 362 public static void main(final String[] args) throws Exception { 363 final JFrame window = new JFrame(); 364 window.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); 365 366 window.setLayout(new GridBagLayout()); 367 final JPanel c = new JPanel(); 368 c.setLayout(new GridBagLayout()); 369 window.getContentPane().add(c); 370 371 final VideoSIFT vs = new VideoSIFT(c); 372 SwingUtilities.getRoot(window).addKeyListener(vs); 373 window.pack(); 374 window.setVisible(true); 375 } 376}