001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.demos.video; 031 032import java.awt.Dimension; 033import java.awt.GridBagConstraints; 034import java.awt.GridBagLayout; 035import java.awt.Insets; 036import java.awt.event.KeyEvent; 037import java.awt.event.KeyListener; 038import java.io.IOException; 039 040import javax.swing.BorderFactory; 041import javax.swing.JComponent; 042import javax.swing.JFrame; 043import javax.swing.JLabel; 044import javax.swing.JPanel; 045import javax.swing.SwingUtilities; 046 047import org.openimaj.demos.Demo; 048import org.openimaj.demos.video.utils.PolygonDrawingListener; 049import org.openimaj.demos.video.utils.PolygonExtractionProcessor; 050import org.openimaj.feature.local.list.LocalFeatureList; 051import org.openimaj.feature.local.matcher.FastBasicKeypointMatcher; 052import org.openimaj.feature.local.matcher.MatchingUtilities; 053import org.openimaj.feature.local.matcher.consistent.ConsistentLocalFeatureMatcher2d; 054import org.openimaj.image.DisplayUtilities.ImageComponent; 055import org.openimaj.image.ImageUtilities; 056import org.openimaj.image.MBFImage; 057import org.openimaj.image.colour.RGBColour; 058import org.openimaj.image.feature.local.engine.DoGColourSIFTEngine; 059import org.openimaj.image.feature.local.keypoints.Keypoint; 060import org.openimaj.image.processing.transform.MBFProjectionProcessor; 061import org.openimaj.image.renderer.MBFImageRenderer; 062import org.openimaj.math.geometry.point.Point2d; 063import org.openimaj.math.geometry.shape.Polygon; 064import org.openimaj.math.geometry.shape.Rectangle; 065import org.openimaj.math.geometry.transforms.HomographyModel; 066import org.openimaj.math.geometry.transforms.MatrixTransformProvider; 067import org.openimaj.math.geometry.transforms.TransformUtilities; 068import org.openimaj.math.geometry.transforms.residuals.SingleImageTransferResidual2d; 069import org.openimaj.math.model.fit.RANSAC; 070import org.openimaj.video.VideoDisplay; 071import org.openimaj.video.VideoDisplayListener; 072import org.openimaj.video.capture.VideoCapture; 073import org.openimaj.video.xuggle.XuggleVideo; 074 075import Jama.Matrix; 076 077/** 078 * OpenIMAJ Real-time (ish) SIFT tracking and matching demo 079 * 080 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk) 081 * @author Sina Samangooei (ss@ecs.soton.ac.uk) 082 */ 083@Demo( 084 author = "Jonathon Hare and Sina Samangooei", 085 description = "Realtime-ish SIFT-based tracking demonstration." + 086 "Hold an object in front of the camera, and press space. Select" + 087 "the outline of the object by clicking points on the frozen video " + 088 "image, and press C when you're done. Press space to start the video " + 089 "again, and the object should be tracked. This demo uses a homography " + 090 "to constrain the matches.", 091 keywords = { "video", "sift", "object tracking" }, 092 title = "VideoColourSIFT") 093public class VideoColourSIFT implements KeyListener, VideoDisplayListener<MBFImage> { 094 enum RenderMode { 095 SQUARE { 096 @Override 097 public void render(final MBFImageRenderer renderer, final Matrix transform, final Rectangle rectangle) { 098 renderer.drawShape(rectangle.transform(transform), 3, RGBColour.BLUE); 099 } 100 }, 101 PICTURE { 102 MBFImage toRender = null; 103 private Matrix renderToBounds; 104 105 @Override 106 public void render(final MBFImageRenderer renderer, final Matrix transform, final Rectangle rectangle) { 107 if (this.toRender == null) { 108 try { 109 this.toRender = ImageUtilities.readMBF(VideoColourSIFT.class 110 .getResource("/org/openimaj/demos/OpenIMAJ.png")); 111 } catch (final IOException e) { 112 System.err.println("Can't load image to render"); 113 } 114 this.renderToBounds = TransformUtilities.makeTransform(this.toRender.getBounds(), rectangle); 115 } 116 117 final MBFProjectionProcessor mbfPP = new MBFProjectionProcessor(); 118 mbfPP.setMatrix(transform.times(this.renderToBounds)); 119 mbfPP.accumulate(this.toRender); 120 mbfPP.performProjection(0, 0, renderer.getImage()); 121 122 } 123 }, 124 VIDEO { 125 private XuggleVideo toRender; 126 private Matrix renderToBounds; 127 128 @Override 129 public void render(final MBFImageRenderer renderer, final Matrix transform, final Rectangle rectangle) { 130 if (this.toRender == null) { 131 this.toRender = new XuggleVideo( 132 VideoColourSIFT.class.getResource("/org/openimaj/demos/video/keyboardcat.flv"), true); 133 this.renderToBounds = TransformUtilities.makeTransform(new Rectangle(0, 0, this.toRender.getWidth(), 134 this.toRender.getHeight()), rectangle); 135 } 136 137 final MBFProjectionProcessor mbfPP = new MBFProjectionProcessor(); 138 mbfPP.setMatrix(transform.times(this.renderToBounds)); 139 mbfPP.accumulate(this.toRender.getNextFrame()); 140 mbfPP.performProjection(0, 0, renderer.getImage()); 141 } 142 }; 143 public abstract void render(MBFImageRenderer renderer, Matrix transform, Rectangle rectangle); 144 } 145 146 private final VideoCapture capture; 147 private final VideoDisplay<MBFImage> videoFrame; 148 private final ImageComponent modelFrame; 149 private final ImageComponent matchFrame; 150 151 private MBFImage modelImage; 152 153 private ConsistentLocalFeatureMatcher2d<Keypoint> matcher; 154 private final DoGColourSIFTEngine engine; 155 private final PolygonDrawingListener polygonListener; 156 private final JPanel vidPanel; 157 private final JPanel modelPanel; 158 private final JPanel matchPanel; 159 private RenderMode renderMode = RenderMode.SQUARE; 160 161 /** 162 * Construct the demo 163 * 164 * @param window 165 * @throws Exception 166 */ 167 public VideoColourSIFT(final JComponent window) throws Exception { 168 this(window, new VideoCapture(320, 240)); 169 } 170 171 /** 172 * Construct the demo 173 * 174 * @param window 175 * @param capture 176 * @throws Exception 177 */ 178 public VideoColourSIFT(final JComponent window, final VideoCapture capture) throws Exception { 179 final int width = capture.getWidth(); 180 final int height = capture.getHeight(); 181 this.capture = capture; 182 this.polygonListener = new PolygonDrawingListener(); 183 184 GridBagConstraints gbc = new GridBagConstraints(); 185 186 final JLabel label = new JLabel( 187 "<html><body><p>Hold an object in front of the camera, and press space. Select<br/>" + 188 "the outline of the object by clicking points on the frozen video<br/>" + 189 "image, and press C when you're done. Press space to start the video<br/>" + 190 "again, and the object should be tracked.</p></body></html>"); 191 gbc.gridx = 0; 192 gbc.gridy = 0; 193 gbc.gridwidth = 2; 194 gbc.insets = new Insets(8, 8, 8, 8); 195 window.add(label, gbc); 196 197 this.vidPanel = new JPanel(new GridBagLayout()); 198 this.vidPanel.setBorder(BorderFactory.createTitledBorder("Live Video")); 199 this.videoFrame = VideoDisplay.createVideoDisplay(capture, this.vidPanel); 200 gbc = new GridBagConstraints(); 201 gbc.gridy = 1; 202 gbc.gridx = 0; 203 gbc.gridwidth = 1; 204 window.add(this.vidPanel, gbc); 205 206 this.modelPanel = new JPanel(new GridBagLayout()); 207 this.modelPanel.setBorder(BorderFactory.createTitledBorder("Model")); 208 this.modelFrame = new ImageComponent(true, false); 209 this.modelFrame.setSize(width, height); 210 this.modelFrame.setPreferredSize(new Dimension(width, height)); 211 this.modelPanel.add(this.modelFrame); 212 gbc = new GridBagConstraints(); 213 gbc.anchor = GridBagConstraints.PAGE_START; 214 gbc.gridy = 1; 215 gbc.gridx = 1; 216 window.add(this.modelPanel, gbc); 217 218 this.matchPanel = new JPanel(new GridBagLayout()); 219 this.matchPanel.setBorder(BorderFactory.createTitledBorder("Matches")); 220 this.matchFrame = new ImageComponent(true, false); 221 this.matchFrame.setSize(width * 2, height); 222 this.matchFrame.setPreferredSize(new Dimension(width * 2, height)); 223 this.matchPanel.add(this.matchFrame); 224 gbc = new GridBagConstraints(); 225 gbc.anchor = GridBagConstraints.PAGE_END; 226 gbc.gridy = 2; 227 gbc.gridx = 0; 228 gbc.gridwidth = 2; 229 window.add(this.matchPanel, gbc); 230 231 this.videoFrame.getScreen().addMouseListener(this.polygonListener); 232 233 this.videoFrame.addVideoListener(this); 234 this.engine = new DoGColourSIFTEngine(); 235 this.engine.getOptions().setDoubleInitialImage(false); 236 } 237 238 @Override 239 public synchronized void keyPressed(final KeyEvent key) { 240 if (key.getKeyCode() == KeyEvent.VK_SPACE) { 241 this.videoFrame.togglePause(); 242 } else if (key.getKeyChar() == 'c' && this.polygonListener.getPolygon().getVertices().size() > 2) { 243 try { 244 final Polygon p = this.polygonListener.getPolygon().clone(); 245 this.polygonListener.reset(); 246 this.modelImage = this.capture.getCurrentFrame().process( 247 new PolygonExtractionProcessor<Float[], MBFImage>(p, RGBColour.BLACK)); 248 249 if (this.matcher == null) { 250 // configure the matcher 251 final HomographyModel model = new HomographyModel(); 252 final RANSAC<Point2d, Point2d, HomographyModel> ransac = new RANSAC<Point2d, Point2d, HomographyModel>( 253 model, new SingleImageTransferResidual2d<HomographyModel>(), 254 3.0, 1500, new RANSAC.ProbabilisticMinInliersStoppingCondition(0.01), true); 255 this.matcher = new ConsistentLocalFeatureMatcher2d<Keypoint>( 256 new FastBasicKeypointMatcher<Keypoint>(8)); 257 this.matcher.setFittingModel(ransac); 258 259 this.modelPanel.setPreferredSize(this.modelPanel.getSize()); 260 } 261 262 this.modelFrame.setImage(ImageUtilities.createBufferedImageForDisplay(this.modelImage)); 263 264 final DoGColourSIFTEngine engine = new DoGColourSIFTEngine(); 265 engine.getOptions().setDoubleInitialImage(true); 266 267 this.matcher.setModelFeatures(engine.findFeatures(this.modelImage)); 268 } catch (final Exception e) { 269 e.printStackTrace(); 270 } 271 } else if (key.getKeyChar() == '1') { 272 this.renderMode = RenderMode.SQUARE; 273 } else if (key.getKeyChar() == '2') { 274 this.renderMode = RenderMode.PICTURE; 275 } else if (key.getKeyChar() == '3') { 276 this.renderMode = RenderMode.VIDEO; 277 } 278 } 279 280 @Override 281 public void keyReleased(final KeyEvent arg0) { 282 } 283 284 @Override 285 public void keyTyped(final KeyEvent arg0) { 286 } 287 288 @Override 289 public synchronized void afterUpdate(final VideoDisplay<MBFImage> display) { 290 if (this.matcher != null && !this.videoFrame.isPaused()) { 291 final MBFImage capImg = this.videoFrame.getVideo().getCurrentFrame(); 292 final LocalFeatureList<Keypoint> kpl = this.engine.findFeatures(capImg); 293 294 final MBFImageRenderer renderer = capImg.createRenderer(); 295 renderer.drawPoints(kpl, RGBColour.MAGENTA, 3); 296 297 MBFImage matches; 298 if (this.matcher.findMatches(kpl)) { 299 try { 300 // Shape sh = 301 // modelImage.getBounds().transform(((MatrixTransformProvider) 302 // matcher.getModel()).getTransform().inverse()); 303 // renderer.drawShape(sh, 3, RGBColour.BLUE); 304 final Matrix boundsToPoly = ((MatrixTransformProvider) this.matcher.getModel()).getTransform() 305 .inverse(); 306 this.renderMode.render(renderer, boundsToPoly, this.modelImage.getBounds()); 307 } catch (final RuntimeException e) { 308 } 309 310 matches = MatchingUtilities 311 .drawMatches(this.modelImage, capImg, this.matcher.getMatches(), RGBColour.RED); 312 } else { 313 matches = MatchingUtilities 314 .drawMatches(this.modelImage, capImg, this.matcher.getMatches(), RGBColour.RED); 315 } 316 317 this.matchPanel.setPreferredSize(this.matchPanel.getSize()); 318 this.matchFrame.setImage(ImageUtilities.createBufferedImageForDisplay(matches)); 319 } 320 } 321 322 @Override 323 public void beforeUpdate(final MBFImage frame) { 324 this.polygonListener.drawPoints(frame); 325 } 326 327 /** 328 * Stop capture 329 */ 330 public void stop() { 331 this.videoFrame.close(); 332 this.capture.stopCapture(); 333 } 334 335 /** 336 * Main method 337 * 338 * @param args 339 * ignored 340 * @throws Exception 341 */ 342 public static void main(final String[] args) throws Exception { 343 final JFrame window = new JFrame(); 344 window.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); 345 346 window.setLayout(new GridBagLayout()); 347 final JPanel c = new JPanel(); 348 c.setLayout(new GridBagLayout()); 349 window.getContentPane().add(c); 350 351 final VideoColourSIFT vs = new VideoColourSIFT(c); 352 SwingUtilities.getRoot(window).addKeyListener(vs); 353 window.pack(); 354 window.setVisible(true); 355 } 356 357}