001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.demos.video; 031 032import java.awt.event.KeyEvent; 033import java.awt.event.KeyListener; 034 035import javax.swing.SwingUtilities; 036 037import org.openimaj.demos.video.utils.PolygonDrawingListener; 038import org.openimaj.demos.video.utils.PolygonExtractionProcessor; 039import org.openimaj.feature.local.list.LocalFeatureList; 040import org.openimaj.feature.local.matcher.FastBasicKeypointMatcher; 041import org.openimaj.feature.local.matcher.consistent.ConsistentLocalFeatureMatcher2d; 042import org.openimaj.image.FImage; 043import org.openimaj.image.MBFImage; 044import org.openimaj.image.colour.RGBColour; 045import org.openimaj.image.colour.Transforms; 046import org.openimaj.image.feature.local.engine.DoGSIFTEngine; 047import org.openimaj.image.feature.local.keypoints.Keypoint; 048import org.openimaj.math.geometry.shape.Polygon; 049import org.openimaj.math.geometry.transforms.HomographyRefinement; 050import org.openimaj.math.geometry.transforms.MatrixTransformProvider; 051import org.openimaj.math.geometry.transforms.estimation.RobustHomographyEstimator; 052import org.openimaj.math.model.fit.RANSAC; 053import org.openimaj.video.VideoDisplay; 054import org.openimaj.video.VideoDisplayListener; 055 056public class CaptureVideoSIFT implements KeyListener, VideoDisplayListener<MBFImage> { 057 058 private VideoWithinVideo vwv; 059 private PolygonDrawingListener polygonListener; 060 private DoGSIFTEngine engine; 061 private VideoDisplay<MBFImage> videoFrame; 062 private MBFImage modelImage; 063 private ConsistentLocalFeatureMatcher2d<Keypoint> matcher; 064 private boolean ransacReader = false; 065 066 public CaptureVideoSIFT(VideoWithinVideo videoWithinVideo) { 067 this.vwv = videoWithinVideo; 068 polygonListener = new PolygonDrawingListener(); 069 this.vwv.display.getScreen().addMouseListener(polygonListener); 070 SwingUtilities.getRoot(this.vwv.display.getScreen()).addKeyListener(this); 071 engine = new DoGSIFTEngine(); 072 engine.getOptions().setDoubleInitialImage(false); 073 074 this.videoFrame = VideoDisplay.createOffscreenVideoDisplay(vwv.capture); 075 this.videoFrame.addVideoListener(this); 076 } 077 078 @Override 079 public void keyPressed(KeyEvent key) { 080 if (key.getKeyCode() == KeyEvent.VK_SPACE) { 081 this.videoFrame.togglePause(); 082 } 083 else if (key.getKeyChar() == 'r') { 084 vwv.display.seek(0); 085 } 086 else if (key.getKeyChar() == 'c' && this.polygonListener.getPolygon().getVertices().size() > 2) { 087 try { 088 ransacReader = false; 089 final Polygon p = this.polygonListener.getPolygon().clone(); 090 this.polygonListener.reset(); 091 modelImage = this.vwv.capture.getCurrentFrame().process( 092 new PolygonExtractionProcessor<Float[], MBFImage>(p, RGBColour.BLACK)); 093 094 // configure the matcher 095 matcher = new ConsistentLocalFeatureMatcher2d<Keypoint>(new FastBasicKeypointMatcher<Keypoint>(8)); 096 matcher.setFittingModel(new RobustHomographyEstimator(3.0, 1500, 097 new RANSAC.PercentageInliersStoppingCondition(0.01), HomographyRefinement.NONE)); 098 099 final DoGSIFTEngine engine = new DoGSIFTEngine(); 100 engine.getOptions().setDoubleInitialImage(false); 101 102 final FImage modelF = Transforms.calculateIntensityNTSC(modelImage); 103 matcher.setModelFeatures(engine.findFeatures(modelF)); 104 vwv.display.seek(0); 105 ransacReader = true; 106 107 } catch (final Exception e) { 108 e.printStackTrace(); 109 } 110 } 111 } 112 113 @Override 114 public void keyReleased(KeyEvent arg0) { 115 } 116 117 @Override 118 public void keyTyped(KeyEvent arg0) { 119 } 120 121 @Override 122 public void afterUpdate(VideoDisplay<MBFImage> display) { 123 if (ransacReader && matcher != null && !videoFrame.isPaused()) { 124 final MBFImage capImg = videoFrame.getVideo().getCurrentFrame(); 125 final LocalFeatureList<Keypoint> kpl = engine.findFeatures(Transforms.calculateIntensityNTSC(capImg)); 126 if (matcher.findMatches(kpl)) { 127 try { 128 final Polygon poly = modelImage.getBounds() 129 .transform(((MatrixTransformProvider) matcher.getModel()).getTransform().inverse()) 130 .asPolygon(); 131 132 this.vwv.targetArea = poly; 133 } catch (final RuntimeException e) { 134 } 135 136 } else { 137 this.vwv.targetArea = null; 138 } 139 } 140 } 141 142 @Override 143 public void beforeUpdate(MBFImage frame) { 144 final MBFImage frameWrite = frame; 145 this.polygonListener.drawPoints(frameWrite); 146 this.vwv.copyToCaptureFrame(frameWrite); 147 148 } 149 150}