001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.demos.acmmm11.presentation.slides; 031 032import java.awt.Component; 033import java.awt.Dimension; 034import java.awt.GridBagLayout; 035import java.awt.event.KeyEvent; 036import java.awt.event.KeyListener; 037import java.io.IOException; 038import java.util.List; 039 040import javax.swing.JPanel; 041 042import org.openimaj.content.slideshow.Slide; 043import org.openimaj.feature.local.list.LocalFeatureList; 044import org.openimaj.feature.local.matcher.FastBasicKeypointMatcher; 045import org.openimaj.feature.local.matcher.consistent.ConsistentLocalFeatureMatcher2d; 046import org.openimaj.image.DisplayUtilities.ImageComponent; 047import org.openimaj.image.FImage; 048import org.openimaj.image.ImageUtilities; 049import org.openimaj.image.MBFImage; 050import org.openimaj.image.colour.RGBColour; 051import org.openimaj.image.feature.local.engine.ALTDoGSIFTEngine; 052import org.openimaj.image.feature.local.engine.DoGSIFTEngine; 053import org.openimaj.image.feature.local.keypoints.Keypoint; 054import org.openimaj.image.processing.resize.ResizeProcessor; 055import org.openimaj.math.geometry.line.Line2d; 056import org.openimaj.math.geometry.point.Point2d; 057import org.openimaj.math.geometry.transforms.AffineTransformModel; 058import org.openimaj.math.geometry.transforms.TransformUtilities; 059import org.openimaj.math.geometry.transforms.residuals.SingleImageTransferResidual2d; 060import org.openimaj.math.model.fit.RANSAC; 061import org.openimaj.util.pair.Pair; 062import org.openimaj.video.VideoDisplay; 063import org.openimaj.video.VideoDisplayListener; 064 065import Jama.Matrix; 066 067/** 068 * Slide illustrating two sift implementation 069 * 070 * @author Sina Samangooei (ss@ecs.soton.ac.uk) 071 * 072 */ 073public class SIFTAltSIFTSlide implements Slide, VideoDisplayListener<MBFImage>, KeyListener { 074 075 private MBFImage outFrame; 076 private DoGSIFTEngine normalEngine; 077 private DoGSIFTEngine altEngine; 078 private FImage carpetGrey; 079 private MBFImage carpet; 080 private ConsistentLocalFeatureMatcher2d<Keypoint> normalmatcher; 081 private ConsistentLocalFeatureMatcher2d<Keypoint> altmatcher; 082 private ImageComponent ic; 083 private SpinningImageVideo spinning; 084 private VideoDisplay<MBFImage> display; 085 086 @Override 087 public Component getComponent(int width, int height) throws IOException { 088 carpet = ImageUtilities.readMBF(SIFTAltSIFTSlide.class.getResource("rabbit.jpeg")); 089 final double wh = Math.sqrt(carpet.getWidth() * carpet.getWidth() + carpet.getHeight() * carpet.getHeight()); 090 if (wh * 2 > Math.min(width, height)) { 091 final float prop = (float) (Math.min(width, height) / (wh * 2)); 092 carpet.processInplace(new ResizeProcessor(prop)); 093 } 094 095 this.carpetGrey = carpet.flatten(); 096 097 spinning = new SpinningImageVideo(carpet, -0.5f, 0.005f); 098 099 outFrame = new MBFImage(spinning.getWidth() * 2, spinning.getHeight() * 2, 3); 100 normalEngine = new DoGSIFTEngine(); 101 normalEngine.getOptions().setDoubleInitialImage(false); 102 altEngine = new ALTDoGSIFTEngine(); 103 altEngine.getOptions().setDoubleInitialImage(false); 104 105 final LocalFeatureList<Keypoint> carpetNormalKPTs = normalEngine.findFeatures(carpetGrey); 106 normalmatcher = new ConsistentLocalFeatureMatcher2d<Keypoint>( 107 new FastBasicKeypointMatcher<Keypoint>(8), 108 new RANSAC<Point2d, Point2d, AffineTransformModel>(new AffineTransformModel(), 109 new SingleImageTransferResidual2d<AffineTransformModel>(), 5.0, 100, 110 new RANSAC.ProbabilisticMinInliersStoppingCondition(0.01), true) 111 ); 112 normalmatcher.setModelFeatures(carpetNormalKPTs); 113 114 final LocalFeatureList<Keypoint> carpetAltKPTs = altEngine.findFeatures(carpetGrey); 115 altmatcher = new ConsistentLocalFeatureMatcher2d<Keypoint>( 116 new FastBasicKeypointMatcher<Keypoint>(8), 117 new RANSAC<Point2d, Point2d, AffineTransformModel>(new AffineTransformModel(), 118 new SingleImageTransferResidual2d<AffineTransformModel>(), 5.0, 100, 119 new RANSAC.ProbabilisticMinInliersStoppingCondition(0.01), true) 120 ); 121 altmatcher.setModelFeatures(carpetAltKPTs); 122 123 display = VideoDisplay.createOffscreenVideoDisplay(spinning); 124 display.addVideoListener(this); 125 126 final JPanel c = new JPanel(); 127 c.setOpaque(false); 128 c.setPreferredSize(new Dimension(width, height)); 129 c.setLayout(new GridBagLayout()); 130 ic = new ImageComponent(true, false); 131 c.add(ic); 132 for (final Component cc : c.getComponents()) { 133 if (cc instanceof JPanel) { 134 ((JPanel) cc).setOpaque(false); 135 } 136 } 137 return c; 138 } 139 140 @Override 141 public void close() { 142 this.spinning.stop(); 143 this.display.close(); 144 } 145 146 @Override 147 public void afterUpdate(VideoDisplay<MBFImage> display) { 148 // do nothing 149 } 150 151 @Override 152 public void beforeUpdate(MBFImage frame) { 153 outFrame.fill(RGBColour.BLACK); 154 final FImage fGrey = frame.flatten(); 155 final LocalFeatureList<Keypoint> normalKPTs = normalEngine.findFeatures(fGrey); 156 final LocalFeatureList<Keypoint> altKPTs = altEngine.findFeatures(fGrey); 157 158 final Matrix carpetOffset = TransformUtilities.translateMatrix(frame.getWidth() / 4, frame.getHeight() * 2 / 3); 159 final Matrix normalOffset = TransformUtilities.translateMatrix(frame.getWidth(), 0); 160 final Matrix altOffset = TransformUtilities.translateMatrix(frame.getWidth(), frame.getHeight()); 161 162 outFrame.drawImage(carpet, frame.getWidth() / 4, frame.getHeight() * 2 / 3); 163 outFrame.drawImage(frame, frame.getWidth(), 0); 164 outFrame.drawImage(frame, frame.getWidth(), frame.getHeight()); 165 166 if (normalmatcher.findMatches(normalKPTs)) 167 { 168 final List<Pair<Keypoint>> normalMatches = normalmatcher.getMatches(); 169 for (final Pair<Keypoint> kps : normalMatches) { 170 final Keypoint p1 = kps.firstObject().transform(normalOffset); 171 final Keypoint p2 = kps.secondObject().transform(carpetOffset); 172 173 outFrame.drawPoint(p1, RGBColour.RED, 3); 174 outFrame.drawPoint(p2, RGBColour.RED, 3); 175 176 outFrame.drawLine(new Line2d(p1, p2), 3, RGBColour.GREEN); 177 } 178 } 179 180 if (altmatcher.findMatches(altKPTs)) { 181 final List<Pair<Keypoint>> altMatches = altmatcher.getMatches(); 182 for (final Pair<Keypoint> kps : altMatches) { 183 final Keypoint p1 = kps.firstObject().transform(altOffset); 184 final Keypoint p2 = kps.secondObject().transform(carpetOffset); 185 186 outFrame.drawPoint(p1, RGBColour.RED, 3); 187 outFrame.drawPoint(p2, RGBColour.RED, 3); 188 189 outFrame.drawLine(new Line2d(p1, p2), 3, RGBColour.BLUE); 190 } 191 } 192 193 this.ic.setImage(ImageUtilities.createBufferedImageForDisplay(outFrame)); 194 } 195 196 @Override 197 public void keyPressed(KeyEvent key) { 198 if (key.getKeyChar() == 'x') { 199 this.spinning.adjustSpeed(0.005f); 200 } 201 else if (key.getKeyChar() == 'z') { 202 this.spinning.adjustSpeed(-0.005f); 203 } 204 else if (key.getKeyCode() == KeyEvent.VK_SPACE) { 205 this.display.togglePause(); 206 this.spinning.togglePause(); 207 } 208 } 209 210 @Override 211 public void keyReleased(KeyEvent arg0) { 212 // do nothing 213 } 214 215 @Override 216 public void keyTyped(KeyEvent arg0) { 217 // do nothing 218 } 219 220}