001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.demos.acmmm11.presentation.slides;
031
032import java.awt.Component;
033import java.awt.Dimension;
034import java.awt.GridBagLayout;
035import java.awt.event.KeyEvent;
036import java.awt.event.KeyListener;
037import java.io.IOException;
038import java.util.List;
039
040import javax.swing.JPanel;
041
042import org.openimaj.content.slideshow.Slide;
043import org.openimaj.feature.local.list.LocalFeatureList;
044import org.openimaj.feature.local.matcher.FastBasicKeypointMatcher;
045import org.openimaj.feature.local.matcher.consistent.ConsistentLocalFeatureMatcher2d;
046import org.openimaj.image.DisplayUtilities.ImageComponent;
047import org.openimaj.image.FImage;
048import org.openimaj.image.ImageUtilities;
049import org.openimaj.image.MBFImage;
050import org.openimaj.image.colour.RGBColour;
051import org.openimaj.image.feature.local.engine.ALTDoGSIFTEngine;
052import org.openimaj.image.feature.local.engine.DoGSIFTEngine;
053import org.openimaj.image.feature.local.keypoints.Keypoint;
054import org.openimaj.image.processing.resize.ResizeProcessor;
055import org.openimaj.math.geometry.line.Line2d;
056import org.openimaj.math.geometry.point.Point2d;
057import org.openimaj.math.geometry.transforms.AffineTransformModel;
058import org.openimaj.math.geometry.transforms.TransformUtilities;
059import org.openimaj.math.geometry.transforms.residuals.SingleImageTransferResidual2d;
060import org.openimaj.math.model.fit.RANSAC;
061import org.openimaj.util.pair.Pair;
062import org.openimaj.video.VideoDisplay;
063import org.openimaj.video.VideoDisplayListener;
064
065import Jama.Matrix;
066
067/**
068 * Slide illustrating two sift implementation
069 * 
070 * @author Sina Samangooei (ss@ecs.soton.ac.uk)
071 * 
072 */
073public class SIFTAltSIFTSlide implements Slide, VideoDisplayListener<MBFImage>, KeyListener {
074
075        private MBFImage outFrame;
076        private DoGSIFTEngine normalEngine;
077        private DoGSIFTEngine altEngine;
078        private FImage carpetGrey;
079        private MBFImage carpet;
080        private ConsistentLocalFeatureMatcher2d<Keypoint> normalmatcher;
081        private ConsistentLocalFeatureMatcher2d<Keypoint> altmatcher;
082        private ImageComponent ic;
083        private SpinningImageVideo spinning;
084        private VideoDisplay<MBFImage> display;
085
086        @Override
087        public Component getComponent(int width, int height) throws IOException {
088                carpet = ImageUtilities.readMBF(SIFTAltSIFTSlide.class.getResource("rabbit.jpeg"));
089                final double wh = Math.sqrt(carpet.getWidth() * carpet.getWidth() + carpet.getHeight() * carpet.getHeight());
090                if (wh * 2 > Math.min(width, height)) {
091                        final float prop = (float) (Math.min(width, height) / (wh * 2));
092                        carpet.processInplace(new ResizeProcessor(prop));
093                }
094
095                this.carpetGrey = carpet.flatten();
096
097                spinning = new SpinningImageVideo(carpet, -0.5f, 0.005f);
098
099                outFrame = new MBFImage(spinning.getWidth() * 2, spinning.getHeight() * 2, 3);
100                normalEngine = new DoGSIFTEngine();
101                normalEngine.getOptions().setDoubleInitialImage(false);
102                altEngine = new ALTDoGSIFTEngine();
103                altEngine.getOptions().setDoubleInitialImage(false);
104
105                final LocalFeatureList<Keypoint> carpetNormalKPTs = normalEngine.findFeatures(carpetGrey);
106                normalmatcher = new ConsistentLocalFeatureMatcher2d<Keypoint>(
107                                new FastBasicKeypointMatcher<Keypoint>(8),
108                                new RANSAC<Point2d, Point2d, AffineTransformModel>(new AffineTransformModel(),
109                                                new SingleImageTransferResidual2d<AffineTransformModel>(), 5.0, 100,
110                                                new RANSAC.ProbabilisticMinInliersStoppingCondition(0.01), true)
111                                );
112                normalmatcher.setModelFeatures(carpetNormalKPTs);
113
114                final LocalFeatureList<Keypoint> carpetAltKPTs = altEngine.findFeatures(carpetGrey);
115                altmatcher = new ConsistentLocalFeatureMatcher2d<Keypoint>(
116                                new FastBasicKeypointMatcher<Keypoint>(8),
117                                new RANSAC<Point2d, Point2d, AffineTransformModel>(new AffineTransformModel(),
118                                                new SingleImageTransferResidual2d<AffineTransformModel>(), 5.0, 100,
119                                                new RANSAC.ProbabilisticMinInliersStoppingCondition(0.01), true)
120                                );
121                altmatcher.setModelFeatures(carpetAltKPTs);
122
123                display = VideoDisplay.createOffscreenVideoDisplay(spinning);
124                display.addVideoListener(this);
125
126                final JPanel c = new JPanel();
127                c.setOpaque(false);
128                c.setPreferredSize(new Dimension(width, height));
129                c.setLayout(new GridBagLayout());
130                ic = new ImageComponent(true, false);
131                c.add(ic);
132                for (final Component cc : c.getComponents()) {
133                        if (cc instanceof JPanel) {
134                                ((JPanel) cc).setOpaque(false);
135                        }
136                }
137                return c;
138        }
139
140        @Override
141        public void close() {
142                this.spinning.stop();
143                this.display.close();
144        }
145
146        @Override
147        public void afterUpdate(VideoDisplay<MBFImage> display) {
148                // do nothing
149        }
150
151        @Override
152        public void beforeUpdate(MBFImage frame) {
153                outFrame.fill(RGBColour.BLACK);
154                final FImage fGrey = frame.flatten();
155                final LocalFeatureList<Keypoint> normalKPTs = normalEngine.findFeatures(fGrey);
156                final LocalFeatureList<Keypoint> altKPTs = altEngine.findFeatures(fGrey);
157
158                final Matrix carpetOffset = TransformUtilities.translateMatrix(frame.getWidth() / 4, frame.getHeight() * 2 / 3);
159                final Matrix normalOffset = TransformUtilities.translateMatrix(frame.getWidth(), 0);
160                final Matrix altOffset = TransformUtilities.translateMatrix(frame.getWidth(), frame.getHeight());
161
162                outFrame.drawImage(carpet, frame.getWidth() / 4, frame.getHeight() * 2 / 3);
163                outFrame.drawImage(frame, frame.getWidth(), 0);
164                outFrame.drawImage(frame, frame.getWidth(), frame.getHeight());
165
166                if (normalmatcher.findMatches(normalKPTs))
167                {
168                        final List<Pair<Keypoint>> normalMatches = normalmatcher.getMatches();
169                        for (final Pair<Keypoint> kps : normalMatches) {
170                                final Keypoint p1 = kps.firstObject().transform(normalOffset);
171                                final Keypoint p2 = kps.secondObject().transform(carpetOffset);
172
173                                outFrame.drawPoint(p1, RGBColour.RED, 3);
174                                outFrame.drawPoint(p2, RGBColour.RED, 3);
175
176                                outFrame.drawLine(new Line2d(p1, p2), 3, RGBColour.GREEN);
177                        }
178                }
179
180                if (altmatcher.findMatches(altKPTs)) {
181                        final List<Pair<Keypoint>> altMatches = altmatcher.getMatches();
182                        for (final Pair<Keypoint> kps : altMatches) {
183                                final Keypoint p1 = kps.firstObject().transform(altOffset);
184                                final Keypoint p2 = kps.secondObject().transform(carpetOffset);
185
186                                outFrame.drawPoint(p1, RGBColour.RED, 3);
187                                outFrame.drawPoint(p2, RGBColour.RED, 3);
188
189                                outFrame.drawLine(new Line2d(p1, p2), 3, RGBColour.BLUE);
190                        }
191                }
192
193                this.ic.setImage(ImageUtilities.createBufferedImageForDisplay(outFrame));
194        }
195
196        @Override
197        public void keyPressed(KeyEvent key) {
198                if (key.getKeyChar() == 'x') {
199                        this.spinning.adjustSpeed(0.005f);
200                }
201                else if (key.getKeyChar() == 'z') {
202                        this.spinning.adjustSpeed(-0.005f);
203                }
204                else if (key.getKeyCode() == KeyEvent.VK_SPACE) {
205                        this.display.togglePause();
206                        this.spinning.togglePause();
207                }
208        }
209
210        @Override
211        public void keyReleased(KeyEvent arg0) {
212                // do nothing
213        }
214
215        @Override
216        public void keyTyped(KeyEvent arg0) {
217                // do nothing
218        }
219
220}