1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30 package org.openimaj.demos.video.videosift;
31
32 import java.awt.Point;
33 import java.awt.event.KeyEvent;
34 import java.awt.event.KeyListener;
35
36 import javax.swing.JFrame;
37 import javax.swing.SwingUtilities;
38
39 import org.openimaj.demos.video.utils.FeatureClickListener;
40 import org.openimaj.demos.video.utils.PolygonDrawingListener;
41 import org.openimaj.demos.video.utils.PolygonExtractionProcessor;
42 import org.openimaj.feature.local.list.LocalFeatureList;
43 import org.openimaj.feature.local.matcher.FastBasicKeypointMatcher;
44 import org.openimaj.feature.local.matcher.MatchingUtilities;
45 import org.openimaj.feature.local.matcher.consistent.ConsistentLocalFeatureMatcher2d;
46 import org.openimaj.image.DisplayUtilities;
47 import org.openimaj.image.FImage;
48 import org.openimaj.image.MBFImage;
49 import org.openimaj.image.colour.RGBColour;
50 import org.openimaj.image.colour.Transforms;
51 import org.openimaj.image.feature.local.engine.ipd.FinderMode;
52 import org.openimaj.image.feature.local.engine.ipd.IPDSIFTEngine;
53 import org.openimaj.image.feature.local.interest.HarrisIPD;
54 import org.openimaj.image.feature.local.interest.IPDSelectionMode;
55 import org.openimaj.image.feature.local.interest.InterestPointData;
56 import org.openimaj.image.feature.local.interest.InterestPointVisualiser;
57 import org.openimaj.image.feature.local.keypoints.InterestPointKeypoint;
58 import org.openimaj.image.feature.local.keypoints.KeypointVisualizer;
59 import org.openimaj.image.renderer.MBFImageRenderer;
60 import org.openimaj.math.geometry.shape.Polygon;
61 import org.openimaj.math.geometry.shape.Shape;
62 import org.openimaj.math.geometry.transforms.HomographyRefinement;
63 import org.openimaj.math.geometry.transforms.MatrixTransformProvider;
64 import org.openimaj.math.geometry.transforms.estimation.RobustHomographyEstimator;
65 import org.openimaj.math.model.fit.RANSAC;
66 import org.openimaj.video.VideoDisplay;
67 import org.openimaj.video.VideoDisplayListener;
68 import org.openimaj.video.capture.VideoCapture;
69
70
71
72
73
74
75
76 public class VideoIPD implements KeyListener, VideoDisplayListener<MBFImage> {
77 private VideoCapture capture;
78 private VideoDisplay<MBFImage> videoFrame;
79 private JFrame modelFrame;
80 private JFrame matchFrame;
81 private MBFImage modelImage;
82
83 private ConsistentLocalFeatureMatcher2d<InterestPointKeypoint<InterestPointData>> matcher;
84 private IPDSIFTEngine engine;
85 private PolygonDrawingListener polygonListener;
86 private FeatureClickListener<Float[], MBFImage> featureClickListener;
87
88 public VideoIPD() throws Exception {
89
90 engine = getNewEngine();
91
92 capture = new VideoCapture(320, 240);
93 polygonListener = new PolygonDrawingListener();
94 videoFrame = VideoDisplay.createVideoDisplay(capture);
95 SwingUtilities.getRoot(videoFrame.getScreen()).addKeyListener(this);
96 videoFrame.getScreen().addMouseListener(polygonListener);
97 this.featureClickListener = new FeatureClickListener<Float[], MBFImage>();
98 videoFrame.getScreen().addMouseListener(featureClickListener);
99
100 videoFrame.addVideoListener(this);
101
102 }
103
104 private IPDSIFTEngine getNewEngine() {
105 final int derScale = 1;
106 final int intScale = 3;
107 final HarrisIPD ipd = new HarrisIPD(derScale, intScale);
108 engine = new IPDSIFTEngine(ipd);
109 engine.setSelectionMode(new IPDSelectionMode.Threshold(10000f));
110 engine.setFinderMode(new FinderMode.Characteristic<InterestPointData>());
111
112 engine.setAcrossScales(true);
113
114 return engine;
115 }
116
117 @Override
118 public void keyPressed(KeyEvent key) {
119 if (key.getKeyCode() == KeyEvent.VK_SPACE) {
120 this.videoFrame.togglePause();
121 } else if (key.getKeyChar() == 'c' && this.polygonListener.getPolygon().getVertices().size() > 2) {
122 try {
123 final Polygon p = this.polygonListener.getPolygon().clone();
124 this.polygonListener.reset();
125 modelImage = capture.getCurrentFrame().process(
126 new PolygonExtractionProcessor<Float[], MBFImage>(p, RGBColour.BLACK));
127
128 if (modelFrame == null) {
129 modelFrame = DisplayUtilities.display(modelImage, "model");
130 modelFrame.addKeyListener(this);
131
132
133 final Point pt = modelFrame.getLocation();
134 modelFrame.setLocation(pt.x + this.videoFrame.getScreen().getWidth(), pt.y);
135
136
137 matcher = new ConsistentLocalFeatureMatcher2d<InterestPointKeypoint<InterestPointData>>(
138 new FastBasicKeypointMatcher<InterestPointKeypoint<InterestPointData>>(8));
139 matcher.setFittingModel(new RobustHomographyEstimator(10.0, 1500,
140 new RANSAC.PercentageInliersStoppingCondition(0.5), HomographyRefinement.NONE));
141 } else {
142 DisplayUtilities.display(modelImage, modelFrame);
143 }
144
145 final FImage modelF = Transforms.calculateIntensityNTSC(modelImage);
146 matcher.setModelFeatures(getNewEngine().findFeatures(modelF));
147 } catch (final Exception e) {
148 e.printStackTrace();
149 }
150 }
151 }
152
153 @Override
154 public void keyReleased(KeyEvent arg0) {
155 }
156
157 @Override
158 public void keyTyped(KeyEvent arg0) {
159 }
160
161 public static void main(String[] args) throws Exception {
162 new VideoIPD();
163 }
164
165 @Override
166 public void afterUpdate(VideoDisplay<MBFImage> display) {
167 if (matcher != null && !videoFrame.isPaused()) {
168 final MBFImage capImg = videoFrame.getVideo().getCurrentFrame();
169 final LocalFeatureList<InterestPointKeypoint<InterestPointData>> kpl = engine.findFeatures(Transforms
170 .calculateIntensityNTSC(capImg));
171
172 final MBFImageRenderer renderer = capImg.createRenderer();
173 renderer.drawPoints(kpl, RGBColour.MAGENTA, 3);
174
175 MBFImage matches;
176 if (matcher.findMatches(kpl)) {
177 try {
178 final Shape sh = modelImage.getBounds().transform(
179 ((MatrixTransformProvider) matcher.getModel()).getTransform().inverse());
180 renderer.drawShape(sh, 3, RGBColour.BLUE);
181 } catch (final RuntimeException e) {
182 }
183
184 matches = MatchingUtilities.drawMatches(modelImage, capImg, matcher.getMatches(), RGBColour.RED);
185 } else {
186 matches = MatchingUtilities.drawMatches(modelImage, capImg, null, RGBColour.RED);
187 }
188
189 if (matchFrame == null) {
190 matchFrame = DisplayUtilities.display(matches, "matches");
191 matchFrame.addKeyListener(this);
192
193 final Point pt = matchFrame.getLocation();
194 matchFrame.setLocation(pt.x, pt.y + matchFrame.getHeight());
195 } else {
196 DisplayUtilities.display(matches, matchFrame);
197 }
198 }
199 }
200
201 @Override
202 public void beforeUpdate(MBFImage frame) {
203 drawKeypoints(frame);
204 }
205
206 private void drawKeypoints(MBFImage frame) {
207 final MBFImage capImg = frame;
208 final LocalFeatureList<InterestPointKeypoint<InterestPointData>> kpl = engine.findFeatures(Transforms
209 .calculateIntensityNTSC(capImg));
210 this.featureClickListener.setImage(kpl, frame.clone());
211 final KeypointVisualizer<Float[], MBFImage> kpv = new KeypointVisualizer<Float[], MBFImage>(capImg, kpl);
212 final InterestPointVisualiser<Float[], MBFImage> ipv = InterestPointVisualiser.visualiseKeypoints(
213 kpv.drawPatches(null, RGBColour.GREEN), kpl);
214 frame.internalAssign(ipv.drawPatches(RGBColour.GREEN, RGBColour.BLUE));
215 }
216 }