1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30 package org.openimaj.demos.video.videosift;
31
32 import java.awt.event.KeyEvent;
33 import java.awt.event.KeyListener;
34 import java.util.ArrayList;
35 import java.util.List;
36
37 import javax.swing.SwingUtilities;
38
39 import org.openimaj.demos.video.utils.PolygonDrawingListener;
40 import org.openimaj.demos.video.utils.PolygonExtractionProcessor;
41 import org.openimaj.feature.local.list.LocalFeatureList;
42 import org.openimaj.feature.local.matcher.FastBasicKeypointMatcher;
43 import org.openimaj.feature.local.matcher.MatchingUtilities;
44 import org.openimaj.feature.local.matcher.consistent.ConsistentLocalFeatureMatcher2d;
45 import org.openimaj.image.FImage;
46 import org.openimaj.image.MBFImage;
47 import org.openimaj.image.colour.RGBColour;
48 import org.openimaj.image.colour.Transforms;
49 import org.openimaj.image.feature.local.engine.DoGSIFTEngine;
50 import org.openimaj.image.feature.local.keypoints.Keypoint;
51 import org.openimaj.image.processing.resize.ResizeProcessor;
52 import org.openimaj.image.processing.transform.ProjectionProcessor;
53 import org.openimaj.image.renderer.MBFImageRenderer;
54 import org.openimaj.math.geometry.point.Point2d;
55 import org.openimaj.math.geometry.point.Point2dImpl;
56 import org.openimaj.math.geometry.shape.Polygon;
57 import org.openimaj.math.geometry.shape.Shape;
58 import org.openimaj.math.geometry.transforms.HomographyModel;
59 import org.openimaj.math.geometry.transforms.HomographyRefinement;
60 import org.openimaj.math.geometry.transforms.MatrixTransformProvider;
61 import org.openimaj.math.geometry.transforms.TransformUtilities;
62 import org.openimaj.math.geometry.transforms.estimation.RobustHomographyEstimator;
63 import org.openimaj.math.geometry.transforms.residuals.SingleImageTransferResidual2d;
64 import org.openimaj.math.model.fit.RANSAC;
65 import org.openimaj.util.pair.IndependentPair;
66 import org.openimaj.video.VideoDisplay;
67 import org.openimaj.video.VideoDisplayListener;
68 import org.openimaj.video.capture.VideoCapture;
69 import org.openimaj.video.tracking.klt.Feature;
70 import org.openimaj.video.tracking.klt.FeatureList;
71 import org.openimaj.video.tracking.klt.KLTTracker;
72 import org.openimaj.video.tracking.klt.TrackingContext;
73
74 import Jama.Matrix;
75
76 public class VideoKLTSIFT implements KeyListener, VideoDisplayListener<MBFImage> {
77 enum Mode {
78 TRACKING, LOOKING, NONE, START_LOOKING
79 }
80
81 private VideoCapture capture;
82 private VideoDisplay<MBFImage> videoFrame;
83 private KLTTracker tracker;
84 private FeatureList fl;
85
86 private FImage oldFrame;
87 private int frameNumber = 0;
88 private int nFeatures = 50;
89 private int nOriginalFoundFeatures = -1;
90 private DoGSIFTEngine engine;
91 private PolygonDrawingListener polygonListener;
92 private Mode mode = Mode.NONE;
93 private FeatureList initialFeatures;
94 private Polygon initialShape;
95 private ConsistentLocalFeatureMatcher2d<Keypoint> siftMatcher;
96 private MBFImage modelImage;
97 private MBFImage overlayFrame = null;
98
99 public VideoKLTSIFT() throws Exception {
100 capture = new VideoCapture(640, 480);
101 polygonListener = new PolygonDrawingListener();
102 videoFrame = VideoDisplay.createVideoDisplay(capture);
103 videoFrame.getScreen().addMouseListener(polygonListener);
104
105 videoFrame.addVideoListener(this);
106 SwingUtilities.getRoot(videoFrame.getScreen()).addKeyListener(this);
107
108 reinitTracker();
109 }
110
111 @Override
112 public void afterUpdate(VideoDisplay<MBFImage> arg0) {
113
114
115 }
116
117 @Override
118 public synchronized void beforeUpdate(MBFImage image) {
119 this.polygonListener.drawPoints(image);
120 if (videoFrame.isPaused())
121 {
122 return;
123 }
124 FImage greyFrame = null;
125 greyFrame = Transforms.calculateIntensityNTSC(image);
126
127
128 if (this.mode == Mode.LOOKING) {
129
130 final Shape shape = findObject(greyFrame);
131 if (shape == null) {
132 this.oldFrame = greyFrame;
133 return;
134 }
135 System.out.println("Object FOUND, switcihg to tracking mode");
136
137
138 initTracking(greyFrame, shape);
139 this.mode = Mode.TRACKING;
140 }
141
142 else if (this.mode == Mode.TRACKING) {
143
144 continueTracking(greyFrame);
145
146 if (fl.countRemainingFeatures() == 0 || fl.countRemainingFeatures() < nOriginalFoundFeatures * 0.2)
147 {
148 System.out.println("Object LOST, switching to LOOKING mode");
149 this.mode = Mode.LOOKING;
150 reinitTracker();
151 }
152
153
154
155
156 }
157 else if (this.mode == Mode.START_LOOKING) {
158 this.reinitTracker();
159 final Polygon p = this.polygonListener.getPolygon().clone();
160 this.polygonListener.reset();
161 final MBFImage modelImage = capture.getCurrentFrame();
162 this.initTracking(greyFrame, p);
163 this.initObjectFinder(modelImage, p);
164 this.oldFrame = greyFrame;
165 mode = Mode.LOOKING;
166 return;
167 }
168
169 if (overlayFrame != null) {
170 drawOverlay(image);
171 }
172 else {
173 drawDebug(image, greyFrame);
174 }
175
176 this.oldFrame = greyFrame;
177
178 }
179
180 private void drawOverlay(MBFImage image) {
181 final ProjectionProcessor<Float[], MBFImage> proc = new ProjectionProcessor<Float[], MBFImage>();
182 image.accumulateWith(proc);
183 final Matrix model = this.estimateModel();
184 if (model != null) {
185 proc.setMatrix(model);
186 this.overlayFrame.accumulateWith(proc);
187 }
188 image.internalAssign(proc.performProjection());
189 }
190
191 private void drawDebug(MBFImage image, FImage greyFrame) {
192 this.polygonListener.drawPoints(image);
193
194 final MBFImageRenderer renderer = image.createRenderer();
195
196 if (this.initialShape != null) {
197 renderer.drawPolygon(initialShape, RGBColour.RED);
198 }
199 if (this.initialFeatures != null) {
200 image.internalAssign(MatchingUtilities.drawMatches(image, this.findAllMatchedPairs(), RGBColour.WHITE));
201 final Matrix esitmatedModel = this.estimateModel();
202 if (esitmatedModel != null)
203 {
204 final Polygon newPolygon = initialShape.transform(esitmatedModel);
205 renderer.drawPolygon(newPolygon, RGBColour.GREEN);
206 if (fl.countRemainingFeatures() < nOriginalFoundFeatures * 0.5) {
207 reinitTracker();
208 initTracking(greyFrame, newPolygon);
209 }
210 }
211 estimateMovement();
212 }
213 }
214
215 private Shape findObject(FImage capImg) {
216 final float scaleImage = .5f;
217
218 final ResizeProcessor resize = new ResizeProcessor(scaleImage);
219 capImg = capImg.process(resize);
220 Shape sh = null;
221 if (siftMatcher != null && !videoFrame.isPaused() && engine != null) {
222 final LocalFeatureList<Keypoint> kpl = engine.findFeatures(capImg);
223 if (siftMatcher.findMatches(kpl)) {
224 Matrix shTransform = ((MatrixTransformProvider) siftMatcher.getModel()).getTransform().copy();
225 if (shTransform != null)
226 {
227 try {
228 shTransform = TransformUtilities.scaleMatrix(1f / scaleImage, 1f / scaleImage).times(
229 shTransform.inverse());
230 sh = modelImage.getBounds().transform(shTransform);
231 } catch (final Exception e) {
232 e.printStackTrace();
233 }
234 }
235
236 }
237
238 }
239 return sh;
240
241 }
242
243 private void reinitTracker() {
244 final TrackingContext tc = new TrackingContext();
245 fl = new FeatureList(nFeatures);
246 tracker = new KLTTracker(tc, fl);
247 tracker.setVerbosity(0);
248
249 tc.setSequentialMode(true);
250 tc.setWriteInternalImages(false);
251 tc.setAffineConsistencyCheck(-1);
252
253
254
255 this.initialFeatures = null;
256 this.initialShape = null;
257 }
258
259 public void initTracking(FImage greyFrame, Shape location) {
260 frameNumber = 0;
261 tracker.getTrackingContext().setTargetArea(location);
262 tracker.selectGoodFeatures(greyFrame);
263 nOriginalFoundFeatures = fl.countRemainingFeatures();
264 initialFeatures = fl.clone();
265 initialShape = location.asPolygon().clone();
266 }
267
268 public void continueTracking(FImage greyFrame) {
269 tracker.trackFeatures(oldFrame, greyFrame);
270 this.frameNumber++;
271 }
272
273 private Matrix estimateModel() {
274 if (this.initialFeatures == null) {
275 return null;
276 }
277 final List<? extends IndependentPair<Point2d, Point2d>> pairs = findAllMatchedPairs();
278 final HomographyModel model = new HomographyModel();
279
280 final RANSAC<Point2d, Point2d, HomographyModel> fitter = new RANSAC<Point2d, Point2d, HomographyModel>(model,
281 new SingleImageTransferResidual2d<HomographyModel>(),
282 10.0, 1500,
283 new RANSAC.PercentageInliersStoppingCondition(0.5), false);
284 if (!fitter.fitData(pairs))
285 return null;
286
287 model.getTransform().print(5, 5);
288 return model.getTransform();
289 }
290
291 private List<IndependentPair<Point2d, Point2d>> findAllMatchedPairs() {
292 final List<IndependentPair<Point2d, Point2d>> pairs = new ArrayList<IndependentPair<Point2d, Point2d>>();
293 for (int i = 0; i < this.initialFeatures.features.length; i++) {
294 final Feature oldFeature = this.initialFeatures.features[i].clone();
295 final Feature newFeature = fl.features[i].clone();
296 if (oldFeature.val >= 0 && newFeature.val >= 0) {
297 pairs.add(new IndependentPair<Point2d, Point2d>(oldFeature, newFeature));
298 }
299 }
300 return pairs;
301 }
302
303 public Point2dImpl estimateMovement() {
304 final Feature[] oldFeatures = this.initialFeatures.features;
305 float sumX = 0;
306 float sumY = 0;
307 float total = 0;
308 if (oldFeatures != null) {
309 for (int i = 0; i < oldFeatures.length; i++) {
310 final Feature oldFeature = oldFeatures[i];
311 final Feature newFeature = fl.features[i];
312 if (oldFeature.val >= 0 && newFeature.val >= 0) {
313 sumX += newFeature.x - oldFeature.x;
314 sumY += newFeature.y - oldFeature.y;
315 total += 1f;
316 }
317 }
318 sumX /= total;
319 sumY /= total;
320 System.out.println("Average displacement: " + sumX + "," + sumY);
321 }
322 return new Point2dImpl(sumX, sumY);
323 }
324
325 @Override
326 public void keyTyped(KeyEvent e) {
327
328 }
329
330 @Override
331 public synchronized void keyPressed(KeyEvent key) {
332 if (key.getKeyCode() == KeyEvent.VK_SPACE) {
333 this.videoFrame.togglePause();
334 } else if (key.getKeyChar() == 'c' && this.polygonListener.getPolygon().getVertices().size() > 2) {
335 mode = Mode.START_LOOKING;
336 } else if (key.getKeyChar() == 'd' && this.polygonListener.getPolygon().getVertices().size() > 2) {
337 final Polygon p = this.polygonListener.getPolygon().clone();
338 this.polygonListener.reset();
339 overlayFrame = this.capture.getCurrentFrame().process(
340 new PolygonExtractionProcessor<Float[], MBFImage>(p, RGBColour.BLACK));
341 }
342 else if (key.getKeyChar() == 'r') {
343 this.mode = Mode.NONE;
344
345 }
346 }
347
348 private void initObjectFinder(MBFImage frame, Polygon p) {
349 modelImage = frame.process(new PolygonExtractionProcessor<Float[], MBFImage>(p, RGBColour.BLACK));
350
351
352 siftMatcher = new ConsistentLocalFeatureMatcher2d<Keypoint>(new FastBasicKeypointMatcher<Keypoint>(8));
353 siftMatcher.setFittingModel(new RobustHomographyEstimator(10.0, 1500,
354 new RANSAC.PercentageInliersStoppingCondition(0.5), HomographyRefinement.NONE));
355
356 engine = new DoGSIFTEngine();
357 engine.getOptions().setDoubleInitialImage(true);
358
359 final FImage modelF = Transforms.calculateIntensityNTSC(modelImage);
360 siftMatcher.setModelFeatures(engine.findFeatures(modelF));
361 }
362
363 @Override
364 public void keyReleased(KeyEvent e) {
365
366
367 }
368
369 public static void main(String args[]) throws Exception {
370 new VideoKLTSIFT();
371 }
372
373 }