1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30 package org.openimaj.demos.image;
31
32 import java.awt.event.MouseEvent;
33 import java.awt.event.MouseListener;
34 import java.awt.event.MouseMotionListener;
35 import java.io.IOException;
36 import java.util.ArrayList;
37 import java.util.List;
38
39 import javax.swing.JFrame;
40
41 import org.openimaj.demos.Demo;
42 import org.openimaj.image.DisplayUtilities;
43 import org.openimaj.image.MBFImage;
44 import org.openimaj.image.colour.ColourSpace;
45 import org.openimaj.image.colour.RGBColour;
46 import org.openimaj.image.colour.Transforms;
47 import org.openimaj.image.feature.local.interest.HarrisIPD;
48 import org.openimaj.image.processing.convolution.FGaussianConvolve;
49 import org.openimaj.image.processing.resize.ResizeProcessor;
50 import org.openimaj.image.processing.transform.ProjectionProcessor;
51 import org.openimaj.image.renderer.MBFImageRenderer;
52 import org.openimaj.math.geometry.line.Line2d;
53 import org.openimaj.math.geometry.point.Point2d;
54 import org.openimaj.math.geometry.point.Point2dImpl;
55 import org.openimaj.math.geometry.shape.Ellipse;
56 import org.openimaj.math.geometry.shape.EllipseUtilities;
57 import org.openimaj.math.geometry.shape.Rectangle;
58 import org.openimaj.math.geometry.shape.Shape;
59 import org.openimaj.math.geometry.transforms.TransformUtilities;
60 import org.openimaj.util.pair.Pair;
61
62 import Jama.EigenvalueDecomposition;
63 import Jama.Matrix;
64
65
66
67
68
69
70
71 @Demo(
72 author = "Sina Samangeooi",
73 description = "Demonstrates the second moment extractor in an interactive"
74 + " way. Move the mouse over the edges of the box in the first image "
75 + "and the moments are displayed in the other images.",
76 keywords = {
77 "image", "moments" },
78 title = "Second Moment Visualiser",
79 icon = "/org/openimaj/demos/icons/image/moment-icon.png")
80 public class SecondMomentVisualiser implements MouseListener, MouseMotionListener {
81 private MBFImage image;
82 private HarrisIPD ipd;
83 private Point2d drawPoint = null;
84 private double derivscale;
85 private JFrame projectFrame;
86 private ResizeProcessor resizeProject;
87 private List<Ellipse> ellipses;
88 private List<Pair<Line2d>> lines;
89 private Matrix transformMatrix;
90 private JFrame mouseFrame;
91 private int windowSize;
92 private int featureWindowSize;
93 private JFrame featureFrame;
94 private double visFactor = 4;
95
96
97
98
99
100
101 public SecondMomentVisualiser() throws IOException {
102
103
104
105 image = new MBFImage(400, 400, ColourSpace.RGB);
106 image.fill(RGBColour.WHITE);
107 final Shape shapeToDraw = new Rectangle(100, 100, 200, 200)
108 .transform(TransformUtilities.rotationMatrixAboutPoint(
109 Math.PI / 4, 200, 200));
110
111
112 image.createRenderer().drawShapeFilled(shapeToDraw, RGBColour.BLACK);
113 derivscale = 5;
114 ipd = new HarrisIPD((float) derivscale, (float) derivscale * 2);
115 ipd.findInterestPoints(Transforms.calculateIntensityNTSC(image));
116
117 class Updater implements Runnable {
118
119 private SecondMomentVisualiser frame;
120
121 Updater(SecondMomentVisualiser frame) {
122 this.frame = frame;
123 }
124
125 @Override
126 public void run() {
127 while (true) {
128 frame.draw();
129 try {
130 Thread.sleep(1000 / 30);
131 } catch (final InterruptedException e) {
132 }
133 }
134 }
135 }
136 image = image.process(new FGaussianConvolve(5));
137
138 this.mouseFrame = DisplayUtilities.displaySimple(image.clone());
139
140 this.mouseFrame.getContentPane().addMouseListener(this);
141 this.mouseFrame.getContentPane().addMouseMotionListener(this);
142
143 projectFrame = DisplayUtilities.display(image.clone());
144 projectFrame.setBounds(image.getWidth(), 0, image.getWidth(),
145 image.getHeight());
146 featureFrame = DisplayUtilities.display(image.clone());
147 featureFrame.setBounds(image.getWidth() * 2, 0, image.getWidth(),
148 image.getHeight());
149 ellipses = new ArrayList<Ellipse>();
150 lines = new ArrayList<Pair<Line2d>>();
151 resizeProject = new ResizeProcessor(256, 256);
152 final Thread t = new Thread(new Updater(this));
153 t.start();
154
155 }
156
157
158
159
160 public synchronized void draw() {
161 final MBFImage toDraw = image.clone();
162 final MBFImageRenderer renderer = toDraw.createRenderer();
163
164 if (this.drawPoint != null)
165 renderer.drawPoint(this.drawPoint, RGBColour.RED, 3);
166
167 for (final Ellipse ellipse : ellipses) {
168 renderer.drawShape(ellipse, 1, RGBColour.GREEN);
169 }
170 for (final Pair<Line2d> line : lines) {
171 renderer.drawLine(line.firstObject(), 3, RGBColour.BLUE);
172 renderer.drawLine(line.secondObject(), 3, RGBColour.RED);
173 }
174 if (this.transformMatrix != null) {
175 try {
176
177 final ProjectionProcessor<Float[], MBFImage> pp = new ProjectionProcessor<Float[], MBFImage>();
178 pp.setMatrix(this.transformMatrix);
179 this.image.accumulateWith(pp);
180 final MBFImage patch = pp.performProjection(-windowSize,
181 windowSize, -windowSize, windowSize,
182 RGBColour.RED);
183 if (patch.getWidth() > 0 && patch.getHeight() > 0) {
184 DisplayUtilities.display(patch.process(this.resizeProject),
185 this.projectFrame);
186 DisplayUtilities.display(
187 patch.extractCenter(this.featureWindowSize,
188 this.featureWindowSize).process(
189 this.resizeProject), this.featureFrame);
190
191 }
192 } catch (final Exception e) {
193 e.printStackTrace();
194 }
195
196 }
197
198 DisplayUtilities.display(toDraw.clone(), this.mouseFrame);
199 }
200
201 private synchronized void setEBowl() {
202 final Matrix secondMoments = ipd.getSecondMomentsAt(
203 (int) this.drawPoint.getX(), (int) this.drawPoint.getY());
204
205
206
207 this.ellipses.clear();
208 this.lines.clear();
209 try {
210 getBowlEllipse(secondMoments);
211 } catch (final Exception e) {
212 e.printStackTrace();
213 }
214 }
215
216 private void getBowlEllipse(Matrix secondMoments) {
217 double rotation = 0;
218 double d1 = 0, d2 = 0;
219 if (secondMoments.det() == 0)
220 return;
221
222
223
224
225
226
227
228 final Matrix noblur = new Matrix(new double[][] {
229 {
230 ipd.lxmxblur.getPixel((int) this.drawPoint.getX(),
231 (int) this.drawPoint.getY()),
232 ipd.lxmyblur.getPixel((int) this.drawPoint.getX(),
233 (int) this.drawPoint.getY()) },
234 {
235 ipd.lxmyblur.getPixel((int) this.drawPoint.getX(),
236 (int) this.drawPoint.getY()),
237 ipd.lxmxblur.getPixel((int) this.drawPoint.getX(),
238 (int) this.drawPoint.getY()) } });
239 System.out.println("NO BLUR SECOND MOMENTS MATRIX");
240 noblur.print(5, 5);
241 System.out.println("det is: " + noblur.det());
242 if (noblur.det() < 0.00001)
243 return;
244
245 final double divFactor = 1 / Math.sqrt(secondMoments.det());
246 final double scaleFctor = derivscale;
247 final EigenvalueDecomposition rdr = secondMoments.times(divFactor).eig();
248 secondMoments.times(divFactor).print(5, 5);
249
250 System.out.println("D1(before)= " + rdr.getD().get(0, 0));
251 System.out.println("D2(before) = " + rdr.getD().get(1, 1));
252
253 if (rdr.getD().get(0, 0) == 0)
254 d1 = 0;
255 else
256 d1 = 1.0 / Math.sqrt(rdr.getD().get(0, 0));
257
258 if (rdr.getD().get(1, 1) == 0)
259 d2 = 0;
260 else
261 d2 = 1.0 / Math.sqrt(rdr.getD().get(1, 1));
262
263
264 final double scaleCorrectedD1 = d1 * scaleFctor * visFactor;
265 final double scaleCorrectedD2 = d2 * scaleFctor * visFactor;
266
267 final Matrix eigenMatrix = rdr.getV();
268 System.out.println("D1 = " + d1);
269 System.out.println("D2 = " + d2);
270 eigenMatrix.print(5, 5);
271
272 rotation = Math.atan2(eigenMatrix.get(1, 0), eigenMatrix.get(0, 0));
273 final Ellipse ellipseToAdd = EllipseUtilities.ellipseFromEquation(
274 this.drawPoint.getX(),
275 this.drawPoint.getY(),
276 scaleCorrectedD1,
277 scaleCorrectedD2,
278 rotation
279 );
280 ellipses.add(ellipseToAdd);
281
282 if (d1 != 0 && d2 != 0) {
283 this.windowSize = (int) (scaleFctor * d1 / d2) / 2;
284 this.featureWindowSize = (int) scaleFctor;
285 if (this.windowSize > 256)
286 this.windowSize = 256;
287
288
289
290
291
292 this.transformMatrix = ellipseToAdd
293 .transformMatrix()
294 .times(TransformUtilities.scaleMatrix(1 / scaleFctor,
295 1 / scaleFctor)).inverse();
296 for (final double d : transformMatrix.getRowPackedCopy())
297 if (d == Double.NaN) {
298 this.transformMatrix = null;
299 break;
300 }
301 } else {
302 transformMatrix = null;
303 }
304 if (transformMatrix != null) {
305 System.out.println("Transform matrix:");
306 transformMatrix.print(5, 5);
307 }
308
309 final Line2d major = Line2d.lineFromRotation((int) this.drawPoint.getX(),
310 (int) this.drawPoint.getY(), (float) rotation,
311 (int) scaleCorrectedD1);
312 final Line2d minor = Line2d.lineFromRotation((int) this.drawPoint.getX(),
313 (int) this.drawPoint.getY(), (float) (rotation + Math.PI / 2),
314 (int) scaleCorrectedD2);
315 lines.add(new Pair<Line2d>(major, minor));
316 }
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357 @Override
358 public void mouseClicked(MouseEvent event) {
359 drawPoint = new Point2dImpl(event.getX(), event.getY());
360 if (this.drawPoint != null) {
361 setEBowl();
362 }
363 }
364
365 @Override
366 public void mouseEntered(MouseEvent event) {
367
368 }
369
370 @Override
371 public void mouseExited(MouseEvent event) {
372
373 }
374
375 @Override
376 public void mousePressed(MouseEvent event) {
377
378 }
379
380 @Override
381 public void mouseReleased(MouseEvent event) {
382
383 }
384
385 @Override
386 public void mouseDragged(MouseEvent e) {
387
388 }
389
390 @Override
391 public void mouseMoved(MouseEvent e) {
392 drawPoint = new Point2dImpl(e.getX(), e.getY());
393 if (this.drawPoint != null) {
394 setEBowl();
395 }
396 }
397
398
399
400
401
402
403
404
405 public static void main(String args[]) throws IOException {
406 new SecondMomentVisualiser();
407 }
408 }