001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.demos.image; 031 032import java.awt.event.MouseEvent; 033import java.awt.event.MouseListener; 034import java.awt.event.MouseMotionListener; 035import java.io.IOException; 036import java.util.ArrayList; 037import java.util.List; 038 039import javax.swing.JFrame; 040 041import org.openimaj.demos.Demo; 042import org.openimaj.image.DisplayUtilities; 043import org.openimaj.image.MBFImage; 044import org.openimaj.image.colour.ColourSpace; 045import org.openimaj.image.colour.RGBColour; 046import org.openimaj.image.colour.Transforms; 047import org.openimaj.image.feature.local.interest.HarrisIPD; 048import org.openimaj.image.processing.convolution.FGaussianConvolve; 049import org.openimaj.image.processing.resize.ResizeProcessor; 050import org.openimaj.image.processing.transform.ProjectionProcessor; 051import org.openimaj.image.renderer.MBFImageRenderer; 052import org.openimaj.math.geometry.line.Line2d; 053import org.openimaj.math.geometry.point.Point2d; 054import org.openimaj.math.geometry.point.Point2dImpl; 055import org.openimaj.math.geometry.shape.Ellipse; 056import org.openimaj.math.geometry.shape.EllipseUtilities; 057import org.openimaj.math.geometry.shape.Rectangle; 058import org.openimaj.math.geometry.shape.Shape; 059import org.openimaj.math.geometry.transforms.TransformUtilities; 060import org.openimaj.util.pair.Pair; 061 062import Jama.EigenvalueDecomposition; 063import Jama.Matrix; 064 065/** 066 * Demonstrate the second moments 067 * 068 * @author Sina Samangeooi (ss@ecs.soton.ac.uk) 069 * 070 */ 071@Demo( 072 author = "Sina Samangeooi", 073 description = "Demonstrates the second moment extractor in an interactive" 074 + " way. Move the mouse over the edges of the box in the first image " 075 + "and the moments are displayed in the other images.", 076 keywords = { 077 "image", "moments" }, 078 title = "Second Moment Visualiser", 079 icon = "/org/openimaj/demos/icons/image/moment-icon.png") 080public class SecondMomentVisualiser implements MouseListener, MouseMotionListener { 081 private MBFImage image; 082 private HarrisIPD ipd; 083 private Point2d drawPoint = null; 084 private double derivscale; 085 private JFrame projectFrame; 086 private ResizeProcessor resizeProject; 087 private List<Ellipse> ellipses; 088 private List<Pair<Line2d>> lines; 089 private Matrix transformMatrix; 090 private JFrame mouseFrame; 091 private int windowSize; 092 private int featureWindowSize; 093 private JFrame featureFrame; 094 private double visFactor = 4; 095 096 /** 097 * Construct the demo 098 * 099 * @throws IOException 100 */ 101 public SecondMomentVisualiser() throws IOException { 102 // image = ImageUtilities.readMBF( 103 // SecondMomentVisualiser.class.getResourceAsStream("/org/openimaj/image/data/square_rot.png") 104 // ); 105 image = new MBFImage(400, 400, ColourSpace.RGB); 106 image.fill(RGBColour.WHITE); 107 final Shape shapeToDraw = new Rectangle(100, 100, 200, 200) 108 .transform(TransformUtilities.rotationMatrixAboutPoint( 109 Math.PI / 4, 200, 200)); 110 // Shape shapeToDraw = new Rectangle(100,100,200,200); 111 // Shape shapeToDraw = new Circle(200f,200f,100f); 112 image.createRenderer().drawShapeFilled(shapeToDraw, RGBColour.BLACK); 113 derivscale = 5; 114 ipd = new HarrisIPD((float) derivscale, (float) derivscale * 2); 115 ipd.findInterestPoints(Transforms.calculateIntensityNTSC(image)); 116 117 class Updater implements Runnable { 118 119 private SecondMomentVisualiser frame; 120 121 Updater(SecondMomentVisualiser frame) { 122 this.frame = frame; 123 } 124 125 @Override 126 public void run() { 127 while (true) { 128 frame.draw(); 129 try { 130 Thread.sleep(1000 / 30); 131 } catch (final InterruptedException e) { 132 } 133 } 134 } 135 } 136 image = image.process(new FGaussianConvolve(5)); 137 138 this.mouseFrame = DisplayUtilities.displaySimple(image.clone()); 139 140 this.mouseFrame.getContentPane().addMouseListener(this); 141 this.mouseFrame.getContentPane().addMouseMotionListener(this); 142 143 projectFrame = DisplayUtilities.display(image.clone()); 144 projectFrame.setBounds(image.getWidth(), 0, image.getWidth(), 145 image.getHeight()); 146 featureFrame = DisplayUtilities.display(image.clone()); 147 featureFrame.setBounds(image.getWidth() * 2, 0, image.getWidth(), 148 image.getHeight()); 149 ellipses = new ArrayList<Ellipse>(); 150 lines = new ArrayList<Pair<Line2d>>(); 151 resizeProject = new ResizeProcessor(256, 256); 152 final Thread t = new Thread(new Updater(this)); 153 t.start(); 154 155 } 156 157 /** 158 * Draw 159 */ 160 public synchronized void draw() { 161 final MBFImage toDraw = image.clone(); 162 final MBFImageRenderer renderer = toDraw.createRenderer(); 163 164 if (this.drawPoint != null) 165 renderer.drawPoint(this.drawPoint, RGBColour.RED, 3); 166 167 for (final Ellipse ellipse : ellipses) { 168 renderer.drawShape(ellipse, 1, RGBColour.GREEN); 169 } 170 for (final Pair<Line2d> line : lines) { 171 renderer.drawLine(line.firstObject(), 3, RGBColour.BLUE); 172 renderer.drawLine(line.secondObject(), 3, RGBColour.RED); 173 } 174 if (this.transformMatrix != null) { 175 try { 176 177 final ProjectionProcessor<Float[], MBFImage> pp = new ProjectionProcessor<Float[], MBFImage>(); 178 pp.setMatrix(this.transformMatrix); 179 this.image.accumulateWith(pp); 180 final MBFImage patch = pp.performProjection(-windowSize, 181 windowSize, -windowSize, windowSize, 182 RGBColour.RED); 183 if (patch.getWidth() > 0 && patch.getHeight() > 0) { 184 DisplayUtilities.display(patch.process(this.resizeProject), 185 this.projectFrame); 186 DisplayUtilities.display( 187 patch.extractCenter(this.featureWindowSize, 188 this.featureWindowSize).process( 189 this.resizeProject), this.featureFrame); 190 191 } 192 } catch (final Exception e) { 193 e.printStackTrace(); 194 } 195 196 } 197 198 DisplayUtilities.display(toDraw.clone(), this.mouseFrame); 199 } 200 201 private synchronized void setEBowl() { 202 final Matrix secondMoments = ipd.getSecondMomentsAt( 203 (int) this.drawPoint.getX(), (int) this.drawPoint.getY()); 204 // System.out.println(secondMoments.det()); 205 // secondMoments = secondMoments.times(1/secondMoments.det()); 206 // System.out.println(secondMoments.det()); 207 this.ellipses.clear(); 208 this.lines.clear(); 209 try { 210 getBowlEllipse(secondMoments); 211 } catch (final Exception e) { 212 e.printStackTrace(); 213 } 214 } 215 216 private void getBowlEllipse(Matrix secondMoments) { 217 double rotation = 0; 218 double d1 = 0, d2 = 0; 219 if (secondMoments.det() == 0) 220 return; 221 // If [u v] M [u v]' = E(u,v) 222 // THEN 223 // [u v] (M / E(u,v)) [u v]' = 1 224 // THEN you can go ahead and do the eigen decomp s.t. 225 // (M / E(u,v)) = R' D R 226 // where R is the rotation and D is the size of the ellipse 227 // double divFactor = 1/E; 228 final Matrix noblur = new Matrix(new double[][] { 229 { 230 ipd.lxmxblur.getPixel((int) this.drawPoint.getX(), 231 (int) this.drawPoint.getY()), 232 ipd.lxmyblur.getPixel((int) this.drawPoint.getX(), 233 (int) this.drawPoint.getY()) }, 234 { 235 ipd.lxmyblur.getPixel((int) this.drawPoint.getX(), 236 (int) this.drawPoint.getY()), 237 ipd.lxmxblur.getPixel((int) this.drawPoint.getX(), 238 (int) this.drawPoint.getY()) } }); 239 System.out.println("NO BLUR SECOND MOMENTS MATRIX"); 240 noblur.print(5, 5); 241 System.out.println("det is: " + noblur.det()); 242 if (noblur.det() < 0.00001) 243 return; 244 245 final double divFactor = 1 / Math.sqrt(secondMoments.det()); 246 final double scaleFctor = derivscale; 247 final EigenvalueDecomposition rdr = secondMoments.times(divFactor).eig(); 248 secondMoments.times(divFactor).print(5, 5); 249 250 System.out.println("D1(before)= " + rdr.getD().get(0, 0)); 251 System.out.println("D2(before) = " + rdr.getD().get(1, 1)); 252 253 if (rdr.getD().get(0, 0) == 0) 254 d1 = 0; 255 else 256 d1 = 1.0 / Math.sqrt(rdr.getD().get(0, 0)); 257 // d1 = Math.sqrt(rdr.getD().get(0,0)); 258 if (rdr.getD().get(1, 1) == 0) 259 d2 = 0; 260 else 261 d2 = 1.0 / Math.sqrt(rdr.getD().get(1, 1)); 262 // d2 = Math.sqrt(rdr.getD().get(1,1)); 263 264 final double scaleCorrectedD1 = d1 * scaleFctor * visFactor; 265 final double scaleCorrectedD2 = d2 * scaleFctor * visFactor; 266 267 final Matrix eigenMatrix = rdr.getV(); 268 System.out.println("D1 = " + d1); 269 System.out.println("D2 = " + d2); 270 eigenMatrix.print(5, 5); 271 272 rotation = Math.atan2(eigenMatrix.get(1, 0), eigenMatrix.get(0, 0)); 273 final Ellipse ellipseToAdd = EllipseUtilities.ellipseFromEquation( 274 this.drawPoint.getX(), // center x 275 this.drawPoint.getY(), // center y 276 scaleCorrectedD1, // semi-major axis 277 scaleCorrectedD2, // semi-minor axis 278 rotation// rotation 279 ); 280 ellipses.add(ellipseToAdd); 281 282 if (d1 != 0 && d2 != 0) { 283 this.windowSize = (int) (scaleFctor * d1 / d2) / 2; 284 this.featureWindowSize = (int) scaleFctor; 285 if (this.windowSize > 256) 286 this.windowSize = 256; 287 // this.transformMatrix = affineIPDTransformMatrix(secondMoments); 288 // this.transformMatrix = 289 // secondMomentsTransformMatrix(secondMoments); 290 // this.transformMatrix = 291 // usingEllipseTransformMatrix(d1,d2,rotation); 292 this.transformMatrix = ellipseToAdd 293 .transformMatrix() 294 .times(TransformUtilities.scaleMatrix(1 / scaleFctor, 295 1 / scaleFctor)).inverse(); 296 for (final double d : transformMatrix.getRowPackedCopy()) 297 if (d == Double.NaN) { 298 this.transformMatrix = null; 299 break; 300 } 301 } else { 302 transformMatrix = null; 303 } 304 if (transformMatrix != null) { 305 System.out.println("Transform matrix:"); 306 transformMatrix.print(5, 5); 307 } 308 309 final Line2d major = Line2d.lineFromRotation((int) this.drawPoint.getX(), 310 (int) this.drawPoint.getY(), (float) rotation, 311 (int) scaleCorrectedD1); 312 final Line2d minor = Line2d.lineFromRotation((int) this.drawPoint.getX(), 313 (int) this.drawPoint.getY(), (float) (rotation + Math.PI / 2), 314 (int) scaleCorrectedD2); 315 lines.add(new Pair<Line2d>(major, minor)); 316 } 317 318 // private Matrix usingEllipseTransformMatrix(double major, double minor, 319 // double rotation){ 320 // Matrix rotate = TransformUtilities.rotationMatrix(rotation); 321 // Matrix scale = TransformUtilities.scaleMatrix(major, minor); 322 // Matrix translation = 323 // TransformUtilities.translateMatrix(this.drawPoint.getX(), 324 // this.drawPoint.getY()); 325 // // Matrix transformMatrix = scale.times(translation).times(rotation); 326 // Matrix transformMatrix = translation.times(rotate.times(scale)); 327 // return transformMatrix.inverse(); 328 // } 329 // 330 // private Matrix secondMomentsTransformMatrix(Matrix secondMoments) { 331 // secondMoments = secondMoments.times(1/Math.sqrt(secondMoments.det())); 332 // Matrix eigenMatrix = MatrixUtils.sqrt(secondMoments).inverse(); // This 333 // is simply the rotation (eig vectors) and the scaling by the semi major 334 // and semi minor axis (eig values) 335 // // eigenMatrix = eigenMatrix.inverse(); 336 // Matrix transformMatrix = new Matrix(new double[][]{ 337 // {eigenMatrix.get(0, 0),eigenMatrix.get(0, 1),this.drawPoint.getX()}, 338 // {eigenMatrix.get(1, 0),eigenMatrix.get(1, 1),this.drawPoint.getY()}, 339 // {0,0,1}, 340 // }); 341 // return transformMatrix.inverse(); 342 // } 343 // 344 // 345 // private Matrix affineIPDTransformMatrix(Matrix secondMoments) { 346 // Matrix covar = 347 // AbstractIPD.InterestPointData.getCovarianceMatrix(secondMoments); 348 // Matrix sqrt = MatrixUtils.sqrt(covar); 349 // Matrix transform = new Matrix(new double[][]{ 350 // {sqrt.get(0, 0),sqrt.get(0,1),this.drawPoint.getX()}, 351 // {sqrt.get(1, 0),sqrt.get(1,1),this.drawPoint.getY()}, 352 // {0,0,1}, 353 // }); 354 // return transform.inverse(); 355 // } 356 357 @Override 358 public void mouseClicked(MouseEvent event) { 359 drawPoint = new Point2dImpl(event.getX(), event.getY()); 360 if (this.drawPoint != null) { 361 setEBowl(); 362 } 363 } 364 365 @Override 366 public void mouseEntered(MouseEvent event) { 367 368 } 369 370 @Override 371 public void mouseExited(MouseEvent event) { 372 373 } 374 375 @Override 376 public void mousePressed(MouseEvent event) { 377 378 } 379 380 @Override 381 public void mouseReleased(MouseEvent event) { 382 383 } 384 385 @Override 386 public void mouseDragged(MouseEvent e) { 387 388 } 389 390 @Override 391 public void mouseMoved(MouseEvent e) { 392 drawPoint = new Point2dImpl(e.getX(), e.getY()); 393 if (this.drawPoint != null) { 394 setEBowl(); 395 } 396 } 397 398 /** 399 * The main method 400 * 401 * @param args 402 * ignored 403 * @throws IOException 404 */ 405 public static void main(String args[]) throws IOException { 406 new SecondMomentVisualiser(); 407 } 408}