001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.image.processing.transform; 031 032import java.util.ArrayList; 033import java.util.HashMap; 034import java.util.List; 035import java.util.Map; 036 037import org.openimaj.image.FImage; 038import org.openimaj.image.Image; 039import org.openimaj.image.MBFImage; 040import org.openimaj.image.combiner.AccumulatingImageCombiner; 041import org.openimaj.math.geometry.point.Point2d; 042import org.openimaj.math.geometry.point.Point2dImpl; 043import org.openimaj.math.geometry.shape.Rectangle; 044import org.openimaj.math.geometry.shape.Shape; 045 046import Jama.Matrix; 047 048/** 049 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk) 050 * @author Sina Samangooei (ss@ecs.soton.ac.uk) 051 * 052 * Perform a set of matrix transforms on a set of images and construct a 053 * single image containing all the pixels (or a window of the pixels) in 054 * the projected space. 055 * 056 * @param <Q> 057 * The image pixel type 058 * @param <T> 059 * the image type 060 */ 061public class ProjectionProcessor<Q, T extends Image<Q, T>> 062 implements 063 AccumulatingImageCombiner<T, T> 064{ 065 protected int minc; 066 protected int minr; 067 protected int maxc; 068 protected int maxr; 069 protected boolean unset; 070 protected List<Matrix> transforms; 071 protected List<Matrix> transformsInverted; 072 protected List<T> images; 073 protected List<Shape> projectedShapes; 074 protected List<Rectangle> projectedRectangles; 075 076 protected Matrix currentMatrix = new Matrix(new double[][] { { 1, 0, 0 }, { 0, 1, 0 }, { 0, 0, 1 } }); 077 078 /** 079 * Construct a projection processor starting with an identity matrix for any 080 * images processed (i.e., don't do anything) 081 */ 082 public ProjectionProcessor() { 083 unset = true; 084 this.minc = 0; 085 this.minr = 0; 086 this.maxc = 0; 087 this.maxr = 0; 088 089 transforms = new ArrayList<Matrix>(); 090 this.transformsInverted = new ArrayList<Matrix>(); 091 images = new ArrayList<T>(); 092 this.projectedShapes = new ArrayList<Shape>(); 093 this.projectedRectangles = new ArrayList<Rectangle>(); 094 } 095 096 /** 097 * Set the matrix, any images processed from this point forward will be 098 * projected using this matrix 099 * 100 * @param matrix 101 * a 3x3 matrix representing a 2d transform 102 */ 103 public void setMatrix(Matrix matrix) { 104 if (matrix.getRowDimension() == 2) { 105 final int c = matrix.getColumnDimension() - 1; 106 107 currentMatrix = new Matrix(3, 3); 108 currentMatrix.setMatrix(0, 1, 0, c, matrix); 109 currentMatrix.set(2, 2, 1); 110 } else { 111 this.currentMatrix = matrix; 112 } 113 } 114 115 /** 116 * Prepare an image to be transformed using the current matrix. The bounds 117 * of the image post transform are calculated so the default 118 * {@link ProjectionProcessor#performProjection} knows what range of pixels 119 * to draw 120 * 121 * @param image 122 * to be transformed 123 */ 124 @Override 125 public void accumulate(T image) { 126 final Rectangle actualBounds = image.getBounds(); 127 final Shape transformedActualBounds = actualBounds.transform(this.currentMatrix); 128 final double tminX = transformedActualBounds.minX(); 129 final double tmaxX = transformedActualBounds.maxX(); 130 final double tminY = transformedActualBounds.minY(); 131 final double tmaxY = transformedActualBounds.maxY(); 132 if (unset) { 133 this.minc = (int) Math.floor(tminX); 134 this.minr = (int) Math.floor(tminY); 135 this.maxc = (int) Math.floor(tmaxX); 136 this.maxr = (int) Math.floor(tmaxY); 137 unset = false; 138 } 139 else { 140 if (tminX < minc) 141 minc = (int) Math.floor(tminX); 142 if (tmaxX > maxc) 143 maxc = (int) Math.floor(tmaxX); 144 if (tminY < minr) 145 minr = (int) Math.floor(tminY); 146 if (tmaxY > maxr) 147 maxr = (int) Math.floor(tmaxY); 148 } 149 // Expand the borders by 1 pixel so we get a nicer effect around the 150 // edges 151 final float padding = 1f; 152 final Rectangle expandedBounds = new Rectangle(actualBounds.x - padding, actualBounds.y - padding, 153 actualBounds.width + padding * 2, actualBounds.height + padding * 2); 154 final Shape transformedExpandedBounds = expandedBounds.transform(this.currentMatrix); 155 Matrix minv = null, m = null; 156 try { 157 m = this.currentMatrix.copy(); 158 minv = this.currentMatrix.copy().inverse(); 159 } catch (final Throwable e) { 160 // the matrix might be singular, return 161 return; 162 } 163 164 this.images.add(image); 165 this.transforms.add(m); 166 this.transformsInverted.add(minv); 167 // this.projectedShapes.add(new 168 // TriangulatedPolygon(transformedExpandedBounds)); 169 this.projectedShapes.add(transformedExpandedBounds); 170 this.projectedRectangles.add(transformedExpandedBounds.calculateRegularBoundingBox()); 171 172 // System.out.println("added image with transform: "); 173 // this.currentMatrix.print(5,5); 174 // System.out.println("and the inverse:"); 175 // this.currentMatrix.inverse().print(5,5); 176 // System.out.println("New min/max become:" + minc + "x" + minr + "/" + 177 // maxc + "x" + maxr); 178 } 179 180 /** 181 * Using all the images currently processed, perform the projection on each 182 * image and draw every pixel with valid data. Pixels within the bounding 183 * box but with no data are set to black (more specifically 0, whatever that 184 * may mean for this kind of image) 185 * 186 * @return the image containing all the pixels drawn 187 */ 188 public T performProjection() { 189 // The most long winded way to get a black pixel EVER 190 return performProjection(false, this.images.get(0).newInstance(1, 1).getPixel(0, 0)); 191 } 192 193 /** 194 * Perform projection specifying the background colour (i.e. the colour of 195 * pixels with no data). 196 * 197 * @param backgroundColour 198 * the background colour 199 * @return projected images 200 */ 201 public T performProjection(Q backgroundColour) { 202 final int projectionMinC = minc, projectionMaxC = maxc, projectionMinR = minr, projectionMaxR = maxr; 203 return performProjection(projectionMinC, projectionMaxC, projectionMinR, projectionMaxR, backgroundColour); 204 } 205 206 /** 207 * Perform projection specifying the background colour (i.e. the colour of 208 * pixels with no data) and whether the original window size should be kept. 209 * If set to true the window of pixels drawn post projection are within the 210 * window of the first image processed. 211 * 212 * @param keepOriginalWindow 213 * whether to keep the original image's window 214 * @param backgroundColour 215 * the background colour 216 * @return projected images 217 */ 218 public T performProjection(boolean keepOriginalWindow, Q backgroundColour) { 219 int projectionMinC = minc, projectionMaxC = maxc, projectionMinR = minr, projectionMaxR = maxr; 220 if (keepOriginalWindow) 221 { 222 projectionMinC = 0; 223 projectionMinR = 0; 224 projectionMaxR = images.get(0).getRows(); 225 projectionMaxC = images.get(0).getCols(); 226 } 227 return performProjection(projectionMinC, projectionMaxC, projectionMinR, projectionMaxR, backgroundColour); 228 } 229 230 /** 231 * Perform projection but only request data for pixels within the windowed 232 * range provided. Specify the background colour, i.e. the value of pixels 233 * with no data post projection. 234 * 235 * @param windowMinC 236 * left X 237 * @param windowMaxC 238 * right X 239 * @param windowMinR 240 * top Y 241 * @param windowMaxR 242 * bottom Y 243 * @return projected image within the window 244 */ 245 public T performProjection(int windowMinC, int windowMaxC, int windowMinR, int windowMaxR) { 246 return performProjection(windowMinC, windowMaxC, windowMinR, windowMaxR, this.images.get(0).newInstance(1, 1) 247 .getPixel(0, 0)); 248 } 249 250 /** 251 * Perform projection but only request data for pixels within the windowed 252 * range provided. Specify the background colour, i.e. the value of pixels 253 * with no data post projection. 254 * 255 * @param windowMinC 256 * left X 257 * @param windowMaxC 258 * right X 259 * @param windowMinR 260 * top Y 261 * @param windowMaxR 262 * bottom Y 263 * @param backgroundColour 264 * background colour of pixels with no data 265 * @return projected image within the window 266 */ 267 public T performProjection(int windowMinC, int windowMaxC, int windowMinR, int windowMaxR, Q backgroundColour) { 268 T output = null; 269 output = images.get(0).newInstance(windowMaxC - windowMinC, windowMaxR - windowMinR); 270 if (backgroundColour != null) 271 output.fill(backgroundColour); 272 273 final Shape[][] projectRectangleShapes = getCurrentShapes(); 274 275 for (int y = 0; y < output.getHeight(); y++) 276 { 277 for (int x = 0; x < output.getWidth(); x++) { 278 final Point2d realPoint = new Point2dImpl(windowMinC + x, windowMinR + y); 279 int i = 0; 280 for (int shapeIndex = 0; shapeIndex < this.projectedShapes.size(); shapeIndex++) { 281 if (backgroundColour == null || isInside(shapeIndex, projectRectangleShapes, realPoint)) { 282 final double[][] transform = this.transformsInverted.get(i).getArray(); 283 284 float xt = (float) transform[0][0] * realPoint.getX() + (float) transform[0][1] 285 * realPoint.getY() + (float) transform[0][2]; 286 float yt = (float) transform[1][0] * realPoint.getX() + (float) transform[1][1] 287 * realPoint.getY() + (float) transform[1][2]; 288 final float zt = (float) transform[2][0] * realPoint.getX() + (float) transform[2][1] 289 * realPoint.getY() + (float) transform[2][2]; 290 291 xt /= zt; 292 yt /= zt; 293 final T im = this.images.get(i); 294 if (backgroundColour != null) 295 output.setPixel(x, y, im.getPixelInterp(xt, yt, backgroundColour)); 296 else 297 output.setPixel(x, y, im.getPixelInterp(xt, yt)); 298 } 299 i++; 300 } 301 } 302 } 303 return output; 304 } 305 306 /** 307 * Get the current shapes as an array for efficient access, first entry for 308 * each shape is its rectangle, second entry is the shape 309 * 310 * @return 311 */ 312 protected Shape[][] getCurrentShapes() { 313 final Shape[][] currentShapes = new Shape[this.projectedShapes.size()][2]; 314 for (int i = 0; i < this.projectedShapes.size(); i++) { 315 currentShapes[i][0] = this.projectedRectangles.get(i); 316 currentShapes[i][1] = this.projectedShapes.get(i); 317 } 318 return currentShapes; 319 } 320 321 protected boolean isInside(int shapeIndex, Shape[][] projectRectangleShapes, Point2d realPoint) { 322 return projectRectangleShapes[shapeIndex][0].isInside(realPoint) 323 && projectRectangleShapes[shapeIndex][1].isInside(realPoint); 324 } 325 326 /** 327 * Perform projection but only request data for pixels within the windowed 328 * range provided. Specify the background colour, i.e. the value of pixels 329 * with no data post projection. 330 * 331 * @param windowMinC 332 * left X 333 * @param windowMinR 334 * top Y 335 * @param output 336 * the target image in which to project 337 * @return projected image within the window 338 */ 339 public T performProjection(int windowMinC, int windowMinR, T output) { 340 341 for (int y = 0; y < output.getHeight(); y++) 342 { 343 for (int x = 0; x < output.getWidth(); x++) { 344 final Point2d realPoint = new Point2dImpl(windowMinC + x, windowMinR + y); 345 int i = 0; 346 for (final Shape s : this.projectedShapes) { 347 if (s.calculateRegularBoundingBox().isInside(realPoint) && s.isInside(realPoint)) { 348 final double[][] transform = this.transformsInverted.get(i).getArray(); 349 350 float xt = (float) transform[0][0] * realPoint.getX() + (float) transform[0][1] 351 * realPoint.getY() + (float) transform[0][2]; 352 float yt = (float) transform[1][0] * realPoint.getX() + (float) transform[1][1] 353 * realPoint.getY() + (float) transform[1][2]; 354 final float zt = (float) transform[2][0] * realPoint.getX() + (float) transform[2][1] 355 * realPoint.getY() + (float) transform[2][2]; 356 357 xt /= zt; 358 yt /= zt; 359 final T im = this.images.get(i); 360 output.setPixel(x, y, im.getPixelInterp(xt, yt, output.getPixel(x, y))); 361 } 362 i++; 363 } 364 } 365 } 366 367 return output; 368 } 369 370 /** 371 * Perform blended projection but only request data for pixels within the 372 * windowed range provided. Specify the background colour, i.e. the value of 373 * pixels with no data post projection. This blends any existing pixels to 374 * newly added pixels 375 * 376 * @param windowMinC 377 * left X 378 * @param windowMaxC 379 * right X 380 * @param windowMinR 381 * top Y 382 * @param windowMaxR 383 * bottom Y 384 * @param backgroundColour 385 * background colour of pixels with no data 386 * @return projected image within the window 387 */ 388 public T performBlendedProjection(int windowMinC, int windowMaxC, int windowMinR, int windowMaxR, Q backgroundColour) 389 { 390 T output = null; 391 output = images.get(0).newInstance(windowMaxC - windowMinC, windowMaxR - windowMinR); 392 final Map<Integer, Boolean> setMap = new HashMap<Integer, Boolean>(); 393 final T blendingPallet = output.newInstance(2, 1); 394 for (int y = 0; y < output.getHeight(); y++) 395 { 396 for (int x = 0; x < output.getWidth(); x++) { 397 final Point2d realPoint = new Point2dImpl(windowMinC + x, windowMinR + y); 398 int i = 0; 399 for (final Shape s : this.projectedShapes) { 400 if (s.isInside(realPoint)) { 401 final double[][] transform = this.transformsInverted.get(i).getArray(); 402 403 float xt = (float) transform[0][0] * realPoint.getX() + (float) transform[0][1] 404 * realPoint.getY() + (float) transform[0][2]; 405 float yt = (float) transform[1][0] * realPoint.getX() + (float) transform[1][1] 406 * realPoint.getY() + (float) transform[1][2]; 407 final float zt = (float) transform[2][0] * realPoint.getX() + (float) transform[2][1] 408 * realPoint.getY() + (float) transform[2][2]; 409 410 xt /= zt; 411 yt /= zt; 412 Q toSet = null; 413 if (backgroundColour != null) 414 toSet = this.images.get(i).getPixelInterp(xt, yt, backgroundColour); 415 else if (setMap.get(y * output.getWidth() + x) != null) 416 toSet = this.images.get(i).getPixelInterp(xt, yt, output.getPixelInterp(x, y)); 417 else 418 toSet = this.images.get(i).getPixelInterp(xt, yt); 419 // Blend the pixel with the existing pixel 420 if (setMap.get(y * output.getWidth() + x) != null) { 421 blendingPallet.setPixel(1, 0, toSet); 422 blendingPallet.setPixel(0, 0, output.getPixel(x, y)); 423 424 toSet = blendingPallet.getPixelInterp(0.1, 0.5); 425 } 426 setMap.put(y * output.getWidth() + x, true); 427 output.setPixel(x, y, toSet); 428 } 429 i++; 430 } 431 } 432 } 433 return output; 434 } 435 436 /** 437 * @return Current matrix 438 */ 439 public Matrix getMatrix() { 440 return this.currentMatrix; 441 } 442 443 /** 444 * Utility function, project one image with one matrix. Every valid pixel in 445 * the space the image is projected into is displayed in the final image. 446 * 447 * @param <Q> 448 * the image pixel type 449 * @param <T> 450 * image type 451 * @param image 452 * the image to project 453 * @param matrix 454 * the matrix to project against 455 * @return projected image 456 */ 457 @SuppressWarnings("unchecked") 458 public static <Q, T extends Image<Q, T>> T project(T image, Matrix matrix) { 459 // Note: extra casts to work around compiler bug 460 if ((Image<?, ?>) image instanceof FImage) { 461 final FProjectionProcessor proc = new FProjectionProcessor(); 462 proc.setMatrix(matrix); 463 ((FImage) (Image<?, ?>) image).accumulateWith(proc); 464 return (T) (Image<?, ?>) proc.performProjection(); 465 } 466 if ((Image<?, ?>) image instanceof MBFImage) { 467 final MBFProjectionProcessor proc = new MBFProjectionProcessor(); 468 proc.setMatrix(matrix); 469 ((MBFImage) (Image<?, ?>) image).accumulateWith(proc); 470 return (T) (Image<?, ?>) proc.performProjection(); 471 } else { 472 final ProjectionProcessor<Q, T> proc = new ProjectionProcessor<Q, T>(); 473 proc.setMatrix(matrix); 474 image.accumulateWith(proc); 475 return proc.performProjection(); 476 } 477 } 478 479 /** 480 * Utility function, project one image with one matrix. Every valid pixel in 481 * the space the image is projected into is displayed in the final image. 482 * 483 * @param <Q> 484 * the image pixel type 485 * @param <T> 486 * image type 487 * @param image 488 * the image to project 489 * @param matrix 490 * the matrix to project against 491 * @param backgroundColour 492 * The colour of pixels with no data 493 * @return projected image 494 */ 495 public static <Q, T extends Image<Q, T>> T project(T image, Matrix matrix, Q backgroundColour) { 496 final ProjectionProcessor<Q, T> proc = new ProjectionProcessor<Q, T>(); 497 proc.setMatrix(matrix); 498 image.accumulateWith(proc); 499 return proc.performProjection(backgroundColour); 500 } 501 502 @Override 503 public T combine() { 504 return performProjection(); 505 } 506}