View Javadoc

1   /**
2    * Copyright (c) 2011, The University of Southampton and the individual contributors.
3    * All rights reserved.
4    *
5    * Redistribution and use in source and binary forms, with or without modification,
6    * are permitted provided that the following conditions are met:
7    *
8    *   * 	Redistributions of source code must retain the above copyright notice,
9    * 	this list of conditions and the following disclaimer.
10   *
11   *   *	Redistributions in binary form must reproduce the above copyright notice,
12   * 	this list of conditions and the following disclaimer in the documentation
13   * 	and/or other materials provided with the distribution.
14   *
15   *   *	Neither the name of the University of Southampton nor the names of its
16   * 	contributors may be used to endorse or promote products derived from this
17   * 	software without specific prior written permission.
18   *
19   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
20   * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21   * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22   * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
23   * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24   * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
26   * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27   * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28   * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29   */
30  package org.openimaj.image.processing.transform;
31  
32  import java.util.ArrayList;
33  import java.util.HashMap;
34  import java.util.List;
35  import java.util.Map;
36  
37  import org.openimaj.image.FImage;
38  import org.openimaj.image.Image;
39  import org.openimaj.image.MBFImage;
40  import org.openimaj.image.combiner.AccumulatingImageCombiner;
41  import org.openimaj.math.geometry.point.Point2d;
42  import org.openimaj.math.geometry.point.Point2dImpl;
43  import org.openimaj.math.geometry.shape.Rectangle;
44  import org.openimaj.math.geometry.shape.Shape;
45  
46  import Jama.Matrix;
47  
48  /**
49   * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
50   * @author Sina Samangooei (ss@ecs.soton.ac.uk)
51   * 
52   *         Perform a set of matrix transforms on a set of images and construct a
53   *         single image containing all the pixels (or a window of the pixels) in
54   *         the projected space.
55   * 
56   * @param <Q>
57   *            The image pixel type
58   * @param <T>
59   *            the image type
60   */
61  public class ProjectionProcessor<Q, T extends Image<Q, T>>
62  		implements
63  		AccumulatingImageCombiner<T, T>
64  {
65  	protected int minc;
66  	protected int minr;
67  	protected int maxc;
68  	protected int maxr;
69  	protected boolean unset;
70  	protected List<Matrix> transforms;
71  	protected List<Matrix> transformsInverted;
72  	protected List<T> images;
73  	protected List<Shape> projectedShapes;
74  	protected List<Rectangle> projectedRectangles;
75  
76  	protected Matrix currentMatrix = new Matrix(new double[][] { { 1, 0, 0 }, { 0, 1, 0 }, { 0, 0, 1 } });
77  
78  	/**
79  	 * Construct a projection processor starting with an identity matrix for any
80  	 * images processed (i.e., don't do anything)
81  	 */
82  	public ProjectionProcessor() {
83  		unset = true;
84  		this.minc = 0;
85  		this.minr = 0;
86  		this.maxc = 0;
87  		this.maxr = 0;
88  
89  		transforms = new ArrayList<Matrix>();
90  		this.transformsInverted = new ArrayList<Matrix>();
91  		images = new ArrayList<T>();
92  		this.projectedShapes = new ArrayList<Shape>();
93  		this.projectedRectangles = new ArrayList<Rectangle>();
94  	}
95  
96  	/**
97  	 * Set the matrix, any images processed from this point forward will be
98  	 * projected using this matrix
99  	 * 
100 	 * @param matrix
101 	 *            a 3x3 matrix representing a 2d transform
102 	 */
103 	public void setMatrix(Matrix matrix) {
104 		if (matrix.getRowDimension() == 2) {
105 			final int c = matrix.getColumnDimension() - 1;
106 
107 			currentMatrix = new Matrix(3, 3);
108 			currentMatrix.setMatrix(0, 1, 0, c, matrix);
109 			currentMatrix.set(2, 2, 1);
110 		} else {
111 			this.currentMatrix = matrix;
112 		}
113 	}
114 
115 	/**
116 	 * Prepare an image to be transformed using the current matrix. The bounds
117 	 * of the image post transform are calculated so the default
118 	 * {@link ProjectionProcessor#performProjection} knows what range of pixels
119 	 * to draw
120 	 * 
121 	 * @param image
122 	 *            to be transformed
123 	 */
124 	@Override
125 	public void accumulate(T image) {
126 		final Rectangle actualBounds = image.getBounds();
127 		final Shape transformedActualBounds = actualBounds.transform(this.currentMatrix);
128 		final double tminX = transformedActualBounds.minX();
129 		final double tmaxX = transformedActualBounds.maxX();
130 		final double tminY = transformedActualBounds.minY();
131 		final double tmaxY = transformedActualBounds.maxY();
132 		if (unset) {
133 			this.minc = (int) Math.floor(tminX);
134 			this.minr = (int) Math.floor(tminY);
135 			this.maxc = (int) Math.floor(tmaxX);
136 			this.maxr = (int) Math.floor(tmaxY);
137 			unset = false;
138 		}
139 		else {
140 			if (tminX < minc)
141 				minc = (int) Math.floor(tminX);
142 			if (tmaxX > maxc)
143 				maxc = (int) Math.floor(tmaxX);
144 			if (tminY < minr)
145 				minr = (int) Math.floor(tminY);
146 			if (tmaxY > maxr)
147 				maxr = (int) Math.floor(tmaxY);
148 		}
149 		// Expand the borders by 1 pixel so we get a nicer effect around the
150 		// edges
151 		final float padding = 1f;
152 		final Rectangle expandedBounds = new Rectangle(actualBounds.x - padding, actualBounds.y - padding,
153 				actualBounds.width + padding * 2, actualBounds.height + padding * 2);
154 		final Shape transformedExpandedBounds = expandedBounds.transform(this.currentMatrix);
155 		Matrix minv = null, m = null;
156 		try {
157 			m = this.currentMatrix.copy();
158 			minv = this.currentMatrix.copy().inverse();
159 		} catch (final Throwable e) {
160 			// the matrix might be singular, return
161 			return;
162 		}
163 
164 		this.images.add(image);
165 		this.transforms.add(m);
166 		this.transformsInverted.add(minv);
167 		// this.projectedShapes.add(new
168 		// TriangulatedPolygon(transformedExpandedBounds));
169 		this.projectedShapes.add(transformedExpandedBounds);
170 		this.projectedRectangles.add(transformedExpandedBounds.calculateRegularBoundingBox());
171 
172 		// System.out.println("added image with transform: ");
173 		// this.currentMatrix.print(5,5);
174 		// System.out.println("and the inverse:");
175 		// this.currentMatrix.inverse().print(5,5);
176 		// System.out.println("New min/max become:" + minc + "x" + minr + "/" +
177 		// maxc + "x" + maxr);
178 	}
179 
180 	/**
181 	 * Using all the images currently processed, perform the projection on each
182 	 * image and draw every pixel with valid data. Pixels within the bounding
183 	 * box but with no data are set to black (more specifically 0, whatever that
184 	 * may mean for this kind of image)
185 	 * 
186 	 * @return the image containing all the pixels drawn
187 	 */
188 	public T performProjection() {
189 		// The most long winded way to get a black pixel EVER
190 		return performProjection(false, this.images.get(0).newInstance(1, 1).getPixel(0, 0));
191 	}
192 
193 	/**
194 	 * Perform projection specifying the background colour (i.e. the colour of
195 	 * pixels with no data).
196 	 * 
197 	 * @param backgroundColour
198 	 *            the background colour
199 	 * @return projected images
200 	 */
201 	public T performProjection(Q backgroundColour) {
202 		final int projectionMinC = minc, projectionMaxC = maxc, projectionMinR = minr, projectionMaxR = maxr;
203 		return performProjection(projectionMinC, projectionMaxC, projectionMinR, projectionMaxR, backgroundColour);
204 	}
205 
206 	/**
207 	 * Perform projection specifying the background colour (i.e. the colour of
208 	 * pixels with no data) and whether the original window size should be kept.
209 	 * If set to true the window of pixels drawn post projection are within the
210 	 * window of the first image processed.
211 	 * 
212 	 * @param keepOriginalWindow
213 	 *            whether to keep the original image's window
214 	 * @param backgroundColour
215 	 *            the background colour
216 	 * @return projected images
217 	 */
218 	public T performProjection(boolean keepOriginalWindow, Q backgroundColour) {
219 		int projectionMinC = minc, projectionMaxC = maxc, projectionMinR = minr, projectionMaxR = maxr;
220 		if (keepOriginalWindow)
221 		{
222 			projectionMinC = 0;
223 			projectionMinR = 0;
224 			projectionMaxR = images.get(0).getRows();
225 			projectionMaxC = images.get(0).getCols();
226 		}
227 		return performProjection(projectionMinC, projectionMaxC, projectionMinR, projectionMaxR, backgroundColour);
228 	}
229 
230 	/**
231 	 * Perform projection but only request data for pixels within the windowed
232 	 * range provided. Specify the background colour, i.e. the value of pixels
233 	 * with no data post projection.
234 	 * 
235 	 * @param windowMinC
236 	 *            left X
237 	 * @param windowMaxC
238 	 *            right X
239 	 * @param windowMinR
240 	 *            top Y
241 	 * @param windowMaxR
242 	 *            bottom Y
243 	 * @return projected image within the window
244 	 */
245 	public T performProjection(int windowMinC, int windowMaxC, int windowMinR, int windowMaxR) {
246 		return performProjection(windowMinC, windowMaxC, windowMinR, windowMaxR, this.images.get(0).newInstance(1, 1)
247 				.getPixel(0, 0));
248 	}
249 
250 	/**
251 	 * Perform projection but only request data for pixels within the windowed
252 	 * range provided. Specify the background colour, i.e. the value of pixels
253 	 * with no data post projection.
254 	 * 
255 	 * @param windowMinC
256 	 *            left X
257 	 * @param windowMaxC
258 	 *            right X
259 	 * @param windowMinR
260 	 *            top Y
261 	 * @param windowMaxR
262 	 *            bottom Y
263 	 * @param backgroundColour
264 	 *            background colour of pixels with no data
265 	 * @return projected image within the window
266 	 */
267 	public T performProjection(int windowMinC, int windowMaxC, int windowMinR, int windowMaxR, Q backgroundColour) {
268 		T output = null;
269 		output = images.get(0).newInstance(windowMaxC - windowMinC, windowMaxR - windowMinR);
270 		if (backgroundColour != null)
271 			output.fill(backgroundColour);
272 
273 		final Shape[][] projectRectangleShapes = getCurrentShapes();
274 
275 		for (int y = 0; y < output.getHeight(); y++)
276 		{
277 			for (int x = 0; x < output.getWidth(); x++) {
278 				final Point2d realPoint = new Point2dImpl(windowMinC + x, windowMinR + y);
279 				int i = 0;
280 				for (int shapeIndex = 0; shapeIndex < this.projectedShapes.size(); shapeIndex++) {
281 					if (backgroundColour == null || isInside(shapeIndex, projectRectangleShapes, realPoint)) {
282 						final double[][] transform = this.transformsInverted.get(i).getArray();
283 
284 						float xt = (float) transform[0][0] * realPoint.getX() + (float) transform[0][1]
285 								* realPoint.getY() + (float) transform[0][2];
286 						float yt = (float) transform[1][0] * realPoint.getX() + (float) transform[1][1]
287 								* realPoint.getY() + (float) transform[1][2];
288 						final float zt = (float) transform[2][0] * realPoint.getX() + (float) transform[2][1]
289 								* realPoint.getY() + (float) transform[2][2];
290 
291 						xt /= zt;
292 						yt /= zt;
293 						final T im = this.images.get(i);
294 						if (backgroundColour != null)
295 							output.setPixel(x, y, im.getPixelInterp(xt, yt, backgroundColour));
296 						else
297 							output.setPixel(x, y, im.getPixelInterp(xt, yt));
298 					}
299 					i++;
300 				}
301 			}
302 		}
303 		return output;
304 	}
305 
306 	/**
307 	 * Get the current shapes as an array for efficient access, first entry for
308 	 * each shape is its rectangle, second entry is the shape
309 	 * 
310 	 * @return
311 	 */
312 	protected Shape[][] getCurrentShapes() {
313 		final Shape[][] currentShapes = new Shape[this.projectedShapes.size()][2];
314 		for (int i = 0; i < this.projectedShapes.size(); i++) {
315 			currentShapes[i][0] = this.projectedRectangles.get(i);
316 			currentShapes[i][1] = this.projectedShapes.get(i);
317 		}
318 		return currentShapes;
319 	}
320 
321 	protected boolean isInside(int shapeIndex, Shape[][] projectRectangleShapes, Point2d realPoint) {
322 		return projectRectangleShapes[shapeIndex][0].isInside(realPoint)
323 				&& projectRectangleShapes[shapeIndex][1].isInside(realPoint);
324 	}
325 
326 	/**
327 	 * Perform projection but only request data for pixels within the windowed
328 	 * range provided. Specify the background colour, i.e. the value of pixels
329 	 * with no data post projection.
330 	 * 
331 	 * @param windowMinC
332 	 *            left X
333 	 * @param windowMinR
334 	 *            top Y
335 	 * @param output
336 	 *            the target image in which to project
337 	 * @return projected image within the window
338 	 */
339 	public T performProjection(int windowMinC, int windowMinR, T output) {
340 
341 		for (int y = 0; y < output.getHeight(); y++)
342 		{
343 			for (int x = 0; x < output.getWidth(); x++) {
344 				final Point2d realPoint = new Point2dImpl(windowMinC + x, windowMinR + y);
345 				int i = 0;
346 				for (final Shape s : this.projectedShapes) {
347 					if (s.calculateRegularBoundingBox().isInside(realPoint) && s.isInside(realPoint)) {
348 						final double[][] transform = this.transformsInverted.get(i).getArray();
349 
350 						float xt = (float) transform[0][0] * realPoint.getX() + (float) transform[0][1]
351 								* realPoint.getY() + (float) transform[0][2];
352 						float yt = (float) transform[1][0] * realPoint.getX() + (float) transform[1][1]
353 								* realPoint.getY() + (float) transform[1][2];
354 						final float zt = (float) transform[2][0] * realPoint.getX() + (float) transform[2][1]
355 								* realPoint.getY() + (float) transform[2][2];
356 
357 						xt /= zt;
358 						yt /= zt;
359 						final T im = this.images.get(i);
360 						output.setPixel(x, y, im.getPixelInterp(xt, yt, output.getPixel(x, y)));
361 					}
362 					i++;
363 				}
364 			}
365 		}
366 
367 		return output;
368 	}
369 
370 	/**
371 	 * Perform blended projection but only request data for pixels within the
372 	 * windowed range provided. Specify the background colour, i.e. the value of
373 	 * pixels with no data post projection. This blends any existing pixels to
374 	 * newly added pixels
375 	 * 
376 	 * @param windowMinC
377 	 *            left X
378 	 * @param windowMaxC
379 	 *            right X
380 	 * @param windowMinR
381 	 *            top Y
382 	 * @param windowMaxR
383 	 *            bottom Y
384 	 * @param backgroundColour
385 	 *            background colour of pixels with no data
386 	 * @return projected image within the window
387 	 */
388 	public T performBlendedProjection(int windowMinC, int windowMaxC, int windowMinR, int windowMaxR, Q backgroundColour)
389 	{
390 		T output = null;
391 		output = images.get(0).newInstance(windowMaxC - windowMinC, windowMaxR - windowMinR);
392 		final Map<Integer, Boolean> setMap = new HashMap<Integer, Boolean>();
393 		final T blendingPallet = output.newInstance(2, 1);
394 		for (int y = 0; y < output.getHeight(); y++)
395 		{
396 			for (int x = 0; x < output.getWidth(); x++) {
397 				final Point2d realPoint = new Point2dImpl(windowMinC + x, windowMinR + y);
398 				int i = 0;
399 				for (final Shape s : this.projectedShapes) {
400 					if (s.isInside(realPoint)) {
401 						final double[][] transform = this.transformsInverted.get(i).getArray();
402 
403 						float xt = (float) transform[0][0] * realPoint.getX() + (float) transform[0][1]
404 								* realPoint.getY() + (float) transform[0][2];
405 						float yt = (float) transform[1][0] * realPoint.getX() + (float) transform[1][1]
406 								* realPoint.getY() + (float) transform[1][2];
407 						final float zt = (float) transform[2][0] * realPoint.getX() + (float) transform[2][1]
408 								* realPoint.getY() + (float) transform[2][2];
409 
410 						xt /= zt;
411 						yt /= zt;
412 						Q toSet = null;
413 						if (backgroundColour != null)
414 							toSet = this.images.get(i).getPixelInterp(xt, yt, backgroundColour);
415 						else if (setMap.get(y * output.getWidth() + x) != null)
416 							toSet = this.images.get(i).getPixelInterp(xt, yt, output.getPixelInterp(x, y));
417 						else
418 							toSet = this.images.get(i).getPixelInterp(xt, yt);
419 						// Blend the pixel with the existing pixel
420 						if (setMap.get(y * output.getWidth() + x) != null) {
421 							blendingPallet.setPixel(1, 0, toSet);
422 							blendingPallet.setPixel(0, 0, output.getPixel(x, y));
423 
424 							toSet = blendingPallet.getPixelInterp(0.1, 0.5);
425 						}
426 						setMap.put(y * output.getWidth() + x, true);
427 						output.setPixel(x, y, toSet);
428 					}
429 					i++;
430 				}
431 			}
432 		}
433 		return output;
434 	}
435 
436 	/**
437 	 * @return Current matrix
438 	 */
439 	public Matrix getMatrix() {
440 		return this.currentMatrix;
441 	}
442 
443 	/**
444 	 * Utility function, project one image with one matrix. Every valid pixel in
445 	 * the space the image is projected into is displayed in the final image.
446 	 * 
447 	 * @param <Q>
448 	 *            the image pixel type
449 	 * @param <T>
450 	 *            image type
451 	 * @param image
452 	 *            the image to project
453 	 * @param matrix
454 	 *            the matrix to project against
455 	 * @return projected image
456 	 */
457 	@SuppressWarnings("unchecked")
458 	public static <Q, T extends Image<Q, T>> T project(T image, Matrix matrix) {
459 		// Note: extra casts to work around compiler bug
460 		if ((Image<?, ?>) image instanceof FImage) {
461 			final FProjectionProcessor proc = new FProjectionProcessor();
462 			proc.setMatrix(matrix);
463 			((FImage) (Image<?, ?>) image).accumulateWith(proc);
464 			return (T) (Image<?, ?>) proc.performProjection();
465 		}
466 		if ((Image<?, ?>) image instanceof MBFImage) {
467 			final MBFProjectionProcessor proc = new MBFProjectionProcessor();
468 			proc.setMatrix(matrix);
469 			((MBFImage) (Image<?, ?>) image).accumulateWith(proc);
470 			return (T) (Image<?, ?>) proc.performProjection();
471 		} else {
472 			final ProjectionProcessor<Q, T> proc = new ProjectionProcessor<Q, T>();
473 			proc.setMatrix(matrix);
474 			image.accumulateWith(proc);
475 			return proc.performProjection();
476 		}
477 	}
478 
479 	/**
480 	 * Utility function, project one image with one matrix. Every valid pixel in
481 	 * the space the image is projected into is displayed in the final image.
482 	 * 
483 	 * @param <Q>
484 	 *            the image pixel type
485 	 * @param <T>
486 	 *            image type
487 	 * @param image
488 	 *            the image to project
489 	 * @param matrix
490 	 *            the matrix to project against
491 	 * @param backgroundColour
492 	 *            The colour of pixels with no data
493 	 * @return projected image
494 	 */
495 	public static <Q, T extends Image<Q, T>> T project(T image, Matrix matrix, Q backgroundColour) {
496 		final ProjectionProcessor<Q, T> proc = new ProjectionProcessor<Q, T>();
497 		proc.setMatrix(matrix);
498 		image.accumulateWith(proc);
499 		return proc.performProjection(backgroundColour);
500 	}
501 
502 	@Override
503 	public T combine() {
504 		return performProjection();
505 	}
506 }