001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.image.processing.transform;
031
032import java.util.ArrayList;
033import java.util.HashMap;
034import java.util.List;
035import java.util.Map;
036
037import org.openimaj.image.FImage;
038import org.openimaj.image.Image;
039import org.openimaj.image.MBFImage;
040import org.openimaj.image.combiner.AccumulatingImageCombiner;
041import org.openimaj.math.geometry.point.Point2d;
042import org.openimaj.math.geometry.point.Point2dImpl;
043import org.openimaj.math.geometry.shape.Rectangle;
044import org.openimaj.math.geometry.shape.Shape;
045
046import Jama.Matrix;
047
048/**
049 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
050 * @author Sina Samangooei (ss@ecs.soton.ac.uk)
051 * 
052 *         Perform a set of matrix transforms on a set of images and construct a
053 *         single image containing all the pixels (or a window of the pixels) in
054 *         the projected space.
055 * 
056 * @param <Q>
057 *            The image pixel type
058 * @param <T>
059 *            the image type
060 */
061public class ProjectionProcessor<Q, T extends Image<Q, T>>
062                implements
063                AccumulatingImageCombiner<T, T>
064{
065        protected int minc;
066        protected int minr;
067        protected int maxc;
068        protected int maxr;
069        protected boolean unset;
070        protected List<Matrix> transforms;
071        protected List<Matrix> transformsInverted;
072        protected List<T> images;
073        protected List<Shape> projectedShapes;
074        protected List<Rectangle> projectedRectangles;
075
076        protected Matrix currentMatrix = new Matrix(new double[][] { { 1, 0, 0 }, { 0, 1, 0 }, { 0, 0, 1 } });
077
078        /**
079         * Construct a projection processor starting with an identity matrix for any
080         * images processed (i.e., don't do anything)
081         */
082        public ProjectionProcessor() {
083                unset = true;
084                this.minc = 0;
085                this.minr = 0;
086                this.maxc = 0;
087                this.maxr = 0;
088
089                transforms = new ArrayList<Matrix>();
090                this.transformsInverted = new ArrayList<Matrix>();
091                images = new ArrayList<T>();
092                this.projectedShapes = new ArrayList<Shape>();
093                this.projectedRectangles = new ArrayList<Rectangle>();
094        }
095
096        /**
097         * Set the matrix, any images processed from this point forward will be
098         * projected using this matrix
099         * 
100         * @param matrix
101         *            a 3x3 matrix representing a 2d transform
102         */
103        public void setMatrix(Matrix matrix) {
104                if (matrix.getRowDimension() == 2) {
105                        final int c = matrix.getColumnDimension() - 1;
106
107                        currentMatrix = new Matrix(3, 3);
108                        currentMatrix.setMatrix(0, 1, 0, c, matrix);
109                        currentMatrix.set(2, 2, 1);
110                } else {
111                        this.currentMatrix = matrix;
112                }
113        }
114
115        /**
116         * Prepare an image to be transformed using the current matrix. The bounds
117         * of the image post transform are calculated so the default
118         * {@link ProjectionProcessor#performProjection} knows what range of pixels
119         * to draw
120         * 
121         * @param image
122         *            to be transformed
123         */
124        @Override
125        public void accumulate(T image) {
126                final Rectangle actualBounds = image.getBounds();
127                final Shape transformedActualBounds = actualBounds.transform(this.currentMatrix);
128                final double tminX = transformedActualBounds.minX();
129                final double tmaxX = transformedActualBounds.maxX();
130                final double tminY = transformedActualBounds.minY();
131                final double tmaxY = transformedActualBounds.maxY();
132                if (unset) {
133                        this.minc = (int) Math.floor(tminX);
134                        this.minr = (int) Math.floor(tminY);
135                        this.maxc = (int) Math.floor(tmaxX);
136                        this.maxr = (int) Math.floor(tmaxY);
137                        unset = false;
138                }
139                else {
140                        if (tminX < minc)
141                                minc = (int) Math.floor(tminX);
142                        if (tmaxX > maxc)
143                                maxc = (int) Math.floor(tmaxX);
144                        if (tminY < minr)
145                                minr = (int) Math.floor(tminY);
146                        if (tmaxY > maxr)
147                                maxr = (int) Math.floor(tmaxY);
148                }
149                // Expand the borders by 1 pixel so we get a nicer effect around the
150                // edges
151                final float padding = 1f;
152                final Rectangle expandedBounds = new Rectangle(actualBounds.x - padding, actualBounds.y - padding,
153                                actualBounds.width + padding * 2, actualBounds.height + padding * 2);
154                final Shape transformedExpandedBounds = expandedBounds.transform(this.currentMatrix);
155                Matrix minv = null, m = null;
156                try {
157                        m = this.currentMatrix.copy();
158                        minv = this.currentMatrix.copy().inverse();
159                } catch (final Throwable e) {
160                        // the matrix might be singular, return
161                        return;
162                }
163
164                this.images.add(image);
165                this.transforms.add(m);
166                this.transformsInverted.add(minv);
167                // this.projectedShapes.add(new
168                // TriangulatedPolygon(transformedExpandedBounds));
169                this.projectedShapes.add(transformedExpandedBounds);
170                this.projectedRectangles.add(transformedExpandedBounds.calculateRegularBoundingBox());
171
172                // System.out.println("added image with transform: ");
173                // this.currentMatrix.print(5,5);
174                // System.out.println("and the inverse:");
175                // this.currentMatrix.inverse().print(5,5);
176                // System.out.println("New min/max become:" + minc + "x" + minr + "/" +
177                // maxc + "x" + maxr);
178        }
179
180        /**
181         * Using all the images currently processed, perform the projection on each
182         * image and draw every pixel with valid data. Pixels within the bounding
183         * box but with no data are set to black (more specifically 0, whatever that
184         * may mean for this kind of image)
185         * 
186         * @return the image containing all the pixels drawn
187         */
188        public T performProjection() {
189                // The most long winded way to get a black pixel EVER
190                return performProjection(false, this.images.get(0).newInstance(1, 1).getPixel(0, 0));
191        }
192
193        /**
194         * Perform projection specifying the background colour (i.e. the colour of
195         * pixels with no data).
196         * 
197         * @param backgroundColour
198         *            the background colour
199         * @return projected images
200         */
201        public T performProjection(Q backgroundColour) {
202                final int projectionMinC = minc, projectionMaxC = maxc, projectionMinR = minr, projectionMaxR = maxr;
203                return performProjection(projectionMinC, projectionMaxC, projectionMinR, projectionMaxR, backgroundColour);
204        }
205
206        /**
207         * Perform projection specifying the background colour (i.e. the colour of
208         * pixels with no data) and whether the original window size should be kept.
209         * If set to true the window of pixels drawn post projection are within the
210         * window of the first image processed.
211         * 
212         * @param keepOriginalWindow
213         *            whether to keep the original image's window
214         * @param backgroundColour
215         *            the background colour
216         * @return projected images
217         */
218        public T performProjection(boolean keepOriginalWindow, Q backgroundColour) {
219                int projectionMinC = minc, projectionMaxC = maxc, projectionMinR = minr, projectionMaxR = maxr;
220                if (keepOriginalWindow)
221                {
222                        projectionMinC = 0;
223                        projectionMinR = 0;
224                        projectionMaxR = images.get(0).getRows();
225                        projectionMaxC = images.get(0).getCols();
226                }
227                return performProjection(projectionMinC, projectionMaxC, projectionMinR, projectionMaxR, backgroundColour);
228        }
229
230        /**
231         * Perform projection but only request data for pixels within the windowed
232         * range provided. Specify the background colour, i.e. the value of pixels
233         * with no data post projection.
234         * 
235         * @param windowMinC
236         *            left X
237         * @param windowMaxC
238         *            right X
239         * @param windowMinR
240         *            top Y
241         * @param windowMaxR
242         *            bottom Y
243         * @return projected image within the window
244         */
245        public T performProjection(int windowMinC, int windowMaxC, int windowMinR, int windowMaxR) {
246                return performProjection(windowMinC, windowMaxC, windowMinR, windowMaxR, this.images.get(0).newInstance(1, 1)
247                                .getPixel(0, 0));
248        }
249
250        /**
251         * Perform projection but only request data for pixels within the windowed
252         * range provided. Specify the background colour, i.e. the value of pixels
253         * with no data post projection.
254         * 
255         * @param windowMinC
256         *            left X
257         * @param windowMaxC
258         *            right X
259         * @param windowMinR
260         *            top Y
261         * @param windowMaxR
262         *            bottom Y
263         * @param backgroundColour
264         *            background colour of pixels with no data
265         * @return projected image within the window
266         */
267        public T performProjection(int windowMinC, int windowMaxC, int windowMinR, int windowMaxR, Q backgroundColour) {
268                T output = null;
269                output = images.get(0).newInstance(windowMaxC - windowMinC, windowMaxR - windowMinR);
270                if (backgroundColour != null)
271                        output.fill(backgroundColour);
272
273                final Shape[][] projectRectangleShapes = getCurrentShapes();
274
275                for (int y = 0; y < output.getHeight(); y++)
276                {
277                        for (int x = 0; x < output.getWidth(); x++) {
278                                final Point2d realPoint = new Point2dImpl(windowMinC + x, windowMinR + y);
279                                int i = 0;
280                                for (int shapeIndex = 0; shapeIndex < this.projectedShapes.size(); shapeIndex++) {
281                                        if (backgroundColour == null || isInside(shapeIndex, projectRectangleShapes, realPoint)) {
282                                                final double[][] transform = this.transformsInverted.get(i).getArray();
283
284                                                float xt = (float) transform[0][0] * realPoint.getX() + (float) transform[0][1]
285                                                                * realPoint.getY() + (float) transform[0][2];
286                                                float yt = (float) transform[1][0] * realPoint.getX() + (float) transform[1][1]
287                                                                * realPoint.getY() + (float) transform[1][2];
288                                                final float zt = (float) transform[2][0] * realPoint.getX() + (float) transform[2][1]
289                                                                * realPoint.getY() + (float) transform[2][2];
290
291                                                xt /= zt;
292                                                yt /= zt;
293                                                final T im = this.images.get(i);
294                                                if (backgroundColour != null)
295                                                        output.setPixel(x, y, im.getPixelInterp(xt, yt, backgroundColour));
296                                                else
297                                                        output.setPixel(x, y, im.getPixelInterp(xt, yt));
298                                        }
299                                        i++;
300                                }
301                        }
302                }
303                return output;
304        }
305
306        /**
307         * Get the current shapes as an array for efficient access, first entry for
308         * each shape is its rectangle, second entry is the shape
309         * 
310         * @return
311         */
312        protected Shape[][] getCurrentShapes() {
313                final Shape[][] currentShapes = new Shape[this.projectedShapes.size()][2];
314                for (int i = 0; i < this.projectedShapes.size(); i++) {
315                        currentShapes[i][0] = this.projectedRectangles.get(i);
316                        currentShapes[i][1] = this.projectedShapes.get(i);
317                }
318                return currentShapes;
319        }
320
321        protected boolean isInside(int shapeIndex, Shape[][] projectRectangleShapes, Point2d realPoint) {
322                return projectRectangleShapes[shapeIndex][0].isInside(realPoint)
323                                && projectRectangleShapes[shapeIndex][1].isInside(realPoint);
324        }
325
326        /**
327         * Perform projection but only request data for pixels within the windowed
328         * range provided. Specify the background colour, i.e. the value of pixels
329         * with no data post projection.
330         * 
331         * @param windowMinC
332         *            left X
333         * @param windowMinR
334         *            top Y
335         * @param output
336         *            the target image in which to project
337         * @return projected image within the window
338         */
339        public T performProjection(int windowMinC, int windowMinR, T output) {
340
341                for (int y = 0; y < output.getHeight(); y++)
342                {
343                        for (int x = 0; x < output.getWidth(); x++) {
344                                final Point2d realPoint = new Point2dImpl(windowMinC + x, windowMinR + y);
345                                int i = 0;
346                                for (final Shape s : this.projectedShapes) {
347                                        if (s.calculateRegularBoundingBox().isInside(realPoint) && s.isInside(realPoint)) {
348                                                final double[][] transform = this.transformsInverted.get(i).getArray();
349
350                                                float xt = (float) transform[0][0] * realPoint.getX() + (float) transform[0][1]
351                                                                * realPoint.getY() + (float) transform[0][2];
352                                                float yt = (float) transform[1][0] * realPoint.getX() + (float) transform[1][1]
353                                                                * realPoint.getY() + (float) transform[1][2];
354                                                final float zt = (float) transform[2][0] * realPoint.getX() + (float) transform[2][1]
355                                                                * realPoint.getY() + (float) transform[2][2];
356
357                                                xt /= zt;
358                                                yt /= zt;
359                                                final T im = this.images.get(i);
360                                                output.setPixel(x, y, im.getPixelInterp(xt, yt, output.getPixel(x, y)));
361                                        }
362                                        i++;
363                                }
364                        }
365                }
366
367                return output;
368        }
369
370        /**
371         * Perform blended projection but only request data for pixels within the
372         * windowed range provided. Specify the background colour, i.e. the value of
373         * pixels with no data post projection. This blends any existing pixels to
374         * newly added pixels
375         * 
376         * @param windowMinC
377         *            left X
378         * @param windowMaxC
379         *            right X
380         * @param windowMinR
381         *            top Y
382         * @param windowMaxR
383         *            bottom Y
384         * @param backgroundColour
385         *            background colour of pixels with no data
386         * @return projected image within the window
387         */
388        public T performBlendedProjection(int windowMinC, int windowMaxC, int windowMinR, int windowMaxR, Q backgroundColour)
389        {
390                T output = null;
391                output = images.get(0).newInstance(windowMaxC - windowMinC, windowMaxR - windowMinR);
392                final Map<Integer, Boolean> setMap = new HashMap<Integer, Boolean>();
393                final T blendingPallet = output.newInstance(2, 1);
394                for (int y = 0; y < output.getHeight(); y++)
395                {
396                        for (int x = 0; x < output.getWidth(); x++) {
397                                final Point2d realPoint = new Point2dImpl(windowMinC + x, windowMinR + y);
398                                int i = 0;
399                                for (final Shape s : this.projectedShapes) {
400                                        if (s.isInside(realPoint)) {
401                                                final double[][] transform = this.transformsInverted.get(i).getArray();
402
403                                                float xt = (float) transform[0][0] * realPoint.getX() + (float) transform[0][1]
404                                                                * realPoint.getY() + (float) transform[0][2];
405                                                float yt = (float) transform[1][0] * realPoint.getX() + (float) transform[1][1]
406                                                                * realPoint.getY() + (float) transform[1][2];
407                                                final float zt = (float) transform[2][0] * realPoint.getX() + (float) transform[2][1]
408                                                                * realPoint.getY() + (float) transform[2][2];
409
410                                                xt /= zt;
411                                                yt /= zt;
412                                                Q toSet = null;
413                                                if (backgroundColour != null)
414                                                        toSet = this.images.get(i).getPixelInterp(xt, yt, backgroundColour);
415                                                else if (setMap.get(y * output.getWidth() + x) != null)
416                                                        toSet = this.images.get(i).getPixelInterp(xt, yt, output.getPixelInterp(x, y));
417                                                else
418                                                        toSet = this.images.get(i).getPixelInterp(xt, yt);
419                                                // Blend the pixel with the existing pixel
420                                                if (setMap.get(y * output.getWidth() + x) != null) {
421                                                        blendingPallet.setPixel(1, 0, toSet);
422                                                        blendingPallet.setPixel(0, 0, output.getPixel(x, y));
423
424                                                        toSet = blendingPallet.getPixelInterp(0.1, 0.5);
425                                                }
426                                                setMap.put(y * output.getWidth() + x, true);
427                                                output.setPixel(x, y, toSet);
428                                        }
429                                        i++;
430                                }
431                        }
432                }
433                return output;
434        }
435
436        /**
437         * @return Current matrix
438         */
439        public Matrix getMatrix() {
440                return this.currentMatrix;
441        }
442
443        /**
444         * Utility function, project one image with one matrix. Every valid pixel in
445         * the space the image is projected into is displayed in the final image.
446         * 
447         * @param <Q>
448         *            the image pixel type
449         * @param <T>
450         *            image type
451         * @param image
452         *            the image to project
453         * @param matrix
454         *            the matrix to project against
455         * @return projected image
456         */
457        @SuppressWarnings("unchecked")
458        public static <Q, T extends Image<Q, T>> T project(T image, Matrix matrix) {
459                // Note: extra casts to work around compiler bug
460                if ((Image<?, ?>) image instanceof FImage) {
461                        final FProjectionProcessor proc = new FProjectionProcessor();
462                        proc.setMatrix(matrix);
463                        ((FImage) (Image<?, ?>) image).accumulateWith(proc);
464                        return (T) (Image<?, ?>) proc.performProjection();
465                }
466                if ((Image<?, ?>) image instanceof MBFImage) {
467                        final MBFProjectionProcessor proc = new MBFProjectionProcessor();
468                        proc.setMatrix(matrix);
469                        ((MBFImage) (Image<?, ?>) image).accumulateWith(proc);
470                        return (T) (Image<?, ?>) proc.performProjection();
471                } else {
472                        final ProjectionProcessor<Q, T> proc = new ProjectionProcessor<Q, T>();
473                        proc.setMatrix(matrix);
474                        image.accumulateWith(proc);
475                        return proc.performProjection();
476                }
477        }
478
479        /**
480         * Utility function, project one image with one matrix. Every valid pixel in
481         * the space the image is projected into is displayed in the final image.
482         * 
483         * @param <Q>
484         *            the image pixel type
485         * @param <T>
486         *            image type
487         * @param image
488         *            the image to project
489         * @param matrix
490         *            the matrix to project against
491         * @param backgroundColour
492         *            The colour of pixels with no data
493         * @return projected image
494         */
495        public static <Q, T extends Image<Q, T>> T project(T image, Matrix matrix, Q backgroundColour) {
496                final ProjectionProcessor<Q, T> proc = new ProjectionProcessor<Q, T>();
497                proc.setMatrix(matrix);
498                image.accumulateWith(proc);
499                return proc.performProjection(backgroundColour);
500        }
501
502        @Override
503        public T combine() {
504                return performProjection();
505        }
506}