001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.demos.sandbox;
031
032import java.io.File;
033import java.io.IOException;
034import java.util.List;
035
036import org.openimaj.feature.local.matcher.FastBasicKeypointMatcher;
037import org.openimaj.feature.local.matcher.consistent.ConsistentLocalFeatureMatcher2d;
038import org.openimaj.image.DisplayUtilities;
039import org.openimaj.image.FImage;
040import org.openimaj.image.ImageUtilities;
041import org.openimaj.image.MBFImage;
042import org.openimaj.image.colour.RGBColour;
043import org.openimaj.image.colour.Transforms;
044import org.openimaj.image.feature.local.engine.DoGSIFTEngine;
045import org.openimaj.image.feature.local.keypoints.Keypoint;
046import org.openimaj.image.processing.resize.ResizeProcessor;
047import org.openimaj.image.processing.transform.ProjectionProcessor;
048import org.openimaj.image.renderer.MBFImageRenderer;
049import org.openimaj.math.geometry.point.Point2d;
050import org.openimaj.math.geometry.transforms.HomographyModel;
051import org.openimaj.math.geometry.transforms.residuals.SingleImageTransferResidual2d;
052import org.openimaj.math.model.fit.RANSAC;
053import org.openimaj.util.pair.Pair;
054
055import Jama.Matrix;
056
057public class Mosaic {
058        public static void main(String[] args) throws IOException {
059                final ProjectionProcessor<Float[], MBFImage> bpp = new ProjectionProcessor<Float[], MBFImage>();
060                final File[] imagesToCombineInOrder = new File[] {
061                                new File("/Users/jon/Work/openimaj/trunk/demos/SimpleMosaic/data/trento-view-0.jpg"),
062                                new File("/Users/jon/Work/openimaj/trunk/demos/SimpleMosaic/data/trento-view-1.jpg"),
063                                new File("/Users/jon/Work/openimaj/trunk/demos/SimpleMosaic/data/trento-view-2.jpg"),
064                                new File("/Users/jon/Work/openimaj/trunk/demos/SimpleMosaic/data/trento-view-3.jpg"),
065                };
066                // AffineTransformModel model = new AffineTransformModel(6.0f);
067                final HomographyModel model = new HomographyModel();
068                final RANSAC<Point2d, Point2d, HomographyModel> ransac = new RANSAC<Point2d, Point2d, HomographyModel>(model,
069                                new SingleImageTransferResidual2d<HomographyModel>(), 12.0, 600,
070                                new RANSAC.BestFitStoppingCondition(), true);
071                final ConsistentLocalFeatureMatcher2d<Keypoint> matcher = new ConsistentLocalFeatureMatcher2d<Keypoint>(
072                                new FastBasicKeypointMatcher<Keypoint>(8));
073                matcher.setFittingModel(ransac);
074                final int centerImageIndex = 1;
075                // double scaleFactor = 1;
076                final ResizeProcessor displayResize = new ResizeProcessor(800, 600, false);
077                final ResizeProcessor analysisResize = new ResizeProcessor(800, 600);
078
079                final MBFImage centerImage = ImageUtilities.readMBF(imagesToCombineInOrder[centerImageIndex]).process(
080                                analysisResize);
081                final FImage centerImagef = Transforms.calculateIntensityNTSC(centerImage);
082
083                final DoGSIFTEngine engine = new DoGSIFTEngine();
084                final List<Keypoint> centerKeys = engine.findFeatures(centerImagef);
085
086                // GO LEFT
087                MBFImage currentImage = centerImage;
088                FImage currentImagef = centerImagef;
089                List<Keypoint> currentKeys = centerKeys;
090                bpp.setMatrix(new Matrix(new double[][] { { 1, 0, 0 }, { 0, 1, 0 }, { 0, 0, 1 } }));
091                bpp.accumulate(centerImage);
092                int steps = 1;
093                for (int i = centerImageIndex - 1; i >= 0; i--) {
094
095                        final MBFImage nextImage = ImageUtilities.readMBF(imagesToCombineInOrder[i]).process(analysisResize);
096                        final FImage nextImagef = Transforms.calculateIntensityNTSC(nextImage);
097
098                        final List<Keypoint> keys2 = engine.findFeatures(nextImagef);
099                        matcher.setModelFeatures(currentKeys);
100                        matcher.findMatches(keys2);
101
102                        // FIXME: there should be a class/method for drawing matches
103                        final MBFImage tmp = new MBFImage(currentImagef.width + nextImagef.width, Math.max(currentImagef.height,
104                                        nextImagef.height), 3);
105                        final MBFImageRenderer r = tmp.createRenderer();
106                        r.drawImage(currentImage, 0, 0);
107                        r.drawImage(nextImage, currentImagef.width, 0);
108                        for (final Pair<Keypoint> m : matcher.getMatches()) {
109                                r.drawLine((int) m.secondObject().x, (int) m.secondObject().y,
110                                                (int) m.firstObject().x + currentImagef.width, (int) m.firstObject().y,
111                                                RGBColour.RED);
112                        }
113
114                        DisplayUtilities.display(tmp);
115                        System.out.println("GOING LEFT");
116                        final Matrix transform = model.getTransform();
117                        model.getTransform().print(5, 5);
118
119                        bpp.setMatrix(bpp.getMatrix().times(transform));
120                        final int propWidth = (int) (nextImage.getWidth() * Math.pow(0.5, 0));
121                        bpp.accumulate(nextImage.extractROI(nextImage.getWidth() - propWidth, 0, propWidth, nextImage.getHeight()));
122                        currentImage = nextImage;
123                        currentImagef = nextImagef;
124                        currentKeys = keys2;
125                        steps++;
126
127                }
128
129                // GO RIGHT
130                currentImage = centerImage;
131                currentImagef = centerImagef;
132                currentKeys = centerKeys;
133                bpp.setMatrix(new Matrix(new double[][] { { 1, 0, 0 }, { 0, 1, 0 }, { 0, 0, 1 } }));
134                steps = 1;
135                for (int i = centerImageIndex + 1; i < imagesToCombineInOrder.length; i++) {
136                        final MBFImage nextImage = ImageUtilities.readMBF(imagesToCombineInOrder[i]).process(analysisResize);
137                        final FImage nextImagef = Transforms.calculateIntensityNTSC(nextImage);
138
139                        final List<Keypoint> keys2 = engine.findFeatures(nextImagef);
140                        matcher.setModelFeatures(currentKeys);
141                        matcher.findMatches(keys2);
142
143                        // FIXME: there should be a class/method for drawing matches
144                        final MBFImage tmp = new MBFImage(currentImagef.width + nextImagef.width, Math.max(currentImagef.height,
145                                        nextImagef.height), 3);
146                        final MBFImageRenderer r = tmp.createRenderer();
147                        r.drawImage(currentImage, 0, 0);
148                        r.drawImage(nextImage, currentImagef.width, 0);
149                        for (final Pair<Keypoint> m : matcher.getMatches()) {
150                                r.drawLine((int) m.secondObject().x, (int) m.secondObject().y,
151                                                (int) m.firstObject().x + currentImagef.width, (int) m.firstObject().y,
152                                                RGBColour.RED);
153                        }
154
155                        DisplayUtilities.display(tmp);
156
157                        final Matrix transform = model.getTransform();
158                        System.out.println("GOING RIGHT");
159                        transform.print(5, 5);
160
161                        bpp.setMatrix(bpp.getMatrix().times(transform));
162                        final int propWidth = (int) (nextImage.getWidth() * Math.pow(0.5, steps));
163                        bpp.accumulate(nextImage.extractROI(0, 0, propWidth, nextImage.getHeight()));
164                        currentImage = nextImage;
165                        currentImagef = nextImagef;
166                        currentKeys = keys2;
167                        steps++;
168
169                }
170
171                DisplayUtilities.display(bpp.performProjection().process(displayResize));
172
173                // MBFImage image1 = ImageUtilities.readMBF(new
174                // File("/Users/ss/Desktop/trento-view-0.jpg")).halfImageSize();
175                // MBFImage image2 = ImageUtilities.readMBF(new
176                // File("/Users/ss/Desktop/trento-view-1.jpg")).halfImageSize();
177                //
178                // FImage image1f = Transforms.calculateIntensityNTSC(image1);
179                // FImage image2f = Transforms.calculateIntensityNTSC(image2);
180                //
181                // KeypointEngine engine = new KeypointEngine();
182                // List<Keypoint> keys1 = engine.findKeypoints(image1f);
183                // List<Keypoint> keys2 = engine.findKeypoints(image2f);
184                //
185                // // HomographyModel model = new HomographyModel(6.0f);
186                // // RANSAC<Point2d, Point2d> ransac = new RANSAC<Point2d,
187                // Point2d>(model, 100, 10, false);
188                // //
189                // // ConsistentKeypointMatcher<Keypoint> matcher = new
190                // ConsistentKeypointMatcher<Keypoint>(8,0);
191                // matcher.setFittingModel(ransac);
192                // matcher.setModelKeypoints(keys1);
193                // matcher.findMatches(keys2);
194                //
195                // //FIXME: there should be a class/method for drawing matches
196                // MBFImage tmp = new MBFImage(Math.max(image1f.rows, image2f.rows),
197                // image1f.cols + image2f.cols, 3);
198                // tmp.drawImage(image1, 0, 0);
199                // tmp.drawImage(image2, image1f.cols, 0);
200                // // It is not drawing any matches??
201                // for (Pair<Keypoint> m : matcher.getMatches()) {
202                // tmp.drawLine((int)m.secondObject().col, (int)m.secondObject().row,
203                // (int)m.firstObject().col + image1f.cols, (int)m.firstObject().row,
204                // RGBColour.RED);
205                // }
206                //
207                // DisplayUtilities.display(tmp);
208                //
209                // Matrix transform = model.getTransform();
210                // model.getTransform().print(5,5);
211                //
212                // BackProjectionProcessor<Float[],MBFImage> bpp = new
213                // BackProjectionProcessor<Float[],MBFImage>();
214                // bpp.processImage(image1);
215                // bpp.setMatrix(transform);
216                // bpp.processImage(image2);
217                // //FIXME: there should be a class to do back-projection...
218                // MBFImage img = new MBFImage(4000,2524,3);
219                // for(int x = 0 ; x < img.getWidth(); x ++ ) {
220                // for(int y = 0 ; y < img.getHeight(); y ++ ) {
221                // int xx = x - 1500;
222                // int yy = y - 1500;
223                //
224                // double xt = transform.get(0, 0) * xx + transform.get(0, 1) * yy +
225                // transform.get(0, 2);
226                // double yt = transform.get(1, 0) * xx + transform.get(1, 1) * yy +
227                // transform.get(1, 2);
228                //
229                // double zt = transform.get(2, 0) * xx + transform.get(2, 1) * yy +
230                // transform.get(2, 2);
231                // xt /= zt;
232                // yt /= zt;
233                //
234                // if (xt >=0 && yt>=0 && xt<=image1.getCols() && yt<= image1.getRows())
235                // img.setPixel(x, y, image1.getPixelInterp(xt, yt));
236                // }
237                // }
238                // img.drawImage(image2, 1500, 1500);
239                // MBFImage img = bpp.performBackProjection();
240                // img = img.trim();
241
242                // DisplayUtilities.display(img.process(displayResize));
243                // DisplayUtilities.display(img.halfImageSize());
244        }
245}