001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.demos.sandbox; 031 032import java.io.File; 033import java.io.IOException; 034import java.util.List; 035 036import org.openimaj.feature.local.matcher.FastBasicKeypointMatcher; 037import org.openimaj.feature.local.matcher.consistent.ConsistentLocalFeatureMatcher2d; 038import org.openimaj.image.DisplayUtilities; 039import org.openimaj.image.FImage; 040import org.openimaj.image.ImageUtilities; 041import org.openimaj.image.MBFImage; 042import org.openimaj.image.colour.RGBColour; 043import org.openimaj.image.colour.Transforms; 044import org.openimaj.image.feature.local.engine.DoGSIFTEngine; 045import org.openimaj.image.feature.local.keypoints.Keypoint; 046import org.openimaj.image.processing.resize.ResizeProcessor; 047import org.openimaj.image.processing.transform.ProjectionProcessor; 048import org.openimaj.image.renderer.MBFImageRenderer; 049import org.openimaj.math.geometry.point.Point2d; 050import org.openimaj.math.geometry.transforms.HomographyModel; 051import org.openimaj.math.geometry.transforms.residuals.SingleImageTransferResidual2d; 052import org.openimaj.math.model.fit.RANSAC; 053import org.openimaj.util.pair.Pair; 054 055import Jama.Matrix; 056 057public class Mosaic { 058 public static void main(String[] args) throws IOException { 059 final ProjectionProcessor<Float[], MBFImage> bpp = new ProjectionProcessor<Float[], MBFImage>(); 060 final File[] imagesToCombineInOrder = new File[] { 061 new File("/Users/jon/Work/openimaj/trunk/demos/SimpleMosaic/data/trento-view-0.jpg"), 062 new File("/Users/jon/Work/openimaj/trunk/demos/SimpleMosaic/data/trento-view-1.jpg"), 063 new File("/Users/jon/Work/openimaj/trunk/demos/SimpleMosaic/data/trento-view-2.jpg"), 064 new File("/Users/jon/Work/openimaj/trunk/demos/SimpleMosaic/data/trento-view-3.jpg"), 065 }; 066 // AffineTransformModel model = new AffineTransformModel(6.0f); 067 final HomographyModel model = new HomographyModel(); 068 final RANSAC<Point2d, Point2d, HomographyModel> ransac = new RANSAC<Point2d, Point2d, HomographyModel>(model, 069 new SingleImageTransferResidual2d<HomographyModel>(), 12.0, 600, 070 new RANSAC.BestFitStoppingCondition(), true); 071 final ConsistentLocalFeatureMatcher2d<Keypoint> matcher = new ConsistentLocalFeatureMatcher2d<Keypoint>( 072 new FastBasicKeypointMatcher<Keypoint>(8)); 073 matcher.setFittingModel(ransac); 074 final int centerImageIndex = 1; 075 // double scaleFactor = 1; 076 final ResizeProcessor displayResize = new ResizeProcessor(800, 600, false); 077 final ResizeProcessor analysisResize = new ResizeProcessor(800, 600); 078 079 final MBFImage centerImage = ImageUtilities.readMBF(imagesToCombineInOrder[centerImageIndex]).process( 080 analysisResize); 081 final FImage centerImagef = Transforms.calculateIntensityNTSC(centerImage); 082 083 final DoGSIFTEngine engine = new DoGSIFTEngine(); 084 final List<Keypoint> centerKeys = engine.findFeatures(centerImagef); 085 086 // GO LEFT 087 MBFImage currentImage = centerImage; 088 FImage currentImagef = centerImagef; 089 List<Keypoint> currentKeys = centerKeys; 090 bpp.setMatrix(new Matrix(new double[][] { { 1, 0, 0 }, { 0, 1, 0 }, { 0, 0, 1 } })); 091 bpp.accumulate(centerImage); 092 int steps = 1; 093 for (int i = centerImageIndex - 1; i >= 0; i--) { 094 095 final MBFImage nextImage = ImageUtilities.readMBF(imagesToCombineInOrder[i]).process(analysisResize); 096 final FImage nextImagef = Transforms.calculateIntensityNTSC(nextImage); 097 098 final List<Keypoint> keys2 = engine.findFeatures(nextImagef); 099 matcher.setModelFeatures(currentKeys); 100 matcher.findMatches(keys2); 101 102 // FIXME: there should be a class/method for drawing matches 103 final MBFImage tmp = new MBFImage(currentImagef.width + nextImagef.width, Math.max(currentImagef.height, 104 nextImagef.height), 3); 105 final MBFImageRenderer r = tmp.createRenderer(); 106 r.drawImage(currentImage, 0, 0); 107 r.drawImage(nextImage, currentImagef.width, 0); 108 for (final Pair<Keypoint> m : matcher.getMatches()) { 109 r.drawLine((int) m.secondObject().x, (int) m.secondObject().y, 110 (int) m.firstObject().x + currentImagef.width, (int) m.firstObject().y, 111 RGBColour.RED); 112 } 113 114 DisplayUtilities.display(tmp); 115 System.out.println("GOING LEFT"); 116 final Matrix transform = model.getTransform(); 117 model.getTransform().print(5, 5); 118 119 bpp.setMatrix(bpp.getMatrix().times(transform)); 120 final int propWidth = (int) (nextImage.getWidth() * Math.pow(0.5, 0)); 121 bpp.accumulate(nextImage.extractROI(nextImage.getWidth() - propWidth, 0, propWidth, nextImage.getHeight())); 122 currentImage = nextImage; 123 currentImagef = nextImagef; 124 currentKeys = keys2; 125 steps++; 126 127 } 128 129 // GO RIGHT 130 currentImage = centerImage; 131 currentImagef = centerImagef; 132 currentKeys = centerKeys; 133 bpp.setMatrix(new Matrix(new double[][] { { 1, 0, 0 }, { 0, 1, 0 }, { 0, 0, 1 } })); 134 steps = 1; 135 for (int i = centerImageIndex + 1; i < imagesToCombineInOrder.length; i++) { 136 final MBFImage nextImage = ImageUtilities.readMBF(imagesToCombineInOrder[i]).process(analysisResize); 137 final FImage nextImagef = Transforms.calculateIntensityNTSC(nextImage); 138 139 final List<Keypoint> keys2 = engine.findFeatures(nextImagef); 140 matcher.setModelFeatures(currentKeys); 141 matcher.findMatches(keys2); 142 143 // FIXME: there should be a class/method for drawing matches 144 final MBFImage tmp = new MBFImage(currentImagef.width + nextImagef.width, Math.max(currentImagef.height, 145 nextImagef.height), 3); 146 final MBFImageRenderer r = tmp.createRenderer(); 147 r.drawImage(currentImage, 0, 0); 148 r.drawImage(nextImage, currentImagef.width, 0); 149 for (final Pair<Keypoint> m : matcher.getMatches()) { 150 r.drawLine((int) m.secondObject().x, (int) m.secondObject().y, 151 (int) m.firstObject().x + currentImagef.width, (int) m.firstObject().y, 152 RGBColour.RED); 153 } 154 155 DisplayUtilities.display(tmp); 156 157 final Matrix transform = model.getTransform(); 158 System.out.println("GOING RIGHT"); 159 transform.print(5, 5); 160 161 bpp.setMatrix(bpp.getMatrix().times(transform)); 162 final int propWidth = (int) (nextImage.getWidth() * Math.pow(0.5, steps)); 163 bpp.accumulate(nextImage.extractROI(0, 0, propWidth, nextImage.getHeight())); 164 currentImage = nextImage; 165 currentImagef = nextImagef; 166 currentKeys = keys2; 167 steps++; 168 169 } 170 171 DisplayUtilities.display(bpp.performProjection().process(displayResize)); 172 173 // MBFImage image1 = ImageUtilities.readMBF(new 174 // File("/Users/ss/Desktop/trento-view-0.jpg")).halfImageSize(); 175 // MBFImage image2 = ImageUtilities.readMBF(new 176 // File("/Users/ss/Desktop/trento-view-1.jpg")).halfImageSize(); 177 // 178 // FImage image1f = Transforms.calculateIntensityNTSC(image1); 179 // FImage image2f = Transforms.calculateIntensityNTSC(image2); 180 // 181 // KeypointEngine engine = new KeypointEngine(); 182 // List<Keypoint> keys1 = engine.findKeypoints(image1f); 183 // List<Keypoint> keys2 = engine.findKeypoints(image2f); 184 // 185 // // HomographyModel model = new HomographyModel(6.0f); 186 // // RANSAC<Point2d, Point2d> ransac = new RANSAC<Point2d, 187 // Point2d>(model, 100, 10, false); 188 // // 189 // // ConsistentKeypointMatcher<Keypoint> matcher = new 190 // ConsistentKeypointMatcher<Keypoint>(8,0); 191 // matcher.setFittingModel(ransac); 192 // matcher.setModelKeypoints(keys1); 193 // matcher.findMatches(keys2); 194 // 195 // //FIXME: there should be a class/method for drawing matches 196 // MBFImage tmp = new MBFImage(Math.max(image1f.rows, image2f.rows), 197 // image1f.cols + image2f.cols, 3); 198 // tmp.drawImage(image1, 0, 0); 199 // tmp.drawImage(image2, image1f.cols, 0); 200 // // It is not drawing any matches?? 201 // for (Pair<Keypoint> m : matcher.getMatches()) { 202 // tmp.drawLine((int)m.secondObject().col, (int)m.secondObject().row, 203 // (int)m.firstObject().col + image1f.cols, (int)m.firstObject().row, 204 // RGBColour.RED); 205 // } 206 // 207 // DisplayUtilities.display(tmp); 208 // 209 // Matrix transform = model.getTransform(); 210 // model.getTransform().print(5,5); 211 // 212 // BackProjectionProcessor<Float[],MBFImage> bpp = new 213 // BackProjectionProcessor<Float[],MBFImage>(); 214 // bpp.processImage(image1); 215 // bpp.setMatrix(transform); 216 // bpp.processImage(image2); 217 // //FIXME: there should be a class to do back-projection... 218 // MBFImage img = new MBFImage(4000,2524,3); 219 // for(int x = 0 ; x < img.getWidth(); x ++ ) { 220 // for(int y = 0 ; y < img.getHeight(); y ++ ) { 221 // int xx = x - 1500; 222 // int yy = y - 1500; 223 // 224 // double xt = transform.get(0, 0) * xx + transform.get(0, 1) * yy + 225 // transform.get(0, 2); 226 // double yt = transform.get(1, 0) * xx + transform.get(1, 1) * yy + 227 // transform.get(1, 2); 228 // 229 // double zt = transform.get(2, 0) * xx + transform.get(2, 1) * yy + 230 // transform.get(2, 2); 231 // xt /= zt; 232 // yt /= zt; 233 // 234 // if (xt >=0 && yt>=0 && xt<=image1.getCols() && yt<= image1.getRows()) 235 // img.setPixel(x, y, image1.getPixelInterp(xt, yt)); 236 // } 237 // } 238 // img.drawImage(image2, 1500, 1500); 239 // MBFImage img = bpp.performBackProjection(); 240 // img = img.trim(); 241 242 // DisplayUtilities.display(img.process(displayResize)); 243 // DisplayUtilities.display(img.halfImageSize()); 244 } 245}