001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.image.model.asm; 031 032import java.util.ArrayList; 033import java.util.List; 034 035import org.openimaj.citation.annotation.Reference; 036import org.openimaj.citation.annotation.ReferenceType; 037import org.openimaj.citation.annotation.References; 038import org.openimaj.image.Image; 039import org.openimaj.image.model.landmark.LandmarkModel; 040import org.openimaj.image.model.landmark.LandmarkModelFactory; 041import org.openimaj.math.geometry.point.Point2d; 042import org.openimaj.math.geometry.point.PointList; 043import org.openimaj.math.geometry.shape.PointDistributionModel; 044import org.openimaj.math.geometry.shape.PointDistributionModel.Constraint; 045import org.openimaj.math.matrix.algorithm.pca.PrincipalComponentAnalysis.ComponentSelector; 046import org.openimaj.util.pair.IndependentPair; 047import org.openimaj.util.pair.ObjectFloatPair; 048 049import Jama.Matrix; 050 051/** 052 * Implementation of a basic Active Shape Model. The implementation allows 053 * different types of landmark appearance models and can work with both colour 054 * and greylevel images. 055 * 056 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk) 057 * 058 * @param <I> 059 * Concrete type of {@link Image} 060 */ 061@References(references = { 062 @Reference( 063 author = { "Cootes, T. F.", "Taylor, C. J." }, 064 title = "Statistical Models of Appearance for Computer Vision", 065 type = ReferenceType.Unpublished, 066 month = "October", 067 year = "2001", 068 url = "http://isbe.man.ac.uk/~bim/Models/app_model.ps.gz" 069 ), 070 @Reference( 071 type = ReferenceType.Inproceedings, 072 author = { "T. F. Cootes", "C. J. Taylor" }, 073 title = "Active Shape Models", 074 year = "1992", 075 booktitle = "in Proceedings of the British Machine Vision Conference" 076 ) 077}) 078public class ActiveShapeModel<I extends Image<?, I>> { 079 private PointDistributionModel pdm; 080 private LandmarkModel<I>[] landmarkModels; 081 private int maxIter = 50; 082 private double inlierPercentage = 0.9; 083 084 /** 085 * Construct an {@link ActiveShapeModel} from a pre-trained 086 * {@link PointDistributionModel} and set of {@link LandmarkModel}s. 087 * 088 * @param pdm 089 * the {@link PointDistributionModel}. 090 * @param landmarkModels 091 * the {@link LandmarkModel}s. 092 */ 093 public ActiveShapeModel(PointDistributionModel pdm, LandmarkModel<I>[] landmarkModels) { 094 this.pdm = pdm; 095 this.landmarkModels = landmarkModels; 096 } 097 098 /** 099 * Train a new {@link ActiveShapeModel} using the given data and parameters. 100 * 101 * @param <I> 102 * The concrete image type. 103 * @param selector 104 * the selector for choosing the number of principal components / 105 * modes of the model. 106 * @param data 107 * the data to train the model from 108 * @param constraint 109 * the constraint to apply to restrict the model to plausible 110 * shapes. 111 * @param factory 112 * the {@link LandmarkModelFactory} for learning local appearance 113 * models 114 * @return a newly trained {@link ActiveShapeModel}. 115 */ 116 public static <I extends Image<?, I>> ActiveShapeModel<I> trainModel(ComponentSelector selector, 117 List<IndependentPair<PointList, I>> data, Constraint constraint, LandmarkModelFactory<I> factory) 118 { 119 final int nPoints = data.get(0).firstObject().size(); 120 121 @SuppressWarnings("unchecked") 122 final LandmarkModel<I>[] ppms = new LandmarkModel[nPoints]; 123 124 for (int i = 0; i < data.size(); i++) { 125 for (int j = 0; j < nPoints; j++) { 126 if (ppms[j] == null) { 127 ppms[j] = factory.createLandmarkModel(); 128 } 129 130 final PointList pl = data.get(i).firstObject(); 131 132 ppms[j].updateModel(data.get(i).secondObject(), pl.get(j), pl); 133 } 134 } 135 136 final List<PointList> pls = new ArrayList<PointList>(); 137 for (final IndependentPair<PointList, I> i : data) 138 pls.add(i.firstObject()); 139 140 final PointDistributionModel pdm = new PointDistributionModel(constraint, pls); 141 pdm.setNumComponents(selector); 142 143 return new ActiveShapeModel<I>(pdm, ppms); 144 } 145 146 /** 147 * Class to hold the response of a single iteration of model fitting. 148 * 149 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk) 150 * 151 */ 152 public static class IterationResult { 153 /** 154 * The percentage of points that moved less than 50% of their allowed 155 * distance 156 */ 157 public double fit; 158 /** 159 * The updated shape in image coordinates 160 */ 161 public PointList shape; 162 /** 163 * The model pose from model coordinates to image coordinates 164 */ 165 public Matrix pose; 166 /** 167 * The model weight parameters 168 */ 169 public double[] parameters; 170 171 protected IterationResult(Matrix pose, PointList shape, double fit, double[] parameters) { 172 this.pose = pose; 173 this.shape = shape; 174 this.fit = fit; 175 this.parameters = parameters; 176 } 177 } 178 179 /** 180 * Perform a single iteration of model fitting. 181 * 182 * @param image 183 * the image to fit to 184 * @param currentShape 185 * the starting shape in image coordinates 186 * @return the updated shape and parameters 187 */ 188 public IterationResult performIteration(I image, PointList currentShape) { 189 PointList newShape = new PointList(); 190 191 int inliers = 0; 192 int outliers = 0; 193 // compute updated points and a score based on how far they moved 194 for (int i = 0; i < landmarkModels.length; i++) { 195 final ObjectFloatPair<Point2d> newBest = landmarkModels[i].updatePosition(image, currentShape.get(i), 196 currentShape); 197 newShape.points.add(newBest.first); 198 199 final float percentageFromStart = newBest.second; 200 if (percentageFromStart < 0.5) 201 inliers++; 202 else 203 outliers++; 204 } 205 final double score = ((double) inliers) / ((double) (inliers + outliers)); 206 207 // find the parameters and pose that "best" model the updated points 208 final IndependentPair<Matrix, double[]> newModelParams = pdm.fitModel(newShape); 209 210 final Matrix pose = newModelParams.firstObject(); 211 final double[] parameters = newModelParams.secondObject(); 212 213 // apply model parameters to get final shape for the iteration 214 newShape = pdm.generateNewShape(parameters).transform(pose); 215 216 return new IterationResult(pose, newShape, score, parameters); 217 } 218 219 /** 220 * Iteratively apply {@link #performIteration(Image, PointList)} until the 221 * maximum number of iterations is exceeded, or the number of points that 222 * moved less than 0.5 of their maximum distance in an iteration is less 223 * than the target inlier percentage. 224 * 225 * @see #setInlierPercentage(double) 226 * @see #setMaxIterations(int) 227 * 228 * @param image 229 * the image to fit the shape to 230 * @param initialShape 231 * the initial shape in image coordinates 232 * @return the fitted shape and parameters 233 */ 234 public IterationResult fit(I image, PointList initialShape) { 235 IterationResult ir = performIteration(image, initialShape); 236 int count = 0; 237 238 while (ir.fit < inlierPercentage && count < maxIter) { 239 ir = performIteration(image, ir.shape); 240 count++; 241 } 242 243 return ir; 244 } 245 246 /** 247 * @return the maxIter 248 */ 249 public int getMaxIterations() { 250 return maxIter; 251 } 252 253 /** 254 * Set the maximum allowed number of iterations in fitting the model 255 * 256 * @param maxIter 257 * the maxIter to set 258 */ 259 public void setMaxIterations(int maxIter) { 260 this.maxIter = maxIter; 261 } 262 263 /** 264 * @return the inlierPercentage 265 */ 266 public double getInlierPercentage() { 267 return inlierPercentage; 268 } 269 270 /** 271 * Set the target percentage of the number of points that move less than 0.5 272 * of their total possible distance within an iteration to stop fitting. 273 * 274 * @param inlierPercentage 275 * the inlierPercentage to set 276 */ 277 public void setInlierPercentage(double inlierPercentage) { 278 this.inlierPercentage = inlierPercentage; 279 } 280 281 /** 282 * @return the learnt {@link PointDistributionModel} 283 */ 284 public PointDistributionModel getPDM() { 285 return pdm; 286 } 287 288 /** 289 * @return the local landmark appearance models; one for each point in the 290 * shape. 291 */ 292 public LandmarkModel<I>[] getLandmarkModels() { 293 return landmarkModels; 294 } 295}