View Javadoc

1   /**
2    * Copyright (c) 2011, The University of Southampton and the individual contributors.
3    * All rights reserved.
4    *
5    * Redistribution and use in source and binary forms, with or without modification,
6    * are permitted provided that the following conditions are met:
7    *
8    *   * 	Redistributions of source code must retain the above copyright notice,
9    * 	this list of conditions and the following disclaimer.
10   *
11   *   *	Redistributions in binary form must reproduce the above copyright notice,
12   * 	this list of conditions and the following disclaimer in the documentation
13   * 	and/or other materials provided with the distribution.
14   *
15   *   *	Neither the name of the University of Southampton nor the names of its
16   * 	contributors may be used to endorse or promote products derived from this
17   * 	software without specific prior written permission.
18   *
19   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
20   * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21   * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22   * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
23   * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24   * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
26   * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27   * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28   * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29   */
30  package org.openimaj.image.model.asm;
31  
32  import java.util.ArrayList;
33  import java.util.List;
34  
35  import org.openimaj.citation.annotation.Reference;
36  import org.openimaj.citation.annotation.ReferenceType;
37  import org.openimaj.citation.annotation.References;
38  import org.openimaj.image.Image;
39  import org.openimaj.image.model.landmark.LandmarkModel;
40  import org.openimaj.image.model.landmark.LandmarkModelFactory;
41  import org.openimaj.math.geometry.point.Point2d;
42  import org.openimaj.math.geometry.point.PointList;
43  import org.openimaj.math.geometry.shape.PointDistributionModel;
44  import org.openimaj.math.geometry.shape.PointDistributionModel.Constraint;
45  import org.openimaj.math.matrix.algorithm.pca.PrincipalComponentAnalysis.ComponentSelector;
46  import org.openimaj.util.pair.IndependentPair;
47  import org.openimaj.util.pair.ObjectFloatPair;
48  
49  import Jama.Matrix;
50  
51  /**
52   * Implementation of a basic Active Shape Model. The implementation allows
53   * different types of landmark appearance models and can work with both colour
54   * and greylevel images.
55   *
56   * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
57   *
58   * @param <I>
59   *            Concrete type of {@link Image}
60   */
61  @References(references = {
62  		@Reference(
63  				author = { "Cootes, T. F.", "Taylor, C. J." },
64  				title = "Statistical Models of Appearance for Computer Vision",
65  				type = ReferenceType.Unpublished,
66  				month = "October",
67  				year = "2001",
68  				url = "http://isbe.man.ac.uk/~bim/Models/app_model.ps.gz"
69  				),
70  		@Reference(
71  						type = ReferenceType.Inproceedings,
72  						author = { "T. F. Cootes", "C. J. Taylor" },
73  						title = "Active Shape Models",
74  						year = "1992",
75  						booktitle = "in Proceedings of the British Machine Vision Conference"
76  						)
77  })
78  public class ActiveShapeModel<I extends Image<?, I>> {
79  	private PointDistributionModel pdm;
80  	private LandmarkModel<I>[] landmarkModels;
81  	private int maxIter = 50;
82  	private double inlierPercentage = 0.9;
83  
84  	/**
85  	 * Construct an {@link ActiveShapeModel} from a pre-trained
86  	 * {@link PointDistributionModel} and set of {@link LandmarkModel}s.
87  	 * 
88  	 * @param pdm
89  	 *            the {@link PointDistributionModel}.
90  	 * @param landmarkModels
91  	 *            the {@link LandmarkModel}s.
92  	 */
93  	public ActiveShapeModel(PointDistributionModel pdm, LandmarkModel<I>[] landmarkModels) {
94  		this.pdm = pdm;
95  		this.landmarkModels = landmarkModels;
96  	}
97  
98  	/**
99  	 * Train a new {@link ActiveShapeModel} using the given data and parameters.
100 	 *
101 	 * @param <I>
102 	 *            The concrete image type.
103 	 * @param selector
104 	 *            the selector for choosing the number of principal components /
105 	 *            modes of the model.
106 	 * @param data
107 	 *            the data to train the model from
108 	 * @param constraint
109 	 *            the constraint to apply to restrict the model to plausible
110 	 *            shapes.
111 	 * @param factory
112 	 *            the {@link LandmarkModelFactory} for learning local appearance
113 	 *            models
114 	 * @return a newly trained {@link ActiveShapeModel}.
115 	 */
116 	public static <I extends Image<?, I>> ActiveShapeModel<I> trainModel(ComponentSelector selector,
117 			List<IndependentPair<PointList, I>> data, Constraint constraint, LandmarkModelFactory<I> factory)
118 	{
119 		final int nPoints = data.get(0).firstObject().size();
120 
121 		@SuppressWarnings("unchecked")
122 		final LandmarkModel<I>[] ppms = new LandmarkModel[nPoints];
123 
124 		for (int i = 0; i < data.size(); i++) {
125 			for (int j = 0; j < nPoints; j++) {
126 				if (ppms[j] == null) {
127 					ppms[j] = factory.createLandmarkModel();
128 				}
129 
130 				final PointList pl = data.get(i).firstObject();
131 
132 				ppms[j].updateModel(data.get(i).secondObject(), pl.get(j), pl);
133 			}
134 		}
135 
136 		final List<PointList> pls = new ArrayList<PointList>();
137 		for (final IndependentPair<PointList, I> i : data)
138 			pls.add(i.firstObject());
139 
140 		final PointDistributionModel pdm = new PointDistributionModel(constraint, pls);
141 		pdm.setNumComponents(selector);
142 
143 		return new ActiveShapeModel<I>(pdm, ppms);
144 	}
145 
146 	/**
147 	 * Class to hold the response of a single iteration of model fitting.
148 	 *
149 	 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
150 	 *
151 	 */
152 	public static class IterationResult {
153 		/**
154 		 * The percentage of points that moved less than 50% of their allowed
155 		 * distance
156 		 */
157 		public double fit;
158 		/**
159 		 * The updated shape in image coordinates
160 		 */
161 		public PointList shape;
162 		/**
163 		 * The model pose from model coordinates to image coordinates
164 		 */
165 		public Matrix pose;
166 		/**
167 		 * The model weight parameters
168 		 */
169 		public double[] parameters;
170 
171 		protected IterationResult(Matrix pose, PointList shape, double fit, double[] parameters) {
172 			this.pose = pose;
173 			this.shape = shape;
174 			this.fit = fit;
175 			this.parameters = parameters;
176 		}
177 	}
178 
179 	/**
180 	 * Perform a single iteration of model fitting.
181 	 *
182 	 * @param image
183 	 *            the image to fit to
184 	 * @param currentShape
185 	 *            the starting shape in image coordinates
186 	 * @return the updated shape and parameters
187 	 */
188 	public IterationResult performIteration(I image, PointList currentShape) {
189 		PointList newShape = new PointList();
190 
191 		int inliers = 0;
192 		int outliers = 0;
193 		// compute updated points and a score based on how far they moved
194 		for (int i = 0; i < landmarkModels.length; i++) {
195 			final ObjectFloatPair<Point2d> newBest = landmarkModels[i].updatePosition(image, currentShape.get(i),
196 					currentShape);
197 			newShape.points.add(newBest.first);
198 
199 			final float percentageFromStart = newBest.second;
200 			if (percentageFromStart < 0.5)
201 				inliers++;
202 			else
203 				outliers++;
204 		}
205 		final double score = ((double) inliers) / ((double) (inliers + outliers));
206 
207 		// find the parameters and pose that "best" model the updated points
208 		final IndependentPair<Matrix, double[]> newModelParams = pdm.fitModel(newShape);
209 
210 		final Matrix pose = newModelParams.firstObject();
211 		final double[] parameters = newModelParams.secondObject();
212 
213 		// apply model parameters to get final shape for the iteration
214 		newShape = pdm.generateNewShape(parameters).transform(pose);
215 
216 		return new IterationResult(pose, newShape, score, parameters);
217 	}
218 
219 	/**
220 	 * Iteratively apply {@link #performIteration(Image, PointList)} until the
221 	 * maximum number of iterations is exceeded, or the number of points that
222 	 * moved less than 0.5 of their maximum distance in an iteration is less
223 	 * than the target inlier percentage.
224 	 *
225 	 * @see #setInlierPercentage(double)
226 	 * @see #setMaxIterations(int)
227 	 *
228 	 * @param image
229 	 *            the image to fit the shape to
230 	 * @param initialShape
231 	 *            the initial shape in image coordinates
232 	 * @return the fitted shape and parameters
233 	 */
234 	public IterationResult fit(I image, PointList initialShape) {
235 		IterationResult ir = performIteration(image, initialShape);
236 		int count = 0;
237 
238 		while (ir.fit < inlierPercentage && count < maxIter) {
239 			ir = performIteration(image, ir.shape);
240 			count++;
241 		}
242 
243 		return ir;
244 	}
245 
246 	/**
247 	 * @return the maxIter
248 	 */
249 	public int getMaxIterations() {
250 		return maxIter;
251 	}
252 
253 	/**
254 	 * Set the maximum allowed number of iterations in fitting the model
255 	 * 
256 	 * @param maxIter
257 	 *            the maxIter to set
258 	 */
259 	public void setMaxIterations(int maxIter) {
260 		this.maxIter = maxIter;
261 	}
262 
263 	/**
264 	 * @return the inlierPercentage
265 	 */
266 	public double getInlierPercentage() {
267 		return inlierPercentage;
268 	}
269 
270 	/**
271 	 * Set the target percentage of the number of points that move less than 0.5
272 	 * of their total possible distance within an iteration to stop fitting.
273 	 * 
274 	 * @param inlierPercentage
275 	 *            the inlierPercentage to set
276 	 */
277 	public void setInlierPercentage(double inlierPercentage) {
278 		this.inlierPercentage = inlierPercentage;
279 	}
280 
281 	/**
282 	 * @return the learnt {@link PointDistributionModel}
283 	 */
284 	public PointDistributionModel getPDM() {
285 		return pdm;
286 	}
287 
288 	/**
289 	 * @return the local landmark appearance models; one for each point in the
290 	 *         shape.
291 	 */
292 	public LandmarkModel<I>[] getLandmarkModels() {
293 		return landmarkModels;
294 	}
295 }