001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.image.model.asm;
031
032import java.util.ArrayList;
033import java.util.List;
034
035import org.openimaj.citation.annotation.Reference;
036import org.openimaj.citation.annotation.ReferenceType;
037import org.openimaj.citation.annotation.References;
038import org.openimaj.image.Image;
039import org.openimaj.image.model.landmark.LandmarkModel;
040import org.openimaj.image.model.landmark.LandmarkModelFactory;
041import org.openimaj.math.geometry.point.Point2d;
042import org.openimaj.math.geometry.point.PointList;
043import org.openimaj.math.geometry.shape.PointDistributionModel;
044import org.openimaj.math.geometry.shape.PointDistributionModel.Constraint;
045import org.openimaj.math.matrix.algorithm.pca.PrincipalComponentAnalysis.ComponentSelector;
046import org.openimaj.util.pair.IndependentPair;
047import org.openimaj.util.pair.ObjectFloatPair;
048
049import Jama.Matrix;
050
051/**
052 * Implementation of a basic Active Shape Model. The implementation allows
053 * different types of landmark appearance models and can work with both colour
054 * and greylevel images.
055 *
056 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
057 *
058 * @param <I>
059 *            Concrete type of {@link Image}
060 */
061@References(references = {
062                @Reference(
063                                author = { "Cootes, T. F.", "Taylor, C. J." },
064                                title = "Statistical Models of Appearance for Computer Vision",
065                                type = ReferenceType.Unpublished,
066                                month = "October",
067                                year = "2001",
068                                url = "http://isbe.man.ac.uk/~bim/Models/app_model.ps.gz"
069                                ),
070                @Reference(
071                                                type = ReferenceType.Inproceedings,
072                                                author = { "T. F. Cootes", "C. J. Taylor" },
073                                                title = "Active Shape Models",
074                                                year = "1992",
075                                                booktitle = "in Proceedings of the British Machine Vision Conference"
076                                                )
077})
078public class ActiveShapeModel<I extends Image<?, I>> {
079        private PointDistributionModel pdm;
080        private LandmarkModel<I>[] landmarkModels;
081        private int maxIter = 50;
082        private double inlierPercentage = 0.9;
083
084        /**
085         * Construct an {@link ActiveShapeModel} from a pre-trained
086         * {@link PointDistributionModel} and set of {@link LandmarkModel}s.
087         * 
088         * @param pdm
089         *            the {@link PointDistributionModel}.
090         * @param landmarkModels
091         *            the {@link LandmarkModel}s.
092         */
093        public ActiveShapeModel(PointDistributionModel pdm, LandmarkModel<I>[] landmarkModels) {
094                this.pdm = pdm;
095                this.landmarkModels = landmarkModels;
096        }
097
098        /**
099         * Train a new {@link ActiveShapeModel} using the given data and parameters.
100         *
101         * @param <I>
102         *            The concrete image type.
103         * @param selector
104         *            the selector for choosing the number of principal components /
105         *            modes of the model.
106         * @param data
107         *            the data to train the model from
108         * @param constraint
109         *            the constraint to apply to restrict the model to plausible
110         *            shapes.
111         * @param factory
112         *            the {@link LandmarkModelFactory} for learning local appearance
113         *            models
114         * @return a newly trained {@link ActiveShapeModel}.
115         */
116        public static <I extends Image<?, I>> ActiveShapeModel<I> trainModel(ComponentSelector selector,
117                        List<IndependentPair<PointList, I>> data, Constraint constraint, LandmarkModelFactory<I> factory)
118        {
119                final int nPoints = data.get(0).firstObject().size();
120
121                @SuppressWarnings("unchecked")
122                final LandmarkModel<I>[] ppms = new LandmarkModel[nPoints];
123
124                for (int i = 0; i < data.size(); i++) {
125                        for (int j = 0; j < nPoints; j++) {
126                                if (ppms[j] == null) {
127                                        ppms[j] = factory.createLandmarkModel();
128                                }
129
130                                final PointList pl = data.get(i).firstObject();
131
132                                ppms[j].updateModel(data.get(i).secondObject(), pl.get(j), pl);
133                        }
134                }
135
136                final List<PointList> pls = new ArrayList<PointList>();
137                for (final IndependentPair<PointList, I> i : data)
138                        pls.add(i.firstObject());
139
140                final PointDistributionModel pdm = new PointDistributionModel(constraint, pls);
141                pdm.setNumComponents(selector);
142
143                return new ActiveShapeModel<I>(pdm, ppms);
144        }
145
146        /**
147         * Class to hold the response of a single iteration of model fitting.
148         *
149         * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
150         *
151         */
152        public static class IterationResult {
153                /**
154                 * The percentage of points that moved less than 50% of their allowed
155                 * distance
156                 */
157                public double fit;
158                /**
159                 * The updated shape in image coordinates
160                 */
161                public PointList shape;
162                /**
163                 * The model pose from model coordinates to image coordinates
164                 */
165                public Matrix pose;
166                /**
167                 * The model weight parameters
168                 */
169                public double[] parameters;
170
171                protected IterationResult(Matrix pose, PointList shape, double fit, double[] parameters) {
172                        this.pose = pose;
173                        this.shape = shape;
174                        this.fit = fit;
175                        this.parameters = parameters;
176                }
177        }
178
179        /**
180         * Perform a single iteration of model fitting.
181         *
182         * @param image
183         *            the image to fit to
184         * @param currentShape
185         *            the starting shape in image coordinates
186         * @return the updated shape and parameters
187         */
188        public IterationResult performIteration(I image, PointList currentShape) {
189                PointList newShape = new PointList();
190
191                int inliers = 0;
192                int outliers = 0;
193                // compute updated points and a score based on how far they moved
194                for (int i = 0; i < landmarkModels.length; i++) {
195                        final ObjectFloatPair<Point2d> newBest = landmarkModels[i].updatePosition(image, currentShape.get(i),
196                                        currentShape);
197                        newShape.points.add(newBest.first);
198
199                        final float percentageFromStart = newBest.second;
200                        if (percentageFromStart < 0.5)
201                                inliers++;
202                        else
203                                outliers++;
204                }
205                final double score = ((double) inliers) / ((double) (inliers + outliers));
206
207                // find the parameters and pose that "best" model the updated points
208                final IndependentPair<Matrix, double[]> newModelParams = pdm.fitModel(newShape);
209
210                final Matrix pose = newModelParams.firstObject();
211                final double[] parameters = newModelParams.secondObject();
212
213                // apply model parameters to get final shape for the iteration
214                newShape = pdm.generateNewShape(parameters).transform(pose);
215
216                return new IterationResult(pose, newShape, score, parameters);
217        }
218
219        /**
220         * Iteratively apply {@link #performIteration(Image, PointList)} until the
221         * maximum number of iterations is exceeded, or the number of points that
222         * moved less than 0.5 of their maximum distance in an iteration is less
223         * than the target inlier percentage.
224         *
225         * @see #setInlierPercentage(double)
226         * @see #setMaxIterations(int)
227         *
228         * @param image
229         *            the image to fit the shape to
230         * @param initialShape
231         *            the initial shape in image coordinates
232         * @return the fitted shape and parameters
233         */
234        public IterationResult fit(I image, PointList initialShape) {
235                IterationResult ir = performIteration(image, initialShape);
236                int count = 0;
237
238                while (ir.fit < inlierPercentage && count < maxIter) {
239                        ir = performIteration(image, ir.shape);
240                        count++;
241                }
242
243                return ir;
244        }
245
246        /**
247         * @return the maxIter
248         */
249        public int getMaxIterations() {
250                return maxIter;
251        }
252
253        /**
254         * Set the maximum allowed number of iterations in fitting the model
255         * 
256         * @param maxIter
257         *            the maxIter to set
258         */
259        public void setMaxIterations(int maxIter) {
260                this.maxIter = maxIter;
261        }
262
263        /**
264         * @return the inlierPercentage
265         */
266        public double getInlierPercentage() {
267                return inlierPercentage;
268        }
269
270        /**
271         * Set the target percentage of the number of points that move less than 0.5
272         * of their total possible distance within an iteration to stop fitting.
273         * 
274         * @param inlierPercentage
275         *            the inlierPercentage to set
276         */
277        public void setInlierPercentage(double inlierPercentage) {
278                this.inlierPercentage = inlierPercentage;
279        }
280
281        /**
282         * @return the learnt {@link PointDistributionModel}
283         */
284        public PointDistributionModel getPDM() {
285                return pdm;
286        }
287
288        /**
289         * @return the local landmark appearance models; one for each point in the
290         *         shape.
291         */
292        public LandmarkModel<I>[] getLandmarkModels() {
293                return landmarkModels;
294        }
295}