001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.image.processing.face.detection;
031
032import java.io.DataInput;
033import java.io.DataOutput;
034import java.io.IOException;
035import java.util.ArrayList;
036import java.util.List;
037
038import org.openimaj.citation.annotation.Reference;
039import org.openimaj.citation.annotation.ReferenceType;
040import org.openimaj.feature.DoubleFV;
041import org.openimaj.image.FImage;
042import org.openimaj.image.MBFImage;
043import org.openimaj.image.processing.face.tracking.clm.MultiTracker.TrackedFace;
044import org.openimaj.image.processing.face.tracking.clm.MultiTracker.TrackerVars;
045import org.openimaj.io.IOUtils;
046import org.openimaj.math.geometry.shape.Rectangle;
047
048import Jama.Matrix;
049
050/**
051 * A constrained local model detected face. In addition to the patch and
052 * detection rectangle, also provides the shape matrix (describing the 2D point
053 * positions in the patch image), and the weight vectors for the model pose
054 * (relative to the detection image) and shape.
055 *
056 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
057 */
058@Reference(
059                type = ReferenceType.Inproceedings,
060                author = { "Jason M. Saragih", "Simon Lucey", "Jeffrey F. Cohn" },
061                title = "Face alignment through subspace constrained mean-shifts",
062                year = "2009",
063                booktitle = "IEEE 12th International Conference on Computer Vision, ICCV 2009, Kyoto, Japan, September 27 - October 4, 2009",
064                pages = { "1034", "1041" },
065                publisher = "IEEE",
066                customData = {
067                                "doi", "http://dx.doi.org/10.1109/ICCV.2009.5459377",
068                                "researchr", "http://researchr.org/publication/SaragihLC09",
069                                "cites", "0",
070                                "citedby", "0"
071                })
072public class CLMDetectedFace extends DetectedFace {
073        private Matrix shape;
074        private Matrix poseParameters;
075        private Matrix shapeParameters;
076        private Matrix visibility;
077
078        protected CLMDetectedFace() {
079        }
080
081        /**
082         * Construct a {@link CLMDetectedFace} by copying the state from a
083         * {@link TrackedFace}
084         *
085         * @param face
086         *            the {@link TrackedFace}
087         * @param image
088         *            the image in which the tracked face was detected
089         */
090        public CLMDetectedFace(final TrackedFace face, final FImage image) {
091                this(face.redetectedBounds, face.shape.copy(), face.clm._pglobl.copy(), face.clm._plocal.copy(),
092                                face.clm._visi[face.clm.getViewIdx()].copy(), image);
093        }
094
095        /**
096         * Construct with the given bounds, shape and pose parameters and detection
097         * image. The face patch is extracted automatically.
098         *
099         * @param bounds
100         * @param shape
101         * @param poseParameters
102         * @param shapeParameters
103         * @param visibility
104         * @param fullImage
105         */
106        public CLMDetectedFace(final Rectangle bounds, final Matrix shape, final Matrix poseParameters, final Matrix shapeParameters,
107                        final Matrix visibility, final FImage fullImage)
108        {
109                super(bounds, fullImage.extractROI(bounds), 1);
110
111                this.poseParameters = poseParameters;
112                this.shapeParameters = shapeParameters;
113                this.visibility = visibility;
114
115                this.shape = shape;
116
117                // translate the shape
118                final int n = shape.getRowDimension() / 2;
119                final double[][] shapeData = shape.getArray();
120                for (int i = 0; i < n; i++) {
121                        shapeData[i][0] -= bounds.x;
122                        shapeData[i + n][0] -= bounds.y;
123                }
124        }
125
126
127
128        /**
129         * Helper method to convert a list of {@link TrackedFace}s to
130         * {@link CLMDetectedFace}s.
131         *
132         * @param faces
133         *            the {@link TrackedFace}s.
134         * @param image
135         *            the image the {@link TrackedFace}s came from.
136         * @return the list of {@link CLMDetectedFace}s
137         */
138        public static List<CLMDetectedFace> convert(final List<TrackedFace> faces, final MBFImage image) {
139                final FImage fimage = image.flatten();
140
141                return CLMDetectedFace.convert(faces, fimage);
142        }
143
144        /**
145         * Helper method to convert a list of {@link TrackedFace}s to
146         * {@link CLMDetectedFace}s.
147         *
148         * @param faces
149         *            the {@link TrackedFace}s.
150         * @param image
151         *            the image the {@link TrackedFace}s came from.
152         * @return the list of {@link CLMDetectedFace}s
153         */
154        public static List<CLMDetectedFace> convert(final List<TrackedFace> faces, final FImage image) {
155                final List<CLMDetectedFace> cvt = new ArrayList<CLMDetectedFace>();
156
157                for (final TrackedFace f : faces) {
158                        cvt.add(new CLMDetectedFace(f, image));
159                }
160
161                return cvt;
162        }
163
164        /**
165         *      Helper method that converts this {@link CLMDetectedFace} into
166         *      a {@link TrackedFace}.
167         *      @return A {@link TrackedFace}
168         */
169        public TrackedFace convert()
170        {
171                final TrackerVars tv = new TrackerVars();
172                tv.clm._pglobl = this.poseParameters.copy();
173                tv.clm._plocal = this.shapeParameters.copy();
174                tv.shape = this.shape.copy();
175                tv.clm._visi[ tv.clm.getViewIdx() ] = this.visibility.copy();
176                return new TrackedFace( this.bounds, tv );
177        }
178
179        @Override
180        public void writeBinary(final DataOutput out) throws IOException {
181                super.writeBinary(out);
182
183                IOUtils.write(this.getShape(), out);
184                IOUtils.write(this.poseParameters, out);
185                IOUtils.write(this.shapeParameters, out);
186        }
187
188        @Override
189        public byte[] binaryHeader() {
190                return "DF".getBytes();
191        }
192
193        @Override
194        public void readBinary(final DataInput in) throws IOException {
195                super.readBinary(in);
196                this.shape = IOUtils.read(in);
197                this.poseParameters = IOUtils.read(in);
198                this.shapeParameters = IOUtils.read(in);
199        }
200
201        /**
202         * Returns the scale (size) of the face
203         * @return the scale of the model
204         */
205        public double getScale() {
206                return this.poseParameters.get(0, 0);
207        }
208
209        /**
210         * Returns the pitch of the model (that is the look up/down, noddy head movement).
211         * @return the pitch of the model
212         */
213        public double getPitch() {
214                return this.poseParameters.get(1, 0);
215        }
216
217        /**
218         * Returns the yaw of the face (that is the side-to-side, shakey head movement).
219         * @return the yaw of the model
220         */
221        public double getYaw() {
222                return this.poseParameters.get(2, 0);
223        }
224
225        /**
226         * Returns the roll of the model (that is the spinning, standy on the head movement)
227         * @return the roll of the model
228         */
229        public double getRoll() {
230                return this.poseParameters.get(3, 0);
231        }
232
233        /**
234         * Returns the x-translation in the model
235         * @return the x-translation of the model
236         */
237        public double getTranslationX() {
238                return this.poseParameters.get(4, 0);
239        }
240
241        /**
242         * Returns the y-translation in the model
243         * @return the y-translation of the model
244         */
245        public double getTranslationY() {
246                return this.poseParameters.get(5, 0);
247        }
248
249        /**
250         * Get the parameters describing the pose of the face. This doesn't include
251         * the translation or scale. The values are {pitch, yaw, roll}
252         *
253         * @return the pose parameters
254         */
255        public DoubleFV getPoseParameters() {
256                return new DoubleFV(new double[] { this.getPitch(), this.getYaw(), this.getRoll() });
257        }
258
259        /**
260         * Get the parameters describing the shape model (i.e. the weights for the
261         * eigenvectors of the point distribution model)
262         *
263         * @return the shape parameters
264         */
265        public DoubleFV getShapeParameters() {
266                final int len = this.shapeParameters.getRowDimension();
267                final double[] vector = new double[len];
268
269                for (int i = 0; i < len; i++) {
270                        vector[i] = this.shapeParameters.get(i, 0);
271                }
272
273                return new DoubleFV(vector);
274        }
275
276        /**
277         * Get a vector describing the pose (pitch, yaw and roll only) and shape of
278         * the model.
279         *
280         * @return the combined pose and shape vector
281         */
282        public DoubleFV getPoseShapeParameters() {
283                final int len = this.shapeParameters.getRowDimension();
284                final double[] vector = new double[len + 3];
285
286                vector[0] = this.getPitch();
287                vector[1] = this.getYaw();
288                vector[2] = this.getRoll();
289
290                for (int i = 3; i < len + 3; i++) {
291                        vector[i] = this.shapeParameters.get(i, 0);
292                }
293
294                return new DoubleFV(vector);
295        }
296
297        /**
298         * Get the matrix of points describing the model. The points are relative to
299         * the image given by {@link #getFacePatch()}.
300         *
301         * @return the shape matrix
302         */
303        public Matrix getShapeMatrix() {
304                return this.shape;
305        }
306
307        /**
308         * Get the visibility matrix
309         *
310         * @return the visibility matrix
311         */
312        public Matrix getVisibility() {
313                return this.visibility;
314        }
315}