001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.image.processing.face.detection;
031
032import java.io.BufferedReader;
033import java.io.DataInput;
034import java.io.DataOutput;
035import java.io.IOException;
036import java.io.InputStream;
037import java.io.InputStreamReader;
038import java.util.ArrayList;
039import java.util.List;
040import java.util.Scanner;
041
042import org.openimaj.citation.annotation.Reference;
043import org.openimaj.citation.annotation.ReferenceType;
044import org.openimaj.image.FImage;
045import org.openimaj.io.IOUtils;
046import org.openimaj.math.geometry.shape.Rectangle;
047
048import Jama.Matrix;
049
050import com.jsaragih.CLM;
051import com.jsaragih.FDet;
052import com.jsaragih.IO;
053import com.jsaragih.MFCheck;
054import com.jsaragih.Tracker;
055
056/**
057 * Face detector based on a constrained local model. Fits a 3D face model to
058 * each detection.
059 * 
060 * @see CLM
061 * 
062 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
063 * 
064 */
065@Reference(
066                type = ReferenceType.Inproceedings,
067                author = { "Jason M. Saragih", "Simon Lucey", "Jeffrey F. Cohn" },
068                title = "Face alignment through subspace constrained mean-shifts",
069                year = "2009",
070                booktitle = "IEEE 12th International Conference on Computer Vision, ICCV 2009, Kyoto, Japan, September 27 - October 4, 2009",
071                pages = { "1034", "1041" },
072                publisher = "IEEE",
073                customData = {
074                                "doi", "http://dx.doi.org/10.1109/ICCV.2009.5459377",
075                                "researchr", "http://researchr.org/publication/SaragihLC09",
076                                "cites", "0",
077                                "citedby", "0"
078                })
079public class CLMFaceDetector implements FaceDetector<CLMDetectedFace, FImage> {
080        /**
081         * Configuration for the face detector
082         * 
083         * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
084         * 
085         */
086        public static class Configuration {
087                /** The constrained local model */
088                public CLM clm;
089
090                /** The reference shape */
091                public Matrix referenceShape;
092
093                /** The current shape */
094                public Matrix shape;
095
096                /** The Face detector */
097                public FDet faceDetector;
098
099                /** The failure checker */
100                public MFCheck failureCheck;
101
102                /** Initialisation similarity */
103                double[] similarity;
104
105                /** The face mesh */
106                public int[][] triangles = null;
107
108                /** The face connections */
109                public int[][] connections = null;
110
111                /** Whether to use the face check (using pixels as a face classifier) */
112                public boolean fcheck = false;
113
114                /** Search window sizes */
115                public int[] windowSize = { 11, 9, 7 };
116
117                /** Number of iterations to use for model fitting */
118                public int nIter = 5;
119
120                /**
121                 * Number of standard deviations from the mean face to allow in the
122                 * model
123                 */
124                public double clamp = 3;
125
126                /** Model fitting optimisation tolerance */
127                public double fTol = 0.01;
128
129                private void read(final InputStream in) {
130                        BufferedReader br = null;
131                        try {
132                                br = new BufferedReader(new InputStreamReader(in));
133                                final Scanner sc = new Scanner(br);
134                                read(sc, true);
135                        } finally {
136                                try {
137                                        br.close();
138                                } catch (final IOException e) {
139                                }
140                        }
141                }
142
143                private void read(Scanner s, boolean readType) {
144                        if (readType) {
145                                final int type = s.nextInt();
146                                assert (type == IO.Types.TRACKER.ordinal());
147                        }
148
149                        clm = CLM.read(s, true);
150                        faceDetector = FDet.read(s, true);
151                        failureCheck = MFCheck.read(s, true);
152                        referenceShape = IO.readMat(s);
153                        similarity = new double[] { s.nextDouble(), s.nextDouble(), s.nextDouble(), s.nextDouble() };
154                        shape = new Matrix(2 * clm._pdm.nPoints(), 1);
155                        clm._pdm.identity(clm._plocal, clm._pglobl);
156                }
157
158                /**
159                 * Construct with the default model parameters
160                 */
161                public Configuration() {
162                        read(Tracker.class.getResourceAsStream("face2.tracker"));
163                        triangles = IO.loadTri(Tracker.class.getResourceAsStream("face.tri"));
164                        connections = IO.loadCon(Tracker.class.getResourceAsStream("face.con"));
165                }
166        }
167
168        private Configuration config;
169
170        /**
171         * Default constructor
172         */
173        public CLMFaceDetector() {
174                config = new Configuration();
175        }
176
177        @Override
178        public void readBinary(DataInput in) throws IOException {
179                config = IOUtils.read(in);
180        }
181
182        @Override
183        public byte[] binaryHeader() {
184                return this.getClass().getName().getBytes();
185        }
186
187        @Override
188        public void writeBinary(DataOutput out) throws IOException {
189                IOUtils.write(config, out);
190        }
191
192        @Override
193        public List<CLMDetectedFace> detectFaces(FImage image) {
194                final List<Rectangle> detRects = config.faceDetector.detect(image);
195
196                return detectFaces(image, detRects);
197        }
198
199        /**
200         * Detect faces in the image using the given rectangles as the seeds from
201         * which to start fitting the model.
202         * 
203         * @param image
204         *            the image
205         * @param detRects
206         *            the seed rectangles
207         * @return the detected faces
208         */
209        public List<CLMDetectedFace> detectFaces(FImage image, List<Rectangle> detRects) {
210                final List<CLMDetectedFace> faces = new ArrayList<CLMDetectedFace>();
211
212                for (final Rectangle f : detRects) {
213                        if ((f.width == 0) || (f.height == 0)) {
214                                continue;
215                        }
216
217                        initShape(f, config.shape, config.referenceShape);
218                        config.clm._pdm.calcParams(config.shape, config.clm._plocal, config.clm._pglobl);
219
220                        config.clm.fit(image, config.windowSize, config.nIter, config.clamp, config.fTol);
221                        config.clm._pdm.calcShape2D(config.shape, config.clm._plocal, config.clm._pglobl);
222
223                        if (config.fcheck) {
224                                if (!config.failureCheck.check(config.clm.getViewIdx(), image, config.shape)) {
225                                        continue;
226                                }
227                        }
228
229                        faces.add(new CLMDetectedFace(f, config.shape.copy(), config.clm._pglobl.copy(), config.clm._plocal.copy(),
230                                        config.clm._visi[config.clm.getViewIdx()].copy(), image));
231                }
232
233                return faces;
234        }
235
236        /**
237         * Initialise the shape within the given rectangle based on the given
238         * reference shape.
239         * 
240         * @param r
241         *            The rectangle
242         * @param shape
243         *            The shape to initialise
244         * @param _rshape
245         *            The reference shape
246         */
247        private void initShape(final Rectangle r, final Matrix shape, final Matrix _rshape) {
248                assert ((shape.getRowDimension() == _rshape.getRowDimension()) && (shape
249                                .getColumnDimension() == _rshape.getColumnDimension()));
250
251                final int n = _rshape.getRowDimension() / 2;
252
253                final double a = r.width * Math.cos(config.similarity[1]) * config.similarity[0] + 1;
254                final double b = r.width * Math.sin(config.similarity[1]) * config.similarity[0];
255
256                final double tx = r.x + (int) (r.width / 2) + r.width * config.similarity[2];
257                final double ty = r.y + (int) (r.height / 2) + r.height * config.similarity[3];
258
259                final double[][] s = _rshape.getArray();
260                final double[][] d = shape.getArray();
261
262                for (int i = 0; i < n; i++) {
263                        d[i][0] = a * s[i][0] - b * s[i + n][0] + tx;
264                        d[i + n][0] = b * s[i][0] + a * s[i + n][0] + ty;
265                }
266        }
267
268        /**
269         * Get the internal configuration of the detector.
270         * 
271         * @return the internal configuration of the detector.
272         */
273        public Configuration getConfiguration() {
274                return config;
275        }
276}