001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.image.processing.face.alignment;
031
032import java.io.DataInput;
033import java.io.DataOutput;
034import java.io.IOException;
035import java.util.ArrayList;
036import java.util.List;
037
038import org.openimaj.image.FImage;
039import org.openimaj.image.ImageUtilities;
040import org.openimaj.image.processing.face.detection.keypoints.FKEFaceDetector;
041import org.openimaj.image.processing.face.detection.keypoints.FacialKeypoint;
042import org.openimaj.image.processing.face.detection.keypoints.FacialKeypoint.FacialKeypointType;
043import org.openimaj.image.processing.face.detection.keypoints.KEDetectedFace;
044import org.openimaj.image.processing.transform.PiecewiseMeshWarp;
045import org.openimaj.math.geometry.point.Point2d;
046import org.openimaj.math.geometry.point.Point2dImpl;
047import org.openimaj.math.geometry.shape.Polygon;
048import org.openimaj.math.geometry.shape.Shape;
049import org.openimaj.math.geometry.transforms.TransformUtilities;
050import org.openimaj.util.pair.Pair;
051
052import Jama.Matrix;
053
054/**
055 * A MeshWarpAligner aligns facial images using a piecewise mesh warping such
056 * that all detected facial keypoints are moved to their canonical coordinates.
057 * The warping is accomplished by defining a mesh of triangles and
058 * quadrilaterals over the facial keypoints and using bi-linear interpolation to
059 * get corrected pixel values.
060 * 
061 * @see PiecewiseMeshWarp
062 * 
063 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
064 * 
065 */
066public class MeshWarpAligner implements FaceAligner<KEDetectedFace> {
067        // Define the default mesh
068        private static final String[][] DEFAULT_MESH_DEFINITION = {
069                        { "EYE_LEFT_RIGHT", "EYE_RIGHT_LEFT", "NOSE_MIDDLE" },
070                        { "EYE_LEFT_LEFT", "EYE_LEFT_RIGHT", "NOSE_LEFT" },
071                        { "EYE_RIGHT_RIGHT", "EYE_RIGHT_LEFT", "NOSE_RIGHT" },
072                        { "EYE_LEFT_RIGHT", "NOSE_LEFT", "NOSE_MIDDLE" },
073                        { "EYE_RIGHT_LEFT", "NOSE_RIGHT", "NOSE_MIDDLE" },
074                        { "MOUTH_LEFT", "MOUTH_RIGHT", "NOSE_MIDDLE" },
075                        { "MOUTH_LEFT", "NOSE_LEFT", "NOSE_MIDDLE" },
076                        { "MOUTH_RIGHT", "NOSE_RIGHT", "NOSE_MIDDLE" },
077                        { "MOUTH_LEFT", "NOSE_LEFT", "EYE_LEFT_LEFT" },
078                        { "MOUTH_RIGHT", "NOSE_RIGHT", "EYE_RIGHT_RIGHT" },
079
080                        // { "P0", "EYE_LEFT_LEFT", "EYE_LEFT_RIGHT" },
081                        // { "P1", "EYE_RIGHT_RIGHT", "EYE_RIGHT_LEFT" },
082                        // { "P0", "EYE_LEFT_RIGHT", "EYE_RIGHT_LEFT", "P1" },
083                        // { "P3", "MOUTH_LEFT", "MOUTH_RIGHT", "P2" },
084
085                        // { "P0", "EYE_LEFT_LEFT", "MOUTH_LEFT" },
086                        // { "P1", "EYE_RIGHT_RIGHT", "MOUTH_RIGHT" },
087                        // {"P0", "P3", "MOUTH_LEFT"},
088                        // {"P1", "P2", "MOUTH_RIGHT"},
089
090                        // { "P0", "EYE_LEFT_RIGHT", "EYE_RIGHT_LEFT" },
091                        // { "P1", "EYE_RIGHT_LEFT", "EYE_LEFT_RIGHT" },
092                        //
093                        // { "P3", "MOUTH_LEFT", "MOUTH_RIGHT" },
094                        // { "P2", "MOUTH_RIGHT", "MOUTH_LEFT" },
095
096                        // {"P3", "EYE_LEFT_LEFT", "MOUTH_LEFT"},
097                        // {"P2", "EYE_RIGHT_RIGHT", "MOUTH_RIGHT"},
098        };
099
100        // Define the outer edges
101        private static final Point2d P0 = new Point2dImpl(0, 0);
102        private static final Point2d P1 = new Point2dImpl(80, 0);
103        private static final Point2d P2 = new Point2dImpl(80, 80);
104        private static final Point2d P3 = new Point2dImpl(0, 80);
105
106        // Define the canonical point positions
107        private static FacialKeypoint[] canonical = loadCanonicalPoints();
108
109        // Define the mesh
110        String[][] meshDefinition = DEFAULT_MESH_DEFINITION;
111
112        FImage mask;
113
114        /**
115         * Default constructor
116         */
117        public MeshWarpAligner() {
118                this(DEFAULT_MESH_DEFINITION);
119        }
120
121        /**
122         * Construct with the given mesh definition
123         * 
124         * @param meshDefinition
125         *            The mesh definition
126         */
127        public MeshWarpAligner(String[][] meshDefinition) {
128                this.meshDefinition = meshDefinition;
129
130                final List<Pair<Shape>> mesh = createMesh(canonical);
131
132                // build mask by mapping the canonical coords to themselves on a white
133                // image
134                mask = new FImage((int) P2.getX(), (int) P2.getY());
135                mask.fill(1f);
136                mask = mask.processInplace(new PiecewiseMeshWarp<Float, FImage>(mesh));
137        }
138
139        private static FacialKeypoint[] loadCanonicalPoints() {
140                final FacialKeypoint[] points = new FacialKeypoint[AffineAligner.Pmu[0].length];
141
142                for (int i = 0; i < points.length; i++) {
143                        points[i] = new FacialKeypoint(FacialKeypointType.valueOf(i));
144                        points[i].position = new Point2dImpl(2 * AffineAligner.Pmu[0][i] - 40, 2 * AffineAligner.Pmu[1][i] - 40);
145                }
146
147                return points;
148        }
149
150        protected FacialKeypoint[] getActualPoints(FacialKeypoint[] keys, Matrix tf0) {
151                final FacialKeypoint[] points = new FacialKeypoint[AffineAligner.Pmu[0].length];
152
153                for (int i = 0; i < points.length; i++) {
154                        points[i] = new FacialKeypoint(FacialKeypointType.valueOf(i));
155                        points[i].position = new Point2dImpl(
156                                        FacialKeypoint.getKeypoint(keys, FacialKeypointType.valueOf(i)).position.transform(tf0));
157                }
158
159                return points;
160        }
161
162        protected List<Pair<Shape>> createMesh(FacialKeypoint[] det) {
163                final List<Pair<Shape>> shapes = new ArrayList<Pair<Shape>>();
164
165                for (final String[] vertDefs : meshDefinition) {
166                        final Polygon p1 = new Polygon();
167                        final Polygon p2 = new Polygon();
168
169                        for (final String v : vertDefs) {
170                                p1.getVertices().add(lookupVertex(v, det));
171                                p2.getVertices().add(lookupVertex(v, canonical));
172                        }
173                        shapes.add(new Pair<Shape>(p1, p2));
174                }
175
176                return shapes;
177        }
178
179        private Point2d lookupVertex(String v, FacialKeypoint[] pts) {
180                if (v.equals("P0"))
181                        return P0;
182                if (v.equals("P1"))
183                        return P1;
184                if (v.equals("P2"))
185                        return P2;
186                if (v.equals("P3"))
187                        return P3;
188
189                return FacialKeypoint.getKeypoint(pts, FacialKeypointType.valueOf(v)).position;
190        }
191
192        @Override
193        public FImage align(KEDetectedFace descriptor) {
194                final float scalingX = P2.getX() / descriptor.getFacePatch().width;
195                final float scalingY = P2.getY() / descriptor.getFacePatch().height;
196                final Matrix tf0 = TransformUtilities.scaleMatrix(scalingX, scalingY);
197                final Matrix tf = tf0.inverse();
198
199                final FImage J = FKEFaceDetector.pyramidResize(descriptor.getFacePatch(), tf);
200                final FImage smallpatch = FKEFaceDetector.extractPatch(J, tf, 80, 0);
201
202                return getWarpedImage(descriptor.getKeypoints(), smallpatch, tf0);
203        }
204
205        protected FImage getWarpedImage(FacialKeypoint[] kpts, FImage patch, Matrix tf0) {
206                final FacialKeypoint[] det = getActualPoints(kpts, tf0);
207                final List<Pair<Shape>> mesh = createMesh(det);
208
209                final FImage newpatch = patch.process(new PiecewiseMeshWarp<Float, FImage>(mesh));
210
211                return newpatch;
212        }
213
214        @Override
215        public FImage getMask() {
216                return mask;
217        }
218
219        @Override
220        public void readBinary(DataInput in) throws IOException {
221                int sz = in.readInt();
222                meshDefinition = new String[sz][];
223                for (int i = 0; i < meshDefinition.length; i++) {
224                        sz = in.readInt();
225                        meshDefinition[i] = new String[sz];
226                        for (int j = 0; j < meshDefinition[i].length; j++)
227                                meshDefinition[i][j] = in.readUTF();
228                }
229
230                mask = ImageUtilities.readF(in);
231        }
232
233        @Override
234        public byte[] binaryHeader() {
235                return this.getClass().getName().getBytes();
236        }
237
238        @Override
239        public void writeBinary(DataOutput out) throws IOException {
240                out.writeInt(meshDefinition.length);
241                for (final String[] def : meshDefinition) {
242                        out.writeInt(def.length);
243                        for (final String s : def)
244                                out.writeUTF(s);
245                }
246
247                ImageUtilities.write(mask, "png", out);
248        }
249}