001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.image.processing.face.alignment; 031 032import java.io.DataInput; 033import java.io.DataOutput; 034import java.io.IOException; 035import java.util.ArrayList; 036import java.util.List; 037 038import org.openimaj.image.FImage; 039import org.openimaj.image.ImageUtilities; 040import org.openimaj.image.processing.face.detection.keypoints.FKEFaceDetector; 041import org.openimaj.image.processing.face.detection.keypoints.FacialKeypoint; 042import org.openimaj.image.processing.face.detection.keypoints.FacialKeypoint.FacialKeypointType; 043import org.openimaj.image.processing.face.detection.keypoints.KEDetectedFace; 044import org.openimaj.image.processing.transform.PiecewiseMeshWarp; 045import org.openimaj.math.geometry.point.Point2d; 046import org.openimaj.math.geometry.point.Point2dImpl; 047import org.openimaj.math.geometry.shape.Polygon; 048import org.openimaj.math.geometry.shape.Shape; 049import org.openimaj.math.geometry.transforms.TransformUtilities; 050import org.openimaj.util.pair.Pair; 051 052import Jama.Matrix; 053 054/** 055 * A MeshWarpAligner aligns facial images using a piecewise mesh warping such 056 * that all detected facial keypoints are moved to their canonical coordinates. 057 * The warping is accomplished by defining a mesh of triangles and 058 * quadrilaterals over the facial keypoints and using bi-linear interpolation to 059 * get corrected pixel values. 060 * 061 * @see PiecewiseMeshWarp 062 * 063 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk) 064 * 065 */ 066public class MeshWarpAligner implements FaceAligner<KEDetectedFace> { 067 // Define the default mesh 068 private static final String[][] DEFAULT_MESH_DEFINITION = { 069 { "EYE_LEFT_RIGHT", "EYE_RIGHT_LEFT", "NOSE_MIDDLE" }, 070 { "EYE_LEFT_LEFT", "EYE_LEFT_RIGHT", "NOSE_LEFT" }, 071 { "EYE_RIGHT_RIGHT", "EYE_RIGHT_LEFT", "NOSE_RIGHT" }, 072 { "EYE_LEFT_RIGHT", "NOSE_LEFT", "NOSE_MIDDLE" }, 073 { "EYE_RIGHT_LEFT", "NOSE_RIGHT", "NOSE_MIDDLE" }, 074 { "MOUTH_LEFT", "MOUTH_RIGHT", "NOSE_MIDDLE" }, 075 { "MOUTH_LEFT", "NOSE_LEFT", "NOSE_MIDDLE" }, 076 { "MOUTH_RIGHT", "NOSE_RIGHT", "NOSE_MIDDLE" }, 077 { "MOUTH_LEFT", "NOSE_LEFT", "EYE_LEFT_LEFT" }, 078 { "MOUTH_RIGHT", "NOSE_RIGHT", "EYE_RIGHT_RIGHT" }, 079 080 // { "P0", "EYE_LEFT_LEFT", "EYE_LEFT_RIGHT" }, 081 // { "P1", "EYE_RIGHT_RIGHT", "EYE_RIGHT_LEFT" }, 082 // { "P0", "EYE_LEFT_RIGHT", "EYE_RIGHT_LEFT", "P1" }, 083 // { "P3", "MOUTH_LEFT", "MOUTH_RIGHT", "P2" }, 084 085 // { "P0", "EYE_LEFT_LEFT", "MOUTH_LEFT" }, 086 // { "P1", "EYE_RIGHT_RIGHT", "MOUTH_RIGHT" }, 087 // {"P0", "P3", "MOUTH_LEFT"}, 088 // {"P1", "P2", "MOUTH_RIGHT"}, 089 090 // { "P0", "EYE_LEFT_RIGHT", "EYE_RIGHT_LEFT" }, 091 // { "P1", "EYE_RIGHT_LEFT", "EYE_LEFT_RIGHT" }, 092 // 093 // { "P3", "MOUTH_LEFT", "MOUTH_RIGHT" }, 094 // { "P2", "MOUTH_RIGHT", "MOUTH_LEFT" }, 095 096 // {"P3", "EYE_LEFT_LEFT", "MOUTH_LEFT"}, 097 // {"P2", "EYE_RIGHT_RIGHT", "MOUTH_RIGHT"}, 098 }; 099 100 // Define the outer edges 101 private static final Point2d P0 = new Point2dImpl(0, 0); 102 private static final Point2d P1 = new Point2dImpl(80, 0); 103 private static final Point2d P2 = new Point2dImpl(80, 80); 104 private static final Point2d P3 = new Point2dImpl(0, 80); 105 106 // Define the canonical point positions 107 private static FacialKeypoint[] canonical = loadCanonicalPoints(); 108 109 // Define the mesh 110 String[][] meshDefinition = DEFAULT_MESH_DEFINITION; 111 112 FImage mask; 113 114 /** 115 * Default constructor 116 */ 117 public MeshWarpAligner() { 118 this(DEFAULT_MESH_DEFINITION); 119 } 120 121 /** 122 * Construct with the given mesh definition 123 * 124 * @param meshDefinition 125 * The mesh definition 126 */ 127 public MeshWarpAligner(String[][] meshDefinition) { 128 this.meshDefinition = meshDefinition; 129 130 final List<Pair<Shape>> mesh = createMesh(canonical); 131 132 // build mask by mapping the canonical coords to themselves on a white 133 // image 134 mask = new FImage((int) P2.getX(), (int) P2.getY()); 135 mask.fill(1f); 136 mask = mask.processInplace(new PiecewiseMeshWarp<Float, FImage>(mesh)); 137 } 138 139 private static FacialKeypoint[] loadCanonicalPoints() { 140 final FacialKeypoint[] points = new FacialKeypoint[AffineAligner.Pmu[0].length]; 141 142 for (int i = 0; i < points.length; i++) { 143 points[i] = new FacialKeypoint(FacialKeypointType.valueOf(i)); 144 points[i].position = new Point2dImpl(2 * AffineAligner.Pmu[0][i] - 40, 2 * AffineAligner.Pmu[1][i] - 40); 145 } 146 147 return points; 148 } 149 150 protected FacialKeypoint[] getActualPoints(FacialKeypoint[] keys, Matrix tf0) { 151 final FacialKeypoint[] points = new FacialKeypoint[AffineAligner.Pmu[0].length]; 152 153 for (int i = 0; i < points.length; i++) { 154 points[i] = new FacialKeypoint(FacialKeypointType.valueOf(i)); 155 points[i].position = new Point2dImpl( 156 FacialKeypoint.getKeypoint(keys, FacialKeypointType.valueOf(i)).position.transform(tf0)); 157 } 158 159 return points; 160 } 161 162 protected List<Pair<Shape>> createMesh(FacialKeypoint[] det) { 163 final List<Pair<Shape>> shapes = new ArrayList<Pair<Shape>>(); 164 165 for (final String[] vertDefs : meshDefinition) { 166 final Polygon p1 = new Polygon(); 167 final Polygon p2 = new Polygon(); 168 169 for (final String v : vertDefs) { 170 p1.getVertices().add(lookupVertex(v, det)); 171 p2.getVertices().add(lookupVertex(v, canonical)); 172 } 173 shapes.add(new Pair<Shape>(p1, p2)); 174 } 175 176 return shapes; 177 } 178 179 private Point2d lookupVertex(String v, FacialKeypoint[] pts) { 180 if (v.equals("P0")) 181 return P0; 182 if (v.equals("P1")) 183 return P1; 184 if (v.equals("P2")) 185 return P2; 186 if (v.equals("P3")) 187 return P3; 188 189 return FacialKeypoint.getKeypoint(pts, FacialKeypointType.valueOf(v)).position; 190 } 191 192 @Override 193 public FImage align(KEDetectedFace descriptor) { 194 final float scalingX = P2.getX() / descriptor.getFacePatch().width; 195 final float scalingY = P2.getY() / descriptor.getFacePatch().height; 196 final Matrix tf0 = TransformUtilities.scaleMatrix(scalingX, scalingY); 197 final Matrix tf = tf0.inverse(); 198 199 final FImage J = FKEFaceDetector.pyramidResize(descriptor.getFacePatch(), tf); 200 final FImage smallpatch = FKEFaceDetector.extractPatch(J, tf, 80, 0); 201 202 return getWarpedImage(descriptor.getKeypoints(), smallpatch, tf0); 203 } 204 205 protected FImage getWarpedImage(FacialKeypoint[] kpts, FImage patch, Matrix tf0) { 206 final FacialKeypoint[] det = getActualPoints(kpts, tf0); 207 final List<Pair<Shape>> mesh = createMesh(det); 208 209 final FImage newpatch = patch.process(new PiecewiseMeshWarp<Float, FImage>(mesh)); 210 211 return newpatch; 212 } 213 214 @Override 215 public FImage getMask() { 216 return mask; 217 } 218 219 @Override 220 public void readBinary(DataInput in) throws IOException { 221 int sz = in.readInt(); 222 meshDefinition = new String[sz][]; 223 for (int i = 0; i < meshDefinition.length; i++) { 224 sz = in.readInt(); 225 meshDefinition[i] = new String[sz]; 226 for (int j = 0; j < meshDefinition[i].length; j++) 227 meshDefinition[i][j] = in.readUTF(); 228 } 229 230 mask = ImageUtilities.readF(in); 231 } 232 233 @Override 234 public byte[] binaryHeader() { 235 return this.getClass().getName().getBytes(); 236 } 237 238 @Override 239 public void writeBinary(DataOutput out) throws IOException { 240 out.writeInt(meshDefinition.length); 241 for (final String[] def : meshDefinition) { 242 out.writeInt(def.length); 243 for (final String s : def) 244 out.writeUTF(s); 245 } 246 247 ImageUtilities.write(mask, "png", out); 248 } 249}