001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.image.processing.face.detection;
031
032import java.io.ByteArrayInputStream;
033import java.io.ByteArrayOutputStream;
034import java.io.DataInput;
035import java.io.DataOutput;
036import java.io.File;
037import java.io.IOException;
038import java.io.ObjectInputStream;
039import java.io.ObjectOutputStream;
040import java.util.ArrayList;
041import java.util.List;
042
043import org.openimaj.citation.annotation.Reference;
044import org.openimaj.citation.annotation.ReferenceType;
045import org.openimaj.image.FImage;
046import org.openimaj.image.ImageUtilities;
047import org.openimaj.image.MBFImage;
048import org.openimaj.image.colour.Transforms;
049import org.openimaj.image.connectedcomponent.ConnectedComponentLabeler;
050import org.openimaj.image.model.pixel.HistogramPixelModel;
051import org.openimaj.image.model.pixel.MBFPixelClassificationModel;
052import org.openimaj.image.pixel.ConnectedComponent;
053import org.openimaj.image.pixel.ConnectedComponent.ConnectMode;
054import org.openimaj.image.processing.convolution.FSobelMagnitude;
055import org.openimaj.image.processor.connectedcomponent.render.OrientatedBoundingBoxRenderer;
056import org.openimaj.math.geometry.shape.Rectangle;
057
058/**
059 * Implementation of a face detector along the lines of "Human Face Detection in
060 * Cluttered Color Images Using Skin Color and Edge Information" K. Sandeep and
061 * A. N. Rajagopalan (IIT/Madras)
062 *
063 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
064 *
065 */
066@Reference(
067                type = ReferenceType.Article,
068                author = { "Sandeep, K", "Rajagopalan, A N" },
069                title = "Human Face Detection in Cluttered Color Images Using Skin Color and Edge Information",
070                year = "2002",
071                journal = "Electrical Engineering",
072                url = "http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.12.730&rep=rep1&type=pdf",
073                publisher = "Citeseer")
074public class SandeepFaceDetector implements FaceDetector<CCDetectedFace, MBFImage> {
075        /**
076         * The golden ratio (for comparing facial height/width)
077         */
078        public final static double GOLDEN_RATIO = 1.618033989; // ((1 + sqrt(5) / 2)
079
080        private final String DEFAULT_MODEL = "/org/openimaj/image/processing/face/detection/skin-histogram-16-6.bin";
081
082        private ConnectedComponentLabeler ccl;
083
084        MBFPixelClassificationModel skinModel;
085        float skinThreshold = 0.1F;
086        float edgeThreshold = 125F / 255F;
087        float goldenRatioThreshold = 0.65F;
088        float percentageThreshold = 0.55F;
089
090        /**
091         * Construct a new {@link SandeepFaceDetector} with the default skin-tone
092         * model.
093         */
094        public SandeepFaceDetector() {
095                ccl = new ConnectedComponentLabeler(ConnectMode.CONNECT_8);
096
097                try {
098                        if (this.getClass().getResource(DEFAULT_MODEL) == null) {
099                                // This is to create the skin model
100                                skinModel = new HistogramPixelModel(16, 6);
101                                final MBFImage rgb = ImageUtilities.readMBF(this.getClass().getResourceAsStream("skin.png"));
102                                skinModel.learnModel(Transforms.RGB_TO_HS(rgb));
103                                // final ObjectOutputStream oos = new ObjectOutputStream(new
104                                // FileOutputStream(new File(
105                                // "src/main/resources" + DEFAULT_MODEL)));
106                                // oos.writeObject(skinModel);
107                                // oos.close();
108                        } else {
109                                // Load in the skin model
110                                final ObjectInputStream ois = new ObjectInputStream(this.getClass().getResourceAsStream(DEFAULT_MODEL));
111                                skinModel = (MBFPixelClassificationModel) ois.readObject();
112                        }
113                } catch (final Exception e) {
114                        e.printStackTrace();
115                        throw new RuntimeException(e);
116                }
117        }
118
119        /**
120         * Construct the detector with the given pixel classification model.
121         *
122         * @param skinModel
123         *            the underlying classification model.
124         */
125        public SandeepFaceDetector(MBFPixelClassificationModel skinModel) {
126                ccl = new ConnectedComponentLabeler(ConnectMode.CONNECT_8);
127                this.skinModel = skinModel;
128        }
129
130        protected FImage generateSkinColorMap(MBFImage inputHS) {
131                final FImage map = skinModel.predict(inputHS);
132
133                map.clipMin(skinThreshold);
134                return map;
135        }
136
137        protected FImage generateSobelMagnitudes(MBFImage inputRGB) {
138                final MBFImage mag = inputRGB.process(new FSobelMagnitude());
139                final FImage ret = mag.flattenMax().clipMax(edgeThreshold);
140                return ret;
141        }
142
143        protected FImage generateFaceMap(FImage skin, FImage edge) {
144                for (int y = 0; y < skin.height; y++) {
145                        for (int x = 0; x < skin.width; x++) {
146
147                                if (edge.pixels[y][x] != 0 && skin.pixels[y][x] != 0)
148                                        skin.pixels[y][x] = 1f;
149                                else
150                                        skin.pixels[y][x] = 0f;
151                        }
152                }
153
154                return skin;
155        }
156
157        protected List<CCDetectedFace> extractFaces(FImage faceMap, FImage skinMap, FImage image) {
158                final List<ConnectedComponent> blobs = ccl.findComponents(faceMap);
159                final List<CCDetectedFace> faces = new ArrayList<CCDetectedFace>();
160
161                for (final ConnectedComponent blob : blobs) {
162                        if (blob.calculateArea() > 1000) {
163                                final double[] centroid = blob.calculateCentroid();
164                                final double[] hw = blob.calculateAverageHeightWidth(centroid);
165
166                                final double percentageSkin = calculatePercentageSkin(skinMap,
167                                                (int) Math.round(centroid[0] - (hw[0] / 2)),
168                                                (int) Math.round(centroid[1] - (hw[1] / 2)),
169                                                (int) Math.round(centroid[0] + (hw[0] / 2)),
170                                                (int) Math.round(centroid[1] + (hw[1] / 2)));
171
172                                final double ratio = hw[0] / hw[1];
173
174                                if (Math.abs(ratio - GOLDEN_RATIO) < goldenRatioThreshold && percentageSkin > percentageThreshold) {
175                                        final Rectangle r = blob.calculateRegularBoundingBox();
176                                        faces.add(new CCDetectedFace(
177                                                        r,
178                                                        image.extractROI(r),
179                                                        blob,
180                                                        (float) ((percentageSkin / percentageThreshold)
181                                                                        * (Math.abs(ratio - GOLDEN_RATIO) / goldenRatioThreshold))));
182                                }
183                        }
184                }
185
186                return faces;
187        }
188
189        private double calculatePercentageSkin(FImage skinMap, int l, int t, int r, int b) {
190                int npix = 0;
191                int nskin = 0;
192
193                l = Math.max(l, 0);
194                t = Math.max(t, 0);
195                r = Math.min(r, skinMap.getWidth());
196                b = Math.min(b, skinMap.getHeight());
197
198                for (int y = t; y < b; y++) {
199                        for (int x = l; x < r; x++) {
200                                npix++;
201                                if (skinMap.pixels[y][x] != 0)
202                                        nskin++;
203                        }
204                }
205
206                return (double) nskin / (double) npix;
207        }
208
209        @Override
210        public List<CCDetectedFace> detectFaces(MBFImage inputRGB) {
211                final FImage skin = generateSkinColorMap(Transforms.RGB_TO_HS(inputRGB));
212                final FImage edge = generateSobelMagnitudes(inputRGB);
213
214                final FImage map = generateFaceMap(skin, edge);
215
216                return extractFaces(map, skin, Transforms.calculateIntensityNTSC(inputRGB));
217        }
218
219        /**
220         * @return The underlying skin-tone classifier
221         */
222        public MBFPixelClassificationModel getSkinModel() {
223                return skinModel;
224        }
225
226        /**
227         * Set the underlying skin-tone classifier
228         *
229         * @param skinModel
230         */
231        public void setSkinModel(MBFPixelClassificationModel skinModel) {
232                this.skinModel = skinModel;
233        }
234
235        /**
236         * @return the detection threshold.
237         */
238        public float getSkinThreshold() {
239                return skinThreshold;
240        }
241
242        /**
243         * Set the detection threshold.
244         *
245         * @param skinThreshold
246         */
247        public void setSkinThreshold(float skinThreshold) {
248                this.skinThreshold = skinThreshold;
249        }
250
251        /**
252         * @return The edge threshold.
253         */
254        public float getEdgeThreshold() {
255                return edgeThreshold;
256        }
257
258        /**
259         * Set the edge threshold.
260         *
261         * @param edgeThreshold
262         */
263        public void setEdgeThreshold(float edgeThreshold) {
264                this.edgeThreshold = edgeThreshold;
265        }
266
267        /**
268         * @return The percentage threshold
269         */
270        public float getPercentageThreshold() {
271                return percentageThreshold;
272        }
273
274        /**
275         * Set the percentage threshold
276         *
277         * @param percentageThreshold
278         */
279        public void setPercentageThreshold(float percentageThreshold) {
280                this.percentageThreshold = percentageThreshold;
281        }
282
283        /**
284         * Run the face detector following the conventions of the ocv detector
285         *
286         * @param args
287         * @throws IOException
288         */
289        public static void main(String[] args) throws IOException {
290                if (args.length < 1 || args.length > 2) {
291                        System.err.println("Usage: SandeepFaceDetector filename [filename_out]");
292                        return;
293                }
294
295                final String inputImage = args[0];
296                String outputImage = null;
297                if (args.length == 2)
298                        outputImage = args[1];
299
300                final SandeepFaceDetector sfd = new SandeepFaceDetector();
301
302                // tweek the default settings
303                sfd.edgeThreshold = 0.39F;
304                sfd.ccl = new ConnectedComponentLabeler(ConnectMode.CONNECT_4);
305
306                final MBFImage image = ImageUtilities.readMBF(new File(inputImage));
307                final List<CCDetectedFace> faces = sfd.detectFaces(image);
308
309                if (outputImage != null) {
310                        final OrientatedBoundingBoxRenderer<Float> render = new OrientatedBoundingBoxRenderer<Float>(
311                                        image.getWidth(), image.getHeight(), 1.0F);
312                        for (final CCDetectedFace f : faces)
313                                f.connectedComponent.process(render);
314                        image.multiplyInplace(render.getImage().inverse());
315
316                        ImageUtilities.write(image, outputImage.substring(outputImage.lastIndexOf('.') + 1), new File(outputImage));
317                }
318
319                for (final CCDetectedFace f : faces) {
320                        System.out.format("%s, %d, %d, %d, %d\n",
321                                        "uk.ac.soton.ecs.jsh2.image.proc.tools.face.detection.skin-histogram-16-6.bin",
322                                        (int) f.bounds.x,
323                                        (int) f.bounds.y,
324                                        (int) f.bounds.width,
325                                        (int) f.bounds.height);
326                }
327        }
328
329        @Override
330        public void readBinary(DataInput in) throws IOException {
331                // ccl;
332
333                try {
334                        final byte[] bytes = new byte[in.readInt()];
335                        in.readFully(bytes);
336                        skinModel = (MBFPixelClassificationModel) new ObjectInputStream(new ByteArrayInputStream(bytes)).readObject();
337                } catch (final ClassNotFoundException e) {
338                        throw new IOException(e);
339                }
340
341                skinThreshold = in.readFloat();
342                edgeThreshold = in.readFloat();
343                goldenRatioThreshold = in.readFloat();
344                percentageThreshold = in.readFloat();
345        }
346
347        @Override
348        public byte[] binaryHeader() {
349                return "SdFD".getBytes();
350        }
351
352        @Override
353        public void writeBinary(DataOutput out) throws IOException {
354                // ccl;
355
356                final ByteArrayOutputStream baos = new ByteArrayOutputStream();
357                final ObjectOutputStream oos = new ObjectOutputStream(baos);
358                oos.writeObject(skinModel);
359                oos.close();
360
361                out.writeInt(baos.size());
362                out.write(baos.toByteArray());
363
364                out.writeFloat(skinThreshold);
365                out.writeFloat(edgeThreshold);
366                out.writeFloat(goldenRatioThreshold);
367                out.writeFloat(percentageThreshold);
368        }
369}