001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.image.processing.face.similarity;
031
032import java.util.HashMap;
033import java.util.LinkedHashMap;
034import java.util.List;
035import java.util.Map;
036import java.util.Set;
037
038import org.openimaj.image.Image;
039import org.openimaj.image.processing.face.detection.DetectedFace;
040import org.openimaj.image.processing.face.detection.FaceDetector;
041import org.openimaj.image.processing.face.feature.FacialFeature;
042import org.openimaj.image.processing.face.feature.FacialFeatureExtractor;
043import org.openimaj.image.processing.face.feature.comparison.FacialFeatureComparator;
044import org.openimaj.math.geometry.shape.Rectangle;
045import org.openimaj.math.matrix.similarity.SimilarityMatrix;
046import org.openimaj.math.matrix.similarity.processor.InvertData;
047
048/**
049 * The {@link FaceSimilarityEngine} allows computation of the similarity
050 * between faces in two images.
051 * 
052 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
053 *
054 * @param <D> The type of {@link DetectedFace} 
055 * @param <F> the type of {@link FacialFeature} 
056 * @param <I> The type of {@link Image}
057 */
058public class FaceSimilarityEngine<D extends DetectedFace, F extends FacialFeature, I extends Image<?, I>> {
059        private FaceDetector<D, I> detector;
060        private FacialFeatureExtractor<F, D> extractor;
061        private FacialFeatureComparator<F> comparator;
062        private Map<String, Rectangle> boundingBoxes;
063        private Map<String, F> featureCache;
064        private Map<String, List<D>> detectedFaceCache;
065        private LinkedHashMap<String, Map<String, Double>> similarityMatrix;
066        private List<D> queryfaces;
067        private List<D> testfaces;
068        private String queryId;
069        private String testId;
070        private boolean cache;
071
072        /**
073         * Construct a new {@link FaceSimilarityEngine} from the
074         * specified detector, extractor and comparator.
075         * 
076         * @param detector The face detector
077         * @param extractor The feature extractor
078         * @param comparator The feature comparator 
079         */
080        public FaceSimilarityEngine(FaceDetector<D, I> detector,
081                        FacialFeatureExtractor<F, D> extractor,
082                        FacialFeatureComparator<F> comparator) {
083                this.detector = detector;
084                this.extractor = extractor;
085                this.comparator = comparator;
086                this.similarityMatrix = new LinkedHashMap<String, Map<String,Double>>();
087                this.boundingBoxes = new HashMap<String, Rectangle>();
088                featureCache = new HashMap<String,F>();
089                detectedFaceCache = new HashMap<String,List<D>>();
090        }
091
092        /**
093         * @return the detector
094         */
095        public FaceDetector<D, I> detector() {
096                return detector;
097        }
098
099        /**
100         * @return the featureFactory
101         */
102        public FacialFeatureExtractor<F, D> extractor() {
103                return extractor;
104        }
105
106        /**
107         * @return the comparator
108         */
109        public FacialFeatureComparator<F> comparator() {
110                return comparator;
111        }
112
113        /**
114         * Create a new {@link FaceSimilarityEngine} from the
115         * specified detector, extractor and comparator.
116         * 
117         * @param <D> The type of {@link DetectedFace} 
118         * @param <F> the type of {@link FacialFeature} 
119         * @param <I> The type of {@link Image}
120         * 
121         * @param detector The face detector
122         * @param extractor The feature extractor
123         * @param comparator The feature comparator 
124         * @return the new {@link FaceSimilarityEngine}
125         */
126        public static <D extends DetectedFace, F extends FacialFeature, I extends Image<?, I>> 
127                FaceSimilarityEngine<D, F, I> create(
128                        FaceDetector<D, I> detector,
129                        FacialFeatureExtractor<F, D> extractor,
130                        FacialFeatureComparator<F> comparator) 
131        {
132                return new FaceSimilarityEngine<D, F, I>(detector, extractor,
133                                comparator);
134        }
135
136        /**
137         * Set the query image.
138         * @param queryImage the query image
139         * @param queryId the identifier of the query image 
140         */
141        public void setQuery(I queryImage, String queryId) {
142                this.queryfaces = getDetectedFaces(queryId,queryImage);
143                this.queryId = queryId;
144                updateBoundingBox(this.queryfaces, queryId);
145        }
146
147        private List<D> getDetectedFaces(String faceId, I faceImage) {
148                List<D> toRet = null;
149                if(!this.cache){
150                        toRet = this.detector.detectFaces(faceImage);
151                }
152                else{
153                        toRet = this.detectedFaceCache.get(faceId);
154                        if(toRet == null){
155//                              System.out.println("Redetected face: " + faceId);
156                                toRet = this.detector.detectFaces(faceImage);;
157                                this.detectedFaceCache.put(faceId, toRet);
158                        }
159                }
160                return toRet;
161        }
162
163        private void updateBoundingBox(List<D> faces, String imageId) {
164                // We need to store the first one if we're running withFirst = true
165                if (boundingBoxes != null)
166                        for (int ff = 0; ff < faces.size(); ff++)
167                                if (boundingBoxes.get(imageId + ":" + ff) == null)
168                                        boundingBoxes.put(imageId + ":" + ff, faces.get(ff)
169                                                        .getBounds());
170        }
171
172        /**
173         * Set the image against which the query will be compared to next
174         * 
175         * @param testImage
176         * @param testId
177         */
178        public void setTest(I testImage, String testId) {
179                this.testId = testId;
180                this.testfaces = getDetectedFaces(testId,testImage);
181                updateBoundingBox(this.testfaces, testId);
182        }
183
184        /**
185         * Compare the query to itself for the next test
186         */
187        public void setQueryTest() {
188                this.testfaces = this.queryfaces;
189                this.testId = this.queryId;
190        }
191
192        /**
193         * Compute the similarities between faces in the query and target
194         */
195        public void performTest() {
196                // Now compare all the faces in the first image
197                // with all the faces in the second image.
198                for (int ii = 0; ii < queryfaces.size(); ii++) {
199                        String face1id = queryId + ":" + ii;
200                        D f1f = queryfaces.get(ii);
201                        
202                        F f1fv = getFeature(face1id, f1f);
203                        // 
204                        // NOTE that the distance matrix will be symmetrical
205                        // so we only have to do half the comparisons.
206                        for (int jj = 0; jj < testfaces.size(); jj++) {
207                                double d = 0;
208                                String face2id = null;
209
210                                // If we're comparing the same face in the same image
211                                // we can assume the distance is zero. Saves doing a match.
212                                if (queryfaces == testfaces && ii == jj) {
213                                        d = 0;
214                                        face2id = face1id;
215                                } else {
216                                        // Compare the two feature vectors using the chosen
217                                        // distance metric.
218                                        D f2f = testfaces.get(jj);
219                                        face2id = testId + ":" + jj;
220                                        
221                                        // F f2fv = featureFactory.createFeature(f2f, false);
222                                        F f2fv = getFeature(face2id, f2f);
223
224                                        d = comparator.compare(f1fv, f2fv);
225                                }
226
227                                // Put the result in the result map
228                                Map<String, Double> mm = this.similarityMatrix.get(face1id);
229                                if (mm == null)
230                                        this.similarityMatrix.put(face1id, mm = new HashMap<String, Double>());
231                                mm.put(face2id, d);
232                        }
233                }
234        }
235
236        private F getFeature(String id, D face) {
237                F toRet = null;
238                
239                if (!cache) {
240                        toRet = extractor.extractFeature(face);
241                } else {
242                        String combinedID = String.format("%s:%b", id);
243                        toRet = this.featureCache.get(combinedID);
244                        
245                        if(toRet == null){
246                                toRet = extractor.extractFeature(face);
247                                this.featureCache.put(combinedID, toRet);
248                        }
249                }
250                return toRet;
251        }
252
253        /**
254         * @return The similarity dictionary structured as: {image0:face0 => {image0:face0 => DISTANCE,...},...,}
255         */
256        public Map<String, Map<String, Double>> getSimilarityDictionary() {
257                return this.similarityMatrix;
258        }
259        
260        /**
261         * Get the similarity matrix computed by {@link #performTest()}.
262         * @param invertIfRequired invert distances into similarities if required.
263         * @return the similarity matrix
264         */
265        public SimilarityMatrix getSimilarityMatrix(boolean invertIfRequired) {
266                Set<String> keys = this.similarityMatrix.keySet();
267                String[] indexArr = keys.toArray(new String[keys.size()]);
268                SimilarityMatrix simMatrix = new SimilarityMatrix(indexArr);
269                for (int i = 0; i < indexArr.length; i++) {
270                        String x = indexArr[i];
271                        for (int j = 0; j < indexArr.length; j++) {
272                                String y = indexArr[j];
273                                simMatrix.set(i, j, this.similarityMatrix.get(x).get(y));
274                        }
275                }
276                
277                if(this.comparator.isDistance() && invertIfRequired) {
278                        simMatrix.processInplace(new InvertData());
279                }
280                return simMatrix;
281        }
282
283        /**
284         * @return the bounding boxes of the detected faces 
285         */
286        public Map<String,Rectangle> getBoundingBoxes() {
287                return this.boundingBoxes;
288        }
289
290        /**
291         * Set whether detections should be cached
292         * @param cache enable cache if true
293         */
294        public void setCache(boolean cache) {
295                this.cache = cache;
296        }
297}