001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.image.processing.face.similarity; 031 032import java.util.HashMap; 033import java.util.LinkedHashMap; 034import java.util.List; 035import java.util.Map; 036import java.util.Set; 037 038import org.openimaj.image.Image; 039import org.openimaj.image.processing.face.detection.DetectedFace; 040import org.openimaj.image.processing.face.detection.FaceDetector; 041import org.openimaj.image.processing.face.feature.FacialFeature; 042import org.openimaj.image.processing.face.feature.FacialFeatureExtractor; 043import org.openimaj.image.processing.face.feature.comparison.FacialFeatureComparator; 044import org.openimaj.math.geometry.shape.Rectangle; 045import org.openimaj.math.matrix.similarity.SimilarityMatrix; 046import org.openimaj.math.matrix.similarity.processor.InvertData; 047 048/** 049 * The {@link FaceSimilarityEngine} allows computation of the similarity 050 * between faces in two images. 051 * 052 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk) 053 * 054 * @param <D> The type of {@link DetectedFace} 055 * @param <F> the type of {@link FacialFeature} 056 * @param <I> The type of {@link Image} 057 */ 058public class FaceSimilarityEngine<D extends DetectedFace, F extends FacialFeature, I extends Image<?, I>> { 059 private FaceDetector<D, I> detector; 060 private FacialFeatureExtractor<F, D> extractor; 061 private FacialFeatureComparator<F> comparator; 062 private Map<String, Rectangle> boundingBoxes; 063 private Map<String, F> featureCache; 064 private Map<String, List<D>> detectedFaceCache; 065 private LinkedHashMap<String, Map<String, Double>> similarityMatrix; 066 private List<D> queryfaces; 067 private List<D> testfaces; 068 private String queryId; 069 private String testId; 070 private boolean cache; 071 072 /** 073 * Construct a new {@link FaceSimilarityEngine} from the 074 * specified detector, extractor and comparator. 075 * 076 * @param detector The face detector 077 * @param extractor The feature extractor 078 * @param comparator The feature comparator 079 */ 080 public FaceSimilarityEngine(FaceDetector<D, I> detector, 081 FacialFeatureExtractor<F, D> extractor, 082 FacialFeatureComparator<F> comparator) { 083 this.detector = detector; 084 this.extractor = extractor; 085 this.comparator = comparator; 086 this.similarityMatrix = new LinkedHashMap<String, Map<String,Double>>(); 087 this.boundingBoxes = new HashMap<String, Rectangle>(); 088 featureCache = new HashMap<String,F>(); 089 detectedFaceCache = new HashMap<String,List<D>>(); 090 } 091 092 /** 093 * @return the detector 094 */ 095 public FaceDetector<D, I> detector() { 096 return detector; 097 } 098 099 /** 100 * @return the featureFactory 101 */ 102 public FacialFeatureExtractor<F, D> extractor() { 103 return extractor; 104 } 105 106 /** 107 * @return the comparator 108 */ 109 public FacialFeatureComparator<F> comparator() { 110 return comparator; 111 } 112 113 /** 114 * Create a new {@link FaceSimilarityEngine} from the 115 * specified detector, extractor and comparator. 116 * 117 * @param <D> The type of {@link DetectedFace} 118 * @param <F> the type of {@link FacialFeature} 119 * @param <I> The type of {@link Image} 120 * 121 * @param detector The face detector 122 * @param extractor The feature extractor 123 * @param comparator The feature comparator 124 * @return the new {@link FaceSimilarityEngine} 125 */ 126 public static <D extends DetectedFace, F extends FacialFeature, I extends Image<?, I>> 127 FaceSimilarityEngine<D, F, I> create( 128 FaceDetector<D, I> detector, 129 FacialFeatureExtractor<F, D> extractor, 130 FacialFeatureComparator<F> comparator) 131 { 132 return new FaceSimilarityEngine<D, F, I>(detector, extractor, 133 comparator); 134 } 135 136 /** 137 * Set the query image. 138 * @param queryImage the query image 139 * @param queryId the identifier of the query image 140 */ 141 public void setQuery(I queryImage, String queryId) { 142 this.queryfaces = getDetectedFaces(queryId,queryImage); 143 this.queryId = queryId; 144 updateBoundingBox(this.queryfaces, queryId); 145 } 146 147 private List<D> getDetectedFaces(String faceId, I faceImage) { 148 List<D> toRet = null; 149 if(!this.cache){ 150 toRet = this.detector.detectFaces(faceImage); 151 } 152 else{ 153 toRet = this.detectedFaceCache.get(faceId); 154 if(toRet == null){ 155// System.out.println("Redetected face: " + faceId); 156 toRet = this.detector.detectFaces(faceImage);; 157 this.detectedFaceCache.put(faceId, toRet); 158 } 159 } 160 return toRet; 161 } 162 163 private void updateBoundingBox(List<D> faces, String imageId) { 164 // We need to store the first one if we're running withFirst = true 165 if (boundingBoxes != null) 166 for (int ff = 0; ff < faces.size(); ff++) 167 if (boundingBoxes.get(imageId + ":" + ff) == null) 168 boundingBoxes.put(imageId + ":" + ff, faces.get(ff) 169 .getBounds()); 170 } 171 172 /** 173 * Set the image against which the query will be compared to next 174 * 175 * @param testImage 176 * @param testId 177 */ 178 public void setTest(I testImage, String testId) { 179 this.testId = testId; 180 this.testfaces = getDetectedFaces(testId,testImage); 181 updateBoundingBox(this.testfaces, testId); 182 } 183 184 /** 185 * Compare the query to itself for the next test 186 */ 187 public void setQueryTest() { 188 this.testfaces = this.queryfaces; 189 this.testId = this.queryId; 190 } 191 192 /** 193 * Compute the similarities between faces in the query and target 194 */ 195 public void performTest() { 196 // Now compare all the faces in the first image 197 // with all the faces in the second image. 198 for (int ii = 0; ii < queryfaces.size(); ii++) { 199 String face1id = queryId + ":" + ii; 200 D f1f = queryfaces.get(ii); 201 202 F f1fv = getFeature(face1id, f1f); 203 // 204 // NOTE that the distance matrix will be symmetrical 205 // so we only have to do half the comparisons. 206 for (int jj = 0; jj < testfaces.size(); jj++) { 207 double d = 0; 208 String face2id = null; 209 210 // If we're comparing the same face in the same image 211 // we can assume the distance is zero. Saves doing a match. 212 if (queryfaces == testfaces && ii == jj) { 213 d = 0; 214 face2id = face1id; 215 } else { 216 // Compare the two feature vectors using the chosen 217 // distance metric. 218 D f2f = testfaces.get(jj); 219 face2id = testId + ":" + jj; 220 221 // F f2fv = featureFactory.createFeature(f2f, false); 222 F f2fv = getFeature(face2id, f2f); 223 224 d = comparator.compare(f1fv, f2fv); 225 } 226 227 // Put the result in the result map 228 Map<String, Double> mm = this.similarityMatrix.get(face1id); 229 if (mm == null) 230 this.similarityMatrix.put(face1id, mm = new HashMap<String, Double>()); 231 mm.put(face2id, d); 232 } 233 } 234 } 235 236 private F getFeature(String id, D face) { 237 F toRet = null; 238 239 if (!cache) { 240 toRet = extractor.extractFeature(face); 241 } else { 242 String combinedID = String.format("%s:%b", id); 243 toRet = this.featureCache.get(combinedID); 244 245 if(toRet == null){ 246 toRet = extractor.extractFeature(face); 247 this.featureCache.put(combinedID, toRet); 248 } 249 } 250 return toRet; 251 } 252 253 /** 254 * @return The similarity dictionary structured as: {image0:face0 => {image0:face0 => DISTANCE,...},...,} 255 */ 256 public Map<String, Map<String, Double>> getSimilarityDictionary() { 257 return this.similarityMatrix; 258 } 259 260 /** 261 * Get the similarity matrix computed by {@link #performTest()}. 262 * @param invertIfRequired invert distances into similarities if required. 263 * @return the similarity matrix 264 */ 265 public SimilarityMatrix getSimilarityMatrix(boolean invertIfRequired) { 266 Set<String> keys = this.similarityMatrix.keySet(); 267 String[] indexArr = keys.toArray(new String[keys.size()]); 268 SimilarityMatrix simMatrix = new SimilarityMatrix(indexArr); 269 for (int i = 0; i < indexArr.length; i++) { 270 String x = indexArr[i]; 271 for (int j = 0; j < indexArr.length; j++) { 272 String y = indexArr[j]; 273 simMatrix.set(i, j, this.similarityMatrix.get(x).get(y)); 274 } 275 } 276 277 if(this.comparator.isDistance() && invertIfRequired) { 278 simMatrix.processInplace(new InvertData()); 279 } 280 return simMatrix; 281 } 282 283 /** 284 * @return the bounding boxes of the detected faces 285 */ 286 public Map<String,Rectangle> getBoundingBoxes() { 287 return this.boundingBoxes; 288 } 289 290 /** 291 * Set whether detections should be cached 292 * @param cache enable cache if true 293 */ 294 public void setCache(boolean cache) { 295 this.cache = cache; 296 } 297}