001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030/** 031 * 032 */ 033package org.openimaj.tools.faces; 034 035import java.io.File; 036import java.io.IOException; 037import java.util.ArrayList; 038import java.util.List; 039import java.util.Map; 040 041import org.kohsuke.args4j.CmdLineException; 042import org.kohsuke.args4j.CmdLineParser; 043import org.openimaj.image.FImage; 044import org.openimaj.image.ImageUtilities; 045import org.openimaj.image.processing.face.similarity.FaceSimilarityEngine; 046import org.openimaj.io.IOUtils; 047import org.openimaj.math.geometry.shape.Rectangle; 048import org.openimaj.math.matrix.similarity.SimilarityMatrix; 049 050/** 051 * Face similarity tool compares the faces in all given images and gives a score 052 * for each comparison. The tool can be made to match only the first image 053 * against all other images or comparing all images against all others. 054 * <p> 055 * This tool can be used both from the command-line and programmatically. 056 * Programmatically, there are some convenience functions for comparing 057 * {@link List}s of {@link File}s and lists of {@link FImage}s, however, you are 058 * welcome to give the 059 * {@link #getDistances(List, boolean, ImageGetter, FaceSimilarityEngine)} 060 * method any {@link List} as long as you supply an {@link ImageGetter} that can 061 * return {@link FImage}s from that list. 062 * 063 * @author David Dupplaw (dpd@ecs.soton.ac.uk) 064 * 065 * @created 30 May 2011 066 */ 067public class FaceSimilarityTool { 068 /** 069 * An interface to get images and names from a list of things. This allows 070 * us to pass in a list of stuff and compare against them without having to 071 * assume what the stuff is until later. 072 * 073 * @author David Dupplaw (dpd@ecs.soton.ac.uk) 074 * 075 * @created 3 Jun 2011 076 * @param <T> 077 */ 078 public interface ImageGetter<T> { 079 /** 080 * Get the image associated with the item 081 * @param list list of items 082 * @param index index of item we're interested in 083 * @return the image 084 */ 085 public FImage getImage(List<T> list, int index); 086 087 /** 088 * Get the name of the item 089 * @param list list of items 090 * @param index index of item we're interested in 091 * @return the name 092 */ 093 public String getName(List<T> list, int index); 094 } 095 096 /** 097 * Calculates the distance between all the faces in the first image with all 098 * the faces in the given images. 099 * 100 * Faces are identified by their image file and the index into the file, so 101 * if an image does not contain any faces, it will return null from the map 102 * for that filename. 103 * 104 * @param first The query image 105 * @param others The list of files to compare against 106 * @param strategy The strategy 107 * @return A Map giving the distance of every face with every other. 108 */ 109 public Map<String, Map<String, Double>> getDistances(File first, 110 List<File> others, FaceSimilarityEngine<?, ?, FImage> strategy) { 111 List<File> x = new ArrayList<File>(); 112 x.add(first); 113 x.addAll(others); 114 115 return this.getDistances(x, true, strategy); 116 } 117 118 /** 119 * Calculates the distance between faces in the given images. Faces are 120 * identified by their image file and the index into the file, so if an 121 * image does not contain any faces, it will return null from the map for 122 * that filename. 123 * 124 * @param inputFiles The list of files to process 125 * @param strategy the strategy 126 * @return A Map giving the distance of every face with every other. 127 */ 128 public Map<String, Map<String, Double>> getDistances(List<File> inputFiles, 129 FaceSimilarityEngine<?, ?, FImage> strategy) { 130 return getDistances(inputFiles, false, strategy); 131 } 132 133 /** 134 * Calculates the distance between faces in the given images. Faces are 135 * identified by their image file and the index into the file, so if an 136 * image does not contain any faces, it will return null from the map for 137 * that filename. 138 * 139 * @param inputFiles 140 * The list of files to process 141 * @param withFirst 142 * if TRUE, the first image in the list will be matched against 143 * all others, otherwise all images are matches against each 144 * other. 145 * @param strategy The strategy 146 * @return A Map giving the distance of every face with every other. 147 */ 148 public Map<String, Map<String, Double>> getDistances(List<File> inputFiles, 149 boolean withFirst, 150 FaceSimilarityEngine<?, ?, FImage> strategy) { 151 return getDistances(inputFiles, withFirst, 152 new ImageGetter<File>() { 153 @Override 154 public FImage getImage(List<File> list, int index) { 155 try { 156// System.out.println("Reading: " + list.get(index)); 157 FImage fi = ImageUtilities.readF(list.get(index)); 158 return fi; 159 } catch (IOException e) { 160 e.printStackTrace(); 161 return null; 162 } 163 } 164 165 @Override 166 public String getName(List<File> list, int index) { 167 return list.get(index).getName(); 168 } 169 170 }, strategy); 171 } 172 173 /** 174 * Calculates the distance between faces in the given images. Faces are 175 * identified by their image file and the index into the file, so if an 176 * image does not contain any faces, it will return null from the map for 177 * that filename. 178 * 179 * @param imageIdentifiers 180 * A list of image names 181 * @param inputImages 182 * The list of images to process 183 * @param withFirst 184 * if TRUE, the first image in the list will be matched against 185 * all others, otherwise all images are matches against each 186 * other. 187 * @param strategy The strategy. 188 * @return A Map giving the distance of every face with every other. 189 */ 190 public Map<String, Map<String, Double>> getDistances( 191 List<String> imageIdentifiers, List<FImage> inputImages, 192 boolean withFirst, 193 FaceSimilarityEngine<?, ?, FImage> strategy) { 194 return getDistances(inputImages, withFirst, 195 new ImageGetter<FImage>() { 196 @Override 197 public FImage getImage(List<FImage> list, int index) { 198 return list.get(index); 199 } 200 201 @Override 202 public String getName(List<FImage> list, int index) { 203 return "image" + index; 204 } 205 }, strategy); 206 } 207 208 /** 209 * This is the actual comparison function that performs the nested loops as 210 * necessary to match all the faces against each other. 211 * 212 * @param <T> 213 * The type of thing in the input list 214 * @param inputList 215 * A list of things to process 216 * @param withFirst 217 * Whether to compare the first against all others (TRUE) or 218 * compare all against each other (FALSE) 219 * @param iGetter 220 * The getter that can make FImages from the input list. 221 * @param strategy The strategy 222 * @return A Map giving the distance of every face with every other. 223 */ 224 public <T> Map<String, Map<String, Double>> getDistances(List<T> inputList, 225 boolean withFirst, 226 ImageGetter<T> iGetter, 227 FaceSimilarityEngine<?, ?, FImage> strategy) { 228 229 // If we're only comparing the images against the first one, 230 // the outer loop only needs to be perfomed once. 231 int xx = 0; 232 if (withFirst) 233 xx = 1; 234 else 235 xx = inputList.size(); 236 237 for (int i = 0; i < xx; i++) { 238 // Read the first image and extract the faces. 239 FImage f1 = iGetter.getImage(inputList, i); 240 String f1id = iGetter.getName(inputList, i); 241 242 strategy.setQuery(f1, f1id); 243 244 // Now loop through all the other images. 245 for (int j = 0; j < inputList.size(); j++) { 246 // If the two images we're comparing are the same one, 247 // we can avoid doing an extra extraction here. 248 if (i != j) { 249 // Read the other image and imageid. 250 FImage f2 = iGetter.getImage( inputList, j); 251 String f2id = iGetter.getName(inputList,j); 252 strategy.setTest(f2, f2id); 253 } else { 254 strategy.setQueryTest(); 255 } 256 257 strategy.performTest(); 258 } 259 } 260 261 return strategy.getSimilarityDictionary(); 262 } 263 264 // 265 // /** 266 // * Compares one set of facial features against another. 267 // * Side-affects (and returns) the input results map that maps 268 // * face to other face and score. 269 // * 270 // * @param m The results map to populate 271 // * @param file1id The identifier of the first image 272 // * @param file2id The identifier of the second image 273 // * @param f1faces The faces in the first image 274 // * @param f2faces The faces in the second image 275 // * @return The input parameter <code>m</code> 276 // */ 277 // public Map<String,Map<String,Double>> compareFaces( 278 // Map<String,Map<String,Double>> m, 279 // String file1id, String file2id, 280 // List<KEDetectedFace> f1faces, 281 // List<KEDetectedFace> f2faces, 282 // FloatFVComparison comparisonFunction ) 283 // { 284 // // Now compare all the faces in the first image 285 // // with all the faces in the second image. 286 // for( int ii = 0; ii < f1faces.size(); ii++ ) 287 // { 288 // String face1id = file1id+":"+ii; 289 // KEDetectedFace f1f = f1faces.get(ii); 290 // 291 // // NOTE that the distance matrix will be symmetrical 292 // // so we only have to do half the comparisons. 293 // for( int jj = 0; jj < f2faces.size(); jj++ ) 294 // { 295 // double d = 0; 296 // String face2id = null; 297 // 298 // // If we're comparing the same face in the same image 299 // // we can assume the distance is zero. Saves doing a match. 300 // if( f1faces == f2faces && ii == jj ) 301 // { 302 // d = 0; 303 // face2id = face1id; 304 // } 305 // else 306 // { 307 // // Compare the two feature vectors using the chosen 308 // // distance metric. 309 // KEDetectedFace f2f = f2faces.get(jj); 310 // face2id = file2id+":"+jj; 311 // 312 // //TODO: other types of feature 313 // FaceFVComparator<FacePatchFeature> comparator = new 314 // FaceFVComparator<FacePatchFeature>(comparisonFunction); 315 // FacePatchFeature.Factory factory = new FacePatchFeature.Factory(); 316 // FacePatchFeature f1fv = factory.createFeature(f1f, false); 317 // FacePatchFeature f2fv = factory.createFeature(f2f, false); 318 // 319 // d = comparator.compare( f1fv, f2fv ); 320 // } 321 // 322 // // Put the result in the result map 323 // Map<String,Double> mm = m.get( face1id ); 324 // if( mm == null ) 325 // m.put( face1id, 326 // mm = new HashMap<String,Double>() ); 327 // mm.put( face2id, d ); 328 // } 329 // } 330 // 331 // return m; 332 // } 333 334 /** 335 * Parses the command line arguments. 336 * 337 * @param args 338 * The arguments to parse 339 * @return The tool options class 340 */ 341 private static FaceSimilarityToolOptions parseArgs(String[] args) { 342 FaceSimilarityToolOptions fdto = new FaceSimilarityToolOptions(); 343 CmdLineParser parser = new CmdLineParser(fdto); 344 345 try { 346 parser.parseArgument(args); 347 } catch (CmdLineException e) { 348 System.err.println(e.getMessage()); 349 System.err 350 .println("java FaceSimilarityTool [options...] IMAGE-FILES"); 351 parser.printUsage(System.err); 352 return null; 353 } 354 355 return fdto; 356 } 357 358 /** 359 * 360 * @param args 361 */ 362 public static void main(String[] args) { 363 FaceSimilarityToolOptions o = parseArgs(args); 364 if(o == null) return; 365 366// Map<String, Rectangle> bb = new HashMap<String, Rectangle>(); 367 FaceSimilarityEngine<?, ?, FImage> strat = o.strategy.strategy(); 368 strat.setCache(o.cache); 369 new FaceSimilarityTool().getDistances(o.inputFiles, o.withFirst, strat); 370 // System.out.println( "Map:" +m); 371 372 if (o.boundingBoxes) { 373 Map<String, Rectangle> bb = strat.getBoundingBoxes(); 374 for (String k : bb.keySet()) { 375 Rectangle r = bb.get(k); 376 System.out.println(k + ":" + r.x + "," + r.y + "," + r.width 377 + "," + r.height); 378 } 379 } 380 SimilarityMatrix similarityMatrix = strat.getSimilarityMatrix(o.invertIfRequired); 381 if(o.output!=null){ 382 try { 383 IOUtils.writeBinary(o.output, similarityMatrix); 384 } catch (IOException e) { 385 System.err.println("Couldn't output file: " + e.getMessage()); 386 } 387 } 388 389 System.out.println(similarityMatrix); 390 391// // Pretty print the matrix 392// Set<String> xx = null; 393// if (o.withFirst) 394// xx = m.get(m.keySet().iterator().next()).keySet(); 395// else 396// xx = m.keySet(); 397// 398// int maxLen = 0; 399// for (String f : xx) 400// if (f.length() > maxLen) 401// maxLen = f.length(); 402// 403// System.out.print(new PrintfFormat("%+" + maxLen + "s").sprintf("")); 404// for (String f : m.keySet()) 405// System.out.print(new PrintfFormat("%8s").sprintf(f.substring(f 406// .lastIndexOf(":")))); 407// System.out.println(); 408// 409// for (String f : xx) { 410// String s = f; 411// if (f.length() < maxLen) 412// s = new PrintfFormat("%+" + maxLen + "s").sprintf(f); 413// System.out.print(s + ":"); 414// 415// Set<String> zz = m.keySet(); 416// 417// // Iterate over the outer map so that we get the same ordering 418// // of the inner map 419// boolean first = true; 420// for (String ff : zz) { 421// Double d = null; 422// if (m.get(f) != null) 423// d = m.get(f).get(ff); 424// 425// if (d == null) 426// if (m.get(ff) != null) 427// d = m.get(ff).get(f); 428// 429// if (d == null) 430// System.out.print(" "); 431// else { 432// if (!first) 433// System.out.print(","); 434// System.out.print(new PrintfFormat("%7.2f").sprintf(d)); 435// first = false; 436// } 437// } 438// 439// System.out.println(); 440// } 441 } 442}