001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030/**
031 * 
032 */
033package org.openimaj.tools.faces;
034
035import java.io.File;
036import java.io.IOException;
037import java.util.ArrayList;
038import java.util.List;
039import java.util.Map;
040
041import org.kohsuke.args4j.CmdLineException;
042import org.kohsuke.args4j.CmdLineParser;
043import org.openimaj.image.FImage;
044import org.openimaj.image.ImageUtilities;
045import org.openimaj.image.processing.face.similarity.FaceSimilarityEngine;
046import org.openimaj.io.IOUtils;
047import org.openimaj.math.geometry.shape.Rectangle;
048import org.openimaj.math.matrix.similarity.SimilarityMatrix;
049
050/**
051 * Face similarity tool compares the faces in all given images and gives a score
052 * for each comparison. The tool can be made to match only the first image
053 * against all other images or comparing all images against all others.
054 * <p>
055 * This tool can be used both from the command-line and programmatically.
056 * Programmatically, there are some convenience functions for comparing
057 * {@link List}s of {@link File}s and lists of {@link FImage}s, however, you are
058 * welcome to give the
059 * {@link #getDistances(List, boolean, ImageGetter, FaceSimilarityEngine)}
060 * method any {@link List} as long as you supply an {@link ImageGetter} that can
061 * return {@link FImage}s from that list.
062 * 
063 * @author David Dupplaw (dpd@ecs.soton.ac.uk)
064 * 
065 * @created 30 May 2011
066 */
067public class FaceSimilarityTool {
068        /**
069         * An interface to get images and names from a list of things. This allows
070         * us to pass in a list of stuff and compare against them without having to
071         * assume what the stuff is until later.
072         * 
073         * @author David Dupplaw (dpd@ecs.soton.ac.uk)
074         * 
075         * @created 3 Jun 2011
076         * @param <T>
077         */
078        public interface ImageGetter<T> {
079                /**
080                 * Get the image associated with the item
081                 * @param list list of items
082                 * @param index index of item we're interested in
083                 * @return the image
084                 */
085                public FImage getImage(List<T> list, int index);
086
087                /**
088                 * Get the name of the item
089                 * @param list list of items
090                 * @param index index of item we're interested in
091                 * @return the name
092                 */
093                public String getName(List<T> list, int index);
094        }
095
096        /**
097         * Calculates the distance between all the faces in the first image with all
098         * the faces in the given images.
099         * 
100         * Faces are identified by their image file and the index into the file, so
101         * if an image does not contain any faces, it will return null from the map
102         * for that filename.
103         * 
104         * @param first The query image
105         * @param others The list of files to compare against
106         * @param strategy The strategy
107         * @return A Map giving the distance of every face with every other.
108         */
109        public Map<String, Map<String, Double>> getDistances(File first,
110                        List<File> others, FaceSimilarityEngine<?, ?, FImage> strategy) {
111                List<File> x = new ArrayList<File>();
112                x.add(first);
113                x.addAll(others);
114
115                return this.getDistances(x, true, strategy);
116        }
117
118        /**
119         * Calculates the distance between faces in the given images. Faces are
120         * identified by their image file and the index into the file, so if an
121         * image does not contain any faces, it will return null from the map for
122         * that filename.
123         * 
124         * @param inputFiles The list of files to process
125         * @param strategy the strategy
126         * @return A Map giving the distance of every face with every other.
127         */
128        public Map<String, Map<String, Double>> getDistances(List<File> inputFiles,
129                        FaceSimilarityEngine<?, ?, FImage> strategy) {
130                return getDistances(inputFiles, false, strategy);
131        }
132
133        /**
134         * Calculates the distance between faces in the given images. Faces are
135         * identified by their image file and the index into the file, so if an
136         * image does not contain any faces, it will return null from the map for
137         * that filename.
138         * 
139         * @param inputFiles
140         *            The list of files to process
141         * @param withFirst
142         *            if TRUE, the first image in the list will be matched against
143         *            all others, otherwise all images are matches against each
144         *            other.
145         * @param strategy The strategy
146         * @return A Map giving the distance of every face with every other.
147         */
148        public Map<String, Map<String, Double>> getDistances(List<File> inputFiles,
149                        boolean withFirst,
150                        FaceSimilarityEngine<?, ?, FImage> strategy) {
151                return getDistances(inputFiles, withFirst, 
152                                new ImageGetter<File>() {
153                                        @Override
154                                        public FImage getImage(List<File> list, int index) {
155                                                try {
156//                                                      System.out.println("Reading: " + list.get(index));
157                                                        FImage fi = ImageUtilities.readF(list.get(index));
158                                                        return fi;
159                                                } catch (IOException e) {
160                                                        e.printStackTrace();
161                                                        return null;
162                                                }
163                                        }
164
165                                        @Override
166                                        public String getName(List<File> list, int index) {
167                                                return list.get(index).getName();
168                                        }
169
170                                }, strategy);
171        }
172
173        /**
174         * Calculates the distance between faces in the given images. Faces are
175         * identified by their image file and the index into the file, so if an
176         * image does not contain any faces, it will return null from the map for
177         * that filename.
178         * 
179         * @param imageIdentifiers
180         *            A list of image names
181         * @param inputImages
182         *            The list of images to process
183         * @param withFirst
184         *            if TRUE, the first image in the list will be matched against
185         *            all others, otherwise all images are matches against each
186         *            other.
187         * @param strategy The strategy.
188         * @return A Map giving the distance of every face with every other.
189         */
190        public Map<String, Map<String, Double>> getDistances(
191                        List<String> imageIdentifiers, List<FImage> inputImages,
192                        boolean withFirst, 
193                        FaceSimilarityEngine<?, ?, FImage> strategy) {
194                return getDistances(inputImages, withFirst, 
195                                new ImageGetter<FImage>() {
196                                        @Override
197                                        public FImage getImage(List<FImage> list, int index) {
198                                                return list.get(index);
199                                        }
200
201                                        @Override
202                                        public String getName(List<FImage> list, int index) {
203                                                return "image" + index;
204                                        }
205                                }, strategy);
206        }
207
208        /**
209         * This is the actual comparison function that performs the nested loops as
210         * necessary to match all the faces against each other.
211         * 
212         * @param <T>
213         *            The type of thing in the input list
214         * @param inputList
215         *            A list of things to process
216         * @param withFirst
217         *            Whether to compare the first against all others (TRUE) or
218         *            compare all against each other (FALSE)
219         * @param iGetter
220         *            The getter that can make FImages from the input list.
221         * @param strategy The strategy
222         * @return A Map giving the distance of every face with every other.
223         */
224        public <T> Map<String, Map<String, Double>> getDistances(List<T> inputList,
225                        boolean withFirst, 
226                        ImageGetter<T> iGetter,
227                        FaceSimilarityEngine<?, ?, FImage> strategy) {
228
229                // If we're only comparing the images against the first one,
230                // the outer loop only needs to be perfomed once.
231                int xx = 0;
232                if (withFirst)
233                        xx = 1;
234                else
235                        xx = inputList.size();
236
237                for (int i = 0; i < xx; i++) {
238                        // Read the first image and extract the faces.
239                        FImage f1 = iGetter.getImage(inputList, i);
240                        String f1id = iGetter.getName(inputList, i);
241
242                        strategy.setQuery(f1, f1id);
243
244                        // Now loop through all the other images.
245                        for (int j = 0; j < inputList.size(); j++) {
246                                // If the two images we're comparing are the same one,
247                                // we can avoid doing an extra extraction here.
248                                if (i != j) {
249                                        // Read the other image and imageid.
250                        FImage f2 = iGetter.getImage( inputList, j);
251                        String f2id = iGetter.getName(inputList,j);
252                                        strategy.setTest(f2, f2id);
253                                } else {
254                                        strategy.setQueryTest();
255                                }
256
257                                strategy.performTest();
258                        }
259                }
260
261                return strategy.getSimilarityDictionary();
262        }
263
264        //
265        // /**
266        // * Compares one set of facial features against another.
267        // * Side-affects (and returns) the input results map that maps
268        // * face to other face and score.
269        // *
270        // * @param m The results map to populate
271        // * @param file1id The identifier of the first image
272        // * @param file2id The identifier of the second image
273        // * @param f1faces The faces in the first image
274        // * @param f2faces The faces in the second image
275        // * @return The input parameter <code>m</code>
276        // */
277        // public Map<String,Map<String,Double>> compareFaces(
278        // Map<String,Map<String,Double>> m,
279        // String file1id, String file2id,
280        // List<KEDetectedFace> f1faces,
281        // List<KEDetectedFace> f2faces,
282        // FloatFVComparison comparisonFunction )
283        // {
284        // // Now compare all the faces in the first image
285        // // with all the faces in the second image.
286        // for( int ii = 0; ii < f1faces.size(); ii++ )
287        // {
288        // String face1id = file1id+":"+ii;
289        // KEDetectedFace f1f = f1faces.get(ii);
290        //
291        // // NOTE that the distance matrix will be symmetrical
292        // // so we only have to do half the comparisons.
293        // for( int jj = 0; jj < f2faces.size(); jj++ )
294        // {
295        // double d = 0;
296        // String face2id = null;
297        //
298        // // If we're comparing the same face in the same image
299        // // we can assume the distance is zero. Saves doing a match.
300        // if( f1faces == f2faces && ii == jj )
301        // {
302        // d = 0;
303        // face2id = face1id;
304        // }
305        // else
306        // {
307        // // Compare the two feature vectors using the chosen
308        // // distance metric.
309        // KEDetectedFace f2f = f2faces.get(jj);
310        // face2id = file2id+":"+jj;
311        //
312        // //TODO: other types of feature
313        // FaceFVComparator<FacePatchFeature> comparator = new
314        // FaceFVComparator<FacePatchFeature>(comparisonFunction);
315        // FacePatchFeature.Factory factory = new FacePatchFeature.Factory();
316        // FacePatchFeature f1fv = factory.createFeature(f1f, false);
317        // FacePatchFeature f2fv = factory.createFeature(f2f, false);
318        //
319        // d = comparator.compare( f1fv, f2fv );
320        // }
321        //
322        // // Put the result in the result map
323        // Map<String,Double> mm = m.get( face1id );
324        // if( mm == null )
325        // m.put( face1id,
326        // mm = new HashMap<String,Double>() );
327        // mm.put( face2id, d );
328        // }
329        // }
330        //
331        // return m;
332        // }
333
334        /**
335         * Parses the command line arguments.
336         * 
337         * @param args
338         *            The arguments to parse
339         * @return The tool options class
340         */
341        private static FaceSimilarityToolOptions parseArgs(String[] args) {
342                FaceSimilarityToolOptions fdto = new FaceSimilarityToolOptions();
343                CmdLineParser parser = new CmdLineParser(fdto);
344
345                try {
346                        parser.parseArgument(args);
347                } catch (CmdLineException e) {
348                        System.err.println(e.getMessage());
349                        System.err
350                                        .println("java FaceSimilarityTool [options...] IMAGE-FILES");
351                        parser.printUsage(System.err);
352                        return null;
353                }
354
355                return fdto;
356        }
357
358        /**
359         * 
360         * @param args
361         */
362        public static void main(String[] args) {
363                FaceSimilarityToolOptions o = parseArgs(args);
364                if(o == null) return;
365
366//              Map<String, Rectangle> bb = new HashMap<String, Rectangle>();
367                FaceSimilarityEngine<?, ?, FImage> strat = o.strategy.strategy();
368                strat.setCache(o.cache);
369                new FaceSimilarityTool().getDistances(o.inputFiles, o.withFirst, strat);
370                // System.out.println( "Map:" +m);
371
372                if (o.boundingBoxes) {
373                        Map<String, Rectangle> bb = strat.getBoundingBoxes();
374                        for (String k : bb.keySet()) {
375                                Rectangle r = bb.get(k);
376                                System.out.println(k + ":" + r.x + "," + r.y + "," + r.width
377                                                + "," + r.height);
378                        }
379                }
380                SimilarityMatrix similarityMatrix = strat.getSimilarityMatrix(o.invertIfRequired);
381                if(o.output!=null){
382                        try {
383                                IOUtils.writeBinary(o.output, similarityMatrix);
384                        } catch (IOException e) {
385                                System.err.println("Couldn't output file: " + e.getMessage());
386                        }
387                }
388                
389                System.out.println(similarityMatrix);
390                
391//              // Pretty print the matrix
392//              Set<String> xx = null;
393//              if (o.withFirst)
394//                      xx = m.get(m.keySet().iterator().next()).keySet();
395//              else
396//                      xx = m.keySet();
397//
398//              int maxLen = 0;
399//              for (String f : xx)
400//                      if (f.length() > maxLen)
401//                              maxLen = f.length();
402//
403//              System.out.print(new PrintfFormat("%+" + maxLen + "s").sprintf(""));
404//              for (String f : m.keySet())
405//                      System.out.print(new PrintfFormat("%8s").sprintf(f.substring(f
406//                                      .lastIndexOf(":"))));
407//              System.out.println();
408//
409//              for (String f : xx) {
410//                      String s = f;
411//                      if (f.length() < maxLen)
412//                              s = new PrintfFormat("%+" + maxLen + "s").sprintf(f);
413//                      System.out.print(s + ":");
414//
415//                      Set<String> zz = m.keySet();
416//
417//                      // Iterate over the outer map so that we get the same ordering
418//                      // of the inner map
419//                      boolean first = true;
420//                      for (String ff : zz) {
421//                              Double d = null;
422//                              if (m.get(f) != null)
423//                                      d = m.get(f).get(ff);
424//
425//                              if (d == null)
426//                                      if (m.get(ff) != null)
427//                                              d = m.get(ff).get(f);
428//
429//                              if (d == null)
430//                                      System.out.print("        ");
431//                              else {
432//                                      if (!first)
433//                                              System.out.print(",");
434//                                      System.out.print(new PrintfFormat("%7.2f").sprintf(d));
435//                                      first = false;
436//                              }
437//                      }
438//
439//                      System.out.println();
440//              }
441        }
442}