001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.image.processing.face.feature; 031 032import java.io.DataInput; 033import java.io.DataOutput; 034import java.io.IOException; 035import java.util.AbstractList; 036import java.util.ArrayList; 037import java.util.List; 038import java.util.Map; 039import java.util.Map.Entry; 040 041import org.openimaj.citation.annotation.Reference; 042import org.openimaj.citation.annotation.ReferenceType; 043import org.openimaj.data.dataset.GroupedDataset; 044import org.openimaj.data.dataset.ListDataset; 045import org.openimaj.feature.DoubleFV; 046import org.openimaj.feature.FeatureVectorProvider; 047import org.openimaj.image.FImage; 048import org.openimaj.image.model.FisherImages; 049import org.openimaj.image.processing.face.alignment.FaceAligner; 050import org.openimaj.image.processing.face.detection.DetectedFace; 051import org.openimaj.io.IOUtils; 052import org.openimaj.ml.training.BatchTrainer; 053import org.openimaj.util.pair.IndependentPair; 054 055/** 056 * A {@link FacialFeature} for FisherFaces. 057 * 058 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk) 059 */ 060@Reference( 061 type = ReferenceType.Article, 062 author = { "Belhumeur, Peter N.", "Hespanha, Jo\\~{a}o P.", "Kriegman, David J." }, 063 title = "Fisherfaces vs. Fisherfaces: Recognition Using Class Specific Linear Projection", 064 year = "1997", 065 journal = "IEEE Trans. Pattern Anal. Mach. Intell.", 066 pages = { "711", "", "720" }, 067 url = "http://dx.doi.org/10.1109/34.598228", 068 month = "July", 069 number = "7", 070 publisher = "IEEE Computer Society", 071 volume = "19", 072 customData = { 073 "issn", "0162-8828", 074 "numpages", "10", 075 "doi", "10.1109/34.598228", 076 "acmid", "261512", 077 "address", "Washington, DC, USA", 078 "keywords", 079 "Appearance-based vision, face recognition, illumination invariance, Fisher's linear discriminant." 080 }) 081public class FisherFaceFeature implements FacialFeature, FeatureVectorProvider<DoubleFV> { 082 /** 083 * A {@link FacialFeatureExtractor} for producing FisherFaces. Unlike most 084 * {@link FacialFeatureExtractor}s, this one either needs to be trained or 085 * provided with a pre-trained {@link FisherImages} object. 086 * <p> 087 * A {@link FaceAligner} can be used to produce aligned faces for training 088 * and feature extraction. 089 * 090 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk) 091 * 092 * @param <T> 093 * 094 */ 095 public static class Extractor<T extends DetectedFace> 096 implements 097 FacialFeatureExtractor<FisherFaceFeature, T>, 098 BatchTrainer<IndependentPair<?, T>> 099 { 100 FisherImages fisher = null; 101 FaceAligner<T> aligner = null; 102 103 /** 104 * Construct with the requested number of components (the number of PCs 105 * to keep) and a face aligner. The principal components must be learned 106 * by calling {@link #train(List)}. 107 * 108 * @param numComponents 109 * the number of principal components to keep. 110 * @param aligner 111 * the face aligner 112 */ 113 public Extractor(int numComponents, FaceAligner<T> aligner) { 114 this(new FisherImages(numComponents), aligner); 115 } 116 117 /** 118 * Construct with given pre-trained {@link FisherImages} basis and a 119 * face aligner. 120 * 121 * @param basis 122 * the pre-trained basis 123 * @param aligner 124 * the face aligner 125 */ 126 public Extractor(FisherImages basis, FaceAligner<T> aligner) { 127 this.fisher = basis; 128 this.aligner = aligner; 129 } 130 131 @Override 132 public FisherFaceFeature extractFeature(T face) { 133 final FImage patch = aligner.align(face); 134 135 final DoubleFV fv = fisher.extractFeature(patch); 136 137 return new FisherFaceFeature(fv); 138 } 139 140 @Override 141 public void readBinary(DataInput in) throws IOException { 142 fisher.readBinary(in); 143 144 final String alignerClass = in.readUTF(); 145 aligner = IOUtils.newInstance(alignerClass); 146 aligner.readBinary(in); 147 } 148 149 @Override 150 public byte[] binaryHeader() { 151 return this.getClass().getName().getBytes(); 152 } 153 154 @Override 155 public void writeBinary(DataOutput out) throws IOException { 156 fisher.writeBinary(out); 157 158 out.writeUTF(aligner.getClass().getName()); 159 aligner.writeBinary(out); 160 } 161 162 @Override 163 public void train(final List<? extends IndependentPair<?, T>> data) { 164 final List<IndependentPair<?, FImage>> patches = new AbstractList<IndependentPair<?, FImage>>() { 165 166 @Override 167 public IndependentPair<?, FImage> get(int index) { 168 return IndependentPair.pair(data.get(index).firstObject(), 169 aligner.align(data.get(index).secondObject())); 170 } 171 172 @Override 173 public int size() { 174 return data.size(); 175 } 176 177 }; 178 179 fisher.train(patches); 180 } 181 182 /** 183 * Train on a map of data. 184 * 185 * @param data 186 * the data 187 */ 188 public void train(Map<?, ? extends List<T>> data) { 189 final List<IndependentPair<?, FImage>> list = new ArrayList<IndependentPair<?, FImage>>(); 190 191 for (final Entry<?, ? extends List<T>> e : data.entrySet()) { 192 for (final T i : e.getValue()) { 193 list.add(IndependentPair.pair(e.getKey(), aligner.align(i))); 194 } 195 } 196 197 fisher.train(list); 198 } 199 200 /** 201 * Train on a grouped dataset. 202 * 203 * @param <KEY> 204 * The group type 205 * @param data 206 * the data 207 */ 208 public <KEY> void train(GroupedDataset<KEY, ? extends ListDataset<T>, T> data) { 209 final List<IndependentPair<?, FImage>> list = new ArrayList<IndependentPair<?, FImage>>(); 210 211 for (final KEY e : data.getGroups()) { 212 for (final T i : data.getInstances(e)) { 213 if (i != null) 214 list.add(IndependentPair.pair(e, aligner.align(i))); 215 } 216 } 217 218 fisher.train(list); 219 } 220 } 221 222 private DoubleFV fv; 223 224 protected FisherFaceFeature() { 225 this(null); 226 } 227 228 /** 229 * Construct the FisherFaceFeature with the given feature vector. 230 * 231 * @param fv 232 * the feature vector 233 */ 234 public FisherFaceFeature(DoubleFV fv) { 235 this.fv = fv; 236 } 237 238 @Override 239 public void readBinary(DataInput in) throws IOException { 240 fv = new DoubleFV(); 241 fv.readBinary(in); 242 } 243 244 @Override 245 public byte[] binaryHeader() { 246 return getClass().getName().getBytes(); 247 } 248 249 @Override 250 public void writeBinary(DataOutput out) throws IOException { 251 fv.writeBinary(out); 252 } 253 254 @Override 255 public DoubleFV getFeatureVector() { 256 return fv; 257 } 258}