001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.image.processing.face.feature; 031 032import java.io.DataInput; 033import java.io.DataOutput; 034import java.io.IOException; 035 036import org.openimaj.feature.FeatureVectorProvider; 037import org.openimaj.feature.FloatFV; 038import org.openimaj.image.FImage; 039import org.openimaj.image.feature.dense.binarypattern.ExtendedLocalBinaryPattern; 040import org.openimaj.image.feature.dense.binarypattern.UniformBinaryPattern; 041import org.openimaj.image.processing.face.alignment.FaceAligner; 042import org.openimaj.image.processing.face.alignment.IdentityAligner; 043import org.openimaj.image.processing.face.detection.DetectedFace; 044import org.openimaj.io.IOUtils; 045 046/** 047 * A {@link FacialFeature} built from decomposing the face image into 048 * (non-overlapping) blocks and building histograms of the 049 * {@link ExtendedLocalBinaryPattern}s for each block and then concatenating to 050 * form the final feature. 051 * 052 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk) 053 */ 054public class LocalLBPHistogram implements FacialFeature, FeatureVectorProvider<FloatFV> { 055 /** 056 * A {@link FacialFeatureExtractor} for building {@link LocalLBPHistogram}s. 057 * 058 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk) 059 * 060 * @param <T> 061 * Type of {@link DetectedFace}. 062 */ 063 public static class Extractor<T extends DetectedFace> implements FacialFeatureExtractor<LocalLBPHistogram, T> { 064 FaceAligner<T> aligner; 065 int blocksX = 25; 066 int blocksY = 25; 067 int samples = 8; 068 int radius = 1; 069 070 /** 071 * Construct with a {@link IdentityAligner} 072 */ 073 public Extractor() { 074 this.aligner = new IdentityAligner<T>(); 075 } 076 077 /** 078 * Construct with the given aligner. 079 * 080 * @param aligner 081 * the aligner 082 */ 083 public Extractor(FaceAligner<T> aligner) { 084 this.aligner = aligner; 085 } 086 087 /** 088 * Construct with the given aligner, parameters describing how the image 089 * is broken into blocks and parameters describing the radius of the LBP 090 * extraction circle, and how many samples are made. 091 * 092 * @param aligner 093 * The face aligner 094 * @param blocksX 095 * The number of blocks in the x-direction 096 * @param blocksY 097 * The number of blocks in the y-direction 098 * @param samples 099 * The number of samples around the circle for the 100 * {@link ExtendedLocalBinaryPattern} 101 * @param radius 102 * the radius used for the {@link ExtendedLocalBinaryPattern} 103 * . 104 */ 105 public Extractor(FaceAligner<T> aligner, int blocksX, int blocksY, int samples, int radius) { 106 this.aligner = aligner; 107 this.blocksX = blocksX; 108 this.blocksY = blocksY; 109 this.samples = samples; 110 this.radius = radius; 111 } 112 113 @Override 114 public LocalLBPHistogram extractFeature(T detectedFace) { 115 final LocalLBPHistogram f = new LocalLBPHistogram(); 116 117 final FImage face = aligner.align(detectedFace); 118 final FImage mask = aligner.getMask(); 119 120 f.initialise(face, mask, blocksX, blocksY, samples, radius); 121 122 return f; 123 } 124 125 @Override 126 public void readBinary(DataInput in) throws IOException { 127 final String alignerClass = in.readUTF(); 128 aligner = IOUtils.newInstance(alignerClass); 129 aligner.readBinary(in); 130 131 blocksX = in.readInt(); 132 blocksY = in.readInt(); 133 radius = in.readInt(); 134 samples = in.readInt(); 135 } 136 137 @Override 138 public byte[] binaryHeader() { 139 return this.getClass().getName().getBytes(); 140 } 141 142 @Override 143 public void writeBinary(DataOutput out) throws IOException { 144 out.writeUTF(aligner.getClass().getName()); 145 aligner.writeBinary(out); 146 147 out.writeInt(blocksX); 148 out.writeInt(blocksY); 149 out.writeInt(radius); 150 out.writeInt(samples); 151 } 152 153 @Override 154 public String toString() { 155 return String.format("LocalLBPHistogram.Factory[blocksX=%d,blocksY=%d,samples=%d,radius=%d]", blocksX, 156 blocksY, samples, radius); 157 } 158 } 159 160 float[][][] histograms; 161 transient FloatFV featureVector; 162 163 protected void initialise(FImage face, FImage mask, int blocksX, int blocksY, int samples, int radius) { 164 final int[][] pattern = ExtendedLocalBinaryPattern.calculateLBP(face, radius, samples); 165 final boolean[][][] maps = UniformBinaryPattern.extractPatternMaps(pattern, samples); 166 167 final int bx = face.width / blocksX; 168 final int by = face.height / blocksY; 169 histograms = new float[blocksY][blocksX][maps.length]; 170 171 // build histogram 172 for (int p = 0; p < maps.length; p++) { 173 for (int y = 0; y < blocksY; y++) { 174 for (int x = 0; x < blocksX; x++) { 175 176 for (int j = 0; j < by; j++) { 177 for (int i = 0; i < bx; i++) { 178 if (maps[p][y * by + j][x * bx + i]) 179 histograms[y][x][p]++; 180 } 181 } 182 } 183 } 184 } 185 186 // normalise 187 for (int y = 0; y < blocksY; y++) { 188 for (int x = 0; x < blocksX; x++) { 189 float count = 0; 190 for (int p = 0; p < maps.length; p++) { 191 count += histograms[y][x][p]; 192 } 193 for (int p = 0; p < maps.length; p++) { 194 histograms[y][x][p] /= count; 195 } 196 } 197 } 198 199 updateFeatureVector(); 200 } 201 202 protected void updateFeatureVector() { 203 featureVector = new FloatFV(histograms.length * histograms[0].length * histograms[0][0].length); 204 205 int i = 0; 206 for (int y = 0; y < histograms.length; y++) { 207 for (int x = 0; x < histograms[0].length; x++) { 208 for (int p = 0; p < histograms[0][0].length; p++) { 209 featureVector.values[i] = histograms[y][x][p]; 210 i++; 211 } 212 } 213 } 214 } 215 216 @Override 217 public byte[] binaryHeader() { 218 return "LBPH".getBytes(); 219 } 220 221 @Override 222 public void readBinary(DataInput in) throws IOException { 223 final int by = in.readInt(); 224 final int bx = in.readInt(); 225 final int p = in.readInt(); 226 227 histograms = new float[by][bx][p]; 228 229 for (int j = 0; j < by; j++) { 230 for (int i = 0; i < bx; i++) { 231 for (int k = 0; k < p; k++) { 232 histograms[j][i][k] = in.readFloat(); 233 } 234 } 235 } 236 updateFeatureVector(); 237 } 238 239 @Override 240 public void writeBinary(DataOutput out) throws IOException { 241 out.writeInt(histograms.length); 242 out.writeInt(histograms[0].length); 243 out.writeInt(histograms[0][0].length); 244 245 for (final float[][] hist1 : histograms) { 246 for (final float[] hist2 : hist1) { 247 for (final float h : hist2) { 248 out.writeFloat(h); 249 } 250 } 251 } 252 } 253 254 @Override 255 public FloatFV getFeatureVector() { 256 if (featureVector == null) 257 updateFeatureVector(); 258 259 return featureVector; 260 } 261}