001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.image.processing.face.feature;
031
032import java.io.DataInput;
033import java.io.DataOutput;
034import java.io.IOException;
035
036import org.openimaj.feature.FeatureVectorProvider;
037import org.openimaj.feature.FloatFV;
038import org.openimaj.image.FImage;
039import org.openimaj.image.feature.dense.binarypattern.ExtendedLocalBinaryPattern;
040import org.openimaj.image.feature.dense.binarypattern.UniformBinaryPattern;
041import org.openimaj.image.processing.face.alignment.FaceAligner;
042import org.openimaj.image.processing.face.alignment.IdentityAligner;
043import org.openimaj.image.processing.face.detection.DetectedFace;
044import org.openimaj.io.IOUtils;
045
046/**
047 * A {@link FacialFeature} built from decomposing the face image into
048 * (non-overlapping) blocks and building histograms of the
049 * {@link ExtendedLocalBinaryPattern}s for each block and then concatenating to
050 * form the final feature.
051 * 
052 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
053 */
054public class LocalLBPHistogram implements FacialFeature, FeatureVectorProvider<FloatFV> {
055        /**
056         * A {@link FacialFeatureExtractor} for building {@link LocalLBPHistogram}s.
057         * 
058         * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
059         * 
060         * @param <T>
061         *            Type of {@link DetectedFace}.
062         */
063        public static class Extractor<T extends DetectedFace> implements FacialFeatureExtractor<LocalLBPHistogram, T> {
064                FaceAligner<T> aligner;
065                int blocksX = 25;
066                int blocksY = 25;
067                int samples = 8;
068                int radius = 1;
069
070                /**
071                 * Construct with a {@link IdentityAligner}
072                 */
073                public Extractor() {
074                        this.aligner = new IdentityAligner<T>();
075                }
076
077                /**
078                 * Construct with the given aligner.
079                 * 
080                 * @param aligner
081                 *            the aligner
082                 */
083                public Extractor(FaceAligner<T> aligner) {
084                        this.aligner = aligner;
085                }
086
087                /**
088                 * Construct with the given aligner, parameters describing how the image
089                 * is broken into blocks and parameters describing the radius of the LBP
090                 * extraction circle, and how many samples are made.
091                 * 
092                 * @param aligner
093                 *            The face aligner
094                 * @param blocksX
095                 *            The number of blocks in the x-direction
096                 * @param blocksY
097                 *            The number of blocks in the y-direction
098                 * @param samples
099                 *            The number of samples around the circle for the
100                 *            {@link ExtendedLocalBinaryPattern}
101                 * @param radius
102                 *            the radius used for the {@link ExtendedLocalBinaryPattern}
103                 *            .
104                 */
105                public Extractor(FaceAligner<T> aligner, int blocksX, int blocksY, int samples, int radius) {
106                        this.aligner = aligner;
107                        this.blocksX = blocksX;
108                        this.blocksY = blocksY;
109                        this.samples = samples;
110                        this.radius = radius;
111                }
112
113                @Override
114                public LocalLBPHistogram extractFeature(T detectedFace) {
115                        final LocalLBPHistogram f = new LocalLBPHistogram();
116
117                        final FImage face = aligner.align(detectedFace);
118                        final FImage mask = aligner.getMask();
119
120                        f.initialise(face, mask, blocksX, blocksY, samples, radius);
121
122                        return f;
123                }
124
125                @Override
126                public void readBinary(DataInput in) throws IOException {
127                        final String alignerClass = in.readUTF();
128                        aligner = IOUtils.newInstance(alignerClass);
129                        aligner.readBinary(in);
130
131                        blocksX = in.readInt();
132                        blocksY = in.readInt();
133                        radius = in.readInt();
134                        samples = in.readInt();
135                }
136
137                @Override
138                public byte[] binaryHeader() {
139                        return this.getClass().getName().getBytes();
140                }
141
142                @Override
143                public void writeBinary(DataOutput out) throws IOException {
144                        out.writeUTF(aligner.getClass().getName());
145                        aligner.writeBinary(out);
146
147                        out.writeInt(blocksX);
148                        out.writeInt(blocksY);
149                        out.writeInt(radius);
150                        out.writeInt(samples);
151                }
152
153                @Override
154                public String toString() {
155                        return String.format("LocalLBPHistogram.Factory[blocksX=%d,blocksY=%d,samples=%d,radius=%d]", blocksX,
156                                        blocksY, samples, radius);
157                }
158        }
159
160        float[][][] histograms;
161        transient FloatFV featureVector;
162
163        protected void initialise(FImage face, FImage mask, int blocksX, int blocksY, int samples, int radius) {
164                final int[][] pattern = ExtendedLocalBinaryPattern.calculateLBP(face, radius, samples);
165                final boolean[][][] maps = UniformBinaryPattern.extractPatternMaps(pattern, samples);
166
167                final int bx = face.width / blocksX;
168                final int by = face.height / blocksY;
169                histograms = new float[blocksY][blocksX][maps.length];
170
171                // build histogram
172                for (int p = 0; p < maps.length; p++) {
173                        for (int y = 0; y < blocksY; y++) {
174                                for (int x = 0; x < blocksX; x++) {
175
176                                        for (int j = 0; j < by; j++) {
177                                                for (int i = 0; i < bx; i++) {
178                                                        if (maps[p][y * by + j][x * bx + i])
179                                                                histograms[y][x][p]++;
180                                                }
181                                        }
182                                }
183                        }
184                }
185
186                // normalise
187                for (int y = 0; y < blocksY; y++) {
188                        for (int x = 0; x < blocksX; x++) {
189                                float count = 0;
190                                for (int p = 0; p < maps.length; p++) {
191                                        count += histograms[y][x][p];
192                                }
193                                for (int p = 0; p < maps.length; p++) {
194                                        histograms[y][x][p] /= count;
195                                }
196                        }
197                }
198
199                updateFeatureVector();
200        }
201
202        protected void updateFeatureVector() {
203                featureVector = new FloatFV(histograms.length * histograms[0].length * histograms[0][0].length);
204
205                int i = 0;
206                for (int y = 0; y < histograms.length; y++) {
207                        for (int x = 0; x < histograms[0].length; x++) {
208                                for (int p = 0; p < histograms[0][0].length; p++) {
209                                        featureVector.values[i] = histograms[y][x][p];
210                                        i++;
211                                }
212                        }
213                }
214        }
215
216        @Override
217        public byte[] binaryHeader() {
218                return "LBPH".getBytes();
219        }
220
221        @Override
222        public void readBinary(DataInput in) throws IOException {
223                final int by = in.readInt();
224                final int bx = in.readInt();
225                final int p = in.readInt();
226
227                histograms = new float[by][bx][p];
228
229                for (int j = 0; j < by; j++) {
230                        for (int i = 0; i < bx; i++) {
231                                for (int k = 0; k < p; k++) {
232                                        histograms[j][i][k] = in.readFloat();
233                                }
234                        }
235                }
236                updateFeatureVector();
237        }
238
239        @Override
240        public void writeBinary(DataOutput out) throws IOException {
241                out.writeInt(histograms.length);
242                out.writeInt(histograms[0].length);
243                out.writeInt(histograms[0][0].length);
244
245                for (final float[][] hist1 : histograms) {
246                        for (final float[] hist2 : hist1) {
247                                for (final float h : hist2) {
248                                        out.writeFloat(h);
249                                }
250                        }
251                }
252        }
253
254        @Override
255        public FloatFV getFeatureVector() {
256                if (featureVector == null)
257                        updateFeatureVector();
258
259                return featureVector;
260        }
261}