001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.image.processing.face.recognition.benchmarking;
031
032import org.openimaj.data.dataset.GroupedDataset;
033import org.openimaj.data.dataset.ListDataset;
034import org.openimaj.experiment.ExperimentContext;
035import org.openimaj.experiment.RunnableExperiment;
036import org.openimaj.experiment.annotations.DependentVariable;
037import org.openimaj.experiment.annotations.Experiment;
038import org.openimaj.experiment.annotations.IndependentVariable;
039import org.openimaj.experiment.annotations.Time;
040import org.openimaj.experiment.evaluation.classification.ClassificationEvaluator;
041import org.openimaj.experiment.evaluation.classification.analysers.confusionmatrix.AggregatedCMResult;
042import org.openimaj.experiment.evaluation.classification.analysers.confusionmatrix.CMAggregator;
043import org.openimaj.experiment.evaluation.classification.analysers.confusionmatrix.CMAnalyser;
044import org.openimaj.experiment.evaluation.classification.analysers.confusionmatrix.CMResult;
045import org.openimaj.experiment.validation.ValidationOperation;
046import org.openimaj.experiment.validation.ValidationRunner;
047import org.openimaj.experiment.validation.cross.CrossValidator;
048import org.openimaj.image.Image;
049import org.openimaj.image.processing.face.detection.DatasetFaceDetector;
050import org.openimaj.image.processing.face.detection.DetectedFace;
051import org.openimaj.image.processing.face.detection.FaceDetector;
052import org.openimaj.image.processing.face.recognition.FaceRecogniser;
053
054/**
055 * An {@link RunnableExperiment} for performing cross-validation experiments on
056 * face recognisers & classifiers.
057 * 
058 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
059 * 
060 * @param <PERSON>
061 *            type representing a person or class
062 * @param <IMAGE>
063 *            type of image containing the face of each person
064 * @param <FACE>
065 *            type of {@link DetectedFace}
066 */
067@Experiment(
068                author = "Jonathon Hare",
069                dateCreated = "2012-07-26",
070                description = "Face recognition cross validation experiment")
071public class CrossValidationBenchmark<PERSON, IMAGE extends Image<?, IMAGE>, FACE extends DetectedFace>
072                implements
073                RunnableExperiment
074{
075        @IndependentVariable
076        protected CrossValidator<GroupedDataset<PERSON, ListDataset<FACE>, FACE>> crossValidator;
077
078        @IndependentVariable
079        protected GroupedDataset<PERSON, ? extends ListDataset<IMAGE>, IMAGE> dataset;
080
081        @IndependentVariable
082        protected FaceDetector<FACE, IMAGE> faceDetector;
083
084        @IndependentVariable
085        protected FaceRecogniserProvider<FACE, PERSON> engine;
086
087        @DependentVariable
088        protected AggregatedCMResult<PERSON> result;
089
090        /**
091         * Construct the {@link CrossValidationBenchmark} experiment with the given
092         * dependent variables.
093         * 
094         * @param dataset
095         *            the dataset
096         * @param crossValidator
097         *            the cross-validator
098         * @param faceDetector
099         *            the face detector
100         * @param engine
101         *            the recogniser
102         */
103        public CrossValidationBenchmark(
104                        CrossValidator<GroupedDataset<PERSON, ListDataset<FACE>, FACE>> crossValidator,
105                        GroupedDataset<PERSON, ? extends ListDataset<IMAGE>, IMAGE> dataset,
106                        FaceDetector<FACE, IMAGE> faceDetector,
107                        FaceRecogniserProvider<FACE, PERSON> engine)
108        {
109                this.dataset = dataset;
110                this.crossValidator = crossValidator;
111                this.faceDetector = faceDetector;
112                this.engine = engine;
113        }
114
115        @Override
116        public void perform() {
117                final CMAggregator<PERSON> aggregator = new CMAggregator<PERSON>();
118
119                final GroupedDataset<PERSON, ListDataset<FACE>, FACE> faceDataset = DatasetFaceDetector.process(dataset,
120                                faceDetector);
121
122                result = ValidationRunner.run(
123                                aggregator,
124                                faceDataset,
125                                crossValidator,
126                                new ValidationOperation<GroupedDataset<PERSON, ListDataset<FACE>, FACE>, CMResult<PERSON>>()
127                                {
128                                        @Time(identifier = "Train and Evaluate recogniser")
129                                        @Override
130                                        public CMResult<PERSON> evaluate(
131                                                        GroupedDataset<PERSON, ListDataset<FACE>, FACE> training,
132                                                        GroupedDataset<PERSON, ListDataset<FACE>, FACE> validation)
133                                        {
134                                                final FaceRecogniser<FACE, PERSON> rec = engine.create(training);
135
136                                                final ClassificationEvaluator<CMResult<PERSON>, PERSON, FACE> eval =
137                                                                new ClassificationEvaluator<CMResult<PERSON>, PERSON, FACE>(
138                                                                                rec, validation, new CMAnalyser<FACE, PERSON>(CMAnalyser.Strategy.SINGLE)
139                                                                );
140
141                                                return eval.analyse(eval.evaluate());
142                                        }
143                                });
144        }
145
146        @Override
147        public void setup() {
148                // TODO Auto-generated method stub
149
150        }
151
152        @Override
153        public void finish(ExperimentContext context) {
154                // TODO Auto-generated method stub
155
156        }
157}