001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.experiment.evaluation.classification;
031
032import java.util.Collection;
033import java.util.HashMap;
034import java.util.HashSet;
035import java.util.Map;
036import java.util.Set;
037
038import org.openimaj.data.dataset.GroupedDataset;
039import org.openimaj.data.dataset.ListDataset;
040import org.openimaj.experiment.dataset.util.DatasetAdaptors;
041import org.openimaj.experiment.evaluation.AnalysisResult;
042import org.openimaj.experiment.evaluation.Evaluator;
043
044/**
045 * Implementation of an {@link Evaluator} for the evaluation of classification
046 * experiments.
047 * 
048 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
049 * 
050 * @param <RESULT>
051 *            Type of analysed data
052 * @param <CLASS>
053 *            Type of classes predicted by the classifier
054 * @param <OBJECT>
055 *            Type of objects classified by the classifier
056 */
057public class ClassificationEvaluator<RESULT extends AnalysisResult, CLASS, OBJECT>
058                implements Evaluator<
059                Map<OBJECT, ClassificationResult<CLASS>>, RESULT>
060{
061        protected Classifier<CLASS, OBJECT> classifier;
062        protected ClassificationAnalyser<RESULT, CLASS, OBJECT> analyser;
063        protected Map<OBJECT, Set<CLASS>> actual;
064        protected Collection<OBJECT> objects;
065
066        /**
067         * Construct a new {@link ClassificationEvaluator} with the given
068         * classifier, set of objects to classify, ground truth ("actual") data and
069         * an {@link ClassificationAnalyser}.
070         * 
071         * @param classifier
072         *            the classifier
073         * @param objects
074         *            the objects to classify
075         * @param actual
076         *            the ground truth
077         * @param analyser
078         *            the analyser
079         */
080        public ClassificationEvaluator(Classifier<CLASS, OBJECT> classifier, Collection<OBJECT> objects,
081                        Map<OBJECT, Set<CLASS>> actual, ClassificationAnalyser<RESULT, CLASS, OBJECT> analyser)
082        {
083                this.classifier = classifier;
084                this.objects = objects;
085                this.actual = actual;
086                this.analyser = analyser;
087        }
088
089        /**
090         * Construct a new {@link ClassificationEvaluator} with the given
091         * classifier, ground truth ("actual") data and an
092         * {@link ClassificationAnalyser}.
093         * <p>
094         * The objects to classify are taken from the {@link Map#keySet()} of the
095         * ground truth.
096         * 
097         * @param classifier
098         *            the classifier
099         * @param actual
100         *            the ground truth
101         * @param analyser
102         *            the analyser
103         */
104        public ClassificationEvaluator(Classifier<CLASS, OBJECT> classifier, Map<OBJECT, Set<CLASS>> actual,
105                        ClassificationAnalyser<RESULT, CLASS, OBJECT> analyser)
106        {
107                this.classifier = classifier;
108                this.objects = actual.keySet();
109                this.actual = actual;
110                this.analyser = analyser;
111        }
112
113        /**
114         * Construct a new {@link ClassificationEvaluator} with the given
115         * classifier, ground truth ("actual") data and an
116         * {@link ClassificationAnalyser}.
117         * <p>
118         * The ground-truth classes to are taken from the
119         * {@link GroupedDataset#getGroups()} of the "actual" {@link GroupedDataset}
120         * , and the objects are assembled by concatenating all of the
121         * {@link ListDataset}s within the "actual" dataset.
122         * 
123         * @param classifier
124         *            the classifier
125         * @param actual
126         *            the dataset containing instances and ground truths
127         * @param analyser
128         *            the analyser
129         */
130        public ClassificationEvaluator(Classifier<CLASS, OBJECT> classifier,
131                        GroupedDataset<CLASS, ? extends ListDataset<OBJECT>, OBJECT> actual,
132                        ClassificationAnalyser<RESULT, CLASS, OBJECT> analyser)
133        {
134                this.classifier = classifier;
135                this.objects = DatasetAdaptors.asList(actual);
136                this.actual = new HashMap<OBJECT, Set<CLASS>>();
137                for (final CLASS clazz : actual.getGroups()) {
138                        final HashSet<CLASS> cset = new HashSet<CLASS>();
139                        cset.add(clazz);
140                        for (final OBJECT instance : actual.getInstances(clazz)) {
141                                this.actual.put(instance, cset);
142                        }
143                }
144                this.analyser = analyser;
145        }
146
147        /**
148         * Construct a new {@link ClassificationEvaluator} with the given
149         * pre-classified results, the ground truth ("actual") data and an
150         * {@link ClassificationAnalyser}.
151         * <p>
152         * Internally, this constructor wraps a simple {@link Classifier}
153         * implementation around the results.
154         * 
155         * @param results
156         *            the pre-classified results
157         * @param actual
158         *            the ground truth
159         * @param analyser
160         *            the analyser
161         */
162        public ClassificationEvaluator(final Map<OBJECT, ClassificationResult<CLASS>> results,
163                        Map<OBJECT, Set<CLASS>> actual, ClassificationAnalyser<RESULT, CLASS, OBJECT> analyser)
164        {
165                this.classifier = new Classifier<CLASS, OBJECT>() {
166                        @Override
167                        public ClassificationResult<CLASS> classify(OBJECT object) {
168                                return results.get(object);
169                        }
170                };
171
172                this.objects = actual.keySet();
173                this.actual = actual;
174                this.analyser = analyser;
175        }
176
177        @Override
178        public Map<OBJECT, ClassificationResult<CLASS>> evaluate() {
179                final Map<OBJECT, ClassificationResult<CLASS>> results = new HashMap<OBJECT, ClassificationResult<CLASS>>();
180
181                for (final OBJECT object : objects) {
182                        results.put(object, classifier.classify(object));
183                }
184
185                return results;
186        }
187
188        @Override
189        public RESULT analyse(Map<OBJECT, ClassificationResult<CLASS>> predicted) {
190                return analyser.analyse(predicted, actual);
191        }
192
193        /**
194         * Get the expected classes for each instance
195         * 
196         * @return the map of instances to expected classes
197         */
198        public Map<OBJECT, Set<CLASS>> getExpected() {
199                return actual;
200        }
201}