001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.experiment.evaluation.retrieval.analysers;
031
032import java.util.ArrayList;
033import java.util.List;
034import java.util.Map;
035import java.util.Map.Entry;
036import java.util.Set;
037import java.util.TreeMap;
038
039import org.lemurproject.ireval.IREval;
040import org.lemurproject.ireval.RetrievalEvaluator.Document;
041import org.lemurproject.ireval.RetrievalEvaluator.Judgment;
042import org.lemurproject.ireval.SetRetrievalEvaluator;
043import org.openimaj.data.identity.Identifiable;
044import org.openimaj.experiment.evaluation.retrieval.Ranked;
045import org.openimaj.experiment.evaluation.retrieval.RetrievalAnalyser;
046import org.openimaj.experiment.evaluation.retrieval.RetrievalEngine;
047import org.openimaj.experiment.evaluation.retrieval.Scored;
048
049/**
050 * An evaluator suitable for any kind of result document from the
051 * {@link RetrievalEngine}. Retrieval statistics are calculated using
052 * {@link IREval}.
053 * <p>
054 * If the retrieved documents are instances of {@link Ranked}, then the rank is
055 * determined through {@link Ranked#getRank()}, otherwise the rank of each
056 * result document is automatically determined from its position in the results
057 * list.
058 * <p>
059 * Similarly, if the retrieved documents are instances of {@link Scored}, then
060 * the score is determined through {@link Scored#getScore()}, otherwise the
061 * score of each result document is set as 1.0/rank.
062 * <p>
063 * If the queries are {@link Identifiable}, then the query ids in the outputted
064 * {@link SetRetrievalEvaluator} are the ID of the query; if the queries are not
065 * {@link Identifiable}, then the {@link Object#toString()} method is used
066 * instead.
067 *
068 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
069 *
070 * @param <QUERY>
071 *            Type of query
072 * @param <DOCUMENT>
073 *            Type of document
074 */
075public class IREvalAnalyser<QUERY, DOCUMENT extends Identifiable>
076implements RetrievalAnalyser<
077IREvalResult,
078QUERY,
079DOCUMENT>
080{
081        protected static <Q, D extends Identifiable> TreeMap<String, ArrayList<Document>> convertResults(
082                        Map<Q, List<D>> results)
083                        {
084                final TreeMap<String, ArrayList<Document>> allRankings = new TreeMap<String, ArrayList<Document>>();
085
086                for (final Entry<Q, List<D>> entry : results.entrySet()) {
087                        String key;
088
089                        if (entry.getKey() instanceof Identifiable)
090                                key = ((Identifiable) entry.getKey()).getID();
091                        else
092                                key = entry.getKey().toString();
093
094                        final ArrayList<Document> docs = new ArrayList<Document>();
095                        for (int i = 0; i < entry.getValue().size(); i++) {
096                                final D doc = entry.getValue().get(i);
097
098                                int rnk = i + 1;
099                                if (doc instanceof Ranked)
100                                        rnk = ((Ranked) doc).getRank();
101
102                                double score = (1.0 / rnk);
103                                if (doc instanceof Scored)
104                                        score = ((Scored) doc).getScore();
105
106                                docs.add(new Document(doc.getID(), rnk, score));
107                        }
108
109                        allRankings.put(key, docs);
110                }
111
112                return allRankings;
113                        }
114
115        protected static <Q, D extends Identifiable> TreeMap<String, ArrayList<Judgment>> convertRelevant(
116                        Map<Q, Set<D>> relevant)
117                        {
118                final TreeMap<String, ArrayList<Judgment>> allJudgments = new TreeMap<String, ArrayList<Judgment>>();
119
120                for (final Entry<Q, Set<D>> entry : relevant.entrySet()) {
121                        String key;
122
123                        if (entry.getKey() instanceof Identifiable)
124                                key = ((Identifiable) entry.getKey()).getID();
125                        else
126                                key = entry.getKey().toString();
127
128                        final ArrayList<Judgment> docs = new ArrayList<Judgment>();
129                        for (final D doc : entry.getValue()) {
130                                docs.add(new Judgment(doc.getID(), 1));
131                        }
132
133                        allJudgments.put(key, docs);
134                }
135
136                return allJudgments;
137                        }
138
139        @Override
140        public IREvalResult analyse(Map<QUERY, List<DOCUMENT>> results, Map<QUERY, Set<DOCUMENT>> relevant) {
141                return new IREvalResult(IREval.create(convertResults(results), convertRelevant(relevant)));
142        }
143}