001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030/*
031 * SetRetrievalEvaluator
032 *
033 * Use of this software is governed by the Creative Commons Attribution license:
034 *    http://creativecommons.org/licenses/by/2.5/
035 *
036 * Trevor Strohman, September 23, 2005
037 */
038
039package org.lemurproject.ireval;
040
041import java.util.Collection;
042import java.util.Map;
043import java.util.TreeMap;
044
045/**
046 * Computes summary statistics over a set of queries.
047 *
048 * @author Trevor Strohman
049 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
050 */
051public class SetRetrievalEvaluator {
052    Collection<RetrievalEvaluator> _evaluators;
053    
054    /** 
055     * Creates a new instance of SetRetrievalEvaluator 
056     * @param evaluators 
057     */
058    public SetRetrievalEvaluator( Collection<RetrievalEvaluator> evaluators ) {
059        _evaluators = evaluators;
060    }
061    
062    /**
063     * @return Returns a collection of evaluators.
064     */
065    public Collection<RetrievalEvaluator> getEvaluators() {
066        return _evaluators;
067    }
068    
069    /**
070     * @return the levels at which precision is calculated
071     */
072    public static int[] getFixedPoints() {
073        return RetrievalEvaluator.getFixedPoints();
074    }
075    
076    /**
077     * @return the precision at fixed values
078     */
079    public double[] precisionAtFixedPoints() {
080        // precision at fixed points
081        int []fixedPoints = RetrievalEvaluator.getFixedPoints();
082        double[] retVal = new double [fixedPoints.length];
083        int i = 0;
084        for( int point : fixedPoints ) {
085            retVal[i++] = meanPrecision( point );
086        }
087        return retVal;
088    }
089    
090    /**
091     * @return the interpolated precision values
092     */
093    public double[] interpolatedPrecision() {
094        double[] precs = {0,0,0,0,0,0,0,0,0,0,0};
095        for (RetrievalEvaluator evaluator : _evaluators) {
096            double[] vals = evaluator.interpolatedPrecision();
097            for (int j = 0; j < vals.length; j++) precs[j] += vals[j];
098        }
099        for (int j = 0; j < precs.length; j++) precs[j] /= _evaluators.size();
100        return precs;
101    }
102    /**
103     * @return the mean average precision; the mean of the average precision values for all queries.
104     */
105    public double meanAveragePrecision() {
106        double sumAveragePrecision = 0;
107        if (_evaluators.size() == 0) return 0;
108
109        for( RetrievalEvaluator evaluator : _evaluators ) {
110            sumAveragePrecision += evaluator.averagePrecision();
111        }
112        
113        return sumAveragePrecision / (double) _evaluators.size();
114    }
115    
116    /**
117     * @return the mean of the binary preference values for all queries.
118     */
119    public double meanBinaryPreference() {
120        double sumBinaryPreference = 0;
121        if (_evaluators.size() == 0) return 0;
122        for( RetrievalEvaluator evaluator : _evaluators ) {
123            sumBinaryPreference += evaluator.binaryPreference();
124        }
125        
126        return sumBinaryPreference / (double) _evaluators.size();
127    }
128
129    /**
130     * @return the geometric mean of average precision values for all queries.
131     */
132    public double geometricMeanAveragePrecision() {
133        double productAveragePrecision = 1.0;
134        if (_evaluators.size() == 0) return 0;
135
136        for( RetrievalEvaluator evaluator : _evaluators ) {
137            double p = evaluator.averagePrecision();
138            // don't let a 0 value into the product.
139            if (p > 0.0)
140                productAveragePrecision *= p;
141        }
142        return Math.pow( productAveragePrecision, 1.0 / _evaluators.size() );
143    }
144    
145    /**
146     * @param documentsRetrieved 
147     * @return the mean of the precision values for all queries.
148     */
149    public double meanPrecision( int documentsRetrieved ) {
150        double sumPrecision = 0;
151        if (_evaluators.size() == 0) return 0;
152
153        for( RetrievalEvaluator evaluator : _evaluators ) {
154            sumPrecision += evaluator.precision( documentsRetrieved );
155        }
156        
157        return sumPrecision / _evaluators.size();
158    }
159    
160    /** 
161     * @return the mean of the reciprocal rank values for all queries.
162     */
163    public double meanReciprocalRank( ) {
164        double sumRR = 0;
165        if (_evaluators.size() == 0) return 0;
166        
167        for( RetrievalEvaluator evaluator : _evaluators ) {
168            sumRR += evaluator.reciprocalRank();
169        }
170        
171        return sumRR / _evaluators.size();
172    }
173    
174    /** 
175     * @return the mean of the R-precision values for all queries.
176     */
177    public double meanRPrecision( ) {
178        double sumRPrecision = 0;
179        if (_evaluators.size() == 0) return 0;
180        
181        for( RetrievalEvaluator evaluator : _evaluators ) {
182            sumRPrecision += evaluator.rPrecision();
183        }
184        
185        return sumRPrecision / _evaluators.size();
186    }  
187    
188    /**
189     * @return the mean of the NDCG values for all queries.
190     */
191    public double meanNormalizedDiscountedCumulativeGain( ) {
192        double sumNDCG = 0;
193        if (_evaluators.size() == 0) return 0;
194        
195        for( RetrievalEvaluator evaluator : _evaluators ) {
196            sumNDCG += evaluator.normalizedDiscountedCumulativeGain();
197        }                                                             
198        
199        return sumNDCG / _evaluators.size();
200    }
201
202    /**
203     * @param documentsRetrieved 
204     * @return the mean of the NDCG values for all queries.
205     */
206    public double meanNormalizedDiscountedCumulativeGain( int documentsRetrieved ) {
207        double sumNDCG = 0;
208        if (_evaluators.size() == 0) return 0;
209        
210        for( RetrievalEvaluator evaluator : _evaluators ) {
211            sumNDCG += evaluator.normalizedDiscountedCumulativeGain( documentsRetrieved );
212        }                                                             
213        
214        return sumNDCG / _evaluators.size();
215    }
216
217    /**
218     * Get a {@link Map} containing a particular metric value for each query.
219     * For instance, if metric == "averagePrecision", this returns 
220     * a map where the keys are query identifiers and the values are the
221     * average precision metric evaluated for each query.
222     * 
223     * @param metric the metric name.
224     * 
225     * @return A {@link Map} containing a particular metric value for each query.
226     * @throws IllegalArgumentException 
227     */
228    public Map<String, Double> evaluateAll( String metric ) throws IllegalArgumentException {
229        TreeMap<String, Double> result = new TreeMap<String, Double>();
230        
231        for( RetrievalEvaluator evaluator : _evaluators ) {
232            double value = 0;
233            
234            if( metric.equals("averagePrecision") || metric.equals("ap") || metric.equals("map") ) {
235                value = evaluator.averagePrecision();
236            } else if( metric.equals("ndcg") ) {
237                value = evaluator.normalizedDiscountedCumulativeGain();
238            } else if( metric.equals("reciprocalRank") || metric.equals("mrr") ) {
239                value = evaluator.reciprocalRank();
240            } else if( metric.equals("rPrecision") ) {
241                value = evaluator.rPrecision();
242            } else if( metric.equals( "bpref" ) ) {
243                value = evaluator.binaryPreference();
244            } else if( metric.startsWith( "P" ) ) {
245                value = evaluator.precision( Integer.parseInt( metric.substring(1) ) );
246            } else if( metric.startsWith( "R" ) ) {
247                value = evaluator.recall( Integer.parseInt( metric.substring(1) ) );
248            } else {
249                throw new IllegalArgumentException( "Unknown metric: " + metric );
250            }
251            
252            result.put( evaluator.queryName(), value );
253        }
254        
255        return result;
256    }
257
258    /**
259     * @return The number of documents retrieved for all queries.
260     */
261    public int numberRetrieved() {
262        int sumRetrieved = 0;
263        
264        for( RetrievalEvaluator evaluator : _evaluators ) {
265            sumRetrieved += evaluator.retrievedDocuments().size();
266        }
267        
268        return sumRetrieved;
269    }
270    
271    /**
272     * @return The total number of relevant documents to any of the queries.
273     */
274    public int numberRelevant() {
275        int sumRelevant = 0;
276        
277        for( RetrievalEvaluator evaluator : _evaluators ) {
278            sumRelevant += evaluator.relevantDocuments().size();
279        }
280        
281        return sumRelevant;
282    }
283    
284    /**
285     * @return The total number of relevant documents retrieved for any of the queries.
286     */
287    public int numberRelevantRetrieved() {
288        int sumRelevantRetrieved = 0;
289        
290        for( RetrievalEvaluator evaluator : _evaluators ) {
291            sumRelevantRetrieved += evaluator.relevantRetrievedDocuments().size();
292        }
293        
294        return sumRelevantRetrieved;
295    }
296    
297    @Override
298        public String toString() {
299        return IREval.singleEvaluation(this, true);
300    }
301}