001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.experiment.evaluation.retrieval.analysers; 031 032import java.io.File; 033import java.io.FileWriter; 034import java.io.IOException; 035import java.io.InputStream; 036import java.util.ArrayList; 037import java.util.HashMap; 038import java.util.List; 039import java.util.Map; 040 041import net.sf.jasperreports.engine.JRException; 042import net.sf.jasperreports.engine.JasperCompileManager; 043import net.sf.jasperreports.engine.JasperFillManager; 044import net.sf.jasperreports.engine.JasperPrint; 045import net.sf.jasperreports.engine.JasperReport; 046import net.sf.jasperreports.engine.data.JRBeanCollectionDataSource; 047import net.sf.jasperreports.engine.design.JasperDesign; 048import net.sf.jasperreports.engine.xml.JRXmlLoader; 049 050import org.lemurproject.ireval.IREval; 051import org.lemurproject.ireval.RetrievalEvaluator; 052import org.lemurproject.ireval.SetRetrievalEvaluator; 053import org.openimaj.experiment.evaluation.AnalysisResult; 054import org.openimaj.util.pair.IndependentPair; 055 056import com.googlecode.jatl.Html; 057 058/** 059 * An {@link AnalysisResult} that is bascked by a {@link SetRetrievalEvaluator} to 060 * capture the results of a retrieval experiment. 061 * 062 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk) 063 */ 064public class IREvalResult implements AnalysisResult { 065 protected SetRetrievalEvaluator eval; 066 067 /** 068 * Construct with the given {@link SetRetrievalEvaluator} result. 069 * @param sre the {@link SetRetrievalEvaluator}. 070 */ 071 public IREvalResult(SetRetrievalEvaluator sre) { 072 this.eval = sre; 073 } 074 075 @Override 076 public String toString() { 077 return this.getSummaryReport(); 078 } 079 080 void writeHTML(File file, final String title, final String info) throws IOException { 081 FileWriter fw = null; 082 try { 083 fw = new FileWriter(file); 084 085 new Html(fw) {{ 086 html(); 087 head(); 088 title(title); 089 end(); 090 body(); 091 h1().text(title).end(); 092 div().text(info).end(); 093 hr(); 094 writeEvaluation(); 095 hr(); 096 h2().text("Individual Query Results:").end(); 097 writeIndividualQueries(); 098 endAll(); 099 done(); 100 } 101 102 void writeIndividualQueries() { 103 for( RetrievalEvaluator evaluator : eval.getEvaluators() ) { 104 String query = evaluator.queryName(); 105 writeIndividual(query, evaluator); 106 } 107 } 108 109 void writeIndividual(String name, RetrievalEvaluator re) { 110 table(); 111 tr(); 112 td().colspan("2"); 113 h3().text(name).end(); 114 end(); 115 end(); 116 117 // counts 118 tr().td().text("num_ret").end().td().text(re.retrievedDocuments().size() + "").end().end(); 119 tr().td().text("num_rel").end().td().text(re.relevantDocuments().size() + "").end().end(); 120 tr().td().text("num_rel_ret").end().td().text(re.relevantRetrievedDocuments().size() + "").end().end(); 121 122 // aggregate measures 123 tr().td().text("map").end().td().text(String.format("%3.4f", re.averagePrecision())).end().end(); 124 tr().td().text("ndcg").end().td().text(String.format("%3.4f", re.normalizedDiscountedCumulativeGain())).end().end(); 125 tr().td().text("ndcg15").end().td().text(String.format("%3.4f", re.normalizedDiscountedCumulativeGain( 15 ))).end().end(); 126 tr().td().text("R-prec").end().td().text(String.format("%3.4f", re.rPrecision())).end().end(); 127 tr().td().text("bpref").end().td().text(String.format("%3.4f", re.binaryPreference())).end().end(); 128 tr().td().text("recip_rank").end().td().text(String.format("%3.4f", re.reciprocalRank())).end().end(); 129 130 // precision at fixed points 131 int[] fixedPoints = RetrievalEvaluator.getFixedPoints(); 132 double [] vals = re.precisionAtFixedPoints(); 133 for( int i=0; i<fixedPoints.length; i++ ) { 134 int point = fixedPoints[i]; 135 tr().td().text("P" + point).end().td().text(String.format("%3.4f", vals[i])).end().end(); 136 } 137 double[] precs = re.interpolatedPrecision(); 138 double prec = 0; 139 for( int i=0; i<precs.length; i++ ) { 140 tr().td().text(String.format("ircl_prn.%3.2f", prec)).end().td().text(String.format("%3.4f", precs[i])).end().end(); 141 prec += 0.1; 142 } 143 end(); 144 } 145 146 void writeEvaluation() { 147 table(); 148 tr(); 149 td().colspan("2"); 150 h2().text("Summary Results:").end(); 151 end(); 152 end(); 153 154 // print summary data 155 tr().td().text("num_q").end().td().text(eval.getEvaluators().size() + "").end().end(); 156 tr().td().text("num_ret").end().td().text(eval.numberRetrieved() + "").end().end(); 157 tr().td().text("num_rel").end().td().text(eval.numberRetrieved() + "").end().end(); 158 tr().td().text("num_rel_ret").end().td().text(eval.numberRelevantRetrieved() + "").end().end(); 159 160 tr().td().text("map").end().td().text(String.format("%3.4f", eval.meanAveragePrecision())).end().end(); 161 tr().td().text("gm_ap").end().td().text(String.format("%3.4f", eval.geometricMeanAveragePrecision())).end().end(); 162 tr().td().text("ndcg").end().td().text(String.format("%3.4f", eval.meanNormalizedDiscountedCumulativeGain())).end().end(); 163 tr().td().text("R-prec").end().td().text(String.format("%3.4f", eval.meanRPrecision())).end().end(); 164 tr().td().text("bpref").end().td().text(String.format("%3.4f", eval.meanBinaryPreference())).end().end(); 165 tr().td().text("recip_rank").end().td().text(String.format("%3.4f", eval.meanReciprocalRank())).end().end(); 166 167 // precision at fixed points 168 int[] fixedPoints = SetRetrievalEvaluator.getFixedPoints(); 169 double [] precs = eval.precisionAtFixedPoints(); 170 171 for( int i=0; i<fixedPoints.length; i++ ) { 172 int point = fixedPoints[i]; 173 tr().td().text("P" + point).end().td().text(String.format("%3.4f", precs[i])).end().end(); 174 } 175 double prec = 0; 176 precs = eval.interpolatedPrecision(); 177 for( int i=0; i<precs.length; i++ ) { 178 tr().td().text(String.format("ircl_prn.%3.2f", prec)).end().td().text(String.format("%3.4f", precs[i])).end().end(); 179 prec += 0.1; 180 } 181 end(); 182 } 183 }; 184 } finally { 185 if (fw != null) fw.close(); 186 } 187 } 188 189 /** 190 * @return a summary of the evaluation 191 */ 192 public List<IndependentPair<String, Number>> getSummaryData() { 193 List<IndependentPair<String, Number>> data = new ArrayList<IndependentPair<String, Number>>(); 194 195 data.add(new IndependentPair<String, Number>("num_q", eval.getEvaluators().size())); 196 data.add(new IndependentPair<String, Number>("num_ret", eval.numberRetrieved())); 197 data.add(new IndependentPair<String, Number>("num_rel", eval.numberRetrieved())); 198 data.add(new IndependentPair<String, Number>("num_rel_ret", eval.numberRelevantRetrieved())); 199 200 data.add(new IndependentPair<String, Number>("map", eval.meanAveragePrecision())); 201 data.add(new IndependentPair<String, Number>("gm_ap", eval.geometricMeanAveragePrecision())); 202 data.add(new IndependentPair<String, Number>("ndcg", eval.meanNormalizedDiscountedCumulativeGain())); 203 data.add(new IndependentPair<String, Number>("R-prec", eval.meanRPrecision())); 204 data.add(new IndependentPair<String, Number>("bpref", eval.meanBinaryPreference())); 205 data.add(new IndependentPair<String, Number>("recip_rank", eval.meanReciprocalRank())); 206 207 return data; 208 } 209 210 /** 211 * @return the interpolated PR data 212 */ 213 public List<IndependentPair<Double, Double>> getInterpolatedPRData() { 214 List<IndependentPair<Double, Double>> data = new ArrayList<IndependentPair<Double, Double>>(); 215 216 double prec = 0; 217 double[] precs = eval.interpolatedPrecision(); 218 for( int i=0; i<precs.length; i++ ) { 219 data.add(new IndependentPair<Double, Double>(prec, precs[i])); 220 prec += 0.1; 221 } 222 223 return data; 224 } 225 226 @Override 227 public JasperPrint getSummaryReport(String title, String info) throws JRException { 228 InputStream inputStream = IREvalResult.class.getResourceAsStream("IREvalSummaryReport.jrxml"); 229 ArrayList<IREvalResult> list = new ArrayList<IREvalResult>(); 230 list.add(this); 231 JRBeanCollectionDataSource beanColDataSource = new JRBeanCollectionDataSource(list); 232 233 Map<String, Object> parameters = new HashMap<String, Object>(); 234 235 JasperDesign jasperDesign = JRXmlLoader.load(inputStream); 236 JasperReport jasperReport = JasperCompileManager.compileReport(jasperDesign); 237 JasperPrint jasperPrint = JasperFillManager.fillReport(jasperReport, parameters, beanColDataSource); 238 239 return jasperPrint; 240 } 241 242 @Override 243 public JasperPrint getDetailReport(String title, String info) { 244 //FIXME 245 throw new UnsupportedOperationException(); 246 } 247 248 @Override 249 public String getSummaryReport() { 250 return IREval.singleEvaluation(eval, false); 251 } 252 253 @Override 254 public String getDetailReport() { 255 return IREval.singleEvaluation(eval, true); 256 } 257}