001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030/** 031 * 032 */ 033package org.openimaj.demos.sandbox.audio; 034 035import java.io.IOException; 036 037import org.openimaj.audio.AudioAnnotator; 038import org.openimaj.data.dataset.GroupedDataset; 039import org.openimaj.data.dataset.ListDataset; 040import org.openimaj.data.dataset.MapBackedDataset; 041import org.openimaj.experiment.ExperimentContext; 042import org.openimaj.experiment.ExperimentRunner; 043import org.openimaj.experiment.RunnableExperiment; 044import org.openimaj.experiment.agent.ExperimentAgent; 045import org.openimaj.experiment.annotations.Experiment; 046import org.openimaj.experiment.annotations.IndependentVariable; 047import org.openimaj.experiment.annotations.Time; 048import org.openimaj.experiment.evaluation.classification.ClassificationEvaluator; 049import org.openimaj.experiment.evaluation.classification.analysers.confusionmatrix.AggregatedCMResult; 050import org.openimaj.experiment.evaluation.classification.analysers.confusionmatrix.CMAggregator; 051import org.openimaj.experiment.evaluation.classification.analysers.confusionmatrix.CMAnalyser; 052import org.openimaj.experiment.evaluation.classification.analysers.confusionmatrix.CMAnalyser.Strategy; 053import org.openimaj.experiment.evaluation.classification.analysers.confusionmatrix.CMResult; 054import org.openimaj.experiment.validation.ValidationOperation; 055import org.openimaj.experiment.validation.ValidationRunner; 056import org.openimaj.experiment.validation.cross.StratifiedGroupedKFold; 057import org.openimaj.feature.DoubleFV; 058 059/** 060 * An experiment for testing how good the trained speech detector is. It uses a 061 * stratified 10-fold cross-validation and uses a 1-k KNN annotator. The dataset 062 * is created during the {@link #setup()} stage by the detector. The cross-validation 063 * uses the audio annotator delivered by the detector. 064 * 065 * @author David Dupplaw (dpd@ecs.soton.ac.uk) 066 * @created 06 Mar 2013 067 * @version $Author$, $Revision$, $Date$ 068 */ 069@Experiment( 070 author = "David Dupplaw <dpd@ecs.soton.ac.uk>", 071 dateCreated = "2013-03-08", 072 description = "Speech detection cross validation experiment") 073public class SpeechDetectorExperiment implements RunnableExperiment 074{ 075 /** The speech detector we're testing */ 076 @IndependentVariable 077 private final SpeechDetector speechDetector; 078 079 /** The dataset to use for the experiment */ 080 @IndependentVariable 081 private MapBackedDataset<String, ListDataset<DoubleFV>, DoubleFV> dataset = null; 082 083 /** The final result of the experiment */ 084 @IndependentVariable 085 private AggregatedCMResult<String> result; 086 087 /** 088 * Takes the speech detector to test 089 * @param sd The speech detector to test 090 */ 091 public SpeechDetectorExperiment( final SpeechDetector sd ) 092 { 093 this.speechDetector = sd; 094 } 095 096 /** 097 * Creates the datasets from which the validations will take place 098 * 099 * {@inheritDoc} 100 * @see org.openimaj.experiment.RunnableExperiment#setup() 101 */ 102 @Override 103 public void setup() 104 { 105 this.dataset = this.speechDetector.generateDataset(); 106 } 107 108 /** 109 * Uses a stratified 10-fold grouped cross-validation 110 * 111 * {@inheritDoc} 112 * @see org.openimaj.experiment.RunnableExperiment#perform() 113 */ 114 @Override 115 public void perform() 116 { 117 // We'll use a stratified K-fold cross-validation with 10 subsets 118 final StratifiedGroupedKFold<String,DoubleFV> k = 119 new StratifiedGroupedKFold<String, DoubleFV>( 10 ); 120 121 // We'll create a confusion matrix for the results 122 final CMAggregator<String> confusionMatrixAggregator = 123 new CMAggregator<String>(); 124 125 this.result = ValidationRunner.run( confusionMatrixAggregator, this.dataset, k, 126 new ValidationOperation<GroupedDataset<String,ListDataset<DoubleFV>,DoubleFV>, 127 CMResult<String>>() 128 { 129 /** 130 * The validation operation uses a ClassificationEvaluator to evaluate the 131 * confusion matrix. 132 * 133 * @param training Training data 134 * @param validation Validation data 135 * @return Confusion Matrix for the classes 136 */ 137 @Time(identifier = "Train and Evaluate detector") 138 @Override 139 public CMResult<String> evaluate( 140 final GroupedDataset<String, ListDataset<DoubleFV>, DoubleFV> training, 141 final GroupedDataset<String, ListDataset<DoubleFV>, DoubleFV> validation ) 142 { 143 for( final String group : training.getGroups() ) 144 { 145 for( int i = 0; i < training.getInstances( group ).size(); i++ ) 146 { 147 final DoubleFV dfv = training.getInstances( group ).getInstance( i ); 148 if( dfv == null ) 149 System.out.println( "TraningSet: Null in group "+group+" at "+i ); 150 } 151 } 152 for( final String group : validation.getGroups() ) 153 { 154 for( int i = 0; i < validation.getInstances( group ).size(); i++ ) 155 { 156 final DoubleFV dfv = validation.getInstances( group ).getInstance( i ); 157 if( dfv == null ) 158 System.out.println( "ValidationSet: Null in group "+group+" at "+i ); 159 } 160 } 161 162 // Get a new annotator and train it with the training set. 163 final AudioAnnotator aa = SpeechDetectorExperiment.this. 164 speechDetector.getNewAnnotator(); 165 aa.train( training ); 166 167 // Now perform the validation using the validation set 168 final ClassificationEvaluator<CMResult<String>, String, DoubleFV> 169 eval = new ClassificationEvaluator<CMResult<String>, String, DoubleFV>( 170 aa, validation, new CMAnalyser<DoubleFV,String>( Strategy.SINGLE ) ); 171 172 return eval.analyse( eval.evaluate() ); 173 } 174 }); 175 176 System.out.println( this.result ); 177 } 178 179 @Override 180 public void finish( final ExperimentContext context ) 181 { 182 System.out.println( "Finished experiment" ); 183 } 184 185 // ===================================================================================== // 186 187 188 /** 189 * Run this experiment. 190 * @param args The command-line arguments 191 * @throws IOException 192 */ 193 public static void main( final String[] args ) throws IOException 194 { 195 ExperimentAgent.initialise(); 196 197 // This is the detector that we'll test 198 final SpeechDetector sd = new SpeechDetector(); 199 sd.parseArgs( args ); 200 201 // Create the experiment 202 final SpeechDetectorExperiment experiment = new SpeechDetectorExperiment( sd ); 203 204 // Now run it! 205 final ExperimentContext ctx = ExperimentRunner.runExperiment( experiment ); 206 System.out.println( ctx ); 207 } 208}