View Javadoc

1   /**
2    * Copyright (c) 2011, The University of Southampton and the individual contributors.
3    * All rights reserved.
4    *
5    * Redistribution and use in source and binary forms, with or without modification,
6    * are permitted provided that the following conditions are met:
7    *
8    *   * 	Redistributions of source code must retain the above copyright notice,
9    * 	this list of conditions and the following disclaimer.
10   *
11   *   *	Redistributions in binary form must reproduce the above copyright notice,
12   * 	this list of conditions and the following disclaimer in the documentation
13   * 	and/or other materials provided with the distribution.
14   *
15   *   *	Neither the name of the University of Southampton nor the names of its
16   * 	contributors may be used to endorse or promote products derived from this
17   * 	software without specific prior written permission.
18   *
19   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
20   * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21   * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22   * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
23   * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24   * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
26   * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27   * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28   * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29   */
30  /**
31   *
32   */
33  package org.openimaj.video.processing.shotdetector;
34  
35  import org.openimaj.citation.annotation.Reference;
36  import org.openimaj.citation.annotation.ReferenceType;
37  import org.openimaj.feature.DoubleFVComparison;
38  import org.openimaj.image.MBFImage;
39  import org.openimaj.image.colour.RGBColour;
40  import org.openimaj.image.pixel.statistics.HistogramModel;
41  import org.openimaj.math.geometry.shape.Rectangle;
42  import org.openimaj.util.array.ArrayUtils;
43  import org.openimaj.video.Video;
44  
45  /**
46   *	A shot detector implementing the Steiner et al. local histogram comparison.
47   *
48   *	@author David Dupplaw (dpd@ecs.soton.ac.uk)
49   *  @created 28 Jan 2013
50   *	@version $Author$, $Revision$, $Date$
51   */
52  @Reference(
53  		type = ReferenceType.Inproceedings,
54  		author = { "{S}teiner, {T}homas", "{V}erborgh, {R}uben", "{G}abarr{\'o} {V}all{\'e}s, {J}oaquim", "{T}roncy, {R}apha{\"e}l", "{H}ausenblas, {M}ichael", "{V}an de {W}alle, {R}ik", "{B}rousseau, {A}rnaud" },
55  		title = "{E}nabling on-the-fly video shot detection on {Y}ou{T}ube",
56  		year = "2012",
57  		booktitle = "{WWW} 2012, 21st {I}nternational {W}orld {W}ide {W}eb {C}onference {D}eveloper's {T}rack, {A}pril 16-20, 2012, {L}yon, {F}rance",
58  		url = "http://www.eurecom.fr/publication/3676",
59  		month = "04",
60  		customData = {
61  			"address", "{L}yon, {FRANCE}"
62  		}
63  	)
64  public class LocalHistogramVideoShotDetector
65  	extends VideoShotDetector<MBFImage>
66  {
67  	/** The histogram model used to estimate the histogram for each frame */
68  	private final HistogramModel histogramModel = new HistogramModel( 4,4,4 );
69  
70  	/** The histogram of the previous frame */
71  	private double[][][] lastHistogram = null;
72  
73  	/** The number of grid elements in the grid */
74  	private int nGridElements = 20;
75  
76  	/** The percentage of tiles to use as the most similar */
77  	private final double pcMostSimilar = 0.1;
78  
79  	/** The percentage of tiles to use as the most dissimilar */
80  	private final double pcMostDissimilar = 0.1;
81  
82  	/** The boosting factor to use */
83  	private final double boostFactor = 1.1;
84  
85  	/** The limiting factor to use */
86  	private final double limitingFactor = 0.9;
87  
88  	/**
89  	 * 	If you use this constructor, your timecodes will be messed up
90  	 * 	unless you call {@link #setFPS(double)} before you process
91  	 * 	any frames.
92  	 * 	@param nGridElements The number of x and y grid elements
93  	 *		(there will be this number squared in total)
94  	 */
95  	public LocalHistogramVideoShotDetector( final int nGridElements )
96  	{
97  		this.nGridElements = nGridElements;
98  		this.lastHistogram = new double[this.nGridElements][this.nGridElements][];
99  		this.threshold = 0.2;
100 	}
101 
102 	/**
103 	 *
104 	 *	@param video The video to process
105 	 * 	@param nGridElements The number of x and y grid elements
106 	 *		(there will be this number squared in total)
107 	 */
108 	public LocalHistogramVideoShotDetector( final Video<MBFImage> video,
109 			final int nGridElements )
110 	{
111 		super( video );
112 		this.nGridElements = nGridElements;
113 		this.lastHistogram = new double[nGridElements][nGridElements][];
114 		this.threshold = 0.2;
115 	}
116 
117 	/**
118 	 *	{@inheritDoc}
119 	 * 	@see org.openimaj.video.processing.shotdetector.VideoShotDetector#getInterframeDistance(org.openimaj.image.Image)
120 	 */
121 	@Override
122 	protected double getInterframeDistance( final MBFImage frame )
123 	{
124 		// The histogram distance at each element
125 		final double[][] avgHisto = new double[this.nGridElements][this.nGridElements];
126 
127 		// Work out the size of each grid element in pixels
128 		final int gw = frame.getWidth()  / this.nGridElements;
129 		final int gh = frame.getHeight() / this.nGridElements;
130 
131 		// Loop through the grid elements
132 		for( int y = 0; y < this.nGridElements; y++ )
133 		{
134 			for( int x = 0; x < this.nGridElements; x++ )
135 			{
136 				// Extract the local image
137 				final MBFImage img = frame.extractROI( x*gw, y*gh, gw, gh );
138 
139 				// Estimate the histogram
140 				this.histogramModel.estimateModel( img );
141 				final double[] histogram = this.histogramModel.
142 						histogram.asDoubleVector();
143 
144 				// If we have a grid element histogram to compare against,
145 				// we will implement the algorithm
146 				if( this.lastHistogram[y][x] != null )
147 				{
148 					final double dist = DoubleFVComparison.EUCLIDEAN.compare(
149 							histogram, this.lastHistogram[y][x] );
150 					avgHisto[y][x] = dist;
151 				}
152 
153 				// Store the histogram for this grid element for next time
154 				this.lastHistogram[y][x] = histogram;
155 			}
156 
157 		}
158 
159 		// We'll work now with a 1D array
160 		double[] flattenedAvgHisto = ArrayUtils.reshape( avgHisto );
161 
162 		// --- Calculate most similar and dissimilar tiles for boosting ---
163 		// Sort the avgHisto values retaining their original indices
164 		final int[] indices = new int[this.nGridElements*this.nGridElements];
165 
166 		// Sort the histogram distance array. The smallest values will end
167 		// up at the end of the array - the smallest values are the most similar.
168 		ArrayUtils.parallelQuicksortDescending(	flattenedAvgHisto,
169 				ArrayUtils.fill( indices ) );
170 
171 		// Create an array with the boost/limit factors in it
172 		final double[][] similarDissimilar =
173 				new double[this.nGridElements][this.nGridElements];
174 
175 		// Set the boost/limit factors
176 		for( int index = 0; index < indices.length; index++ )
177 		{
178 			double factor = 1;
179 			if( index < this.nGridElements * this.nGridElements * this.pcMostDissimilar )
180 					factor = this.limitingFactor;
181 			else
182 			if( index >= this.nGridElements * this.nGridElements * (1-this.pcMostSimilar) )
183 					factor = this.boostFactor;
184 			else	factor = 1;
185 
186 			final int y = indices[index] / this.nGridElements;
187 			final int x = indices[index] % this.nGridElements;
188 			similarDissimilar[y][x] = factor;
189 		}
190 
191 		// DEBUG
192 //		this.drawBoxes( frame, similarDissimilar );
193 
194 		// Boost the avgHisto values based on the distance measures
195 		for( int y = 0; y < this.nGridElements; y++ )
196 			for( int x = 0; x < this.nGridElements; x++ )
197 				avgHisto[y][x] *= similarDissimilar[y][x];
198 
199 		// Calculate the average histogram distance (over all the grid elements)
200 		flattenedAvgHisto = ArrayUtils.reshape( avgHisto );
201 		double avgDist = ArrayUtils.sumValues( flattenedAvgHisto );
202 		avgDist /= this.nGridElements * this.nGridElements;
203 
204 		// Calculate the stddev
205 		ArrayUtils.subtract( flattenedAvgHisto, avgDist );
206 		final double stdDev = Math.sqrt( ArrayUtils.sumValuesSquared(
207 				flattenedAvgHisto ) / (this.nGridElements*this.nGridElements) );
208 
209 		return stdDev;
210 	}
211 
212 	/**
213 	 * 	Draws the boxes to show movements.
214 	 *	@param img The image to draw to
215 	 */
216 	protected void drawBoxes( final MBFImage img, final double[][] sim )
217 	{
218 		final int gw = img.getWidth()  / this.nGridElements;
219 		final int gh = img.getHeight() / this.nGridElements;
220 		for( int y = 0; y < this.nGridElements; y++ )
221 		{
222 			for( int x = 0; x < this.nGridElements; x++ )
223 			{
224 				Float[] c = new Float[]{0f,0f,0f,0f};
225 				if( sim[y][x] == this.boostFactor )
226 						c = RGBColour.RED;
227 				else
228 				if( sim[y][x] == this.limitingFactor )
229 						c = RGBColour.BLUE;
230 				else	c = RGBColour.BLACK;
231 				img.drawShape( new Rectangle(x*gw,y*gh,gw,gh), c );
232 			}
233 		}
234 	}
235 }