001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030/** 031 * 032 */ 033package org.openimaj.demos.sandbox.video; 034 035import java.io.File; 036import java.util.ArrayList; 037import java.util.Collections; 038import java.util.List; 039 040import org.openimaj.demos.Demo; 041import org.openimaj.image.DisplayUtilities; 042import org.openimaj.image.MBFImage; 043import org.openimaj.image.colour.RGBColour; 044import org.openimaj.image.processing.face.detection.DetectedFace; 045import org.openimaj.image.processing.face.detection.HaarCascadeDetector; 046import org.openimaj.image.processing.resize.ResizeProcessor; 047import org.openimaj.math.geometry.shape.Rectangle; 048import org.openimaj.util.pair.IndependentPair; 049import org.openimaj.video.Video; 050import org.openimaj.video.processing.shotdetector.ShotBoundary; 051import org.openimaj.video.processing.shotdetector.HistogramVideoShotDetector; 052import org.openimaj.video.processing.timefinder.ObjectTimeFinder; 053import org.openimaj.video.processing.tracking.BasicMBFImageObjectTracker; 054import org.openimaj.video.timecode.HrsMinSecFrameTimecode; 055import org.openimaj.video.timecode.VideoTimecode; 056import org.openimaj.video.xuggle.XuggleVideo; 057 058/** 059 * Uses Jon's face keyframes (or at least the frame number from them) to 060 * make this visualisation. Requires the video from which they came too. 061 * 062 * @author David Dupplaw (dpd@ecs.soton.ac.uk) 063 * 064 * @created 14 Oct 2011 065 */ 066@Demo( 067 author = "David Dupplaw", 068 description = "Demonstrates the face range finder component which " + 069 "splits a video into shots, detects faces from the keyframe at the " + 070 "centre of the shot then tracks those faces " + 071 "through the video (back and forward) to give the timecodes between which " + 072 "faces appear in the video.", 073 keywords = { "face", "video", "tracking", "shot boundary" }, 074 title = "Face Range Finder" 075) 076public class VideoFaceRangeFinderDemo 077{ 078 /** The video we'll be reading from */ 079 private Video<MBFImage> video = null; 080 081 /** Set this to false if you simply want the face ranges to be printed */ 082 private boolean doNiceVis = true; 083 084 /** The number of frames in the video */ 085 private long nFrames = 0; 086 087 /** The shot boundaries found in the video, if we're looking */ 088 private List<ShotBoundary<MBFImage>> shotBoundaries = null; 089 090 /** The visualisation image */ 091 private MBFImage outputImage = null; 092 093 private int tnSize = 64; 094 095 /** 096 * 097 */ 098 public VideoFaceRangeFinderDemo() 099 { 100 // The video we're going to use 101 video = new XuggleVideo( new File( 102 "src/main/resources/org/openimaj/demos/rttr/07161859-rttr-16k-news10-rttr-16k.mpg" ) ); 103 104 // If we're going to do a nice visualisation, we'll do a shot boundary 105 // detection so that we can plot the timeline of the video 106 if( doNiceVis ) 107 { 108 // Work out the number of frames so that we can do a display 109 nFrames = video.countFrames(); 110 111 // Work out the shot boundaries (just for visualisation) 112 HistogramVideoShotDetector sd = new HistogramVideoShotDetector( video ); 113 sd.process(); 114 shotBoundaries = sd.getShotBoundaries(); 115 116 // Create the output image 117 int w = 1500; 118 int h = 150; 119 outputImage = new MBFImage( w, h, 3 ); 120 121 // Draw boxes into the output image to represent shots 122 Float[][] colours = new Float[][]{ 123 {0.7f,0.7f,0.8f}, 124 {0.5f,0.5f,0.6f} 125 }; 126 127 int i = 0; 128 ShotBoundary<MBFImage> last = null; 129 for( ShotBoundary<MBFImage> kf : shotBoundaries ) 130 { 131 if( last != null ) 132 { 133 int x = (int)(last.getTimecode().getFrameNumber() / (double)nFrames * w); 134 int xw = (int)(kf.getTimecode().getFrameNumber() / (double)nFrames * w); 135 outputImage.drawShapeFilled( 136 new Rectangle( x, 0, xw-x, h ), 137 colours[i++%colours.length]); 138 } 139 140 last = kf; 141 } 142 143 // Display the shots 144 DisplayUtilities.displayName( outputImage, "vis", true ); 145 } 146 147 // Get the frames in which we think there are faces 148 List<VideoTimecode> startFrames = getStartFrames(); 149 150 // Initialise our object trackers 151 ObjectTimeFinder of = new ObjectTimeFinder(); 152 BasicMBFImageObjectTracker objectTracker = new BasicMBFImageObjectTracker(); 153 154 // For each of the keyframes in which we think there are faces... 155 for( VideoTimecode keyframeTime : startFrames ) 156 { 157 // Skip to the keyframe time and get the current frame 158 System.out.println( "Seeking to frame "+keyframeTime ); 159 video.setCurrentFrameIndex( keyframeTime.getFrameNumber() ); 160 MBFImage image = video.getCurrentFrame(); 161 162 // Detect faces in this frame 163 HaarCascadeDetector faceDetector = new HaarCascadeDetector( 40 ); 164 List<DetectedFace> faces = faceDetector.detectFaces( image.flatten() ); 165 166 System.out.println( "In frame "+keyframeTime+" found "+faces.size()+" faces."); 167 168 // TODO: Check faces against keyframe 169 DetectedFace df = faces.get(0); 170 171 if( doNiceVis ) 172 { 173 int x = (int)(keyframeTime.getFrameNumber() / 174 (double)nFrames * outputImage.getWidth() ); 175 176 // Extract the face patch and draw it into the visualisation 177 MBFImage facePatch = image.extractROI( df.getBounds() ); 178 outputImage.drawImage( facePatch.process( 179 new ResizeProcessor(tnSize, tnSize, true) ), x-(tnSize/2), 0 ); 180 181 // Update the display 182 DisplayUtilities.displayName( outputImage, "vis", true ); 183 } 184 185 // Track the face 186 System.out.println( "Tracking face..." ); 187 IndependentPair<VideoTimecode, VideoTimecode> times = 188 of.trackObject( objectTracker, video, keyframeTime, df.getBounds(), null ); 189 190 System.out.println( "Got times: "+times ); 191 192 if( doNiceVis ) 193 { 194 int x = (int)(keyframeTime.getFrameNumber() / 195 (double)nFrames * outputImage.getWidth() ); 196 int x1 = (int)(times.firstObject().getFrameNumber() / 197 (double)nFrames * outputImage.getWidth() ); 198 int x2 = (int)(times.secondObject().getFrameNumber() / 199 (double)nFrames * outputImage.getWidth() ); 200 201 outputImage.drawLine( x, tnSize, x, tnSize+4, 2, RGBColour.RED ); 202 outputImage.drawLine( x1, tnSize+4, x2, tnSize+4, 2, RGBColour.RED ); 203 outputImage.drawLine( x1, tnSize+2, x1, tnSize+6, 1, RGBColour.RED ); 204 outputImage.drawLine( x2, tnSize+2, x2, tnSize+6, 1, RGBColour.RED ); 205 206 // Update the display 207 DisplayUtilities.displayName( outputImage, "vis", true ); 208 } 209 } 210 } 211 212 /** 213 * Returns the keyframes in which we think there are faces. 214 * @return 215 */ 216 private List<VideoTimecode> getStartFrames() 217 { 218 List<VideoTimecode> timecodes = new ArrayList<VideoTimecode>(); 219 220 // We're just going to read these from disk. 221 File[] files = new File( "src/main/resources/org/openimaj/demos/rttr/shots" ).listFiles(); 222 223 for( File f : files ) 224 { 225 String frameNumStr = f.getName().substring( 226 f.getName().lastIndexOf( '#' )+1, 227 f.getName().lastIndexOf( '.' ) ); 228 int frameNum = Integer.parseInt( frameNumStr ); 229 230 HrsMinSecFrameTimecode t = new HrsMinSecFrameTimecode( frameNum, 231 video.getFPS() ); 232 timecodes.add( t ); 233 } 234 235 Collections.sort( timecodes ); 236 237 return timecodes; 238 } 239 240 /** 241 * Default main 242 * @param args Command-line arguments 243 */ 244 public static void main( String[] args ) 245 { 246 new VideoFaceRangeFinderDemo(); 247 } 248}