001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030/**
031 * 
032 */
033package org.openimaj.demos.sandbox.video;
034
035import java.io.File;
036import java.util.ArrayList;
037import java.util.Collections;
038import java.util.List;
039
040import org.openimaj.demos.Demo;
041import org.openimaj.image.DisplayUtilities;
042import org.openimaj.image.MBFImage;
043import org.openimaj.image.colour.RGBColour;
044import org.openimaj.image.processing.face.detection.DetectedFace;
045import org.openimaj.image.processing.face.detection.HaarCascadeDetector;
046import org.openimaj.image.processing.resize.ResizeProcessor;
047import org.openimaj.math.geometry.shape.Rectangle;
048import org.openimaj.util.pair.IndependentPair;
049import org.openimaj.video.Video;
050import org.openimaj.video.processing.shotdetector.ShotBoundary;
051import org.openimaj.video.processing.shotdetector.HistogramVideoShotDetector;
052import org.openimaj.video.processing.timefinder.ObjectTimeFinder;
053import org.openimaj.video.processing.tracking.BasicMBFImageObjectTracker;
054import org.openimaj.video.timecode.HrsMinSecFrameTimecode;
055import org.openimaj.video.timecode.VideoTimecode;
056import org.openimaj.video.xuggle.XuggleVideo;
057
058/**
059 *      Uses Jon's face keyframes (or at least the frame number from them) to
060 *      make this visualisation. Requires the video from which they came too.
061 *
062 *      @author David Dupplaw (dpd@ecs.soton.ac.uk)
063 *      
064 *      @created 14 Oct 2011
065 */
066@Demo(
067        author = "David Dupplaw", 
068        description = "Demonstrates the face range finder component which " +
069                        "splits a video into shots, detects faces from the keyframe at the " +
070                        "centre of the shot then tracks those faces " +
071                        "through the video (back and forward) to give the timecodes between which " +
072                        "faces appear in the video.", 
073        keywords = { "face", "video", "tracking", "shot boundary" }, 
074        title = "Face Range Finder"
075)
076public class VideoFaceRangeFinderDemo 
077{
078        /** The video we'll be reading from */
079        private Video<MBFImage> video = null;
080        
081        /** Set this to false if you simply want the face ranges to be printed */
082        private boolean doNiceVis = true;
083        
084        /** The number of frames in the video */
085        private long nFrames = 0;
086
087        /** The shot boundaries found in the video, if we're looking */
088        private List<ShotBoundary<MBFImage>> shotBoundaries = null;
089
090        /** The visualisation image */
091        private MBFImage outputImage = null;
092        
093        private int tnSize = 64;
094        
095        /**
096         * 
097         */
098        public VideoFaceRangeFinderDemo()
099        {
100                // The video we're going to use
101                video = new XuggleVideo( new File(
102                        "src/main/resources/org/openimaj/demos/rttr/07161859-rttr-16k-news10-rttr-16k.mpg" ) );
103        
104                // If we're going to do a nice visualisation, we'll do a shot boundary
105                // detection so that we can plot the timeline of the video
106                if( doNiceVis )
107                {
108                        // Work out the number of frames so that we can do a display
109                        nFrames = video.countFrames();
110                        
111                        // Work out the shot boundaries (just for visualisation)
112                        HistogramVideoShotDetector sd = new HistogramVideoShotDetector( video );
113                        sd.process();
114                        shotBoundaries = sd.getShotBoundaries();
115
116                        // Create the output image
117                        int w = 1500;
118                        int h = 150;
119                        outputImage  = new MBFImage( w, h, 3 );
120                        
121                        // Draw boxes into the output image to represent shots
122                        Float[][] colours = new Float[][]{
123                                        {0.7f,0.7f,0.8f},
124                                        {0.5f,0.5f,0.6f}
125                        };
126                        
127                        int i = 0;
128                        ShotBoundary<MBFImage> last = null;
129                        for( ShotBoundary<MBFImage> kf : shotBoundaries )
130                        {
131                                if( last != null )
132                                {
133                                        int x  = (int)(last.getTimecode().getFrameNumber() / (double)nFrames * w);
134                                        int xw = (int)(kf.getTimecode().getFrameNumber() / (double)nFrames * w);
135                                        outputImage.drawShapeFilled( 
136                                                new Rectangle( x, 0, xw-x, h ), 
137                                                colours[i++%colours.length]);
138                                }
139                                
140                                last = kf;
141                        }
142                        
143                        // Display the shots
144                        DisplayUtilities.displayName( outputImage, "vis", true );
145                }
146                
147                // Get the frames in which we think there are faces
148                List<VideoTimecode> startFrames = getStartFrames();
149                
150                // Initialise our object trackers
151                ObjectTimeFinder of = new ObjectTimeFinder();
152                BasicMBFImageObjectTracker objectTracker = new BasicMBFImageObjectTracker();
153                
154                // For each of the keyframes in which we think there are faces...
155                for( VideoTimecode keyframeTime : startFrames )
156                {
157                        // Skip to the keyframe time and get the current frame
158                        System.out.println( "Seeking to frame "+keyframeTime );
159                        video.setCurrentFrameIndex( keyframeTime.getFrameNumber() );
160                        MBFImage image = video.getCurrentFrame();
161                        
162                        // Detect faces in this frame
163                        HaarCascadeDetector faceDetector = new HaarCascadeDetector( 40 );
164                        List<DetectedFace> faces = faceDetector.detectFaces( image.flatten() );
165                        
166                        System.out.println( "In frame "+keyframeTime+" found "+faces.size()+" faces.");
167
168                        // TODO: Check faces against keyframe
169                        DetectedFace df = faces.get(0);
170                        
171                        if( doNiceVis )
172                        {
173                                int x  = (int)(keyframeTime.getFrameNumber() / 
174                                                (double)nFrames * outputImage.getWidth() );
175
176                                // Extract the face patch and draw it into the visualisation
177                                MBFImage facePatch = image.extractROI( df.getBounds() );
178                                outputImage.drawImage( facePatch.process( 
179                                                new ResizeProcessor(tnSize, tnSize, true) ), x-(tnSize/2), 0 );
180                                
181                                // Update the display
182                                DisplayUtilities.displayName( outputImage, "vis", true );
183                        }
184                        
185                        // Track the face
186                        System.out.println( "Tracking face..." );
187                        IndependentPair<VideoTimecode, VideoTimecode> times = 
188                                of.trackObject( objectTracker, video, keyframeTime, df.getBounds(), null );
189                        
190                        System.out.println( "Got times: "+times );
191                        
192                        if( doNiceVis )
193                        {
194                                int x  = (int)(keyframeTime.getFrameNumber() / 
195                                                (double)nFrames * outputImage.getWidth() );
196                                int x1  = (int)(times.firstObject().getFrameNumber() / 
197                                                (double)nFrames * outputImage.getWidth() );
198                                int x2  = (int)(times.secondObject().getFrameNumber() / 
199                                                (double)nFrames * outputImage.getWidth() );
200                                
201                                outputImage.drawLine(  x,   tnSize,  x, tnSize+4, 2, RGBColour.RED );
202                                outputImage.drawLine( x1, tnSize+4, x2, tnSize+4, 2, RGBColour.RED );
203                                outputImage.drawLine( x1, tnSize+2, x1, tnSize+6, 1, RGBColour.RED );
204                                outputImage.drawLine( x2, tnSize+2, x2, tnSize+6, 1, RGBColour.RED );
205
206                                // Update the display
207                                DisplayUtilities.displayName( outputImage, "vis", true );
208                        }
209                }
210        }
211        
212        /**
213         *      Returns the keyframes in which we think there are faces.
214         *      @return
215         */
216        private List<VideoTimecode> getStartFrames()
217        {
218                List<VideoTimecode> timecodes = new ArrayList<VideoTimecode>();
219                
220                // We're just going to read these from disk.
221                File[] files = new File( "src/main/resources/org/openimaj/demos/rttr/shots" ).listFiles();
222
223                for( File f : files )
224                {
225                        String frameNumStr = f.getName().substring( 
226                                        f.getName().lastIndexOf( '#' )+1,
227                                        f.getName().lastIndexOf( '.' ) );
228                        int frameNum = Integer.parseInt( frameNumStr );
229                        
230                        HrsMinSecFrameTimecode t = new HrsMinSecFrameTimecode( frameNum, 
231                                        video.getFPS() );
232                        timecodes.add( t );
233                }
234                
235                Collections.sort( timecodes );
236                
237                return timecodes;
238        }
239        
240        /**
241         *      Default main
242         *  @param args Command-line arguments
243         */
244        public static void main( String[] args )
245        {
246                new VideoFaceRangeFinderDemo();
247        }
248}