1 /** 2 * Copyright (c) 2011, The University of Southampton and the individual contributors. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without modification, 6 * are permitted provided that the following conditions are met: 7 * 8 * * Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 11 * * Redistributions in binary form must reproduce the above copyright notice, 12 * this list of conditions and the following disclaimer in the documentation 13 * and/or other materials provided with the distribution. 14 * 15 * * Neither the name of the University of Southampton nor the names of its 16 * contributors may be used to endorse or promote products derived from this 17 * software without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 23 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 24 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 26 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 28 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 /** 31 * 32 */ 33 package org.openimaj.video.processing.timefinder; 34 35 import java.util.List; 36 37 import org.openimaj.image.Image; 38 import org.openimaj.math.geometry.shape.Rectangle; 39 import org.openimaj.util.pair.IndependentPair; 40 import org.openimaj.video.Video; 41 import org.openimaj.video.VideoCache; 42 import org.openimaj.video.processing.tracking.ObjectTracker; 43 import org.openimaj.video.timecode.HrsMinSecFrameTimecode; 44 import org.openimaj.video.timecode.VideoTimecode; 45 46 /** 47 * This class is a general class for finding the time range in which a specific 48 * object is found within a video. Given a seed frame in which the object appears, 49 * this class will track forwards and backwards in the video stream to find when 50 * the object disappears. It is then able to provide the timecodes at which the 51 * object can be found within the video. 52 * 53 * @author David Dupplaw (dpd@ecs.soton.ac.uk) 54 * 55 * @created 14 Oct 2011 56 */ 57 public class ObjectTimeFinder 58 { 59 /** 60 * An interface for objects that wish to be informed of object tracking 61 * events during the time finder's execution. 62 * 63 * @author David Dupplaw (dpd@ecs.soton.ac.uk) 64 * 65 * @param <O> Type of object tracked 66 * @param <I> Type of {@link Image} 67 * @created 7 Nov 2011 68 */ 69 public interface TimeFinderListener<O,I> 70 { 71 /** 72 * Called when an object is tracked in the video. 73 * 74 * @param objects The objects being tracked 75 * @param time The timecode of the frame in which the object was found. 76 * @param frame The frame in which the object was found. 77 */ 78 public void objectTracked( List<O> objects, VideoTimecode time, I frame ); 79 } 80 81 /** 82 * Given a video, a keyframe (timecode) and a region of the image, 83 * this method will attempt to track the the contents of the rectangle 84 * from the given frame, back and forth to find the place 85 * at which the object appears and disappears from the video. 86 * 87 * @param <I> The type of image returned from the video 88 * @param <O> The type of object being tracked 89 * @param objectTracker The object tracker that will track the object for us 90 * @param video The video 91 * @param keyframeTime The keyframe timecode to start at 92 * @param bounds The bounding box of the object you wish to track 93 * @param listener A listener for object tracking events 94 * @return An {@link IndependentPair} of timecodes delineating the start 95 * and end of the video which contains the object 96 */ 97 public <I extends Image<?,I>,O> 98 IndependentPair<VideoTimecode,VideoTimecode> trackObject( 99 ObjectTracker<O,I> objectTracker, Video<I> video, 100 VideoTimecode keyframeTime, Rectangle bounds, 101 TimeFinderListener<O,I> listener ) 102 { 103 // Set up the initial start and end timecodes. We'll update these 104 // as we scan through the video 105 HrsMinSecFrameTimecode startTime = new HrsMinSecFrameTimecode( 106 keyframeTime.getFrameNumber(), video.getFPS() ); 107 HrsMinSecFrameTimecode endTime = new HrsMinSecFrameTimecode( 108 keyframeTime.getFrameNumber(), video.getFPS() ); 109 110 // Now set the video to the start frame 111 video.setCurrentFrameIndex( startTime.getFrameNumber() ); 112 I image = video.getCurrentFrame(); 113 I keyframeImage = image.clone(); 114 115 // Initialise the tracking with the start frame 116 objectTracker.initialiseTracking( bounds, image ); 117 118 // Now we'll scan forwards in the video to find out when the object 119 // disappears from view 120 boolean foundObject = true; 121 while( foundObject && image != null ) 122 { 123 // Track the object 124 List<O> objects = objectTracker.trackObject( image ); 125 126 // If we've found no objects in this frame, then we've lost 127 // sight of the object and must stop 128 if( objects.size() == 0 ) 129 foundObject = false; 130 else 131 { 132 // Move on to the next frame 133 image = video.getNextFrame(); 134 135 // Update the end time 136 endTime.setFrameNumber( video.getCurrentFrameIndex() ); 137 138 if( listener != null ) 139 listener.objectTracked( objects, endTime, image ); 140 } 141 } 142 143 // Reinitialise the object tracker to our start point 144 objectTracker.initialiseTracking( bounds, keyframeImage ); 145 foundObject = true; 146 147 // Now we're going to scan backwards through the video. However, scanning 148 // backwards can be really slow, so what we're going to do is scan back 149 // by a big chunk and then cache that chunk, before we track back through 150 // it frame by frame. This reduces the number of times we need to do the 151 // expensive backwards track. 152 int nFramesToJumpBack = (int)video.getFPS()*2; // We'll go 2 seconds at a time 153 154 // currentTimecode will give the start of the chunk 155 HrsMinSecFrameTimecode currentTimecode = new HrsMinSecFrameTimecode( 156 Math.max( startTime.getFrameNumber() - nFramesToJumpBack, 0 ), 157 video.getFPS() ); 158 159 // Now keep looping until we either lose the object or 160 // we hit the start of the video 161 while( foundObject && currentTimecode.getFrameNumber() >= 0 ) 162 { 163 // Slightly confusingly here, the startTime is the end of the chunk 164 VideoCache<I> vc = VideoCache.cacheVideo( video, currentTimecode, startTime ); 165 166 // Now track back through the cached video 167 for( int n = 1; n <= vc.getNumberOfFrames(); n++ ) 168 { 169 // Track backwards through the frames 170 image = vc.getFrame( vc.getNumberOfFrames()-n ); 171 172 // Track the object 173 List<O> objects = objectTracker.trackObject( image ); 174 175 // If we've found no objects in this frame, then we've lost 176 // sight of the object and must stop 177 if( objects.size() == 0 ) 178 { 179 foundObject = false; 180 break; 181 } 182 else 183 { 184 // Update the start time 185 startTime.setFrameNumber( startTime.getFrameNumber()-1 ); 186 187 if( listener != null ) 188 listener.objectTracked( objects, startTime, image ); 189 } 190 } 191 192 // If we're already at frame zero, we should end this loop 193 if( currentTimecode.getFrameNumber() == 0 ) 194 foundObject = false; 195 196 // Update the chunk timecode if we need to keep going 197 currentTimecode.setFrameNumber( 198 Math.max( startTime.getFrameNumber() - nFramesToJumpBack, 0 ) ); 199 } 200 201 // Return the videos 202 return new IndependentPair<VideoTimecode,VideoTimecode>( startTime, endTime ); 203 } 204 }