001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030/** 031 * 032 */ 033package org.openimaj.video.processing.timefinder; 034 035import java.util.List; 036 037import org.openimaj.image.Image; 038import org.openimaj.math.geometry.shape.Rectangle; 039import org.openimaj.util.pair.IndependentPair; 040import org.openimaj.video.Video; 041import org.openimaj.video.VideoCache; 042import org.openimaj.video.processing.tracking.ObjectTracker; 043import org.openimaj.video.timecode.HrsMinSecFrameTimecode; 044import org.openimaj.video.timecode.VideoTimecode; 045 046/** 047 * This class is a general class for finding the time range in which a specific 048 * object is found within a video. Given a seed frame in which the object appears, 049 * this class will track forwards and backwards in the video stream to find when 050 * the object disappears. It is then able to provide the timecodes at which the 051 * object can be found within the video. 052 * 053 * @author David Dupplaw (dpd@ecs.soton.ac.uk) 054 * 055 * @created 14 Oct 2011 056 */ 057public class ObjectTimeFinder 058{ 059 /** 060 * An interface for objects that wish to be informed of object tracking 061 * events during the time finder's execution. 062 * 063 * @author David Dupplaw (dpd@ecs.soton.ac.uk) 064 * 065 * @param <O> Type of object tracked 066 * @param <I> Type of {@link Image} 067 * @created 7 Nov 2011 068 */ 069 public interface TimeFinderListener<O,I> 070 { 071 /** 072 * Called when an object is tracked in the video. 073 * 074 * @param objects The objects being tracked 075 * @param time The timecode of the frame in which the object was found. 076 * @param frame The frame in which the object was found. 077 */ 078 public void objectTracked( List<O> objects, VideoTimecode time, I frame ); 079 } 080 081 /** 082 * Given a video, a keyframe (timecode) and a region of the image, 083 * this method will attempt to track the the contents of the rectangle 084 * from the given frame, back and forth to find the place 085 * at which the object appears and disappears from the video. 086 * 087 * @param <I> The type of image returned from the video 088 * @param <O> The type of object being tracked 089 * @param objectTracker The object tracker that will track the object for us 090 * @param video The video 091 * @param keyframeTime The keyframe timecode to start at 092 * @param bounds The bounding box of the object you wish to track 093 * @param listener A listener for object tracking events 094 * @return An {@link IndependentPair} of timecodes delineating the start 095 * and end of the video which contains the object 096 */ 097 public <I extends Image<?,I>,O> 098 IndependentPair<VideoTimecode,VideoTimecode> trackObject( 099 ObjectTracker<O,I> objectTracker, Video<I> video, 100 VideoTimecode keyframeTime, Rectangle bounds, 101 TimeFinderListener<O,I> listener ) 102 { 103 // Set up the initial start and end timecodes. We'll update these 104 // as we scan through the video 105 HrsMinSecFrameTimecode startTime = new HrsMinSecFrameTimecode( 106 keyframeTime.getFrameNumber(), video.getFPS() ); 107 HrsMinSecFrameTimecode endTime = new HrsMinSecFrameTimecode( 108 keyframeTime.getFrameNumber(), video.getFPS() ); 109 110 // Now set the video to the start frame 111 video.setCurrentFrameIndex( startTime.getFrameNumber() ); 112 I image = video.getCurrentFrame(); 113 I keyframeImage = image.clone(); 114 115 // Initialise the tracking with the start frame 116 objectTracker.initialiseTracking( bounds, image ); 117 118 // Now we'll scan forwards in the video to find out when the object 119 // disappears from view 120 boolean foundObject = true; 121 while( foundObject && image != null ) 122 { 123 // Track the object 124 List<O> objects = objectTracker.trackObject( image ); 125 126 // If we've found no objects in this frame, then we've lost 127 // sight of the object and must stop 128 if( objects.size() == 0 ) 129 foundObject = false; 130 else 131 { 132 // Move on to the next frame 133 image = video.getNextFrame(); 134 135 // Update the end time 136 endTime.setFrameNumber( video.getCurrentFrameIndex() ); 137 138 if( listener != null ) 139 listener.objectTracked( objects, endTime, image ); 140 } 141 } 142 143 // Reinitialise the object tracker to our start point 144 objectTracker.initialiseTracking( bounds, keyframeImage ); 145 foundObject = true; 146 147 // Now we're going to scan backwards through the video. However, scanning 148 // backwards can be really slow, so what we're going to do is scan back 149 // by a big chunk and then cache that chunk, before we track back through 150 // it frame by frame. This reduces the number of times we need to do the 151 // expensive backwards track. 152 int nFramesToJumpBack = (int)video.getFPS()*2; // We'll go 2 seconds at a time 153 154 // currentTimecode will give the start of the chunk 155 HrsMinSecFrameTimecode currentTimecode = new HrsMinSecFrameTimecode( 156 Math.max( startTime.getFrameNumber() - nFramesToJumpBack, 0 ), 157 video.getFPS() ); 158 159 // Now keep looping until we either lose the object or 160 // we hit the start of the video 161 while( foundObject && currentTimecode.getFrameNumber() >= 0 ) 162 { 163 // Slightly confusingly here, the startTime is the end of the chunk 164 VideoCache<I> vc = VideoCache.cacheVideo( video, currentTimecode, startTime ); 165 166 // Now track back through the cached video 167 for( int n = 1; n <= vc.getNumberOfFrames(); n++ ) 168 { 169 // Track backwards through the frames 170 image = vc.getFrame( vc.getNumberOfFrames()-n ); 171 172 // Track the object 173 List<O> objects = objectTracker.trackObject( image ); 174 175 // If we've found no objects in this frame, then we've lost 176 // sight of the object and must stop 177 if( objects.size() == 0 ) 178 { 179 foundObject = false; 180 break; 181 } 182 else 183 { 184 // Update the start time 185 startTime.setFrameNumber( startTime.getFrameNumber()-1 ); 186 187 if( listener != null ) 188 listener.objectTracked( objects, startTime, image ); 189 } 190 } 191 192 // If we're already at frame zero, we should end this loop 193 if( currentTimecode.getFrameNumber() == 0 ) 194 foundObject = false; 195 196 // Update the chunk timecode if we need to keep going 197 currentTimecode.setFrameNumber( 198 Math.max( startTime.getFrameNumber() - nFramesToJumpBack, 0 ) ); 199 } 200 201 // Return the videos 202 return new IndependentPair<VideoTimecode,VideoTimecode>( startTime, endTime ); 203 } 204}