1 /**
2 * Copyright (c) 2011, The University of Southampton and the individual contributors.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without modification,
6 * are permitted provided that the following conditions are met:
7 *
8 * * Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 *
11 * * Redistributions in binary form must reproduce the above copyright notice,
12 * this list of conditions and the following disclaimer in the documentation
13 * and/or other materials provided with the distribution.
14 *
15 * * Neither the name of the University of Southampton nor the names of its
16 * contributors may be used to endorse or promote products derived from this
17 * software without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
23 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
26 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30 /**
31 *
32 */
33 package org.openimaj.video.processing.timefinder;
34
35 import java.util.List;
36
37 import org.openimaj.image.Image;
38 import org.openimaj.math.geometry.shape.Rectangle;
39 import org.openimaj.util.pair.IndependentPair;
40 import org.openimaj.video.Video;
41 import org.openimaj.video.VideoCache;
42 import org.openimaj.video.processing.tracking.ObjectTracker;
43 import org.openimaj.video.timecode.HrsMinSecFrameTimecode;
44 import org.openimaj.video.timecode.VideoTimecode;
45
46 /**
47 * This class is a general class for finding the time range in which a specific
48 * object is found within a video. Given a seed frame in which the object appears,
49 * this class will track forwards and backwards in the video stream to find when
50 * the object disappears. It is then able to provide the timecodes at which the
51 * object can be found within the video.
52 *
53 * @author David Dupplaw (dpd@ecs.soton.ac.uk)
54 *
55 * @created 14 Oct 2011
56 */
57 public class ObjectTimeFinder
58 {
59 /**
60 * An interface for objects that wish to be informed of object tracking
61 * events during the time finder's execution.
62 *
63 * @author David Dupplaw (dpd@ecs.soton.ac.uk)
64 *
65 * @param <O> Type of object tracked
66 * @param <I> Type of {@link Image}
67 * @created 7 Nov 2011
68 */
69 public interface TimeFinderListener<O,I>
70 {
71 /**
72 * Called when an object is tracked in the video.
73 *
74 * @param objects The objects being tracked
75 * @param time The timecode of the frame in which the object was found.
76 * @param frame The frame in which the object was found.
77 */
78 public void objectTracked( List<O> objects, VideoTimecode time, I frame );
79 }
80
81 /**
82 * Given a video, a keyframe (timecode) and a region of the image,
83 * this method will attempt to track the the contents of the rectangle
84 * from the given frame, back and forth to find the place
85 * at which the object appears and disappears from the video.
86 *
87 * @param <I> The type of image returned from the video
88 * @param <O> The type of object being tracked
89 * @param objectTracker The object tracker that will track the object for us
90 * @param video The video
91 * @param keyframeTime The keyframe timecode to start at
92 * @param bounds The bounding box of the object you wish to track
93 * @param listener A listener for object tracking events
94 * @return An {@link IndependentPair} of timecodes delineating the start
95 * and end of the video which contains the object
96 */
97 public <I extends Image<?,I>,O>
98 IndependentPair<VideoTimecode,VideoTimecode> trackObject(
99 ObjectTracker<O,I> objectTracker, Video<I> video,
100 VideoTimecode keyframeTime, Rectangle bounds,
101 TimeFinderListener<O,I> listener )
102 {
103 // Set up the initial start and end timecodes. We'll update these
104 // as we scan through the video
105 HrsMinSecFrameTimecode startTime = new HrsMinSecFrameTimecode(
106 keyframeTime.getFrameNumber(), video.getFPS() );
107 HrsMinSecFrameTimecode endTime = new HrsMinSecFrameTimecode(
108 keyframeTime.getFrameNumber(), video.getFPS() );
109
110 // Now set the video to the start frame
111 video.setCurrentFrameIndex( startTime.getFrameNumber() );
112 I image = video.getCurrentFrame();
113 I keyframeImage = image.clone();
114
115 // Initialise the tracking with the start frame
116 objectTracker.initialiseTracking( bounds, image );
117
118 // Now we'll scan forwards in the video to find out when the object
119 // disappears from view
120 boolean foundObject = true;
121 while( foundObject && image != null )
122 {
123 // Track the object
124 List<O> objects = objectTracker.trackObject( image );
125
126 // If we've found no objects in this frame, then we've lost
127 // sight of the object and must stop
128 if( objects.size() == 0 )
129 foundObject = false;
130 else
131 {
132 // Move on to the next frame
133 image = video.getNextFrame();
134
135 // Update the end time
136 endTime.setFrameNumber( video.getCurrentFrameIndex() );
137
138 if( listener != null )
139 listener.objectTracked( objects, endTime, image );
140 }
141 }
142
143 // Reinitialise the object tracker to our start point
144 objectTracker.initialiseTracking( bounds, keyframeImage );
145 foundObject = true;
146
147 // Now we're going to scan backwards through the video. However, scanning
148 // backwards can be really slow, so what we're going to do is scan back
149 // by a big chunk and then cache that chunk, before we track back through
150 // it frame by frame. This reduces the number of times we need to do the
151 // expensive backwards track.
152 int nFramesToJumpBack = (int)video.getFPS()*2; // We'll go 2 seconds at a time
153
154 // currentTimecode will give the start of the chunk
155 HrsMinSecFrameTimecode currentTimecode = new HrsMinSecFrameTimecode(
156 Math.max( startTime.getFrameNumber() - nFramesToJumpBack, 0 ),
157 video.getFPS() );
158
159 // Now keep looping until we either lose the object or
160 // we hit the start of the video
161 while( foundObject && currentTimecode.getFrameNumber() >= 0 )
162 {
163 // Slightly confusingly here, the startTime is the end of the chunk
164 VideoCache<I> vc = VideoCache.cacheVideo( video, currentTimecode, startTime );
165
166 // Now track back through the cached video
167 for( int n = 1; n <= vc.getNumberOfFrames(); n++ )
168 {
169 // Track backwards through the frames
170 image = vc.getFrame( vc.getNumberOfFrames()-n );
171
172 // Track the object
173 List<O> objects = objectTracker.trackObject( image );
174
175 // If we've found no objects in this frame, then we've lost
176 // sight of the object and must stop
177 if( objects.size() == 0 )
178 {
179 foundObject = false;
180 break;
181 }
182 else
183 {
184 // Update the start time
185 startTime.setFrameNumber( startTime.getFrameNumber()-1 );
186
187 if( listener != null )
188 listener.objectTracked( objects, startTime, image );
189 }
190 }
191
192 // If we're already at frame zero, we should end this loop
193 if( currentTimecode.getFrameNumber() == 0 )
194 foundObject = false;
195
196 // Update the chunk timecode if we need to keep going
197 currentTimecode.setFrameNumber(
198 Math.max( startTime.getFrameNumber() - nFramesToJumpBack, 0 ) );
199 }
200
201 // Return the videos
202 return new IndependentPair<VideoTimecode,VideoTimecode>( startTime, endTime );
203 }
204 }