001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.video.capture;
031
032import java.util.List;
033
034import javax.swing.SwingUtilities;
035
036import org.bridj.Pointer;
037import org.openimaj.image.ImageUtilities;
038import org.openimaj.image.MBFImage;
039import org.openimaj.image.colour.ColourSpace;
040import org.openimaj.video.Video;
041import org.openimaj.video.VideoDisplay;
042
043/**
044 * VideoCapture is a type of {@link Video} that can capture live video streams
045 * from a webcam or other video device. On OSX and Windows, this is completely
046 * dependency-free and no extra software needs to be installed. On linux you
047 * need to have video4linux installed.
048 * <p>
049 * <strong>Environment variables</strong>
050 * <ul>
051 * <li>The environment variable OPENIMAJ_GRABBER_VERBOSE can be set (to any
052 * non-zero length value) on windows to make the native library print lots of
053 * debugging information</li>
054 * <li>The environment variable OPENIMAJ_GRABBER_READ can be set on linux to
055 * force the native library use v4l in read-mode rather than through memory
056 * mapping the device. This can be useful if you have lots of cameras attached
057 * as it reduces the bandwidth required.</li>
058 * </ul>
059 * <p>
060 * <strong>System properties</strong>
061 * <ul>
062 * <li>The system property with the name given by
063 * {@link #DEFAULT_DEVICE_NUMBER_PROPERTY} can be used to set the default
064 * capture device and can either be a device number, or device identifer string.
065 * See {@link #VideoCapture(int, int)} for more details.</li>
066 * </ul>
067 * 
068 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
069 */
070public class VideoCapture extends Video<MBFImage> {
071        /**
072         * The property key for overriding the default device number.
073         */
074        public static final String DEFAULT_DEVICE_NUMBER_PROPERTY = "openimaj.grabber.camera";
075
076        private OpenIMAJGrabber grabber;
077        private MBFImage frame;
078        private int width;
079        private int height;
080        private boolean isStopped = true;
081        private double fps = 25;
082
083        /** The timestamp at which the capture (session) was started */
084        private long captureStartedTimestamp = 0;
085
086        /** The timestamp of the current image */
087        private long currentTimestamp = 0;
088
089        /**
090         * Construct a VideoCapture instance with the requested width and height.
091         * The default video device will be used. The actual height and width of the
092         * captured frames may not equal the requested size if the underlying
093         * platform-specific grabber is not able to honor the request. The actual
094         * size can be inspected through the {@link #getWidth()} and
095         * {@link #getHeight()} methods.
096         * <p>
097         * The default device is usually the first device listed by
098         * {@link #getVideoDevices()}, however this is down to the underlying native
099         * libraries and operating system. The
100         * {@link #DEFAULT_DEVICE_NUMBER_PROPERTY} system property allows the
101         * selection of the default device to be overridden. The property value can
102         * be an integer representing the index of the default device in the list
103         * produced by {@link #DEFAULT_DEVICE_NUMBER_PROPERTY} or a {@link String}
104         * that includes part of the device identifier. In the case of a String
105         * value, the device with the identifier that first contains the value is
106         * selected.
107         * 
108         * @param width
109         *            the requested video width
110         * @param height
111         *            the requested video height
112         * @throws VideoCaptureException
113         *             if no webcam is found, or there is a problem opening it
114         */
115        public VideoCapture(int width, int height) throws VideoCaptureException {
116                // on 32 bit osx a deadlock seems to occur between the
117                // initialisation of the native library and AWT. This
118                // seems to fix it...
119                final List<Device> devices = VideoCapture.getVideoDevices();
120
121                Device defaultDevice = null;
122                final String defaultDeviceStr = System.getProperty(DEFAULT_DEVICE_NUMBER_PROPERTY);
123                if (defaultDeviceStr != null) {
124                        try {
125                                final int i = Integer.parseInt(defaultDeviceStr);
126
127                                if (i >= 0 && i < devices.size()) {
128                                        defaultDevice = devices.get(i);
129                                } else {
130                                        System.err.println("Warning: The " + DEFAULT_DEVICE_NUMBER_PROPERTY
131                                                        + " property setting is out of range (0..<" + devices.size() + ") and will be ignored.");
132                                        System.err.println("Valid devices are:");
133                                        for (int x = 0; x < devices.size(); x++)
134                                                System.err.println(x + " : " + devices.get(x).getIdentifierStr());
135                                }
136                        } catch (final NumberFormatException e) {
137                                for (final Device d : devices) {
138                                        if (d.getIdentifierStr().contains(defaultDeviceStr)) {
139                                                defaultDevice = d;
140                                                break;
141                                        }
142                                }
143
144                                if (defaultDevice == null) {
145                                        System.err.println("Warning: The device name given by the " + DEFAULT_DEVICE_NUMBER_PROPERTY
146                                                        + " property (" + defaultDeviceStr + ") setting was not found and has been ignored.");
147                                        System.err.println("Valid devices are:");
148                                        for (int x = 0; x < devices.size(); x++)
149                                                System.err.println(x + " : " + devices.get(x).getIdentifierStr());
150                                }
151                        }
152                }
153
154                grabber = new OpenIMAJGrabber();
155
156                if (defaultDevice == null) {
157                        if (!startSession(width, height, 0))
158                                throw new VideoCaptureException("No webcams found!");
159                } else {
160                        if (!startSession(width, height, 0, defaultDevice))
161                                throw new VideoCaptureException("An error occured opening the capture device");
162                }
163        }
164
165        /**
166         * Construct a VideoCapture instance with the requested width and height
167         * using the specified video device. The actual height and width of the
168         * captured frames may not equal the requested size if the underlying
169         * platform-specific grabber is not able to honor the request. The actual
170         * size can be inspected through the {@link #getWidth()} and
171         * {@link #getHeight()} methods.
172         * 
173         * @param width
174         *            the requested video width.
175         * @param height
176         *            the requested video height.
177         * @param device
178         *            the requested video device.
179         * @throws VideoCaptureException
180         *             if there is a problem opening the webcam
181         */
182        public VideoCapture(int width, int height, Device device) throws VideoCaptureException {
183                grabber = new OpenIMAJGrabber();
184                if (!startSession(width, height, 0, device))
185                        throw new VideoCaptureException("An error occured opening the capture device");
186        }
187
188        /**
189         * Construct a VideoCapture instance with the requested width and height
190         * using the specified video device. The actual height and width of the
191         * captured frames may not equal the requested size if the underlying
192         * platform-specific grabber is not able to honor the request. The actual
193         * size can be inspected through the {@link #getWidth()} and
194         * {@link #getHeight()} methods.
195         * 
196         * @param width
197         *            the requested video width.
198         * @param height
199         *            the requested video height.
200         * @param fps
201         *            the requested frame rate
202         * @param device
203         *            the requested video device.
204         * @throws VideoCaptureException
205         *             if there is a problem opening the webcam
206         */
207        public VideoCapture(int width, int height, double fps, Device device) throws VideoCaptureException {
208                this.fps = fps;
209                grabber = new OpenIMAJGrabber();
210                if (!startSession(width, height, fps, device))
211                        throw new VideoCaptureException("An error occured opening the capture device");
212        }
213
214        /**
215         * Get a list of all compatible video devices attached to the machine.
216         * 
217         * @return a list of devices.
218         */
219        public static List<Device> getVideoDevices() {
220                final OpenIMAJGrabber grabber = new OpenIMAJGrabber();
221                final DeviceList list = grabber.getVideoDevices().get();
222
223                return list.asArrayList();
224        }
225
226        protected synchronized boolean startSession(final int requestedWidth, final int requestedHeight, double requestedFPS,
227                        Device device)
228        {
229                final int millisPerFrame = requestedFPS == 0 ? 0 : (int) (1000.0 / requestedFPS);
230
231                if (grabber.startSession(requestedWidth, requestedHeight, millisPerFrame, Pointer.getPointer(device))) {
232                        width = grabber.getWidth();
233                        height = grabber.getHeight();
234                        frame = new MBFImage(width, height, ColourSpace.RGB);
235
236                        isStopped = false;
237                        return true;
238                }
239                return false;
240        }
241
242        protected synchronized boolean startSession(int requestedWidth, int requestedHeight, double requestedFPS) {
243                final int millisPerFrame = requestedFPS == 0 ? 0 : (int) (1000.0 / requestedFPS);
244
245                if (grabber.startSession(requestedWidth, requestedHeight, millisPerFrame)) {
246                        width = grabber.getWidth();
247                        height = grabber.getHeight();
248                        frame = new MBFImage(width, height, ColourSpace.RGB);
249
250                        isStopped = false;
251                        return true;
252                }
253                return false;
254        }
255
256        /**
257         * Stop the video capture system. Once stopped, it can only be started again
258         * by constructing a new instance of VideoCapture.
259         */
260        public synchronized void stopCapture() {
261                if (!isStopped) {
262                        isStopped = true;
263                        grabber.stopSession();
264                }
265        }
266
267        @Override
268        public MBFImage getCurrentFrame() {
269                return frame;
270        }
271
272        /**
273         * {@inheritDoc}
274         * 
275         * @throws RuntimeException
276         *             wrapping a {@link VideoCaptureException} with the message
277         *             "Timed out waiting for next frame" if there is a timeout
278         *             waiting for the next frame. This could potentially be caught
279         *             and ignored (i.e. the frame is dropped).
280         * @throws RuntimeException
281         *             wrapping a {@link VideoCaptureException} with the message
282         *             "Error occurred getting next frame" if there is an error the
283         *             next frame. Currently this can only occur on linux.
284         */
285        @Override
286        public synchronized MBFImage getNextFrame() {
287                if (isStopped)
288                        return frame;
289
290                final int err = grabber.nextFrame();
291                if (err == -1)
292                        throw new RuntimeException(new VideoCaptureException("Timed out waiting for next frame"));
293                if (err < -1)
294                        throw new RuntimeException(new VideoCaptureException("Error occurred getting next frame"));
295
296                final Pointer<Byte> data = grabber.getImage();
297                if (data == null) {
298                        return frame;
299                }
300
301                final byte[] d = data.getBytes(width * height * 3);
302                final float[][] r = frame.bands.get(0).pixels;
303                final float[][] g = frame.bands.get(1).pixels;
304                final float[][] b = frame.bands.get(2).pixels;
305
306                for (int i = 0, y = 0; y < height; y++) {
307                        for (int x = 0; x < width; x++, i += 3) {
308                                final int red = d[i + 0] & 0xFF;
309                                final int green = d[i + 1] & 0xFF;
310                                final int blue = d[i + 2] & 0xFF;
311                                r[y][x] = ImageUtilities.BYTE_TO_FLOAT_LUT[red];
312                                g[y][x] = ImageUtilities.BYTE_TO_FLOAT_LUT[green];
313                                b[y][x] = ImageUtilities.BYTE_TO_FLOAT_LUT[blue];
314                        }
315                }
316
317                super.currentFrame++;
318
319                if (captureStartedTimestamp == 0)
320                        captureStartedTimestamp = System.currentTimeMillis();
321                currentTimestamp = System.currentTimeMillis() - captureStartedTimestamp;
322
323                return frame;
324        }
325
326        @Override
327        public boolean hasNextFrame() {
328                return true;
329        }
330
331        @Override
332        public long countFrames() {
333                return -1;
334        }
335
336        /**
337         * Test main method. Lists the available devices, and then opens the first
338         * and second capture devices if they are available and displays their
339         * video.
340         * 
341         * @param args
342         *            ignored.
343         * @throws VideoCaptureException
344         */
345        public static void main(String[] args) throws VideoCaptureException {
346                final List<Device> devices = VideoCapture.getVideoDevices();
347                for (final Device d : devices)
348                        System.out.println(d);
349
350                if (devices.size() == 1) {
351                        final VideoCapture grabber1 = new VideoCapture(640, 480, devices.get(0));
352                        VideoDisplay.createVideoDisplay(grabber1);
353                } else {
354                        final int w = 320;
355                        final int h = 240;
356                        final double rate = 10.0;
357
358                        for (int y = 0, i = 0; y < 3; y++) {
359                                for (int x = 0; x < 3 && i < devices.size(); x++, i++) {
360                                        final VideoCapture grabber2 = new VideoCapture(w, h, rate, devices.get(i));
361                                        final VideoDisplay<MBFImage> disp = VideoDisplay.createVideoDisplay(grabber2);
362                                        SwingUtilities.getRoot(disp.getScreen()).setLocation(320 * x, 240 * y);
363                                }
364                        }
365                }
366        }
367
368        /**
369         * {@inheritDoc}
370         * 
371         * @see org.openimaj.video.Video#getWidth()
372         */
373        @Override
374        public int getWidth()
375        {
376                return width;
377        }
378
379        /**
380         * {@inheritDoc}
381         * 
382         * @see org.openimaj.video.Video#getHeight()
383         */
384        @Override
385        public int getHeight()
386        {
387                return height;
388        }
389
390        @Override
391        public void reset()
392        {
393                stopCapture();
394                startSession(width, height, fps);
395        }
396
397        /**
398         * {@inheritDoc}
399         * 
400         * @see org.openimaj.video.Video#getTimeStamp()
401         */
402        @Override
403        public long getTimeStamp()
404        {
405                return currentTimestamp;
406                // return (long)(super.currentFrame * 1000 / this.fps);
407        }
408
409        /*
410         * (non-Javadoc)
411         * 
412         * @see org.openimaj.video.Video#setCurrentFrameIndex(long)
413         */
414        @Override
415        public void setCurrentFrameIndex(long newFrame) {
416                // do nothing
417        }
418
419        /**
420         * {@inheritDoc}
421         * 
422         * @see org.openimaj.video.Video#getFPS()
423         */
424        @Override
425        public double getFPS()
426        {
427                return fps;
428        }
429
430        /**
431         * Set the number of frames per second.
432         * 
433         * @param fps
434         *            The number of frames per second.
435         */
436        public void setFPS(double fps)
437        {
438                this.fps = fps;
439        }
440
441        @Override
442        public void close() {
443                this.stopCapture();
444        }
445
446        @Override
447        protected void finalize() throws Throwable {
448                this.close();
449        }
450}