001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030/**
031 *
032 */
033package org.openimaj.video.xuggle;
034
035import java.io.File;
036import java.io.IOException;
037import java.io.InputStream;
038import java.net.MalformedURLException;
039import java.net.URI;
040import java.net.URISyntaxException;
041import java.net.URL;
042import java.util.concurrent.TimeUnit;
043
044import org.apache.log4j.Logger;
045import org.openimaj.audio.AudioFormat;
046import org.openimaj.audio.AudioStream;
047import org.openimaj.audio.SampleChunk;
048import org.openimaj.audio.timecode.AudioTimecode;
049
050import com.xuggle.mediatool.IMediaReader;
051import com.xuggle.mediatool.MediaToolAdapter;
052import com.xuggle.mediatool.ToolFactory;
053import com.xuggle.mediatool.event.IAudioSamplesEvent;
054import com.xuggle.xuggler.Global;
055import com.xuggle.xuggler.IAudioSamples;
056import com.xuggle.xuggler.ICodec;
057import com.xuggle.xuggler.IContainer;
058import com.xuggle.xuggler.IError;
059import com.xuggle.xuggler.IStream;
060import com.xuggle.xuggler.IStreamCoder;
061import com.xuggle.xuggler.io.URLProtocolManager;
062
063/**
064 * A wrapper for the Xuggle audio decoding system into the OpenIMAJ audio
065 * system.
066 *
067 * @author David Dupplaw (dpd@ecs.soton.ac.uk)
068 * @created 8 Jun 2011
069 *
070 */
071public class XuggleAudio extends AudioStream
072{
073        static Logger logger = Logger.getLogger(XuggleAudio.class);
074
075        static {
076                URLProtocolManager.getManager().registerFactory("jar", new JarURLProtocolHandlerFactory());
077        }
078
079        /** The reader used to read the video */
080        private IMediaReader reader = null;
081
082        /** The stream index that we'll be reading from */
083        private int streamIndex = -1;
084
085        /** The current sample chunk - note this is reused */
086        private SampleChunk currentSamples = null;
087
088        /** Whether we've read a complete chunk */
089        private boolean chunkAvailable = false;
090
091        /** The timecode of the current sample chunk */
092        private final AudioTimecode currentTimecode = new AudioTimecode(0);
093
094        /** The length of the media */
095        private long length = -1;
096
097        /** The URL being read */
098        private final String url;
099
100        /** Whether to loop the file */
101        private final boolean loop;
102
103        /**
104         * Whether this class was constructed from a stream. Some functions are
105         * unavailable
106         */
107        private boolean constructedFromStream = false;
108
109        /**
110         *
111         *
112         * @author David Dupplaw (dpd@ecs.soton.ac.uk)
113         * @created 8 Jun 2011
114         *
115         */
116        protected class ChunkGetter extends MediaToolAdapter
117        {
118                /**
119                 * {@inheritDoc}
120                 *
121                 * @see com.xuggle.mediatool.MediaToolAdapter#onAudioSamples(com.xuggle.mediatool.event.IAudioSamplesEvent)
122                 */
123                @Override
124                public void onAudioSamples(final IAudioSamplesEvent event)
125                {
126                        // Get the samples
127                        final IAudioSamples aSamples = event.getAudioSamples();
128                        final byte[] rawBytes = aSamples.getData().
129                                        getByteArray(0, aSamples.getSize());
130                        XuggleAudio.this.currentSamples.setSamples(rawBytes);
131
132                        // Set the timecode of these samples
133                        // double timestampMillisecs =
134                        // rawBytes.length/format.getNumChannels() /
135                        // format.getSampleRateKHz();
136                        final long timestampMillisecs = TimeUnit.MILLISECONDS.convert(
137                                        event.getTimeStamp().longValue(), event.getTimeUnit());
138
139                        XuggleAudio.this.currentTimecode.setTimecodeInMilliseconds(
140                                        timestampMillisecs);
141
142                        XuggleAudio.this.currentSamples.setStartTimecode(
143                                        XuggleAudio.this.currentTimecode);
144
145                        XuggleAudio.this.currentSamples.getFormat().setNumChannels(
146                                        XuggleAudio.this.getFormat().getNumChannels());
147
148                        XuggleAudio.this.currentSamples.getFormat().setSigned(
149                                        XuggleAudio.this.getFormat().isSigned());
150
151                        XuggleAudio.this.currentSamples.getFormat().setBigEndian(
152                                        XuggleAudio.this.getFormat().isBigEndian());
153
154                        XuggleAudio.this.currentSamples.getFormat().setSampleRateKHz(
155                                        XuggleAudio.this.getFormat().getSampleRateKHz());
156
157                        XuggleAudio.this.chunkAvailable = true;
158                }
159        }
160
161        /**
162         * Default constructor that takes the file to read.
163         *
164         * @param file
165         *            The file to read.
166         */
167        public XuggleAudio(final File file)
168        {
169                this(file.toURI().toString(), false);
170        }
171
172        /**
173         * Default constructor that takes the file to read.
174         *
175         * @param file
176         *            The file to read.
177         * @param loop
178         *            Whether to loop indefinitely
179         */
180        public XuggleAudio(final File file, final boolean loop)
181        {
182                this(file.toURI().toString(), loop);
183        }
184
185        /**
186         * Default constructor that takes the location of a file to read. This can
187         * either be a filename or a URL.
188         *
189         * @param u
190         *            The URL of the file to read
191         */
192        public XuggleAudio(final URL u)
193        {
194                this(u.toString(), false);
195        }
196
197        /**
198         * Default constructor that takes the location of a file to read. This can
199         * either be a filename or a URL.
200         *
201         * @param u
202         *            The URL of the file to read
203         * @param loop
204         *            Whether to loop indefinitely
205         */
206        public XuggleAudio(final URL u, final boolean loop)
207        {
208                this(u.toString(), loop);
209        }
210
211        /**
212         * Default constructor that takes the location of a file to read. This can
213         * either be a filename or a URL.
214         *
215         * @param url
216         *            The URL of the file to read
217         */
218        public XuggleAudio(final String url)
219        {
220                this(url, false);
221        }
222
223        /**
224         * Default constructor that takes the location of a file to read. This can
225         * either be a filename or a URL. The second parameter determines whether
226         * the file will loop indefinitely. If so, {@link #nextSampleChunk()} will
227         * never return null; otherwise this method will return null at the end of
228         * the video.
229         *
230         * @param u
231         *            The URL of the file to read
232         * @param loop
233         *            Whether to loop indefinitely
234         */
235        public XuggleAudio(final String u, final boolean loop)
236        {
237                this.url = u;
238                this.loop = loop;
239                this.create(null);
240        }
241
242        /**
243         * Construct a xuggle audio object from the stream.
244         *
245         * @param stream
246         *            The stream
247         */
248        public XuggleAudio(final InputStream stream)
249        {
250                this.url = "stream://local";
251                this.loop = false;
252                this.constructedFromStream = true;
253                this.create(stream);
254        }
255
256        /**
257         * Create the Xuggler reader
258         *
259         * @param stream
260         *            Can be NULL; else the stream to create from.
261         */
262        private void create(final InputStream stream)
263        {
264                // If the reader is already open, we'll close it first and
265                // reinstantiate it.
266                if (this.reader != null && this.reader.isOpen())
267                {
268                        this.reader.close();
269                        this.reader = null;
270                }
271
272                // Check whether the string we have is a valid URI
273                IContainer container = null;
274                int openResult = 0;
275                try
276                {
277                        // Create the container to read our audio file
278                        container = IContainer.make();
279
280                        // If we have a stream, we'll create from the stream...
281                        if (stream != null)
282                        {
283                                openResult = container.open(stream, null, true, true);
284
285                                if (openResult < 0)
286                                        logger.info("XuggleAudio could not open InputStream to audio.");
287                        }
288                        // otherwise we'll use the URL in the class
289                        else
290                        {
291                                final URI uri = new URI(this.url);
292
293                                // If it's a valid URI, we'll try to open the container using
294                                // the URI string.
295                                openResult = container.open(uri.toString(),
296                                                IContainer.Type.READ, null, true, true);
297
298                                // If there was an error trying to open the container in this
299                                // way,
300                                // it may be that we have a resource URL (which ffmpeg doesn't
301                                // understand), so we'll try opening an InputStream to the
302                                // resource.
303                                if (openResult < 0)
304                                {
305                                        logger.trace("URL " + this.url + " could not be opened by ffmpeg. " +
306                                                        "Trying to open a stream to the URL instead.");
307                                        final InputStream is = uri.toURL().openStream();
308                                        openResult = container.open(is, null, true, true);
309
310                                        if (openResult < 0)
311                                        {
312                                                logger.error("Error opening container. Error " + openResult +
313                                                                " (" + IError.errorNumberToType(openResult).toString() + ")");
314                                                return;
315                                        }
316                                }
317                                else
318                                        logger.info("Opened XuggleAudio stream ok: " + openResult);
319                        }
320                } catch (final URISyntaxException e2)
321                {
322                        e2.printStackTrace();
323                        return;
324                } catch (final MalformedURLException e)
325                {
326                        e.printStackTrace();
327                        return;
328                } catch (final IOException e)
329                {
330                        e.printStackTrace();
331                        return;
332                }
333
334                // Set up a new reader using the container that reads the images.
335                this.reader = ToolFactory.makeReader(container);
336                this.reader.addListener(new ChunkGetter());
337                this.reader.setCloseOnEofOnly(!this.loop);
338
339                // Find the audio stream.
340                IStream s = null;
341                int i = 0;
342                while (i < container.getNumStreams())
343                {
344                        s = container.getStream(i);
345                        if (s != null &&
346                                        s.getStreamCoder().getCodecType() == ICodec.Type.CODEC_TYPE_AUDIO)
347                        {
348                                // Save the stream index so that we only get frames from
349                                // this stream in the FrameGetter
350                                this.streamIndex = i;
351                                break;
352                        }
353                        i++;
354                }
355                logger.info("Using audio stream " + this.streamIndex);
356
357                if (container.getDuration() == Global.NO_PTS)
358                        this.length = -1;
359                else
360                        this.length = (long) (s.getDuration() *
361                                        s.getTimeBase().getDouble() * 1000d);
362
363                // Get the coder for the audio stream
364                final IStreamCoder aAudioCoder = container.
365                                getStream(this.streamIndex).getStreamCoder();
366
367                logger.info("Using stream code: " + aAudioCoder);
368
369                // Create an audio format object suitable for the audio
370                // samples from Xuggle files
371                final AudioFormat af = new AudioFormat(
372                                (int) IAudioSamples.findSampleBitDepth(aAudioCoder.getSampleFormat()),
373                                aAudioCoder.getSampleRate() / 1000d,
374                                aAudioCoder.getChannels());
375                af.setSigned(true);
376                af.setBigEndian(false);
377                super.format = af;
378
379                logger.info("XuggleAudio using audio format: " + af);
380
381                this.currentSamples = new SampleChunk(af.clone());
382        }
383
384        // protected int retries = 0;
385        // protected int maxRetries = 0;
386        //
387        // /**
388        // * Set the maximum allowed number of retries in case of an error reading a
389        // * packet. Only use this on live streams; if you do it on a file-based
390        // * stream it might cause looping at the end of file.
391        // *
392        // * @param retries
393        // * maximum number of retries
394        // */
395        // public void setMaxRetries(int retries) {
396        // this.maxRetries = retries;
397        // }
398
399        /**
400         * {@inheritDoc}
401         *
402         * @see org.openimaj.audio.AudioStream#nextSampleChunk()
403         */
404        @Override
405        public SampleChunk nextSampleChunk()
406        {
407                try
408                {
409                        IError e = null;
410                        while ((e = this.reader.readPacket()) == null && !this.chunkAvailable)
411                                ;
412
413                        if (!this.chunkAvailable) {
414                                this.reader.close();
415                                this.reader = null;
416                                return null;
417                        }
418
419                        if (e != null)
420                        {
421                                this.reader.close();
422                                this.reader = null;
423
424                                // // We might be reading from a live stream & if we hit an
425                                // error
426                                // // we'll retry
427                                // if (e != null && e.getType() != IError.Type.ERROR_EOF)
428                                // {
429                                // logger.error("Got audio demux error " + e.getDescription());
430                                // this.create(null);
431                                // this.retries++;
432                                // }
433                                // logger.info("Closing audio stream " + this.url);
434                                return null;
435                        }
436
437                        this.chunkAvailable = false;
438                        return this.currentSamples;
439                } catch (final Exception e) {
440                }
441
442                return null;
443        }
444
445        /**
446         * {@inheritDoc}
447         *
448         * @see org.openimaj.audio.AudioStream#reset()
449         */
450        @Override
451        public void reset()
452        {
453                if (this.constructedFromStream)
454                {
455                        logger.info("Cannot reset a stream of audio.");
456                        return;
457                }
458
459                if (this.reader == null || this.reader.getContainer() == null)
460                        this.create(null);
461                else
462                        this.seek(0);
463        }
464
465        /**
466         * {@inheritDoc}
467         *
468         * @see org.openimaj.audio.AudioStream#getLength()
469         */
470        @Override
471        public long getLength()
472        {
473                return this.length;
474        }
475
476        /**
477         * {@inheritDoc}
478         *
479         * @see org.openimaj.audio.AudioStream#seek(long)
480         */
481        @Override
482        public void seek(final long timestamp)
483        {
484                if (this.constructedFromStream)
485                {
486                        logger.info("Cannot seek within a stream of audio.");
487                        return;
488                }
489
490                if (this.reader == null || this.reader.getContainer() == null)
491                        this.create(null);
492
493                // Convert from milliseconds to stream timestamps
494                final double timebase = this.reader.getContainer().getStream(
495                                this.streamIndex).getTimeBase().getDouble();
496                final long position = (long) (timestamp / timebase);
497
498                final long min = Math.max(0, position - 100);
499                final long max = position;
500
501                // logger.info( "Timebase: "+timebase+" of a second second");
502                // logger.info( "Position to seek to (timebase units): "+position
503                // );
504                // logger.info( "max: "+max+", min: "+min );
505
506                final int i = this.reader.getContainer().seekKeyFrame(this.streamIndex,
507                                min, position, max, 0);
508
509                // Check for errors
510                if (i < 0)
511                        logger.error("Audio seek error (" + i + "): " + IError.errorNumberToType(i));
512                else
513                        this.nextSampleChunk();
514        }
515
516        /**
517         * Close the audio stream.
518         */
519        public synchronized void close()
520        {
521                if (this.reader != null)
522                {
523                        synchronized (this.reader)
524                        {
525                                if (this.reader.isOpen())
526                                {
527                                        this.reader.close();
528                                        this.reader = null;
529                                }
530                        }
531                }
532        }
533}