View Javadoc

1   /**
2    * Copyright (c) 2011, The University of Southampton and the individual contributors.
3    * All rights reserved.
4    *
5    * Redistribution and use in source and binary forms, with or without modification,
6    * are permitted provided that the following conditions are met:
7    *
8    *   * 	Redistributions of source code must retain the above copyright notice,
9    * 	this list of conditions and the following disclaimer.
10   *
11   *   *	Redistributions in binary form must reproduce the above copyright notice,
12   * 	this list of conditions and the following disclaimer in the documentation
13   * 	and/or other materials provided with the distribution.
14   *
15   *   *	Neither the name of the University of Southampton nor the names of its
16   * 	contributors may be used to endorse or promote products derived from this
17   * 	software without specific prior written permission.
18   *
19   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
20   * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21   * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22   * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
23   * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24   * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
26   * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27   * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28   * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29   */
30  /**
31   *
32   */
33  package org.openimaj.video.xuggle;
34  
35  import java.io.File;
36  import java.io.IOException;
37  import java.io.InputStream;
38  import java.net.MalformedURLException;
39  import java.net.URI;
40  import java.net.URISyntaxException;
41  import java.net.URL;
42  import java.util.concurrent.TimeUnit;
43  
44  import org.apache.log4j.Logger;
45  import org.openimaj.audio.AudioFormat;
46  import org.openimaj.audio.AudioStream;
47  import org.openimaj.audio.SampleChunk;
48  import org.openimaj.audio.timecode.AudioTimecode;
49  
50  import com.xuggle.mediatool.IMediaReader;
51  import com.xuggle.mediatool.MediaToolAdapter;
52  import com.xuggle.mediatool.ToolFactory;
53  import com.xuggle.mediatool.event.IAudioSamplesEvent;
54  import com.xuggle.xuggler.Global;
55  import com.xuggle.xuggler.IAudioSamples;
56  import com.xuggle.xuggler.ICodec;
57  import com.xuggle.xuggler.IContainer;
58  import com.xuggle.xuggler.IError;
59  import com.xuggle.xuggler.IStream;
60  import com.xuggle.xuggler.IStreamCoder;
61  import com.xuggle.xuggler.io.URLProtocolManager;
62  
63  /**
64   * A wrapper for the Xuggle audio decoding system into the OpenIMAJ audio
65   * system.
66   *
67   * @author David Dupplaw (dpd@ecs.soton.ac.uk)
68   * @created 8 Jun 2011
69   *
70   */
71  public class XuggleAudio extends AudioStream
72  {
73  	static Logger logger = Logger.getLogger(XuggleAudio.class);
74  
75  	static {
76  		URLProtocolManager.getManager().registerFactory("jar", new JarURLProtocolHandlerFactory());
77  	}
78  
79  	/** The reader used to read the video */
80  	private IMediaReader reader = null;
81  
82  	/** The stream index that we'll be reading from */
83  	private int streamIndex = -1;
84  
85  	/** The current sample chunk - note this is reused */
86  	private SampleChunk currentSamples = null;
87  
88  	/** Whether we've read a complete chunk */
89  	private boolean chunkAvailable = false;
90  
91  	/** The timecode of the current sample chunk */
92  	private final AudioTimecode currentTimecode = new AudioTimecode(0);
93  
94  	/** The length of the media */
95  	private long length = -1;
96  
97  	/** The URL being read */
98  	private final String url;
99  
100 	/** Whether to loop the file */
101 	private final boolean loop;
102 
103 	/**
104 	 * Whether this class was constructed from a stream. Some functions are
105 	 * unavailable
106 	 */
107 	private boolean constructedFromStream = false;
108 
109 	/**
110 	 *
111 	 *
112 	 * @author David Dupplaw (dpd@ecs.soton.ac.uk)
113 	 * @created 8 Jun 2011
114 	 *
115 	 */
116 	protected class ChunkGetter extends MediaToolAdapter
117 	{
118 		/**
119 		 * {@inheritDoc}
120 		 *
121 		 * @see com.xuggle.mediatool.MediaToolAdapter#onAudioSamples(com.xuggle.mediatool.event.IAudioSamplesEvent)
122 		 */
123 		@Override
124 		public void onAudioSamples(final IAudioSamplesEvent event)
125 		{
126 			// Get the samples
127 			final IAudioSamples aSamples = event.getAudioSamples();
128 			final byte[] rawBytes = aSamples.getData().
129 					getByteArray(0, aSamples.getSize());
130 			XuggleAudio.this.currentSamples.setSamples(rawBytes);
131 
132 			// Set the timecode of these samples
133 			// double timestampMillisecs =
134 			// rawBytes.length/format.getNumChannels() /
135 			// format.getSampleRateKHz();
136 			final long timestampMillisecs = TimeUnit.MILLISECONDS.convert(
137 					event.getTimeStamp().longValue(), event.getTimeUnit());
138 
139 			XuggleAudio.this.currentTimecode.setTimecodeInMilliseconds(
140 					timestampMillisecs);
141 
142 			XuggleAudio.this.currentSamples.setStartTimecode(
143 					XuggleAudio.this.currentTimecode);
144 
145 			XuggleAudio.this.currentSamples.getFormat().setNumChannels(
146 					XuggleAudio.this.getFormat().getNumChannels());
147 
148 			XuggleAudio.this.currentSamples.getFormat().setSigned(
149 					XuggleAudio.this.getFormat().isSigned());
150 
151 			XuggleAudio.this.currentSamples.getFormat().setBigEndian(
152 					XuggleAudio.this.getFormat().isBigEndian());
153 
154 			XuggleAudio.this.currentSamples.getFormat().setSampleRateKHz(
155 					XuggleAudio.this.getFormat().getSampleRateKHz());
156 
157 			XuggleAudio.this.chunkAvailable = true;
158 		}
159 	}
160 
161 	/**
162 	 * Default constructor that takes the file to read.
163 	 *
164 	 * @param file
165 	 *            The file to read.
166 	 */
167 	public XuggleAudio(final File file)
168 	{
169 		this(file.toURI().toString(), false);
170 	}
171 
172 	/**
173 	 * Default constructor that takes the file to read.
174 	 *
175 	 * @param file
176 	 *            The file to read.
177 	 * @param loop
178 	 *            Whether to loop indefinitely
179 	 */
180 	public XuggleAudio(final File file, final boolean loop)
181 	{
182 		this(file.toURI().toString(), loop);
183 	}
184 
185 	/**
186 	 * Default constructor that takes the location of a file to read. This can
187 	 * either be a filename or a URL.
188 	 *
189 	 * @param u
190 	 *            The URL of the file to read
191 	 */
192 	public XuggleAudio(final URL u)
193 	{
194 		this(u.toString(), false);
195 	}
196 
197 	/**
198 	 * Default constructor that takes the location of a file to read. This can
199 	 * either be a filename or a URL.
200 	 *
201 	 * @param u
202 	 *            The URL of the file to read
203 	 * @param loop
204 	 *            Whether to loop indefinitely
205 	 */
206 	public XuggleAudio(final URL u, final boolean loop)
207 	{
208 		this(u.toString(), loop);
209 	}
210 
211 	/**
212 	 * Default constructor that takes the location of a file to read. This can
213 	 * either be a filename or a URL.
214 	 *
215 	 * @param url
216 	 *            The URL of the file to read
217 	 */
218 	public XuggleAudio(final String url)
219 	{
220 		this(url, false);
221 	}
222 
223 	/**
224 	 * Default constructor that takes the location of a file to read. This can
225 	 * either be a filename or a URL. The second parameter determines whether
226 	 * the file will loop indefinitely. If so, {@link #nextSampleChunk()} will
227 	 * never return null; otherwise this method will return null at the end of
228 	 * the video.
229 	 *
230 	 * @param u
231 	 *            The URL of the file to read
232 	 * @param loop
233 	 *            Whether to loop indefinitely
234 	 */
235 	public XuggleAudio(final String u, final boolean loop)
236 	{
237 		this.url = u;
238 		this.loop = loop;
239 		this.create(null);
240 	}
241 
242 	/**
243 	 * Construct a xuggle audio object from the stream.
244 	 *
245 	 * @param stream
246 	 *            The stream
247 	 */
248 	public XuggleAudio(final InputStream stream)
249 	{
250 		this.url = "stream://local";
251 		this.loop = false;
252 		this.constructedFromStream = true;
253 		this.create(stream);
254 	}
255 
256 	/**
257 	 * Create the Xuggler reader
258 	 *
259 	 * @param stream
260 	 *            Can be NULL; else the stream to create from.
261 	 */
262 	private void create(final InputStream stream)
263 	{
264 		// If the reader is already open, we'll close it first and
265 		// reinstantiate it.
266 		if (this.reader != null && this.reader.isOpen())
267 		{
268 			this.reader.close();
269 			this.reader = null;
270 		}
271 
272 		// Check whether the string we have is a valid URI
273 		IContainer container = null;
274 		int openResult = 0;
275 		try
276 		{
277 			// Create the container to read our audio file
278 			container = IContainer.make();
279 
280 			// If we have a stream, we'll create from the stream...
281 			if (stream != null)
282 			{
283 				openResult = container.open(stream, null, true, true);
284 
285 				if (openResult < 0)
286 					logger.info("XuggleAudio could not open InputStream to audio.");
287 			}
288 			// otherwise we'll use the URL in the class
289 			else
290 			{
291 				final URI uri = new URI(this.url);
292 
293 				// If it's a valid URI, we'll try to open the container using
294 				// the URI string.
295 				openResult = container.open(uri.toString(),
296 						IContainer.Type.READ, null, true, true);
297 
298 				// If there was an error trying to open the container in this
299 				// way,
300 				// it may be that we have a resource URL (which ffmpeg doesn't
301 				// understand), so we'll try opening an InputStream to the
302 				// resource.
303 				if (openResult < 0)
304 				{
305 					logger.trace("URL " + this.url + " could not be opened by ffmpeg. " +
306 							"Trying to open a stream to the URL instead.");
307 					final InputStream is = uri.toURL().openStream();
308 					openResult = container.open(is, null, true, true);
309 
310 					if (openResult < 0)
311 					{
312 						logger.error("Error opening container. Error " + openResult +
313 								" (" + IError.errorNumberToType(openResult).toString() + ")");
314 						return;
315 					}
316 				}
317 				else
318 					logger.info("Opened XuggleAudio stream ok: " + openResult);
319 			}
320 		} catch (final URISyntaxException e2)
321 		{
322 			e2.printStackTrace();
323 			return;
324 		} catch (final MalformedURLException e)
325 		{
326 			e.printStackTrace();
327 			return;
328 		} catch (final IOException e)
329 		{
330 			e.printStackTrace();
331 			return;
332 		}
333 
334 		// Set up a new reader using the container that reads the images.
335 		this.reader = ToolFactory.makeReader(container);
336 		this.reader.addListener(new ChunkGetter());
337 		this.reader.setCloseOnEofOnly(!this.loop);
338 
339 		// Find the audio stream.
340 		IStream s = null;
341 		int i = 0;
342 		while (i < container.getNumStreams())
343 		{
344 			s = container.getStream(i);
345 			if (s != null &&
346 					s.getStreamCoder().getCodecType() == ICodec.Type.CODEC_TYPE_AUDIO)
347 			{
348 				// Save the stream index so that we only get frames from
349 				// this stream in the FrameGetter
350 				this.streamIndex = i;
351 				break;
352 			}
353 			i++;
354 		}
355 		logger.info("Using audio stream " + this.streamIndex);
356 
357 		if (container.getDuration() == Global.NO_PTS)
358 			this.length = -1;
359 		else
360 			this.length = (long) (s.getDuration() *
361 					s.getTimeBase().getDouble() * 1000d);
362 
363 		// Get the coder for the audio stream
364 		final IStreamCoder aAudioCoder = container.
365 				getStream(this.streamIndex).getStreamCoder();
366 
367 		logger.info("Using stream code: " + aAudioCoder);
368 
369 		// Create an audio format object suitable for the audio
370 		// samples from Xuggle files
371 		final AudioFormat af = new AudioFormat(
372 				(int) IAudioSamples.findSampleBitDepth(aAudioCoder.getSampleFormat()),
373 				aAudioCoder.getSampleRate() / 1000d,
374 				aAudioCoder.getChannels());
375 		af.setSigned(true);
376 		af.setBigEndian(false);
377 		super.format = af;
378 
379 		logger.info("XuggleAudio using audio format: " + af);
380 
381 		this.currentSamples = new SampleChunk(af.clone());
382 	}
383 
384 	// protected int retries = 0;
385 	// protected int maxRetries = 0;
386 	//
387 	// /**
388 	// * Set the maximum allowed number of retries in case of an error reading a
389 	// * packet. Only use this on live streams; if you do it on a file-based
390 	// * stream it might cause looping at the end of file.
391 	// *
392 	// * @param retries
393 	// * maximum number of retries
394 	// */
395 	// public void setMaxRetries(int retries) {
396 	// this.maxRetries = retries;
397 	// }
398 
399 	/**
400 	 * {@inheritDoc}
401 	 *
402 	 * @see org.openimaj.audio.AudioStream#nextSampleChunk()
403 	 */
404 	@Override
405 	public SampleChunk nextSampleChunk()
406 	{
407 		try
408 		{
409 			IError e = null;
410 			while ((e = this.reader.readPacket()) == null && !this.chunkAvailable)
411 				;
412 
413 			if (!this.chunkAvailable) {
414 				this.reader.close();
415 				this.reader = null;
416 				return null;
417 			}
418 
419 			if (e != null)
420 			{
421 				this.reader.close();
422 				this.reader = null;
423 
424 				// // We might be reading from a live stream & if we hit an
425 				// error
426 				// // we'll retry
427 				// if (e != null && e.getType() != IError.Type.ERROR_EOF)
428 				// {
429 				// logger.error("Got audio demux error " + e.getDescription());
430 				// this.create(null);
431 				// this.retries++;
432 				// }
433 				// logger.info("Closing audio stream " + this.url);
434 				return null;
435 			}
436 
437 			this.chunkAvailable = false;
438 			return this.currentSamples;
439 		} catch (final Exception e) {
440 		}
441 
442 		return null;
443 	}
444 
445 	/**
446 	 * {@inheritDoc}
447 	 *
448 	 * @see org.openimaj.audio.AudioStream#reset()
449 	 */
450 	@Override
451 	public void reset()
452 	{
453 		if (this.constructedFromStream)
454 		{
455 			logger.info("Cannot reset a stream of audio.");
456 			return;
457 		}
458 
459 		if (this.reader == null || this.reader.getContainer() == null)
460 			this.create(null);
461 		else
462 			this.seek(0);
463 	}
464 
465 	/**
466 	 * {@inheritDoc}
467 	 *
468 	 * @see org.openimaj.audio.AudioStream#getLength()
469 	 */
470 	@Override
471 	public long getLength()
472 	{
473 		return this.length;
474 	}
475 
476 	/**
477 	 * {@inheritDoc}
478 	 *
479 	 * @see org.openimaj.audio.AudioStream#seek(long)
480 	 */
481 	@Override
482 	public void seek(final long timestamp)
483 	{
484 		if (this.constructedFromStream)
485 		{
486 			logger.info("Cannot seek within a stream of audio.");
487 			return;
488 		}
489 
490 		if (this.reader == null || this.reader.getContainer() == null)
491 			this.create(null);
492 
493 		// Convert from milliseconds to stream timestamps
494 		final double timebase = this.reader.getContainer().getStream(
495 				this.streamIndex).getTimeBase().getDouble();
496 		final long position = (long) (timestamp / timebase);
497 
498 		final long min = Math.max(0, position - 100);
499 		final long max = position;
500 
501 		// logger.info( "Timebase: "+timebase+" of a second second");
502 		// logger.info( "Position to seek to (timebase units): "+position
503 		// );
504 		// logger.info( "max: "+max+", min: "+min );
505 
506 		final int i = this.reader.getContainer().seekKeyFrame(this.streamIndex,
507 				min, position, max, 0);
508 
509 		// Check for errors
510 		if (i < 0)
511 			logger.error("Audio seek error (" + i + "): " + IError.errorNumberToType(i));
512 		else
513 			this.nextSampleChunk();
514 	}
515 
516 	/**
517 	 * Close the audio stream.
518 	 */
519 	public synchronized void close()
520 	{
521 		if (this.reader != null)
522 		{
523 			synchronized (this.reader)
524 			{
525 				if (this.reader.isOpen())
526 				{
527 					this.reader.close();
528 					this.reader = null;
529 				}
530 			}
531 		}
532 	}
533 }