001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030/** 031 * 032 */ 033package org.openimaj.video.xuggle; 034 035import java.io.File; 036import java.io.IOException; 037import java.io.InputStream; 038import java.net.MalformedURLException; 039import java.net.URI; 040import java.net.URISyntaxException; 041import java.net.URL; 042import java.util.concurrent.TimeUnit; 043 044import org.apache.log4j.Logger; 045import org.openimaj.audio.AudioFormat; 046import org.openimaj.audio.AudioStream; 047import org.openimaj.audio.SampleChunk; 048import org.openimaj.audio.timecode.AudioTimecode; 049 050import com.xuggle.mediatool.IMediaReader; 051import com.xuggle.mediatool.MediaToolAdapter; 052import com.xuggle.mediatool.ToolFactory; 053import com.xuggle.mediatool.event.IAudioSamplesEvent; 054import com.xuggle.xuggler.Global; 055import com.xuggle.xuggler.IAudioSamples; 056import com.xuggle.xuggler.ICodec; 057import com.xuggle.xuggler.IContainer; 058import com.xuggle.xuggler.IError; 059import com.xuggle.xuggler.IStream; 060import com.xuggle.xuggler.IStreamCoder; 061import com.xuggle.xuggler.io.URLProtocolManager; 062 063/** 064 * A wrapper for the Xuggle audio decoding system into the OpenIMAJ audio 065 * system. 066 * 067 * @author David Dupplaw (dpd@ecs.soton.ac.uk) 068 * @created 8 Jun 2011 069 * 070 */ 071public class XuggleAudio extends AudioStream 072{ 073 static Logger logger = Logger.getLogger(XuggleAudio.class); 074 075 static { 076 URLProtocolManager.getManager().registerFactory("jar", new JarURLProtocolHandlerFactory()); 077 } 078 079 /** The reader used to read the video */ 080 private IMediaReader reader = null; 081 082 /** The stream index that we'll be reading from */ 083 private int streamIndex = -1; 084 085 /** The current sample chunk - note this is reused */ 086 private SampleChunk currentSamples = null; 087 088 /** Whether we've read a complete chunk */ 089 private boolean chunkAvailable = false; 090 091 /** The timecode of the current sample chunk */ 092 private final AudioTimecode currentTimecode = new AudioTimecode(0); 093 094 /** The length of the media */ 095 private long length = -1; 096 097 /** The URL being read */ 098 private final String url; 099 100 /** Whether to loop the file */ 101 private final boolean loop; 102 103 /** 104 * Whether this class was constructed from a stream. Some functions are 105 * unavailable 106 */ 107 private boolean constructedFromStream = false; 108 109 /** 110 * 111 * 112 * @author David Dupplaw (dpd@ecs.soton.ac.uk) 113 * @created 8 Jun 2011 114 * 115 */ 116 protected class ChunkGetter extends MediaToolAdapter 117 { 118 /** 119 * {@inheritDoc} 120 * 121 * @see com.xuggle.mediatool.MediaToolAdapter#onAudioSamples(com.xuggle.mediatool.event.IAudioSamplesEvent) 122 */ 123 @Override 124 public void onAudioSamples(final IAudioSamplesEvent event) 125 { 126 // Get the samples 127 final IAudioSamples aSamples = event.getAudioSamples(); 128 final byte[] rawBytes = aSamples.getData(). 129 getByteArray(0, aSamples.getSize()); 130 XuggleAudio.this.currentSamples.setSamples(rawBytes); 131 132 // Set the timecode of these samples 133 // double timestampMillisecs = 134 // rawBytes.length/format.getNumChannels() / 135 // format.getSampleRateKHz(); 136 final long timestampMillisecs = TimeUnit.MILLISECONDS.convert( 137 event.getTimeStamp().longValue(), event.getTimeUnit()); 138 139 XuggleAudio.this.currentTimecode.setTimecodeInMilliseconds( 140 timestampMillisecs); 141 142 XuggleAudio.this.currentSamples.setStartTimecode( 143 XuggleAudio.this.currentTimecode); 144 145 XuggleAudio.this.currentSamples.getFormat().setNumChannels( 146 XuggleAudio.this.getFormat().getNumChannels()); 147 148 XuggleAudio.this.currentSamples.getFormat().setSigned( 149 XuggleAudio.this.getFormat().isSigned()); 150 151 XuggleAudio.this.currentSamples.getFormat().setBigEndian( 152 XuggleAudio.this.getFormat().isBigEndian()); 153 154 XuggleAudio.this.currentSamples.getFormat().setSampleRateKHz( 155 XuggleAudio.this.getFormat().getSampleRateKHz()); 156 157 XuggleAudio.this.chunkAvailable = true; 158 } 159 } 160 161 /** 162 * Default constructor that takes the file to read. 163 * 164 * @param file 165 * The file to read. 166 */ 167 public XuggleAudio(final File file) 168 { 169 this(file.toURI().toString(), false); 170 } 171 172 /** 173 * Default constructor that takes the file to read. 174 * 175 * @param file 176 * The file to read. 177 * @param loop 178 * Whether to loop indefinitely 179 */ 180 public XuggleAudio(final File file, final boolean loop) 181 { 182 this(file.toURI().toString(), loop); 183 } 184 185 /** 186 * Default constructor that takes the location of a file to read. This can 187 * either be a filename or a URL. 188 * 189 * @param u 190 * The URL of the file to read 191 */ 192 public XuggleAudio(final URL u) 193 { 194 this(u.toString(), false); 195 } 196 197 /** 198 * Default constructor that takes the location of a file to read. This can 199 * either be a filename or a URL. 200 * 201 * @param u 202 * The URL of the file to read 203 * @param loop 204 * Whether to loop indefinitely 205 */ 206 public XuggleAudio(final URL u, final boolean loop) 207 { 208 this(u.toString(), loop); 209 } 210 211 /** 212 * Default constructor that takes the location of a file to read. This can 213 * either be a filename or a URL. 214 * 215 * @param url 216 * The URL of the file to read 217 */ 218 public XuggleAudio(final String url) 219 { 220 this(url, false); 221 } 222 223 /** 224 * Default constructor that takes the location of a file to read. This can 225 * either be a filename or a URL. The second parameter determines whether 226 * the file will loop indefinitely. If so, {@link #nextSampleChunk()} will 227 * never return null; otherwise this method will return null at the end of 228 * the video. 229 * 230 * @param u 231 * The URL of the file to read 232 * @param loop 233 * Whether to loop indefinitely 234 */ 235 public XuggleAudio(final String u, final boolean loop) 236 { 237 this.url = u; 238 this.loop = loop; 239 this.create(null); 240 } 241 242 /** 243 * Construct a xuggle audio object from the stream. 244 * 245 * @param stream 246 * The stream 247 */ 248 public XuggleAudio(final InputStream stream) 249 { 250 this.url = "stream://local"; 251 this.loop = false; 252 this.constructedFromStream = true; 253 this.create(stream); 254 } 255 256 /** 257 * Create the Xuggler reader 258 * 259 * @param stream 260 * Can be NULL; else the stream to create from. 261 */ 262 private void create(final InputStream stream) 263 { 264 // If the reader is already open, we'll close it first and 265 // reinstantiate it. 266 if (this.reader != null && this.reader.isOpen()) 267 { 268 this.reader.close(); 269 this.reader = null; 270 } 271 272 // Check whether the string we have is a valid URI 273 IContainer container = null; 274 int openResult = 0; 275 try 276 { 277 // Create the container to read our audio file 278 container = IContainer.make(); 279 280 // If we have a stream, we'll create from the stream... 281 if (stream != null) 282 { 283 openResult = container.open(stream, null, true, true); 284 285 if (openResult < 0) 286 logger.info("XuggleAudio could not open InputStream to audio."); 287 } 288 // otherwise we'll use the URL in the class 289 else 290 { 291 final URI uri = new URI(this.url); 292 293 // If it's a valid URI, we'll try to open the container using 294 // the URI string. 295 openResult = container.open(uri.toString(), 296 IContainer.Type.READ, null, true, true); 297 298 // If there was an error trying to open the container in this 299 // way, 300 // it may be that we have a resource URL (which ffmpeg doesn't 301 // understand), so we'll try opening an InputStream to the 302 // resource. 303 if (openResult < 0) 304 { 305 logger.trace("URL " + this.url + " could not be opened by ffmpeg. " + 306 "Trying to open a stream to the URL instead."); 307 final InputStream is = uri.toURL().openStream(); 308 openResult = container.open(is, null, true, true); 309 310 if (openResult < 0) 311 { 312 logger.error("Error opening container. Error " + openResult + 313 " (" + IError.errorNumberToType(openResult).toString() + ")"); 314 return; 315 } 316 } 317 else 318 logger.info("Opened XuggleAudio stream ok: " + openResult); 319 } 320 } catch (final URISyntaxException e2) 321 { 322 e2.printStackTrace(); 323 return; 324 } catch (final MalformedURLException e) 325 { 326 e.printStackTrace(); 327 return; 328 } catch (final IOException e) 329 { 330 e.printStackTrace(); 331 return; 332 } 333 334 // Set up a new reader using the container that reads the images. 335 this.reader = ToolFactory.makeReader(container); 336 this.reader.addListener(new ChunkGetter()); 337 this.reader.setCloseOnEofOnly(!this.loop); 338 339 // Find the audio stream. 340 IStream s = null; 341 int i = 0; 342 while (i < container.getNumStreams()) 343 { 344 s = container.getStream(i); 345 if (s != null && 346 s.getStreamCoder().getCodecType() == ICodec.Type.CODEC_TYPE_AUDIO) 347 { 348 // Save the stream index so that we only get frames from 349 // this stream in the FrameGetter 350 this.streamIndex = i; 351 break; 352 } 353 i++; 354 } 355 logger.info("Using audio stream " + this.streamIndex); 356 357 if (container.getDuration() == Global.NO_PTS) 358 this.length = -1; 359 else 360 this.length = (long) (s.getDuration() * 361 s.getTimeBase().getDouble() * 1000d); 362 363 // Get the coder for the audio stream 364 final IStreamCoder aAudioCoder = container. 365 getStream(this.streamIndex).getStreamCoder(); 366 367 logger.info("Using stream code: " + aAudioCoder); 368 369 // Create an audio format object suitable for the audio 370 // samples from Xuggle files 371 final AudioFormat af = new AudioFormat( 372 (int) IAudioSamples.findSampleBitDepth(aAudioCoder.getSampleFormat()), 373 aAudioCoder.getSampleRate() / 1000d, 374 aAudioCoder.getChannels()); 375 af.setSigned(true); 376 af.setBigEndian(false); 377 super.format = af; 378 379 logger.info("XuggleAudio using audio format: " + af); 380 381 this.currentSamples = new SampleChunk(af.clone()); 382 } 383 384 // protected int retries = 0; 385 // protected int maxRetries = 0; 386 // 387 // /** 388 // * Set the maximum allowed number of retries in case of an error reading a 389 // * packet. Only use this on live streams; if you do it on a file-based 390 // * stream it might cause looping at the end of file. 391 // * 392 // * @param retries 393 // * maximum number of retries 394 // */ 395 // public void setMaxRetries(int retries) { 396 // this.maxRetries = retries; 397 // } 398 399 /** 400 * {@inheritDoc} 401 * 402 * @see org.openimaj.audio.AudioStream#nextSampleChunk() 403 */ 404 @Override 405 public SampleChunk nextSampleChunk() 406 { 407 try 408 { 409 IError e = null; 410 while ((e = this.reader.readPacket()) == null && !this.chunkAvailable) 411 ; 412 413 if (!this.chunkAvailable) { 414 this.reader.close(); 415 this.reader = null; 416 return null; 417 } 418 419 if (e != null) 420 { 421 this.reader.close(); 422 this.reader = null; 423 424 // // We might be reading from a live stream & if we hit an 425 // error 426 // // we'll retry 427 // if (e != null && e.getType() != IError.Type.ERROR_EOF) 428 // { 429 // logger.error("Got audio demux error " + e.getDescription()); 430 // this.create(null); 431 // this.retries++; 432 // } 433 // logger.info("Closing audio stream " + this.url); 434 return null; 435 } 436 437 this.chunkAvailable = false; 438 return this.currentSamples; 439 } catch (final Exception e) { 440 } 441 442 return null; 443 } 444 445 /** 446 * {@inheritDoc} 447 * 448 * @see org.openimaj.audio.AudioStream#reset() 449 */ 450 @Override 451 public void reset() 452 { 453 if (this.constructedFromStream) 454 { 455 logger.info("Cannot reset a stream of audio."); 456 return; 457 } 458 459 if (this.reader == null || this.reader.getContainer() == null) 460 this.create(null); 461 else 462 this.seek(0); 463 } 464 465 /** 466 * {@inheritDoc} 467 * 468 * @see org.openimaj.audio.AudioStream#getLength() 469 */ 470 @Override 471 public long getLength() 472 { 473 return this.length; 474 } 475 476 /** 477 * {@inheritDoc} 478 * 479 * @see org.openimaj.audio.AudioStream#seek(long) 480 */ 481 @Override 482 public void seek(final long timestamp) 483 { 484 if (this.constructedFromStream) 485 { 486 logger.info("Cannot seek within a stream of audio."); 487 return; 488 } 489 490 if (this.reader == null || this.reader.getContainer() == null) 491 this.create(null); 492 493 // Convert from milliseconds to stream timestamps 494 final double timebase = this.reader.getContainer().getStream( 495 this.streamIndex).getTimeBase().getDouble(); 496 final long position = (long) (timestamp / timebase); 497 498 final long min = Math.max(0, position - 100); 499 final long max = position; 500 501 // logger.info( "Timebase: "+timebase+" of a second second"); 502 // logger.info( "Position to seek to (timebase units): "+position 503 // ); 504 // logger.info( "max: "+max+", min: "+min ); 505 506 final int i = this.reader.getContainer().seekKeyFrame(this.streamIndex, 507 min, position, max, 0); 508 509 // Check for errors 510 if (i < 0) 511 logger.error("Audio seek error (" + i + "): " + IError.errorNumberToType(i)); 512 else 513 this.nextSampleChunk(); 514 } 515 516 /** 517 * Close the audio stream. 518 */ 519 public synchronized void close() 520 { 521 if (this.reader != null) 522 { 523 synchronized (this.reader) 524 { 525 if (this.reader.isOpen()) 526 { 527 this.reader.close(); 528 this.reader = null; 529 } 530 } 531 } 532 } 533}