001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030/** 031 * 032 */ 033package org.openimaj.demos.audio; 034 035import org.openimaj.audio.AudioFormat; 036import org.openimaj.audio.AudioMixer; 037import org.openimaj.audio.AudioMixer.MixEventListener; 038import org.openimaj.audio.AudioPlayer; 039import org.openimaj.audio.analysis.BeatDetector; 040import org.openimaj.audio.analysis.EffectiveSoundPressure; 041import org.openimaj.audio.samples.SampleBuffer; 042import org.openimaj.audio.timecode.MeasuresBeatsTicksTimecode; 043import org.openimaj.audio.util.MusicUtils; 044import org.openimaj.demos.Demo; 045import org.openimaj.image.DisplayUtilities; 046import org.openimaj.image.MBFImage; 047import org.openimaj.image.colour.RGBColour; 048import org.openimaj.math.geometry.shape.Rectangle; 049import org.openimaj.time.Sequencer; 050import org.openimaj.video.xuggle.XuggleAudio; 051 052/** 053 * A demonstration of the {@link AudioMixer} function in OpenIMAJ. 054 * Also demonstrates the {@link EffectiveSoundPressure} processor 055 * (for calculating the loudness of each channel), the {@link BeatDetector} 056 * processor (for displaying beats) and the {@link Sequencer} for 057 * sequencing actions in time (in this case triggering the audio loops). 058 * <p> 059 * Note: if this demo makes a horrible noise as the number of channels being 060 * mixed increases, then it's probably because the sample buffer isn't long 061 * enough for your system. On line 70 it's set to 720 samples. You may increase 062 * this if it doesn't work. 063 * 064 * @author David Dupplaw (dpd@ecs.soton.ac.uk) 065 * 066 * @created 26 Nov 2011 067 */ 068@Demo( 069 title = "Audio Mixing and Beat Detection", 070 author = "David Dupplaw", 071 description = "Demonstrates some of the OpenIMAJ audio functionality for " + 072 "audio processing. Includes the mixing of audio streams, the " + 073 "sequencing of events (in this case audio events), the calculation " + 074 "of sound levels and the detection of beats within music.", 075 keywords = { "audio", "sound", "vu", "loudness", "beat detection", "mixing", 076 "sequencing", "events", "pressure", "dB" }, 077 icon = "/org/openimaj/demos/icons/audio/vumeter.png", 078 screenshot = "/org/openimaj/demos/screens/audio/mixing.png" 079) 080public class AudioMixerDemo 081{ 082 /** 083 * Construct the demo for the audio mixer 084 */ 085 public AudioMixerDemo() 086 { 087 // The image will contain the VU meters 088 this.img = new MBFImage( 300, 400, 3 ); 089 DisplayUtilities.displayName( this.img, "VU Meters" ); 090 091 // Create a new audio mixer than mixes audio streams 092 final AudioFormat mixerFormat = new AudioFormat( 16, 44.1, 2 ); 093 final AudioMixer am = new AudioMixer( mixerFormat ); 094 am.addMixEventListener( new VURenderer( mixerFormat ) ); 095 am.setMixEvents( true ); 096 097 // Due to the way we sequence the loops, the size of this buffer will 098 // cause a lag in the sequenced loops, so the smaller it is the better. 099 // However, the smaller it is, the less time we'll have to mix and process 100 // the sample chunks and display the VU meters (because in this demo it's 101 // all done in one thread). 102 am.setBufferSize( 720 ); 103 104 // Create a new audio player (this will be the timekeeper for the sequencer) 105 final AudioPlayer ap = new AudioPlayer( am /*, "M44 [plughw:0,0]" */ ); 106 ap.setTimecodeObject( new MeasuresBeatsTicksTimecode( 140 ) ); 107 108 // Create a new sequencer that will set up the different streams 109 final Sequencer seq = new Sequencer( ap, MusicUtils.millisPerBeat( 140 )/4 ); 110 111 // We instantiate a XuggleAudio here as it seems to take a long time 112 // to start this class up for the first time and the synchronisation 113 // goes out unless we preload the class and everything it needs here. 114 new XuggleAudio( AudioMixer.class.getResource("/org/openimaj/demos/audio/140bpm-2205.mp3") ); 115 116 // Set up the various events in the sequencer 117 // First we set up all the actions (which can be reused) 118 final Sequencer.SequencedAction drums = this.getAction( 119 "/org/openimaj/demos/audio/140bpm_formware_psytech.mp3", am ); 120 final Sequencer.SequencedAction bass = this.getAction( 121 "/org/openimaj/demos/audio/140bpm-Arp.mp3", am ); 122 final Sequencer.SequencedAction tb303 = this.getAction( 123 "/org/openimaj/demos/audio/140bpm-303.mp3", am ); 124 final Sequencer.SequencedAction tb2205 = this.getAction( 125 "/org/openimaj/demos/audio/140bpm-2205.mp3", am ); 126 127 // Add the drums events 128 for( int i = 1; i < 29; i += 4 ) 129 seq.addEvent( new Sequencer.SequencerEvent( 130 new MeasuresBeatsTicksTimecode( 140,i,0,0 ), drums ) ); 131 seq.addEvent( new Sequencer.SequencerEvent( 132 new MeasuresBeatsTicksTimecode( 140,5,0,0 ), bass ) ); 133 seq.addEvent( new Sequencer.SequencerEvent( 134 new MeasuresBeatsTicksTimecode( 140,13,0,0 ), tb303 ) ); 135 seq.addEvent( new Sequencer.SequencerEvent( 136 new MeasuresBeatsTicksTimecode( 140,17,0,0 ), tb303 ) ); 137 seq.addEvent( new Sequencer.SequencerEvent( 138 new MeasuresBeatsTicksTimecode( 140,21,0,0 ), bass ) ); 139 seq.addEvent( new Sequencer.SequencerEvent( 140 new MeasuresBeatsTicksTimecode( 140,21,0,0 ), tb303 ) ); 141 for( int i = 21; i < 33; i+=2 ) 142 seq.addEvent( new Sequencer.SequencerEvent( 143 new MeasuresBeatsTicksTimecode( 140,i,0,0 ), tb2205 ) ); 144 145 // Run the sequencer (and the audio player which is the time keeper) 146 seq.run(); 147 } 148 149 /** 150 * Returns a sequenced action that will play a given sound file on 151 * the given mixer. 152 * 153 * @param soundFile The sound file to play 154 * @param am The mixer to play it on 155 * @return A {@link SequencedAction} 156 */ 157 private Sequencer.SequencedAction getAction( final String soundFile, 158 final AudioMixer am ) 159 { 160 return new Sequencer.SequencedAction() 161 { 162 @Override 163 public boolean performAction() 164 { 165 final XuggleAudio xa5 = new XuggleAudio( 166 AudioMixer.class.getResource( soundFile ) ); 167 am.addStream( xa5, 0.3f ); 168 return true; 169 } 170 }; 171 } 172 173 /** The image we'll draw into */ 174 private MBFImage img = null; 175 176 /** 177 * Class that will render the channels in VU meters when 178 * a mix event occurs. 179 * 180 * @author David Dupplaw (dpd@ecs.soton.ac.uk) 181 * 182 * @created 29 Nov 2011 183 */ 184 private class VURenderer implements MixEventListener 185 { 186 /** Processor to get RMS */ 187 private final EffectiveSoundPressure rms = new EffectiveSoundPressure(); 188 189 /** The beat detector processor */ 190 private BeatDetector beatDetector = null; 191 192 /** 0db in our RMS range */ 193 private final double ref = 82; 194 195 /** db value where we go red on the VU */ 196 private final double redAbove = 0.8; 197 198 /** Size of each VU block */ 199 private final int blockPadding = 4; 200 private final int blockHeight = 6; 201 private final int blockSize = this.blockHeight + this.blockPadding; 202 203 /** Width of the VU Display */ 204 private final int blockWidth = 30; 205 206 /** Space between each VU */ 207 private final int padding = 10; 208 209 /** Gap between channel VUs */ 210 private final int intraChannelGap = 2; 211 212 /** Size of total VU meter */ 213 private final int vuSize = 300; 214 215 /** Where we start drawing the VU meters */ 216 private final int yOffset = 350; 217 218 /** Where we start drawing the VU Meters */ 219 private final int xOffset = 50; 220 221 private final Float[] colourAbove = RGBColour.RED; 222 private final Float[] colourBelow = RGBColour.GREEN; 223 224 /** A hysteresis for the beat detector lights */ 225 private final int[] beatDetectorLightCount = new int[10]; 226 227 /** 228 * Instantiate the VU renderer. 229 * @param af The audio format of the mixer 230 */ 231 public VURenderer( final AudioFormat af ) 232 { 233 this.beatDetector = new BeatDetector( af ); 234 } 235 236 /** 237 * {@inheritDoc} 238 * @see org.openimaj.audio.AudioMixer.MixEventListener#mix(org.openimaj.audio.samples.SampleBuffer[], org.openimaj.audio.samples.SampleBuffer) 239 */ 240 @Override 241 public void mix( final SampleBuffer[] channels, final SampleBuffer mix ) 242 { 243 // Really, the drawing should be done in another thread and the 244 // samples which are being drawn buffered. However, that would 245 // somewhat complicate matters, so we're just going to try and 246 // draw the VU meters between each mix of the mixer. That means 247 // the VUs will get a bit flickery and it's possible that if this 248 // process takes too long the mixer will start to stutter. 249 250 AudioMixerDemo.this.img.zero(); 251 252 for( int i = 0; i < channels.length; i++ ) 253 { 254 try 255 { 256 final int redAboveY = (int)(this.yOffset - this.redAbove*this.vuSize); 257 final int x = this.xOffset + i * (this.blockWidth+this.padding); 258 final int nc = channels[i].getFormat().getNumChannels(); 259 for( int c = 0; c < nc; c++ ) 260 { 261 // Convert the samples into a dB value 262 this.rms.process( channels[i].getSampleChunk(c) ); 263 double d = 6/Math.log(2)* 264 Math.log(this.rms.getEffectiveSoundPressure())-this.ref; 265 d = Math.exp(Math.log(1.055)*d); 266 267 // Draw the VU Meters 268 for( int y = this.yOffset; y > this.yOffset-(d*this.vuSize); y -= this.blockSize ) 269 AudioMixerDemo.this.img.drawShapeFilled( 270 new Rectangle( x+(c*this.blockWidth/nc), y-this.blockHeight, 271 this.blockWidth/nc-this.intraChannelGap, this.blockHeight ), 272 y < redAboveY? this.colourAbove : this.colourBelow ); 273 } 274 275 // Do beat detection and show the beats as a yellow 276 // light underneath the VUs 277 if( this.beatDetectorLightCount[i] <= 0 ) 278 { 279 this.beatDetector.process( channels[i].getSampleChunk() ); 280 if( this.beatDetector.beatDetected() ) 281 this.beatDetectorLightCount[i] = 20; 282 } 283 284 // The light count is a hysteresis for the lights so that 285 // they stay on for longer than the pulse detected as a beat. 286 if( this.beatDetectorLightCount[i]-- >= 0 ) 287 AudioMixerDemo.this.img.drawShapeFilled( 288 new Rectangle( x, this.yOffset+4, this.blockWidth, 15 ), 289 RGBColour.YELLOW ); 290 else 291 AudioMixerDemo.this.img.drawShapeFilled( 292 new Rectangle( x, this.yOffset+4, this.blockWidth, 15 ), 293 new Float[]{0.2f,0.2f,0f} ); 294 295 DisplayUtilities.displayName( AudioMixerDemo.this.img, "VU Meters" ); 296 } 297 catch( final Exception e ) 298 { 299 e.printStackTrace(); 300 } 301 } 302 } 303 } 304 305 /** 306 * Default main 307 * @param args Command-line arguments 308 */ 309 public static void main(final String[] args) 310 { 311 new AudioMixerDemo(); 312 } 313}