001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030/** 031 * 032 */ 033package org.openimaj.demos.audio; 034 035import java.nio.ByteBuffer; 036import java.nio.ShortBuffer; 037 038import org.openimaj.audio.AudioFormat; 039import org.openimaj.audio.JavaSoundAudioGrabber; 040import org.openimaj.audio.SampleChunk; 041import org.openimaj.audio.analysis.FourierTransform; 042import org.openimaj.audio.filters.HanningAudioProcessor; 043import org.openimaj.demos.Demo; 044import org.openimaj.image.DisplayUtilities; 045import org.openimaj.image.FImage; 046import org.openimaj.image.MBFImage; 047import org.openimaj.image.colour.ColourMap; 048import org.openimaj.image.colour.RGBColour; 049import org.openimaj.image.typography.hershey.HersheyFont; 050 051/** 052 * @author David Dupplaw (dpd@ecs.soton.ac.uk) 053 * 054 * @created 28 Oct 2011 055 */ 056@Demo( 057 title = "Audio Spectrum Processing", 058 author = "David Dupplaw", 059 description = "Demonstrates the basic FFT audio processing in OpenIMAJ", 060 keywords = { "audio", "fft", "spectra" }, 061 icon = "/org/openimaj/demos/icons/audio/microphone-icon.png") 062public class AudioCaptureDemo { 063 /** We'll at first ask for a sample chunk size of 1024. We might not get it */ 064 private int sampleChunkSize = 512; 065 066 /** The image displaying the waveform */ 067 private FImage img = null; 068 069 /** The image displaying the FFT bins */ 070 private FImage fft = null; 071 072 /** The image displaying the spectragram */ 073 private FImage spectra = null; 074 075 /** The frequency bands to mark on the spectragram */ 076 private final double[] Hz = { 100, 500, 1000, 5000, 10000, 20000, 40000 }; 077 078 /** Whether to mark the frequency bands on the spectragram */ 079 private final boolean drawFreqBands = true; 080 081 /** The Fourier transform processor we're going to use */ 082 private FourierTransform fftp = null; 083 084 /** 085 * 086 */ 087 public AudioCaptureDemo() { 088 this.img = new FImage(512, 400); 089 DisplayUtilities.displayName(this.img, "display"); 090 091 this.fft = new FImage(this.img.getWidth(), 400); 092 DisplayUtilities.displayName(this.fft, "fft"); 093 DisplayUtilities.positionNamed("fft", 0, this.img.getHeight()); 094 095 this.fftp = new FourierTransform(); 096 this.spectra = new FImage(800, this.sampleChunkSize / 2); 097 DisplayUtilities.displayName(this.spectra, "spectra", true); 098 DisplayUtilities.positionNamed("spectra", this.img.getWidth(), 0); 099 100 // Uncomment the below to read from a file 101 // final XuggleAudio xa = new XuggleAudio( AudioCaptureDemo.class. 102 // getResource("/org/openimaj/demos/audio/140bpm-Arp.mp3" ) ); 103 104 // Uncomment the below for grabbing audio live 105 final JavaSoundAudioGrabber xa = new JavaSoundAudioGrabber(new AudioFormat(16, 44.1, 1)); 106 xa.setMaxBufferSize(this.sampleChunkSize); 107 new Thread(xa).start(); 108 109 // Hanning processor on top of the main audio stream 110 final HanningAudioProcessor g = 111 new HanningAudioProcessor(xa, this.img.getWidth() * xa.getFormat().getNumChannels()) 112 { 113 @Override 114 public SampleChunk processSamples(final SampleChunk sample) 115 { 116 AudioCaptureDemo.this.updateDisplay(sample); 117 return sample; 118 } 119 }; 120 121 System.out.println("Using audio stream: " + g.getFormat()); 122 123 try { 124 Thread.sleep(500); 125 SampleChunk s = null; 126 while ((s = g.nextSampleChunk()) != null) 127 this.updateDisplay(s); 128 } catch (final InterruptedException e) { 129 e.printStackTrace(); 130 } 131 } 132 133 /** 134 * Updates the visualisation each time a sample chunk comes in. 135 * 136 * @param s 137 * The sample chunk to display 138 */ 139 public void updateDisplay(final SampleChunk s) { 140 ShortBuffer sb = null; 141 ByteBuffer bb = null; 142 if ((bb = s.getSamplesAsByteBuffer()) != null) 143 sb = bb.asShortBuffer(); 144 else 145 return; 146 147 // ------------------------------------------------- 148 // Draw waveform 149 // ------------------------------------------------- 150 this.img.zero(); 151 final int yOffset = this.img.getHeight() / 2; 152 for (int i = 1; i < s.getNumberOfSamples() / s.getFormat().getNumChannels(); i++) { 153 this.img.drawLine( 154 i - 1, sb.get((i - 1) * s.getFormat().getNumChannels()) / 256 + yOffset, 155 i, sb.get(i * s.getFormat().getNumChannels()) / 256 + yOffset, 1f); 156 } 157 DisplayUtilities.displayName(this.img, "display"); 158 159 // ------------------------------------------------- 160 // Draw FFT 161 // ------------------------------------------------- 162 this.fft.zero(); 163 this.fftp.process(s); 164 165 final float[] f = this.fftp.getLastFFT()[0]; 166 final double binSize = (s.getFormat().getSampleRateKHz() * 1000) / (f.length / 2); 167 168 for (int i = 0; i < f.length / 4; i++) { 169 final float re = f[i * 2]; 170 final float im = f[i * 2 + 1]; 171 final float mag = (float) Math.log(Math.sqrt(re * re + im * im) + 1) / 50f; 172 this.fft.drawLine(i * 2, this.fft.getHeight(), i * 2, this.fft.getHeight() - (int) (mag * this.fft.getHeight()), 2, 1f); 173 } 174 DisplayUtilities.displayName(this.fft, "fft"); 175 176 // ------------------------------------------------- 177 // Draw Spectra 178 // ------------------------------------------------- 179 // System.out.println( "Sample chunk size: "+sampleChunkSize ); 180 // System.out.println( "Number of samples: "+s.getNumberOfSamples() ); 181 // System.out.println( "FFT size: "+f.length ); 182 if (s.getNumberOfSamples() != this.sampleChunkSize) { 183 this.sampleChunkSize = s.getNumberOfSamples(); 184 this.spectra = new FImage(800, this.sampleChunkSize / 2); 185 DisplayUtilities.displayName(this.spectra, "spectra"); 186 DisplayUtilities.positionNamed("spectra", this.img.getWidth(), 0); 187 } 188 189 this.spectra.shiftLeftInplace(); 190 191 // Draw the spectra 192 for (int i = 0; i < f.length / 4; i++) { 193 final float re = f[i * 2]; 194 final float im = f[i * 2 + 1]; 195 float mag = (float) Math.log(Math.sqrt(re * re + im * im) + 1) / 25f; 196 if (mag > 1) { 197 mag = 1; 198 } 199 this.spectra.setPixel(this.spectra.getWidth() - 1, this.spectra.getHeight() - i, mag); 200 } 201 202 final MBFImage drawSpectra = ColourMap.Jet.apply(this.spectra); 203 if (this.drawFreqBands) { 204 // drawSpectra = spectra.clone(); 205 206 // Draw the frequency bands 207 for (final double freq : this.Hz) { 208 final int y = drawSpectra.getHeight() - (int) (freq / binSize); 209 drawSpectra.drawLine(0, y, this.spectra.getWidth(), y, RGBColour.GREEN); 210 drawSpectra.drawText("" + freq + "Hz", 4, y, HersheyFont.TIMES_BOLD, 10, RGBColour.GREEN); 211 } 212 } 213 214 DisplayUtilities.displayName(drawSpectra, "spectra"); 215 } 216 217 /** 218 * 219 * @param args 220 */ 221 public static void main(final String[] args) { 222 new AudioCaptureDemo(); 223 } 224}