001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030/**
031 *
032 */
033package org.openimaj.vis.audio;
034
035import java.awt.Dimension;
036import java.util.ArrayList;
037import java.util.List;
038
039import org.openimaj.audio.AudioFormat;
040import org.openimaj.audio.AudioStream;
041import org.openimaj.audio.SampleChunk;
042import org.openimaj.audio.analysis.FourierTransform;
043import org.openimaj.audio.filters.HanningAudioProcessor;
044import org.openimaj.image.FImage;
045import org.openimaj.image.MBFImage;
046import org.openimaj.image.typography.hershey.HersheyFont;
047import org.openimaj.vis.VisualisationImpl;
048
049/**
050 * A spectrogram visualisation that scrolls the audio visualisation as the audio
051 * is processed. Vertical axis of the visualisation represents frequency,
052 * horizontal represents time (newest on the right), and pixel intensity
053 * represents frequency amplitude.
054 *
055 * @author David Dupplaw (dpd@ecs.soton.ac.uk)
056 * @created 19 Jul 2012
057 * @version $Author$, $Revision$, $Date$
058 */
059public class AudioSpectrogram extends VisualisationImpl<float[]>
060{
061        /**
062         * A listener for when the spectrogram has completed processing.
063         *
064         * @author David Dupplaw (dpd@ecs.soton.ac.uk)
065         * @created 19 Jul 2012
066         * @version $Author$, $Revision$, $Date$
067         */
068        public static interface SpectrogramCompleteListener
069        {
070                /**
071                 * Called when the spectragram is complete
072                 *
073                 * @param as The spectragram that completed.
074                 */
075                public void spectrogramComplete( AudioSpectrogram as );
076        }
077
078        /** */
079        private static final long serialVersionUID = 1L;
080
081        /** The Fourier transformer we'll use */
082        private final FourierTransform fftp = new FourierTransform();
083
084        /** Whether to draw the frequency bands, or not */
085        private boolean drawFreqBands = true;
086
087        /** The frequency bands to mark on the spectragram */
088        private double[] frequencyBands =
089        { 100, 500, 1000, 5000, 10000, 20000, 40000 };
090
091        /** Is the processing complete */
092        private boolean isComplete = false;
093
094        /** The size of the FFT bins (in Hz) */
095        private double binSize = 0;
096
097        /** The listeners */
098        private final List<SpectrogramCompleteListener> listeners =
099                        new ArrayList<AudioSpectrogram.SpectrogramCompleteListener>();
100
101        /** The format of the audio being processed */
102        private AudioFormat audioFormat = null;
103
104        /** Whether to draw the line at the current position of drawing */
105        private boolean drawCurrentPositionLine = true;
106
107        /** Colour of the line delineating the end of the current spectrogram */
108        private Float[] currentPositionLineColour = new Float[]
109                        { 0.5f, 0.5f, 0.5f, 1f };
110
111        /** The total number of frames we've drawn */
112        private int nFrames = 0;
113
114        /** The current position we're drawing at */
115        private int currentDrawPosition;
116
117        /** The last spectrogram image */
118        private FImage previousSpecImage = null;
119
120        /**
121         * Create a spectrogram that can be added to as and when it's necessary.
122         */
123        public AudioSpectrogram()
124        {
125                this.setPreferredSize( new Dimension( -1, 100 ) );
126                super.clearBeforeDraw = true;
127        }
128
129        /**
130         * Construct a visualisation of the given size
131         *
132         * @param w Width of the required visualisation
133         * @param h Height of the required visualisation
134         */
135        public AudioSpectrogram( final int w, final int h )
136        {
137                super( w, h );
138                super.clearBeforeDraw = true;
139                this.setPreferredSize( new Dimension( w, h ) );
140        }
141
142        /**
143         * Add the given listener
144         *
145         * @param l The listener
146         */
147        public void addListener( final SpectrogramCompleteListener l )
148        {
149                this.listeners.add( l );
150        }
151
152        /**
153         *      Remove the given listener
154         *      @param l The listener to remove
155         */
156        public void removeListener( final SpectrogramCompleteListener l )
157        {
158                this.listeners.remove( l );
159        }
160
161        /**
162         * Process the entire stream (or as much data will fit into the
163         * visualisation window). The last transform to be processed will
164         * be available in the <code>data</code> field of this class.
165         *
166         * @param as The stream to process
167         */
168        public void processStream( final AudioStream as )
169        {
170                this.audioFormat = as.getFormat().clone();
171                new Thread( new Runnable()
172                {
173                        @Override
174                        public void run()
175                        {
176                                // We'll process the incoming stream with a hanning processor
177                                final HanningAudioProcessor hanningProcessor = new HanningAudioProcessor(
178                                                as, AudioSpectrogram.this.visImage.getHeight()*8 );
179
180                                // Loop through all the sample chunks and process them.
181                                SampleChunk s = null;
182                                AudioSpectrogram.this.currentDrawPosition = 0;
183                                while( (s = hanningProcessor.nextSampleChunk()) != null
184                                                && AudioSpectrogram.this.currentDrawPosition < AudioSpectrogram.this.visImage.getWidth() )
185                                {
186                                        AudioSpectrogram.this.process( s );
187                                }
188
189                                // We're done.
190                                AudioSpectrogram.this.isComplete = true;
191
192                                // So, fire the complete listener.
193                                for( final SpectrogramCompleteListener l : AudioSpectrogram.this.listeners )
194                                        l.spectrogramComplete( AudioSpectrogram.this );
195                        }
196                } ).start();
197        }
198
199        /**
200         *      Processes a single sample chunk: calculates the FFT, gets the magnitudes,
201         *      copies the format (if it's the first chunk), and then goes on to update the image.
202         *
203         *      @param s The sample chunk to process
204         */
205        public void process( final SampleChunk s )
206        {
207                // Process the FFT
208                this.fftp.process( s.getSampleBuffer() );
209
210                // Get the magnitudes to show in the spectrogram
211                final float[] f = this.fftp.getNormalisedMagnitudes( 1f/Integer.MAX_VALUE )[0];
212
213                // Store the format of this sample chunk if we don't have one yet.
214                // This allows us to continue to draw the frequency bands on the image
215                // (if it's configured to do that).
216                if( this.audioFormat == null )
217                        this.audioFormat = s.getFormat().clone();
218
219                // Store this FFT into the data member. Note this calls a method in this class.
220                this.setData( f );
221        }
222
223        /**
224         *      {@inheritDoc}
225         *      @see org.openimaj.vis.VisualisationImpl#setData(java.lang.Object)
226         */
227        @Override
228        public void setData( final float[] data )
229        {
230                // Set the data into the data field
231                super.setData( data );
232
233                // We count the number of frames so we can stop
234                // if we get to the edge of the window
235                this.nFrames++;
236
237                // Shift the data along (if the window's too small)
238                this.shiftData();
239
240                // Repaint the visualisation
241                this.updateVis();
242        }
243
244        /**
245         *      Add the given sample chunk into the spectrogram. This is a
246         *      handy method for processing an audio stream outside of this class.
247         *
248         *      @param sc The sample chunk to add.
249         */
250        public void setData( final SampleChunk sc )
251        {
252                this.process( sc );
253        }
254
255        /**
256         * Draw the given spectra into the image at the given x coordinate.
257         *
258         * @param freqs The FFT output
259         * @param x The x position to draw it at
260         */
261        private void drawSpectra( final FImage img, final float[] f, final int x )
262        {
263                if( img == null || f == null ) return;
264
265//              final double ps = img.getHeight()/f.length;
266                for( int i = 0; i < f.length; i++ )
267                {
268//                      img.drawLine( x, img.getHeight()-i, x, (int)(img.getHeight()-i-ps), mag );
269                        final int y = img.getHeight() - i -1;
270                        img.setPixel( x, y, f[i] );
271                }
272        }
273
274        /**
275         * Returns whether the spectragram image is complete.
276         *
277         * @return Whether the image is complete.
278         */
279        public boolean isComplete()
280        {
281                return this.isComplete;
282        }
283
284        /**
285         *      Checks whether the visualisation needs to be shifted to the left.
286         *      If not it draws the spectra from the left of the image. If it
287         *      reaches the right of the image, the image will be scrolled to the
288         *      left and the new spectra drawn at the right of the image.
289         */
290        private void shiftData()
291        {
292                // Check if we should be drawing outside of the image. If so,
293                // shift the image to the left and draw at the right hand edge.
294                // (our draw position is given in currentDrawPosition and it's
295                // not updated if we enter this if clause).
296                if( this.nFrames > this.visImage.getWidth() )
297                        this.previousSpecImage = this.previousSpecImage.shiftLeft();
298                else
299                {
300                        // Blat the previous spectrogram and update where we're going to draw
301                        // the newest spectra.
302                        if( this.nFrames > 0 )
303                        {
304                                final FImage t = new FImage( this.nFrames, this.visImage.getHeight() );
305                                if( this.previousSpecImage != null )
306                                        t.drawImage( this.previousSpecImage, 0, t.getHeight()-this.previousSpecImage.getHeight() );
307                                this.previousSpecImage = t;
308                                this.currentDrawPosition++;
309                        }
310                }
311
312                // Draw the newest spectra. Note that we draw onto the "previousSpecImage"
313                // memory image and we blat this out in the update() method.
314                synchronized( this.data )
315                {
316                        // Draw spectra onto image
317                        this.drawSpectra( this.previousSpecImage, this.data, this.currentDrawPosition-1 );
318                }
319        }
320
321        /**
322         *      {@inheritDoc}
323         *      @see org.openimaj.vis.VisualisationImpl#update()
324         */
325        @Override
326        public void update()
327        {
328                if( this.data != null )
329                {
330                        synchronized( this.visImage )
331                        {
332                                // Draw the spectra image if we have one.
333                                if( this.previousSpecImage != null )
334                                {
335                                        this.visImage.drawImage( MBFImage.createRGB(this.previousSpecImage), 0,
336                                                this.visImage.getHeight()-this.previousSpecImage.getHeight() );
337                                }
338
339                                // Draw the frequency bands onto the image.
340                                if( this.drawFreqBands && this.audioFormat != null )
341                                {
342                                        // Work out where to plot the next spectra
343                                        this.binSize = (this.audioFormat.getSampleRateKHz() * 500) / this.data.length;
344
345                                        // Draw the frequency bands
346                                        for( final double freq : this.frequencyBands )
347                                        {
348                                                final Float[] fbc = new Float[] { 0.2f, 0.2f, 0.2f };
349                                                final Float[] fbtc = fbc;
350                                                final int y = (int)(this.visImage.getHeight() -
351                                                                freq/this.binSize);
352
353                                                this.visImage.drawLine( 0, y, this.visImage.getWidth(), y, fbc );
354                                                this.visImage.drawText( "" + freq + "Hz", 4, y, HersheyFont.TIMES_BOLD, 10, fbtc );
355                                        }
356                                }
357
358                                // Draw the bar showing where we're drawing at.
359                                if( this.drawCurrentPositionLine )
360                                        this.visImage.drawLine( this.currentDrawPosition + 1, 0,
361                                                        this.currentDrawPosition + 1, this.visImage.getHeight(),
362                                                        this.currentPositionLineColour );
363                        }
364                }
365        }
366
367        /**
368         *      Returns whether the frequency bands are being drawn.
369         *      @return TRUE if the bands are being drawn; FALSE otherwise
370         */
371        public boolean isDrawFreqBands()
372        {
373                return this.drawFreqBands;
374        }
375
376        /**
377         *      Set whether to overlay the frequency bands onto the image.
378         *      @param drawFreqBands TRUE to draw the frequency bands; FALSE otherwise
379         */
380        public void setDrawFreqBands( final boolean drawFreqBands )
381        {
382                this.drawFreqBands = drawFreqBands;
383        }
384
385        /**
386         *      Get the frequency bands which are being drawn in Hz.
387         *      @return The frequency bands which are being drawn.
388         */
389        public double[] getFrequencyBands()
390        {
391                return this.frequencyBands;
392        }
393
394        /**
395         *      Set the frequency bands to overlay on the image in Hz.
396         *      @param frequencyBands the frequency bands to draw
397         */
398        public void setFrequencyBands( final double[] frequencyBands )
399        {
400                this.frequencyBands = frequencyBands;
401        }
402
403        /**
404         *      Returns whether the current position line is being drawn
405         *      @return Whether the current position line is being drawn
406         */
407        public boolean isDrawCurrentPositionLine()
408        {
409                return this.drawCurrentPositionLine;
410        }
411
412        /**
413         *      Set whether to draw the position at which the current spectra will be drawn
414         *      @param drawCurrentPositionLine TRUE to draw the current position line
415         */
416        public void setDrawCurrentPositionLine( final boolean drawCurrentPositionLine )
417        {
418                this.drawCurrentPositionLine = drawCurrentPositionLine;
419        }
420
421        /**
422         *      Get the colour of the current position line
423         *      @return The current position line colour
424         */
425        public Float[] getCurrentPositionLineColour()
426        {
427                return this.currentPositionLineColour;
428        }
429
430        /**
431         *      Set the colour of the line which is showning the current draw position.
432         *      @param currentPositionLineColour The colour of the current draw position.
433         */
434        public void setCurrentPositionLineColour( final Float[] currentPositionLineColour )
435        {
436                this.currentPositionLineColour = currentPositionLineColour;
437        }
438
439        /**
440         *      Get the position at which the next spectrum will be drawn.
441         *      @return the position at which the next spectrum will be drawn.
442         */
443        public int getCurrentDrawPosition()
444        {
445                return this.currentDrawPosition;
446        }
447}