View Javadoc

1   /**
2    * Copyright (c) 2011, The University of Southampton and the individual contributors.
3    * All rights reserved.
4    *
5    * Redistribution and use in source and binary forms, with or without modification,
6    * are permitted provided that the following conditions are met:
7    *
8    *   * 	Redistributions of source code must retain the above copyright notice,
9    * 	this list of conditions and the following disclaimer.
10   *
11   *   *	Redistributions in binary form must reproduce the above copyright notice,
12   * 	this list of conditions and the following disclaimer in the documentation
13   * 	and/or other materials provided with the distribution.
14   *
15   *   *	Neither the name of the University of Southampton nor the names of its
16   * 	contributors may be used to endorse or promote products derived from this
17   * 	software without specific prior written permission.
18   *
19   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
20   * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21   * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22   * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
23   * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24   * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
26   * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27   * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28   * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29   */
30  /**
31   *
32   */
33  package org.openimaj.audio.features;
34  
35  import org.openimaj.audio.AudioStream;
36  import org.openimaj.audio.SampleChunk;
37  import org.openimaj.audio.processor.AudioProcessor;
38  import org.openimaj.audio.samples.SampleBuffer;
39  import org.openimaj.feature.DoubleFV;
40  import org.openimaj.feature.FeatureExtractor;
41  import org.openimaj.util.array.ArrayUtils;
42  
43  /**
44   *	This class provides an OpenIMAJ wrapper for the JAudio library of feature extractors.
45   *	It provides the marshalling of data from the OpenIMAJ audio streams into
46   *	the data structures necessary for the jAudio FeatureExtractor interface.
47   *
48   *	@author David Dupplaw (dpd@ecs.soton.ac.uk)
49   *  @created 23 May 2013
50   *	@version $Author$, $Revision$, $Date$
51   */
52  public abstract class JAudioFeatureExtractor extends AudioProcessor
53  	implements FeatureExtractor<DoubleFV, SampleChunk>
54  {
55  	/** The jaudio feature extractor */
56  	protected jAudioFeatureExtractor.AudioFeatures.FeatureExtractor featureExtractor;
57  
58  	/** The feature that was last calculated */
59  	private double[][] lastCalculatedFeature;
60  
61  	/**
62  	 * 	Default constructor for ad-hoc processing.
63  	 */
64  	public JAudioFeatureExtractor()
65  	{
66  	}
67  
68  	/**
69  	 * 	Chainable constructor
70  	 *	@param as The audio stream to chain to.
71  	 */
72  	public JAudioFeatureExtractor( final AudioStream as )
73  	{
74  		super( as );
75  	}
76  
77  	/**
78  	 * 	Process the given sample buffer.
79  	 *	@param sb The sample buffer
80  	 *	@return The sample buffer
81  	 */
82  	public SampleBuffer process( final SampleBuffer sb )
83  	{
84  		final double[][] chanSamples = sb.asDoubleChannelArray();
85  		this.lastCalculatedFeature = new double[chanSamples.length][];
86  		for( int c = 0; c < sb.getFormat().getNumChannels(); c++ )
87  			this.lastCalculatedFeature[c] = this.process( chanSamples[c], sb.getFormat().getSampleRateKHz()*1000d );
88  		return sb;
89  	}
90  
91  	/**
92  	 * 	Process the given sample data.
93  	 * 	@param samples The samples
94  	 * 	@param sampleRate The sample rate of the data
95  	 *	@return The features
96  	 */
97  	public double[][] process( final double[][] samples, final double sampleRate )
98  	{
99  		final double[][] featureVectors = new double[samples.length][];
100 		for( int i = 0; i < samples.length; i++ )
101 			featureVectors[i] = this.process( samples[i], sampleRate );
102 		return featureVectors;
103 	}
104 
105 	/**
106 	 * 	Process the given sample array for a single channel
107 	 *	@param samples The samples for a single channel
108 	 *	@param sampleRate The sample rate of the data
109 	 *	@return The feature for the single channel
110 	 */
111 	public double[] process( final double[] samples, final double sampleRate )
112 	{
113 		// Process the feature
114 		try
115 		{
116 			final double[] f = this.featureExtractor.extractFeature( samples, sampleRate,
117 					this.getExtraInputs( samples, sampleRate ) );
118 			return f;
119 		}
120 		catch( final Exception e )
121 		{
122 			e.printStackTrace();
123 		}
124 
125 		// If an exception occurs we return null
126 		return null;
127 	}
128 
129 	/**
130 	 * 	Returns the extra inputs required by a specific feature extractor
131 	 *	@param samples The samples for a single channel
132 	 *	@param sampleRate The sample rate of the data
133 	 *	@return The extra input
134 	 */
135 	public abstract double[][] getExtraInputs( double[] samples, double sampleRate );
136 
137 	/**
138 	 *	{@inheritDoc}
139 	 * 	@see org.openimaj.audio.processor.AudioProcessor#process(org.openimaj.audio.SampleChunk)
140 	 */
141 	@Override
142 	public SampleChunk process( final SampleChunk sample ) throws Exception
143 	{
144 		this.process( sample.getSampleBuffer() );
145 		return sample;
146 	}
147 
148 	/**
149 	 * 	Calculates the feature for each channel, then flattens the channel arrays
150 	 * 	into a single {@link DoubleFV}.
151 	 *
152 	 *	{@inheritDoc}
153 	 * 	@see org.openimaj.feature.FeatureExtractor#extractFeature(java.lang.Object)
154 	 */
155 	@Override
156 	public DoubleFV extractFeature( final SampleChunk sc )
157 	{
158 		// Calculate the feature vector for this frame.
159 		this.process( sc.getSampleBuffer() );
160 		return new DoubleFV( ArrayUtils.reshape( this.lastCalculatedFeature ) );
161 	}
162 
163 	/**
164 	 *	@return the lastCalculatedFeature
165 	 */
166 	public double[][] getLastCalculatedFeature()
167 	{
168 		return this.lastCalculatedFeature;
169 	}
170 
171 	/**
172 	 *	@param lastCalculatedFeature the lastCalculatedFeature to set
173 	 */
174 	public void setLastCalculatedFeature( final double[][] lastCalculatedFeature )
175 	{
176 		this.lastCalculatedFeature = lastCalculatedFeature;
177 	}
178 
179 }