1 /**
2 * Copyright (c) 2011, The University of Southampton and the individual contributors.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without modification,
6 * are permitted provided that the following conditions are met:
7 *
8 * * Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 *
11 * * Redistributions in binary form must reproduce the above copyright notice,
12 * this list of conditions and the following disclaimer in the documentation
13 * and/or other materials provided with the distribution.
14 *
15 * * Neither the name of the University of Southampton nor the names of its
16 * contributors may be used to endorse or promote products derived from this
17 * software without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
23 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
26 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30 /**
31 *
32 */
33 package org.openimaj.audio.features;
34
35 import org.openimaj.audio.AudioStream;
36 import org.openimaj.audio.SampleChunk;
37 import org.openimaj.audio.processor.AudioProcessor;
38 import org.openimaj.audio.samples.SampleBuffer;
39 import org.openimaj.feature.DoubleFV;
40 import org.openimaj.feature.FeatureExtractor;
41 import org.openimaj.util.array.ArrayUtils;
42
43 /**
44 * This class provides an OpenIMAJ wrapper for the JAudio library of feature extractors.
45 * It provides the marshalling of data from the OpenIMAJ audio streams into
46 * the data structures necessary for the jAudio FeatureExtractor interface.
47 *
48 * @author David Dupplaw (dpd@ecs.soton.ac.uk)
49 * @created 23 May 2013
50 * @version $Author$, $Revision$, $Date$
51 */
52 public abstract class JAudioFeatureExtractor extends AudioProcessor
53 implements FeatureExtractor<DoubleFV, SampleChunk>
54 {
55 /** The jaudio feature extractor */
56 protected jAudioFeatureExtractor.AudioFeatures.FeatureExtractor featureExtractor;
57
58 /** The feature that was last calculated */
59 private double[][] lastCalculatedFeature;
60
61 /**
62 * Default constructor for ad-hoc processing.
63 */
64 public JAudioFeatureExtractor()
65 {
66 }
67
68 /**
69 * Chainable constructor
70 * @param as The audio stream to chain to.
71 */
72 public JAudioFeatureExtractor( final AudioStream as )
73 {
74 super( as );
75 }
76
77 /**
78 * Process the given sample buffer.
79 * @param sb The sample buffer
80 * @return The sample buffer
81 */
82 public SampleBuffer process( final SampleBuffer sb )
83 {
84 final double[][] chanSamples = sb.asDoubleChannelArray();
85 this.lastCalculatedFeature = new double[chanSamples.length][];
86 for( int c = 0; c < sb.getFormat().getNumChannels(); c++ )
87 this.lastCalculatedFeature[c] = this.process( chanSamples[c], sb.getFormat().getSampleRateKHz()*1000d );
88 return sb;
89 }
90
91 /**
92 * Process the given sample data.
93 * @param samples The samples
94 * @param sampleRate The sample rate of the data
95 * @return The features
96 */
97 public double[][] process( final double[][] samples, final double sampleRate )
98 {
99 final double[][] featureVectors = new double[samples.length][];
100 for( int i = 0; i < samples.length; i++ )
101 featureVectors[i] = this.process( samples[i], sampleRate );
102 return featureVectors;
103 }
104
105 /**
106 * Process the given sample array for a single channel
107 * @param samples The samples for a single channel
108 * @param sampleRate The sample rate of the data
109 * @return The feature for the single channel
110 */
111 public double[] process( final double[] samples, final double sampleRate )
112 {
113 // Process the feature
114 try
115 {
116 final double[] f = this.featureExtractor.extractFeature( samples, sampleRate,
117 this.getExtraInputs( samples, sampleRate ) );
118 return f;
119 }
120 catch( final Exception e )
121 {
122 e.printStackTrace();
123 }
124
125 // If an exception occurs we return null
126 return null;
127 }
128
129 /**
130 * Returns the extra inputs required by a specific feature extractor
131 * @param samples The samples for a single channel
132 * @param sampleRate The sample rate of the data
133 * @return The extra input
134 */
135 public abstract double[][] getExtraInputs( double[] samples, double sampleRate );
136
137 /**
138 * {@inheritDoc}
139 * @see org.openimaj.audio.processor.AudioProcessor#process(org.openimaj.audio.SampleChunk)
140 */
141 @Override
142 public SampleChunk process( final SampleChunk sample ) throws Exception
143 {
144 this.process( sample.getSampleBuffer() );
145 return sample;
146 }
147
148 /**
149 * Calculates the feature for each channel, then flattens the channel arrays
150 * into a single {@link DoubleFV}.
151 *
152 * {@inheritDoc}
153 * @see org.openimaj.feature.FeatureExtractor#extractFeature(java.lang.Object)
154 */
155 @Override
156 public DoubleFV extractFeature( final SampleChunk sc )
157 {
158 // Calculate the feature vector for this frame.
159 this.process( sc.getSampleBuffer() );
160 return new DoubleFV( ArrayUtils.reshape( this.lastCalculatedFeature ) );
161 }
162
163 /**
164 * @return the lastCalculatedFeature
165 */
166 public double[][] getLastCalculatedFeature()
167 {
168 return this.lastCalculatedFeature;
169 }
170
171 /**
172 * @param lastCalculatedFeature the lastCalculatedFeature to set
173 */
174 public void setLastCalculatedFeature( final double[][] lastCalculatedFeature )
175 {
176 this.lastCalculatedFeature = lastCalculatedFeature;
177 }
178
179 }