001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.image.indexing.vlad;
031
032import java.io.DataInputStream;
033import java.io.DataOutputStream;
034import java.io.File;
035import java.io.IOException;
036import java.io.InputStream;
037import java.io.OutputStream;
038import java.util.List;
039
040import org.openimaj.feature.DoubleFV;
041import org.openimaj.feature.MultidimensionalFloatFV;
042import org.openimaj.feature.local.FloatLocalFeatureAdaptor;
043import org.openimaj.feature.local.LocalFeature;
044import org.openimaj.feature.local.LocalFeatureExtractor;
045import org.openimaj.image.MBFImage;
046import org.openimaj.image.feature.local.aggregate.VLAD;
047import org.openimaj.io.IOUtils;
048import org.openimaj.knn.pq.FloatProductQuantiser;
049import org.openimaj.knn.pq.IncrementalFloatADCNearestNeighbours;
050import org.openimaj.ml.pca.FeatureVectorPCA;
051import org.openimaj.util.array.ArrayUtils;
052import org.openimaj.util.function.Function;
053
054/**
055 * Class representing the data required to build a VLAD + PCA +
056 * product-quantisation based image index.
057 * 
058 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
059 * 
060 */
061public class VLADIndexerData {
062        private VLAD<float[]> vlad;
063        private FeatureVectorPCA pca;
064        private FloatProductQuantiser pq;
065        private LocalFeatureExtractor<LocalFeature<?, ?>, MBFImage> extractor;
066        private Function<List<? extends LocalFeature<?, ?>>, List<FloatLocalFeatureAdaptor<?>>> postProcess;
067
068        /**
069         * Construct with the given data
070         * 
071         * @param vlad
072         *            the VLAD extractor
073         * @param pca
074         *            the PCA basis
075         * @param pq
076         *            the product quantiser
077         * @param extractor
078         *            the raw local feature extractor
079         * @param postProcess
080         *            the process to apply to the raw features before VLAD
081         *            aggregation
082         */
083        public VLADIndexerData(VLAD<float[]> vlad, FeatureVectorPCA pca, FloatProductQuantiser pq,
084                        LocalFeatureExtractor<LocalFeature<?, ?>, MBFImage> extractor,
085                        Function<List<? extends LocalFeature<?, ?>>, List<FloatLocalFeatureAdaptor<?>>> postProcess)
086        {
087                this.vlad = vlad;
088                this.pca = pca;
089                this.pq = pq;
090                this.extractor = extractor;
091                this.postProcess = postProcess;
092        }
093
094        /**
095         * Extract the PCA-projected VLAD feature from the given raw local features.
096         * The local features will be post-processed before being aggregated using
097         * {@link VLAD} and projected by the PCA basis.
098         * 
099         * @param features
100         *            the raw local features
101         * @return the pca-vlad aggregated representation of the image
102         */
103        public float[] extractPcaVlad(List<? extends LocalFeature<?, ?>> features) {
104                final MultidimensionalFloatFV keys = vlad.aggregate(postProcess.apply(features));
105
106                if (keys == null)
107                        return null;
108
109                final DoubleFV subspaceVector = pca.project(keys).normaliseFV(2);
110                return ArrayUtils.convertToFloat(subspaceVector.values);
111        }
112
113        /**
114         * Extract the PCA-projected VLAD feature from the given image. The local
115         * features will be post-processed before being aggregated using
116         * {@link VLAD} and projected by the PCA basis.
117         * 
118         * @param image
119         *            the image to extract from
120         * @return the pca-vlad aggregated representation of the image
121         */
122        public float[] extractPcaVlad(MBFImage image) {
123                return extractPcaVlad(extractor.extractFeature(image));
124        }
125
126        /**
127         * Extract the product-quantised PCA-projected VLAD feature from the given
128         * raw local features. The local features will be post-processed before
129         * being aggregated using {@link VLAD} and projected by the PCA basis.
130         * 
131         * @param features
132         *            the raw local features
133         * @return the product-quantised pca-vlad aggregated representation of the
134         *         image
135         */
136        public byte[] extractPQPcaVlad(List<? extends LocalFeature<?, ?>> features) {
137                final MultidimensionalFloatFV keys = vlad.aggregate(postProcess.apply(features));
138
139                if (keys == null)
140                        return null;
141
142                final DoubleFV subspaceVector = pca.project(keys).normaliseFV(2);
143                return pq.quantise(ArrayUtils.convertToFloat(subspaceVector.values));
144        }
145
146        /**
147         * Extract the product-quantisedPCA-projected VLAD feature from the given
148         * image. The local features will be post-processed before being aggregated
149         * using {@link VLAD} and projected by the PCA basis.
150         * 
151         * @param image
152         *            the image to extract from
153         * @return the product-quantised pca-vlad aggregated representation of the
154         *         image
155         */
156        public byte[] extractPQPcaVlad(MBFImage image) {
157                return extractPQPcaVlad(extractor.extractFeature(image));
158        }
159
160        /**
161         * Get the product quantiser
162         * 
163         * @return get the product quantiser
164         */
165        public FloatProductQuantiser getProductQuantiser() {
166                return pq;
167        }
168
169        /**
170         * Create an {@link IncrementalFloatADCNearestNeighbours} pre-prepared to
171         * index data
172         * 
173         * @return a new {@link IncrementalFloatADCNearestNeighbours}
174         */
175        public IncrementalFloatADCNearestNeighbours createIncrementalIndex() {
176                return new IncrementalFloatADCNearestNeighbours(pq, pca.getMean().length);
177        }
178
179        /**
180         * Index the given features into the given nearest neighbours object by
181         * converting them to the PCA-VLAD representation and then
182         * product-quantising.
183         * 
184         * @param features
185         *            the features to index
186         * @param nn
187         *            the nearest neighbours object
188         * @return the index at which the features were added in the nearest
189         *         neighbours object
190         */
191        public int index(List<? extends LocalFeature<?, ?>> features, IncrementalFloatADCNearestNeighbours nn) {
192                return nn.add(extractPcaVlad(features));
193        }
194
195        /**
196         * Index the given image into the given nearest neighbours object by
197         * extracting the PCA-VLAD representation and then product-quantising.
198         * 
199         * @param image
200         *            the image to index
201         * @param nn
202         *            the nearest neighbours object
203         * @return the index at which the features were added in the nearest
204         *         neighbours object
205         */
206        public int index(MBFImage image, IncrementalFloatADCNearestNeighbours nn) {
207                return nn.add(extractPcaVlad(image));
208        }
209
210        /**
211         * Write this {@link VLADIndexerData} object to the given file. The file can
212         * be re-read using the {@link #read(File)} method.
213         * 
214         * @param file
215         *            the file to write to
216         * @throws IOException
217         *             if an error occurs
218         */
219        public void write(File file) throws IOException {
220                IOUtils.writeToFile(this, file);
221        }
222
223        /**
224         * Write this {@link VLADIndexerData} object to the given stream. The
225         * {@link #read(InputStream)} can read from a stream to reconstruct the
226         * {@link VLADIndexerData}.
227         * 
228         * @param os
229         *            the stream
230         * @throws IOException
231         *             if an error occurs
232         */
233        public void write(OutputStream os) throws IOException {
234                IOUtils.write(this, new DataOutputStream(os));
235        }
236
237        /**
238         * Read a {@link VLADIndexerData} object to the given file created with the
239         * {@link #write(File)} method.
240         * 
241         * @param file
242         *            the file to read from
243         * @return the newly read {@link VLADIndexerData} object.
244         * @throws IOException
245         *             if an error occurs
246         */
247        public static VLADIndexerData read(File file) throws IOException {
248                return IOUtils.readFromFile(file);
249        }
250
251        /**
252         * Read a {@link VLADIndexerData} object to the given stream created with
253         * the {@link #write(OutputStream)} method.
254         * 
255         * @param is
256         *            the stream to read from
257         * @return the newly read {@link VLADIndexerData} object.
258         * @throws IOException
259         *             if an error occurs
260         */
261        public static VLADIndexerData read(InputStream is) throws IOException {
262                return IOUtils.read(new DataInputStream(is));
263        }
264
265        /**
266         * Get the {@link VLAD} aggregator instance
267         * 
268         * @return the {@link VLAD} aggregator
269         */
270        public VLAD<float[]> getVLAD() {
271                return vlad;
272        }
273
274        /**
275         * Get the dimensionality of the float vectors extracted from the pca-vlad
276         * process.
277         * 
278         * @return the dimensionality.
279         */
280        public int numDimensions() {
281                return this.pca.getEigenValues().length;
282        }
283
284        /**
285         * @return the pca
286         */
287        public FeatureVectorPCA getPCA() {
288                return pca;
289        }
290
291        /**
292         * @return the extractor
293         */
294        public LocalFeatureExtractor<LocalFeature<?, ?>, MBFImage> getExtractor() {
295                return extractor;
296        }
297
298        /**
299         * @return the postProcess
300         */
301        public Function<List<? extends LocalFeature<?, ?>>, List<FloatLocalFeatureAdaptor<?>>> getPostProcess() {
302                return postProcess;
303        }
304
305}