001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.image.feature.dense.gradient.dsift;
031
032import org.openimaj.feature.local.list.LocalFeatureList;
033import org.openimaj.feature.local.list.MemoryLocalFeatureList;
034import org.openimaj.image.MBFImage;
035import org.openimaj.image.colour.ColourSpace;
036import org.openimaj.math.geometry.shape.Rectangle;
037
038/**
039 * Implementation of colour dense-sift. The algorithm works by applying the
040 * {@link DenseSIFT} extractor to each channel of the image (in the target
041 * {@link ColourSpace}) and then aggregating the descriptors across all
042 * {@link ColourSpace}s for the same spatial location.
043 * <p>
044 * Any {@link ColourSpace} can be used. To compute the contrast energy of a
045 * descriptor, the luminance is extracted across channels using
046 * {@link ColourSpace#computeIntensity(float[])}. This means that if you choose
047 * a {@link ColourSpace} that doesn't support intensities, then the contrast
048 * energy will be zero. In practice this should not matter as it's most usual to
049 * use the {@link ColourSpace#RGB}, {@link ColourSpace#HSV} or
050 * {@link ColourSpace#OPPONENT} colour spaces, which can compute intensities.
051 * 
052 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
053 * 
054 */
055public class ColourDenseSIFT extends AbstractDenseSIFT<MBFImage> {
056        DenseSIFT dsift;
057        ColourSpace colourSpace;
058
059        volatile float[][] descriptors;
060        volatile float[] energies;
061
062        /**
063         * Construct with the given internal {@link DenseSIFT} extractor to apply to
064         * each band of the image created by converting the input to
065         * {@link #analyseImage(MBFImage)} or
066         * {@link #analyseImage(MBFImage, Rectangle)} to the given
067         * {@link ColourSpace}.
068         * 
069         * @param dsift
070         *            the dense sift extractor
071         * @param colourSpace
072         *            the target colour space
073         */
074        public ColourDenseSIFT(DenseSIFT dsift, ColourSpace colourSpace) {
075                this.dsift = dsift;
076                this.colourSpace = colourSpace;
077        }
078
079        @Override
080        public void analyseImage(MBFImage image, Rectangle bounds) {
081                // handle colour conversion?
082                final MBFImage cimg = colourSpace.convert(image);
083
084                // first band:
085                dsift.analyseImage(cimg.bands.get(0), bounds);
086                final int len = dsift.descriptors[0].length;
087                descriptors = new float[dsift.descriptors.length][len * cimg.bands.size()];
088                final float[][] tmpEnergies = new float[dsift.descriptors.length][cimg.bands.size()];
089
090                for (int i = 0; i < descriptors.length; i++) {
091                        System.arraycopy(dsift.descriptors[i], 0, descriptors[i], 0, len);
092                        tmpEnergies[i][0] = dsift.energies[i];
093                }
094
095                // other bands
096                for (int j = 1; j < cimg.bands.size(); j++) {
097                        dsift.analyseImage(cimg.bands.get(j), bounds);
098                        for (int i = 0; i < descriptors.length; i++) {
099                                System.arraycopy(dsift.descriptors[i], 0, descriptors[i], j * len, len);
100                                tmpEnergies[i][j] = dsift.energies[i];
101                        }
102                }
103
104                // deal with energies
105                energies = new float[descriptors.length];
106                for (int i = 0; i < descriptors.length; i++) {
107                        energies[i] = colourSpace.computeIntensity(tmpEnergies[i]);
108                }
109        }
110
111        @Override
112        public LocalFeatureList<FloatDSIFTKeypoint> getFloatKeypoints() {
113                final MemoryLocalFeatureList<FloatDSIFTKeypoint> keys = new MemoryLocalFeatureList<FloatDSIFTKeypoint>(
114                                dsift.numOriBins
115                                                * dsift.numBinsX * dsift.numBinsY, descriptors.length);
116
117                final int frameSizeX = dsift.binWidth * (dsift.numBinsX - 1) + 1;
118                final int frameSizeY = dsift.binHeight * (dsift.numBinsY - 1) + 1;
119
120                final float deltaCenterX = 0.5F * dsift.binWidth * (dsift.numBinsX - 1);
121                final float deltaCenterY = 0.5F * dsift.binHeight * (dsift.numBinsY - 1);
122
123                for (int framey = dsift.data.boundMinY, i = 0; framey <= dsift.data.boundMaxY - frameSizeY + 1; framey += dsift.stepY)
124                {
125                        for (int framex = dsift.data.boundMinX; framex <= dsift.data.boundMaxX - frameSizeX + 1; framex += dsift.stepX, i++)
126                        {
127                                keys.add(new FloatDSIFTKeypoint(framex + deltaCenterX, framey + deltaCenterY, descriptors[i], energies[i]));
128                        }
129                }
130
131                return keys;
132        }
133
134        @Override
135        public LocalFeatureList<ByteDSIFTKeypoint> getByteKeypoints() {
136                final MemoryLocalFeatureList<ByteDSIFTKeypoint> keys = new MemoryLocalFeatureList<ByteDSIFTKeypoint>(
137                                dsift.numOriBins
138                                                * dsift.numBinsX * dsift.numBinsY, descriptors.length);
139
140                final int frameSizeX = dsift.binWidth * (dsift.numBinsX - 1) + 1;
141                final int frameSizeY = dsift.binHeight * (dsift.numBinsY - 1) + 1;
142
143                final float deltaCenterX = 0.5F * dsift.binWidth * (dsift.numBinsX - 1);
144                final float deltaCenterY = 0.5F * dsift.binHeight * (dsift.numBinsY - 1);
145
146                for (int framey = dsift.data.boundMinY, i = 0; framey <= dsift.data.boundMaxY - frameSizeY + 1; framey += dsift.stepY)
147                {
148                        for (int framex = dsift.data.boundMinX; framex <= dsift.data.boundMaxX - frameSizeX + 1; framex += dsift.stepX, i++)
149                        {
150                                keys.add(new ByteDSIFTKeypoint(framex + deltaCenterX, framey + deltaCenterY, descriptors[i], energies[i]));
151                        }
152                }
153
154                return keys;
155        }
156
157        @Override
158        public LocalFeatureList<FloatDSIFTKeypoint> getFloatKeypoints(float energyThreshold) {
159                final MemoryLocalFeatureList<FloatDSIFTKeypoint> keys = new MemoryLocalFeatureList<FloatDSIFTKeypoint>(
160                                dsift.numOriBins
161                                                * dsift.numBinsX * dsift.numBinsY);
162
163                final int frameSizeX = dsift.binWidth * (dsift.numBinsX - 1) + 1;
164                final int frameSizeY = dsift.binHeight * (dsift.numBinsY - 1) + 1;
165
166                final float deltaCenterX = 0.5F * dsift.binWidth * (dsift.numBinsX - 1);
167                final float deltaCenterY = 0.5F * dsift.binHeight * (dsift.numBinsY - 1);
168
169                for (int framey = dsift.data.boundMinY, i = 0; framey <= dsift.data.boundMaxY - frameSizeY + 1; framey += dsift.stepY)
170                {
171                        for (int framex = dsift.data.boundMinX; framex <= dsift.data.boundMaxX - frameSizeX + 1; framex += dsift.stepX, i++)
172                        {
173                                if (energies[i] >= energyThreshold)
174                                        keys.add(new FloatDSIFTKeypoint(framex + deltaCenterX, framey + deltaCenterY, descriptors[i],
175                                                        energies[i]));
176                        }
177                }
178
179                return keys;
180        }
181
182        @Override
183        public LocalFeatureList<ByteDSIFTKeypoint> getByteKeypoints(float energyThreshold) {
184                final MemoryLocalFeatureList<ByteDSIFTKeypoint> keys = new MemoryLocalFeatureList<ByteDSIFTKeypoint>(
185                                dsift.numOriBins
186                                                * dsift.numBinsX * dsift.numBinsY);
187
188                final int frameSizeX = dsift.binWidth * (dsift.numBinsX - 1) + 1;
189                final int frameSizeY = dsift.binHeight * (dsift.numBinsY - 1) + 1;
190
191                final float deltaCenterX = 0.5F * dsift.binWidth * (dsift.numBinsX - 1);
192                final float deltaCenterY = 0.5F * dsift.binHeight * (dsift.numBinsY - 1);
193
194                for (int framey = dsift.data.boundMinY, i = 0; framey <= dsift.data.boundMaxY - frameSizeY + 1; framey += dsift.stepY)
195                {
196                        for (int framex = dsift.data.boundMinX; framex <= dsift.data.boundMaxX - frameSizeX + 1; framex += dsift.stepX, i++)
197                        {
198                                if (energies[i] >= energyThreshold)
199                                        keys.add(new ByteDSIFTKeypoint(framex + deltaCenterX, framey + deltaCenterY, descriptors[i],
200                                                        energies[i]));
201                        }
202                }
203
204                return keys;
205        }
206
207        @Override
208        public void setBinWidth(int size) {
209                this.dsift.setBinWidth(size);
210        }
211
212        @Override
213        public void setBinHeight(int size) {
214                this.dsift.setBinHeight(size);
215        }
216
217        @Override
218        public int getBinWidth() {
219                // TODO Auto-generated method stub
220                return 0;
221        }
222
223        @Override
224        public int getBinHeight() {
225                // TODO Auto-generated method stub
226                return 0;
227        }
228
229        @Override
230        public int getNumBinsX() {
231                // TODO Auto-generated method stub
232                return 0;
233        }
234
235        @Override
236        public int getNumBinsY() {
237                // TODO Auto-generated method stub
238                return 0;
239        }
240
241        @Override
242        public int getNumOriBins() {
243                // TODO Auto-generated method stub
244                return 0;
245        }
246
247        @Override
248        public float[][] getDescriptors() {
249                return descriptors;
250        }
251}