001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.image.feature.local.interest;
031
032import java.util.ArrayList;
033import java.util.Collections;
034import java.util.Comparator;
035import java.util.List;
036
037import org.openimaj.image.FImage;
038import org.openimaj.image.pixel.FValuePixel;
039import org.openimaj.image.pixel.Pixel;
040import org.openimaj.image.processing.convolution.BasicDerivativeKernels;
041import org.openimaj.image.processing.convolution.FGaussianConvolve;
042import org.openimaj.math.geometry.shape.Rectangle;
043import org.openimaj.math.util.FloatArrayStatsUtils;
044
045import Jama.Matrix;
046
047/**
048 * Abstract base class for an interest point detector which uses derivatives or
049 * the (multiscale) structure tensor.
050 *
051 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
052 * @author Sina Samangooei (ss@ecs.soton.ac.uk)
053 */
054public abstract class AbstractStructureTensorIPD implements
055                MultiscaleInterestPointDetector<InterestPointData>
056{
057
058        protected int borderSkip;
059        FImage originalImage;
060        FImage l, lx, ly, lxmx, lymy, lxmy;
061        public FImage lxmxblur, lymyblur, lxmyblur;
062
063        protected float detectionScale;
064        protected float integrationScale;
065        protected float detIntScaleFactor = 1.4f;
066
067        protected List<Maxima> maxima;
068
069        private boolean blurred;
070
071        /**
072         * Set the scale factor between the integration scale and the detection
073         * scale. When detection scale is set, integration scale = detIntScaleFactor
074         * * detectionScale
075         *
076         * @param detIntScaleFactor
077         */
078        public AbstractStructureTensorIPD(float detIntScaleFactor) {
079                this.detIntScaleFactor = detIntScaleFactor;
080                this.borderSkip = 2;
081        }
082
083        /**
084         * Abstract structure tensor detected at a given scale, the first
085         * derivatives found and a structure tensor combined from these first
086         * derivatives with a gaussian window of sigma = integrationScale
087         *
088         * @param detectionScale
089         * @param integrationScale
090         */
091        public AbstractStructureTensorIPD(float detectionScale,
092                        float integrationScale)
093        {
094                this(detectionScale, integrationScale, 2, false);
095        }
096
097        /**
098         * Abstract structure tensor detected at a given scale, the first
099         * derivatives found and a structure tensor combined from these first
100         * derivatives with a gaussian window of sigma = integrationScale. Also
101         * state whether the image in from which features are extracted is already
102         * blurred to the detection scale, if not it will be blurred to the correct
103         * level
104         *
105         * @param detectionScale
106         * @param integrationScale
107         * @param blurred
108         */
109        public AbstractStructureTensorIPD(float detectionScale,
110                        float integrationScale, boolean blurred)
111        {
112                this(detectionScale, integrationScale, 2, blurred);
113        }
114
115        /**
116         * Abstract structure tensor detected at a given scale, the first
117         * derivatives found and a structure tensor combined from these first
118         * derivatives with a gaussian window of sigma = integrationScale. Also
119         * specify how many pixels to skip around the edge of the image. The kernel
120         * used to extract edges results in a black border so some pixels are better
121         * ignored in terms of corner detection.
122         *
123         * @param detectionScale
124         * @param integrationScale
125         * @param borderSkip
126         */
127        public AbstractStructureTensorIPD(float detectionScale,
128                        float integrationScale, int borderSkip)
129        {
130                this(detectionScale, integrationScale, borderSkip,
131                                false);
132        }
133
134        /**
135         * Abstract structure tensor detected at a given scale, the first
136         * derivatives found and a structure tensor combined from these first
137         * derivatives with a gaussian window of sigma = integrationScale. Also
138         * specify how many pixels to skip around the edge of the image. The kernel
139         * used to extract edges results in a black border so some pixels are better
140         * ignored in terms of corner detection. Also state whether the image in
141         * from which features are extracted is already blurred to the detection
142         * scale, if not it will be blurred to the correct level
143         *
144         * @param detectionScale
145         * @param integrationScale
146         * @param borderSkip
147         * @param blurred
148         */
149        public AbstractStructureTensorIPD(float detectionScale,
150                        float integrationScale, int borderSkip, boolean blurred)
151        {
152                this.blurred = blurred;
153                if (borderSkip < 1)
154                        borderSkip = 1;
155
156                this.detectionScale = detectionScale;
157                this.integrationScale = integrationScale;
158                this.borderSkip = borderSkip;
159        }
160
161        public void prepareInterestPoints(FImage image) {
162                originalImage = image;
163                // // Add padding around the edges of the image (4 pixels all the way
164                // around)
165                // image = image.padding(4,4);
166                // l = image.clone().processInplace(new
167                // FDiscGausConvolve(detectionScale));
168                // lx =
169                // l.process(BasicDerivativeKernels.DX_KERNEL).extractROI(4,4,this.originalImage.getWidth(),
170                // this.originalImage.getHeight()).multiplyInplace((float)Math.sqrt(detectionScale));
171                // ly =
172                // l.process(BasicDerivativeKernels.DY_KERNEL).extractROI(4,4,this.originalImage.getWidth(),
173                // this.originalImage.getHeight()).multiplyInplace((float)Math.sqrt(detectionScale));
174
175                l = image;
176                if (!this.blurred)
177                        l = l.processInplace(new FGaussianConvolve(detectionScale));
178                lx = l.process(BasicDerivativeKernels.DX_KERNEL).multiplyInplace(this.detectionScale);
179                ly = l.process(BasicDerivativeKernels.DY_KERNEL).multiplyInplace(this.detectionScale);
180
181                lxmx = lx.multiply(lx);
182                lymy = ly.multiply(ly);
183                lxmy = lx.multiply(ly);
184                final FGaussianConvolve intConv = new FGaussianConvolve(integrationScale);
185                lxmxblur = lxmx.clone().processInplace(intConv);
186                lymyblur = lymy.clone().processInplace(intConv);
187                lxmyblur = lxmy.clone().processInplace(intConv);
188        }
189
190        public void printStructureTensorStats() {
191                System.out.format("Structure tensor stats for sd/si = %4.2f/%4.2f\n",
192                                detectionScale, integrationScale);
193                System.out.format(
194                                "\tlxmx mean/std = %4.2e/%4.2e max/min = %4.2e/%4.2e\n",
195                                FloatArrayStatsUtils.mean(lxmxblur.pixels),
196                                FloatArrayStatsUtils.std(lxmxblur.pixels), lxmx.max(),
197                                lxmx.min());
198                System.out.format(
199                                "\tlxmy mean/std = %4.2e/%4.2e max/min = %4.2e/%4.2e\n",
200                                FloatArrayStatsUtils.mean(lxmyblur.pixels),
201                                FloatArrayStatsUtils.std(lxmyblur.pixels), lxmy.max(),
202                                lxmy.min());
203                System.out.format(
204                                "\tlymy mean/std = %4.2e/%4.2e max/min = %4.2e/%4.2e\n",
205                                FloatArrayStatsUtils.mean(lymyblur.pixels),
206                                FloatArrayStatsUtils.std(lymyblur.pixels), lymy.max(),
207                                lymy.min());
208        }
209
210        @Override
211        public void findInterestPoints(FImage image) {
212
213                this.prepareInterestPoints(image);
214                final FImage cornerImage = createInterestPointMap();
215
216                detectMaxima(cornerImage, image.getBounds());
217        }
218
219        @Override
220        public void findInterestPoints(FImage image, Rectangle window) {
221
222                this.prepareInterestPoints(image);
223                final FImage cornerImage = createInterestPointMap();
224                System.out.format(
225                                "corner image mean/std = %4.2e/%4.2e max/min = %4.2e/%4.2e\n",
226                                FloatArrayStatsUtils.mean(cornerImage.pixels),
227                                FloatArrayStatsUtils.std(cornerImage.pixels),
228                                cornerImage.max(), cornerImage.min());
229
230                detectMaxima(cornerImage, window);
231        }
232
233        public FValuePixel findMaximum(Rectangle window) {
234                final FImage cornerImage = createInterestPointMap();
235                final FValuePixel c = cornerImage.extractROI(window).maxPixel();
236                c.translate(window.x, window.y);
237                return c;
238        }
239
240        public class Maxima {
241                public int x, y;
242                public float val;
243
244                public Maxima(int x, int y, float v) {
245                        this.x = x;
246                        this.y = y;
247                        this.val = v;
248                }
249        }
250
251        protected void detectMaxima(FImage image, Rectangle window) {
252                maxima = new ArrayList<Maxima>();
253
254                for (int y = borderSkip; y < image.height - borderSkip; y++) {
255                        for (int x = borderSkip; x < image.width - borderSkip; x++) {
256                                if (!window.isInside(new Pixel(x, y)))
257                                        continue;
258                                final float curr = image.pixels[y][x];
259                                if (curr > image.pixels[y - 1][x - 1]
260                                                && curr >= image.pixels[y - 1][x]
261                                                                && curr >= image.pixels[y - 1][x + 1]
262                                                                                && curr >= image.pixels[y][x - 1]
263                                                                                                && curr >= image.pixels[y][x + 1]
264                                                                                                                && curr >= image.pixels[y + 1][x - 1]
265                                                                                                                                && curr >= image.pixels[y + 1][x]
266                                                                                                                                                && curr >= image.pixels[y + 1][x + 1])
267                                {
268                                        maxima.add(new Maxima(x, y, curr));
269                                }
270                        }
271                }
272
273                Collections.sort(maxima, new Comparator<Maxima>() {
274                        @Override
275                        public int compare(Maxima o1, Maxima o2) {
276                                if (o1.val == o2.val)
277                                        return 0;
278                                return o1.val < o2.val ? 1 : -1;
279                        }
280                });
281        }
282
283        public abstract FImage createInterestPointMap();
284
285        @Override
286        public List<InterestPointData> getInterestPoints(int npoints) {
287                if (npoints < 0 || npoints > maxima.size())
288                        npoints = maxima.size();
289                final List<InterestPointData> ipdata = new ArrayList<InterestPointData>();
290
291                for (int i = 0; i < npoints; i++) {
292                        final InterestPointData ipd = new InterestPointData();
293
294                        ipd.x = maxima.get(i).x;
295                        ipd.y = maxima.get(i).y;
296                        ipd.scale = integrationScale;
297                        ipd.score = maxima.get(i).val;
298
299                        ipdata.add(ipd);
300                }
301
302                return ipdata;
303        }
304
305        public float getDetIntScaleFactor() {
306                return detIntScaleFactor;
307        }
308
309        public void setDetIntScaleFactor(float detIntScaleFactor) {
310                this.detIntScaleFactor = detIntScaleFactor;
311        }
312
313        public float getDetectionScale() {
314                return detectionScale;
315        }
316
317        public void setImageBlurred(boolean blurred) {
318                this.blurred = blurred;
319        }
320
321        @Override
322        public void setDetectionScale(float detectionScale) {
323                this.detectionScale = detectionScale;
324                this.integrationScale = this.detectionScale * this.detIntScaleFactor;
325        }
326
327        public float getIntegrationScale() {
328                return integrationScale;
329        }
330
331        public void setIntegrationScale(float integrationScale) {
332                this.integrationScale = integrationScale;
333                this.detectionScale = integrationScale * (1f / this.detIntScaleFactor);
334        }
335
336        @Override
337        public List<InterestPointData> getInterestPoints() {
338                return getInterestPoints(-1);
339        }
340
341        @Override
342        public List<InterestPointData> getInterestPoints(float threshold) {
343                return getInterestPointsThresh(threshold);
344        }
345
346        public List<InterestPointData> getInterestPointsThresh(float thresh) {
347                final List<InterestPointData> ipdata = new ArrayList<InterestPointData>();
348
349                for (final Maxima m : maxima) {
350                        if (m.val < thresh)
351                                continue;
352
353                        final InterestPointData ipd = new InterestPointData();
354
355                        ipd.x = m.x;
356                        ipd.y = m.y;
357                        ipd.scale = integrationScale;
358                        ipd.score = m.val;
359
360                        ipdata.add(ipd);
361                }
362
363                return ipdata;
364        }
365
366        public Matrix getSecondMomentsAt(int x, int y) {
367                final Matrix secondMoments = new Matrix(2, 2);
368                secondMoments.set(0, 0, lxmxblur.pixels[y][x]);
369                secondMoments.set(0, 1, lxmyblur.pixels[y][x]);
370                secondMoments.set(1, 0, lxmyblur.pixels[y][x]);
371                secondMoments.set(1, 1, lymyblur.pixels[y][x]);
372                return secondMoments;
373        }
374
375        @Override
376        public AbstractStructureTensorIPD clone() {
377                AbstractStructureTensorIPD a = null;
378                try {
379                        a = (AbstractStructureTensorIPD) super.clone();
380                } catch (final CloneNotSupportedException e) {
381                        return null;
382                }
383                return a;
384        }
385
386        public int pointsFound() {
387                return this.maxima.size();
388        }
389}