001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.image.feature.local.interest; 031 032import java.util.ArrayList; 033import java.util.Collections; 034import java.util.Comparator; 035import java.util.List; 036 037import org.openimaj.image.FImage; 038import org.openimaj.image.pixel.FValuePixel; 039import org.openimaj.image.pixel.Pixel; 040import org.openimaj.image.processing.convolution.BasicDerivativeKernels; 041import org.openimaj.image.processing.convolution.FGaussianConvolve; 042import org.openimaj.math.geometry.shape.Rectangle; 043import org.openimaj.math.util.FloatArrayStatsUtils; 044 045import Jama.Matrix; 046 047/** 048 * Abstract base class for an interest point detector which uses derivatives or 049 * the (multiscale) structure tensor. 050 * 051 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk) 052 * @author Sina Samangooei (ss@ecs.soton.ac.uk) 053 */ 054public abstract class AbstractStructureTensorIPD implements 055 MultiscaleInterestPointDetector<InterestPointData> 056{ 057 058 protected int borderSkip; 059 FImage originalImage; 060 FImage l, lx, ly, lxmx, lymy, lxmy; 061 public FImage lxmxblur, lymyblur, lxmyblur; 062 063 protected float detectionScale; 064 protected float integrationScale; 065 protected float detIntScaleFactor = 1.4f; 066 067 protected List<Maxima> maxima; 068 069 private boolean blurred; 070 071 /** 072 * Set the scale factor between the integration scale and the detection 073 * scale. When detection scale is set, integration scale = detIntScaleFactor 074 * * detectionScale 075 * 076 * @param detIntScaleFactor 077 */ 078 public AbstractStructureTensorIPD(float detIntScaleFactor) { 079 this.detIntScaleFactor = detIntScaleFactor; 080 this.borderSkip = 2; 081 } 082 083 /** 084 * Abstract structure tensor detected at a given scale, the first 085 * derivatives found and a structure tensor combined from these first 086 * derivatives with a gaussian window of sigma = integrationScale 087 * 088 * @param detectionScale 089 * @param integrationScale 090 */ 091 public AbstractStructureTensorIPD(float detectionScale, 092 float integrationScale) 093 { 094 this(detectionScale, integrationScale, 2, false); 095 } 096 097 /** 098 * Abstract structure tensor detected at a given scale, the first 099 * derivatives found and a structure tensor combined from these first 100 * derivatives with a gaussian window of sigma = integrationScale. Also 101 * state whether the image in from which features are extracted is already 102 * blurred to the detection scale, if not it will be blurred to the correct 103 * level 104 * 105 * @param detectionScale 106 * @param integrationScale 107 * @param blurred 108 */ 109 public AbstractStructureTensorIPD(float detectionScale, 110 float integrationScale, boolean blurred) 111 { 112 this(detectionScale, integrationScale, 2, blurred); 113 } 114 115 /** 116 * Abstract structure tensor detected at a given scale, the first 117 * derivatives found and a structure tensor combined from these first 118 * derivatives with a gaussian window of sigma = integrationScale. Also 119 * specify how many pixels to skip around the edge of the image. The kernel 120 * used to extract edges results in a black border so some pixels are better 121 * ignored in terms of corner detection. 122 * 123 * @param detectionScale 124 * @param integrationScale 125 * @param borderSkip 126 */ 127 public AbstractStructureTensorIPD(float detectionScale, 128 float integrationScale, int borderSkip) 129 { 130 this(detectionScale, integrationScale, borderSkip, 131 false); 132 } 133 134 /** 135 * Abstract structure tensor detected at a given scale, the first 136 * derivatives found and a structure tensor combined from these first 137 * derivatives with a gaussian window of sigma = integrationScale. Also 138 * specify how many pixels to skip around the edge of the image. The kernel 139 * used to extract edges results in a black border so some pixels are better 140 * ignored in terms of corner detection. Also state whether the image in 141 * from which features are extracted is already blurred to the detection 142 * scale, if not it will be blurred to the correct level 143 * 144 * @param detectionScale 145 * @param integrationScale 146 * @param borderSkip 147 * @param blurred 148 */ 149 public AbstractStructureTensorIPD(float detectionScale, 150 float integrationScale, int borderSkip, boolean blurred) 151 { 152 this.blurred = blurred; 153 if (borderSkip < 1) 154 borderSkip = 1; 155 156 this.detectionScale = detectionScale; 157 this.integrationScale = integrationScale; 158 this.borderSkip = borderSkip; 159 } 160 161 public void prepareInterestPoints(FImage image) { 162 originalImage = image; 163 // // Add padding around the edges of the image (4 pixels all the way 164 // around) 165 // image = image.padding(4,4); 166 // l = image.clone().processInplace(new 167 // FDiscGausConvolve(detectionScale)); 168 // lx = 169 // l.process(BasicDerivativeKernels.DX_KERNEL).extractROI(4,4,this.originalImage.getWidth(), 170 // this.originalImage.getHeight()).multiplyInplace((float)Math.sqrt(detectionScale)); 171 // ly = 172 // l.process(BasicDerivativeKernels.DY_KERNEL).extractROI(4,4,this.originalImage.getWidth(), 173 // this.originalImage.getHeight()).multiplyInplace((float)Math.sqrt(detectionScale)); 174 175 l = image; 176 if (!this.blurred) 177 l = l.processInplace(new FGaussianConvolve(detectionScale)); 178 lx = l.process(BasicDerivativeKernels.DX_KERNEL).multiplyInplace(this.detectionScale); 179 ly = l.process(BasicDerivativeKernels.DY_KERNEL).multiplyInplace(this.detectionScale); 180 181 lxmx = lx.multiply(lx); 182 lymy = ly.multiply(ly); 183 lxmy = lx.multiply(ly); 184 final FGaussianConvolve intConv = new FGaussianConvolve(integrationScale); 185 lxmxblur = lxmx.clone().processInplace(intConv); 186 lymyblur = lymy.clone().processInplace(intConv); 187 lxmyblur = lxmy.clone().processInplace(intConv); 188 } 189 190 public void printStructureTensorStats() { 191 System.out.format("Structure tensor stats for sd/si = %4.2f/%4.2f\n", 192 detectionScale, integrationScale); 193 System.out.format( 194 "\tlxmx mean/std = %4.2e/%4.2e max/min = %4.2e/%4.2e\n", 195 FloatArrayStatsUtils.mean(lxmxblur.pixels), 196 FloatArrayStatsUtils.std(lxmxblur.pixels), lxmx.max(), 197 lxmx.min()); 198 System.out.format( 199 "\tlxmy mean/std = %4.2e/%4.2e max/min = %4.2e/%4.2e\n", 200 FloatArrayStatsUtils.mean(lxmyblur.pixels), 201 FloatArrayStatsUtils.std(lxmyblur.pixels), lxmy.max(), 202 lxmy.min()); 203 System.out.format( 204 "\tlymy mean/std = %4.2e/%4.2e max/min = %4.2e/%4.2e\n", 205 FloatArrayStatsUtils.mean(lymyblur.pixels), 206 FloatArrayStatsUtils.std(lymyblur.pixels), lymy.max(), 207 lymy.min()); 208 } 209 210 @Override 211 public void findInterestPoints(FImage image) { 212 213 this.prepareInterestPoints(image); 214 final FImage cornerImage = createInterestPointMap(); 215 216 detectMaxima(cornerImage, image.getBounds()); 217 } 218 219 @Override 220 public void findInterestPoints(FImage image, Rectangle window) { 221 222 this.prepareInterestPoints(image); 223 final FImage cornerImage = createInterestPointMap(); 224 System.out.format( 225 "corner image mean/std = %4.2e/%4.2e max/min = %4.2e/%4.2e\n", 226 FloatArrayStatsUtils.mean(cornerImage.pixels), 227 FloatArrayStatsUtils.std(cornerImage.pixels), 228 cornerImage.max(), cornerImage.min()); 229 230 detectMaxima(cornerImage, window); 231 } 232 233 public FValuePixel findMaximum(Rectangle window) { 234 final FImage cornerImage = createInterestPointMap(); 235 final FValuePixel c = cornerImage.extractROI(window).maxPixel(); 236 c.translate(window.x, window.y); 237 return c; 238 } 239 240 public class Maxima { 241 public int x, y; 242 public float val; 243 244 public Maxima(int x, int y, float v) { 245 this.x = x; 246 this.y = y; 247 this.val = v; 248 } 249 } 250 251 protected void detectMaxima(FImage image, Rectangle window) { 252 maxima = new ArrayList<Maxima>(); 253 254 for (int y = borderSkip; y < image.height - borderSkip; y++) { 255 for (int x = borderSkip; x < image.width - borderSkip; x++) { 256 if (!window.isInside(new Pixel(x, y))) 257 continue; 258 final float curr = image.pixels[y][x]; 259 if (curr > image.pixels[y - 1][x - 1] 260 && curr >= image.pixels[y - 1][x] 261 && curr >= image.pixels[y - 1][x + 1] 262 && curr >= image.pixels[y][x - 1] 263 && curr >= image.pixels[y][x + 1] 264 && curr >= image.pixels[y + 1][x - 1] 265 && curr >= image.pixels[y + 1][x] 266 && curr >= image.pixels[y + 1][x + 1]) 267 { 268 maxima.add(new Maxima(x, y, curr)); 269 } 270 } 271 } 272 273 Collections.sort(maxima, new Comparator<Maxima>() { 274 @Override 275 public int compare(Maxima o1, Maxima o2) { 276 if (o1.val == o2.val) 277 return 0; 278 return o1.val < o2.val ? 1 : -1; 279 } 280 }); 281 } 282 283 public abstract FImage createInterestPointMap(); 284 285 @Override 286 public List<InterestPointData> getInterestPoints(int npoints) { 287 if (npoints < 0 || npoints > maxima.size()) 288 npoints = maxima.size(); 289 final List<InterestPointData> ipdata = new ArrayList<InterestPointData>(); 290 291 for (int i = 0; i < npoints; i++) { 292 final InterestPointData ipd = new InterestPointData(); 293 294 ipd.x = maxima.get(i).x; 295 ipd.y = maxima.get(i).y; 296 ipd.scale = integrationScale; 297 ipd.score = maxima.get(i).val; 298 299 ipdata.add(ipd); 300 } 301 302 return ipdata; 303 } 304 305 public float getDetIntScaleFactor() { 306 return detIntScaleFactor; 307 } 308 309 public void setDetIntScaleFactor(float detIntScaleFactor) { 310 this.detIntScaleFactor = detIntScaleFactor; 311 } 312 313 public float getDetectionScale() { 314 return detectionScale; 315 } 316 317 public void setImageBlurred(boolean blurred) { 318 this.blurred = blurred; 319 } 320 321 @Override 322 public void setDetectionScale(float detectionScale) { 323 this.detectionScale = detectionScale; 324 this.integrationScale = this.detectionScale * this.detIntScaleFactor; 325 } 326 327 public float getIntegrationScale() { 328 return integrationScale; 329 } 330 331 public void setIntegrationScale(float integrationScale) { 332 this.integrationScale = integrationScale; 333 this.detectionScale = integrationScale * (1f / this.detIntScaleFactor); 334 } 335 336 @Override 337 public List<InterestPointData> getInterestPoints() { 338 return getInterestPoints(-1); 339 } 340 341 @Override 342 public List<InterestPointData> getInterestPoints(float threshold) { 343 return getInterestPointsThresh(threshold); 344 } 345 346 public List<InterestPointData> getInterestPointsThresh(float thresh) { 347 final List<InterestPointData> ipdata = new ArrayList<InterestPointData>(); 348 349 for (final Maxima m : maxima) { 350 if (m.val < thresh) 351 continue; 352 353 final InterestPointData ipd = new InterestPointData(); 354 355 ipd.x = m.x; 356 ipd.y = m.y; 357 ipd.scale = integrationScale; 358 ipd.score = m.val; 359 360 ipdata.add(ipd); 361 } 362 363 return ipdata; 364 } 365 366 public Matrix getSecondMomentsAt(int x, int y) { 367 final Matrix secondMoments = new Matrix(2, 2); 368 secondMoments.set(0, 0, lxmxblur.pixels[y][x]); 369 secondMoments.set(0, 1, lxmyblur.pixels[y][x]); 370 secondMoments.set(1, 0, lxmyblur.pixels[y][x]); 371 secondMoments.set(1, 1, lymyblur.pixels[y][x]); 372 return secondMoments; 373 } 374 375 @Override 376 public AbstractStructureTensorIPD clone() { 377 AbstractStructureTensorIPD a = null; 378 try { 379 a = (AbstractStructureTensorIPD) super.clone(); 380 } catch (final CloneNotSupportedException e) { 381 return null; 382 } 383 return a; 384 } 385 386 public int pointsFound() { 387 return this.maxima.size(); 388 } 389}