1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30 package org.openimaj.image.analysis.algorithm;
31
32 import java.io.File;
33 import java.io.IOException;
34 import java.util.Comparator;
35
36 import org.openimaj.image.DisplayUtilities;
37 import org.openimaj.image.FImage;
38 import org.openimaj.image.ImageUtilities;
39 import org.openimaj.image.MBFImage;
40 import org.openimaj.image.analyser.ImageAnalyser;
41 import org.openimaj.image.colour.RGBColour;
42 import org.openimaj.image.pixel.FValuePixel;
43 import org.openimaj.image.processing.algorithm.MeanCenter;
44 import org.openimaj.math.geometry.shape.Rectangle;
45 import org.openimaj.util.queue.BoundedPriorityQueue;
46
47
48
49
50
51
52
53 public class TemplateMatcher implements ImageAnalyser<FImage> {
54
55
56
57
58
59 public enum Mode {
60
61
62
63
64
65
66
67 SUM_SQUARED_DIFFERENCE {
68 @Override
69 protected float computeMatchScore(final FImage image, final FImage template, final int x, final int y, final Object workingSpace) {
70 final float[][] imageData = image.pixels;
71 final float[][] templateData = template.pixels;
72
73 return computeMatchScore(imageData, x, y, templateData, 0, 0, template.width, template.height);
74 }
75
76 @Override
77 public final float computeMatchScore(final float[][] img, int x, int y, final float[][] template, final int templateX, final int templateY, final int templateWidth, final int templateHeight) {
78 final int stopX1 = templateWidth + x;
79 final int stopY1 = templateHeight + y;
80 final int stopX2 = templateWidth + templateX;
81 final int stopY2 = templateHeight + templateY;
82
83 float score = 0;
84 for (int yy1=y, yy2=templateY; yy1<stopY1 && yy2 < stopY2; yy1++, yy2++) {
85 for (int xx1=x, xx2=templateX; xx1<stopX1 && xx2 < stopX2; xx1++, xx2++) {
86 float diff = (img[yy1][xx1] - template[yy2][xx2]);
87
88 score += diff*diff;
89 }
90 }
91 return score;
92 }
93
94
95 @Override
96 public boolean scoresAscending() {
97 return false;
98 }
99 },
100
101
102
103
104
105
106
107 NORM_SUM_SQUARED_DIFFERENCE {
108 @Override
109 protected float computeMatchScore(final FImage image, final FImage template, final int x, final int y, final Object workingSpace) {
110 float score = 0;
111 float si = 0;
112 final float st = (Float)workingSpace;
113
114 final float[][] imageData = image.pixels;
115 final float[][] templateData = template.pixels;
116
117 final int stopX = template.width + x;
118 final int stopY = template.height + y;
119
120 for (int yy=y, j=0; yy<stopY; yy++, j++) {
121 for (int xx=x, i=0; xx<stopX; xx++, i++) {
122 float diff = (imageData[yy][xx] - templateData[j][i]);
123
124 score += diff*diff;
125 si += (imageData[yy][xx] * imageData[yy][xx]);
126 }
127 }
128
129 return (float) (score / Math.sqrt(si*st));
130 }
131
132 @Override
133 public final float computeMatchScore(final float[][] img, final int x, final int y, final float[][] template, final int templateX, final int templateY, final int templateWidth, final int templateHeight) {
134 final int stopX1 = templateWidth + x;
135 final int stopY1 = templateHeight + y;
136 final int stopX2 = templateWidth + templateX;
137 final int stopY2 = templateHeight + templateY;
138
139 float s1 = 0;
140 float s2 = 0;
141 float score = 0;
142
143 for (int yy1=y, yy2=templateY; yy1<stopY1 && yy2 < stopY2; yy1++, yy2++) {
144 for (int xx1=x, xx2=templateX; xx1<stopX1 && xx2 < stopX2; xx1++, xx2++) {
145 float diff = (img[yy1][xx1] - template[yy2][xx2]);
146 score += diff*diff;
147 s1 += (img[yy1][xx1] * img[yy1][xx1]);
148 s2 += (template[yy2][xx2] * template[yy2][xx2]);
149 }
150 }
151
152 return (float) (score / Math.sqrt(s1*s2));
153 }
154
155
156 @Override
157 public boolean scoresAscending() {
158 return false;
159 }
160
161 @Override
162 public Float prepareWorkingSpace(FImage template) {
163 float sumsq = 0;
164
165 for (int y=0; y<template.height; y++)
166 for (int x=0; x<template.width; x++)
167 sumsq += template.pixels[y][x]*template.pixels[y][x];
168
169 return new Float(sumsq);
170 }
171 },
172
173
174
175
176
177
178
179 CORRELATION {
180 @Override
181 protected float computeMatchScore(final FImage image, final FImage template, final int x, final int y, final Object workingSpace) {
182 final float[][] imageData = image.pixels;
183 final float[][] templateData = template.pixels;
184
185 return computeMatchScore(imageData, x, y, templateData, 0, 0, template.width, template.height);
186 }
187
188 @Override
189 public float computeMatchScore(final float[][] img, final int x, final int y, final float[][] template, final int templateX, final int templateY, final int templateWidth, final int templateHeight) {
190 float score = 0;
191
192 final int stopX1 = templateWidth + x;
193 final int stopY1 = templateHeight + y;
194 final int stopX2 = templateWidth + templateX;
195 final int stopY2 = templateHeight + templateY;
196
197 for (int yy1=y, yy2=templateY; yy1<stopY1 && yy2 < stopY2; yy1++, yy2++) {
198 for (int xx1=x, xx2=templateX; xx1<stopX1 && xx2 < stopX2; xx1++, xx2++) {
199 float prod = (img[yy1][xx1] * template[yy2][xx2]);
200
201 score += prod;
202 }
203 }
204
205 return score;
206 }
207
208 @Override
209 public boolean scoresAscending() {
210 return true;
211 }
212 },
213
214
215
216
217
218
219
220 NORM_CORRELATION {
221 @Override
222 protected float computeMatchScore(final FImage image, final FImage template, final int x, final int y, final Object workingSpace) {
223 float score = 0;
224 float si = 0;
225 final float st = (Float)workingSpace;
226
227 final float[][] imageData = image.pixels;
228 final float[][] templateData = template.pixels;
229
230 final int stopX = template.width + x;
231 final int stopY = template.height + y;
232
233 for (int yy=y, j=0; yy<stopY; yy++, j++) {
234 for (int xx=x, i=0; xx<stopX; xx++, i++) {
235 float prod = (imageData[yy][xx] * templateData[j][i]);
236 score += prod;
237 si += (imageData[yy][xx] * imageData[yy][xx]);
238 }
239 }
240
241 return (float) (score / Math.sqrt(si*st));
242 }
243
244 @Override
245 public float computeMatchScore(final float[][] img, final int x, final int y, final float[][] template, final int templateX, final int templateY, final int templateWidth, final int templateHeight) {
246 float score = 0;
247 float s1 = 0;
248 float s2 = 0;
249
250 final int stopX1 = templateWidth + x;
251 final int stopY1 = templateHeight + y;
252 final int stopX2 = templateWidth + templateX;
253 final int stopY2 = templateHeight + templateY;
254
255 int xx1,xx2,yy1,yy2;
256 for (yy1=y, yy2=templateY; yy1<stopY1 && yy2 < stopY2; yy1++, yy2++) {
257 for (xx1=x, xx2=templateX; xx1<stopX1 && xx2 < stopX2; xx1++, xx2++) {
258 float prod = (img[yy1][xx1] * template[yy2][xx2]);
259
260 s1 += (img[yy1][xx1] * img[yy1][xx1]);
261 s2 += (template[yy2][xx2] * template[yy2][xx2]);
262
263 score += prod;
264 }
265 }
266
267 return (float) (score / Math.sqrt(s1*s2));
268 }
269
270 @Override
271 public boolean scoresAscending() {
272 return true;
273 }
274
275 @Override
276 public Float prepareWorkingSpace(FImage template) {
277 float sumsq = 0;
278
279 for (int y=0; y<template.height; y++)
280 for (int x=0; x<template.width; x++)
281 sumsq += template.pixels[y][x]*template.pixels[y][x];
282
283 return new Float(sumsq);
284 }
285 },
286
287
288
289
290
291
292
293 CORRELATION_COEFFICIENT {
294 @Override
295 protected final float computeMatchScore(final FImage image, final FImage template, final int x, final int y, final Object workingSpace) {
296 final float[][] imageData = image.pixels;
297 final float[][] templateData = template.pixels;
298
299 final float templateMean = (Float)workingSpace;
300 final float imgMean = MeanCenter.patchMean(imageData);
301
302 return computeMatchScore(imageData, x, y, imgMean, templateData, 0, 0, template.width, template.height, templateMean);
303 }
304
305 @Override
306 public final float computeMatchScore(final float[][] img, final int x, final int y, final float[][] template, final int templateX, final int templateY, final int templateWidth, final int templateHeight) {
307 float imgMean = MeanCenter.patchMean(img, x, y, templateWidth, templateHeight);
308 float templateMean = MeanCenter.patchMean(template, templateX, templateY, templateWidth, templateHeight);
309
310 return computeMatchScore(img, x, y, imgMean, template, templateX, templateY, templateWidth, templateHeight, templateMean);
311 }
312
313 private final float computeMatchScore(final float[][] img, final int x, final int y, final float imgMean, final float[][] template, final int templateX, final int templateY, final int templateWidth, final int templateHeight, final float templateMean) {
314 final int stopX1 = templateWidth + x;
315 final int stopY1 = templateHeight + y;
316 final int stopX2 = templateWidth + templateX;
317 final int stopY2 = templateHeight + templateY;
318
319 float score = 0;
320 for (int yy1=y, yy2=templateY; yy1<stopY1 && yy2 < stopY2; yy1++, yy2++) {
321 for (int xx1=x, xx2=templateX; xx1<stopX1 && xx2 < stopX2; xx1++, xx2++) {
322 float prod = ((img[yy1][xx1] - imgMean) * (template[yy2][xx2] - templateMean));
323 score += prod;
324 }
325 }
326
327 return score;
328 }
329
330 @Override
331 public boolean scoresAscending() {
332 return true;
333 }
334
335 @Override
336 public Float prepareWorkingSpace(FImage template) {
337 return MeanCenter.patchMean(template.pixels);
338 }
339 },
340
341
342
343
344
345
346
347 NORM_CORRELATION_COEFFICIENT {
348 @Override
349 protected final float computeMatchScore(final FImage image, final FImage template, final int x, final int y, final Object workingSpace) {
350 final int width = template.width;
351 final int height = template.height;
352
353 float imgMean = MeanCenter.patchMean(image.pixels, x, y, width, height);
354
355 float score = 0;
356 float si = 0;
357 final float st = (Float)workingSpace;
358
359 final float[][] imageData = image.pixels;
360 final float[][] templateData = template.pixels;
361
362 for (int j=0; j<height; j++) {
363 for (int i=0; i<width; i++) {
364 float ival = imageData[j+y][i+x] - imgMean;
365
366 float prod = (ival * templateData[j][i]);
367
368 score += prod;
369
370 si += (ival * ival);
371 }
372 }
373
374 double norm = Math.sqrt(si * st);
375
376 if (norm == 0) return 0;
377
378 return (float) (score / norm);
379 }
380
381 @Override
382 public final float computeMatchScore(final float[][] img, final int x, final int y, final float[][] template, final int templateX, final int templateY, final int templateWidth, final int templateHeight) {
383 float imgMean = MeanCenter.patchMean(img, x,y,templateWidth, templateHeight);
384 float templateMean = MeanCenter.patchMean(template, templateX, templateY, templateWidth, templateHeight);
385
386 final int stopX1 = templateWidth + x;
387 final int stopY1 = templateHeight + y;
388 final int stopX2 = templateWidth + templateX;
389 final int stopY2 = templateHeight + templateY;
390
391 float score = 0;
392 float s1 = 0;
393 float s2 = 0;
394
395 for (int yy1=y, yy2=templateY; yy1<stopY1 && yy2 < stopY2; yy1++, yy2++) {
396 for (int xx1=x, xx2=templateX; xx1<stopX1 && xx2 < stopX2; xx1++, xx2++) {
397 float ival = (img[yy1][xx1] - imgMean);
398 float tval = (template[yy2][xx2] - templateMean);
399
400 float prod = (ival * tval);
401
402 score += prod;
403
404 s1 += (ival * ival);
405 s2 += (tval * tval);
406 }
407 }
408
409 double norm = Math.sqrt(s1*s2);
410
411 if (norm == 0) return 0;
412
413 return (float) (score / norm);
414 }
415
416 @Override
417 public boolean scoresAscending() {
418 return true;
419 }
420
421 @Override
422 public FImage prepareTemplate(FImage template) {
423 return template.process(new MeanCenter());
424 }
425
426 @Override
427 public Float prepareWorkingSpace(FImage template) {
428 float sumsq = 0;
429
430 for (int y=0; y<template.height; y++)
431 for (int x=0; x<template.width; x++)
432 sumsq += template.pixels[y][x]*template.pixels[y][x];
433
434 return sumsq;
435 }
436 }
437 ;
438
439
440
441
442
443
444
445
446
447
448
449 protected abstract float computeMatchScore(final FImage image, final FImage template, final int x, final int y, final Object workingSpace);
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466 public abstract float computeMatchScore(final float[][] img, int x, int y, final float[][] template, final int templateX, final int templateY, final int templateWidth, final int templateHeight);
467
468
469
470
471
472 public abstract boolean scoresAscending();
473
474
475
476
477
478
479
480 protected FImage prepareTemplate(FImage template) {
481 return template;
482 }
483
484
485
486
487
488
489
490
491
492
493
494 protected Object prepareWorkingSpace(FImage template) {
495 return null;
496 }
497 }
498
499 private FImage template;
500 private Mode mode;
501 private Object workingSpace;
502 private Rectangle searchBounds;
503 private FImage responseMap;
504
505
506
507
508
509
510
511
512
513
514 public TemplateMatcher(FImage template, Mode mode) {
515 this.mode = mode;
516 this.template = mode.prepareTemplate(template);
517 this.workingSpace = mode.prepareWorkingSpace(this.template);
518 }
519
520
521
522
523
524
525
526
527
528
529
530 public TemplateMatcher(FImage template, Mode mode, Rectangle bounds) {
531 this.searchBounds = bounds;
532 this.mode = mode;
533 this.template = mode.prepareTemplate(template);
534 this.workingSpace = mode.prepareWorkingSpace(this.template);
535 }
536
537
538
539
540 public Rectangle getSearchBounds() {
541 return searchBounds;
542 }
543
544
545
546
547
548
549
550
551
552 public void setSearchBounds(Rectangle searchBounds) {
553 this.searchBounds = searchBounds;
554 }
555
556
557
558
559
560
561
562
563
564
565 @Override
566 public void analyseImage(FImage image) {
567 Rectangle searchSpace = null;
568
569 if (this.searchBounds != null) {
570 final int halfWidth = template.width / 2;
571 final int halfHeight = template.height / 2;
572
573 float x = Math.max(searchBounds.x - halfWidth, 0);
574 x = Math.min(x, image.width-template.width);
575 float width = searchBounds.width;
576 if (searchBounds.x - halfWidth < 0) {
577 width += (searchBounds.x - halfWidth);
578 }
579 if (x + width > image.width - template.width)
580 width += (image.width - template.width) - (x+width);
581
582 float y = Math.max(searchBounds.y - halfHeight, 0);
583 y = Math.min(y, image.height - template.height);
584 float height = searchBounds.height;
585 if (searchBounds.y - halfHeight < 0) {
586 height += (searchBounds.y - halfHeight);
587 }
588 if (y + height > image.height - template.height)
589 height += (image.height - template.height) - (y+height);
590
591 searchSpace = new Rectangle(
592 x,
593 y,
594 width,
595 height
596 );
597
598 } else {
599 searchSpace = new Rectangle(
600 0,
601 0,
602 image.width - template.width + 1,
603 image.height - template.height + 1
604 );
605 }
606
607 final int scanX = (int) searchSpace.x;
608 final int scanY = (int) searchSpace.y;
609 final int scanWidth = (int)searchSpace.width;
610 final int scanHeight = (int)searchSpace.height;
611
612 responseMap = new FImage(scanWidth, scanHeight);
613 final float[][] responseMapData = responseMap.pixels;
614
615 for (int y=0; y<scanHeight; y++) {
616 for (int x=0; x<scanWidth; x++) {
617 responseMapData[y][x] = mode.computeMatchScore(image, template, x+scanX, y+scanY, workingSpace);
618 }
619 }
620 }
621
622
623
624
625
626
627
628 public FValuePixel[] getBestResponses(int numResponses) {
629 Comparator<FValuePixel> comparator = mode.scoresAscending() ? FValuePixel.ReverseValueComparator.INSTANCE : FValuePixel.ValueComparator.INSTANCE;
630
631 return getBestResponses(numResponses, responseMap, getXOffset(), getYOffset(), comparator);
632 }
633
634
635
636
637
638
639
640
641
642
643
644 public static FValuePixel[] getBestResponses(int numResponses, FImage responseMap, int offsetX, int offsetY, Comparator<FValuePixel> comparator) {
645 BoundedPriorityQueue<FValuePixel> bestResponses = new BoundedPriorityQueue<FValuePixel>(numResponses, comparator);
646
647 final float[][] responseMapData = responseMap.pixels;
648
649 final int scanWidth = responseMap.width;
650 final int scanHeight = responseMap.height;
651
652 FValuePixel tmpPixel = new FValuePixel(0, 0, 0);
653 for (int y=0; y<scanHeight; y++) {
654 for (int x=0; x<scanWidth; x++) {
655 tmpPixel.x = x + offsetX;
656 tmpPixel.y = y + offsetY;
657 tmpPixel.value = responseMapData[y][x];
658
659 FValuePixel removed = bestResponses.offerItem(tmpPixel);
660
661 if (removed == null)
662 tmpPixel = new FValuePixel(0, 0, 0);
663 else
664 tmpPixel = removed;
665 }
666 }
667
668 return bestResponses.toOrderedArray(new FValuePixel[numResponses]);
669 }
670
671
672
673
674
675
676 public int getXOffset() {
677 final int halfWidth = template.width / 2;
678
679 if (this.searchBounds == null)
680 return halfWidth;
681 else
682 return (int) Math.max(searchBounds.x , halfWidth);
683 }
684
685
686
687
688
689
690 public int getYOffset() {
691 final int halfHeight = template.height / 2;
692
693 if (this.searchBounds == null)
694 return halfHeight;
695 else
696 return (int) Math.max(searchBounds.y , halfHeight);
697 }
698
699
700
701
702 public FImage getResponseMap() {
703 return responseMap;
704 }
705
706
707
708
709
710 public FImage getTemplate() {
711 return template;
712 }
713
714
715
716
717
718
719 public static void main(String[] args) throws IOException {
720 FImage image = ImageUtilities.readF(new File("/Users/jsh2/Desktop/image.png"));
721 FImage template = image.extractROI(100, 100, 100, 100);
722 image.fill(0f);
723 image.drawImage(template, 100, 100);
724
725 TemplateMatcher matcher = new TemplateMatcher(template, Mode.CORRELATION);
726 matcher.setSearchBounds(new Rectangle(100,100,200,200));
727 image.analyseWith(matcher);
728 DisplayUtilities.display(matcher.responseMap.normalise());
729
730 MBFImage cimg = image.toRGB();
731 for (FValuePixel p : matcher.getBestResponses(10)) {
732 System.out.println(p);
733 cimg.drawPoint(p, RGBColour.RED, 1);
734 }
735
736 cimg.drawShape(matcher.getSearchBounds(), RGBColour.BLUE);
737 cimg.drawShape(new Rectangle(100,100,100,100), RGBColour.GREEN);
738
739 DisplayUtilities.display(cimg);
740 }
741 }