1
2
3
4
5
6
7
8
9 package org.openimaj.video.tracking.klt;
10 import org.openimaj.image.FImage;
11 import org.openimaj.math.geometry.shape.Shape;
12
13
14
15
16
17
18
19 public class TrackingContext {
20 protected final static int _mindist = 10;
21 protected final static int _window_size = 7;
22 protected final static int _min_eigenvalue = 1;
23 protected final static float _min_determinant = 0.01f;
24 protected final static float _min_displacement = 0.1f;
25 protected final static int _max_iterations = 10;
26 protected final static float _max_residue = 10.0f;
27 protected final static float _grad_sigma = 1.0f;
28 protected final static float _smooth_sigma_fact = 0.1f;
29 protected final static float _pyramid_sigma_fact = 0.9f;
30 protected final static float _step_factor = 1.0f;
31 protected static boolean _sequentialMode = false;
32
33 protected final static boolean _lighting_insensitive = false;
34
35 protected final static int _affineConsistencyCheck = -1;
36 protected final static int _affine_window_size = 15;
37 protected final static int _affine_max_iterations = 10;
38 protected final static float _affine_max_residue = 10.0f;
39 protected final static float _affine_min_displacement = 0.02f;
40 protected final static float _affine_max_displacement_differ = 1.5f;
41
42 protected final static boolean _smoothBeforeSelecting = true;
43 protected final static boolean _writeInternalImages = false;
44 protected final static int _search_range = 15;
45 protected final static int _nSkippedPixels = 0;
46
47
48 int mindist;
49 int window_width, window_height;
50 boolean sequentialMode;
51
52
53 boolean smoothBeforeSelecting;
54
55 boolean writeInternalImages;
56
57
58 boolean lighting_insensitive;
59
60
61
62 int min_eigenvalue;
63 float min_determinant;
64 float min_displacement;
65 int max_iterations;
66 float max_residue;
67 float grad_sigma;
68 float smooth_sigma_fact;
69 float pyramid_sigma_fact;
70 float step_factor;
71 int nSkippedPixels;
72 int borderx;
73 int bordery;
74 int nPyramidLevels;
75 int subsampling;
76
77
78 int affine_window_width, affine_window_height;
79 int affineConsistencyCheck;
80
81
82
83
84
85 int affine_max_iterations;
86 float affine_max_residue;
87 float affine_min_displacement;
88 float affine_max_displacement_differ;
89
90
91 private Shape targetArea = null;
92
93
94 private Pyramid pyramid_last;
95 private Pyramid pyramid_last_gradx;
96 private Pyramid pyramid_last_grady;
97
98
99
100
101 public PyramidSet previousPyramidSet(){
102 if(pyramid_last == null)
103 return null;
104 else
105 return new PyramidSet(pyramid_last,pyramid_last_gradx,pyramid_last_grady);
106 }
107
108
109
110
111
112
113 public TrackingContext()
114 {
115
116 this.mindist = _mindist;
117 this.window_width = _window_size;
118 this.window_height = _window_size;
119 this.sequentialMode = _sequentialMode;
120 this.smoothBeforeSelecting = _smoothBeforeSelecting;
121 this.writeInternalImages = _writeInternalImages;
122
123 this.lighting_insensitive = _lighting_insensitive;
124 this.min_eigenvalue = _min_eigenvalue;
125 this.min_determinant = _min_determinant;
126 this.max_iterations = _max_iterations;
127 this.min_displacement = _min_displacement;
128 this.max_residue = _max_residue;
129 this.grad_sigma = _grad_sigma;
130 this.smooth_sigma_fact = _smooth_sigma_fact;
131 this.pyramid_sigma_fact = _pyramid_sigma_fact;
132 this.step_factor = _step_factor;
133 this.nSkippedPixels = _nSkippedPixels;
134 this.pyramid_last = null;
135 this.pyramid_last_gradx = null;
136 this.pyramid_last_grady = null;
137
138 this.affineConsistencyCheck = _affineConsistencyCheck;
139 this.affine_window_width = _affine_window_size;
140 this.affine_window_height = _affine_window_size;
141 this.affine_max_iterations = _affine_max_iterations;
142 this.affine_max_residue = _affine_max_residue;
143 this.affine_min_displacement = _affine_min_displacement;
144 this.affine_max_displacement_differ = _affine_max_displacement_differ;
145
146
147 changeTCPyramid(_search_range);
148
149
150
151 updateTCBorder();
152 }
153
154
155
156
157 @Override
158 public String toString()
159 {
160 String s = "";
161 s += String.format("\n\nTracking context:\n\n");
162 s += String.format("\tmindist = %d\n", this.mindist);
163 s += String.format("\twindow_width = %d\n", this.window_width);
164 s += String.format("\twindow_height = %d\n", this.window_height);
165 s += String.format("\tsequentialMode = %s\n", this.sequentialMode ? "true" : "false");
166 s += String.format("\tsmoothBeforeSelecting = %s\n", this.smoothBeforeSelecting ? "true" : "false");
167 s += String.format("\twriteInternalImages = %s\n", this.writeInternalImages ? "true" : "false");
168
169 s += String.format("\tmin_eigenvalue = %d\n", this.min_eigenvalue);
170 s += String.format("\tmin_determinant = %f\n", this.min_determinant);
171 s += String.format("\tmin_displacement = %f\n", this.min_displacement);
172 s += String.format("\tmax_iterations = %d\n", this.max_iterations);
173 s += String.format("\tmax_residue = %f\n", this.max_residue);
174 s += String.format("\tgrad_sigma = %f\n", this.grad_sigma);
175 s += String.format("\tsmooth_sigma_fact = %f\n", this.smooth_sigma_fact);
176 s += String.format("\tpyramid_sigma_fact = %f\n", this.pyramid_sigma_fact);
177 s += String.format("\tnSkippedPixels = %d\n", this.nSkippedPixels);
178 s += String.format("\tborderx = %d\n", this.borderx);
179 s += String.format("\tbordery = %d\n", this.bordery);
180 s += String.format("\tnPyramidLevels = %d\n", this.nPyramidLevels);
181 s += String.format("\tsubsampling = %d\n", this.subsampling);
182
183 s += String.format("\n\tpyramid_last = %s\n", (this.pyramid_last!=null) ? "points to old image" : "null");
184 s += String.format("\tpyramid_last_gradx = %s\n", (this.pyramid_last_gradx!=null) ? "points to old image" : "null");
185 s += String.format("\tpyramid_last_grady = %s\n", (this.pyramid_last_grady!=null) ? "points to old image" : "null");
186 s += String.format("\n\n");
187
188 return s;
189 }
190
191
192
193
194
195
196
197 public void changeTCPyramid(int search_range) {
198 float window_halfwidth;
199 float subsampling;
200
201
202 if (this.window_width % 2 != 1) {
203 this.window_width = this.window_width+1;
204 System.err.format("(KLTChangeTCPyramid) Window width must be odd. Changing to %d.\n", this.window_width);
205 }
206 if (this.window_height % 2 != 1) {
207 this.window_height = this.window_height+1;
208 System.err.format("(KLTChangeTCPyramid) Window height must be odd. Changing to %d.\n", this.window_height);
209 }
210 if (this.window_width < 3) {
211 this.window_width = 3;
212 System.err.format("(KLTChangeTCPyramid) Window width must be at least three. \nChanging to %d.\n", this.window_width);
213 }
214 if (this.window_height < 3) {
215 this.window_height = 3;
216 System.err.format("(KLTChangeTCPyramid) Window height must be at least three. \nChanging to %d.\n", this.window_height);
217 }
218 window_halfwidth = Math.min(this.window_width,this.window_height)/2.0f;
219
220 subsampling = search_range / window_halfwidth;
221
222 if (subsampling < 1.0) {
223 this.nPyramidLevels = 1;
224 } else if (subsampling <= 3.0) {
225 this.nPyramidLevels = 2;
226 this.subsampling = 2;
227 } else if (subsampling <= 5.0) {
228 this.nPyramidLevels = 2;
229 this.subsampling = 4;
230 } else if (subsampling <= 9.0) {
231 this.nPyramidLevels = 2;
232 this.subsampling = 8;
233 } else {
234
235
236
237
238
239
240
241 float val = (float) (Math.log(7.0*subsampling+1.0)/Math.log(8.0));
242 this.nPyramidLevels = (int) (val + 0.99);
243 this.subsampling = 8;
244 }
245 }
246
247
248
249
250
251 float _pyramidSigma()
252 {
253 return (this.pyramid_sigma_fact * this.subsampling);
254 }
255
256
257
258
259
260
261
262 public void updateTCBorder() {
263 float val;
264 int pyramid_gauss_hw;
265 int smooth_gauss_hw;
266 int gauss_width;
267 int num_levels = this.nPyramidLevels;
268 int n_invalid_pixels;
269 int window_hw;
270 int ss = this.subsampling;
271 int ss_power;
272 int border;
273 int i;
274
275
276 if (this.window_width % 2 != 1) {
277 this.window_width = this.window_width+1;
278 System.err.format("(KLTUpdateTCBorder) Window width must be odd. Changing to %d.\n", this.window_width);
279 }
280 if (this.window_height % 2 != 1) {
281 this.window_height = this.window_height+1;
282 System.err.format("(KLTUpdateTCBorder) Window height must be odd. Changing to %d.\n", this.window_height);
283 }
284 if (this.window_width < 3) {
285 this.window_width = 3;
286 System.err.format("(KLTUpdateTCBorder) Window width must be at least three. \nChanging to %d.\n", this.window_width);
287 }
288 if (this.window_height < 3) {
289 this.window_height = 3;
290 System.err.format("(KLTUpdateTCBorder) Window height must be at least three. \nChanging to %d.\n", this.window_height);
291 }
292 window_hw = Math.max(this.window_width, this.window_height)/2;
293
294
295 gauss_width=_getKernelWidths(computeSmoothSigma())[0];
296 smooth_gauss_hw = gauss_width/2;
297
298 gauss_width = _getKernelWidths(_pyramidSigma())[0];
299 pyramid_gauss_hw = gauss_width/2;
300
301
302
303
304
305
306
307 n_invalid_pixels = smooth_gauss_hw;
308 for (i = 1 ; i < num_levels ; i++) {
309 val = ((float) n_invalid_pixels + pyramid_gauss_hw) / ss;
310 n_invalid_pixels = (int) (val + 0.99);
311 }
312
313
314 ss_power = 1;
315 for (i = 1 ; i < num_levels ; i++)
316 ss_power *= ss;
317
318
319
320 border = (n_invalid_pixels + window_hw) * ss_power;
321
322 this.borderx = border;
323 this.bordery = border;
324 }
325
326
327
328
329 void stopSequentialMode()
330 {
331 this.sequentialMode = false;
332
333 this.pyramid_last = null;
334 this.pyramid_last_gradx = null;
335 this.pyramid_last_grady = null;
336 }
337
338 float computeSmoothSigma() {
339 return (smooth_sigma_fact * Math.max(window_width, window_height));
340 }
341
342 private class ConvolutionKernel {
343 private static final int MAX_KERNEL_WIDTH = 71;
344
345 int width;
346 float [] data = new float[MAX_KERNEL_WIDTH];
347 }
348
349 ConvolutionKernel gauss_kernel = new ConvolutionKernel();
350 ConvolutionKernel gaussderiv_kernel = new ConvolutionKernel();
351 float sigma_last = -10.0f;
352
353
354
355
356 void _computeKernels(float sigma, ConvolutionKernel gauss, ConvolutionKernel gaussderiv) {
357 final float factor = 0.01f;
358 int i;
359
360
361 {
362 final int hw = ConvolutionKernel.MAX_KERNEL_WIDTH / 2;
363 float max_gauss = 1.0f, max_gaussderiv = (float) (sigma*Math.exp(-0.5f));
364
365
366 for (i = -hw ; i <= hw ; i++) {
367 gauss.data[i+hw] = (float) Math.exp(-i*i / (2*sigma*sigma));
368 gaussderiv.data[i+hw] = -i * gauss.data[i+hw];
369 }
370
371
372 gauss.width = ConvolutionKernel.MAX_KERNEL_WIDTH;
373 for (i = -hw ; Math.abs(gauss.data[i+hw] / max_gauss) < factor ; i++) gauss.width -= 2;
374 gaussderiv.width = ConvolutionKernel.MAX_KERNEL_WIDTH;
375 for (i = -hw ; Math.abs(gaussderiv.data[i+hw] / max_gaussderiv) < factor ; i++) gaussderiv.width -= 2;
376 if (gauss.width == ConvolutionKernel.MAX_KERNEL_WIDTH || gaussderiv.width == ConvolutionKernel.MAX_KERNEL_WIDTH)
377 throw new RuntimeException(
378 String.format("(_computeKernels) MAX_KERNEL_WIDTH %d is too small for a sigma of %f", ConvolutionKernel.MAX_KERNEL_WIDTH, sigma)
379 );
380 }
381
382
383 for (i = 0 ; i < gauss.width ; i++)
384 gauss.data[i] = gauss.data[i+(ConvolutionKernel.MAX_KERNEL_WIDTH-gauss.width)/2];
385 for (i = 0 ; i < gaussderiv.width ; i++)
386 gaussderiv.data[i] = gaussderiv.data[i+(ConvolutionKernel.MAX_KERNEL_WIDTH-gaussderiv.width)/2];
387
388 {
389 final int hw = gaussderiv.width / 2;
390 float den;
391
392 den = 0.0f;
393 for (i = 0 ; i < gauss.width ; i++) den += gauss.data[i];
394 for (i = 0 ; i < gauss.width ; i++) gauss.data[i] /= den;
395 den = 0.0f;
396 for (i = -hw ; i <= hw ; i++) den -= i*gaussderiv.data[i+hw];
397 for (i = -hw ; i <= hw ; i++) gaussderiv.data[i+hw] /= den;
398 }
399
400 sigma_last = sigma;
401 }
402
403
404
405
406
407
408 int [] _getKernelWidths(float sigma)
409 {
410 _computeKernels(sigma, gauss_kernel, gaussderiv_kernel);
411 int gauss_width = gauss_kernel.width;
412 int gaussderiv_width = gaussderiv_kernel.width;
413
414 return new int[] {gauss_width, gaussderiv_width};
415 }
416
417
418
419
420
421 void _convolveImageHoriz(FImage imgin, ConvolutionKernel kernel, FImage imgout)
422 {
423 float sum;
424 int radius = kernel.width / 2;
425 int ncols = imgin.width, nrows = imgin.height;
426 int i, j, k;
427
428
429 assert(kernel.width % 2 == 1);
430
431
432 assert(imgin != imgout);
433
434
435 for (j = 0 ; j < nrows ; j++) {
436 int ptrout = 0;
437
438
439 for (i = 0 ; i < radius ; i++)
440 imgout.pixels[j][ptrout++] = 0.0f; /
441
442
443 for ( ; i < ncols - radius ; i++) {
444
445 int ppp = i - radius;
446
447 sum = 0.0f;
448 for (k = kernel.width-1 ; k >= 0 ; k--)
449 sum += imgin.pixels[j][ppp++] * kernel.data[k];
450 imgout.pixels[j][ptrout++] = sum;
451 }
452
453
454 for ( ; i < ncols ; i++)
455 imgout.pixels[j][ptrout++] = 0.0f; /
456
457
458
459
460
461
462
463 void _convolveImageVert(FImage imgin, ConvolutionKernel kernel, FImage imgout) {
464 float sum;
465 int radius = kernel.width / 2;
466 int ncols = imgin.width, nrows = imgin.height;
467 int i, j, k;
468
469
470 assert(kernel.width % 2 == 1);
471
472
473 assert(imgin != imgout);
474
475
476 for (i = 0 ; i < ncols ; i++) {
477 int ptrout = 0;
478
479 for (j = 0 ; j < radius ; j++) {
480 imgout.pixels[ptrout][i] = 0;
481 ptrout++;
482 }
483
484
485 for ( ; j < nrows - radius ; j++) {
486 int ppp = (j - radius);
487 sum = 0.0f;
488 for (k = kernel.width-1 ; k >= 0 ; k--) {
489 sum += imgin.pixels[ppp][i] * kernel.data[k];
490 ppp++;
491 }
492 imgout.pixels[ptrout][i] = sum;
493 ptrout ++;
494 }
495
496
497 for ( ; j < nrows ; j++) {
498 imgout.pixels[ptrout][i] = 0;
499 ptrout++;
500 }
501 }
502 }
503
504
505
506
507
508 void _convolveSeparate(FImage imgin, ConvolutionKernel horiz_kernel, ConvolutionKernel vert_kernel, FImage imgout)
509 {
510
511 FImage tmpimg = new FImage(imgin.width, imgin.height);
512
513
514 _convolveImageHoriz(imgin, horiz_kernel, tmpimg);
515
516 _convolveImageVert(tmpimg, vert_kernel, imgout);
517 }
518
519
520
521
522
523
524
525
526 public void computeGradients(FImage img, float sigma, FImage gradx, FImage grady) {
527
528 if (Math.abs(sigma - sigma_last) > 0.05)
529 _computeKernels(sigma, gauss_kernel, gaussderiv_kernel);
530
531 _convolveSeparate(img, gaussderiv_kernel, gauss_kernel, gradx);
532 _convolveSeparate(img, gauss_kernel, gaussderiv_kernel, grady);
533 }
534
535
536
537
538 public int getMinDist() {
539 return mindist;
540 }
541
542
543
544
545
546 public void setMinDist(int mindist) {
547 this.mindist = mindist;
548 }
549
550
551
552
553 public int getWindowWidth() {
554 return window_width;
555 }
556
557
558
559
560
561 public void setWindowWidth(int window_width) {
562 this.window_width = window_width;
563 }
564
565
566
567
568 public int getWindowHeight() {
569 return window_height;
570 }
571
572
573
574
575
576 public void setWindowHeight(int window_height) {
577 this.window_height = window_height;
578 }
579
580
581
582
583 public boolean sequentialMode() {
584 return sequentialMode;
585 }
586
587
588
589
590
591 public void setSequentialMode(boolean sequentialMode) {
592 this.sequentialMode = sequentialMode;
593 }
594
595
596
597
598 public boolean writeInternalImages() {
599 return writeInternalImages;
600 }
601
602
603
604
605
606 public void setWriteInternalImages(boolean writeInternalImages) {
607 this.writeInternalImages = writeInternalImages;
608 }
609
610
611
612
613 public boolean isLightingInsensitive() {
614 return lighting_insensitive;
615 }
616
617
618
619
620
621 public void setLightingInsensitive(boolean lighting_insensitive) {
622 this.lighting_insensitive = lighting_insensitive;
623 }
624
625
626
627
628 public int getMinEigenvalue() {
629 return min_eigenvalue;
630 }
631
632
633
634
635
636 public void setMinEigenvalue(int min_eigenvalue) {
637 this.min_eigenvalue = min_eigenvalue;
638 }
639
640
641
642
643 public float getMinDeterminant() {
644 return min_determinant;
645 }
646
647
648
649
650
651 public void setMinDeterminant(float min_determinant) {
652 this.min_determinant = min_determinant;
653 }
654
655
656
657
658 public float getMinDisplacement() {
659 return min_displacement;
660 }
661
662
663
664
665
666 public void setMinDisplacement(float min_displacement) {
667 this.min_displacement = min_displacement;
668 }
669
670
671
672
673 public int getMaxIterations() {
674 return max_iterations;
675 }
676
677
678
679
680
681 public void setMaxIterations(int max_iterations) {
682 this.max_iterations = max_iterations;
683 }
684
685
686
687
688 public float getMaxResidue() {
689 return max_residue;
690 }
691
692
693
694
695
696 public void setMaxResidue(float max_residue) {
697 this.max_residue = max_residue;
698 }
699
700
701
702
703 public float getStepFactor() {
704 return step_factor;
705 }
706
707
708
709
710
711 public void setStepFactor(float step_factor) {
712 this.step_factor = step_factor;
713 }
714
715
716
717
718 public int getSubsampling() {
719 return subsampling;
720 }
721
722
723
724
725
726 public void setSubsampling(int subsampling) {
727 this.subsampling = subsampling;
728 }
729
730
731
732
733 public int getAffineConsistencyCheck() {
734 return affineConsistencyCheck;
735 }
736
737
738
739
740
741 public void setAffineConsistencyCheck(int affineConsistencyCheck) {
742 this.affineConsistencyCheck = affineConsistencyCheck;
743 }
744
745
746
747
748
749 public void setTargetArea(Shape targetArea) {
750 this.targetArea = targetArea;
751 }
752
753
754
755
756 public Shape getTargetArea() {
757 return targetArea;
758 }
759
760
761
762
763 public void setPreviousPyramid(PyramidSet pyr) {
764 this.pyramid_last = pyr.imgPyr;
765 this.pyramid_last_gradx = pyr.gradx;
766 this.pyramid_last_grady = pyr.grady;
767 }
768
769
770
771
772 public PyramidSet getPreviousPyramid() {
773 PyramidSet ret = new PyramidSet();
774 ret.imgPyr = this.pyramid_last;
775 ret.gradx = this.pyramid_last_gradx;
776 ret.grady = this.pyramid_last_grady;
777 return ret;
778 }
779 }