1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43 package com.jsaragih;
44
45 import java.io.BufferedReader;
46 import java.io.BufferedWriter;
47 import java.io.FileNotFoundException;
48 import java.io.FileReader;
49 import java.io.FileWriter;
50 import java.io.IOException;
51 import java.io.InputStream;
52 import java.io.InputStreamReader;
53 import java.util.List;
54 import java.util.Scanner;
55
56 import org.openimaj.citation.annotation.Reference;
57 import org.openimaj.citation.annotation.ReferenceType;
58 import org.openimaj.image.FImage;
59 import org.openimaj.image.analysis.algorithm.FourierTemplateMatcher;
60 import org.openimaj.image.processing.resize.ResizeProcessor;
61 import org.openimaj.math.geometry.shape.Rectangle;
62
63 import Jama.Matrix;
64
65
66
67
68
69
70
71
72
73
74
75
76
77 @Reference(
78 type = ReferenceType.Inproceedings,
79 author = { "Jason M. Saragih", "Simon Lucey", "Jeffrey F. Cohn" },
80 title = "Face alignment through subspace constrained mean-shifts",
81 year = "2009",
82 booktitle = "IEEE 12th International Conference on Computer Vision, ICCV 2009, Kyoto, Japan, September 27 - October 4, 2009",
83 pages = { "1034", "1041" },
84 publisher = "IEEE",
85 customData = {
86 "doi", "http://dx.doi.org/10.1109/ICCV.2009.5459377",
87 "researchr", "http://researchr.org/publication/SaragihLC09",
88 "cites", "0",
89 "citedby", "0"
90 }
91 )
92 public class Tracker {
93 private static boolean init = false;
94 static { Tracker.init(); }
95
96 static synchronized void init() {
97 if (!init) {
98 System.err.println("This software uses the OpenIMAJ port of FaceTracker.");
99 System.err.println("FaceTracker has a different license to the rest of OpenIMAJ:");
100 System.err.println();
101 System.err.println("FaceTracker Licence");
102 System.err.println("-------------------");
103 System.err.println("(Academic, non-commercial, not-for-profit licence)");
104 System.err.println();
105 System.err.println("Copyright (c) 2010 Jason Mora Saragih");
106 System.err.println("All rights reserved.");
107 System.err.println("");
108 System.err.println("Redistribution and use in source and binary forms, with or without ");
109 System.err.println("modification, are permitted provided that the following conditions are met:");
110 System.err.println();
111 System.err.println(" * The software is provided under the terms of this licence stricly for");
112 System.err.println(" academic, non-commercial, not-for-profit purposes.");
113 System.err.println(" * Redistributions of source code must retain the above copyright notice, ");
114 System.err.println(" this list of conditions (licence) and the following disclaimer.");
115 System.err.println(" * Redistributions in binary form must reproduce the above copyright ");
116 System.err.println(" notice, this list of conditions (licence) and the following disclaimer ");
117 System.err.println(" in the documentation and/or other materials provided with the ");
118 System.err.println(" distribution.");
119 System.err.println(" * The name of the author may not be used to endorse or promote products ");
120 System.err.println(" derived from this software without specific prior written permission.");
121 System.err.println(" * As this software depends on other libraries, the user must adhere to and ");
122 System.err.println(" keep in place any licencing terms of those libraries.");
123 System.err.println(" * Any publications arising from the use of this software, including but");
124 System.err.println(" not limited to academic journal and conference publications, technical");
125 System.err.println(" reports and manuals, must cite the following work:");
126 System.err.println();
127 System.err.println(" J. M. Saragih, S. Lucey, and J. F. Cohn. Face Alignment through Subspace ");
128 System.err.println(" Constrained Mean-Shifts. International Journal of Computer Vision ");
129 System.err.println(" (ICCV), September, 2009.");
130 System.err.println();
131 System.err.println("THIS SOFTWARE IS PROVIDED BY THE AUTHOR \"AS IS\" AND ANY EXPRESS OR IMPLIED ");
132 System.err.println("WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF ");
133 System.err.println("MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO ");
134 System.err.println("EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, ");
135 System.err.println("INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, ");
136 System.err.println("BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, ");
137 System.err.println("DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY ");
138 System.err.println("OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING ");
139 System.err.println("NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, ");
140 System.err.println("EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.");
141 init = true;
142 }
143 }
144
145 private static final double TSCALE = 0.3;
146
147
148 public CLM _clm;
149
150
151 FDet _fdet;
152
153
154 long _frame;
155
156
157 MFCheck _fcheck;
158
159
160 public Matrix _shape;
161
162
163 public Matrix _rshape;
164
165
166 Rectangle _rect;
167
168
169 double[] _simil;
170
171 FImage gray_, temp_;
172
173 private FImage small_;
174
175 Tracker(CLM clm, FDet fdet, MFCheck fcheck, Matrix rshape, double[] simil) {
176 _clm = clm;
177 _fdet = fdet;
178 _fcheck = fcheck;
179
180 _rshape = rshape.copy();
181 _simil = simil;
182
183 _shape = new Matrix(2 * _clm._pdm.nPoints(), 1);
184 _rect.x = 0;
185 _rect.y = 0;
186 _rect.width = 0;
187 _rect.height = 0;
188 _frame = -1;
189 _clm._pdm.identity(_clm._plocal, _clm._pglobl);
190 }
191
192 Tracker() {
193 }
194
195
196 public void frameReset() {
197 _frame = -1;
198 }
199
200 static Tracker load(final String fname) throws FileNotFoundException {
201 BufferedReader br = null;
202 try {
203 br = new BufferedReader(new FileReader(fname));
204 Scanner sc = new Scanner(br);
205 return read(sc, true);
206 } finally {
207 try {
208 br.close();
209 } catch (IOException e) {
210 }
211 }
212 }
213
214
215
216
217
218 public static Tracker load(final InputStream in) {
219 BufferedReader br = null;
220 try {
221 br = new BufferedReader(new InputStreamReader(in));
222 Scanner sc = new Scanner(br);
223 return read(sc, true);
224 } finally {
225 try {
226 br.close();
227 } catch (IOException e) {
228 }
229 }
230 }
231
232 void save(final String fname) throws IOException {
233 BufferedWriter bw = null;
234 try {
235 bw = new BufferedWriter(new FileWriter(fname));
236
237 write(bw);
238 } finally {
239 try {
240 if (bw != null)
241 bw.close();
242 } catch (IOException e) {
243 }
244 }
245 }
246
247 void write(BufferedWriter s) throws IOException {
248 s.write(IO.Types.TRACKER.ordinal() + " ");
249
250 _clm.write(s);
251 _fdet.write(s);
252 _fcheck.write(s);
253 IO.writeMat(s, _rshape);
254
255 s.write(_simil[0] + " " + _simil[1] + " " + _simil[2] + " " + _simil[3]
256 + " ");
257 }
258
259 static Tracker read(Scanner s, boolean readType) {
260 if (readType) {
261 int type = s.nextInt();
262 assert (type == IO.Types.TRACKER.ordinal());
263 }
264 Tracker tracker = new Tracker();
265 tracker._clm = CLM.read(s, true);
266 tracker._fdet = FDet.read(s, true);
267 tracker._fcheck = MFCheck.read(s, true);
268 tracker._rshape = IO.readMat(s);
269
270 tracker._simil = new double[] { s.nextDouble(), s.nextDouble(),
271 s.nextDouble(), s.nextDouble() };
272 tracker._shape = new Matrix(2 * tracker._clm._pdm.nPoints(), 1);
273
274 tracker._rect = new Rectangle();
275 tracker._rect.x = 0;
276 tracker._rect.y = 0;
277 tracker._rect.width = 0;
278 tracker._rect.height = 0;
279 tracker._frame = -1;
280 tracker._clm._pdm.identity(tracker._clm._plocal, tracker._clm._pglobl);
281
282 return tracker;
283 }
284
285
286
287
288
289
290
291
292
293
294
295 public int track(FImage im, int[] wSize, final int fpd, final int nIter,
296 final double clamp, final double fTol, final boolean fcheck) {
297 gray_ = im;
298
299 boolean gen, rsize = true;
300 Rectangle R = new Rectangle(0, 0, 0, 0);
301
302 if ((_frame < 0) || (fpd >= 0 && fpd < _frame)) {
303 _frame = 0;
304 List<Rectangle> RL = _fdet.detect(gray_);
305
306
307 double max = 0;
308 for (Rectangle r : RL)
309 if (r.calculateArea() > max) {
310 max = r.calculateArea();
311 R = r;
312 }
313
314 gen = true;
315 } else {
316 R = redetect(gray_);
317 gen = false;
318 }
319
320 if ((R.width == 0) || (R.height == 0)) {
321 _frame = -1;
322 return -1;
323 }
324
325 _frame++;
326
327 if (gen) {
328 initShape(R, _shape);
329 _clm._pdm.calcParams(_shape, _clm._plocal, _clm._pglobl);
330 } else {
331 double tx = R.x - _rect.x;
332 double ty = R.y - _rect.y;
333
334 _clm._pglobl.getArray()[4][0] += tx;
335 _clm._pglobl.getArray()[5][0] += ty;
336
337 rsize = false;
338 }
339
340 _clm.fit(gray_, wSize, nIter, clamp, fTol);
341
342 _clm._pdm.calcShape2D(_shape, _clm._plocal, _clm._pglobl);
343
344 if (fcheck) {
345 if (!_fcheck.check(_clm.getViewIdx(), gray_, _shape))
346 return -1;
347 }
348
349 _rect = updateTemplate(gray_, _shape, rsize);
350
351 if ((_rect.width == 0) || (_rect.height == 0))
352 return -1;
353
354 return 0;
355 }
356
357 void initShape(Rectangle r, Matrix shape) {
358 assert ((shape.getRowDimension() == _rshape.getRowDimension()) && (shape
359 .getColumnDimension() == _rshape.getColumnDimension()));
360
361 int n = _rshape.getRowDimension() / 2;
362
363 double a = r.width * Math.cos(_simil[1]) * _simil[0] + 1;
364 double b = r.width * Math.sin(_simil[1]) * _simil[0];
365
366 double tx = r.x + (int) (r.width / 2) + r.width * _simil[2];
367 double ty = r.y + (int) (r.height / 2) + r.height * _simil[3];
368
369 double[][] s = _rshape.getArray();
370 double[][] d = shape.getArray();
371
372 for (int i = 0; i < n; i++) {
373 d[i][0] = a * s[i][0] - b * s[i + n][0] + tx;
374 d[i + n][0] = b * s[i][0] + a * s[i + n][0] + ty;
375 }
376 }
377
378 Rectangle redetect(FImage im) {
379 final int ww = im.width;
380 final int hh = im.height;
381
382 int w = (int) (TSCALE * ww - temp_.width + 1);
383 int h = (int) (TSCALE * hh - temp_.height + 1);
384
385 small_ = ResizeProcessor.resample(im, (int) (TSCALE * ww),
386 (int) (TSCALE * hh));
387
388 h = small_.height - temp_.height + 1;
389 w = small_.width - temp_.width + 1;
390
391 FourierTemplateMatcher matcher = new FourierTemplateMatcher(temp_,
392 FourierTemplateMatcher.Mode.NORM_CORRELATION_COEFFICIENT);
393 matcher.analyseImage(small_);
394 float[][] ncc_ = matcher.getResponseMap().pixels;
395
396 Rectangle R = temp_.getBounds();
397 float v, vb = -2;
398 for (int y = 0; y < h; y++) {
399 for (int x = 0; x < w; x++) {
400 v = ncc_[y][x];
401
402 if (v > vb) {
403 vb = v;
404 R.x = x;
405 R.y = y;
406 }
407 }
408 }
409
410 R.x *= 1.0 / TSCALE;
411 R.y *= 1.0 / TSCALE;
412
413 R.width *= 1.0 / TSCALE;
414 R.height *= 1.0 / TSCALE;
415
416 return R;
417 }
418
419 Rectangle updateTemplate(FImage im, Matrix s, boolean rsize) {
420 final int n = s.getRowDimension() / 2;
421
422 double[][] sv = s.getArray();
423 double xmax = sv[0][0], ymax = sv[n][0], xmin = sv[0][0], ymin = sv[n][0];
424
425 for (int i = 0; i < n; i++) {
426 double vx = sv[i][0];
427 double vy = sv[i + n][0];
428
429 xmax = Math.max(xmax, vx);
430 ymax = Math.max(ymax, vy);
431
432 xmin = Math.min(xmin, vx);
433 ymin = Math.min(ymin, vy);
434 }
435
436 if ((xmin < 0) || (ymin < 0) || (xmax >= im.width)
437 || (ymax >= im.height) || Double.isNaN(xmin)
438 || Double.isInfinite(xmin) || Double.isNaN(xmax)
439 || Double.isInfinite(xmax) || Double.isNaN(ymin)
440 || Double.isInfinite(ymin) || Double.isNaN(ymax)
441 || Double.isInfinite(ymax)) {
442 return new Rectangle(0, 0, 0, 0);
443 } else {
444 xmin *= TSCALE;
445 ymin *= TSCALE;
446 xmax *= TSCALE;
447 ymax *= TSCALE;
448
449 Rectangle R = new Rectangle((float) Math.floor(xmin),
450 (float) Math.floor(ymin), (float) Math.ceil(xmax - xmin),
451 (float) Math.ceil(ymax - ymin));
452
453 final int ww = im.width;
454 final int hh = im.height;
455
456 if (rsize) {
457 small_ = ResizeProcessor.resample(im, (int) (TSCALE * ww),
458 (int) (TSCALE * hh));
459 }
460
461 temp_ = small_.extractROI(R);
462
463 R.x *= 1.0 / TSCALE;
464 R.y *= 1.0 / TSCALE;
465 R.width *= 1.0 / TSCALE;
466 R.height *= 1.0 / TSCALE;
467
468 return R;
469 }
470 }
471 }