001/** 002 * FaceTracker Licence 003 * ------------------- 004 * (Academic, non-commercial, not-for-profit licence) 005 * 006 * Copyright (c) 2010 Jason Mora Saragih 007 * All rights reserved. 008 * 009 * Redistribution and use in source and binary forms, with or without 010 * modification, are permitted provided that the following conditions are met: 011 * 012 * * The software is provided under the terms of this licence stricly for 013 * academic, non-commercial, not-for-profit purposes. 014 * * Redistributions of source code must retain the above copyright notice, 015 * this list of conditions (licence) and the following disclaimer. 016 * * Redistributions in binary form must reproduce the above copyright 017 * notice, this list of conditions (licence) and the following disclaimer 018 * in the documentation and/or other materials provided with the 019 * distribution. 020 * * The name of the author may not be used to endorse or promote products 021 * derived from this software without specific prior written permission. 022 * * As this software depends on other libraries, the user must adhere to and 023 * keep in place any licencing terms of those libraries. 024 * * Any publications arising from the use of this software, including but 025 * not limited to academic journal and conference publications, technical 026 * reports and manuals, must cite the following work: 027 * 028 * J. M. Saragih, S. Lucey, and J. F. Cohn. Face Alignment through Subspace 029 * Constrained Mean-Shifts. International Journal of Computer Vision 030 * (ICCV), September, 2009. 031 * 032 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR IMPLIED 033 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 034 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 035 * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 036 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 037 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 038 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 039 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 040 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 041 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 042 */ 043package com.jsaragih; 044 045import java.io.BufferedReader; 046import java.io.BufferedWriter; 047import java.io.FileNotFoundException; 048import java.io.FileReader; 049import java.io.FileWriter; 050import java.io.IOException; 051import java.io.InputStream; 052import java.io.InputStreamReader; 053import java.util.List; 054import java.util.Scanner; 055 056import org.openimaj.citation.annotation.Reference; 057import org.openimaj.citation.annotation.ReferenceType; 058import org.openimaj.image.FImage; 059import org.openimaj.image.analysis.algorithm.FourierTemplateMatcher; 060import org.openimaj.image.processing.resize.ResizeProcessor; 061import org.openimaj.math.geometry.shape.Rectangle; 062 063import Jama.Matrix; 064 065/** 066 * The initial ported version of the CLMTracker that can only track a single 067 * face in an image. It's had only a few small changes to allow it to work with 068 * the list which the face detector class now returns. Unless you're sure you 069 * only want to track a single face, you should probably use the new 070 * OpenIMAJ org.openimaj.image.processing.face.tracking.clm.MultiTracker class 071 * from the faces sub-project instead. 072 * 073 * @author Jason Mora Saragih 074 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk) 075 * @author David Dupplaw (dpd@ecs.soton.ac.uk) 076 */ 077@Reference( 078 type = ReferenceType.Inproceedings, 079 author = { "Jason M. Saragih", "Simon Lucey", "Jeffrey F. Cohn" }, 080 title = "Face alignment through subspace constrained mean-shifts", 081 year = "2009", 082 booktitle = "IEEE 12th International Conference on Computer Vision, ICCV 2009, Kyoto, Japan, September 27 - October 4, 2009", 083 pages = { "1034", "1041" }, 084 publisher = "IEEE", 085 customData = { 086 "doi", "http://dx.doi.org/10.1109/ICCV.2009.5459377", 087 "researchr", "http://researchr.org/publication/SaragihLC09", 088 "cites", "0", 089 "citedby", "0" 090 } 091 ) 092public class Tracker { 093 private static boolean init = false; 094 static { Tracker.init(); } 095 096 static synchronized void init() { 097 if (!init) { 098 System.err.println("This software uses the OpenIMAJ port of FaceTracker."); 099 System.err.println("FaceTracker has a different license to the rest of OpenIMAJ:"); 100 System.err.println(); 101 System.err.println("FaceTracker Licence"); 102 System.err.println("-------------------"); 103 System.err.println("(Academic, non-commercial, not-for-profit licence)"); 104 System.err.println(); 105 System.err.println("Copyright (c) 2010 Jason Mora Saragih"); 106 System.err.println("All rights reserved."); 107 System.err.println(""); 108 System.err.println("Redistribution and use in source and binary forms, with or without "); 109 System.err.println("modification, are permitted provided that the following conditions are met:"); 110 System.err.println(); 111 System.err.println(" * The software is provided under the terms of this licence stricly for"); 112 System.err.println(" academic, non-commercial, not-for-profit purposes."); 113 System.err.println(" * Redistributions of source code must retain the above copyright notice, "); 114 System.err.println(" this list of conditions (licence) and the following disclaimer."); 115 System.err.println(" * Redistributions in binary form must reproduce the above copyright "); 116 System.err.println(" notice, this list of conditions (licence) and the following disclaimer "); 117 System.err.println(" in the documentation and/or other materials provided with the "); 118 System.err.println(" distribution."); 119 System.err.println(" * The name of the author may not be used to endorse or promote products "); 120 System.err.println(" derived from this software without specific prior written permission."); 121 System.err.println(" * As this software depends on other libraries, the user must adhere to and "); 122 System.err.println(" keep in place any licencing terms of those libraries."); 123 System.err.println(" * Any publications arising from the use of this software, including but"); 124 System.err.println(" not limited to academic journal and conference publications, technical"); 125 System.err.println(" reports and manuals, must cite the following work:"); 126 System.err.println(); 127 System.err.println(" J. M. Saragih, S. Lucey, and J. F. Cohn. Face Alignment through Subspace "); 128 System.err.println(" Constrained Mean-Shifts. International Journal of Computer Vision "); 129 System.err.println(" (ICCV), September, 2009."); 130 System.err.println(); 131 System.err.println("THIS SOFTWARE IS PROVIDED BY THE AUTHOR \"AS IS\" AND ANY EXPRESS OR IMPLIED "); 132 System.err.println("WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF "); 133 System.err.println("MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO "); 134 System.err.println("EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, "); 135 System.err.println("INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, "); 136 System.err.println("BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, "); 137 System.err.println("DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY "); 138 System.err.println("OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING "); 139 System.err.println("NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, "); 140 System.err.println("EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."); 141 init = true; 142 } 143 } 144 145 private static final double TSCALE = 0.3; 146 147 /** Constrained Local Model */ 148 public CLM _clm; 149 150 /** Face Detector */ 151 FDet _fdet; 152 153 /** Frame number since last detection */ 154 long _frame; 155 156 /** Failure checker */ 157 MFCheck _fcheck; 158 159 /** Current shape */ 160 public Matrix _shape; 161 162 /** Reference shape */ 163 public Matrix _rshape; 164 165 /** Detected rectangle */ 166 Rectangle _rect; 167 168 /** Initialization similarity */ 169 double[] _simil; 170 171 FImage gray_, temp_; 172 173 private FImage small_; 174 175 Tracker(CLM clm, FDet fdet, MFCheck fcheck, Matrix rshape, double[] simil) { 176 _clm = clm; 177 _fdet = fdet; 178 _fcheck = fcheck; 179 180 _rshape = rshape.copy(); 181 _simil = simil; 182 183 _shape = new Matrix(2 * _clm._pdm.nPoints(), 1); 184 _rect.x = 0; 185 _rect.y = 0; 186 _rect.width = 0; 187 _rect.height = 0; 188 _frame = -1; 189 _clm._pdm.identity(_clm._plocal, _clm._pglobl); 190 } 191 192 Tracker() { 193 } 194 195 /** Reset frame number (will perform detection in next image) */ 196 public void frameReset() { 197 _frame = -1; 198 } 199 200 static Tracker load(final String fname) throws FileNotFoundException { 201 BufferedReader br = null; 202 try { 203 br = new BufferedReader(new FileReader(fname)); 204 Scanner sc = new Scanner(br); 205 return read(sc, true); 206 } finally { 207 try { 208 br.close(); 209 } catch (IOException e) { 210 } 211 } 212 } 213 214 /** 215 * @param in 216 * @return a tracker 217 */ 218 public static Tracker load(final InputStream in) { 219 BufferedReader br = null; 220 try { 221 br = new BufferedReader(new InputStreamReader(in)); 222 Scanner sc = new Scanner(br); 223 return read(sc, true); 224 } finally { 225 try { 226 br.close(); 227 } catch (IOException e) { 228 } 229 } 230 } 231 232 void save(final String fname) throws IOException { 233 BufferedWriter bw = null; 234 try { 235 bw = new BufferedWriter(new FileWriter(fname)); 236 237 write(bw); 238 } finally { 239 try { 240 if (bw != null) 241 bw.close(); 242 } catch (IOException e) { 243 } 244 } 245 } 246 247 void write(BufferedWriter s) throws IOException { 248 s.write(IO.Types.TRACKER.ordinal() + " "); 249 250 _clm.write(s); 251 _fdet.write(s); 252 _fcheck.write(s); 253 IO.writeMat(s, _rshape); 254 255 s.write(_simil[0] + " " + _simil[1] + " " + _simil[2] + " " + _simil[3] 256 + " "); 257 } 258 259 static Tracker read(Scanner s, boolean readType) { 260 if (readType) { 261 int type = s.nextInt(); 262 assert (type == IO.Types.TRACKER.ordinal()); 263 } 264 Tracker tracker = new Tracker(); 265 tracker._clm = CLM.read(s, true); 266 tracker._fdet = FDet.read(s, true); 267 tracker._fcheck = MFCheck.read(s, true); 268 tracker._rshape = IO.readMat(s); 269 270 tracker._simil = new double[] { s.nextDouble(), s.nextDouble(), 271 s.nextDouble(), s.nextDouble() }; 272 tracker._shape = new Matrix(2 * tracker._clm._pdm.nPoints(), 1); 273 274 tracker._rect = new Rectangle(); 275 tracker._rect.x = 0; 276 tracker._rect.y = 0; 277 tracker._rect.width = 0; 278 tracker._rect.height = 0; 279 tracker._frame = -1; 280 tracker._clm._pdm.identity(tracker._clm._plocal, tracker._clm._pglobl); 281 282 return tracker; 283 } 284 285 /** 286 * @param im 287 * @param wSize 288 * @param fpd 289 * @param nIter 290 * @param clamp 291 * @param fTol 292 * @param fcheck 293 * @return 0 for success, -1 for redetect 294 */ 295 public int track(FImage im, int[] wSize, final int fpd, final int nIter, 296 final double clamp, final double fTol, final boolean fcheck) { 297 gray_ = im; 298 299 boolean gen, rsize = true; 300 Rectangle R = new Rectangle(0, 0, 0, 0); 301 302 if ((_frame < 0) || (fpd >= 0 && fpd < _frame)) { 303 _frame = 0; 304 List<Rectangle> RL = _fdet.detect(gray_); 305 306 // Get largest 307 double max = 0; 308 for (Rectangle r : RL) 309 if (r.calculateArea() > max) { 310 max = r.calculateArea(); 311 R = r; 312 } 313 314 gen = true; 315 } else { 316 R = redetect(gray_); 317 gen = false; 318 } 319 320 if ((R.width == 0) || (R.height == 0)) { 321 _frame = -1; 322 return -1; 323 } 324 325 _frame++; 326 327 if (gen) { 328 initShape(R, _shape); 329 _clm._pdm.calcParams(_shape, _clm._plocal, _clm._pglobl); 330 } else { 331 double tx = R.x - _rect.x; 332 double ty = R.y - _rect.y; 333 334 _clm._pglobl.getArray()[4][0] += tx; 335 _clm._pglobl.getArray()[5][0] += ty; 336 337 rsize = false; 338 } 339 340 _clm.fit(gray_, wSize, nIter, clamp, fTol); 341 342 _clm._pdm.calcShape2D(_shape, _clm._plocal, _clm._pglobl); 343 344 if (fcheck) { 345 if (!_fcheck.check(_clm.getViewIdx(), gray_, _shape)) 346 return -1; 347 } 348 349 _rect = updateTemplate(gray_, _shape, rsize); 350 351 if ((_rect.width == 0) || (_rect.height == 0)) 352 return -1; 353 354 return 0; 355 } 356 357 void initShape(Rectangle r, Matrix shape) { 358 assert ((shape.getRowDimension() == _rshape.getRowDimension()) && (shape 359 .getColumnDimension() == _rshape.getColumnDimension())); 360 361 int n = _rshape.getRowDimension() / 2; 362 363 double a = r.width * Math.cos(_simil[1]) * _simil[0] + 1; 364 double b = r.width * Math.sin(_simil[1]) * _simil[0]; 365 366 double tx = r.x + (int) (r.width / 2) + r.width * _simil[2]; 367 double ty = r.y + (int) (r.height / 2) + r.height * _simil[3]; 368 369 double[][] s = _rshape.getArray(); 370 double[][] d = shape.getArray(); 371 372 for (int i = 0; i < n; i++) { 373 d[i][0] = a * s[i][0] - b * s[i + n][0] + tx; 374 d[i + n][0] = b * s[i][0] + a * s[i + n][0] + ty; 375 } 376 } 377 378 Rectangle redetect(FImage im) { 379 final int ww = im.width; 380 final int hh = im.height; 381 382 int w = (int) (TSCALE * ww - temp_.width + 1); 383 int h = (int) (TSCALE * hh - temp_.height + 1); 384 385 small_ = ResizeProcessor.resample(im, (int) (TSCALE * ww), 386 (int) (TSCALE * hh)); 387 388 h = small_.height - temp_.height + 1; 389 w = small_.width - temp_.width + 1; 390 391 FourierTemplateMatcher matcher = new FourierTemplateMatcher(temp_, 392 FourierTemplateMatcher.Mode.NORM_CORRELATION_COEFFICIENT); 393 matcher.analyseImage(small_); 394 float[][] ncc_ = matcher.getResponseMap().pixels; 395 396 Rectangle R = temp_.getBounds(); 397 float v, vb = -2; 398 for (int y = 0; y < h; y++) { 399 for (int x = 0; x < w; x++) { 400 v = ncc_[y][x]; 401 402 if (v > vb) { 403 vb = v; 404 R.x = x; 405 R.y = y; 406 } 407 } 408 } 409 410 R.x *= 1.0 / TSCALE; 411 R.y *= 1.0 / TSCALE; 412 413 R.width *= 1.0 / TSCALE; 414 R.height *= 1.0 / TSCALE; 415 416 return R; 417 } 418 419 Rectangle updateTemplate(FImage im, Matrix s, boolean rsize) { 420 final int n = s.getRowDimension() / 2; 421 422 double[][] sv = s.getArray(); // ,y = s.begin<double>()+n; 423 double xmax = sv[0][0], ymax = sv[n][0], xmin = sv[0][0], ymin = sv[n][0]; 424 425 for (int i = 0; i < n; i++) { 426 double vx = sv[i][0]; 427 double vy = sv[i + n][0]; 428 429 xmax = Math.max(xmax, vx); 430 ymax = Math.max(ymax, vy); 431 432 xmin = Math.min(xmin, vx); 433 ymin = Math.min(ymin, vy); 434 } 435 436 if ((xmin < 0) || (ymin < 0) || (xmax >= im.width) 437 || (ymax >= im.height) || Double.isNaN(xmin) 438 || Double.isInfinite(xmin) || Double.isNaN(xmax) 439 || Double.isInfinite(xmax) || Double.isNaN(ymin) 440 || Double.isInfinite(ymin) || Double.isNaN(ymax) 441 || Double.isInfinite(ymax)) { 442 return new Rectangle(0, 0, 0, 0); 443 } else { 444 xmin *= TSCALE; 445 ymin *= TSCALE; 446 xmax *= TSCALE; 447 ymax *= TSCALE; 448 449 Rectangle R = new Rectangle((float) Math.floor(xmin), 450 (float) Math.floor(ymin), (float) Math.ceil(xmax - xmin), 451 (float) Math.ceil(ymax - ymin)); 452 453 final int ww = im.width; 454 final int hh = im.height; 455 456 if (rsize) { 457 small_ = ResizeProcessor.resample(im, (int) (TSCALE * ww), 458 (int) (TSCALE * hh)); 459 } 460 461 temp_ = small_.extractROI(R); 462 463 R.x *= 1.0 / TSCALE; 464 R.y *= 1.0 / TSCALE; 465 R.width *= 1.0 / TSCALE; 466 R.height *= 1.0 / TSCALE; 467 468 return R; 469 } 470 } 471}