001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.image.processing.face.tracking.clm; 031 032import java.io.BufferedReader; 033import java.io.FileNotFoundException; 034import java.io.FileReader; 035import java.io.IOException; 036import java.io.InputStream; 037import java.io.InputStreamReader; 038import java.util.ArrayList; 039import java.util.Iterator; 040import java.util.List; 041import java.util.Scanner; 042 043import org.openimaj.image.FImage; 044import org.openimaj.image.analysis.algorithm.FourierTemplateMatcher; 045import org.openimaj.image.processing.face.detection.DetectedFace; 046import org.openimaj.image.processing.resize.ResizeProcessor; 047import org.openimaj.math.geometry.shape.Rectangle; 048 049import Jama.Matrix; 050 051import com.jsaragih.CLM; 052import com.jsaragih.FDet; 053import com.jsaragih.IO; 054import com.jsaragih.MFCheck; 055 056/** 057 * A CLM Tracker that is able to deal with multiple tracks within the same 058 * video. To instantiate use {@link #load(InputStream)} to get a 059 * {@link TrackerVars} object which can be used to construct the Tracker. 060 * <p> 061 * <code><pre>MultiTracker t = new MultiTracker( MultiTracker.load( new File("face.tracker.file") ) );</pre></code> 062 * 063 * @author David Dupplaw (dpd@ecs.soton.ac.uk) 064 */ 065public class MultiTracker { 066 /** 067 * Encapsulates the variables for a single tracked face. This includes the 068 * model, the shape parameters, the last-matched template and the bounding 069 * rectangle. 070 * 071 * @author David Dupplaw (dpd@ecs.soton.ac.uk) 072 * @created 4 Jul 2012 073 * @version $Author$, $Revision$, $Date$ 074 */ 075 public static class TrackedFace extends DetectedFace { 076 /** The constrained local model */ 077 public CLM clm; 078 079 /** The current shape */ 080 public Matrix shape; 081 082 /** The reference shape */ 083 public Matrix referenceShape; 084 085 /** The template image */ 086 public FImage templateImage; 087 088 /** The last matched bounds: _rect */ 089 public Rectangle lastMatchBounds; 090 091 /** The redetected bounds: R */ 092 public Rectangle redetectedBounds; 093 094 protected boolean gen = true; 095 096 /** 097 * @param r 098 * The rectangle in which the initial face was found 099 * @param tv 100 * The initial tracker vars to use 101 */ 102 public TrackedFace(final Rectangle r, final TrackerVars tv) { 103 this.redetectedBounds = r; 104 this.clm = tv.clm.copy(); 105 this.shape = tv.shape.copy(); 106 this.referenceShape = tv.referenceShape.copy(); 107 } 108 109 @Override 110 public Rectangle getBounds() 111 { 112 return this.lastMatchBounds; 113 } 114 115 @Override 116 public String toString() { 117 return "Face[" 118 + (this.redetectedBounds == null ? "null" : this.redetectedBounds 119 .toString()) + "]"; 120 } 121 } 122 123 /** 124 * This class is used to store the tracker variables when they are loaded 125 * from a file. These variables can then be copied to make specific 126 * trackers. 127 * 128 * @author David Dupplaw (dpd@ecs.soton.ac.uk) 129 * @created 5 Jul 2012 130 * @version $Author$, $Revision$, $Date$ 131 */ 132 public static class TrackerVars { 133 /** The constrained local model */ 134 public CLM clm; 135 136 /** The current shape */ 137 public Matrix shape; 138 139 /** The reference shape */ 140 public Matrix referenceShape; 141 142 /** The Face detector */ 143 public FDet faceDetector; 144 145 /** The failure checker */ 146 public MFCheck failureCheck; 147 148 /** Initialisation similarity */ 149 double[] similarity; 150 } 151 152 /** Scaling of template for template matching */ 153 private static final double TSCALE = 0.3; 154 155 /** */ 156 public List<TrackedFace> trackedFaces = new ArrayList<TrackedFace>(); 157 158 /** The initial tracker */ 159 private TrackerVars initialTracker = null; 160 161 /** < Frame number since last detection */ 162 private long framesSinceLastDetection; 163 164 /** The frame currently being processed */ 165 private FImage currentFrame; 166 167 private FImage small_; 168 169 /** 170 * Create a tracker using the given model, face detector, failure checker, 171 * reference shape and similarity measures. These values will be copied into 172 * all trackers. 173 * 174 * @param clm 175 * The local model 176 * @param fdet 177 * The face detector 178 * @param fcheck 179 * The failure checker 180 * @param rshape 181 * The reference shape 182 * @param simil 183 * The similarity measures 184 */ 185 public MultiTracker(final CLM clm, final FDet fdet, final MFCheck fcheck, final Matrix rshape, 186 final double[] simil) 187 { 188 this.initialTracker = new TrackerVars(); 189 this.initialTracker.clm = clm; 190 this.initialTracker.clm._pdm.identity(clm._plocal, clm._pglobl); 191 this.initialTracker.faceDetector = fdet; 192 this.initialTracker.failureCheck = fcheck; 193 this.initialTracker.referenceShape = rshape.copy(); 194 this.initialTracker.similarity = simil; 195 this.initialTracker.shape = new Matrix(2 * clm._pdm.nPoints(), 1); 196 this.framesSinceLastDetection = -1; 197 } 198 199 /** 200 * Create a tracker with the given variables. 201 * 202 * @param tv 203 * The tracker variables to use for all face trackers. 204 */ 205 public MultiTracker(final TrackerVars tv) { 206 this.initialTracker = tv; 207 this.framesSinceLastDetection = -1; 208 } 209 210 /** 211 * Constructor for making a tracker when loading data. 212 */ 213 protected MultiTracker() { 214 } 215 216 /** 217 * Reset frame number (will perform detection in next image) 218 */ 219 public void frameReset() { 220 this.framesSinceLastDetection = -1; 221 this.trackedFaces.clear(); 222 } 223 224 /** 225 * Track faces from a previous frame to the given frame. 226 * 227 * @param im 228 * The video frame 229 * @param wSize 230 * The window size 231 * @param fpd 232 * The number of frames between forced redetecs 233 * @param nIter 234 * The number of iterations for model fitting 235 * @param clamp 236 * The number s.d.'s in which a model must fit 237 * @param fTol 238 * The tolerance for model fitting 239 * @param fcheck 240 * Whether to automatically check for failed tracking 241 * @param searchAreaSize 242 * The size of the template match search area 243 * @return 0 for success, -1 for failure. 244 */ 245 public int track(final FImage im, final int[] wSize, final int fpd, final int nIter, 246 final double clamp, final double fTol, final boolean fcheck, 247 final float searchAreaSize) 248 { 249 this.currentFrame = im; 250 251 if ((this.framesSinceLastDetection < 0) 252 || (fpd >= 0 && fpd < this.framesSinceLastDetection)) 253 { 254 this.framesSinceLastDetection = 0; 255 final List<Rectangle> RL = this.initialTracker.faceDetector 256 .detect(this.currentFrame); 257 258 // Convert the detected rectangles into face trackers 259 // trackedFaces.clear(); 260 // for (final Rectangle r : RL) 261 // trackedFaces.add(new TrackedFace(r, initialTracker)); 262 if (this.trackedFaces.size() == 0) { 263 for (final Rectangle r : RL) 264 this.trackedFaces.add(new TrackedFace(r, this.initialTracker)); 265 } else { 266 this.trackRedetect(this.currentFrame, searchAreaSize); 267 268 final int sz = this.trackedFaces.size(); 269 for (final Rectangle r : RL) { 270 boolean found = false; 271 for (int i = 0; i < sz; i++) { 272 if (r.percentageOverlap(this.trackedFaces.get(i).redetectedBounds) > 0.5) { 273 found = true; 274 break; 275 } 276 } 277 278 if (!found) 279 this.trackedFaces.add(new TrackedFace(r, this.initialTracker)); 280 } 281 } 282 } else { 283 // Updates the tracked faces 284 this.trackRedetect(this.currentFrame, searchAreaSize); 285 } 286 287 // Didn't find any faces in this frame? Try again next frame. 288 if (this.trackedFaces.size() == 0) 289 return -1; 290 291 boolean resize = true; 292 293 for (final Iterator<TrackedFace> iterator = this.trackedFaces.iterator(); iterator.hasNext();) { 294 final TrackedFace f = iterator.next(); 295 296 if ((f.redetectedBounds.width == 0) 297 || (f.redetectedBounds.height == 0)) 298 { 299 iterator.remove(); 300 this.framesSinceLastDetection = -1; 301 continue; 302 // return -1; 303 } 304 305 if (f.gen) { 306 this.initShape(f.redetectedBounds, f.shape, f.referenceShape); 307 f.clm._pdm.calcParams(f.shape, f.clm._plocal, f.clm._pglobl); 308 } else { 309 final double tx = f.redetectedBounds.x - f.lastMatchBounds.x; 310 final double ty = f.redetectedBounds.y - f.lastMatchBounds.y; 311 312 f.clm._pglobl.getArray()[4][0] += tx; 313 f.clm._pglobl.getArray()[5][0] += ty; 314 315 resize = false; 316 } 317 318 f.clm.fit(this.currentFrame, wSize, nIter, clamp, fTol); 319 f.clm._pdm.calcShape2D(f.shape, f.clm._plocal, f.clm._pglobl); 320 321 if (fcheck) { 322 if (!this.initialTracker.failureCheck.check(f.clm.getViewIdx(), 323 this.currentFrame, f.shape)) 324 { 325 iterator.remove(); 326 continue; 327 // return -1; 328 } 329 } 330 331 f.lastMatchBounds = this.updateTemplate(f, this.currentFrame, f.shape, 332 resize); 333 334 if ((f.lastMatchBounds.width == 0) 335 || (f.lastMatchBounds.height == 0)) 336 { 337 iterator.remove(); 338 this.framesSinceLastDetection = -1; 339 continue; 340 // return -1; 341 } 342 } 343 344 // Didn't find any faces in this frame? Try again next frame. 345 if (this.trackedFaces.size() == 0) 346 return -1; 347 348 this.framesSinceLastDetection++; 349 350 return 0; 351 } 352 353 /** 354 * Initialise the shape within the given rectangle based on the given 355 * reference shape. 356 * 357 * @param r 358 * The rectangle 359 * @param shape 360 * The shape to initialise 361 * @param _rshape 362 * The reference shape 363 */ 364 public void initShape(final Rectangle r, final Matrix shape, 365 final Matrix _rshape) 366 { 367 assert ((shape.getRowDimension() == _rshape.getRowDimension()) && (shape 368 .getColumnDimension() == _rshape.getColumnDimension())); 369 370 final int n = _rshape.getRowDimension() / 2; 371 372 final double a = r.width * Math.cos(this.initialTracker.similarity[1]) 373 * this.initialTracker.similarity[0] + 1; 374 final double b = r.width * Math.sin(this.initialTracker.similarity[1]) 375 * this.initialTracker.similarity[0]; 376 377 final double tx = r.x + (int) (r.width / 2) + r.width 378 * this.initialTracker.similarity[2]; 379 final double ty = r.y + (int) (r.height / 2) + r.height 380 * this.initialTracker.similarity[3]; 381 382 final double[][] s = _rshape.getArray(); 383 final double[][] d = shape.getArray(); 384 385 for (int i = 0; i < n; i++) { 386 d[i][0] = a * s[i][0] - b * s[i + n][0] + tx; 387 d[i + n][0] = b * s[i][0] + a * s[i + n][0] + ty; 388 } 389 } 390 391 /** 392 * Redetect the faces in the new frame. 393 * 394 * @param im 395 * The new frame. 396 * @param searchAreaSize 397 * The search area size 398 */ 399 private void trackRedetect(final FImage im, final float searchAreaSize) { 400 final int ww = im.width; 401 final int hh = im.height; 402 403 // Resize the frame so processing is quicker. 404 this.small_ = ResizeProcessor.resample(im, (int) (MultiTracker.TSCALE * ww), 405 (int) (MultiTracker.TSCALE * hh)); 406 407 for (final TrackedFace f : this.trackedFaces) { 408 f.gen = false; 409 410 // Get the new search area nearby to the last match 411 Rectangle searchAreaBounds = f.lastMatchBounds.clone(); 412 searchAreaBounds.scale((float) MultiTracker.TSCALE); 413 searchAreaBounds.scaleCentroid(searchAreaSize); 414 415 if (searchAreaBounds.overlapping(this.small_.getBounds()) != null) 416 searchAreaBounds = searchAreaBounds.overlapping(this.small_.getBounds()); 417 else 418 searchAreaBounds = this.small_.getBounds(); 419 420 // Get the search image 421 final FImage searchArea = this.small_.extractROI(searchAreaBounds); 422 423 // Template match the template over the reduced size image. 424 final FourierTemplateMatcher matcher = new FourierTemplateMatcher( 425 f.templateImage, 426 FourierTemplateMatcher.Mode.NORM_CORRELATION_COEFFICIENT); 427 matcher.analyseImage(searchArea); 428 429 // Get the response map 430 final float[][] ncc_ = matcher.getResponseMap().pixels; 431 432 // DisplayUtilities.displayName( matcher.getResponseMap(), 433 // "responseMap" ); 434 // DisplayUtilities.displayName( f.templateImage, "template" ); 435 436 f.redetectedBounds = f.templateImage.getBounds(); 437 438 // Find the maximum template match in the image 439 final int h = searchArea.height - f.templateImage.height + 1; 440 final int w = searchArea.width - f.templateImage.width + 1; 441 float vb = -2; 442 for (int y = 0; y < h; y++) { 443 for (int x = 0; x < w; x++) { 444 final float v = ncc_[y][x]; 445 if (v > vb) { 446 vb = v; 447 f.redetectedBounds.x = x + searchAreaBounds.x; 448 f.redetectedBounds.y = y + searchAreaBounds.y; 449 } 450 } 451 } 452 453 // Rescale the rectangle to full-size image coordinates. 454 f.redetectedBounds.scale((float) (1d / MultiTracker.TSCALE)); 455 } 456 } 457 458 protected Rectangle updateTemplate(final TrackedFace f, final FImage im, final Matrix s, 459 final boolean resize) 460 { 461 final int n = s.getRowDimension() / 2; 462 463 final double[][] sv = s.getArray(); 464 double xmax = sv[0][0], ymax = sv[n][0], xmin = sv[0][0], ymin = sv[n][0]; 465 466 for (int i = 0; i < n; i++) { 467 final double vx = sv[i][0]; 468 final double vy = sv[i + n][0]; 469 470 xmax = Math.max(xmax, vx); 471 ymax = Math.max(ymax, vy); 472 473 xmin = Math.min(xmin, vx); 474 ymin = Math.min(ymin, vy); 475 } 476 477 if ((xmin < 0) || (ymin < 0) || (xmax >= im.width) 478 || (ymax >= im.height) || Double.isNaN(xmin) 479 || Double.isInfinite(xmin) || Double.isNaN(xmax) 480 || Double.isInfinite(xmax) || Double.isNaN(ymin) 481 || Double.isInfinite(ymin) || Double.isNaN(ymax) 482 || Double.isInfinite(ymax)) 483 { 484 return new Rectangle(0, 0, 0, 0); 485 } else { 486 xmin *= MultiTracker.TSCALE; 487 ymin *= MultiTracker.TSCALE; 488 xmax *= MultiTracker.TSCALE; 489 ymax *= MultiTracker.TSCALE; 490 491 final Rectangle R = new Rectangle((float) Math.floor(xmin), 492 (float) Math.floor(ymin), (float) Math.ceil(xmax - xmin), 493 (float) Math.ceil(ymax - ymin)); 494 495 final int ww = im.width; 496 final int hh = im.height; 497 498 if (resize) 499 this.small_ = ResizeProcessor.resample(im, (int) (MultiTracker.TSCALE * ww), 500 (int) (MultiTracker.TSCALE * hh)); 501 502 f.templateImage = this.small_.extractROI(R); 503 504 R.x *= 1.0 / MultiTracker.TSCALE; 505 R.y *= 1.0 / MultiTracker.TSCALE; 506 R.width *= 1.0 / MultiTracker.TSCALE; 507 R.height *= 1.0 / MultiTracker.TSCALE; 508 509 return R; 510 } 511 } 512 513 /** 514 * Load a tracker from a file. 515 * 516 * @param fname 517 * File name to read from 518 * @return A tracker variable class 519 * @throws FileNotFoundException 520 */ 521 public static TrackerVars load(final String fname) 522 throws FileNotFoundException 523 { 524 BufferedReader br = null; 525 try { 526 br = new BufferedReader(new FileReader(fname)); 527 final Scanner sc = new Scanner(br); 528 return MultiTracker.read(sc, true); 529 } finally { 530 try { 531 br.close(); 532 } catch (final IOException e) { 533 } 534 } 535 } 536 537 /** 538 * Load a tracker from an input stream. 539 * 540 * @param in 541 * The input stream 542 * @return a tracker 543 */ 544 public static TrackerVars load(final InputStream in) { 545 BufferedReader br = null; 546 try { 547 br = new BufferedReader(new InputStreamReader(in)); 548 final Scanner sc = new Scanner(br); 549 return MultiTracker.read(sc, true); 550 } finally { 551 try { 552 if (br != null) 553 br.close(); 554 } catch (final IOException e) { 555 } 556 } 557 } 558 559 /** 560 * 561 * @param s 562 * @param readType 563 * @return 564 */ 565 private static TrackerVars read(final Scanner s, final boolean readType) { 566 if (readType) { 567 final int type = s.nextInt(); 568 assert (type == IO.Types.TRACKER.ordinal()); 569 } 570 final TrackerVars trackerVars = new TrackerVars(); 571 trackerVars.clm = CLM.read(s, true); 572 trackerVars.faceDetector = FDet.read(s, true); 573 trackerVars.failureCheck = MFCheck.read(s, true); 574 trackerVars.referenceShape = IO.readMat(s); 575 trackerVars.similarity = new double[] { s.nextDouble(), s.nextDouble(), 576 s.nextDouble(), s.nextDouble() }; 577 trackerVars.shape = new Matrix(2 * trackerVars.clm._pdm.nPoints(), 1); 578 trackerVars.clm._pdm.identity(trackerVars.clm._plocal, 579 trackerVars.clm._pglobl); 580 581 return trackerVars; 582 } 583 584 /** 585 * Returns the initial variables used for each face tracker. 586 * 587 * @return The initial variables 588 */ 589 public TrackerVars getInitialVars() { 590 return this.initialTracker; 591 } 592}