001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.demos.faces; 031 032import gnu.trove.map.hash.TObjectIntHashMap; 033 034import java.io.IOException; 035import java.net.MalformedURLException; 036import java.net.URL; 037import java.util.ArrayList; 038import java.util.HashSet; 039import java.util.List; 040import java.util.Set; 041 042import javax.swing.JOptionPane; 043 044import org.openimaj.demos.Demo; 045import org.openimaj.image.FImage; 046import org.openimaj.image.ImageUtilities; 047import org.openimaj.image.MBFImage; 048import org.openimaj.image.processing.face.tracking.clm.CLMFaceTracker; 049import org.openimaj.image.processing.face.tracking.clm.MultiTracker.TrackedFace; 050import org.openimaj.image.processing.transform.PiecewiseMeshWarp; 051import org.openimaj.math.geometry.shape.Rectangle; 052import org.openimaj.math.geometry.shape.Shape; 053import org.openimaj.math.geometry.shape.Triangle; 054import org.openimaj.util.pair.IndependentPair; 055import org.openimaj.util.pair.Pair; 056import org.openimaj.video.VideoDisplay; 057import org.openimaj.video.VideoDisplayListener; 058import org.openimaj.video.capture.VideoCapture; 059import org.openimaj.video.capture.VideoCaptureException; 060 061/** 062 * Demo showing real-time face mapping. 063 * 064 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk) 065 * 066 */ 067@Demo( 068 author = "Jonathon Hare", 069 description = "Real-time face mapping based on the CLM face tracker. Multiple faces are supported.", 070 keywords = { "video", "face", "webcam", "constrained local model" }, 071 title = "Puppeteer") 072public class MultiPuppeteer implements VideoDisplayListener<MBFImage> { 073 private CLMFaceTracker tracker = new CLMFaceTracker(); 074 private List<IndependentPair<MBFImage, List<Triangle>>> puppets = new ArrayList<IndependentPair<MBFImage, List<Triangle>>>(); 075 private TObjectIntHashMap<TrackedFace> puppetAssignments = new TObjectIntHashMap<TrackedFace>(); 076 private int nextPuppet = 0; 077 078 /** 079 * Default constructor. 080 * 081 * @throws MalformedURLException 082 * @throws IOException 083 */ 084 public MultiPuppeteer() throws MalformedURLException, IOException { 085 tracker.scale = 0.5f; 086 tracker.fpd = 120; 087 tracker.fcheck = false; 088 089 final CLMFaceTracker ptracker = new CLMFaceTracker(); 090 091 final URL[] puppetUrls = { 092 MultiPuppeteer.class.getResource("nigel.jpg"), 093 MultiPuppeteer.class.getResource("wendy.png") 094 }; 095 096 for (final URL url : puppetUrls) { 097 MBFImage image = ImageUtilities.readMBF(url); 098 099 final int paddingWidth = Math.max(image.getWidth(), 640); 100 final int paddingHeight = Math.max(image.getHeight(), 480); 101 102 image = image.padding(paddingWidth, paddingHeight); 103 104 ptracker.track(image); 105 106 final TrackedFace face = ptracker.getTrackedFaces().get(0); 107 108 puppets.add(IndependentPair.pair(image, ptracker.getTriangles(face))); 109 110 ptracker.reset(); 111 } 112 } 113 114 @Override 115 public void afterUpdate(VideoDisplay<MBFImage> display) { 116 // do nothing 117 } 118 119 @Override 120 public void beforeUpdate(MBFImage frame) { 121 tracker.track(frame); 122 123 final List<TrackedFace> tracked = tracker.getTrackedFaces(); 124 125 for (final TrackedFace face : tracked) { 126 int asgn; 127 128 if (puppetAssignments.contains(face)) { 129 asgn = puppetAssignments.get(face); 130 } else { 131 asgn = nextPuppet; 132 puppetAssignments.put(face, asgn); 133 nextPuppet++; 134 135 if (nextPuppet >= puppets.size()) 136 nextPuppet = 0; 137 } 138 139 final List<Triangle> triangles = tracker.getTriangles(face); 140 final IndependentPair<MBFImage, List<Triangle>> puppetData = this.puppets.get(asgn); 141 final List<Triangle> puppetTriangles = puppetData.secondObject(); 142 143 final List<Pair<Shape>> matches = computeMatches(puppetTriangles, triangles); 144 145 final PiecewiseMeshWarp<Float[], MBFImage> pmw = new PiecewiseMeshWarp<Float[], MBFImage>(matches); 146 147 final Rectangle bounds = face.redetectedBounds.clone(); 148 bounds.height += 10; 149 bounds.width += 10; 150 bounds.x -= 5; 151 bounds.y -= 5; 152 bounds.scale((float) (1.0 / tracker.scale)); 153 154 final MBFImage puppet = puppetData.firstObject(); 155 final List<FImage> bands = puppet.bands; 156 puppet.processInplace(pmw); 157 158 composite(frame, puppet, bounds); 159 160 puppet.bands = bands; 161 } 162 163 final Set<TrackedFace> toRemove = new HashSet<TrackedFace>(puppetAssignments.keySet()); 164 toRemove.removeAll(tracked); 165 for (final TrackedFace face : toRemove) { 166 puppetAssignments.remove(face); 167 } 168 } 169 170 private List<Pair<Shape>> computeMatches(List<Triangle> from, List<Triangle> to) { 171 final List<Pair<Shape>> mtris = new ArrayList<Pair<Shape>>(); 172 173 for (int i = 0; i < from.size(); i++) { 174 final Triangle t1 = from.get(i); 175 Triangle t2 = to.get(i); 176 177 if (t1 != null && t2 != null) { 178 t2 = t2.clone(); 179 t2.scale((float) (1.0 / tracker.scale)); 180 mtris.add(new Pair<Shape>(t1, t2)); 181 } 182 } 183 184 return mtris; 185 } 186 187 private void composite(MBFImage back, MBFImage fore, Rectangle bounds) { 188 final float[][] rin = fore.bands.get(0).pixels; 189 final float[][] gin = fore.bands.get(1).pixels; 190 final float[][] bin = fore.bands.get(2).pixels; 191 192 final float[][] rout = back.bands.get(0).pixels; 193 final float[][] gout = back.bands.get(1).pixels; 194 final float[][] bout = back.bands.get(2).pixels; 195 196 final int xmin = (int) Math.max(0, bounds.x); 197 final int ymin = (int) Math.max(0, bounds.y); 198 199 final int ymax = (int) Math.min(Math.min(fore.getHeight(), back.getHeight()), bounds.y + bounds.height); 200 final int xmax = (int) Math.min(Math.min(fore.getWidth(), back.getWidth()), bounds.x + bounds.width); 201 202 for (int y = ymin; y < ymax; y++) { 203 for (int x = xmin; x < xmax; x++) { 204 if (rin[y][x] != 0 && gin[y][x] != 0 && bin[y][x] != 0) { 205 rout[y][x] = rin[y][x]; 206 gout[y][x] = gin[y][x]; 207 bout[y][x] = bin[y][x]; 208 } 209 } 210 } 211 } 212 213 /** 214 * The main method. 215 * 216 * @param args 217 * @throws IOException 218 * @throws MalformedURLException 219 */ 220 public static void main(String[] args) throws MalformedURLException, IOException { 221 try { 222 final MultiPuppeteer puppeteer = new MultiPuppeteer(); 223 224 VideoDisplay.createVideoDisplay(new VideoCapture(640, 480)).addVideoListener(puppeteer); 225 } catch (final VideoCaptureException e) { 226 JOptionPane.showMessageDialog(null, "No video capture devices were found!"); 227 } 228 } 229}