001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.demos.faces;
031
032import gnu.trove.map.hash.TObjectIntHashMap;
033
034import java.io.IOException;
035import java.net.MalformedURLException;
036import java.net.URL;
037import java.util.ArrayList;
038import java.util.HashSet;
039import java.util.List;
040import java.util.Set;
041
042import javax.swing.JOptionPane;
043
044import org.openimaj.demos.Demo;
045import org.openimaj.image.FImage;
046import org.openimaj.image.ImageUtilities;
047import org.openimaj.image.MBFImage;
048import org.openimaj.image.processing.face.tracking.clm.CLMFaceTracker;
049import org.openimaj.image.processing.face.tracking.clm.MultiTracker.TrackedFace;
050import org.openimaj.image.processing.transform.PiecewiseMeshWarp;
051import org.openimaj.math.geometry.shape.Rectangle;
052import org.openimaj.math.geometry.shape.Shape;
053import org.openimaj.math.geometry.shape.Triangle;
054import org.openimaj.util.pair.IndependentPair;
055import org.openimaj.util.pair.Pair;
056import org.openimaj.video.VideoDisplay;
057import org.openimaj.video.VideoDisplayListener;
058import org.openimaj.video.capture.VideoCapture;
059import org.openimaj.video.capture.VideoCaptureException;
060
061/**
062 * Demo showing real-time face mapping.
063 * 
064 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
065 * 
066 */
067@Demo(
068                author = "Jonathon Hare",
069                description = "Real-time face mapping based on the CLM face tracker. Multiple faces are supported.",
070                keywords = { "video", "face", "webcam", "constrained local model" },
071                title = "Puppeteer")
072public class MultiPuppeteer implements VideoDisplayListener<MBFImage> {
073        private CLMFaceTracker tracker = new CLMFaceTracker();
074        private List<IndependentPair<MBFImage, List<Triangle>>> puppets = new ArrayList<IndependentPair<MBFImage, List<Triangle>>>();
075        private TObjectIntHashMap<TrackedFace> puppetAssignments = new TObjectIntHashMap<TrackedFace>();
076        private int nextPuppet = 0;
077
078        /**
079         * Default constructor.
080         * 
081         * @throws MalformedURLException
082         * @throws IOException
083         */
084        public MultiPuppeteer() throws MalformedURLException, IOException {
085                tracker.scale = 0.5f;
086                tracker.fpd = 120;
087                tracker.fcheck = false;
088
089                final CLMFaceTracker ptracker = new CLMFaceTracker();
090
091                final URL[] puppetUrls = {
092                                MultiPuppeteer.class.getResource("nigel.jpg"),
093                                MultiPuppeteer.class.getResource("wendy.png")
094                };
095
096                for (final URL url : puppetUrls) {
097                        MBFImage image = ImageUtilities.readMBF(url);
098
099                        final int paddingWidth = Math.max(image.getWidth(), 640);
100                        final int paddingHeight = Math.max(image.getHeight(), 480);
101
102                        image = image.padding(paddingWidth, paddingHeight);
103
104                        ptracker.track(image);
105
106                        final TrackedFace face = ptracker.getTrackedFaces().get(0);
107
108                        puppets.add(IndependentPair.pair(image, ptracker.getTriangles(face)));
109
110                        ptracker.reset();
111                }
112        }
113
114        @Override
115        public void afterUpdate(VideoDisplay<MBFImage> display) {
116                // do nothing
117        }
118
119        @Override
120        public void beforeUpdate(MBFImage frame) {
121                tracker.track(frame);
122
123                final List<TrackedFace> tracked = tracker.getTrackedFaces();
124
125                for (final TrackedFace face : tracked) {
126                        int asgn;
127
128                        if (puppetAssignments.contains(face)) {
129                                asgn = puppetAssignments.get(face);
130                        } else {
131                                asgn = nextPuppet;
132                                puppetAssignments.put(face, asgn);
133                                nextPuppet++;
134
135                                if (nextPuppet >= puppets.size())
136                                        nextPuppet = 0;
137                        }
138
139                        final List<Triangle> triangles = tracker.getTriangles(face);
140                        final IndependentPair<MBFImage, List<Triangle>> puppetData = this.puppets.get(asgn);
141                        final List<Triangle> puppetTriangles = puppetData.secondObject();
142
143                        final List<Pair<Shape>> matches = computeMatches(puppetTriangles, triangles);
144
145                        final PiecewiseMeshWarp<Float[], MBFImage> pmw = new PiecewiseMeshWarp<Float[], MBFImage>(matches);
146
147                        final Rectangle bounds = face.redetectedBounds.clone();
148                        bounds.height += 10;
149                        bounds.width += 10;
150                        bounds.x -= 5;
151                        bounds.y -= 5;
152                        bounds.scale((float) (1.0 / tracker.scale));
153
154                        final MBFImage puppet = puppetData.firstObject();
155                        final List<FImage> bands = puppet.bands;
156                        puppet.processInplace(pmw);
157
158                        composite(frame, puppet, bounds);
159
160                        puppet.bands = bands;
161                }
162
163                final Set<TrackedFace> toRemove = new HashSet<TrackedFace>(puppetAssignments.keySet());
164                toRemove.removeAll(tracked);
165                for (final TrackedFace face : toRemove) {
166                        puppetAssignments.remove(face);
167                }
168        }
169
170        private List<Pair<Shape>> computeMatches(List<Triangle> from, List<Triangle> to) {
171                final List<Pair<Shape>> mtris = new ArrayList<Pair<Shape>>();
172
173                for (int i = 0; i < from.size(); i++) {
174                        final Triangle t1 = from.get(i);
175                        Triangle t2 = to.get(i);
176
177                        if (t1 != null && t2 != null) {
178                                t2 = t2.clone();
179                                t2.scale((float) (1.0 / tracker.scale));
180                                mtris.add(new Pair<Shape>(t1, t2));
181                        }
182                }
183
184                return mtris;
185        }
186
187        private void composite(MBFImage back, MBFImage fore, Rectangle bounds) {
188                final float[][] rin = fore.bands.get(0).pixels;
189                final float[][] gin = fore.bands.get(1).pixels;
190                final float[][] bin = fore.bands.get(2).pixels;
191
192                final float[][] rout = back.bands.get(0).pixels;
193                final float[][] gout = back.bands.get(1).pixels;
194                final float[][] bout = back.bands.get(2).pixels;
195
196                final int xmin = (int) Math.max(0, bounds.x);
197                final int ymin = (int) Math.max(0, bounds.y);
198
199                final int ymax = (int) Math.min(Math.min(fore.getHeight(), back.getHeight()), bounds.y + bounds.height);
200                final int xmax = (int) Math.min(Math.min(fore.getWidth(), back.getWidth()), bounds.x + bounds.width);
201
202                for (int y = ymin; y < ymax; y++) {
203                        for (int x = xmin; x < xmax; x++) {
204                                if (rin[y][x] != 0 && gin[y][x] != 0 && bin[y][x] != 0) {
205                                        rout[y][x] = rin[y][x];
206                                        gout[y][x] = gin[y][x];
207                                        bout[y][x] = bin[y][x];
208                                }
209                        }
210                }
211        }
212
213        /**
214         * The main method.
215         * 
216         * @param args
217         * @throws IOException
218         * @throws MalformedURLException
219         */
220        public static void main(String[] args) throws MalformedURLException, IOException {
221                try {
222                        final MultiPuppeteer puppeteer = new MultiPuppeteer();
223
224                        VideoDisplay.createVideoDisplay(new VideoCapture(640, 480)).addVideoListener(puppeteer);
225                } catch (final VideoCaptureException e) {
226                        JOptionPane.showMessageDialog(null, "No video capture devices were found!");
227                }
228        }
229}