001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.demos.sandbox;
031
032import java.awt.event.KeyEvent;
033import java.awt.event.KeyListener;
034import java.io.IOException;
035import java.util.List;
036
037import javax.swing.SwingUtilities;
038
039import org.openimaj.image.FImage;
040import org.openimaj.image.ImageUtilities;
041import org.openimaj.image.MBFImage;
042import org.openimaj.image.colour.RGBColour;
043import org.openimaj.image.model.asm.ActiveShapeModel.IterationResult;
044import org.openimaj.image.model.asm.MultiResolutionActiveShapeModel;
045import org.openimaj.image.model.asm.datasets.IMMFaceDatabase;
046import org.openimaj.image.model.asm.datasets.ShapeModelDataset;
047import org.openimaj.image.model.landmark.FNormalLandmarkModel;
048import org.openimaj.image.pixel.sampling.FLineSampler;
049import org.openimaj.image.processing.face.detection.DetectedFace;
050import org.openimaj.image.processing.face.detection.HaarCascadeDetector;
051import org.openimaj.math.geometry.line.Line2d;
052import org.openimaj.math.geometry.point.Point2d;
053import org.openimaj.math.geometry.point.PointList;
054import org.openimaj.math.geometry.point.PointListConnections;
055import org.openimaj.math.geometry.shape.PointDistributionModel;
056import org.openimaj.math.geometry.transforms.TransformUtilities;
057import org.openimaj.math.matrix.algorithm.pca.PrincipalComponentAnalysis.PercentageEnergyComponentSelector;
058import org.openimaj.video.VideoDisplay;
059import org.openimaj.video.VideoDisplayListener;
060import org.openimaj.video.capture.VideoCapture;
061
062import Jama.Matrix;
063
064public class PDMPlaygroundLive {
065        /**
066         * @param args
067         * @throws IOException
068         */
069        @SuppressWarnings("rawtypes")
070        public static void main(String[] args) throws IOException {
071                // ASFDataset dataset = new ASFDataset(new
072                // File("/Users/jsh2/Work/lmlk/trunk/shared/JAAM-API/data/face-data"));
073                // AMPTSDataset dataset = new AMPTSDataset(
074                // new File("/Users/jsh2/Downloads/am_tools/points"),
075                // new File("/Users/jsh2/Downloads/am_tools/images"),
076                // new File("/Users/jsh2/Downloads/am_tools/models/face.parts"));
077                final ShapeModelDataset<FImage> dataset = IMMFaceDatabase.load(ImageUtilities.FIMAGE_READER);
078
079                final PointListConnections connections = dataset.getConnections();
080                final List<PointList> pointData = dataset.getPointLists();
081
082                final float scale = 0.02f;
083                final FNormalLandmarkModel.Factory factory = new FNormalLandmarkModel.Factory(connections,
084                                FLineSampler.INTERPOLATED_DERIVATIVE, 5, 9, scale);
085                // BlockLandmarkModel.Factory factory = new
086                // BlockLandmarkModel.Factory();
087                final MultiResolutionActiveShapeModel asm = MultiResolutionActiveShapeModel.trainModel(3,
088                                new PercentageEnergyComponentSelector(0.95), dataset, new PointDistributionModel.BoxConstraint(3),
089                                factory);
090                // final ActiveShapeModel asm = ActiveShapeModel.trainModel(10, data,
091                // new PointDistributionModel.BoxConstraint(3), factory);
092
093                final boolean[] tracking = { false };
094
095                final VideoDisplay<MBFImage> vd = VideoDisplay.createVideoDisplay(new VideoCapture(320, 240));
096                SwingUtilities.getRoot(vd.getScreen()).addKeyListener(new KeyListener() {
097
098                        @Override
099                        public void keyTyped(KeyEvent e) {
100                        }
101
102                        @Override
103                        public void keyReleased(KeyEvent e) {
104                        }
105
106                        @Override
107                        public void keyPressed(KeyEvent e) {
108                                if (e.getKeyChar() == 'r')
109                                        tracking[0] = false;
110                        }
111                });
112                vd.addVideoListener(new VideoDisplayListener<MBFImage>() {
113
114                        HaarCascadeDetector detector = new HaarCascadeDetector(80);
115                        PointList shape = null;
116
117                        @Override
118                        public void beforeUpdate(MBFImage frame) {
119                                final FImage image = frame.flatten();
120                                final List<DetectedFace> faces = detector.detectFaces(image);
121
122                                if (faces == null || faces.size() == 0)
123                                        return;
124
125                                // for (DetectedFace face : faces)
126                                final DetectedFace face = faces.get(0);
127                                {
128                                        frame.drawShape(face.getBounds(), RGBColour.GREEN);
129
130                                        final Point2d cog = face.getBounds().calculateCentroid();
131                                        final double facescale = face.getBounds().height / 3.5;
132
133                                        if (!tracking[0]) {
134                                                final Matrix pose = TransformUtilities.translateMatrix(cog.getX(), cog.getY()).times(
135                                                                TransformUtilities.scaleMatrix(facescale, facescale));
136                                                shape = asm.getPDM().getMean().transform(pose);
137                                                tracking[0] = true;
138                                        }
139
140                                        final long t1 = System.currentTimeMillis();
141                                        final IterationResult newData = asm.fit(image, shape);
142                                        final long t2 = System.currentTimeMillis();
143
144                                        shape = newData.shape;
145
146                                        frame.drawLines(connections.getLines(shape), 1, RGBColour.RED);
147
148                                        final float shapeScale = shape.computeIntrinsicScale();
149                                        for (final Point2d pt : shape) {
150                                                final Line2d normal = connections.calculateNormalLine(pt, shape, scale * shapeScale);
151                                                if (normal != null)
152                                                        frame.drawLine(normal, 1, RGBColour.BLUE);
153                                        }
154
155                                        System.out.println(newData.fit);
156                                        System.out.println(t2 - t1);
157                                }
158                        }
159
160                        @Override
161                        public void afterUpdate(VideoDisplay<MBFImage> display) {
162                        }
163                });
164        }
165}