001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.demos.video.videosift; 031 032import java.awt.event.KeyAdapter; 033import java.awt.event.KeyEvent; 034import java.io.File; 035import java.io.IOException; 036import java.util.List; 037 038import javax.swing.JOptionPane; 039import javax.swing.SwingUtilities; 040 041import org.openimaj.feature.DoubleFV; 042import org.openimaj.feature.DoubleFVComparison; 043import org.openimaj.image.FImage; 044import org.openimaj.image.MBFImage; 045import org.openimaj.image.colour.RGBColour; 046import org.openimaj.image.processing.face.detection.CLMDetectedFace; 047import org.openimaj.image.processing.face.feature.CLMShapeFeature; 048import org.openimaj.image.processing.face.feature.comparison.FaceFVComparator; 049import org.openimaj.image.processing.face.feature.comparison.FacialFeatureComparator; 050import org.openimaj.image.processing.face.recognition.AnnotatorFaceRecogniser; 051import org.openimaj.image.processing.face.tracking.clm.CLMFaceTracker; 052import org.openimaj.image.typography.hershey.HersheyFont; 053import org.openimaj.io.IOUtils; 054import org.openimaj.math.geometry.point.Point2d; 055import org.openimaj.ml.annotation.AnnotatedObject; 056import org.openimaj.ml.annotation.ScoredAnnotation; 057import org.openimaj.ml.annotation.basic.KNNAnnotator; 058import org.openimaj.video.VideoDisplay; 059import org.openimaj.video.VideoDisplayListener; 060import org.openimaj.video.capture.VideoCapture; 061 062public class VideoFacialExpressionRecognition extends KeyAdapter implements VideoDisplayListener<MBFImage> { 063 private VideoCapture capture; 064 private VideoDisplay<MBFImage> videoFrame; 065 066 private AnnotatorFaceRecogniser<CLMDetectedFace, String> recogniser; 067 private CLMFaceTracker engine; 068 private FImage currentFrame; 069 070 public VideoFacialExpressionRecognition() throws Exception { 071 capture = new VideoCapture(320, 240); 072 engine = new CLMFaceTracker(); 073 engine.fpd = 120; 074 // engine.fcheck = true; 075 076 videoFrame = VideoDisplay.createVideoDisplay(capture); 077 videoFrame.addVideoListener(this); 078 SwingUtilities.getRoot(videoFrame.getScreen()).addKeyListener(this); 079 080 final CLMShapeFeature.Extractor extractor = new 081 CLMShapeFeature.Extractor(); 082 final FacialFeatureComparator<CLMShapeFeature> comparator = new 083 FaceFVComparator<CLMShapeFeature, DoubleFV>( 084 DoubleFVComparison.EUCLIDEAN); 085 086 final KNNAnnotator<CLMDetectedFace, String, CLMShapeFeature> knn = KNNAnnotator.create(extractor, comparator, 1, 087 5f); 088 089 recogniser = AnnotatorFaceRecogniser.create(knn); 090 } 091 092 @Override 093 public synchronized void keyPressed(KeyEvent key) { 094 if (key.getKeyCode() == KeyEvent.VK_SPACE) { 095 this.videoFrame.togglePause(); 096 } else if (key.getKeyChar() == 'c') { 097 // if (!this.videoFrame.isPaused()) 098 // this.videoFrame.togglePause(); 099 100 final String person = JOptionPane.showInputDialog(this.videoFrame.getScreen(), "", "", 101 JOptionPane.QUESTION_MESSAGE); 102 103 final List<CLMDetectedFace> faces = detectFaces(); 104 if (faces.size() == 1) { 105 recogniser.train(new AnnotatedObject<CLMDetectedFace, String>(faces.get(0), person)); 106 } else { 107 System.out.println("Wrong number of faces found"); 108 } 109 110 // this.videoFrame.togglePause(); 111 } else if (key.getKeyChar() == 'd') { 112 engine.reset(); 113 } 114 // else if (key.getKeyChar() == 'q') { 115 // if (!this.videoFrame.isPaused()) 116 // this.videoFrame.togglePause(); 117 // 118 // final List<CLMDetectedFace> faces = detectFaces(); 119 // if (faces.size() == 1) { 120 // System.out.println("Looks like: " + 121 // recogniser.annotate(faces.get(0))); 122 // } else { 123 // System.out.println("Wrong number of faces found"); 124 // } 125 // 126 // this.videoFrame.togglePause(); 127 // } 128 else if (key.getKeyChar() == 's') { 129 try { 130 final File f = new File("rec.bin"); 131 f.delete(); 132 IOUtils.writeBinaryFull(f, this.recogniser); 133 } catch (final IOException e) { 134 e.printStackTrace(); 135 } 136 } else if (key.getKeyChar() == 'l') { 137 try { 138 final File f = new File("rec.bin"); 139 this.recogniser = IOUtils.read(f); 140 } catch (final IOException e) { 141 // TODO Auto-generated catch block 142 e.printStackTrace(); 143 } 144 } 145 } 146 147 private List<CLMDetectedFace> detectFaces() { 148 return CLMDetectedFace.convert(engine.model.trackedFaces, currentFrame); 149 } 150 151 @Override 152 public void afterUpdate(VideoDisplay<MBFImage> display) { 153 // do nothing 154 } 155 156 @Override 157 public synchronized void beforeUpdate(MBFImage frame) { 158 this.currentFrame = frame.flatten(); 159 engine.track(frame); 160 engine.drawModel(frame, true, true, true, true, true); 161 162 if (recogniser != null && recogniser.listPeople().size() >= 1) { 163 for (final CLMDetectedFace f : detectFaces()) { 164 final List<ScoredAnnotation<String>> name = recogniser.annotate(f); 165 166 if (name.size() > 0) { 167 final Point2d r = f.getBounds().getTopLeft(); 168 frame.drawText(name.get(0).annotation, r, HersheyFont.ROMAN_SIMPLEX, 15, RGBColour.GREEN); 169 } 170 } 171 } 172 } 173 174 public static void main(String[] args) throws Exception { 175 new VideoFacialExpressionRecognition(); 176 } 177}