001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.demos.video.videosift; 031 032import java.awt.event.KeyAdapter; 033import java.awt.event.KeyEvent; 034import java.io.File; 035import java.io.IOException; 036import java.util.List; 037 038import javax.swing.JOptionPane; 039import javax.swing.SwingUtilities; 040 041import org.openimaj.feature.FloatFV; 042import org.openimaj.feature.FloatFVComparison; 043import org.openimaj.image.FImage; 044import org.openimaj.image.MBFImage; 045import org.openimaj.image.colour.RGBColour; 046import org.openimaj.image.processing.face.alignment.CLMAligner; 047import org.openimaj.image.processing.face.detection.CLMDetectedFace; 048import org.openimaj.image.processing.face.feature.LocalLBPHistogram; 049import org.openimaj.image.processing.face.feature.comparison.FaceFVComparator; 050import org.openimaj.image.processing.face.feature.comparison.FacialFeatureComparator; 051import org.openimaj.image.processing.face.recognition.AnnotatorFaceRecogniser; 052import org.openimaj.image.processing.face.tracking.clm.CLMFaceTracker; 053import org.openimaj.image.typography.hershey.HersheyFont; 054import org.openimaj.io.IOUtils; 055import org.openimaj.math.geometry.point.Point2d; 056import org.openimaj.ml.annotation.AnnotatedObject; 057import org.openimaj.ml.annotation.ScoredAnnotation; 058import org.openimaj.ml.annotation.basic.KNNAnnotator; 059import org.openimaj.video.VideoDisplay; 060import org.openimaj.video.VideoDisplayListener; 061import org.openimaj.video.capture.VideoCapture; 062 063public class VideoFaceRecognition extends KeyAdapter implements VideoDisplayListener<MBFImage> { 064 private VideoCapture capture; 065 private VideoDisplay<MBFImage> videoFrame; 066 067 private AnnotatorFaceRecogniser<CLMDetectedFace, String> recogniser; 068 private CLMFaceTracker engine; 069 private FImage currentFrame; 070 071 public VideoFaceRecognition() throws Exception { 072 capture = new VideoCapture(320, 240); 073 engine = new CLMFaceTracker(); 074 engine.fpd = 120; 075 // engine.fcheck = true; 076 077 videoFrame = VideoDisplay.createVideoDisplay(capture); 078 videoFrame.addVideoListener(this); 079 SwingUtilities.getRoot(videoFrame.getScreen()).addKeyListener(this); 080 081 final LocalLBPHistogram.Extractor<CLMDetectedFace> extractor = new 082 LocalLBPHistogram.Extractor<CLMDetectedFace>( 083 new CLMAligner(), 20, 20, 8, 1); 084 final FacialFeatureComparator<LocalLBPHistogram> comparator = new 085 FaceFVComparator<LocalLBPHistogram, FloatFV>( 086 FloatFVComparison.EUCLIDEAN); 087 final KNNAnnotator<CLMDetectedFace, String, LocalLBPHistogram> knn = KNNAnnotator.create(extractor, comparator, 088 1, 5f); 089 // final CLMShapeFeature.Extractor extractor = new 090 // CLMShapeFeature.Extractor(); 091 // final FacialFeatureComparator<CLMShapeFeature> comparator = new 092 // FaceFVComparator<CLMShapeFeature, DoubleFV>( 093 // DoubleFVComparison.EUCLIDEAN); 094 // 095 // final KNNAnnotator<CLMDetectedFace, String, 096 // CLMShapeFeature.Extractor, CLMShapeFeature> knn = 097 // KNNAnnotator.create(extractor, comparator, 1, 5f); 098 099 recogniser = AnnotatorFaceRecogniser.create(knn); 100 } 101 102 @Override 103 public synchronized void keyPressed(KeyEvent key) { 104 if (key.getKeyCode() == KeyEvent.VK_SPACE) { 105 this.videoFrame.togglePause(); 106 } else if (key.getKeyChar() == 'c') { 107 // if (!this.videoFrame.isPaused()) 108 // this.videoFrame.togglePause(); 109 110 final String person = JOptionPane.showInputDialog(this.videoFrame.getScreen(), "", "", 111 JOptionPane.QUESTION_MESSAGE); 112 113 final List<CLMDetectedFace> faces = detectFaces(); 114 if (faces.size() == 1) { 115 recogniser.train(new AnnotatedObject<CLMDetectedFace, String>(faces.get(0), person)); 116 } else { 117 System.out.println("Wrong number of faces found"); 118 } 119 120 // this.videoFrame.togglePause(); 121 } else if (key.getKeyChar() == 'd') { 122 engine.reset(); 123 } 124 // else if (key.getKeyChar() == 'q') { 125 // if (!this.videoFrame.isPaused()) 126 // this.videoFrame.togglePause(); 127 // 128 // final List<CLMDetectedFace> faces = detectFaces(); 129 // if (faces.size() == 1) { 130 // System.out.println("Looks like: " + 131 // recogniser.annotate(faces.get(0))); 132 // } else { 133 // System.out.println("Wrong number of faces found"); 134 // } 135 // 136 // this.videoFrame.togglePause(); 137 // } 138 else if (key.getKeyChar() == 's') { 139 try { 140 final File f = new File("rec.bin"); 141 f.delete(); 142 IOUtils.writeBinaryFull(f, this.recogniser); 143 } catch (final IOException e) { 144 e.printStackTrace(); 145 } 146 } else if (key.getKeyChar() == 'l') { 147 try { 148 final File f = new File("rec.bin"); 149 this.recogniser = IOUtils.read(f); 150 } catch (final IOException e) { 151 // TODO Auto-generated catch block 152 e.printStackTrace(); 153 } 154 } 155 } 156 157 private List<CLMDetectedFace> detectFaces() { 158 return CLMDetectedFace.convert(engine.model.trackedFaces, currentFrame); 159 } 160 161 @Override 162 public void afterUpdate(VideoDisplay<MBFImage> display) { 163 // do nothing 164 } 165 166 @Override 167 public synchronized void beforeUpdate(MBFImage frame) { 168 this.currentFrame = frame.flatten(); 169 engine.track(frame); 170 engine.drawModel(frame, true, true, true, true, true); 171 172 if (recogniser != null && recogniser.listPeople().size() >= 1) { 173 for (final CLMDetectedFace f : detectFaces()) { 174 final List<ScoredAnnotation<String>> name = recogniser.annotate(f); 175 176 if (name.size() > 0) { 177 final Point2d r = f.getBounds().getTopLeft(); 178 frame.drawText(name.get(0).annotation, r, HersheyFont.ROMAN_SIMPLEX, 15, RGBColour.GREEN); 179 } 180 } 181 } 182 } 183 184 public static void main(String[] args) throws Exception { 185 new VideoFaceRecognition(); 186 } 187}