1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30 package org.openimaj.demos.video.videosift;
31
32 import java.awt.event.KeyAdapter;
33 import java.awt.event.KeyEvent;
34 import java.io.File;
35 import java.io.IOException;
36 import java.util.List;
37
38 import javax.swing.JOptionPane;
39 import javax.swing.SwingUtilities;
40
41 import org.openimaj.feature.DoubleFV;
42 import org.openimaj.feature.DoubleFVComparison;
43 import org.openimaj.image.FImage;
44 import org.openimaj.image.MBFImage;
45 import org.openimaj.image.colour.RGBColour;
46 import org.openimaj.image.processing.face.detection.CLMDetectedFace;
47 import org.openimaj.image.processing.face.feature.CLMShapeFeature;
48 import org.openimaj.image.processing.face.feature.comparison.FaceFVComparator;
49 import org.openimaj.image.processing.face.feature.comparison.FacialFeatureComparator;
50 import org.openimaj.image.processing.face.recognition.AnnotatorFaceRecogniser;
51 import org.openimaj.image.processing.face.tracking.clm.CLMFaceTracker;
52 import org.openimaj.image.typography.hershey.HersheyFont;
53 import org.openimaj.io.IOUtils;
54 import org.openimaj.math.geometry.point.Point2d;
55 import org.openimaj.ml.annotation.AnnotatedObject;
56 import org.openimaj.ml.annotation.ScoredAnnotation;
57 import org.openimaj.ml.annotation.basic.KNNAnnotator;
58 import org.openimaj.video.VideoDisplay;
59 import org.openimaj.video.VideoDisplayListener;
60 import org.openimaj.video.capture.VideoCapture;
61
62 public class VideoFacialExpressionRecognition extends KeyAdapter implements VideoDisplayListener<MBFImage> {
63 private VideoCapture capture;
64 private VideoDisplay<MBFImage> videoFrame;
65
66 private AnnotatorFaceRecogniser<CLMDetectedFace, String> recogniser;
67 private CLMFaceTracker engine;
68 private FImage currentFrame;
69
70 public VideoFacialExpressionRecognition() throws Exception {
71 capture = new VideoCapture(320, 240);
72 engine = new CLMFaceTracker();
73 engine.fpd = 120;
74
75
76 videoFrame = VideoDisplay.createVideoDisplay(capture);
77 videoFrame.addVideoListener(this);
78 SwingUtilities.getRoot(videoFrame.getScreen()).addKeyListener(this);
79
80 final CLMShapeFeature.Extractor extractor = new
81 CLMShapeFeature.Extractor();
82 final FacialFeatureComparator<CLMShapeFeature> comparator = new
83 FaceFVComparator<CLMShapeFeature, DoubleFV>(
84 DoubleFVComparison.EUCLIDEAN);
85
86 final KNNAnnotator<CLMDetectedFace, String, CLMShapeFeature> knn = KNNAnnotator.create(extractor, comparator, 1,
87 5f);
88
89 recogniser = AnnotatorFaceRecogniser.create(knn);
90 }
91
92 @Override
93 public synchronized void keyPressed(KeyEvent key) {
94 if (key.getKeyCode() == KeyEvent.VK_SPACE) {
95 this.videoFrame.togglePause();
96 } else if (key.getKeyChar() == 'c') {
97
98
99
100 final String person = JOptionPane.showInputDialog(this.videoFrame.getScreen(), "", "",
101 JOptionPane.QUESTION_MESSAGE);
102
103 final List<CLMDetectedFace> faces = detectFaces();
104 if (faces.size() == 1) {
105 recogniser.train(new AnnotatedObject<CLMDetectedFace, String>(faces.get(0), person));
106 } else {
107 System.out.println("Wrong number of faces found");
108 }
109
110
111 } else if (key.getKeyChar() == 'd') {
112 engine.reset();
113 }
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128 else if (key.getKeyChar() == 's') {
129 try {
130 final File f = new File("rec.bin");
131 f.delete();
132 IOUtils.writeBinaryFull(f, this.recogniser);
133 } catch (final IOException e) {
134 e.printStackTrace();
135 }
136 } else if (key.getKeyChar() == 'l') {
137 try {
138 final File f = new File("rec.bin");
139 this.recogniser = IOUtils.read(f);
140 } catch (final IOException e) {
141
142 e.printStackTrace();
143 }
144 }
145 }
146
147 private List<CLMDetectedFace> detectFaces() {
148 return CLMDetectedFace.convert(engine.model.trackedFaces, currentFrame);
149 }
150
151 @Override
152 public void afterUpdate(VideoDisplay<MBFImage> display) {
153
154 }
155
156 @Override
157 public synchronized void beforeUpdate(MBFImage frame) {
158 this.currentFrame = frame.flatten();
159 engine.track(frame);
160 engine.drawModel(frame, true, true, true, true, true);
161
162 if (recogniser != null && recogniser.listPeople().size() >= 1) {
163 for (final CLMDetectedFace f : detectFaces()) {
164 final List<ScoredAnnotation<String>> name = recogniser.annotate(f);
165
166 if (name.size() > 0) {
167 final Point2d r = f.getBounds().getTopLeft();
168 frame.drawText(name.get(0).annotation, r, HersheyFont.ROMAN_SIMPLEX, 15, RGBColour.GREEN);
169 }
170 }
171 }
172 }
173
174 public static void main(String[] args) throws Exception {
175 new VideoFacialExpressionRecognition();
176 }
177 }