1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30 package org.openimaj.demos.video;
31
32 import java.awt.GraphicsDevice;
33 import java.awt.GraphicsEnvironment;
34 import java.awt.Toolkit;
35 import java.awt.event.KeyEvent;
36 import java.awt.event.KeyListener;
37 import java.util.LinkedList;
38 import java.util.Queue;
39 import javax.swing.JFrame;
40 import javax.swing.SwingUtilities;
41
42 import org.openimaj.demos.Demo;
43 import org.openimaj.hardware.kinect.KinectController;
44 import org.openimaj.hardware.kinect.KinectException;
45 import org.openimaj.image.DisplayUtilities;
46 import org.openimaj.image.FImage;
47 import org.openimaj.image.MBFImage;
48 import org.openimaj.image.processing.resize.ResizeProcessor;
49 import org.openimaj.util.pair.IndependentPair;
50 import org.openimaj.video.Video;
51 import org.openimaj.video.VideoDisplay;
52
53
54
55
56
57
58
59
60
61 @Demo(
62 author = "Jonathon Hare",
63 description = "Kinect integration demo. Shows video and depth. Press t " +
64 "to toggle between rgb and ir mode. Pressing w and x moves the device " +
65 "up or down. Pressing s levels the device.",
66 keywords = { "kinect", "video" },
67 title = "Kinect Integration",
68 screenshot = "/org/openimaj/demos/screens/hardware/kinect.png",
69 icon = "/org/openimaj/demos/icons/hardware/kinect.png"
70 )
71 public class KinectDepthSnapshot extends Video<MBFImage> implements KeyListener {
72 private static final int MAX_HELD_FRAMES = 10;
73 MBFImage currentFrame;
74 KinectController controller;
75 JFrame frame;
76 private double tilt = 0;
77 private boolean irmode = false;
78 private VideoDisplay<MBFImage> videoFrame;
79 private Queue<IndependentPair<FImage, MBFImage>> heldDepthFrames;
80 private IndependentPair<FImage, MBFImage> currentDepthFrame;
81 private int screenWidth;
82 private int screenHeight;
83 protected ResizeProcessor fullScreenResizeProcessor;
84
85
86
87
88
89
90 public KinectDepthSnapshot(int id) throws KinectException {
91 controller = new KinectController(id, irmode,true);
92 GraphicsEnvironment ge = GraphicsEnvironment.getLocalGraphicsEnvironment();
93 GraphicsDevice[] devices = ge.getScreenDevices();
94 JFrame frame=new JFrame("Full Screen JFrame");
95
96 frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
97
98 screenWidth = Toolkit.getDefaultToolkit().getScreenSize().width;
99 screenHeight = Toolkit.getDefaultToolkit().getScreenSize().height;
100 fullScreenResizeProcessor = new ResizeProcessor(screenWidth, screenHeight);
101 frame.setBounds(0,0,screenWidth,screenHeight);
102 videoFrame = VideoDisplay.createVideoDisplay(this, new DisplayUtilities.ImageComponent(true));
103 JFrame wholeWindow = new JFrame();
104 wholeWindow.setUndecorated(true);
105 wholeWindow.setAlwaysOnTop(true);
106 wholeWindow.getContentPane().add(videoFrame.getScreen());
107 devices[0].setFullScreenWindow(wholeWindow);
108 ((JFrame)SwingUtilities.getRoot(videoFrame.getScreen())).setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
109 SwingUtilities.getRoot(videoFrame.getScreen()).addKeyListener(this);
110
111 }
112
113 @Override
114 public MBFImage getNextFrame() {
115 FImage tmp = controller.depthStream.getNextFrame();
116 MBFImage frame = (MBFImage) controller.videoStream.getNextFrame();
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136 if(super.currentFrame % 30 == 0){
137 if (super.currentFrame % (60 * 2 * MAX_HELD_FRAMES) == 0)
138 {
139 removeAllFrames();
140 }
141
142 addDepthFrame(tmp,frame);
143 currentFrame = this.currentDepthFrame.secondObject();
144 }
145 else{
146 FImage oldDepth = this.currentDepthFrame.firstObject().clone();
147 currentFrame = this.currentDepthFrame.secondObject().clone();
148 for (int y = 0; y < tmp.height; y++) {
149 for (int x = 0; x < tmp.width; x++) {
150 if (oldDepth.pixels[y][x] == 0 || (tmp.pixels[y][x] != 0 && oldDepth.pixels[y][x] > tmp.pixels[y][x])) {
151
152 currentFrame.setPixel(x, y, frame.getPixel(x, y));
153 }
154 }
155 }
156 }
157
158 super.currentFrame++;
159
160 return currentFrame;
161 }
162
163 private void removeAllFrames() {
164 if (this.heldDepthFrames!=null)this.heldDepthFrames.clear();
165 }
166
167 private void addDepthFrame(FImage tmp, MBFImage frame) {
168 if(this.heldDepthFrames == null){
169 this.heldDepthFrames = new LinkedList<IndependentPair<FImage,MBFImage>>();
170 }
171 if(this.heldDepthFrames.size() == MAX_HELD_FRAMES){
172 this.heldDepthFrames.poll();
173 }
174
175
176 this.heldDepthFrames.add(IndependentPair.pair(tmp.clone(), frame.clone()));
177 System.out.println("Added frame, new frame size:" + this.heldDepthFrames.size());
178 this.currentDepthFrame = constructDepthFrame();
179 }
180
181 private IndependentPair<FImage, MBFImage> constructDepthFrame() {
182 FImage compiledDepth = null;
183 MBFImage compiledRGB = null;
184 for (IndependentPair<FImage, MBFImage> heldFrame : this.heldDepthFrames) {
185
186 FImage heldDepth = heldFrame.firstObject();
187 MBFImage heldRGB = heldFrame.secondObject();
188 if(compiledDepth == null){
189
190 compiledDepth = heldDepth.clone();
191 compiledRGB = heldRGB .clone();
192 continue;
193 }
194
195 for (int y = 0; y < heldDepth.height; y++) {
196 for (int x = 0; x < heldDepth.width; x++) {
197 if (compiledDepth.pixels[y][x] == 0 || (heldDepth.pixels[y][x] != 0 && compiledDepth.pixels[y][x] > heldDepth.pixels[y][x])) {
198 compiledDepth.pixels[y][x] = heldDepth.pixels[y][x];
199 compiledRGB.setPixel(x, y, heldRGB.getPixel(x, y));
200 }
201 }
202 }
203 }
204 return IndependentPair.pair(compiledDepth, compiledRGB);
205 }
206
207 @Override
208 public MBFImage getCurrentFrame() {
209 return currentFrame;
210 }
211
212 @Override
213 public int getWidth() {
214 return this.screenWidth;
215 }
216
217 @Override
218 public int getHeight() {
219 return this.screenHeight;
220 }
221
222 @Override
223 public boolean hasNextFrame() {
224 return true;
225 }
226
227 @Override
228 public long countFrames() {
229 return -1;
230 }
231
232 @Override
233 public void reset() {
234
235 }
236
237 @Override
238 public void keyTyped(KeyEvent e) {
239
240 }
241
242 @Override
243 public void keyPressed(KeyEvent e) {
244 if (e.getKeyChar() == 'w') {
245 controller.setTilt(tilt+=1);
246 } else if (e.getKeyChar() == 'x') {
247 controller.setTilt(tilt-=1);
248 } else if (e.getKeyChar() == 's') {
249 controller.setTilt(tilt=0);
250 } else if (e.getKeyChar() == 't') {
251 controller.setIRMode(irmode=!irmode );
252 }
253 }
254
255 @Override
256 public void keyReleased(KeyEvent e) {
257
258 }
259
260
261
262
263
264
265 public static void main(String[] args) throws KinectException {
266 new KinectDepthSnapshot(0);
267 }
268
269 @Override
270 public long getTimeStamp()
271 {
272 return (long)(super.currentFrame * 1000 / getFPS());
273 }
274
275 @Override
276 public double getFPS()
277 {
278 return 30;
279 }
280
281
282
283
284
285 public VideoDisplay<MBFImage> getDisplay() {
286 return videoFrame;
287 }
288 }