Source code

001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.demos.sandbox;
031
032import java.awt.GraphicsDevice;
033import java.awt.GraphicsEnvironment;
034import java.awt.event.KeyEvent;
035import java.awt.event.KeyListener;
036import java.awt.event.MouseEvent;
037import java.io.IOException;
038import java.util.ArrayList;
039import java.util.List;
040
041import javax.swing.JFrame;
042import javax.swing.event.MouseInputListener;
043
044import org.openimaj.image.DisplayUtilities;
045import org.openimaj.image.FImage;
046import org.openimaj.image.MBFImage;
047import org.openimaj.image.colour.ColourSpace;
048import org.openimaj.image.colour.RGBColour;
049import org.openimaj.image.colour.Transforms;
050import org.openimaj.image.connectedcomponent.ConnectedComponentLabeler;
051import org.openimaj.image.pixel.ConnectedComponent;
052import org.openimaj.image.pixel.Pixel;
053import org.openimaj.image.pixel.PixelSet;
054import org.openimaj.math.geometry.line.Line2d;
055import org.openimaj.math.geometry.point.Point2d;
056import org.openimaj.math.geometry.point.Point2dImpl;
057import org.openimaj.math.geometry.shape.Circle;
058import org.openimaj.math.geometry.transforms.HomographyModel;
059import org.openimaj.util.pair.IndependentPair;
060import org.openimaj.util.pair.Pair;
061import org.openimaj.video.VideoDisplay;
062import org.openimaj.video.VideoDisplayListener;
063import org.openimaj.video.capture.VideoCapture;
064
065public class DigitalWhiteboard implements VideoDisplayListener<MBFImage>, MouseInputListener, KeyListener {
066        private VideoCapture capture;
067        private VideoDisplay<MBFImage> display;
068        private JFrame drawingFrame;
069        private MBFImage drawingPanel;
070        private Runnable drawingUpdater;
071        private ConnectedComponentLabeler labeler;
072        List<MBFImage> learningFrames = new ArrayList<MBFImage>();
073        // private HistogramPixelModel model = null;
074        private MODE mode = MODE.NONE;
075        private HomographyModel homography = null;
076        private List<Pair<Point2d>> homographyPoints = new ArrayList<Pair<Point2d>>();
077        private List<IndependentPair<String, Point2d>> calibrationPoints = new ArrayList<IndependentPair<String, Point2d>>();
078        private int calibrationPointIndex;
079        private Point2dImpl previousPoint;
080        private double screenDiagonal;
081
082        enum MODE {
083                MODEL, SEARCHING, NONE, LINE_CONSTRUCTING;
084        }
085
086        public DigitalWhiteboard() throws IOException {
087
088                System.out.println(VideoCapture.getVideoDevices());
089                capture = new VideoCapture(320, 240, VideoCapture.getVideoDevices().get(1));
090                final JFrame screen = DisplayUtilities.makeFrame("Video");
091                display = VideoDisplay.createVideoDisplay(capture, screen);
092                display.addVideoListener(this);
093                display.displayMode(true);
094                display.getScreen().addKeyListener(this);
095                // GraphicsDevice device =
096                // GraphicsEnvironment.getLocalGraphicsEnvironment().getDefaultScreenDevice();
097                final GraphicsDevice device = GraphicsEnvironment.getLocalGraphicsEnvironment().getScreenDevices()[1];
098                drawingPanel = new MBFImage(device.getDisplayMode().getWidth(), device.getDisplayMode().getHeight(),
099                                ColourSpace.RGB);
100                // drawingPanel = new MBFImage(640,480,ColourSpace.RGB);
101                drawingPanel.fill(RGBColour.WHITE);
102                drawingFrame = DisplayUtilities.display(drawingPanel);
103                drawingFrame.setBounds(640, 0, drawingPanel.getWidth(), drawingPanel.getHeight());
104                drawingFrame.addKeyListener(this);
105                // drawingFrame.setUndecorated(true);
106                drawingFrame.setIgnoreRepaint(true);
107                drawingFrame.setResizable(false);
108                // drawingFrame.setVisible(false);
109                // drawingFrame.setExtendedState(JFrame.MAXIMIZED_BOTH);
110                // device.setFullScreenWindow(drawingFrame);
111                // GraphicsEnvironment.getLocalGraphicsEnvironment().getScreenDevices()[0].setFullScreenWindow(screen);
112
113                // screenDiagonal =
114                // Math.sqrt(Math.pow(display.getScreen().getWidth()/2,2) +
115                // Math.pow(display.getScreen().getHeight()/2,2));
116                screenDiagonal = 50;
117
118                drawingUpdater = new Runnable() {
119
120                        @Override
121                        public void run() {
122                                while (true) {
123                                        // drawingPanel = new MBFImage(640,480,ColourSpace.RGB);
124                                        try {
125                                                drawWhiteboard(drawingPanel);
126                                                DisplayUtilities.display(drawingPanel, drawingFrame);
127
128                                                Thread.sleep(1000 / 30);
129                                        } catch (final InterruptedException e) {
130                                                // TODO Auto-generated catch block
131                                                e.printStackTrace();
132                                        }
133                                }
134                        }
135
136                };
137                final Thread t = new Thread(drawingUpdater);
138                t.start();
139
140                labeler = new ConnectedComponentLabeler(ConnectedComponent.ConnectMode.CONNECT_4);
141
142                calibrationPoints.add(new IndependentPair<String, Point2d>("TOP LEFT", new Point2dImpl(20, 20)));
143                calibrationPoints.add(new IndependentPair<String, Point2d>("TOP RIGHT", new Point2dImpl(
144                                drawingPanel.getWidth() - 20, 20)));
145                calibrationPoints.add(new IndependentPair<String, Point2d>("BOTTOM LEFT", new Point2dImpl(20, drawingPanel
146                                .getHeight() - 20)));
147                calibrationPoints.add(new IndependentPair<String, Point2d>("BOTTOM RIGHT", new Point2dImpl(drawingPanel
148                                .getWidth() - 20, drawingPanel.getHeight() - 20)));
149                calibrationPointIndex = 0;
150        }
151
152        private synchronized void drawWhiteboard(MBFImage drawingPanel) {
153                // drawingPanel.fill(RGBColour.WHITE);
154                if (mode == MODE.MODEL || this.calibrationPointIndex < this.calibrationPoints.size()) {
155                        drawingPanel.fill(RGBColour.WHITE);
156                        final Point2d waitingForPoint = this.calibrationPoints.get(calibrationPointIndex).secondObject();
157                        drawingPanel.createRenderer().drawShape(new Circle(waitingForPoint.getX(), waitingForPoint.getY(), 10),
158                                        RGBColour.RED);
159                }
160
161        }
162
163        @Override
164        public void afterUpdate(VideoDisplay<MBFImage> display) {
165                // TODO Auto-generated method stub
166
167        }
168
169        @Override
170        public synchronized void beforeUpdate(MBFImage frame) {
171
172                final FImage greyFramePart = Transforms.calculateIntensityNTSC(frame);
173                greyFramePart.threshold(0.99f);
174                // greyFramePart.processInplace(new AdaptiveLocalThresholdMean(5));
175                // greyFramePart.processInplace(new OtsuThreshold());
176                // FImage greyFramePart = frame.getBand(0);
177                final MBFImage greyFrame = new MBFImage(new FImage[] { greyFramePart.clone(), greyFramePart.clone(),
178                                greyFramePart.clone() });
179                if (mode == MODE.MODEL) {
180                        final List<ConnectedComponent> labels = labeler.findComponents(greyFramePart);
181                        if (labels.size() > 0) {
182                                final PixelSet c = largestLabel(labels);
183                                final Pixel centroid = c.calculateCentroidPixel();
184                                int distance = -1;
185                                if (this.homographyPoints.size() != 0) {
186                                        distance = distance(homographyPoints.get(homographyPoints.size() - 1).firstObject(), centroid);
187                                }
188                                if (this.homographyPoints.size() == 0 || distance > screenDiagonal) {
189                                        System.out.println("Point found at: " + centroid);
190                                        System.out.println("Distance was: " + distance);
191                                        final IndependentPair<String, Point2d> calibration = this.calibrationPoints
192                                                        .get(this.calibrationPointIndex);
193                                        System.out.println("Adding point for: " + calibration.firstObject());
194                                        this.homographyPoints.add(new Pair<Point2d>(centroid, calibration.secondObject()));
195                                        this.calibrationPointIndex++;
196                                        if (this.calibrationPointIndex >= this.calibrationPoints.size()) {
197                                                this.homography.estimate(homographyPoints);
198                                                this.mode = MODE.SEARCHING;
199                                                drawingPanel.fill(RGBColour.WHITE);
200                                        }
201                                        else {
202                                                System.out.println("CURRENTLY EXPECTING POINT: "
203                                                                + this.calibrationPoints.get(this.calibrationPointIndex).firstObject());
204                                        }
205                                }
206
207                        }
208                }
209                else if (mode == MODE.SEARCHING) {
210                        System.out.println("TOTALLY SEARHCING");
211                        final List<ConnectedComponent> labels = labeler.findComponents(greyFramePart);
212                        if (labels.size() > 0) {
213                                this.mode = MODE.LINE_CONSTRUCTING;
214                                final PixelSet c = largestLabel(labels);
215                                final Point2dImpl actualPoint = findActualPoint(c);
216                                drawingPanel.createRenderer().drawShapeFilled(new Circle((int) actualPoint.x, (int) actualPoint.y, 5),
217                                                RGBColour.BLACK);
218                                previousPoint = actualPoint;
219                        }
220                }
221                else if (mode == MODE.LINE_CONSTRUCTING) {
222                        System.out.println("TOTALLY LINE DRAWING");
223                        final List<ConnectedComponent> labels = labeler.findComponents(greyFramePart);
224                        if (labels.size() > 0) {
225                                final PixelSet c = largestLabel(labels);
226                                final Point2dImpl actualPoint = findActualPoint(c);
227                                drawingPanel.createRenderer().drawLine(new Line2d(previousPoint, actualPoint), 5, RGBColour.BLACK);
228                                previousPoint = actualPoint;
229                        }
230                        else {
231                                mode = MODE.SEARCHING;
232                                previousPoint = null;
233                        }
234                }
235                frame.internalAssign(greyFrame);
236        }
237
238        private Point2dImpl findActualPoint(PixelSet c) {
239                final double[] centroidDouble = c.calculateCentroid();
240                final Point2dImpl centroid = new Point2dImpl((float) centroidDouble[0], (float) centroidDouble[1]);
241                final Point2dImpl actualPoint = centroid.transform(this.homography.getTransform());
242                return actualPoint;
243        }
244
245        private PixelSet largestLabel(List<ConnectedComponent> labels) {
246                int max = 0;
247                PixelSet r = null;
248                for (final PixelSet c : labels) {
249                        if (c.getPixels().size() > max)
250                        {
251                                max = c.getPixels().size();
252                                r = c;
253                        }
254                }
255                return r;
256        }
257
258        private int distance(Point2d p1, Point2d p2) {
259                final double dx = p1.getX() - p2.getX();
260                final double dy = p1.getY() - p2.getY();
261                return (int) Math.sqrt(dx * dx + dy * dy);
262        }
263
264        public static void main(String args[]) throws IOException {
265                new DigitalWhiteboard();
266        }
267
268        @Override
269        public void mouseClicked(MouseEvent arg0) {
270
271        }
272
273        @Override
274        public void mouseEntered(MouseEvent arg0) {
275                // TODO Auto-generated method stub
276
277        }
278
279        @Override
280        public void mouseExited(MouseEvent arg0) {
281                // TODO Auto-generated method stub
282
283        }
284
285        @Override
286        public void mousePressed(MouseEvent arg0) {
287                // TODO Auto-generated method stub
288
289        }
290
291        @Override
292        public void mouseReleased(MouseEvent arg0) {
293                // TODO Auto-generated method stub
294
295        }
296
297        @Override
298        public void mouseDragged(MouseEvent arg0) {
299                // TODO Auto-generated method stub
300
301        }
302
303        @Override
304        public void mouseMoved(MouseEvent arg0) {
305                // TODO Auto-generated method stub
306
307        }
308
309        @Override
310        public void keyPressed(KeyEvent arg0) {
311                // TODO Auto-generated method stub
312
313        }
314
315        @Override
316        public void keyReleased(KeyEvent arg0) {
317                // TODO Auto-generated method stub
318
319        }
320
321        @Override
322        public void keyTyped(KeyEvent event) {
323                System.out.println("Got a key");
324                if (event.getKeyChar() == 'c') {
325                        drawingPanel.fill(RGBColour.WHITE);
326                        System.out.println("Modelling mode started");
327                        this.mode = MODE.MODEL;
328                        this.calibrationPointIndex = 0;
329                        homographyPoints.clear();
330                        this.homography = new HomographyModel();
331
332                        System.out.println("CURRENTLY EXPECTING POINT: "
333                                        + this.calibrationPoints.get(this.calibrationPointIndex).firstObject());
334
335                }
336                if (event.getKeyChar() == 'd' && this.homographyPoints.size() > 4) {
337
338                }
339        }
340}