001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.demos.touchtable;
031
032import java.awt.GraphicsDevice;
033import java.awt.GraphicsEnvironment;
034import java.awt.event.KeyListener;
035import java.io.IOException;
036import java.util.ArrayList;
037import java.util.List;
038
039import javax.swing.SwingUtilities;
040
041import org.openimaj.image.FImage;
042import org.openimaj.image.MBFImage;
043import org.openimaj.image.analysis.watershed.Component;
044import org.openimaj.image.analysis.watershed.feature.MomentFeature;
045import org.openimaj.image.colour.RGBColour;
046import org.openimaj.image.connectedcomponent.ConnectedComponentLabeler;
047import org.openimaj.image.connectedcomponent.ConnectedComponentLabeler.Algorithm;
048import org.openimaj.image.feature.local.detector.mser.MSERFeatureGenerator;
049import org.openimaj.image.feature.local.detector.mser.MSERFeatureGenerator.MSERDirection;
050import org.openimaj.image.pixel.ConnectedComponent;
051import org.openimaj.image.pixel.PixelSet;
052import org.openimaj.image.processing.morphology.Close;
053import org.openimaj.image.processing.morphology.Open;
054import org.openimaj.math.geometry.shape.Rectangle;
055import org.openimaj.video.VideoDisplay;
056import org.openimaj.video.VideoDisplayListener;
057import org.openimaj.video.capture.Device;
058import org.openimaj.video.capture.VideoCapture;
059
060public class TouchTableDemo implements VideoDisplayListener<MBFImage> {
061
062        private static final int IMAGE_WIDTH = 160;
063        private static final int IMAGE_HEIGHT = 120;
064        public static final int SMALLEST_POINT_AREA = Math.max(1, (IMAGE_WIDTH * IMAGE_HEIGHT) / (480 * 360));
065        public static final int BIGGEST_POINT_AREA = Math.max(1, (IMAGE_WIDTH * IMAGE_HEIGHT) / (30 * 10));
066        public static final int SMALLEST_POINT_DIAMETER = IMAGE_HEIGHT / 30;
067        public static final int BIGGEST_POINT_DIAMETER = SMALLEST_POINT_DIAMETER * 2;
068        private VideoCapture capture;
069        private VideoDisplay<MBFImage> display;
070        private ConnectedComponentLabeler labler;
071        private TouchTableScreen touchTableScreen;
072        private FImageBackgroundLearner backgroundLearner;
073
074        enum DebugMode {
075                DEBUG_DISPLAY,
076                NONE
077        }
078
079        private DebugMode mode = DebugMode.NONE;
080        private KeyListener touchTableKeyboard;
081
082        public Rectangle extractionArea = new Rectangle(IMAGE_WIDTH / 10, IMAGE_HEIGHT / 10,
083                        IMAGE_WIDTH - (IMAGE_WIDTH / 5f), IMAGE_HEIGHT - (IMAGE_HEIGHT / 4.5f));
084        private MSERFeatureGenerator mserDetector;
085
086        public TouchTableDemo() throws IOException {
087                final List<Device> captureDevices = VideoCapture.getVideoDevices();
088                this.capture = new VideoCapture(IMAGE_WIDTH, IMAGE_HEIGHT, 30, captureDevices.get(0));
089                this.display = VideoDisplay.createVideoDisplay(capture);
090                mserDetector = new MSERFeatureGenerator(MomentFeature.class);
091                this.labler = new ConnectedComponentLabeler(Algorithm.SINGLE_PASS, ConnectedComponent.ConnectMode.CONNECT_4);
092
093                final GraphicsEnvironment ge = GraphicsEnvironment.getLocalGraphicsEnvironment();
094                final GraphicsDevice[] devices = ge.getScreenDevices();
095                try {
096                        Thread.sleep(1000);
097                } catch (final InterruptedException e) {
098                        // TODO Auto-generated catch block
099                        e.printStackTrace();
100                }
101                if (devices.length > 1) {
102                        // Then there is a touchscreen attached
103                        this.touchTableScreen = new TouchTableScreen(extractionArea, new Rectangle(0, 0, 800, 600 - 25));
104                        this.touchTableScreen.setUndecorated(true);
105                        this.touchTableScreen.setAlwaysOnTop(true);
106                        devices[1].setFullScreenWindow(this.touchTableScreen);
107                        this.touchTableScreen.init();
108
109                }
110
111                this.backgroundLearner = new FImageBackgroundLearner();
112                this.display.addVideoListener(this);
113                touchTableKeyboard = new TouchTableKeyboard(this, this.touchTableScreen);
114                SwingUtilities.getRoot(this.display.getScreen()).addKeyListener(this.touchTableKeyboard);
115
116        }
117
118        public static void main(String[] args) throws IOException {
119                new TouchTableDemo();
120        }
121
122        @Override
123        public void afterUpdate(VideoDisplay<MBFImage> display) {
124
125        }
126
127        @Override
128        public void beforeUpdate(MBFImage frame) {
129                // Rectangle extractionArea = new
130                // Rectangle(100,100,IMAGE_WIDTH-200,IMAGE_HEIGHT-200);
131
132                if (this.mode == DebugMode.DEBUG_DISPLAY)
133                        return;
134                final FImage grey = frame.extractROI(extractionArea).flatten();
135                if (!this.backgroundLearner.ready()) {
136                        grey.process(this.backgroundLearner);
137                        frame.fill(RGBColour.BLACK);
138                        frame.drawImage(new MBFImage(grey, grey, grey), (int) extractionArea.x, (int) extractionArea.y);
139                        return;
140                }
141                grey.addInplace(this.backgroundLearner.getBackground());
142                grey.threshold(0.07f);
143                // grey.processInplace(new OtsuThreshold());
144                // if(grey.sum() > BIGGEST_POINT_AREA * 2 ){
145                // this.backgroundLearner.relearn();
146                // return;
147                // }
148
149                // List<Circle> filtered = getFilteredCircles(grey);
150                final List<Touch> filtered = getFilteredTouchesFast(grey);
151                if (filtered.size() != 0)
152                        this.fireTouchEvent(filtered);
153                frame.fill(RGBColour.BLACK);
154                frame.drawImage(new MBFImage(grey, grey, grey), (int) extractionArea.x, (int) extractionArea.y);
155        }
156
157        private List<Touch> getFilteredTouchesFast(FImage grey) {
158                final Close morphClose = new Close();
159                final Open morphOpen = new Open();
160                // grey.processInplace(morphOpen);
161                // grey.processInplace(morphClose);
162
163                final List<Component> comps = mserDetector.generateMSERs(grey, MSERDirection.Down);
164                final List<Touch> ret = new ArrayList<Touch>();
165
166                for (final Component component : comps) {
167                        final int nPixels = component.size();
168                        if (nPixels < SMALLEST_POINT_AREA)
169                        {
170                                // if(nPixels > SMALLEST_POINT_AREA/10)
171                                continue;
172                        }
173                        // else if(nPixels > BIGGEST_POINT_AREA) {
174                        // continue;
175                        // }
176                        ret.add(new Touch(((MomentFeature) component.getFeature(0)).getCircle(10.0f)));
177                }
178                return ret;
179        }
180
181        private List<Touch> getFilteredTouches(FImage grey) {
182                final List<ConnectedComponent> comps = labler.findComponents(grey);
183
184                final List<Touch> filtered = new ArrayList<Touch>();
185
186                double[] hw = null;
187                double[] c = null;
188
189                for (final PixelSet connectedComponent : comps) {
190                        final int nPixels = connectedComponent.pixels.size();
191                        if (nPixels < SMALLEST_POINT_AREA)
192                        {
193                                // if(nPixels > SMALLEST_POINT_AREA/10)
194                                continue;
195                        }
196                        else if (nPixels > BIGGEST_POINT_AREA) {
197                                continue;
198                        }
199                        c = connectedComponent.calculateCentroid();
200                        hw = connectedComponent.calculateAverageHeightWidth(c);
201
202                        filtered.add(new Touch((float) c[0], (float) c[1], (float) Math.sqrt(hw[0] * hw[0] + hw[1] * hw[1])));
203                }
204                return filtered;
205        }
206
207        private void fireTouchEvent(List<Touch> filtered) {
208                if (this.touchTableScreen != null)
209                        this.touchTableScreen.touchEvent(filtered);
210        }
211}