001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.demos.touchtable; 031 032import java.awt.GraphicsDevice; 033import java.awt.GraphicsEnvironment; 034import java.awt.event.KeyListener; 035import java.io.IOException; 036import java.util.ArrayList; 037import java.util.List; 038 039import javax.swing.SwingUtilities; 040 041import org.openimaj.image.FImage; 042import org.openimaj.image.MBFImage; 043import org.openimaj.image.analysis.watershed.Component; 044import org.openimaj.image.analysis.watershed.feature.MomentFeature; 045import org.openimaj.image.colour.RGBColour; 046import org.openimaj.image.connectedcomponent.ConnectedComponentLabeler; 047import org.openimaj.image.connectedcomponent.ConnectedComponentLabeler.Algorithm; 048import org.openimaj.image.feature.local.detector.mser.MSERFeatureGenerator; 049import org.openimaj.image.feature.local.detector.mser.MSERFeatureGenerator.MSERDirection; 050import org.openimaj.image.pixel.ConnectedComponent; 051import org.openimaj.image.pixel.PixelSet; 052import org.openimaj.image.processing.morphology.Close; 053import org.openimaj.image.processing.morphology.Open; 054import org.openimaj.math.geometry.shape.Rectangle; 055import org.openimaj.video.VideoDisplay; 056import org.openimaj.video.VideoDisplayListener; 057import org.openimaj.video.capture.Device; 058import org.openimaj.video.capture.VideoCapture; 059 060public class TouchTableDemo implements VideoDisplayListener<MBFImage> { 061 062 private static final int IMAGE_WIDTH = 160; 063 private static final int IMAGE_HEIGHT = 120; 064 public static final int SMALLEST_POINT_AREA = Math.max(1, (IMAGE_WIDTH * IMAGE_HEIGHT) / (480 * 360)); 065 public static final int BIGGEST_POINT_AREA = Math.max(1, (IMAGE_WIDTH * IMAGE_HEIGHT) / (30 * 10)); 066 public static final int SMALLEST_POINT_DIAMETER = IMAGE_HEIGHT / 30; 067 public static final int BIGGEST_POINT_DIAMETER = SMALLEST_POINT_DIAMETER * 2; 068 private VideoCapture capture; 069 private VideoDisplay<MBFImage> display; 070 private ConnectedComponentLabeler labler; 071 private TouchTableScreen touchTableScreen; 072 private FImageBackgroundLearner backgroundLearner; 073 074 enum DebugMode { 075 DEBUG_DISPLAY, 076 NONE 077 } 078 079 private DebugMode mode = DebugMode.NONE; 080 private KeyListener touchTableKeyboard; 081 082 public Rectangle extractionArea = new Rectangle(IMAGE_WIDTH / 10, IMAGE_HEIGHT / 10, 083 IMAGE_WIDTH - (IMAGE_WIDTH / 5f), IMAGE_HEIGHT - (IMAGE_HEIGHT / 4.5f)); 084 private MSERFeatureGenerator mserDetector; 085 086 public TouchTableDemo() throws IOException { 087 final List<Device> captureDevices = VideoCapture.getVideoDevices(); 088 this.capture = new VideoCapture(IMAGE_WIDTH, IMAGE_HEIGHT, 30, captureDevices.get(0)); 089 this.display = VideoDisplay.createVideoDisplay(capture); 090 mserDetector = new MSERFeatureGenerator(MomentFeature.class); 091 this.labler = new ConnectedComponentLabeler(Algorithm.SINGLE_PASS, ConnectedComponent.ConnectMode.CONNECT_4); 092 093 final GraphicsEnvironment ge = GraphicsEnvironment.getLocalGraphicsEnvironment(); 094 final GraphicsDevice[] devices = ge.getScreenDevices(); 095 try { 096 Thread.sleep(1000); 097 } catch (final InterruptedException e) { 098 // TODO Auto-generated catch block 099 e.printStackTrace(); 100 } 101 if (devices.length > 1) { 102 // Then there is a touchscreen attached 103 this.touchTableScreen = new TouchTableScreen(extractionArea, new Rectangle(0, 0, 800, 600 - 25)); 104 this.touchTableScreen.setUndecorated(true); 105 this.touchTableScreen.setAlwaysOnTop(true); 106 devices[1].setFullScreenWindow(this.touchTableScreen); 107 this.touchTableScreen.init(); 108 109 } 110 111 this.backgroundLearner = new FImageBackgroundLearner(); 112 this.display.addVideoListener(this); 113 touchTableKeyboard = new TouchTableKeyboard(this, this.touchTableScreen); 114 SwingUtilities.getRoot(this.display.getScreen()).addKeyListener(this.touchTableKeyboard); 115 116 } 117 118 public static void main(String[] args) throws IOException { 119 new TouchTableDemo(); 120 } 121 122 @Override 123 public void afterUpdate(VideoDisplay<MBFImage> display) { 124 125 } 126 127 @Override 128 public void beforeUpdate(MBFImage frame) { 129 // Rectangle extractionArea = new 130 // Rectangle(100,100,IMAGE_WIDTH-200,IMAGE_HEIGHT-200); 131 132 if (this.mode == DebugMode.DEBUG_DISPLAY) 133 return; 134 final FImage grey = frame.extractROI(extractionArea).flatten(); 135 if (!this.backgroundLearner.ready()) { 136 grey.process(this.backgroundLearner); 137 frame.fill(RGBColour.BLACK); 138 frame.drawImage(new MBFImage(grey, grey, grey), (int) extractionArea.x, (int) extractionArea.y); 139 return; 140 } 141 grey.addInplace(this.backgroundLearner.getBackground()); 142 grey.threshold(0.07f); 143 // grey.processInplace(new OtsuThreshold()); 144 // if(grey.sum() > BIGGEST_POINT_AREA * 2 ){ 145 // this.backgroundLearner.relearn(); 146 // return; 147 // } 148 149 // List<Circle> filtered = getFilteredCircles(grey); 150 final List<Touch> filtered = getFilteredTouchesFast(grey); 151 if (filtered.size() != 0) 152 this.fireTouchEvent(filtered); 153 frame.fill(RGBColour.BLACK); 154 frame.drawImage(new MBFImage(grey, grey, grey), (int) extractionArea.x, (int) extractionArea.y); 155 } 156 157 private List<Touch> getFilteredTouchesFast(FImage grey) { 158 final Close morphClose = new Close(); 159 final Open morphOpen = new Open(); 160 // grey.processInplace(morphOpen); 161 // grey.processInplace(morphClose); 162 163 final List<Component> comps = mserDetector.generateMSERs(grey, MSERDirection.Down); 164 final List<Touch> ret = new ArrayList<Touch>(); 165 166 for (final Component component : comps) { 167 final int nPixels = component.size(); 168 if (nPixels < SMALLEST_POINT_AREA) 169 { 170 // if(nPixels > SMALLEST_POINT_AREA/10) 171 continue; 172 } 173 // else if(nPixels > BIGGEST_POINT_AREA) { 174 // continue; 175 // } 176 ret.add(new Touch(((MomentFeature) component.getFeature(0)).getCircle(10.0f))); 177 } 178 return ret; 179 } 180 181 private List<Touch> getFilteredTouches(FImage grey) { 182 final List<ConnectedComponent> comps = labler.findComponents(grey); 183 184 final List<Touch> filtered = new ArrayList<Touch>(); 185 186 double[] hw = null; 187 double[] c = null; 188 189 for (final PixelSet connectedComponent : comps) { 190 final int nPixels = connectedComponent.pixels.size(); 191 if (nPixels < SMALLEST_POINT_AREA) 192 { 193 // if(nPixels > SMALLEST_POINT_AREA/10) 194 continue; 195 } 196 else if (nPixels > BIGGEST_POINT_AREA) { 197 continue; 198 } 199 c = connectedComponent.calculateCentroid(); 200 hw = connectedComponent.calculateAverageHeightWidth(c); 201 202 filtered.add(new Touch((float) c[0], (float) c[1], (float) Math.sqrt(hw[0] * hw[0] + hw[1] * hw[1]))); 203 } 204 return filtered; 205 } 206 207 private void fireTouchEvent(List<Touch> filtered) { 208 if (this.touchTableScreen != null) 209 this.touchTableScreen.touchEvent(filtered); 210 } 211}