001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.demos.touchtable; 031 032import java.io.BufferedReader; 033import java.io.ByteArrayOutputStream; 034import java.io.IOException; 035import java.io.InputStreamReader; 036import java.io.OutputStream; 037import java.io.PrintWriter; 038import java.io.UnsupportedEncodingException; 039import java.net.ServerSocket; 040import java.net.Socket; 041import java.security.MessageDigest; 042import java.security.NoSuchAlgorithmException; 043import java.util.ArrayList; 044import java.util.HashMap; 045import java.util.List; 046import java.util.Map; 047 048import javax.swing.JFrame; 049 050import org.openimaj.demos.sandbox.Pong; 051import org.openimaj.image.DisplayUtilities; 052import org.openimaj.image.MBFImage; 053import org.openimaj.image.colour.ColourSpace; 054import org.openimaj.image.colour.RGBColour; 055import org.openimaj.math.geometry.line.Line2d; 056import org.openimaj.math.geometry.point.Point2d; 057import org.openimaj.math.geometry.point.Point2dImpl; 058import org.openimaj.math.geometry.shape.Circle; 059import org.openimaj.math.geometry.shape.Rectangle; 060import org.openimaj.math.geometry.transforms.HomographyModel; 061import org.openimaj.util.pair.IndependentPair; 062 063import com.lowagie.text.pdf.codec.Base64; 064 065public class TouchTableScreen extends JFrame implements Runnable { 066 067 /** 068 * A touchtable full screen jframe 069 */ 070 private static final long serialVersionUID = -966931575089952536L; 071 private MBFImage image; 072 Mode mode; 073 public CameraConfig cameraConfig; 074 private Rectangle inputArea; 075 private Rectangle visibleArea; 076 private boolean renderMode = true; 077 private boolean clear = false;; 078 079 interface Mode { 080 public class PONG extends Pong implements Mode { 081 private TouchTableScreen touchScreen; 082 083 public PONG(TouchTableScreen touchScreen) { 084 super((int) touchScreen.visibleArea.width, (int) touchScreen.visibleArea.height); 085 this.touchScreen = touchScreen; 086 reset(); 087 } 088 089 @Override 090 public void acceptTouch(List<Touch> filtered) { 091 for (final Touch tableTouch : filtered) { 092 final Touch touch = this.touchScreen.cameraConfig.transformTouch(tableTouch); 093 if (touch == null) 094 continue; 095 // System.out.println(touch); 096 // System.out.println(this.getWidth()/2); 097 // goToFinger(touch); 098 followFinger(touch); 099 } 100 } 101 102 private void followFinger(Touch touch) { 103 if (touch.intersectionArea(this.paddleLeft) > 0) { 104 this.leftPaddle(touch.getY()); 105 } 106 else if (touch.intersectionArea(this.paddleRight) > 0) { 107 this.rightPaddle(touch.getY()); 108 } 109 } 110 111 private void goToFinger(Touch touch) { 112 if (touch.getX() < this.getWidth() / 2) // left paddle 113 { 114 if (touch.getY() < this.leftPaddleY()) { 115 this.leftPaddleUp(); 116 } 117 else { 118 this.leftPaddleDown(); 119 } 120 } 121 else { // right paddle 122 if (touch.getY() < this.rightPaddleY()) { 123 this.rightPaddleUp(); 124 } 125 else { 126 this.rightPaddleDown(); 127 } 128 } 129 } 130 131 @Override 132 public void drawToImage(MBFImage image) { 133 final MBFImage gFrame = getNextFrame(); 134 image.drawImage(gFrame, 0, 0); 135 } 136 137 } 138 139 public class DRAWING implements Mode { 140 141 protected TouchTableScreen touchScreen; 142 protected List<Touch> points; 143 144 public DRAWING(TouchTableScreen touchScreen) { 145 this.touchScreen = touchScreen; 146 points = new ArrayList<Touch>(); 147 } 148 149 @Override 150 public synchronized void acceptTouch(List<Touch> filtered) { 151 this.points.addAll(filtered); 152 } 153 154 @Override 155 public void drawToImage(MBFImage image) { 156 final List<Touch> toDraw = this.getDrawingPoints(); 157 // if(this.touchScreen.cameraConfig instanceof 158 // TriangleCameraConfig){ 159 // ((TriangleCameraConfig)this.touchScreen.cameraConfig).drawTriangles(image); 160 // 161 // } 162 for (final Touch touch : toDraw) { 163 // Point2d trans = 164 // point2d.transform(this.touchScreen.cameraConfig.homography); 165 166 final Circle trans = this.touchScreen.cameraConfig.transformTouch(touch); 167 if (trans != null) 168 image.drawShapeFilled(trans, RGBColour.BLUE); 169 } 170 } 171 172 protected synchronized List<Touch> getDrawingPoints() { 173 final List<Touch> toRet = this.points; 174 this.points = new ArrayList<Touch>(); 175 return toRet; 176 } 177 } 178 179 public class DRAWING_TRACKED extends DRAWING { 180 Map<Long, Float[]> colours = new HashMap<Long, Float[]>(); 181 ReallyBasicTouchTracker tracker = new ReallyBasicTouchTracker(75); 182 183 public DRAWING_TRACKED(TouchTableScreen touchScreen) { 184 super(touchScreen); 185 } 186 187 @Override 188 public synchronized void acceptTouch(List<Touch> filtered) { 189 List<Touch> tracked = new ArrayList<Touch>(); 190 191 for (final Touch touch : filtered) { 192 final Touch trans = this.touchScreen.cameraConfig.transformTouch(touch); 193 194 if (trans != null) 195 tracked.add(trans); 196 } 197 198 tracked = tracker.trackPoints(tracked); 199 this.points.addAll(tracked); 200 } 201 202 @Override 203 public void drawToImage(MBFImage image) { 204 final List<Touch> toDraw = this.getDrawingPoints(); 205 206 for (final Touch touch : toDraw) { 207 Float[] col = colours.get(touch.touchID); 208 if (touch.getRadius() > 15) 209 col = RGBColour.WHITE; 210 else if (col == null) 211 colours.put(touch.touchID, col = RGBColour.randomColour()); 212 213 image.drawShapeFilled(touch, col); 214 if (touch.motionVector != null) 215 image.drawLine( 216 (int) touch.getX(), (int) touch.getY(), 217 (int) (touch.getX() - touch.motionVector.x), 218 (int) (touch.getY() - touch.motionVector.y), 219 (int) (2 * touch.getRadius()), col); 220 } 221 } 222 } 223 224 public class SERVER extends DRAWING_TRACKED { 225 static List<OutputStream> pws = new ArrayList<OutputStream>(); 226 static ServerSocket serverSocket; 227 228 public SERVER(TouchTableScreen touchScreen) { 229 super(touchScreen); 230 touchScreen.setRenderMode(false); 231 232 new Thread(new Runnable() { 233 234 @Override 235 public void run() { 236 try { 237 if (serverSocket != null) 238 return; 239 serverSocket = new ServerSocket(40000); 240 } catch (final IOException e) { 241 System.out.println("Unable to bind to port"); 242 return; 243 } 244 245 while (true) { 246 PrintWriter pw; 247 try { 248 final Socket conn = serverSocket.accept(); 249 final BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream())); 250 String line = null; 251 String securityBit = ""; 252 String key = ""; 253 while ((line = br.readLine()) != null) { 254 // System.out.println(line); 255 if (line.startsWith("Sec")) { 256 securityBit += line + "\r\n"; 257 if (line.contains("Key")) { 258 key = line.split(":")[1].trim(); 259 } 260 if (line.contains("Version")) { 261 break; 262 } 263 } 264 } 265 System.out.println("Client connected!"); 266 final OutputStream os = conn.getOutputStream(); 267 pw = new PrintWriter(os); 268 pw.print("HTTP/1.1 101 Web Socket Protocol Handshake\r\n"); 269 pw.print("Upgrade: WebSocket\r\n"); 270 pw.print("Connection: Upgrade\r\n"); 271 pw.print("Sec-WebSocket-Origin: null\r\n"); 272 pw.print("Sec-WebSocket-Location: ws://127.0.0.1:40000\r\n"); 273 System.out.println("Key: \"" + key + "\""); 274 final String combined = key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; 275 final byte[] sha1 = MessageDigest.getInstance("SHA-1").digest(combined.getBytes("UTF8")); 276 System.out.println("Number of bytes: " + sha1.length); 277 final String encoded = Base64.encodeBytes(sha1); 278 System.out.println("encoded string: " + encoded); 279 pw.print("Sec-WebSocket-Accept: " + encoded + "\r\n"); 280 // pw.print(securityBit); 281 pw.print("\r\n"); 282 pw.flush(); 283 synchronized (pws) { 284 pws.add(os); 285 } 286 } catch (final IOException e) { 287 e.printStackTrace(); 288 } catch (final NoSuchAlgorithmException e) { 289 // TODO Auto-generated catch block 290 e.printStackTrace(); 291 } 292 } 293 } 294 295 }).start(); 296 } 297 298 @Override 299 public synchronized void acceptTouch(List<Touch> filtered) { 300 super.acceptTouch(filtered); 301 302 // String touches = "" + Math.random() + "\n"; 303 final List<Touch> pointsToPrint = this.getDrawingPoints(); 304 // String touches = createTouchesString(pointsToPrint); 305 final byte[] touches = createTouchesString(pointsToPrint); 306 307 synchronized (pws) { 308 // List<PrintWriter> toKill = new ArrayList<PrintWriter>(); 309 310 for (final OutputStream pw : pws) { 311 // try { 312 try { 313 pw.write(touches); 314 pw.flush(); 315 } catch (final IOException e) { 316 } 317 // } catch (IOException e) { 318 // toKill.add(pw); 319 // } 320 } 321 322 // pws.removeAll(toKill); 323 } 324 } 325 326 private byte[] createTouchesString(List<Touch> pointsToPrint) { 327 final StringBuilder builder = new StringBuilder(); 328 final String touchFormat = "(%f %f %f %d) "; 329 final String timeFormat = "[%d] "; 330 builder.append(String.format(timeFormat, System.currentTimeMillis())); 331 for (final Touch touch : pointsToPrint) { 332 builder.append(String.format(touchFormat, touch.getX(), touch.getY(), touch.getRadius(), 333 touch.touchID)); 334 } 335 // return builder.toString(); 336 // String stringout = "{\"wang\" : \"foo\"}"; 337 final String stringout = builder.toString(); 338 final byte[] bytesraw = stringout.getBytes(); 339 340 byte[] out = null; 341 final int opcode = 129; 342 if (bytesraw.length < 126) { 343 out = new byte[] { (byte) opcode, (byte) bytesraw.length }; 344 } 345 else { 346 final byte first = (byte) ((bytesraw.length >> 8) & 255); 347 final byte second = (byte) (bytesraw.length & 255); 348 349 out = new byte[] { (byte) opcode, 126, first, second }; 350 } 351 try { 352 final ByteArrayOutputStream bos = new ByteArrayOutputStream(); 353 bos.write(out); 354 bos.write(stringout.getBytes("UTF8")); 355 return bos.toByteArray(); 356 } catch (final UnsupportedEncodingException e) { 357 358 } catch (final IOException e) { 359 // TODO Auto-generated catch block 360 e.printStackTrace(); 361 } 362 return new byte[0]; 363 } 364 } 365 366 class CALIBRATION_TRIANGLES implements Mode { 367 368 private static final int GRIDY = 4; 369 private static final int GRIDX = 5; 370 371 private ArrayList<Point2d> touchArray; 372 373 int gridxy = (GRIDX + 1) * (GRIDY + 1); // a 4x4 grid of points 374 private TouchTableScreen touchScreen; 375 376 public CALIBRATION_TRIANGLES(TouchTableScreen touchTableScreen) { 377 this.touchScreen = touchTableScreen; 378 this.touchArray = new ArrayList<Point2d>(); 379 } 380 381 @Override 382 public void acceptTouch(List<Touch> filtered) { 383 final Point2d pixelToAdd = filtered.get(0).calculateCentroid(); 384 Point2d lastPointAdded = null; 385 if (this.touchArray.size() != 0) 386 lastPointAdded = this.touchArray.get(this.touchArray.size() - 1); 387 if (lastPointAdded == null || 388 Line2d.distance(pixelToAdd, lastPointAdded) > TouchTableDemo.SMALLEST_POINT_DIAMETER) 389 { 390 this.touchArray.add(pixelToAdd); 391 } 392 393 if (this.touchArray.size() == this.gridxy) { 394 calibrate(); 395 } 396 } 397 398 @Override 399 public void drawToImage(MBFImage image) { 400 image.fill(RGBColour.WHITE); 401 final int nPoints = touchArray.size(); 402 final float gridX = nPoints % (GRIDX + 1); 403 final float gridY = nPoints / (GRIDX + 1); 404 405 final Point2dImpl currentpoint = new Point2dImpl( 406 (image.getWidth() * (gridX / GRIDX)), 407 ((image.getHeight()) * (gridY / GRIDY)) 408 ); 409 drawTarget(image, currentpoint); 410 } 411 412 private void drawTarget(MBFImage image, Point2d point) { 413 image.drawShapeFilled(new Rectangle(point.getX() - 5, point.getY() - 5, 10, 10), RGBColour.RED); 414 } 415 416 private void calibrate() { 417 this.touchScreen.cameraConfig = new TriangleCameraConfig( 418 this.touchArray, GRIDX, GRIDY, this.touchScreen.visibleArea 419 ); 420 touchScreen.mode = new Mode.DRAWING(touchScreen); 421 } 422 423 } 424 425 class CALIBRATION_HOMOGRAPHY implements Mode { 426 427 private static Point2d TOP_LEFT = null; 428 private static Point2d TOP_RIGHT = null; 429 private static Point2d BOTTOM_LEFT = null; 430 private static Point2d BOTTOM_RIGHT = null; 431 private ArrayList<Point2d> touchArray; 432 private TouchTableScreen touchScreen; 433 434 public CALIBRATION_HOMOGRAPHY(TouchTableScreen touchTableScreen) { 435 this.touchArray = new ArrayList<Point2d>(); 436 TOP_LEFT = new Point2dImpl(30f, 30f); 437 TOP_RIGHT = new Point2dImpl(touchTableScreen.image.getWidth() - 30f, 30f); 438 BOTTOM_LEFT = new Point2dImpl(30f, touchTableScreen.image.getHeight() - 30f); 439 BOTTOM_RIGHT = new Point2dImpl(touchTableScreen.image.getWidth() - 30f, 440 touchTableScreen.image.getHeight() - 30f); 441 this.touchScreen = touchTableScreen; 442 } 443 444 @Override 445 public void drawToImage(MBFImage image) { 446 image.fill(RGBColour.WHITE); 447 switch (this.touchArray.size()) { 448 case 0: 449 drawTarget(image, TOP_LEFT); 450 break; 451 case 1: 452 drawTarget(image, TOP_RIGHT); 453 break; 454 case 2: 455 drawTarget(image, BOTTOM_LEFT); 456 break; 457 case 3: 458 drawTarget(image, BOTTOM_RIGHT); 459 break; 460 default: 461 break; 462 } 463 } 464 465 private void drawTarget(MBFImage image, Point2d point) { 466 image.drawPoint(point, RGBColour.RED, 10); 467 } 468 469 @Override 470 public void acceptTouch(List<Touch> filtered) { 471 final Point2d pixelToAdd = filtered.get(0).calculateCentroid(); 472 Point2d lastPointAdded = null; 473 if (this.touchArray.size() != 0) 474 lastPointAdded = this.touchArray.get(this.touchArray.size() - 1); 475 if (lastPointAdded == null || 476 Line2d.distance(pixelToAdd, lastPointAdded) > TouchTableDemo.SMALLEST_POINT_DIAMETER) 477 { 478 this.touchArray.add(pixelToAdd); 479 } 480 481 if (this.touchArray.size() == 4) { 482 calibrate(); 483 } 484 } 485 486 private void calibrate() { 487 final HomographyModel m = new HomographyModel(); 488 final List<IndependentPair<Point2d, Point2d>> matches = new ArrayList<IndependentPair<Point2d, Point2d>>(); 489 490 matches.add(IndependentPair.pair(TOP_LEFT, this.touchArray.get(0))); 491 matches.add(IndependentPair.pair(TOP_RIGHT, this.touchArray.get(1))); 492 matches.add(IndependentPair.pair(BOTTOM_LEFT, this.touchArray.get(2))); 493 matches.add(IndependentPair.pair(BOTTOM_RIGHT, this.touchArray.get(3))); 494 final HomographyCameraConfig cameraConfig = new HomographyCameraConfig( 495 4.9736307741305950e+002f, 4.9705029823649602e+002f, 496 touchScreen.inputArea.width / 2, touchScreen.inputArea.height / 2, 497 5.8322574816106650e-002f, -1.7482068549377444e-001f, 498 -3.1083477039117124e-003f, -4.3781939644044129e-003f 499 ); 500 m.estimate(matches); 501 cameraConfig.homography = m.getTransform().inverse(); 502 touchScreen.cameraConfig = cameraConfig; 503 touchScreen.mode = new Mode.DRAWING(touchScreen); 504 } 505 }; 506 507 public void drawToImage(MBFImage image); 508 509 public void acceptTouch(List<Touch> filtered); 510 } 511 512 public TouchTableScreen(Rectangle extractionArea, Rectangle visibleArea) { 513 this.setUndecorated(true); 514 this.inputArea = extractionArea; 515 this.visibleArea = visibleArea; 516 } 517 518 public void setRenderMode(boolean renderMode) { 519 this.renderMode = renderMode; 520 this.setVisible(renderMode); 521 522 } 523 524 public void init() { 525 final int width = this.getWidth(); 526 final int height = this.getHeight(); 527 528 image = new MBFImage(width, height, ColourSpace.RGB); 529 this.mode = new Mode.CALIBRATION_TRIANGLES(this); 530 531 final Thread t = new Thread(this); 532 t.start(); 533 } 534 535 public void touchEvent(List<Touch> filtered) { 536 this.mode.acceptTouch(filtered); 537 } 538 539 @Override 540 public void run() { 541 while (true) { 542 if (!renderMode) 543 break; 544 final MBFImage extracted = this.image.extractROI(this.visibleArea); 545 if (clear) { 546 extracted.fill(RGBColour.WHITE); 547 this.clear = false; 548 } 549 this.mode.drawToImage(extracted); 550 551 this.image.drawImage(extracted, 0, 0); 552 DisplayUtilities.display(this.image, this); 553 } 554 } 555 556 public void setCameraConfig(CameraConfig newCC) { 557 this.cameraConfig = newCC; 558 this.mode = new Mode.DRAWING(this); 559 } 560 561 public void clear() { 562 this.clear = true; 563 } 564 565 public static void main(String[] args) throws NoSuchAlgorithmException, UnsupportedEncodingException { 566 final String key = "x3JJHMbDL1EzLkh9GBhXDw=="; 567 final String combined = key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; 568 final byte[] sha1 = MessageDigest.getInstance("SHA-1").digest(combined.getBytes("UTF8")); 569 System.out.println("Number of bytes: " + sha1.length); 570 final String encoded = Base64.encodeBytes(sha1); 571 System.out.println("encoded string: " + encoded); 572 } 573}