001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.demos.video; 031 032import java.awt.Dimension; 033import java.awt.Font; 034import java.awt.HeadlessException; 035import java.awt.image.BufferedImage; 036import java.io.IOException; 037import java.util.ArrayList; 038import java.util.List; 039 040import javax.swing.JFrame; 041import javax.swing.JPanel; 042 043import org.openimaj.demos.Demo; 044import org.openimaj.image.DisplayUtilities.ImageComponent; 045import org.openimaj.image.ImageUtilities; 046import org.openimaj.image.MBFImage; 047import org.openimaj.image.processing.transform.MBFProjectionProcessor; 048import org.openimaj.image.typography.general.GeneralFont; 049import org.openimaj.image.typography.general.GeneralFontRenderer; 050import org.openimaj.image.typography.general.GeneralFontStyle; 051import org.openimaj.math.geometry.point.Point2d; 052import org.openimaj.math.geometry.point.Point2dImpl; 053import org.openimaj.math.geometry.shape.Rectangle; 054import org.openimaj.math.geometry.transforms.TransformUtilities; 055import org.openimaj.util.pair.Pair; 056import org.openimaj.video.Video; 057import org.openimaj.video.VideoDisplay; 058import org.openimaj.video.VideoDisplayListener; 059import org.openimaj.video.capture.VideoCapture; 060 061import cern.colt.Arrays; 062 063import com.google.zxing.BinaryBitmap; 064import com.google.zxing.LuminanceSource; 065import com.google.zxing.Result; 066import com.google.zxing.ResultPoint; 067import com.google.zxing.client.j2se.BufferedImageLuminanceSource; 068import com.google.zxing.common.HybridBinarizer; 069 070/** 071 * A demo of ZXing using OpenIMAJ to grab images from the camera 072 * 073 * @author Mike Cook 074 * @author Sina Samangooei (ss@ecs.soton.ac.uk) 075 * 076 * @created 28 Sep 2011 077 */ 078@Demo(author = "Michael Cook", description = "Using ZXing, QR image rendering with OpenIMAJ", keywords = { "video" }, title = "QR Code Tracking") 079public class QRTrackingDemo extends JPanel implements 080VideoDisplayListener<MBFImage> { 081 /** */ 082 private static final long serialVersionUID = 1L; 083 084 /** The video */ 085 private Video<MBFImage> video; 086 /** The video display which will play the video */ 087 private VideoDisplay<MBFImage> videoDisplay; 088 /** The image component into which the video is being painted (reused) */ 089 private final ImageComponent ic; 090 /** The thread which is running the video playback */ 091 private Thread videoThread; 092 093 /** Mike: Need to deal in BufferedImage's for QR code reader */ 094 private BufferedImage bimg; 095 /** Mike: added MBFImage for processing frames */ 096 private MBFImage toDraw; 097 /** Mike: added qr code reader class */ 098 com.google.zxing.Reader reader; 099 /** Mike: add last image string */ 100 String lastImage; // This will have the cover image 101 /** The QR points matched*/ 102 List<Pair<Point2d>> points = new ArrayList<Pair<Point2d>>(); 103 /** Time QR points were last matched */ 104 long timeLastMatched; 105 106 /** 107 * Default constructor. 108 * 109 * @throws IOException 110 */ 111 public QRTrackingDemo() throws IOException { 112 this.ic = new ImageComponent(true); 113 this.ic.setPreferredSize(new Dimension(320, 240)); 114 this.toDraw = new MBFImage(320, 240, 3); 115 // Now test to see if it has a QR code embedded in it 116 this.reader = new com.google.zxing.qrcode.QRCodeReader(); 117 this.lastImage = ""; 118 this.add(this.ic); 119 } 120 121 /** 122 * 123 Set the video source to be the webcam 124 * 125 * @throws IOException 126 */ 127 public void useWebcam() throws IOException { 128 // Setup a new video from the VideoCapture class 129 this.video = new VideoCapture(320, 240); 130 // Reset the video displayer to use the capture class 131 this.videoDisplay = new VideoDisplay<MBFImage>(this.video, this.ic); 132 // Make sure the listeners are sorted 133 this.videoDisplay.addVideoListener(this); 134 // Start the new video playback thread 135 this.videoThread = new Thread(this.videoDisplay); 136 this.videoThread.start(); 137 } 138 139 140 141 /** 142 * {@inheritDoc} 143 * @see org.openimaj.video.VideoDisplayListener#afterUpdate(org.openimaj.video.VideoDisplay) 144 */ 145 @Override 146 public void afterUpdate(final VideoDisplay<MBFImage> display) { 147 if(System.currentTimeMillis() - this.timeLastMatched > 100){ 148 this.points.clear(); 149 } 150 } 151 152 /** 153 * {@inheritDoc} 154 * @see org.openimaj.video.VideoDisplayListener#beforeUpdate(org.openimaj.image.Image) 155 */ 156 @Override 157 public void beforeUpdate(final MBFImage frame) { 158 this.bimg = ImageUtilities.createBufferedImageForDisplay(frame, this.bimg); 159 final LuminanceSource source = new BufferedImageLuminanceSource(this.bimg); 160 161 final BinaryBitmap bitmap = new BinaryBitmap(new HybridBinarizer(source)); 162 163 try { 164 final Result result = this.reader.decode(bitmap); 165 if (result.getText() != null) { 166 if( !result.getText().equals( this.lastImage )) 167 { 168 this.lastImage = result.getText(); 169 try { 170 final MBFImage img = new MBFImage( 200, 50, 3 ); 171 172 final GeneralFont font = new GeneralFont( "Arial", Font.PLAIN ); 173 final GeneralFontStyle<Float[]> gfs = new GeneralFontStyle<Float[]>( font, img.createRenderer(), false ); 174 final GeneralFontRenderer<Float[]> gfr = new GeneralFontRenderer<Float[]>(); 175 final Rectangle b = gfr.getSize( this.lastImage, gfs ); 176 177 final MBFImage img2 = new MBFImage( (int)b.width, (int)(b.height*1.3), 3 ); 178 179 img2.drawText( this.lastImage, 0, (int)b.height, font, 30 ); 180 this.toDraw = img2; 181 } 182 183 catch (final Exception e) { 184 e.printStackTrace(); 185 System.out.println("could not read url"); 186 } 187 } 188 189 try { 190 final ResultPoint[] rpoints = result.getResultPoints(); 191 System.out.println( Arrays.toString( rpoints ) ); 192 this.points.clear(); 193 if( rpoints.length >= 3 ) 194 { 195 float s = 1; 196 float xx1 = 0; 197 float xx2 = 0; 198 for (int i = 2; i >= 0; i--) { 199 final Point2dImpl pa = new Point2dImpl(rpoints[i].getX(),rpoints[i].getY()); 200 Point2dImpl pb = null; 201 if(i == 1) { 202 pb = new Point2dImpl(0,0); 203 xx1 = pa.x; 204 } 205 if(i == 2) { 206 pb = new Point2dImpl(this.toDraw.getWidth(),0); 207 xx2 = pa.x; 208 } 209 if(i == 0) { 210 s = this.toDraw.getWidth() / (xx2 - xx1) * 3; 211 pb = new Point2dImpl( 0, this.toDraw.getHeight()*s ); 212 } 213 this.points.add(new Pair<Point2d>(pa,pb)); 214 } 215 } 216 this.timeLastMatched = System.currentTimeMillis(); 217 // frame.createRenderer().drawImage(toDraw, x, y); 218 219 220 } catch (final Exception e) { 221 System.out.println("could not find image"); 222 } 223 224 } 225 } catch (final Exception e) { 226 227 } 228 if(this.points.size()>2){ 229 final MBFProjectionProcessor pp = new MBFProjectionProcessor(); 230 pp.accumulate(frame); 231 pp.setMatrix(TransformUtilities.affineMatrix(this.points).inverse()); 232 pp.accumulate(this.toDraw); 233 frame.internalAssign(pp.performProjection()); 234 } 235 236 237 } 238 239 /** 240 * 241 * @param args 242 */ 243 public static void main(final String[] args) { 244 try { 245 246 final QRTrackingDemo demo = new QRTrackingDemo(); 247 final JFrame f = new JFrame("Video Processing Demo -- Mike"); 248 f.getContentPane().add(demo); 249 f.pack(); 250 f.setVisible(true); 251 demo.useWebcam(); 252 // demo.useFile(new 253 // File("/Users/ss/Downloads/20070701_185500_bbcthree_doctor_who_confidential.ts")); 254 } catch (final HeadlessException e) { 255 e.printStackTrace(); 256 } catch (final IOException e) { 257 e.printStackTrace(); 258 } 259 } 260}