View Javadoc

1   /**
2    * Copyright (c) 2011, The University of Southampton and the individual contributors.
3    * All rights reserved.
4    *
5    * Redistribution and use in source and binary forms, with or without modification,
6    * are permitted provided that the following conditions are met:
7    *
8    *   * 	Redistributions of source code must retain the above copyright notice,
9    * 	this list of conditions and the following disclaimer.
10   *
11   *   *	Redistributions in binary form must reproduce the above copyright notice,
12   * 	this list of conditions and the following disclaimer in the documentation
13   * 	and/or other materials provided with the distribution.
14   *
15   *   *	Neither the name of the University of Southampton nor the names of its
16   * 	contributors may be used to endorse or promote products derived from this
17   * 	software without specific prior written permission.
18   *
19   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
20   * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21   * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22   * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
23   * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24   * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
26   * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27   * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28   * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29   */
30  package org.openimaj.demos.video;
31  
32  import java.awt.Dimension;
33  import java.awt.Font;
34  import java.awt.HeadlessException;
35  import java.awt.image.BufferedImage;
36  import java.io.IOException;
37  import java.util.ArrayList;
38  import java.util.List;
39  
40  import javax.swing.JFrame;
41  import javax.swing.JPanel;
42  
43  import org.openimaj.demos.Demo;
44  import org.openimaj.image.DisplayUtilities.ImageComponent;
45  import org.openimaj.image.ImageUtilities;
46  import org.openimaj.image.MBFImage;
47  import org.openimaj.image.processing.transform.MBFProjectionProcessor;
48  import org.openimaj.image.typography.general.GeneralFont;
49  import org.openimaj.image.typography.general.GeneralFontRenderer;
50  import org.openimaj.image.typography.general.GeneralFontStyle;
51  import org.openimaj.math.geometry.point.Point2d;
52  import org.openimaj.math.geometry.point.Point2dImpl;
53  import org.openimaj.math.geometry.shape.Rectangle;
54  import org.openimaj.math.geometry.transforms.TransformUtilities;
55  import org.openimaj.util.pair.Pair;
56  import org.openimaj.video.Video;
57  import org.openimaj.video.VideoDisplay;
58  import org.openimaj.video.VideoDisplayListener;
59  import org.openimaj.video.capture.VideoCapture;
60  
61  import cern.colt.Arrays;
62  
63  import com.google.zxing.BinaryBitmap;
64  import com.google.zxing.LuminanceSource;
65  import com.google.zxing.Result;
66  import com.google.zxing.ResultPoint;
67  import com.google.zxing.client.j2se.BufferedImageLuminanceSource;
68  import com.google.zxing.common.HybridBinarizer;
69  
70  /**
71   * A demo of ZXing using OpenIMAJ to grab images from the camera
72   * 
73   * @author Mike Cook
74   * @author Sina Samangooei (ss@ecs.soton.ac.uk)
75   * 
76   * @created 28 Sep 2011
77   */
78  @Demo(author = "Michael Cook", description = "Using ZXing, QR image rendering with OpenIMAJ", keywords = { "video" }, title = "QR Code Tracking")
79  public class QRTrackingDemo extends JPanel implements
80  VideoDisplayListener<MBFImage> {
81  	/** */
82  	private static final long serialVersionUID = 1L;
83  
84  	/** The video */
85  	private Video<MBFImage> video;
86  	/** The video display which will play the video */
87  	private VideoDisplay<MBFImage> videoDisplay;
88  	/** The image component into which the video is being painted (reused) */
89  	private final ImageComponent ic;
90  	/** The thread which is running the video playback */
91  	private Thread videoThread;
92  
93  	/** Mike: Need to deal in BufferedImage's for QR code reader */
94  	private BufferedImage bimg;
95  	/** Mike: added MBFImage for processing frames */
96  	private MBFImage toDraw;
97  	/** Mike: added qr code reader class */
98  	com.google.zxing.Reader reader;
99  	/** Mike: add last image string */
100 	String lastImage; // This will have the cover image
101 	/** The QR points matched*/
102 	List<Pair<Point2d>> points = new ArrayList<Pair<Point2d>>();
103 	/** Time QR points were last matched */
104 	long timeLastMatched;
105 
106 	/**
107 	 * Default constructor.
108 	 * 
109 	 * @throws IOException
110 	 */
111 	public QRTrackingDemo() throws IOException {
112 		this.ic = new ImageComponent(true);
113 		this.ic.setPreferredSize(new Dimension(320, 240));
114 		this.toDraw = new MBFImage(320, 240, 3);
115 		// Now test to see if it has a QR code embedded in it
116 		this.reader = new com.google.zxing.qrcode.QRCodeReader();
117 		this.lastImage = "";
118 		this.add(this.ic);
119 	}
120 
121 	/**
122 	 * 
123 	 Set the video source to be the webcam
124 	 * 
125 	 * @throws IOException
126 	 */
127 	public void useWebcam() throws IOException {
128 		// Setup a new video from the VideoCapture class
129 		this.video = new VideoCapture(320, 240);
130 		// Reset the video displayer to use the capture class
131 		this.videoDisplay = new VideoDisplay<MBFImage>(this.video, this.ic);
132 		// Make sure the listeners are sorted
133 		this.videoDisplay.addVideoListener(this);
134 		// Start the new video playback thread
135 		this.videoThread = new Thread(this.videoDisplay);
136 		this.videoThread.start();
137 	}
138 
139 
140 
141 	/**
142 	 * {@inheritDoc}
143 	 * @see org.openimaj.video.VideoDisplayListener#afterUpdate(org.openimaj.video.VideoDisplay)
144 	 */
145 	@Override
146 	public void afterUpdate(final VideoDisplay<MBFImage> display) {
147 		if(System.currentTimeMillis() - this.timeLastMatched > 100){
148 			this.points.clear();
149 		}
150 	}
151 
152 	/**
153 	 * {@inheritDoc}
154 	 * @see org.openimaj.video.VideoDisplayListener#beforeUpdate(org.openimaj.image.Image)
155 	 */
156 	@Override
157 	public void beforeUpdate(final MBFImage frame) {
158 		this.bimg = ImageUtilities.createBufferedImageForDisplay(frame, this.bimg);
159 		final LuminanceSource source = new BufferedImageLuminanceSource(this.bimg);
160 
161 		final BinaryBitmap bitmap = new BinaryBitmap(new HybridBinarizer(source));
162 
163 		try {
164 			final Result result = this.reader.decode(bitmap);
165 			if (result.getText() != null) {
166 				if( !result.getText().equals( this.lastImage ))
167 				{
168 					this.lastImage = result.getText();
169 					try {
170 						final MBFImage img = new MBFImage( 200, 50, 3 );
171 
172 						final GeneralFont font = new GeneralFont( "Arial", Font.PLAIN );
173 						final GeneralFontStyle<Float[]> gfs = new GeneralFontStyle<Float[]>( font, img.createRenderer(), false );
174 						final GeneralFontRenderer<Float[]> gfr = new GeneralFontRenderer<Float[]>();
175 						final Rectangle b = gfr.getSize( this.lastImage, gfs );
176 
177 						final MBFImage img2 = new MBFImage( (int)b.width, (int)(b.height*1.3), 3 );
178 
179 						img2.drawText( this.lastImage, 0, (int)b.height, font, 30 );
180 						this.toDraw = img2;
181 					}
182 
183 					catch (final Exception e) {
184 						e.printStackTrace();
185 						System.out.println("could not read url");
186 					}
187 				}
188 
189 				try {
190 					final ResultPoint[] rpoints = result.getResultPoints();
191 					System.out.println( Arrays.toString( rpoints ) );
192 					this.points.clear();
193 					if( rpoints.length >= 3 )
194 					{
195 						float s = 1;
196 						float xx1 = 0;
197 						float xx2 = 0;
198 						for (int i = 2; i >= 0; i--) {
199 							final Point2dImpl pa = new Point2dImpl(rpoints[i].getX(),rpoints[i].getY());
200 							Point2dImpl pb = null;
201 							if(i == 1) {
202 								pb = new Point2dImpl(0,0);
203 								xx1 = pa.x;
204 							}
205 							if(i == 2) {
206 								pb = new Point2dImpl(this.toDraw.getWidth(),0);
207 								xx2 = pa.x;
208 							}
209 							if(i == 0) {
210 								s = this.toDraw.getWidth() / (xx2 - xx1) * 3;
211 								pb = new Point2dImpl( 0, this.toDraw.getHeight()*s );
212 							}
213 							this.points.add(new Pair<Point2d>(pa,pb));
214 						}
215 					}
216 					this.timeLastMatched = System.currentTimeMillis();
217 					//					frame.createRenderer().drawImage(toDraw, x, y);
218 
219 
220 				} catch (final Exception e) {
221 					System.out.println("could not find image");
222 				}
223 
224 			}
225 		} catch (final Exception e) {
226 
227 		}
228 		if(this.points.size()>2){
229 			final MBFProjectionProcessor pp = new MBFProjectionProcessor();
230 			pp.accumulate(frame);
231 			pp.setMatrix(TransformUtilities.affineMatrix(this.points).inverse());
232 			pp.accumulate(this.toDraw);
233 			frame.internalAssign(pp.performProjection());
234 		}
235 
236 
237 	}
238 
239 	/**
240 	 * 
241 	 * @param args
242 	 */
243 	public static void main(final String[] args) {
244 		try {
245 
246 			final QRTrackingDemo demo = new QRTrackingDemo();
247 			final JFrame f = new JFrame("Video Processing Demo -- Mike");
248 			f.getContentPane().add(demo);
249 			f.pack();
250 			f.setVisible(true);
251 			demo.useWebcam();
252 			// demo.useFile(new
253 			// File("/Users/ss/Downloads/20070701_185500_bbcthree_doctor_who_confidential.ts"));
254 		} catch (final HeadlessException e) {
255 			e.printStackTrace();
256 		} catch (final IOException e) {
257 			e.printStackTrace();
258 		}
259 	}
260 }