001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.demos.video; 031 032import java.awt.AlphaComposite; 033import java.awt.BorderLayout; 034import java.awt.Color; 035import java.awt.Dimension; 036import java.awt.FlowLayout; 037import java.awt.Font; 038import java.awt.FontMetrics; 039import java.awt.Graphics; 040import java.awt.Graphics2D; 041import java.awt.Image; 042import java.awt.event.ActionEvent; 043import java.awt.event.ActionListener; 044import java.awt.event.KeyAdapter; 045import java.text.DecimalFormat; 046 047import javax.swing.Box; 048import javax.swing.JButton; 049import javax.swing.JFrame; 050import javax.swing.JLabel; 051import javax.swing.JPanel; 052import javax.swing.JTextField; 053import javax.swing.SwingWorker; 054import javax.swing.Timer; 055 056import org.openimaj.image.DisplayUtilities.ImageComponent; 057import org.openimaj.image.MBFImage; 058import org.openimaj.video.VideoDisplay; 059import org.openimaj.video.VideoDisplayListener; 060import org.openimaj.video.capture.VideoCapture; 061import org.openimaj.video.capture.VideoCaptureException; 062import org.slf4j.Logger; 063import org.slf4j.LoggerFactory; 064 065/** 066 * Example showing how the built-in OpenIMAJ fps counter works and how to 067 * dipsplay real-time fps. Since {@link VideoDisplay#getDisplayFPS()} reports 068 * FPS speed with which display is updated, it is not necessairly the speed 069 * at which FPS is rendered out to the display. This program shows this 070 * discrepancy. Notice that while "capture" FPS reports at the consistent 071 * range, depending on frame delay set, the actual render FPS may be quite 072 * different. This demo implementation uses simple thread sleep to simulate 073 * expensive rendering operation. 074 * 075 * @author Adam Zimowski (mrazjava) 076 */ 077public class VideoCaptureFramesExample extends KeyAdapter { 078 079 private static final Logger log = LoggerFactory.getLogger(VideoCaptureFramesExample.class); 080 081 VideoCapture vc; 082 VideoDisplay<MBFImage> display; 083 Thread displayThread; 084 085 private static final String DELAY_LBL = "Delay (ms): "; 086 087 private int fpsDelayMillis = 0; 088 089 private final JLabel fpsDelayLabel; 090 091 private JTextField fpsDelayText; 092 093 /** 094 * @throws VideoCaptureException 095 */ 096 public VideoCaptureFramesExample() throws VideoCaptureException { 097 098 this.fpsDelayLabel = new JLabel(VideoCaptureFramesExample.DELAY_LBL + this.fpsDelayMillis); 099 100 // open the capture device and create a window to display in 320, 240 101 this.vc = new VideoCapture(1024, 768); 102 103 104 final FrameDemoImageComponent ic = new FrameDemoImageComponent(); 105 ic.setAllowZoom( false ); 106 ic.setAllowPanning( false ); 107 ic.setTransparencyGrid( false ); 108 ic.setShowPixelColours( false ); 109 ic.setShowXYPosition( true ); 110 111 this.buildGui(ic).setVisible(true); 112 113 this.display = new VideoDisplay<MBFImage>( this.vc, null, ic ); 114 this.display.addVideoListener(ic); 115 116 this.displayThread = new Thread(this.display); 117 this.displayThread.start(); 118 } 119 120 /** 121 * @param ic frame displaying component 122 * @return 123 */ 124 private JFrame buildGui(final ImageComponent ic) { 125 126 final JFrame win = new JFrame("Frame Counting Demo"); 127 win.setPreferredSize(new Dimension(640, 480)); 128 win.getContentPane().setLayout(new BorderLayout()); 129 130 final JPanel controlPanel = new JPanel(new BorderLayout()); 131 final JPanel setDelayPanel = new JPanel(new FlowLayout()); 132 133 final JButton setButton = new JButton("Set Delay"); 134 setButton.setPreferredSize(new Dimension(150, 20)); 135 setButton.setMinimumSize(new Dimension(75, 20)); 136 setButton.addActionListener(new ActionListener() { 137 @Override 138 public void actionPerformed(final ActionEvent arg0) { 139 try { 140 final int delay = Integer.valueOf(VideoCaptureFramesExample.this.fpsDelayText.getText()); 141 if(delay < 0) throw new NumberFormatException(); 142 VideoCaptureFramesExample.this.fpsDelayMillis = delay; 143 VideoCaptureFramesExample.this.fpsDelayLabel.setText(VideoCaptureFramesExample.DELAY_LBL + VideoCaptureFramesExample.this.fpsDelayText.getText()); 144 } 145 catch(final NumberFormatException nfe) { 146 VideoCaptureFramesExample.this.fpsDelayText.setText(Integer.toString(VideoCaptureFramesExample.this.fpsDelayMillis)); 147 } 148 } 149 }); 150 this.fpsDelayText = new JTextField(); 151 this.fpsDelayText.setText(Integer.toString(this.fpsDelayMillis)); 152 this.fpsDelayText.setPreferredSize(new Dimension(100, 20)); 153 this.fpsDelayText.setMinimumSize(new Dimension(25, 20)); 154 155 setDelayPanel.add(this.fpsDelayText); 156 setDelayPanel.add(setButton); 157 158 controlPanel.add(this.fpsDelayLabel, BorderLayout.WEST); 159 controlPanel.add(Box.createGlue(), BorderLayout.CENTER); 160 controlPanel.add(setDelayPanel, BorderLayout.EAST); 161 162 win.getContentPane().add(ic, BorderLayout.CENTER); 163 win.getContentPane().add(controlPanel, BorderLayout.SOUTH); 164 win.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); 165 166 win.pack(); 167 168 return win; 169 } 170 171 class FrameDemoImageComponent extends ImageComponent implements VideoDisplayListener<MBFImage> { 172 173 private static final long serialVersionUID = 1618169267341185294L; 174 175 private final float FPS_BOX_ALPHA = 0.45f; 176 177 private final int FONT_SIZE = 10; 178 179 private final int AVG_SAMPLE = 20; 180 181 private long lastFrameTimestamp = 0; 182 183 private double renderedWorstFps = Double.MAX_VALUE; 184 185 private double renderedBestFps = 0d; 186 187 private final double[] renderedSampleFps = new double[this.AVG_SAMPLE]; 188 189 private double capturedWorstFps = Double.MAX_VALUE; 190 191 private double capturedBestFps = 0d; 192 193 private final double[] capturedSampleFps = new double[this.AVG_SAMPLE]; 194 195 private long framesRendered = 0; 196 197 private final DecimalFormat df = new DecimalFormat("#.#######"); 198 199 private double renderedFps; 200 201 private long timedFps; 202 203 204 public FrameDemoImageComponent() { 205 } 206 207 private double getFps() { 208 final long currentFrameTimestamp = System.currentTimeMillis(); 209 final double fps = 1000d/(currentFrameTimestamp - this.lastFrameTimestamp); 210 this.lastFrameTimestamp = currentFrameTimestamp; 211 return fps; 212 } 213 214 private double computeRenderedAvgFps() { 215 double avgFps = 0d; 216 for(final double fps : this.renderedSampleFps) { 217 if(fps > 0d) avgFps += fps; 218 } 219 return avgFps / this.renderedSampleFps.length; 220 } 221 222 private double computeCapturedAvgFps() { 223 double avgFps = 0d; 224 for(final double fps : this.capturedSampleFps) { 225 if(fps > 0d) avgFps += fps; 226 } 227 return avgFps / this.capturedSampleFps.length; 228 } 229 230 private String[] buildFpsMessages() { 231 232 final int currentSampleIndex = (int)(this.framesRendered % this.AVG_SAMPLE); 233 234 final double capturedAvgFps = this.computeCapturedAvgFps(); 235 this.capturedSampleFps[currentSampleIndex] = VideoCaptureFramesExample.this.display.getDisplayFPS(); 236 final double renderedAvgFps = this.computeRenderedAvgFps(); 237 this.renderedSampleFps[currentSampleIndex] = this.renderedFps; 238 239 final double tmpRenderedWorstFps = Math.min(this.renderedSampleFps[currentSampleIndex], this.renderedWorstFps); 240 // we won't report zero as a valid worst fps 241 if(tmpRenderedWorstFps > 7.367059952818556E-10) { 242 this.renderedWorstFps = tmpRenderedWorstFps; 243 } 244 final double tmpRenderedBestFps = Math.max(this.renderedSampleFps[currentSampleIndex], this.renderedBestFps); 245 // consider best within the mans of reasonable deviation 246 //if(tmpRenderedBestFps > renderedBestFps && tmpRenderedBestFps < (renderedAvgFps*2)) { 247 if(tmpRenderedBestFps > this.renderedBestFps) { 248 this.renderedBestFps = tmpRenderedBestFps; 249 } 250 251 final double tmpCapturedWorstFps = Math.min(this.capturedSampleFps[currentSampleIndex], this.capturedWorstFps); 252 // we won't report zero as a valid worst fps 253 if(tmpCapturedWorstFps > 7.367059952818556E-10) { 254 this.capturedWorstFps = tmpCapturedWorstFps; 255 } 256 final double tmpCapturedBestFps = Math.max(this.capturedSampleFps[currentSampleIndex], this.capturedBestFps); 257 // consider best within the mans of reasonable deviation 258 //if(tmpCapturedBestFps > capturedBestFps && tmpCapturedBestFps < (capturedAvgFps*2)) { 259 if(tmpCapturedBestFps > this.capturedBestFps) { 260 this.capturedBestFps = tmpCapturedBestFps; 261 } 262 263 return new String[] { 264 "Rendered Live FPS: " + this.df.format(this.renderedSampleFps[currentSampleIndex]), 265 "Rendered Avg FPS: " + this.df.format(renderedAvgFps), 266 "Rendered Worst FPS: " + this.df.format(this.renderedWorstFps), 267 "Rendered Best FPS: " + this.df.format(this.renderedBestFps), 268 "Captured FSP: " + this.df.format(this.capturedSampleFps[currentSampleIndex]), 269 "Captured Avg FPS: " + this.df.format(capturedAvgFps), 270 "Captured Worst FPS: " + this.df.format(this.capturedWorstFps), 271 "Captured Best FPS: " + this.df.format(this.capturedBestFps) 272 }; 273 } 274 275 private void renderFpsStats(final Graphics g) { 276 // show fps stats 277 if(VideoCaptureFramesExample.this.display != null) { 278 final String[] fps = this.buildFpsMessages(); 279 final int type = AlphaComposite.SRC_OVER; 280 final AlphaComposite composite = AlphaComposite.getInstance(type, this.FPS_BOX_ALPHA); 281 final Graphics2D g2 = (Graphics2D) g.create(); 282 g2.setComposite(composite); 283 g2.setColor(Color.DARK_GRAY); 284 final Font font = new Font("SansSerif", Font.PLAIN, this.FONT_SIZE); 285 final FontMetrics fm = g.getFontMetrics(font); 286 g2.fillRect(0, 0, this.getWidth(), (fm.getHeight()*fps.length)+(this.FONT_SIZE/2)); 287 g2.dispose(); 288 if(this.framesRendered > 0) { 289 g.setFont(font); 290 g.setColor(Color.WHITE); 291 int x = 1; 292 final int xOffset = 3; 293 for(; x<=(fps.length/2); ++x) { 294 g.drawString(fps[x-1], xOffset, fm.getHeight()*x); 295 } 296 g.setColor(Color.YELLOW); 297 for(; x<=fps.length; ++x) { 298 g.drawString(fps[x-1], xOffset, fm.getHeight()*x); 299 } 300 g.setColor(Color.CYAN); 301 final String timedFpsStr = "Timed FPS: "; 302 final int timedFpsStrWidth = (int)(fm.getStringBounds(timedFpsStr + 1000, g).getWidth()); 303 g.drawString(timedFpsStr + this.timedFps, this.getWidth()-timedFpsStrWidth, fm.getHeight()); 304 } 305 else { 306 g.setColor(Color.YELLOW); 307 g.drawString("Initializing ...", 3, fm.getHeight()); 308 } 309 } 310 } 311 312 private boolean slowFrameInProgress = false; 313 private Image slowFrame = null; 314 315 class FrameRenderer extends SwingWorker<Void, Void> { 316 317 private final int width; 318 private final int height; 319 320 public FrameRenderer(final int frameWidth, final int frameHeight) { 321 this.width = frameWidth; 322 this.height = frameHeight; 323 } 324 325 @Override 326 public Void doInBackground() { 327 FrameDemoImageComponent.this.slowFrameInProgress = true; 328 FrameDemoImageComponent.this.slowFrame = FrameDemoImageComponent.this.image.getScaledInstance(this.width, this.height, Image.SCALE_FAST); 329 try { 330 // simulating a really expensive frame rendering operation 331 Thread.sleep(VideoCaptureFramesExample.this.fpsDelayMillis); 332 } catch (final InterruptedException e) { 333 e.printStackTrace(); 334 } 335 return null; 336 } 337 338 @Override 339 protected void done() { 340 FrameDemoImageComponent.this.renderedFps = FrameDemoImageComponent.this.getFps(); 341 FrameDemoImageComponent.this.framesRendered++; 342 FrameDemoImageComponent.this.slowFrameInProgress = false; 343 } 344 } 345 346 @Override 347 public void paint(final Graphics g) { 348 if(VideoCaptureFramesExample.this.display != null) { 349 if(VideoCaptureFramesExample.this.fpsDelayMillis > 0) { 350 if(!this.slowFrameInProgress) { 351 new FrameRenderer(this.getWidth(), this.getHeight()).execute(); 352 } 353 if(this.slowFrame != null) { 354 g.drawImage(this.slowFrame, 0, 0, null); 355 } 356 else { 357 // rather than having blank display that one time when 358 // slow frame scales for the first time, let it render 359 // from base but don't track frame count since this is 360 // just an eye-candy effect 361 super.paint(g); 362 } 363 } 364 else { 365 if(this.framesRendered > 0) { 366 // we are rendering at base (OpenIMAJ) speed. 367 super.paint(g); 368 this.framesRendered++; 369 this.renderedFps = this.getFps(); 370 } 371 else { 372 // container is ready to paint us but cam (display var) 373 // is still initializing 374 VideoCaptureFramesExample.log.debug(null); 375 } 376 } 377 this.renderFpsStats(g); 378 } 379 } 380 381 @Override 382 public void afterUpdate(final VideoDisplay<MBFImage> display) { 383 // tell us when frames begin to appear on display (they likely 384 // will have been already streamed from the device) 385 if(this.framesRendered == 0) { 386 this.framesRendered++; 387 new Timer(1000, this.timerListener).start(); 388 } 389 } 390 391 @Override 392 public void beforeUpdate(final MBFImage frame) { 393 // not needed 394 } 395 396 private final ActionListener timerListener = new ActionListener() { 397 private long lastFrameCount = 0; 398 @Override 399 public void actionPerformed(final ActionEvent e) { 400 final long rendered = FrameDemoImageComponent.this.framesRendered; 401 VideoCaptureFramesExample.log.debug("lastFrameCount: {}, framesRendered: {}", this.lastFrameCount, rendered); 402 FrameDemoImageComponent.this.timedFps = rendered - this.lastFrameCount; 403 this.lastFrameCount = rendered; 404 } 405 }; 406 } 407 408 /** 409 * Runs the program contained in this class. 410 * 411 * @param args not used 412 * @throws VideoCaptureException 413 * if their is a problem with the video capture hardware 414 */ 415 public static void main(final String[] args) throws VideoCaptureException { 416 new VideoCaptureFramesExample(); 417 //double tmp = Math.min(1.2341231234d, Double.MAX_VALUE); 418 //System.out.println(tmp); 419 } 420}