001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.demos.video;
031
032import java.awt.AlphaComposite;
033import java.awt.BorderLayout;
034import java.awt.Color;
035import java.awt.Dimension;
036import java.awt.FlowLayout;
037import java.awt.Font;
038import java.awt.FontMetrics;
039import java.awt.Graphics;
040import java.awt.Graphics2D;
041import java.awt.Image;
042import java.awt.event.ActionEvent;
043import java.awt.event.ActionListener;
044import java.awt.event.KeyAdapter;
045import java.text.DecimalFormat;
046
047import javax.swing.Box;
048import javax.swing.JButton;
049import javax.swing.JFrame;
050import javax.swing.JLabel;
051import javax.swing.JPanel;
052import javax.swing.JTextField;
053import javax.swing.SwingWorker;
054import javax.swing.Timer;
055
056import org.openimaj.image.DisplayUtilities.ImageComponent;
057import org.openimaj.image.MBFImage;
058import org.openimaj.video.VideoDisplay;
059import org.openimaj.video.VideoDisplayListener;
060import org.openimaj.video.capture.VideoCapture;
061import org.openimaj.video.capture.VideoCaptureException;
062import org.slf4j.Logger;
063import org.slf4j.LoggerFactory;
064
065/**
066 * Example showing how the built-in OpenIMAJ fps counter works and how to 
067 * dipsplay real-time fps. Since {@link VideoDisplay#getDisplayFPS()} reports 
068 * FPS speed with which display is updated, it is not necessairly the speed 
069 * at which FPS is rendered out to the display. This program shows this 
070 * discrepancy. Notice that while "capture" FPS reports at the consistent 
071 * range, depending on frame delay set, the actual render FPS may be quite 
072 * different. This demo implementation uses simple thread sleep to simulate 
073 * expensive rendering operation.
074 * 
075 * @author Adam Zimowski (mrazjava)
076 */
077public class VideoCaptureFramesExample extends KeyAdapter {
078        
079        private static final Logger log = LoggerFactory.getLogger(VideoCaptureFramesExample.class);
080
081        VideoCapture vc;
082        VideoDisplay<MBFImage> display;
083        Thread displayThread;
084        
085        private static final String DELAY_LBL = "Delay (ms): ";
086        
087        private int fpsDelayMillis = 0;
088        
089        private final JLabel fpsDelayLabel;
090        
091        private JTextField fpsDelayText;
092
093        /**
094         * @throws VideoCaptureException
095         */
096        public VideoCaptureFramesExample() throws VideoCaptureException {
097                
098                this.fpsDelayLabel = new JLabel(VideoCaptureFramesExample.DELAY_LBL + this.fpsDelayMillis);
099                
100                // open the capture device and create a window to display in 320, 240
101                this.vc = new VideoCapture(1024, 768);
102
103                
104                final FrameDemoImageComponent ic = new FrameDemoImageComponent();
105                ic.setAllowZoom( false );
106                ic.setAllowPanning( false );
107                ic.setTransparencyGrid( false );
108                ic.setShowPixelColours( false );
109                ic.setShowXYPosition( true );           
110
111                this.buildGui(ic).setVisible(true);
112                
113                this.display = new VideoDisplay<MBFImage>( this.vc, null, ic );
114                this.display.addVideoListener(ic);
115                
116                this.displayThread = new Thread(this.display);
117                this.displayThread.start();
118        }
119        
120        /**
121         * @param ic frame displaying component
122         * @return
123         */
124        private JFrame buildGui(final ImageComponent ic) {
125                
126                final JFrame win = new JFrame("Frame Counting Demo");
127                win.setPreferredSize(new Dimension(640, 480));
128                win.getContentPane().setLayout(new BorderLayout());
129                
130                final JPanel controlPanel = new JPanel(new BorderLayout());
131                final JPanel setDelayPanel = new JPanel(new FlowLayout());
132                
133                final JButton setButton = new JButton("Set Delay");
134                setButton.setPreferredSize(new Dimension(150, 20));
135                setButton.setMinimumSize(new Dimension(75, 20));
136                setButton.addActionListener(new ActionListener() {
137                        @Override
138                        public void actionPerformed(final ActionEvent arg0) {
139                                try {
140                                        final int delay = Integer.valueOf(VideoCaptureFramesExample.this.fpsDelayText.getText());
141                                        if(delay < 0) throw new NumberFormatException();
142                                        VideoCaptureFramesExample.this.fpsDelayMillis = delay;
143                                        VideoCaptureFramesExample.this.fpsDelayLabel.setText(VideoCaptureFramesExample.DELAY_LBL + VideoCaptureFramesExample.this.fpsDelayText.getText());
144                                }
145                                catch(final NumberFormatException nfe) {
146                                        VideoCaptureFramesExample.this.fpsDelayText.setText(Integer.toString(VideoCaptureFramesExample.this.fpsDelayMillis));
147                                }
148                        }
149                });
150                this.fpsDelayText = new JTextField();
151                this.fpsDelayText.setText(Integer.toString(this.fpsDelayMillis));
152                this.fpsDelayText.setPreferredSize(new Dimension(100, 20));
153                this.fpsDelayText.setMinimumSize(new Dimension(25, 20));
154                
155                setDelayPanel.add(this.fpsDelayText);
156                setDelayPanel.add(setButton);
157                
158                controlPanel.add(this.fpsDelayLabel, BorderLayout.WEST);
159                controlPanel.add(Box.createGlue(), BorderLayout.CENTER);
160                controlPanel.add(setDelayPanel, BorderLayout.EAST);
161                
162                win.getContentPane().add(ic, BorderLayout.CENTER);
163                win.getContentPane().add(controlPanel, BorderLayout.SOUTH);
164                win.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
165                
166                win.pack();
167
168                return win;
169        }
170        
171        class FrameDemoImageComponent extends ImageComponent implements VideoDisplayListener<MBFImage> {
172
173                private static final long serialVersionUID = 1618169267341185294L;
174
175                private final float FPS_BOX_ALPHA = 0.45f;
176                
177                private final int FONT_SIZE = 10;
178                
179                private final int AVG_SAMPLE = 20;
180                
181                private long lastFrameTimestamp = 0;
182                
183                private double renderedWorstFps = Double.MAX_VALUE;
184                
185                private double renderedBestFps = 0d;
186                
187                private final double[] renderedSampleFps = new double[this.AVG_SAMPLE];
188                
189                private double capturedWorstFps = Double.MAX_VALUE;
190                
191                private double capturedBestFps = 0d;
192                
193                private final double[] capturedSampleFps = new double[this.AVG_SAMPLE];
194                
195                private long framesRendered = 0;
196                
197                private final DecimalFormat df = new DecimalFormat("#.#######");
198                
199                private double renderedFps;
200                
201                private long timedFps;
202                
203                
204                public FrameDemoImageComponent() {
205                }
206                
207                private double getFps() {
208                        final long currentFrameTimestamp = System.currentTimeMillis();
209                        final double fps = 1000d/(currentFrameTimestamp - this.lastFrameTimestamp);
210                        this.lastFrameTimestamp = currentFrameTimestamp;
211                        return fps;
212                }
213                
214                private double computeRenderedAvgFps() {
215                        double avgFps = 0d;
216                        for(final double fps : this.renderedSampleFps) {
217                                if(fps > 0d) avgFps += fps;
218                        }
219                        return avgFps / this.renderedSampleFps.length;
220                }
221                
222                private double computeCapturedAvgFps() {
223                        double avgFps = 0d;
224                        for(final double fps : this.capturedSampleFps) {
225                                if(fps > 0d) avgFps += fps;
226                        }
227                        return avgFps / this.capturedSampleFps.length;                  
228                }
229                
230                private String[] buildFpsMessages() {
231                        
232                        final int currentSampleIndex = (int)(this.framesRendered % this.AVG_SAMPLE);
233                        
234                        final double capturedAvgFps = this.computeCapturedAvgFps();
235                        this.capturedSampleFps[currentSampleIndex] = VideoCaptureFramesExample.this.display.getDisplayFPS();
236                        final double renderedAvgFps = this.computeRenderedAvgFps();
237                        this.renderedSampleFps[currentSampleIndex] = this.renderedFps;
238                        
239                        final double tmpRenderedWorstFps = Math.min(this.renderedSampleFps[currentSampleIndex], this.renderedWorstFps);
240                        // we won't report zero as a valid worst fps
241                        if(tmpRenderedWorstFps > 7.367059952818556E-10) {
242                                this.renderedWorstFps = tmpRenderedWorstFps;
243                        }
244                        final double tmpRenderedBestFps = Math.max(this.renderedSampleFps[currentSampleIndex], this.renderedBestFps);
245                        // consider best within the mans of reasonable deviation
246                        //if(tmpRenderedBestFps > renderedBestFps && tmpRenderedBestFps < (renderedAvgFps*2)) {
247                        if(tmpRenderedBestFps > this.renderedBestFps) {
248                                this.renderedBestFps = tmpRenderedBestFps;
249                        }
250
251                        final double tmpCapturedWorstFps = Math.min(this.capturedSampleFps[currentSampleIndex], this.capturedWorstFps);
252                        // we won't report zero as a valid worst fps
253                        if(tmpCapturedWorstFps > 7.367059952818556E-10) {
254                                this.capturedWorstFps = tmpCapturedWorstFps;
255                        }
256                        final double tmpCapturedBestFps = Math.max(this.capturedSampleFps[currentSampleIndex], this.capturedBestFps);
257                        // consider best within the mans of reasonable deviation
258                        //if(tmpCapturedBestFps > capturedBestFps && tmpCapturedBestFps < (capturedAvgFps*2)) {
259                        if(tmpCapturedBestFps > this.capturedBestFps) {
260                                this.capturedBestFps = tmpCapturedBestFps;
261                        }
262                        
263                        return new String[] {
264                                        "Rendered Live FPS: " + this.df.format(this.renderedSampleFps[currentSampleIndex]),
265                                        "Rendered Avg FPS: " + this.df.format(renderedAvgFps), 
266                                        "Rendered Worst FPS: " + this.df.format(this.renderedWorstFps),
267                                        "Rendered Best FPS: " + this.df.format(this.renderedBestFps),
268                                        "Captured FSP: " + this.df.format(this.capturedSampleFps[currentSampleIndex]), 
269                                        "Captured Avg FPS: " + this.df.format(capturedAvgFps), 
270                                        "Captured Worst FPS: " + this.df.format(this.capturedWorstFps), 
271                                        "Captured Best FPS: " + this.df.format(this.capturedBestFps)
272                        };
273                }
274                
275                private void renderFpsStats(final Graphics g) {
276                        // show fps stats
277                        if(VideoCaptureFramesExample.this.display != null) {
278                                final String[] fps = this.buildFpsMessages();
279                                final int type = AlphaComposite.SRC_OVER; 
280                                final AlphaComposite composite = AlphaComposite.getInstance(type, this.FPS_BOX_ALPHA);
281                                final Graphics2D g2 = (Graphics2D) g.create();
282                            g2.setComposite(composite);
283                                g2.setColor(Color.DARK_GRAY);
284                                final Font font = new Font("SansSerif", Font.PLAIN, this.FONT_SIZE);
285                                final FontMetrics fm = g.getFontMetrics(font);                          
286                                g2.fillRect(0, 0, this.getWidth(), (fm.getHeight()*fps.length)+(this.FONT_SIZE/2));
287                                g2.dispose();
288                                if(this.framesRendered > 0) {
289                                        g.setFont(font);
290                                        g.setColor(Color.WHITE);
291                                        int x = 1;
292                                        final int xOffset = 3;
293                                        for(; x<=(fps.length/2); ++x) {
294                                                g.drawString(fps[x-1], xOffset, fm.getHeight()*x);
295                                        }
296                                        g.setColor(Color.YELLOW);
297                                        for(; x<=fps.length; ++x) {
298                                                g.drawString(fps[x-1], xOffset, fm.getHeight()*x);
299                                        }
300                                        g.setColor(Color.CYAN);
301                                        final String timedFpsStr = "Timed FPS: ";
302                                        final int timedFpsStrWidth = (int)(fm.getStringBounds(timedFpsStr + 1000, g).getWidth());
303                                        g.drawString(timedFpsStr + this.timedFps, this.getWidth()-timedFpsStrWidth, fm.getHeight());
304                                }
305                                else {
306                                        g.setColor(Color.YELLOW);
307                                        g.drawString("Initializing ...", 3, fm.getHeight());
308                                }
309                        }                       
310                }
311                
312                private boolean slowFrameInProgress = false;
313                private Image slowFrame = null;
314                
315                class FrameRenderer extends SwingWorker<Void, Void> {
316                        
317                        private final int width;
318                        private final int height;
319                        
320                        public FrameRenderer(final int frameWidth, final int frameHeight) {
321                                this.width = frameWidth;
322                                this.height = frameHeight;
323                        }
324                        
325                        @Override
326                        public Void doInBackground() {
327                                FrameDemoImageComponent.this.slowFrameInProgress = true;
328                                FrameDemoImageComponent.this.slowFrame = FrameDemoImageComponent.this.image.getScaledInstance(this.width, this.height, Image.SCALE_FAST);
329                                try {
330                                        // simulating a really expensive frame rendering operation
331                                        Thread.sleep(VideoCaptureFramesExample.this.fpsDelayMillis);
332                                } catch (final InterruptedException e) {
333                                        e.printStackTrace();
334                                }
335                                return null;
336                        }
337
338                        @Override
339                        protected void done() {
340                                FrameDemoImageComponent.this.renderedFps = FrameDemoImageComponent.this.getFps();
341                                FrameDemoImageComponent.this.framesRendered++;
342                                FrameDemoImageComponent.this.slowFrameInProgress = false;
343                        }                       
344                }
345
346                @Override
347                public void paint(final Graphics g) {
348                        if(VideoCaptureFramesExample.this.display != null) {
349                                if(VideoCaptureFramesExample.this.fpsDelayMillis > 0) {
350                                        if(!this.slowFrameInProgress) {
351                                                new FrameRenderer(this.getWidth(), this.getHeight()).execute();
352                                        }
353                                        if(this.slowFrame != null) {
354                                                g.drawImage(this.slowFrame, 0, 0, null);
355                                        }
356                                        else {
357                                                // rather than having blank display that one time when 
358                                                // slow frame scales for the first time, let it render 
359                                                // from base but don't track frame count since this is 
360                                                // just an eye-candy effect
361                                                super.paint(g);
362                                        }
363                                }
364                                else {
365                                        if(this.framesRendered > 0) {
366                                                // we are rendering at base (OpenIMAJ) speed.
367                                                super.paint(g);
368                                                this.framesRendered++;
369                                                this.renderedFps = this.getFps();
370                                        }
371                                        else {
372                                                // container is ready to paint us but cam (display var) 
373                                                // is still initializing
374                                                VideoCaptureFramesExample.log.debug(null);
375                                        }
376                                }
377                                this.renderFpsStats(g);
378                        }
379                }
380
381                @Override
382                public void afterUpdate(final VideoDisplay<MBFImage> display) {
383                        // tell us when frames begin to appear on display (they likely 
384                        // will have been already streamed from the device)
385                        if(this.framesRendered == 0) {
386                                this.framesRendered++;
387                                new Timer(1000, this.timerListener).start();
388                        }
389                }
390
391                @Override
392                public void beforeUpdate(final MBFImage frame) {
393                        // not needed
394                }
395                
396                private final ActionListener timerListener = new ActionListener() {
397                        private long lastFrameCount = 0;
398                        @Override
399                        public void actionPerformed(final ActionEvent e) {
400                                final long rendered = FrameDemoImageComponent.this.framesRendered;
401                                VideoCaptureFramesExample.log.debug("lastFrameCount: {}, framesRendered: {}", this.lastFrameCount, rendered);
402                                FrameDemoImageComponent.this.timedFps = rendered - this.lastFrameCount;
403                                this.lastFrameCount = rendered;                 
404                        }
405                };
406        }
407        
408        /**
409         * Runs the program contained in this class.
410         * 
411         * @param args not used
412         * @throws VideoCaptureException
413         *             if their is a problem with the video capture hardware
414         */
415        public static void main(final String[] args) throws VideoCaptureException {
416                new VideoCaptureFramesExample();
417                //double tmp = Math.min(1.2341231234d, Double.MAX_VALUE);
418                //System.out.println(tmp);
419        }
420}