001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030/** 031 * 032 */ 033package org.openimaj.demos.sandbox.video; 034 035import java.io.File; 036import java.util.HashMap; 037import java.util.List; 038 039import org.openimaj.audio.analysis.EffectiveSoundPressure; 040import org.openimaj.image.DisplayUtilities; 041import org.openimaj.image.MBFImage; 042import org.openimaj.image.colour.RGBColour; 043import org.openimaj.image.processing.face.tracking.clm.CLMFaceTracker; 044import org.openimaj.image.processing.face.tracking.clm.MultiTracker.TrackedFace; 045import org.openimaj.video.VideoDisplay; 046import org.openimaj.video.VideoDisplay.EndAction; 047import org.openimaj.video.VideoDisplayListener; 048import org.openimaj.video.processing.shotdetector.HistogramVideoShotDetector; 049import org.openimaj.video.xuggle.XuggleAudio; 050import org.openimaj.video.xuggle.XuggleVideo; 051 052/** 053 * 054 * 055 * @author David Dupplaw (dpd@ecs.soton.ac.uk) 056 * @created 23 Jul 2012 057 * @version $Author$, $Revision$, $Date$ 058 */ 059public class FaceModelAcousticMatch 060{ 061 /** Number of frames average to take sound pressure over */ 062 private final int nFrameAvg = 1; 063 064 /** Colours to draw each of the detected faces in */ 065 private final Float[][] colours = new Float[][]{ 066 RGBColour.YELLOW, 067 RGBColour.RED, 068 RGBColour.MAGENTA, 069 RGBColour.CYAN 070 }; 071 072 /** 073 * 074 * @param filename 075 */ 076 public FaceModelAcousticMatch( final String filename ) 077 { 078 // Create a video object 079 final XuggleVideo v = new XuggleVideo( new File( filename ) ); 080 final double fps = v.getFPS(); 081 082 // Create an audio object 083 final XuggleAudio a = new XuggleAudio( new File( filename ) ); 084 085 // Create an audio processor where the frames will be the same length 086 // (in time) as the video frames. 087 final EffectiveSoundPressure esp = new EffectiveSoundPressure( a, 088 this.nFrameAvg * (int)(1000d/fps), 0 ); 089 090 // The face tracker 091 final CLMFaceTracker faceTracker = new CLMFaceTracker(); 092 093 // The shot detector used to control the face tracker 094 final HistogramVideoShotDetector shotDetector = new HistogramVideoShotDetector( fps ); 095 096 // An image to display the vis. 097 final MBFImage visImage = new MBFImage( 800, 400, 3 ); 098 099 // Create a video display to show the video 100 final VideoDisplay<MBFImage> vd = VideoDisplay.createVideoDisplay( v ); 101 vd.setEndAction( EndAction.STOP_AT_END ); 102 vd.addVideoListener( new VideoDisplayListener<MBFImage>() 103 { 104 private double lastPressure = 0; 105 private int counter = 0; 106 private final HashMap<TrackedFace,Integer> lastPos = 107 new HashMap<TrackedFace, Integer>(); 108 109 @Override 110 public void beforeUpdate( final MBFImage frame ) 111 { 112 // Check whether a shot boundary was found... 113 shotDetector.processFrame( frame ); 114 115 // ...if so, reset the face tracker 116 if( shotDetector.wasLastFrameBoundary() ) 117 { 118 faceTracker.reset(); 119 this.lastPos.clear(); 120 } 121 122 // Track the face 123 faceTracker.track( frame ); 124 final List<TrackedFace> trackedFaces = faceTracker.getModelTracker().trackedFaces; 125 126 // Draw any detected faces onto the image 127 for( int i = 0; i < trackedFaces.size(); i++ ) 128 { 129 final TrackedFace f = trackedFaces.get(i); 130 CLMFaceTracker.drawFaceModel( frame, f, true, true, 131 true, true, true, faceTracker.getReferenceTriangles(), 132 faceTracker.getReferenceConnections(), 1f, 133 FaceModelAcousticMatch.this.colours[i], RGBColour.BLACK, RGBColour.WHITE, 134 RGBColour.RED ); 135 } 136 137 // We aren't actually interested in the samples here, but this 138 // will force the calculation of the pressure for the next 139 // sample chunk - and the sample chunks should be the length 140 // of one frame of video which is why we can do it once per 141 // frame. 142 double pressure = this.lastPressure; 143 if( this.counter % FaceModelAcousticMatch.this.nFrameAvg == 0 ) 144 { 145 esp.nextSampleChunk(); 146 pressure = esp.getEffectiveSoundPressure() / 20d; 147 } 148 149 // Move the vis along 150 visImage.shiftLeftInplace(); 151 152 // Draw the audio pressure 153 visImage.drawLine( visImage.getWidth()-2, (int)(visImage.getHeight() - this.lastPressure), 154 visImage.getWidth()-1, (int)(visImage.getHeight() - pressure), 155 RGBColour.GREEN ); 156 157 // Check the model shape at this frame 158 double modelShape = 0; 159 160 if( trackedFaces.size() > 0 ) 161 { 162 double lastMs = 0; 163 for( int i = 0; i < trackedFaces.size(); i++ ) 164 { 165 // Get the face model 166 final TrackedFace f = trackedFaces.get(i); 167 modelShape = f.clm._plocal.get(0,0); 168 169 // Last position to draw from 170 Integer lp = this.lastPos.get(f); 171 if( lp == null ) 172 lp = 0; 173 174 // Mouth speed (dy/dx of model shape) 175 final int diff = (int)(Math.abs( modelShape-lastMs ) * 10d); 176 177 // Draw the model shape 178 visImage.drawLine( visImage.getWidth()-2, 179 (visImage.getHeight() - lp), 180 visImage.getWidth()-1, 181 (visImage.getHeight() - diff), 182 FaceModelAcousticMatch.this.colours[i%FaceModelAcousticMatch.this.colours.length] ); 183 184 this.lastPos.put( f, diff ); 185 lastMs = modelShape; 186 } 187 } 188 189 // Show the window 190 DisplayUtilities.displayName( visImage, "Pressure vs Shape", true ); 191 192 // Remember where we were last drawing (so we can draw lines) 193 this.lastPressure = pressure; 194 this.counter++; 195 } 196 197 @Override 198 public void afterUpdate( final VideoDisplay<MBFImage> display ) 199 { 200 } 201 } ); 202 } 203 204 /** 205 * Main 206 * @param args CLAs 207 */ 208 public static void main( final String[] args ) 209 { 210 new FaceModelAcousticMatch( "heads1.mpeg" ); 211 } 212}