001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.demos.video; 031 032import java.awt.GridBagConstraints; 033import java.awt.GridBagLayout; 034import java.awt.event.KeyEvent; 035import java.awt.event.KeyListener; 036import java.io.IOException; 037import java.util.Arrays; 038import java.util.List; 039 040import javax.swing.BorderFactory; 041import javax.swing.JComponent; 042import javax.swing.JFrame; 043import javax.swing.JPanel; 044import javax.swing.SwingUtilities; 045 046import org.openimaj.demos.Demo; 047import org.openimaj.feature.DoubleFV; 048import org.openimaj.image.DisplayUtilities.ImageComponent; 049import org.openimaj.image.FImage; 050import org.openimaj.image.ImageUtilities; 051import org.openimaj.image.MBFImage; 052import org.openimaj.image.colour.ColourSpace; 053import org.openimaj.image.colour.RGBColour; 054import org.openimaj.image.colour.Transforms; 055import org.openimaj.image.feature.local.engine.DoGSIFTEngine; 056import org.openimaj.image.feature.local.keypoints.Keypoint; 057import org.openimaj.image.pixel.statistics.HistogramModel; 058import org.openimaj.image.processing.resize.ResizeProcessor; 059import org.openimaj.image.renderer.MBFImageRenderer; 060import org.openimaj.io.IOUtils; 061import org.openimaj.math.geometry.point.Point2dImpl; 062import org.openimaj.math.geometry.shape.Rectangle; 063import org.openimaj.ml.clustering.ByteCentroidsResult; 064import org.openimaj.ml.clustering.assignment.hard.ExactByteAssigner; 065import org.openimaj.video.VideoDisplay; 066import org.openimaj.video.VideoDisplayListener; 067import org.openimaj.video.capture.VideoCapture; 068 069/** 070 * Demonstration of different feature extraction techniques that produce a 071 * single global histogram for a given image. Currently this includes RGB and 072 * HSV colour histograms, and a simple SIFT-based Bag of Visual Words. The demo 073 * opens the first webcam and displays a histogram of features. Press the space 074 * bar to toggle between the different feature types. 075 * 076 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk) 077 * 078 * @created 15 Feb 2012 079 */ 080@Demo( 081 author = "Jonathon Hare", 082 description = "Demonstration of different feature extraction techniques " + 083 "that produce a single global histogram for a given image. Currently " + 084 "this includes RGB and HSV colour histograms, and a simple SIFT-based " + 085 "Bag of Visual Words. The demo opens the first webcam and displays a " + 086 "histogram of features. Press the space bar to toggle between the " + 087 "different feature types.", 088 keywords = { "features", "video", "histogram", "sift", "webcam", "bag-of-visual-words" }, 089 title = "Video Feature Extraction") 090public class VideoFeatureExtraction implements VideoDisplayListener<MBFImage>, KeyListener { 091 private final VideoCapture capture; 092 private final VideoDisplay<MBFImage> videoDisplay; 093 private Mode mode = Mode.RGB_HISTOGRAM; 094 private final MBFImage histogramImage; 095 private final ImageComponent modelFrame; 096 private final JComponent modelPanel; 097 098 /** 099 * Default constructor 100 * 101 * @param window 102 * The window to display the demo in 103 * @throws IOException 104 */ 105 public VideoFeatureExtraction(final JComponent window) throws IOException { 106 this.capture = new VideoCapture(640, 480); 107 108 window.setLayout(new GridBagLayout()); 109 110 final JPanel vidPanel = new JPanel(new GridBagLayout()); 111 vidPanel.setBorder(BorderFactory.createTitledBorder("Live Video")); 112 this.videoDisplay = VideoDisplay.createVideoDisplay(this.capture, vidPanel); 113 this.videoDisplay.addVideoListener(this); 114 GridBagConstraints gbc = new GridBagConstraints(); 115 gbc.anchor = GridBagConstraints.PAGE_START; 116 window.add(vidPanel, gbc); 117 118 this.modelPanel = new JPanel(new GridBagLayout()); 119 this.modelPanel.setBorder(BorderFactory.createTitledBorder("Feature type: " + this.mode.toString())); 120 gbc = new GridBagConstraints(); 121 gbc.anchor = GridBagConstraints.PAGE_END; 122 gbc.gridy = 1; 123 window.add(this.modelPanel, gbc); 124 125 this.modelFrame = new ImageComponent(true, false); 126 this.modelPanel.add(this.modelFrame); 127 this.histogramImage = new MBFImage(640, 60, ColourSpace.RGB); 128 this.modelFrame.setImage(ImageUtilities.createBufferedImageForDisplay(this.histogramImage)); 129 130 ((JFrame) SwingUtilities.getRoot(this.videoDisplay.getScreen())).addKeyListener(this); 131 } 132 133 @Override 134 public void afterUpdate(final VideoDisplay<MBFImage> display) { 135 136 } 137 138 @Override 139 public synchronized void beforeUpdate(final MBFImage frame) { 140 final DoubleFV histogram = this.mode.createFeature(frame); 141 142 this.drawHistogramImage(histogram); 143 this.modelFrame.setImage(ImageUtilities.createBufferedImageForDisplay(this.histogramImage)); 144 } 145 146 private void drawHistogramImage(DoubleFV histogram) { 147 histogram = histogram.normaliseFV(); 148 149 final int width = this.histogramImage.getWidth(); 150 final int height = this.histogramImage.getHeight(); 151 152 final int bw = width / histogram.length(); 153 154 this.histogramImage.zero(); 155 final MBFImageRenderer renderer = this.histogramImage.createRenderer(); 156 final Rectangle s = new Rectangle(); 157 s.width = bw; 158 for (int i = 0; i < histogram.values.length; i++) { 159 final int rectHeight = (int) (histogram.values[i] * height); 160 final int remHeight = height - rectHeight; 161 162 s.x = i * bw; 163 s.y = remHeight; 164 s.height = rectHeight; 165 renderer.drawShapeFilled(s, this.mode.colourForBin(i)); 166 } 167 } 168 169 @Override 170 public void keyTyped(final KeyEvent e) { 171 // do nothing 172 } 173 174 @Override 175 public synchronized void keyPressed(final KeyEvent e) { 176 if (e.getKeyChar() == ' ') { 177 int newOrdinal = this.mode.ordinal() + 1; 178 if (newOrdinal >= Mode.values().length) 179 newOrdinal = 0; 180 181 this.mode = Mode.values()[newOrdinal]; 182 183 this.modelPanel.setBorder(BorderFactory.createTitledBorder("Feature type: " + this.mode.toString())); 184 } 185 } 186 187 @Override 188 public void keyReleased(final KeyEvent e) { 189 // do nothing 190 } 191 192 /** 193 * Default main 194 * 195 * @param args 196 * Command-line arguments 197 * @throws IOException 198 */ 199 public static void main(final String[] args) throws IOException { 200 final JFrame window = new JFrame( "Press SPACE to change feature type" ); 201 window.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); 202 203 window.setLayout(new GridBagLayout()); 204 final JPanel c = new JPanel(); 205 c.setLayout(new GridBagLayout()); 206 window.getContentPane().add(c); 207 208 final VideoFeatureExtraction vfe = new VideoFeatureExtraction(c); 209 window.pack(); 210 window.setVisible(true); 211 window.addKeyListener(vfe); 212 } 213} 214 215enum Mode { 216 RGB_HISTOGRAM { 217 HistogramModel model = new HistogramModel(4, 4, 4); 218 Float[][] binCols = null; 219 220 @Override 221 public String toString() { 222 return "64-bin RGB Histogram"; 223 } 224 225 @Override 226 public DoubleFV createFeature(final MBFImage image) { 227 this.model.estimateModel(image); 228 return this.model.histogram; 229 } 230 231 void buildBinCols() { 232 this.binCols = new Float[4 * 4 * 4][3]; 233 for (int k = 0; k < 4; k++) { 234 for (int j = 0; j < 4; j++) { 235 for (int i = 0; i < 4; i++) { 236 this.binCols[k * 4 * 4 + j * 4 + i][0] = (float) i / 4 + (0.5f / 4); 237 this.binCols[k * 4 * 4 + j * 4 + i][1] = (float) j / 4 + (0.5f / 4); 238 this.binCols[k * 4 * 4 + j * 4 + i][2] = (float) k / 4 + (0.5f / 4); 239 } 240 } 241 } 242 } 243 244 @Override 245 public Float[] colourForBin(final int bin) { 246 if (this.binCols == null) 247 this.buildBinCols(); 248 249 return this.binCols[bin]; 250 } 251 }, 252 HSV_HISTOGRAM { 253 HistogramModel model = new HistogramModel(4, 4, 4); 254 Float[][] binCols = null; 255 256 @Override 257 public String toString() { 258 return "64-bin HSV Histogram"; 259 } 260 261 @Override 262 public DoubleFV createFeature(MBFImage image) { 263 image = Transforms.RGB_TO_HSV(image); 264 this.model.estimateModel(image); 265 return this.model.histogram; 266 } 267 268 void buildBinCols() { 269 this.binCols = new Float[4 * 4 * 4][]; 270 for (int k = 0; k < 4; k++) { 271 for (int j = 0; j < 4; j++) { 272 for (int i = 0; i < 4; i++) { 273 final float h = (float) i / 4 + (0.5f / 4); 274 final float s = (float) j / 4 + (0.5f / 4); 275 final float v = (float) k / 4 + (0.5f / 4); 276 277 MBFImage img = new MBFImage(1, 1, ColourSpace.HSV); 278 img.setPixel(0, 0, new Float[] { h, s, v }); 279 280 img = Transforms.HSV_TO_RGB(img); 281 282 this.binCols[k * 4 * 4 + j * 4 + i] = img.getPixel(0, 0); 283 } 284 } 285 } 286 } 287 288 @Override 289 public Float[] colourForBin(final int bin) { 290 if (this.binCols == null) 291 this.buildBinCols(); 292 293 return this.binCols[bin]; 294 } 295 }, 296 SIFT { 297 ExactByteAssigner rabc = null; 298 DoubleFV fv = null; 299 DoGSIFTEngine engine = new DoGSIFTEngine(); 300 301 @Override 302 public String toString() { 303 return "100-term SIFT BoVW"; 304 } 305 306 @Override 307 public DoubleFV createFeature(final MBFImage image) { 308 if (this.rabc == null) { 309 try { 310 final ByteCentroidsResult clusterer = IOUtils.read(Mode.class 311 .getResourceAsStream("/org/openimaj/demos/codebooks/random-100-highfield-codebook.voc"), 312 ByteCentroidsResult.class); 313 314 this.rabc = new ExactByteAssigner(clusterer); 315 this.fv = new DoubleFV(clusterer.numClusters()); 316 this.engine.getOptions().setDoubleInitialImage(false); 317 } catch (final IOException e) { 318 e.printStackTrace(); 319 } 320 } 321 322 FImage img = Transforms.calculateIntensity(image); 323 img = ResizeProcessor.halfSize(img); 324 final List<Keypoint> keys = this.engine.findFeatures(img); 325 326 for (final Keypoint keypoint : keys) { 327 image.drawPoint(new Point2dImpl(keypoint.x * 2f, keypoint.y * 2f), RGBColour.RED, 3); 328 } 329 330 Arrays.fill(this.fv.values, 0); 331 332 for (final Keypoint k : keys) { 333 this.fv.values[this.rabc.assign(k.ivec)]++; 334 } 335 336 return this.fv; 337 } 338 339 @Override 340 public Float[] colourForBin(final int bin) { 341 return RGBColour.RED; 342 } 343 }; 344 public abstract DoubleFV createFeature(MBFImage image); 345 346 public abstract Float[] colourForBin(int bin); 347}