001package org.openimaj.demos.sandbox.tldcpp.videotld; 002 003import java.util.List; 004 005import org.openimaj.demos.sandbox.tldcpp.detector.NNClassifier; 006import org.openimaj.demos.sandbox.tldcpp.detector.NormalizedPatch; 007import org.openimaj.demos.sandbox.tldcpp.videotld.TLDMain.Command; 008import org.openimaj.image.DisplayUtilities; 009import org.openimaj.image.FImage; 010import org.openimaj.image.MBFImage; 011import org.openimaj.image.colour.RGBColour; 012import org.openimaj.image.processing.resize.ResizeProcessor; 013import org.openimaj.image.typography.hershey.HersheyFont; 014import org.openimaj.math.geometry.shape.Rectangle; 015import org.openimaj.video.VideoDisplay; 016import org.openimaj.video.VideoDisplay.Mode; 017import org.openimaj.video.VideoDisplayListener; 018import org.openimaj.video.tracking.klt.Feature; 019 020public class TLDVideoListener implements VideoDisplayListener<MBFImage> { 021 022 private TLDMain tldMain; 023 private boolean reuseFrameOnce; 024 private boolean skipProcessingOnce; 025 private int currentFrame; 026 027 public TLDVideoListener(TLDMain tldMain) { 028 this.tldMain = tldMain; 029 this.reuseFrameOnce = false; 030 this.skipProcessingOnce = false; 031 currentFrame = 0; 032 } 033 034 @Override 035 public void beforeUpdate(MBFImage frame) { 036 if (this.tldMain.selector.drawRect(frame)) { 037 return; 038 } 039 final long tic = System.currentTimeMillis(); 040 currentFrame++; 041 FImage grey = null; 042 if (!reuseFrameOnce) { 043 grey = frame.flatten(); 044 } 045 046 if (!skipProcessingOnce) { 047 tldMain.tld.processImage(grey); 048 } else { 049 skipProcessingOnce = false; 050 } 051 052 if (tldMain.printResults != null) { 053 if (tldMain.tld.currBB != null) { 054 tldMain.resultsFile.printf("%d %.2f %.2f %.2f %.2f %f\n", currentFrame - 1, tldMain.tld.currBB.x, 055 tldMain.tld.currBB.y, tldMain.tld.currBB.width, tldMain.tld.currBB.height, tldMain.tld.currConf); 056 } else { 057 tldMain.resultsFile.printf("%d NaN NaN NaN NaN NaN\n", currentFrame - 1); 058 } 059 } 060 061 final float fps = 1 / ((System.currentTimeMillis() - tic) / 1000f); 062 063 final boolean confident = (tldMain.tld.currConf >= tldMain.threshold) ? true : false; 064 065 if (tldMain.showOutput || tldMain.saveDir != null) { 066 String learningString = ""; 067 if (tldMain.tld.isLearning()) { 068 learningString = "learning"; 069 } 070 071 final String string = String.format("#%d,Posterior %.2f; fps: %.2f, #numwindows:%d, %s", currentFrame - 1, 072 tldMain.tld.currConf, fps, tldMain.tld.detectorCascade.getNumWindows(), learningString); 073 final Float[] yellow = RGBColour.YELLOW; 074 final Float[] blue = RGBColour.BLUE; 075 final Float[] red = RGBColour.RED; 076 077 if (tldMain.tld.currBB != null) { 078 final Float[] rectangleColor = confident ? blue : yellow; 079 // cvRectangle(img, tld.currBB.tl(), tld.currBB.br(), 080 // rectangleColor, 8, 8, 0); 081 frame.drawShape(tldMain.tld.currBB, rectangleColor); 082 if (tldMain.markerMode && this.tldMain.tld.medianFlowTracker.featuresTrackedToBfromA != null) { 083 // Draw the tracked points 084 final Feature[] from = this.tldMain.tld.medianFlowTracker.featuresTrackedToBfromA.features; 085 final Feature[] to = this.tldMain.tld.medianFlowTracker.featuresTrackedToAviaB.features; 086 for (int i = 0; i < from.length; i++) { 087 frame.drawLine(from[i], to[i], 3, red); 088 } 089 } 090 } 091 092 drawPositivePatches(); 093 094 final HersheyFont font = HersheyFont.ROMAN_SIMPLEX; 095 frame.drawText(string, 25, 25, font, 12); 096 097 } 098 099 if (reuseFrameOnce) { 100 reuseFrameOnce = false; 101 } 102 103 } 104 105 private void drawPositivePatches() { 106 final NNClassifier nnClass = tldMain.tld.detectorCascade.getNNClassifier(); 107 final List<NormalizedPatch> patches = nnClass.getPositivePatches(); 108 final Rectangle inDim = new Rectangle(0, 0, NormalizedPatch.TLD_PATCH_SIZE, NormalizedPatch.TLD_PATCH_SIZE); 109 final int X = 5; 110 final int Y = Math.max(6, (patches.size() / X) + 1); 111 final FImage out = new FImage(50 * X, 50 * Y); 112 out.fill(1f); 113 int i = 0; 114 final Rectangle otRect = new Rectangle(0, 0, 50, 50); 115 for (final NormalizedPatch normalizedPatch : patches) { 116 otRect.x = (i % X) * 50; 117 otRect.y = (i / X) * 50; 118 if ((i / X) >= Y) 119 break; 120 ResizeProcessor.zoom(normalizedPatch.normalisedPatch, inDim, out, otRect); 121 i++; 122 } 123 DisplayUtilities.displayName(out, "patches", true); 124 } 125 126 @Override 127 public void afterUpdate(VideoDisplay<MBFImage> display) { 128 if (this.tldMain.command == Command.NONE) { 129 } 130 else if (this.tldMain.command == Command.ALTERNATING) { 131 } 132 else if (this.tldMain.command == Command.CLEAR) { 133 this.tldMain.tld.release(); 134 } 135 else if (this.tldMain.command == Command.SELECT) { 136 this.tldMain.disp.setMode(Mode.PAUSE); 137 this.tldMain.selector.selectBoundingBox(); 138 } 139 else if (this.tldMain.command == Command.LEARNING) { 140 tldMain.tld.learningEnabled = !tldMain.tld.learningEnabled; 141 } 142 tldMain.command = Command.NONE; 143 } 144 145}