001package org.openimaj.demos.sandbox.tldcpp.videotld;
002
003import java.util.List;
004
005import org.openimaj.demos.sandbox.tldcpp.detector.NNClassifier;
006import org.openimaj.demos.sandbox.tldcpp.detector.NormalizedPatch;
007import org.openimaj.demos.sandbox.tldcpp.videotld.TLDMain.Command;
008import org.openimaj.image.DisplayUtilities;
009import org.openimaj.image.FImage;
010import org.openimaj.image.MBFImage;
011import org.openimaj.image.colour.RGBColour;
012import org.openimaj.image.processing.resize.ResizeProcessor;
013import org.openimaj.image.typography.hershey.HersheyFont;
014import org.openimaj.math.geometry.shape.Rectangle;
015import org.openimaj.video.VideoDisplay;
016import org.openimaj.video.VideoDisplay.Mode;
017import org.openimaj.video.VideoDisplayListener;
018import org.openimaj.video.tracking.klt.Feature;
019
020public class TLDVideoListener implements VideoDisplayListener<MBFImage> {
021
022        private TLDMain tldMain;
023        private boolean reuseFrameOnce;
024        private boolean skipProcessingOnce;
025        private int currentFrame;
026
027        public TLDVideoListener(TLDMain tldMain) {
028                this.tldMain = tldMain;
029                this.reuseFrameOnce = false;
030                this.skipProcessingOnce = false;
031                currentFrame = 0;
032        }
033
034        @Override
035        public void beforeUpdate(MBFImage frame) {
036                if (this.tldMain.selector.drawRect(frame)) {
037                        return;
038                }
039                final long tic = System.currentTimeMillis();
040                currentFrame++;
041                FImage grey = null;
042                if (!reuseFrameOnce) {
043                        grey = frame.flatten();
044                }
045
046                if (!skipProcessingOnce) {
047                        tldMain.tld.processImage(grey);
048                } else {
049                        skipProcessingOnce = false;
050                }
051
052                if (tldMain.printResults != null) {
053                        if (tldMain.tld.currBB != null) {
054                                tldMain.resultsFile.printf("%d %.2f %.2f %.2f %.2f %f\n", currentFrame - 1, tldMain.tld.currBB.x,
055                                                tldMain.tld.currBB.y, tldMain.tld.currBB.width, tldMain.tld.currBB.height, tldMain.tld.currConf);
056                        } else {
057                                tldMain.resultsFile.printf("%d NaN NaN NaN NaN NaN\n", currentFrame - 1);
058                        }
059                }
060
061                final float fps = 1 / ((System.currentTimeMillis() - tic) / 1000f);
062
063                final boolean confident = (tldMain.tld.currConf >= tldMain.threshold) ? true : false;
064
065                if (tldMain.showOutput || tldMain.saveDir != null) {
066                        String learningString = "";
067                        if (tldMain.tld.isLearning()) {
068                                learningString = "learning";
069                        }
070
071                        final String string = String.format("#%d,Posterior %.2f; fps: %.2f, #numwindows:%d, %s", currentFrame - 1,
072                                        tldMain.tld.currConf, fps, tldMain.tld.detectorCascade.getNumWindows(), learningString);
073                        final Float[] yellow = RGBColour.YELLOW;
074                        final Float[] blue = RGBColour.BLUE;
075                        final Float[] red = RGBColour.RED;
076
077                        if (tldMain.tld.currBB != null) {
078                                final Float[] rectangleColor = confident ? blue : yellow;
079                                // cvRectangle(img, tld.currBB.tl(), tld.currBB.br(),
080                                // rectangleColor, 8, 8, 0);
081                                frame.drawShape(tldMain.tld.currBB, rectangleColor);
082                                if (tldMain.markerMode && this.tldMain.tld.medianFlowTracker.featuresTrackedToBfromA != null) {
083                                        // Draw the tracked points
084                                        final Feature[] from = this.tldMain.tld.medianFlowTracker.featuresTrackedToBfromA.features;
085                                        final Feature[] to = this.tldMain.tld.medianFlowTracker.featuresTrackedToAviaB.features;
086                                        for (int i = 0; i < from.length; i++) {
087                                                frame.drawLine(from[i], to[i], 3, red);
088                                        }
089                                }
090                        }
091
092                        drawPositivePatches();
093
094                        final HersheyFont font = HersheyFont.ROMAN_SIMPLEX;
095                        frame.drawText(string, 25, 25, font, 12);
096
097                }
098
099                if (reuseFrameOnce) {
100                        reuseFrameOnce = false;
101                }
102
103        }
104
105        private void drawPositivePatches() {
106                final NNClassifier nnClass = tldMain.tld.detectorCascade.getNNClassifier();
107                final List<NormalizedPatch> patches = nnClass.getPositivePatches();
108                final Rectangle inDim = new Rectangle(0, 0, NormalizedPatch.TLD_PATCH_SIZE, NormalizedPatch.TLD_PATCH_SIZE);
109                final int X = 5;
110                final int Y = Math.max(6, (patches.size() / X) + 1);
111                final FImage out = new FImage(50 * X, 50 * Y);
112                out.fill(1f);
113                int i = 0;
114                final Rectangle otRect = new Rectangle(0, 0, 50, 50);
115                for (final NormalizedPatch normalizedPatch : patches) {
116                        otRect.x = (i % X) * 50;
117                        otRect.y = (i / X) * 50;
118                        if ((i / X) >= Y)
119                                break;
120                        ResizeProcessor.zoom(normalizedPatch.normalisedPatch, inDim, out, otRect);
121                        i++;
122                }
123                DisplayUtilities.displayName(out, "patches", true);
124        }
125
126        @Override
127        public void afterUpdate(VideoDisplay<MBFImage> display) {
128                if (this.tldMain.command == Command.NONE) {
129                }
130                else if (this.tldMain.command == Command.ALTERNATING) {
131                }
132                else if (this.tldMain.command == Command.CLEAR) {
133                        this.tldMain.tld.release();
134                }
135                else if (this.tldMain.command == Command.SELECT) {
136                        this.tldMain.disp.setMode(Mode.PAUSE);
137                        this.tldMain.selector.selectBoundingBox();
138                }
139                else if (this.tldMain.command == Command.LEARNING) {
140                        tldMain.tld.learningEnabled = !tldMain.tld.learningEnabled;
141                }
142                tldMain.command = Command.NONE;
143        }
144
145}