001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.image.feature.local.keypoints;
031
032import java.util.ArrayList;
033import java.util.HashMap;
034import java.util.List;
035import java.util.Map;
036
037import org.openimaj.image.FImage;
038import org.openimaj.image.Image;
039import org.openimaj.image.processing.convolution.FGaussianConvolve;
040import org.openimaj.image.processing.resize.ResizeProcessor;
041import org.openimaj.image.processor.SinglebandImageProcessor;
042import org.openimaj.image.renderer.ImageRenderer;
043import org.openimaj.math.geometry.point.Point2d;
044import org.openimaj.math.geometry.point.Point2dImpl;
045import org.openimaj.math.geometry.shape.Circle;
046import org.openimaj.math.geometry.shape.Polygon;
047
048/**
049 * Helpers for visualising (SIFT) interest points.
050 * 
051 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
052 * 
053 * @param <T>
054 * @param <Q>
055 */
056public class KeypointVisualizer<T, Q extends Image<T, Q> & SinglebandImageProcessor.Processable<Float, FImage, Q>> {
057        Q image;
058        List<? extends Keypoint> keypoints;
059
060        /**
061         * Construct the visualiser with the given image and keypoints.
062         * 
063         * @param image
064         *            the image
065         * @param keys
066         *            the keypoints
067         */
068        public KeypointVisualizer(Q image, List<? extends Keypoint> keys) {
069                this.image = image;
070                this.keypoints = keys;
071        }
072
073        /**
074         * Extract the oriented sampling patches used in the construction of the
075         * keypoints. The patches are normalised to the given fixed size.
076         * 
077         * @param dim
078         *            the patch size.
079         * @return images depicting the sampling patches of each feature
080         */
081        public Map<Keypoint, Q> getPatches(int dim) {
082                final Map<Keypoint, Q> patches = new HashMap<Keypoint, Q>();
083                final Map<Float, Q> blurred = new HashMap<Float, Q>();
084
085                for (final Keypoint k : keypoints) {
086                        // blur image
087                        if (!blurred.containsKey(k.scale)) {
088                                blurred.put(k.scale, image.process(new FGaussianConvolve(k.scale)));
089                        }
090                        final Q blur = blurred.get(k.scale);
091
092                        // make empty patch
093                        final int sz = (int) (2 * 2 * 3 * k.scale);
094                        final Q patch = image.newInstance(sz, sz);
095
096                        // extract pixels
097                        for (int y = 0; y < sz; y++) {
098                                for (int x = 0; x < sz; x++) {
099                                        final double xbar = x - sz / 2.0;
100                                        final double ybar = y - sz / 2.0;
101
102                                        final double xx = (xbar * Math.cos(-k.ori) + ybar * Math.sin(-k.ori)) + k.x;
103                                        final double yy = (-xbar * Math.sin(-k.ori) + ybar * Math.cos(-k.ori)) + k.y;
104
105                                        patch.setPixel(x, y, blur.getPixelInterp(xx, yy));
106                                }
107                        }
108
109                        patches.put(k, patch.processInplace(new ResizeProcessor(dim, dim)));
110                }
111
112                return patches;
113        }
114
115        /**
116         * Draw the sampling boxes on an image.
117         * 
118         * @param boxColour
119         *            the sampling box colour
120         * @param circleColour
121         *            the scale-circle colour
122         * @return an image showing the features.
123         */
124        public Q drawPatches(T boxColour, T circleColour) {
125                return drawPatchesInplace(image.clone(), keypoints, boxColour, circleColour);
126        }
127
128        /**
129         * Draw the SIFT features onto an image. The features are visualised as
130         * circles with orientation lines showing the scale and orientation, and
131         * with oriented squares showing the sampling region.
132         * 
133         * The colours of the squares and circles is controlled individually.
134         * Setting the colour to null will cause the shape not to be displayed.
135         * 
136         * The sizes of the drawn shapes assume the default SIFT settings described
137         * by Lowe. If the parameters used to find the keypoints have been changed,
138         * then the features might not be drawn at the correct size.
139         * 
140         * @param <T>
141         *            the pixel type
142         * @param <Q>
143         *            the image type
144         * @param image
145         *            the image to draw on
146         * @param keypoints
147         *            the features to draw
148         * @param boxColour
149         *            the colour of the sampling boxes
150         * @param circleColour
151         *            the colour of the scale circle
152         * @return the input image
153         */
154        public static <T, Q extends Image<T, Q> & SinglebandImageProcessor.Processable<Float, FImage, Q>>
155                        Q
156                        drawPatchesInplace(Q image, List<? extends Keypoint> keypoints, T boxColour, T circleColour)
157        {
158                final ImageRenderer<T, Q> renderer = image.createRenderer();
159
160                for (final Keypoint k : keypoints) {
161                        if (boxColour != null) {
162                                renderer.drawPolygon(getSamplingBox(k), boxColour);
163                        }
164
165                        if (circleColour != null) {
166                                renderer.drawLine((int) k.x, (int) k.y, -k.ori, (int) k.scale * 5, circleColour);
167                                renderer.drawShape(new Circle(k.x, k.y, k.scale), circleColour);
168                        }
169                }
170
171                return image;
172        }
173
174        /**
175         * Draw the centre point of the keypoints on an image
176         * 
177         * @param col
178         *            the colour to draw with
179         * @return the image
180         */
181        public Q drawCenter(T col) {
182                final Q output = image.clone();
183                final ImageRenderer<T, Q> renderer = output.createRenderer();
184
185                renderer.drawPoints(keypoints, col, 2);
186                return output;
187        }
188
189        /**
190         * Get the sampling area of an single feature as a polygon.
191         * 
192         * @param k
193         *            the feature
194         * @return the polygon representing the sampling area.
195         */
196        public static Polygon getSamplingBox(Keypoint k) {
197                return getSamplingBox(k, 0);
198        }
199
200        /**
201         * Get the sampling area of an single feature as a polygon.
202         * 
203         * @param k
204         *            the feature
205         * @param scincr
206         *            the scaling factor to apply to the sampling area
207         * @return the polygon representing the sampling area.
208         */
209        public static Polygon getSamplingBox(Keypoint k, float scincr) {
210                final List<Point2d> vertices = new ArrayList<Point2d>();
211
212                vertices.add(new Point2dImpl(k.x - (scincr + 2 * 3 * k.scale), k.y - (scincr + 2 * 3 * k.scale)));
213                vertices.add(new Point2dImpl(k.x + (scincr + 2 * 3 * k.scale), k.y - (scincr + 2 * 3 * k.scale)));
214                vertices.add(new Point2dImpl(k.x + (scincr + 2 * 3 * k.scale), k.y + (scincr + 2 * 3 * k.scale)));
215                vertices.add(new Point2dImpl(k.x - (scincr + 2 * 3 * k.scale), k.y + (scincr + 2 * 3 * k.scale)));
216
217                final Polygon poly = new Polygon(vertices);
218
219                poly.rotate(new Point2dImpl(k.x, k.y), -k.ori);
220
221                return poly;
222        }
223}