001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.demos.sandbox.vlad;
031
032import gnu.trove.list.array.TLongArrayList;
033
034import java.io.BufferedInputStream;
035import java.io.DataInputStream;
036import java.io.EOFException;
037import java.io.File;
038import java.io.FileInputStream;
039import java.io.IOException;
040import java.io.RandomAccessFile;
041import java.net.URI;
042
043import javax.ws.rs.GET;
044import javax.ws.rs.Path;
045import javax.ws.rs.Produces;
046import javax.ws.rs.QueryParam;
047import javax.ws.rs.core.UriBuilder;
048
049import org.glassfish.grizzly.http.server.HttpServer;
050import org.openimaj.io.IOUtils;
051import org.openimaj.knn.pq.IncrementalFloatADCNearestNeighbours;
052import org.openimaj.util.array.ArrayUtils;
053
054import scala.actors.threadpool.Arrays;
055
056import com.sun.jersey.api.container.grizzly2.GrizzlyServerFactory;
057import com.sun.jersey.api.core.ClassNamesResourceConfig;
058import com.sun.jersey.api.core.ResourceConfig;
059
060@Path("/search")
061public class FlickrServerTest {
062        static IncrementalFloatADCNearestNeighbours nn;
063        static long[] indexes;
064        static long[] offsets;
065        static String imageFormat;
066        static RandomAccessFile featureData;
067        static RandomAccessFile imageData;
068
069        static {
070                try {
071                        System.out.println("Loading NN");
072
073                        nn = IOUtils.read(new File("/Volumes/My Book/flickr46m-vlad-pqadcnn.dat"),
074                                        IncrementalFloatADCNearestNeighbours.class);
075
076                        System.out.println("Loading index");
077                        indexes = ((TLongArrayList) IOUtils.readFromFile(new File(
078                                        "/Volumes/My Book/flickr46m-vlad-pqadcnn-indexes.dat"))).toArray();
079
080                        imageFormat = "http://farm%d.staticflickr.com/%d/%d_%s.jpg";
081
082                        System.out.println("Creating RAFs");
083                        featureData = new RandomAccessFile("/Volumes/My Book/flickr46m-vlad.dat", "r");
084                        imageData = new RandomAccessFile("/Volumes/My Book/flickr46m-id2farm-server-secret.map", "r");
085
086                        System.out.println("Building index");
087                        offsets = buildIndex();
088                } catch (final Exception e) {
089                        e.printStackTrace();
090                }
091        }
092
093        private float[] getFeature(int i) throws IOException {
094                featureData.seek(i * (Long.SIZE + 128 * Float.SIZE));
095                System.out.println(featureData.readLong());
096
097                final float[] data = new float[128];
098                for (int j = 0; j < 128; j++)
099                        data[j] = featureData.readFloat();
100
101                return data;
102        }
103
104        private static long[] buildIndex() throws IOException {
105                final TLongArrayList aIds = new TLongArrayList(46000000);
106                final TLongArrayList aoffsets = new TLongArrayList(46000000);
107
108                final File input = new File("/Volumes/My Book/flickr46m-id2farm-server-secret.map");
109
110                final DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(input)));
111
112                try {
113                        int offset = 0;
114                        for (int i = 0;; i++) {
115                                if (i % 1000000 == 0)
116                                        System.out.println(i);
117
118                                final long currentOffset = offset;
119                                final long id = dis.readLong();
120                                offset += 8;
121                                dis.readInt();
122                                offset += 4;
123                                dis.readInt();
124                                offset += 4;
125
126                                final int len = dis.readUnsignedShort();
127                                final byte[] tmp = new byte[len];
128                                dis.readFully(tmp);
129                                offset += len + 2;
130
131                                aIds.add(id);
132                                aoffsets.add(currentOffset);
133                        }
134                } catch (final EOFException e) {
135                        dis.close();
136                }
137
138                final long[] ids = aIds.toArray();
139                final long[] offsets = aoffsets.toArray();
140
141                ArrayUtils.parallelQuicksortAscending(ids, offsets);
142
143                final long[] indexedOffsets = new long[indexes.length];
144
145                for (int i = 0; i < indexedOffsets.length; i++) {
146                        final long indexedId = indexes[i];
147
148                        if (i % 1000000 == 0)
149                                System.out.println(i);
150
151                        indexedOffsets[i] = offsets[Arrays.binarySearch(ids, indexedId)];
152                }
153
154                return indexedOffsets;
155        }
156
157        @GET
158        @Produces("text/html")
159        public String search(@QueryParam("i") int i) throws IOException
160        {
161                System.out.println("Searching for " + i);
162
163                final int length = 100;
164                final int[][] argmins = new int[1][length];
165                final float[][] mins = new float[1][length];
166
167                nn.searchKNN(new float[][] { getFeature(i) }, length, argmins, mins);
168
169                final StringBuffer sb = new StringBuffer();
170                sb.append("<html><body>");
171                for (int j = 0; j < length; j++) {
172                        sb.append("<div style='width:100px; float: left;'>");
173                        sb.append("<a href=\"/search?i=" + argmins[0][j] + "\">");
174                        sb.append("<img width=\"100\" src=\"" + getImageId(argmins[0][j]) + "\"/>");
175                        sb.append("</a>");
176                        sb.append("<br />");
177                        sb.append("FID " + indexes[argmins[0][j]]);
178                        sb.append("</div>");
179                }
180                sb.append("</body></html>");
181
182                return sb.toString();
183        }
184
185        private String getImageId(int i) throws IOException {
186                final long seekId = indexes[i];
187                final long offset = offsets[i];
188
189                imageData.seek(offset);
190                final long id = imageData.readLong();
191                final int farm = imageData.readInt();
192                final int server = imageData.readInt();
193                final String secret = imageData.readUTF();
194
195                return String.format(imageFormat, farm, server, id, secret);
196        }
197
198        private static URI getBaseURI() {
199                return UriBuilder.fromUri("http://localhost/").port(9998).build();
200        }
201
202        public static final URI BASE_URI = getBaseURI();
203
204        protected static HttpServer startServer() throws IOException {
205                System.out.println("Starting grizzly...");
206                final ResourceConfig rc = new ClassNamesResourceConfig(FlickrServerTest.class);
207                return GrizzlyServerFactory.createHttpServer(BASE_URI, rc);
208        }
209
210        public static void main(String[] args) throws IOException {
211                final HttpServer httpServer = startServer();
212                System.in.read();
213                httpServer.stop();
214        }
215}