001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.demos.sandbox.vlad; 031 032import gnu.trove.list.array.TLongArrayList; 033 034import java.io.BufferedInputStream; 035import java.io.DataInputStream; 036import java.io.EOFException; 037import java.io.File; 038import java.io.FileInputStream; 039import java.io.IOException; 040import java.io.RandomAccessFile; 041import java.net.URI; 042 043import javax.ws.rs.GET; 044import javax.ws.rs.Path; 045import javax.ws.rs.Produces; 046import javax.ws.rs.QueryParam; 047import javax.ws.rs.core.UriBuilder; 048 049import org.glassfish.grizzly.http.server.HttpServer; 050import org.openimaj.io.IOUtils; 051import org.openimaj.knn.pq.IncrementalFloatADCNearestNeighbours; 052import org.openimaj.util.array.ArrayUtils; 053 054import scala.actors.threadpool.Arrays; 055 056import com.sun.jersey.api.container.grizzly2.GrizzlyServerFactory; 057import com.sun.jersey.api.core.ClassNamesResourceConfig; 058import com.sun.jersey.api.core.ResourceConfig; 059 060@Path("/search") 061public class FlickrServerTest { 062 static IncrementalFloatADCNearestNeighbours nn; 063 static long[] indexes; 064 static long[] offsets; 065 static String imageFormat; 066 static RandomAccessFile featureData; 067 static RandomAccessFile imageData; 068 069 static { 070 try { 071 System.out.println("Loading NN"); 072 073 nn = IOUtils.read(new File("/Volumes/My Book/flickr46m-vlad-pqadcnn.dat"), 074 IncrementalFloatADCNearestNeighbours.class); 075 076 System.out.println("Loading index"); 077 indexes = ((TLongArrayList) IOUtils.readFromFile(new File( 078 "/Volumes/My Book/flickr46m-vlad-pqadcnn-indexes.dat"))).toArray(); 079 080 imageFormat = "http://farm%d.staticflickr.com/%d/%d_%s.jpg"; 081 082 System.out.println("Creating RAFs"); 083 featureData = new RandomAccessFile("/Volumes/My Book/flickr46m-vlad.dat", "r"); 084 imageData = new RandomAccessFile("/Volumes/My Book/flickr46m-id2farm-server-secret.map", "r"); 085 086 System.out.println("Building index"); 087 offsets = buildIndex(); 088 } catch (final Exception e) { 089 e.printStackTrace(); 090 } 091 } 092 093 private float[] getFeature(int i) throws IOException { 094 featureData.seek(i * (Long.SIZE + 128 * Float.SIZE)); 095 System.out.println(featureData.readLong()); 096 097 final float[] data = new float[128]; 098 for (int j = 0; j < 128; j++) 099 data[j] = featureData.readFloat(); 100 101 return data; 102 } 103 104 private static long[] buildIndex() throws IOException { 105 final TLongArrayList aIds = new TLongArrayList(46000000); 106 final TLongArrayList aoffsets = new TLongArrayList(46000000); 107 108 final File input = new File("/Volumes/My Book/flickr46m-id2farm-server-secret.map"); 109 110 final DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(input))); 111 112 try { 113 int offset = 0; 114 for (int i = 0;; i++) { 115 if (i % 1000000 == 0) 116 System.out.println(i); 117 118 final long currentOffset = offset; 119 final long id = dis.readLong(); 120 offset += 8; 121 dis.readInt(); 122 offset += 4; 123 dis.readInt(); 124 offset += 4; 125 126 final int len = dis.readUnsignedShort(); 127 final byte[] tmp = new byte[len]; 128 dis.readFully(tmp); 129 offset += len + 2; 130 131 aIds.add(id); 132 aoffsets.add(currentOffset); 133 } 134 } catch (final EOFException e) { 135 dis.close(); 136 } 137 138 final long[] ids = aIds.toArray(); 139 final long[] offsets = aoffsets.toArray(); 140 141 ArrayUtils.parallelQuicksortAscending(ids, offsets); 142 143 final long[] indexedOffsets = new long[indexes.length]; 144 145 for (int i = 0; i < indexedOffsets.length; i++) { 146 final long indexedId = indexes[i]; 147 148 if (i % 1000000 == 0) 149 System.out.println(i); 150 151 indexedOffsets[i] = offsets[Arrays.binarySearch(ids, indexedId)]; 152 } 153 154 return indexedOffsets; 155 } 156 157 @GET 158 @Produces("text/html") 159 public String search(@QueryParam("i") int i) throws IOException 160 { 161 System.out.println("Searching for " + i); 162 163 final int length = 100; 164 final int[][] argmins = new int[1][length]; 165 final float[][] mins = new float[1][length]; 166 167 nn.searchKNN(new float[][] { getFeature(i) }, length, argmins, mins); 168 169 final StringBuffer sb = new StringBuffer(); 170 sb.append("<html><body>"); 171 for (int j = 0; j < length; j++) { 172 sb.append("<div style='width:100px; float: left;'>"); 173 sb.append("<a href=\"/search?i=" + argmins[0][j] + "\">"); 174 sb.append("<img width=\"100\" src=\"" + getImageId(argmins[0][j]) + "\"/>"); 175 sb.append("</a>"); 176 sb.append("<br />"); 177 sb.append("FID " + indexes[argmins[0][j]]); 178 sb.append("</div>"); 179 } 180 sb.append("</body></html>"); 181 182 return sb.toString(); 183 } 184 185 private String getImageId(int i) throws IOException { 186 final long seekId = indexes[i]; 187 final long offset = offsets[i]; 188 189 imageData.seek(offset); 190 final long id = imageData.readLong(); 191 final int farm = imageData.readInt(); 192 final int server = imageData.readInt(); 193 final String secret = imageData.readUTF(); 194 195 return String.format(imageFormat, farm, server, id, secret); 196 } 197 198 private static URI getBaseURI() { 199 return UriBuilder.fromUri("http://localhost/").port(9998).build(); 200 } 201 202 public static final URI BASE_URI = getBaseURI(); 203 204 protected static HttpServer startServer() throws IOException { 205 System.out.println("Starting grizzly..."); 206 final ResourceConfig rc = new ClassNamesResourceConfig(FlickrServerTest.class); 207 return GrizzlyServerFactory.createHttpServer(BASE_URI, rc); 208 } 209 210 public static void main(String[] args) throws IOException { 211 final HttpServer httpServer = startServer(); 212 System.in.read(); 213 httpServer.stop(); 214 } 215}