001/** 002 * Copyright (c) 2012, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.twitter.finance; 031 032import java.io.BufferedInputStream; 033import java.io.IOException; 034import java.io.InputStream; 035import java.io.PrintWriter; 036import java.io.Reader; 037import java.io.StringReader; 038import java.util.HashMap; 039import java.util.Map; 040import java.util.Map.Entry; 041import java.util.Scanner; 042import java.util.Set; 043 044import org.apache.commons.httpclient.HttpClient; 045import org.apache.commons.httpclient.HttpException; 046import org.apache.commons.httpclient.HttpMethod; 047import org.apache.commons.httpclient.cookie.CookiePolicy; 048import org.apache.commons.httpclient.methods.GetMethod; 049import org.joda.time.DateTime; 050import org.joda.time.format.DateTimeFormat; 051import org.joda.time.format.DateTimeFormatter; 052import org.openimaj.io.CachableASCII; 053import org.openimaj.ml.timeseries.processor.interpolation.LinearInterpolationProcessor; 054import org.openimaj.ml.timeseries.processor.interpolation.util.TimeSpanUtils; 055import org.openimaj.ml.timeseries.series.DoubleTimeSeries; 056 057import com.Ostermiller.util.CSVParser; 058 059/** 060 * A class which doesn't belong here, but I need it so here it lives! 061 * 062 * 063 * @author Sina Samangooei (ss@ecs.soton.ac.uk) 064 * 065 */ 066public class YahooFinanceData implements CachableASCII{ 067 068 private final static String YAHOO_URL = "http://ichart.finance.yahoo.com/table.csv"; 069 private String product; 070 private DateTime start; 071 private DateTime end; 072 private String data; 073 private String[] titles; 074 private Map<String, double[]> datavalues; 075 private int nentries; 076 private boolean loadedFromAPICall = false; 077 078 /** 079 * 080 */ 081 public YahooFinanceData() { 082 } 083 084 /** 085 * Query the yahoo finance api for the product from the start date (inclusive) till the end date (inclusive) 086 * 087 * @param product a stock ticker name e.g. AAPL 088 * @param start the start date 089 * @param end the end date 090 */ 091 public YahooFinanceData(String product, DateTime start, DateTime end){ 092 this.product = product; 093 this.start = start; 094 this.end = end; 095 } 096 097 /** 098 * @param product 099 * @param start 100 * @param end 101 * @param format yoda format 102 */ 103 public YahooFinanceData(String product, String start, String end, String format) { 104 DateTimeFormatter parser= DateTimeFormat.forPattern(format); 105 this.start = parser.parseDateTime(start); 106 this.end = parser.parseDateTime(end); 107 this.product = product; 108 } 109 110 private void prepare() throws IOException{ 111 if(this.data == null){ 112 String uri = buildURI(product, start, end); 113 this.data = doCall(uri); 114 this.loadedFromAPICall = true; 115 readData(); 116 } 117 } 118 119 private void readData() throws IOException { 120 121 StringReader reader = new StringReader(this.data); 122 readData(reader); 123 } 124 125 private void readData(Reader in) throws IOException { 126 CSVParser creader = new CSVParser(in); 127 this.datavalues = new HashMap<String,double[]>(); 128 this.titles = creader.getLine(); 129 for (String title : titles) { 130 this.datavalues.put(title, new double[nentries]); 131 } 132 String[] line = null; 133 DateTimeFormatter parser= DateTimeFormat.forPattern("YYYY-MM-dd"); 134 int entry = nentries - 1; 135 while((line = creader.getLine()) != null){ 136 for (int i = 0; i < titles.length; i++) { 137 String title = titles[i]; 138 if(i == 0){ 139 DateTime dt = parser.parseDateTime(line[i]); 140 this.datavalues.get(title)[entry ] = dt.getMillis(); 141 }else{ 142 143 this.datavalues.get(title)[entry ] = Double.parseDouble(line[i]); 144 } 145 } 146 entry--; 147 } 148 } 149 150 /** 151 * @return obtain the underlying data 152 * @throws IOException 153 */ 154 public String resultsString() throws IOException{ 155 prepare(); 156 return this.data; 157 } 158 159 /** 160 * @return obtain the underlying data 161 * @throws IOException 162 */ 163 public Map<String,double[]> results() throws IOException{ 164 prepare(); 165 return this.datavalues; 166 } 167 168 private String buildURI(String product, DateTime start, DateTime end) { 169 StringBuilder uri = new StringBuilder(); 170 DateTime actualstart = start; 171 uri.append(YAHOO_URL); 172 uri.append("?s=").append(product); 173 uri.append("&a=").append(actualstart.getMonthOfYear()-1); 174 uri.append("&b=").append(actualstart.getDayOfMonth()); 175 uri.append("&c=").append(actualstart.getYear()); 176 uri.append("&d=").append(end.getMonthOfYear()-1); 177 uri.append("&e=").append(end.getDayOfMonth()); 178 uri.append("&f=").append(end.getYear()); 179 uri.append("&g=d"); 180 181 return uri.toString(); 182 } 183 184 private String responseToString(InputStream stream) throws IOException { 185 BufferedInputStream bi = new BufferedInputStream(stream); 186 187 StringBuilder sb = new StringBuilder(); 188 189 byte[] buffer = new byte[1024]; 190 int bytesRead = 0; 191 this.nentries = 0; 192 while ((bytesRead = bi.read(buffer)) != -1) { 193 String s = new String(buffer, 0, bytesRead); 194 for (char b : s.toCharArray()) { 195 if(b == '\n') this.nentries++; 196 } 197 sb.append(s); 198 } 199 this.nentries--; 200 return sb.toString(); 201 } 202 203 private String doCall(String uri) throws IOException { 204 System.out.println("We're calling the uri"); 205 HttpClient httpClient = new HttpClient(); 206 httpClient.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY); 207 HttpMethod getMethod = new GetMethod(uri); 208 209 try { 210 int response = httpClient.executeMethod(getMethod); 211 212 if (response != 200) { 213 throw new IOException("HTTP problem, httpcode: " 214 + response); 215 } 216 217 InputStream stream = getMethod.getResponseBodyAsStream(); 218 String responseText = responseToString(stream); 219 return responseText; 220 221 } catch (HttpException e) { 222 e.printStackTrace(); 223 } catch (IOException e) { 224 e.printStackTrace(); 225 } 226 227 return null; 228 } 229 230 @Override 231 public void readASCII(Scanner in) throws IOException { 232 String[] inputParts = in.nextLine().split(" "); 233 this.product = inputParts[0]; 234 this.start = new DateTime(Long.parseLong(inputParts[1])); 235 this.end = new DateTime(Long.parseLong(inputParts[2])); 236 this.nentries = Integer.parseInt(inputParts[3]); 237 this.data = ""; 238 while(in.hasNextLine()){ 239 String l = in.nextLine(); 240 if(l.length() == 0) continue; 241 this.data += l + "\n"; 242 243 } 244 this.readData(); 245 } 246 247 @Override 248 public String asciiHeader() { 249 return "YAHOO-FINANCE\n"; 250 } 251 252 @Override 253 public void writeASCII(PrintWriter out) throws IOException { 254 this.prepare(); 255 out.printf("%s %s %s %s\n",this.product,start.getMillis(),end.getMillis(),this.nentries); 256 out.println(this.data); 257 } 258 259 /** 260 * @return the timeperiods actually retrieved 261 * @throws IOException 262 */ 263 public long[] timeperiods() throws IOException { 264 prepare(); 265 double[] dates = this.datavalues.get("Date"); 266 long[] times = new long[dates.length]; 267 int i = 0; 268 for (double d : dates) { 269 times[i++] = (long) d; 270 } 271 return times; 272 } 273 274 /** 275 * @param name 276 * @return stocks time series by name 277 * @throws IOException 278 */ 279 public DoubleTimeSeries seriesByName(String name) throws IOException{ 280 prepare(); 281 if(!this.datavalues.containsKey(name))return null; 282 return new DoubleTimeSeries(timeperiods(),this.datavalues.get(name)); 283 } 284 285 /** 286 * @return stocks time series for each name 287 * @throws IOException 288 */ 289 public Map<String,DoubleTimeSeries> seriesMap() throws IOException{ 290 prepare(); 291 Map<String, DoubleTimeSeries> ret = new HashMap<String, DoubleTimeSeries>(); 292 long[] tp = this.timeperiods(); 293 for (Entry<String, double[]> namevalues : this.datavalues.entrySet()) { 294 if(namevalues.getKey().equals("Date"))continue; 295 ret.put(namevalues.getKey(), new DoubleTimeSeries(tp,namevalues.getValue())); 296 } 297 return ret; 298 } 299 300 /** 301 * @param times times to interpolate stocks to 302 * @return stocks time series for each name interpolated to the times 303 * @throws IOException 304 */ 305 public Map<String,DoubleTimeSeries> seriesMapInerp(long[] times) throws IOException{ 306 prepare(); 307 Map<String, DoubleTimeSeries> ret = new HashMap<String, DoubleTimeSeries>(); 308 LinearInterpolationProcessor interp = new LinearInterpolationProcessor(times); 309 long[] tp = this.timeperiods(); 310 for (Entry<String, double[]> namevalues : this.datavalues.entrySet()) { 311 if(namevalues.getKey().equals("Date")) continue; 312 DoubleTimeSeries dt = new DoubleTimeSeries(tp,namevalues.getValue()); 313 interp.process(dt); 314 ret.put(namevalues.getKey(), dt); 315 } 316 return ret; 317 } 318 319 /** 320 * @return all available data for each date 321 */ 322 public Set<String> labels() { 323 return this.datavalues.keySet(); 324 } 325 326 /** 327 * Interpolated finance results from the beggining time till the end in perscribed delta 328 * @param delta 329 * @return a map of stock components to time series 330 * @throws IOException 331 */ 332 public Map<String, DoubleTimeSeries> seriesMapInerp(long delta) throws IOException { 333 long[] financeTimes = this.timeperiods(); 334 long start = financeTimes[0]; 335 long end = financeTimes[financeTimes.length-1]; 336 long[] times = TimeSpanUtils.getTime(start, end, delta); 337 return seriesMapInerp(times); 338 } 339 340 @Override 341 public String identifier() { 342 DateTimeFormatter parser= DateTimeFormat.forPattern("YYYY-MM-dd"); 343 String startDate = this.start.toString(parser); 344 String endDate = this.end.toString(parser); 345 return String.format("%s-%s-%s",this.product,startDate,endDate); 346 } 347 348 @Override 349 public boolean equals(Object obj) { 350 if(!(obj instanceof YahooFinanceData)) return false; 351 YahooFinanceData that = (YahooFinanceData) obj; 352 try { 353 this.prepare(); 354 that.prepare(); 355 } catch (IOException e) { 356 return false; 357 } 358 359 return this.data.equals(that.data); 360 } 361 362 @Override 363 public String toString() { 364 return this.data; 365 } 366 367 /** 368 * @return Whether this data instance was actually loaded from the API or 369 * from a saved instance 370 */ 371 public boolean loadedFromAPI() { 372 return this.loadedFromAPICall; 373 } 374}