001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.demos.sandbox.ml.linear.learner.stream; 031 032import java.io.ByteArrayInputStream; 033import java.io.IOException; 034import java.io.UnsupportedEncodingException; 035import java.net.MalformedURLException; 036import java.net.URL; 037import java.net.URLEncoder; 038import java.util.ArrayList; 039import java.util.HashMap; 040import java.util.HashSet; 041import java.util.List; 042import java.util.Map; 043import java.util.Set; 044 045import org.apache.commons.httpclient.ConnectTimeoutException; 046import org.apache.commons.io.IOUtils; 047import org.apache.commons.lang.StringUtils; 048import org.apache.http.HttpEntity; 049import org.apache.log4j.Logger; 050import org.jboss.netty.handler.timeout.ReadTimeoutException; 051import org.openimaj.io.HttpUtils; 052import org.openimaj.io.HttpUtils.MetaRefreshRedirectStrategy; 053import org.openimaj.util.function.Function; 054import org.openimaj.util.function.Operation; 055import org.openimaj.util.function.Predicate; 056import org.openimaj.util.pair.IndependentPair; 057import org.openimaj.util.stream.AbstractStream; 058 059import com.Ostermiller.util.CSVParser; 060import com.google.gson.Gson; 061 062/** 063 * @author Sina Samangooei (ss@ecs.soton.ac.uk) 064 * 065 */ 066public class YahooFinanceStream extends AbstractStream<Map<String,Double>>{ 067 Logger logger = Logger.getLogger(YahooFinanceStream.class); 068 private static final String YAHOO_URI = "http://finance.yahoo.com/d/quotes.csv?s=%s&f=snl1"; 069 private static final String YAHOO_SUGGEST_URI = "http://d.yimg.com/autoc.finance.yahoo.com/autoc?query=%s&callback=YAHOO.Finance.SymbolSuggest.ssCallback"; 070 private static final int CONNECT_TIMEOUT = 1000; 071 private static final int READ_TIMEOUT = 2000; 072 private static final int ENFORCED_WAIT = 1000; 073 private URL yahooURL; 074 private long lastRead; 075 private boolean expandTickers; 076 077 private class FeedItem { 078 String name; 079 String longname; 080 double value; 081 } 082 083 /** 084 * The tickers you want 085 * @param expand whether an attempt should be made to expand the tickers using #querySymbols 086 * @param tickers 087 * @throws IOException 088 * @throws MalformedURLException 089 */ 090 public YahooFinanceStream(boolean expand, String ... tickers) throws MalformedURLException, IOException { 091 if(expand) tickers = expandTickers(tickers); 092 constructYahooURI(tickers); 093 this.lastRead = 0; 094 } 095 096 private String[] expandTickers(String[] tickers) { 097 Set<String> expanded = new HashSet<String>(); 098 for (String ticker : tickers) { 099 try { 100 List<Map<String, String>> queried = querySymbols(ticker); 101 for (Map<String, String> query : queried) { 102 if(query.containsKey("symbol")) expanded.add(query.get("symbol")); 103 } 104 } catch (IOException e) { 105 // Just add the ticker back 106 expanded.add(ticker); 107 } 108 } 109 return expanded.toArray(new String[expanded.size()]); 110 } 111 112 /** 113 * The tickers you want 114 * @param tickers 115 * @throws IOException 116 * @throws MalformedURLException 117 */ 118 public YahooFinanceStream(String ... tickers) throws MalformedURLException, IOException { 119 constructYahooURI(tickers); 120 this.lastRead = 0; 121 } 122 123 private void constructYahooURI(String[] tickers) throws MalformedURLException { 124 for (int i = 0; i < tickers.length; i++) { 125 try { 126 tickers[i] = URLEncoder.encode(tickers[i], "UTF-8"); 127 } catch (UnsupportedEncodingException e) { 128 // TODO Auto-generated catch block 129 e.printStackTrace(); 130 } 131 } 132 String feeds = StringUtils.join(tickers, "+"); 133 String urlString = String.format(YAHOO_URI,feeds); 134 this.yahooURL = new URL(urlString); 135 136 } 137 @Override 138 public boolean hasNext() { 139 return true; 140 } 141 142 @Override 143 public Map<String,Double> next() { 144 List<FeedItem> items = readItems(); 145 146 Map<String, Double> ret = new HashMap<String, Double>(); 147 for (FeedItem feedItem : items) { 148 ret.put(feedItem.name,feedItem.value); 149 } 150 return ret ; 151 } 152 private List<FeedItem> readItems() { 153 CSVParser parser; 154 List<FeedItem> ret = new ArrayList<YahooFinanceStream.FeedItem>(); 155 if(System.currentTimeMillis() - this.lastRead < ENFORCED_WAIT){ 156 logger.debug(String.format("Haven't waited %d, waiting %d, sending old results",ENFORCED_WAIT,System.currentTimeMillis() - this.lastRead)); 157 try { 158 Thread.sleep(ENFORCED_WAIT - (System.currentTimeMillis() - this.lastRead)); 159 } catch (InterruptedException e) { 160 } 161 return readItems(); 162 } 163 try { 164 IndependentPair<HttpEntity, ByteArrayInputStream> readURL = HttpUtils.readURLAsByteArrayInputStream(this.yahooURL, CONNECT_TIMEOUT, READ_TIMEOUT, new MetaRefreshRedirectStrategy() ,HttpUtils.DEFAULT_USERAGENT); 165 parser = new CSVParser(readURL.secondObject()); 166 String[][] vals = parser.getAllValues(); 167 for (String[] strings : vals) { 168 FeedItem feedItem = new FeedItem(); 169 feedItem.name = strings[0]; 170 feedItem.longname = strings[1]; 171 feedItem.value = Double.parseDouble(strings[2]); 172 ret.add(feedItem); 173 } 174 logger.debug(String.format("Read succesfully!")); 175 this.lastRead = System.currentTimeMillis(); 176 } 177 catch (ConnectTimeoutException e){ 178 logger.debug("Connection timeout, sending old results"); 179 return this.readItems(); 180 } 181 catch (ReadTimeoutException e){ 182 logger.debug("Read timeout!, sending old results"); 183 return this.readItems(); 184 } 185 catch (IOException e) { 186 return this.readItems(); 187 } 188 return ret; 189 } 190 191 /** 192 * Access to the Yahoo! stock symbol suggestion API. For a plain text query returns possible stock ticker 193 * names in the format: 194 * [ 195 * {symbol:, name:, exch:, type:, exchDisp:, typeDisp:}, 196 * ..., 197 * ] 198 * 199 * @param query 200 * @return list of maps described above 201 * @throws IOException 202 */ 203 public static List<Map<String,String>> querySymbols(String query) throws IOException{ 204 Gson g = new Gson(); 205 URL queryURL = new URL(String.format(YAHOO_SUGGEST_URI,query)); 206 IndependentPair<HttpEntity, ByteArrayInputStream> readURL = HttpUtils.readURLAsByteArrayInputStream(queryURL, CONNECT_TIMEOUT, READ_TIMEOUT, new MetaRefreshRedirectStrategy() ,HttpUtils.DEFAULT_USERAGENT); 207 208 String jsonp = IOUtils.toString(readURL.secondObject()); 209 String json = jsonp.substring(jsonp.indexOf("(") + 1, jsonp.lastIndexOf(")")); 210 Map<?,?> parsed = g.fromJson(json, Map.class); 211 212 try{ 213 @SuppressWarnings("unchecked") 214 List<Map<String,String>> ret = (List<Map<String, String>>) ((Map<?,?>)parsed.get("ResultSet")).get("Result"); 215 return ret; 216 }catch(Throwable c){ 217 throw new IOException(c); 218 } 219 } 220 221 /** 222 * @param args 223 * @throws IOException 224 */ 225 public static void main(String[] args) throws IOException { 226 YahooFinanceStream yfs = new YahooFinanceStream(true,"google", "microsoft","yahoo"); 227 yfs.filter(new Predicate<Map<String,Double>>() { 228 229 @Override 230 public boolean test(Map<String, Double> object) { 231 System.out.println(object.keySet()); 232 if(!object.containsKey("GOOG"))return false; 233 return object.get("GOOG") > 10; 234 } 235 }).map(new Function<Map<String,Double>, String>() { 236 237 @Override 238 public String apply(Map<String, Double> in) { 239 return in.get("GOOG") + "cheese"; 240 } 241 }).forEach(new Operation<String>() { 242 243 @Override 244 public void perform(String object) { 245 System.out.println(object); 246 } 247 });; 248// yfs.forEach(new Operation<Map<String,Double>>() { 249// 250// @Override 251// public void perform(Map<String, Double> object) { 252// for (Entry<String, Double> map : object.entrySet()) { 253// System.out.println(map.getKey() + ": " + map.getValue()); 254// } 255// } 256// }); 257 } 258 259}