001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.demos.sandbox.ml.linear.learner.stream;
031
032import java.io.ByteArrayInputStream;
033import java.io.IOException;
034import java.io.UnsupportedEncodingException;
035import java.net.MalformedURLException;
036import java.net.URL;
037import java.net.URLEncoder;
038import java.util.ArrayList;
039import java.util.HashMap;
040import java.util.HashSet;
041import java.util.List;
042import java.util.Map;
043import java.util.Set;
044
045import org.apache.commons.httpclient.ConnectTimeoutException;
046import org.apache.commons.io.IOUtils;
047import org.apache.commons.lang.StringUtils;
048import org.apache.http.HttpEntity;
049import org.apache.log4j.Logger;
050import org.jboss.netty.handler.timeout.ReadTimeoutException;
051import org.openimaj.io.HttpUtils;
052import org.openimaj.io.HttpUtils.MetaRefreshRedirectStrategy;
053import org.openimaj.util.function.Function;
054import org.openimaj.util.function.Operation;
055import org.openimaj.util.function.Predicate;
056import org.openimaj.util.pair.IndependentPair;
057import org.openimaj.util.stream.AbstractStream;
058
059import com.Ostermiller.util.CSVParser;
060import com.google.gson.Gson;
061
062/**
063 * @author Sina Samangooei (ss@ecs.soton.ac.uk)
064 *
065 */
066public class YahooFinanceStream extends AbstractStream<Map<String,Double>>{
067        Logger logger = Logger.getLogger(YahooFinanceStream.class);
068        private static final String YAHOO_URI = "http://finance.yahoo.com/d/quotes.csv?s=%s&f=snl1";
069        private static final String YAHOO_SUGGEST_URI = "http://d.yimg.com/autoc.finance.yahoo.com/autoc?query=%s&callback=YAHOO.Finance.SymbolSuggest.ssCallback";
070        private static final int CONNECT_TIMEOUT = 1000;
071        private static final int READ_TIMEOUT = 2000;
072        private static final int ENFORCED_WAIT = 1000;
073        private URL yahooURL;
074        private long lastRead;
075        private boolean expandTickers;
076
077        private class FeedItem {
078                String name;
079                String longname;
080                double value;
081        }
082
083        /**
084         * The tickers you want
085         * @param expand whether an attempt should be made to expand the tickers using #querySymbols
086         * @param tickers
087         * @throws IOException
088         * @throws MalformedURLException
089         */
090        public YahooFinanceStream(boolean expand, String ... tickers) throws MalformedURLException, IOException {
091                if(expand) tickers = expandTickers(tickers);
092                constructYahooURI(tickers);
093                this.lastRead = 0;
094        }
095
096        private String[] expandTickers(String[] tickers) {
097                Set<String> expanded = new HashSet<String>();
098                for (String ticker : tickers) {
099                        try {
100                                List<Map<String, String>> queried = querySymbols(ticker);
101                                for (Map<String, String> query : queried) {
102                                        if(query.containsKey("symbol")) expanded.add(query.get("symbol"));
103                                }
104                        } catch (IOException e) {
105                                // Just add the ticker back
106                                expanded.add(ticker);
107                        }
108                }
109                return expanded.toArray(new String[expanded.size()]);
110        }
111
112        /**
113         * The tickers you want
114         * @param tickers
115         * @throws IOException
116         * @throws MalformedURLException
117         */
118        public YahooFinanceStream(String ... tickers) throws MalformedURLException, IOException {
119                constructYahooURI(tickers);
120                this.lastRead = 0;
121        }
122
123        private void constructYahooURI(String[] tickers) throws MalformedURLException {
124                for (int i = 0; i < tickers.length; i++) {
125                        try {
126                                tickers[i] = URLEncoder.encode(tickers[i], "UTF-8");
127                        } catch (UnsupportedEncodingException e) {
128                                // TODO Auto-generated catch block
129                                e.printStackTrace();
130                        }
131                }
132                String feeds = StringUtils.join(tickers, "+");
133                String urlString = String.format(YAHOO_URI,feeds);
134                this.yahooURL = new URL(urlString);
135
136        }
137        @Override
138        public boolean hasNext() {
139                return true;
140        }
141
142        @Override
143        public Map<String,Double> next() {
144                List<FeedItem> items = readItems();
145
146                Map<String, Double> ret = new HashMap<String, Double>();
147                for (FeedItem feedItem : items) {
148                        ret.put(feedItem.name,feedItem.value);
149                }
150                return ret ;
151        }
152        private List<FeedItem> readItems() {
153                CSVParser parser;
154                List<FeedItem> ret = new ArrayList<YahooFinanceStream.FeedItem>();
155                if(System.currentTimeMillis() - this.lastRead < ENFORCED_WAIT){
156                        logger.debug(String.format("Haven't waited %d, waiting %d, sending old results",ENFORCED_WAIT,System.currentTimeMillis() - this.lastRead));
157                        try {
158                                Thread.sleep(ENFORCED_WAIT - (System.currentTimeMillis() - this.lastRead));
159                        } catch (InterruptedException e) {
160                        }
161                        return readItems();
162                }
163                try {
164                        IndependentPair<HttpEntity, ByteArrayInputStream> readURL = HttpUtils.readURLAsByteArrayInputStream(this.yahooURL, CONNECT_TIMEOUT, READ_TIMEOUT, new MetaRefreshRedirectStrategy() ,HttpUtils.DEFAULT_USERAGENT);
165                        parser = new CSVParser(readURL.secondObject());
166                        String[][] vals = parser.getAllValues();
167                        for (String[] strings : vals) {
168                                FeedItem feedItem = new FeedItem();
169                                feedItem.name = strings[0];
170                                feedItem.longname = strings[1];
171                                feedItem.value = Double.parseDouble(strings[2]);
172                                ret.add(feedItem);
173                        }
174                        logger.debug(String.format("Read succesfully!"));
175                        this.lastRead = System.currentTimeMillis();
176                }
177                catch (ConnectTimeoutException e){
178                        logger.debug("Connection timeout, sending old results");
179                        return this.readItems();
180                }
181                catch (ReadTimeoutException e){
182                        logger.debug("Read timeout!, sending old results");
183                        return this.readItems();
184                }
185                catch (IOException e) {
186                        return this.readItems();
187                }
188                return ret;
189        }
190
191        /**
192         * Access to the Yahoo! stock symbol suggestion API. For a plain text query returns possible stock ticker
193         * names in the format:
194         * [
195         *      {symbol:, name:, exch:, type:, exchDisp:, typeDisp:},
196         *      ...,
197         * ]
198         *
199         * @param query
200         * @return list of maps described above
201         * @throws IOException
202         */
203        public static List<Map<String,String>> querySymbols(String query) throws IOException{
204                Gson g = new Gson();
205                URL queryURL = new URL(String.format(YAHOO_SUGGEST_URI,query));
206                IndependentPair<HttpEntity, ByteArrayInputStream> readURL = HttpUtils.readURLAsByteArrayInputStream(queryURL, CONNECT_TIMEOUT, READ_TIMEOUT, new MetaRefreshRedirectStrategy() ,HttpUtils.DEFAULT_USERAGENT);
207
208                String jsonp = IOUtils.toString(readURL.secondObject());
209                String json = jsonp.substring(jsonp.indexOf("(") + 1, jsonp.lastIndexOf(")"));
210                Map<?,?> parsed = g.fromJson(json, Map.class);
211
212                try{
213                        @SuppressWarnings("unchecked")
214                        List<Map<String,String>> ret = (List<Map<String, String>>) ((Map<?,?>)parsed.get("ResultSet")).get("Result");
215                        return ret;
216                }catch(Throwable c){
217                        throw new IOException(c);
218                }
219        }
220
221        /**
222         * @param args
223         * @throws IOException
224         */
225        public static void main(String[] args) throws IOException {
226                YahooFinanceStream yfs = new YahooFinanceStream(true,"google", "microsoft","yahoo");
227                yfs.filter(new Predicate<Map<String,Double>>() {
228                        
229                        @Override
230                        public boolean test(Map<String, Double> object) {
231                                System.out.println(object.keySet());
232                                if(!object.containsKey("GOOG"))return false;
233                                return object.get("GOOG") > 10;
234                        }
235                }).map(new Function<Map<String,Double>, String>() {
236
237                        @Override
238                        public String apply(Map<String, Double> in) {
239                                return in.get("GOOG") + "cheese";
240                        }
241                }).forEach(new Operation<String>() {
242                        
243                        @Override
244                        public void perform(String object) {
245                                System.out.println(object);
246                        }
247                });;
248//              yfs.forEach(new Operation<Map<String,Double>>() {
249//
250//                      @Override
251//                      public void perform(Map<String, Double> object) {
252//                              for (Entry<String, Double> map : object.entrySet()) {
253//                                      System.out.println(map.getKey() + ": " + map.getValue());
254//                              }
255//                      }
256//              });
257        }
258
259}