/**
 * Copyright 2010 The University of Southampton, Yahoo Inc., and the
 * individual contributors. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.openimaj.web.readability;

import java.io.IOException;
import java.io.StringReader;
import java.net.URL;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.EnumSet;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.cyberneko.html.parsers.DOMFragmentParser;
import org.cyberneko.html.parsers.DOMParser;
import org.pojava.datetime.DateTime;
import org.w3c.dom.DOMException;
import org.w3c.dom.Document;
import org.w3c.dom.DocumentFragment;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.w3c.dom.bootstrap.DOMImplementationRegistry;
import org.w3c.dom.ls.DOMImplementationLS;
import org.w3c.dom.ls.LSSerializer;
import org.w3c.dom.traversal.DocumentTraversal;
import org.w3c.dom.traversal.NodeFilter;
import org.w3c.dom.traversal.TreeWalker;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

/**
 * Class for extracting the "content" from web-pages, and ignoring adverts, etc.
 * Based upon readability.js (http://lab.arc90.com/experiments/readability/) and
 * modified to behave better for certain sites (and typically better mimic
 * Safari Reader functionality).
 *
 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
 * @author Michael Matthews (mikemat@yahoo-inc.com)
 * @author David Dupplaw (dpd@ecs.soton.ac.uk)
 */
public class Readability
{
        /**
         * Regular expressions for different types of content
         */
        protected static class Regexps {

                public static String unlikelyCandidatesRe = "(?i)combx|comment|disqus|foot|header|menu|rss|shoutbox|sidebar|sponsor|story-feature|banner"; // caption?
                public static String okMaybeItsACandidateRe = "(?i)and|comments|article|body|column|main";
                public static String positiveRe = "(?i)article|body|comments|content|entry|hentry|page|pagination|post|text";
                public static String negativeRe = "(?i)combx|comment|contact|foot|footer|footnote|link|masthead|media|meta|promo|related|scroll|shoutbox|sponsor|tags|widget|warning";
                public static String divToPElementsRe = "(?i)(a|blockquote|dl|div|img|ol|p|pre|table|ul)";
                public static String replaceBrsRe = "(?i)(<br[^>]*>[ \n\r\t]*){2,}";
                public static String replaceFontsRe = "(?i)<(\\/?)font[^>]*>";
                public static String trimRe = "^\\s+|\\s+$";
                public static String normalizeRe = "\\s{2,}";
                public static String killBreaksRe = "(<br\\s*\\/?>(\\s|&nbsp;?)*){1,}";
                public static String videoRe = "(?i)http:\\/\\/(www\\.)?(youtube|vimeo)\\.com";

                public static String titleSeparatorRe = "\\|\\-\\/";

                // this is used to try and find elements that represent sub-headings
                // (that are not h1..h6)
                public static String likelySubheadCandidateRe = "(?i)cross-head";
        }

        enum Flag {
                FLAG_STRIP_UNLIKELYS,
                FLAG_WEIGHT_CLASSES
        }

        /**
         * Threshold for removing elements with lots of links
         */
        public static float LINK_DENSITY_THRESHOLD = 0.33F;

        // IVARS below
        protected Document document;
        private Node bodyCache;
        protected EnumSet<Flag> flags = EnumSet.allOf(Flag.class);

        protected String articleTitle;
        protected Element articleContent;
        protected String article_date_string;
        protected Date article_date;
        protected String article_contentType;

        protected boolean debug = false;

        protected boolean addTitle = false;

        /**
         * Construct with the given document. Debugging is disabled.
         *
         * @param document
         *            The document.
         */
        public Readability(Document document) {
                this(document, false);
        }

        /**
         * Construct with the given document. The second argument can be used to
         * enable debugging output.
         *
         * @param document
         *            The document.
         * @param debug
         *            Enable debugging output.
         */
        public Readability(Document document, boolean debug) {
                this(document, debug, false);
        }

        /**
         * Construct with the given document. The second argument can be used to
         * enable debugging output. The third option controls whether the title
         * should be included in the output.
         *
         * @param document
         *            The document.
         * @param debug
         *            Enable debugging output.
         * @param addTitle
         *            Add title to output.
         */
        public Readability(Document document, boolean debug, boolean addTitle) {
                this.debug = debug;
                this.document = document;
                this.addTitle = addTitle;
                augmentDocument(document);
                init();
        }

        /**
         * Iterates through all the ELEMENT nodes in a document and gives them ids
         * if they don't already have them.
         *
         * @param document
         */
        public static void augmentDocument(Document document) {
                final DocumentTraversal traversal = (DocumentTraversal) document;

                final TreeWalker walker = traversal.createTreeWalker(document, NodeFilter.SHOW_ELEMENT, null, true);

                traverseLevel(walker, 0);
        }

        private static int traverseLevel(TreeWalker walker, int counter) {
                // describe current node:
                final Node parend = walker.getCurrentNode();

                if (parend instanceof Element) {
                        if (((Element) parend).getAttribute("id").length() == 0) {
                                ((Element) parend).setAttribute("id", "gen-id-" + counter);
                                counter++;
                        }
                }

                // traverse children:
                for (Node n = walker.firstChild(); n != null; n = walker.nextSibling()) {
                        counter = traverseLevel(walker, counter);
                }

                // return position to the current (level up):
                walker.setCurrentNode(parend);

                return counter;
        }

        protected void dbg(String s) {
                if (debug)
                        System.err.println(s);
        }

        protected String getTitle() {
                final NodeList l = document.getElementsByTagName("title");

                if (l.getLength() == 0)
                        return "";

                return l.item(0).getTextContent();
        }

        /**
         * Javascript-like String.match
         *
         * @param input
         * @param regex
         * @return
         */
        protected String[] match(String input, String regex) {
                final Matcher matcher = Pattern.compile(regex).matcher(input);
                final List<String> matches = new ArrayList<String>();

                while (matcher.find()) {
                        matches.add(matcher.group(0));
                }

                return matches.toArray(new String[matches.size()]);
        }

        /**
         * @return True if the article has any detected content; false otherwise.
         */
        public boolean hasContent() {
                return articleContent != null;
        }

        /**
         * Javascript-like String.search
         *
         * @param input
         * @param regex
         * @return
         */
        protected int search(String input, String regex) {
                final Matcher matcher = Pattern.compile(regex).matcher(input);

                if (!matcher.find())
                        return -1;
                return matcher.start();
        }

        protected void findArticleEncoding() {
                final NodeList nl = document.getElementsByTagName("meta");
                for (int j = 0; j < nl.getLength(); j++) {
                        if (((Element) nl.item(j)).getAttribute("http-equiv").equals("Content-Type")) {
                                article_contentType = ((Element) nl.item(j)).getAttribute("content");
                                return;
                        }
                }

        }

        protected void findArticleDate() {
                // <meta name="OriginalPublicationDate" content="2010/07/12 14:08:02"/>
                // <meta name="DC.date.issued" content="2010-07-12">
                NodeList nl = document.getElementsByTagName("meta");
                for (int j = 0; j < nl.getLength(); j++) {
                        if (((Element) nl.item(j)).getAttribute("name").equals("OriginalPublicationDate")) {
                                article_date_string = ((Element) nl.item(j)).getAttribute("content");
                                article_date = DateTime.parse(article_date_string).toDate();
                                return;
                        }
                        if (((Element) nl.item(j)).getAttribute("name").equals("DC.date.issued")) {
                                article_date_string = ((Element) nl.item(j)).getAttribute("content");
                                article_date = DateTime.parse(article_date_string).toDate();
                                return;
                        }
                }

                // <time datetime="2010-07-12T10:26BST" pubdate>Monday 12 July 2010
                // 10.26 BST</time>
                nl = document.getElementsByTagName("time");
                for (int j = 0; j < nl.getLength(); j++) {
                        if (((Element) nl.item(j)).getAttributeNode("pubdate") != null) {
                                article_date_string = ((Element) nl.item(j)).getAttribute("datetime");
                                article_date = DateTime.parse(article_date_string).toDate();
                                return;
                        }
                }

                // <span class="date">14:08 GMT, Monday, 12 July 2010 15:08 UK</span>
                // <p class="date">09.07.2010 @ 17:49 CET</p>
                // <p class="date">Today @ 09:29 CET</p>
                nl = document.getElementsByTagName("*");
                for (int j = 0; j < nl.getLength(); j++) {
                        if ((((Element) nl.item(j)).getAttribute("class").contains("date") ||
                                        ((Element) nl.item(j)).getAttribute("class").contains("Date")) &&
                                        !(((Element) nl.item(j)).getAttribute("class").contains("update") ||
                                                        ((Element) nl.item(j)).getAttribute("class").contains("Update")))
                        {
                                article_date_string = getInnerTextSep(nl.item(j)).trim();
                                parseDate();
                                return;
                        }
                }
                for (int j = 0; j < nl.getLength(); j++) {
                        if ((((Element) nl.item(j)).getAttribute("id").contains("date") ||
                                        ((Element) nl.item(j)).getAttribute("id").contains("Date")) &&
                                        !(((Element) nl.item(j)).getAttribute("id").contains("update") ||
                                                        ((Element) nl.item(j)).getAttribute("id").contains("Update")))
                        {
                                article_date_string = getInnerTextSep(nl.item(j)).trim();
                                parseDate();
                                return;
                        }
                }

                // Last updated at 3:05 PM on 12th July 2010
                nl = document.getElementsByTagName("*");
                for (int j = 0; j < nl.getLength(); j++) {
                        final String text = nl.item(j).getTextContent();

                        if (text == null)
                                continue;

                        final Pattern p = Pattern.compile("Last updated at (\\d+:\\d\\d [AP]M on \\d+[thsndr]+ \\w+ \\d\\d\\d\\d)");
                        final Matcher m = p.matcher(text);
                        if (m.find()) {
                                article_date_string = m.group(1);

                                String cpy = article_date_string.replaceAll("th", "");
                                cpy = cpy.replaceAll("st", "");
                                cpy = cpy.replaceAll("nd", "");
                                cpy = cpy.replaceAll("rd", "");

                                final SimpleDateFormat sdf = new SimpleDateFormat("h:mm a 'on' dd MMMM yyyy");
                                try {
                                        article_date = sdf.parse(cpy);
                                } catch (final ParseException e) {
                                }
                                return;
                        }
                }
        }

        @SuppressWarnings("deprecation")
        protected void parseDate() {
                if (article_date_string == null || article_date_string.trim().isEmpty())
                        return;

                if (article_date_string.contains("Today")) {
                        try {
                                final SimpleDateFormat sdf = new SimpleDateFormat("'Today @' HH:mm z");
                                article_date = sdf.parse(article_date_string);
                                final Date now = new Date();
                                article_date.setDate(now.getDate());
                                article_date.setMonth(now.getMonth());
                                article_date.setYear(now.getYear());
                        } catch (final ParseException e) {
                        }
                } else {
                        try {
                                final SimpleDateFormat sdf = new SimpleDateFormat("h:mm z',' E',' dd M yyyy");
                                article_date = sdf.parse(article_date_string);
                        } catch (final ParseException e) {
                                try {
                                        final SimpleDateFormat sdf = new SimpleDateFormat("dd.MM.yyyy '@' HH:mm z");
                                        article_date = sdf.parse(article_date_string);
                                } catch (final ParseException ee) {
                                        try {
                                                final SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy");
                                                article_date = sdf.parse(article_date_string);
                                        } catch (final ParseException eee) {
                                                try {
                                                        article_date = DateTime.parse(article_date_string).toDate();
                                                } catch (final IllegalArgumentException ie) {
                                                } catch (final java.lang.ArrayIndexOutOfBoundsException ie) {
                                                        System.out.println(article_date_string);
                                                }
                                        }
                                }
                        }
                }
        }

        /**
         * Get the article title.
         *
         * @return void
         **/
        protected String findArticleTitle() {
                String curTitle = "", origTitle = "";

                curTitle = origTitle = getTitle();

                //
                final List<String> potentialTitles = new ArrayList<String>();
                for (int i = 1; i <= 6; i++) {
                        final NodeList nl = document.getElementsByTagName("h" + i);
                        if (nl.getLength() > 0) {
                                for (int j = 0; j < nl.getLength(); j++)
                                        potentialTitles.add(nl.item(j).getTextContent().trim());
                        }
                }

                String potentialTitle = null;
                int score = 0;
                for (final String s : potentialTitles) {
                        if (s.length() > score && curTitle.contains(s)) {
                                potentialTitle = s;
                                score = s.length();
                        }
                }
                if (potentialTitle != null)
                        return potentialTitle;
                //

                if (match(curTitle, " [" + Regexps.titleSeparatorRe + "]+ ").length > 0)
                {
                        curTitle = origTitle.replaceAll("(.*) [" + Regexps.titleSeparatorRe + "]+ .*", "$1");

                        if (curTitle.split(" ").length < 3) {
                                curTitle = origTitle.replaceAll("(?i)[^" + Regexps.titleSeparatorRe + "]*[" + Regexps.titleSeparatorRe
                                                + "]+(.*)", "$1");
                        }
                }
                else if (curTitle.indexOf(": ") != -1)
                {
                        curTitle = origTitle.replaceAll("(?i).*:(.*)", "$1");

                        if (curTitle.split(" ").length < 3) {
                                curTitle = origTitle.replaceAll("(?i)[^:]*[:](.*)", "$1");
                        }
                }
                else if (curTitle.length() > 150 || curTitle.length() < 15)
                {
                        final NodeList hOnes = document.getElementsByTagName("h1");
                        if (hOnes.getLength() == 1)
                        {
                                curTitle = getInnerText((Element) hOnes.item(0));
                        }
                }

                curTitle = curTitle.replaceAll(Regexps.trimRe, "");

                if (curTitle.split(" ").length <= 3) {
                        curTitle = origTitle;
                }

                return curTitle;
        }

        /**
         * Equivalent to document.body in JS
         *
         * @return
         */
        protected Element getBody() {
                final NodeList nl = document.getElementsByTagName("body");

                if (nl.getLength() == 0)
                        return null;
                else
                        return (Element) nl.item(0);
        }

        /**
         * Runs readability.
         *
         * Workflow: 1. Prep the document by removing script tags, css, etc. 2.
         * Build readability"s DOM tree. 3. Grab the article content from the
         * current dom tree. 4. Replace the current DOM tree with the new one. 5.
         * Read peacefully.
         *
         **/
        protected void init() {
                if (getBody() != null && bodyCache == null) {
                        bodyCache = getBody().cloneNode(true);
                }

                findArticleDate(); // must be done before prepDocument()

                findArticleEncoding();

                prepDocument();

                /* Build readability"s DOM tree */
                articleTitle = findArticleTitle();
                articleContent = grabArticle();

                /**
                 * If we attempted to strip unlikely candidates on the first run
                 * through, and we ended up with no content, that may mean we stripped
                 * out the actual content so we couldn"t parse it. So re-run init while
                 * preserving unlikely candidates to have a better shot at getting our
                 * content out properly.
                 **/
                if (getInnerText(articleContent, false).length() < 250)
                {
                        if (flags.contains(Flag.FLAG_STRIP_UNLIKELYS)) {
                                flags.remove(Flag.FLAG_STRIP_UNLIKELYS);
                                getBody().getParentNode().replaceChild(bodyCache, getBody());
                                init();
                                return;
                        }
                        else if (flags.contains(Flag.FLAG_WEIGHT_CLASSES)) {
                                flags.remove(Flag.FLAG_WEIGHT_CLASSES);
                                getBody().getParentNode().replaceChild(bodyCache, getBody());
                                init();
                                return;
                        }
                        else {
                                articleContent = null;
                        }
                }

                if (addTitle && articleContent != null) {
                        final Element titleNode = document.createElement("h1");
                        titleNode.setAttribute("id", "title");
                        titleNode.appendChild(document.createTextNode(getArticleTitle()));
                        articleContent.insertBefore(titleNode, articleContent.getFirstChild());
                }
        }

        /**
         * Prepare the HTML document for readability to scrape it. This includes
         * things like stripping javascript, CSS, and handling terrible markup.
         *
         **/
        protected void prepDocument() {
                /**
                 * In some cases a body element can"t be found (if the HTML is totally
                 * hosed for example) so we create a new body node and append it to the
                 * document.
                 */
                if (getBody() == null)
                {
                        final Node body = document.createElement("body");
                        document.appendChild(body);
                }

                // frames are not supported in this version!
                // NodeList frames = document.getElementsByTagName("frame");
                // if(frames.length > 0)
                // {
                // Node bestFrame = null;
                // int bestFrameSize = 0;
                // for(int frameIndex = 0; frameIndex < frames.getLength();
                // frameIndex++)
                // {
                // int frameSize = frames.item(frameIndex).offsetWidth +
                // frames[frameIndex].offsetHeight;
                // var canAccessFrame = false;
                // try {
                // frames[frameIndex].contentWindow.document.body;
                // canAccessFrame = true;
                // }
                // catch(eFrames) {
                // dbg(eFrames);
                // }
                //
                // if(canAccessFrame && frameSize > bestFrameSize)
                // {
                // bestFrame = frames[frameIndex];
                // bestFrameSize = frameSize;
                // }
                // }
                //
                // if(bestFrame)
                // {
                // var newBody = document.createElement("body");
                // newBody.innerHTML = bestFrame.contentWindow.document.body.innerHTML;
                // newBody.style.overflow = "scroll";
                // document.body = newBody;
                //
                // var frameset = document.getElementsByTagName("frameset")[0];
                // if(frameset) {
                // frameset.parentNode.removeChild(frameset); }
                //
                // readability.frameHack = true;
                // }
                // }

                /* remove all scripts that are not readability */
                final NodeList scripts = document.getElementsByTagName("script");
                for (int i = scripts.getLength() - 1; i >= 0; i--)
                {
                        scripts.item(i).getParentNode().removeChild(scripts.item(i));
                }

                /* Remove all style tags in head */
                final NodeList styleTags = document.getElementsByTagName("style");
                for (int st = styleTags.getLength() - 1; st >= 0; st--) {
                        styleTags.item(st).getParentNode().removeChild(styleTags.item(st));
                }

                /* Remove all meta tags */
                final NodeList metaTags = document.getElementsByTagName("meta");
                for (int mt = metaTags.getLength() - 1; mt >= 0; mt--) {
                        metaTags.item(mt).getParentNode().removeChild(metaTags.item(mt));
                }

                /* Turn all double br's into p's */
                /*
                 * Note, this is pretty costly as far as processing goes. Maybe optimize
                 * later.
                 */
                // document.body.innerHTML =
                // document.body.innerHTML.replace(readability.regexps.replaceBrsRe,
                // '</p><p>').replace(readability.regexps.replaceFontsRe, '<$1span>');
                final Element body = getBody();
                // Node rep =
                // stringToNode(nodeToString(body).replaceAll(Regexps.replaceBrsRe,
                // "</P><P>").replaceAll(Regexps.replaceFontsRe, "<$1span>"));
                // body.getParentNode().replaceChild(rep, body);

                // This is slow!
                final Node frag = stringToNode(getInnerHTML(body).replaceAll(Regexps.replaceBrsRe, "</P><P>").replaceAll(
                                Regexps.replaceFontsRe, "<$1span>"));
                removeChildren(body);
                body.appendChild(frag);

                /* Remove all comments */
                removeComments(document);
        }

        protected void removeComments(Node n) {
                if (n.getNodeType() == Node.COMMENT_NODE) {
                        n.getParentNode().removeChild(n);
                } else {
                        final NodeList nl = n.getChildNodes();
                        for (int i = nl.getLength() - 1; i >= 0; i--)
                                removeComments(nl.item(i));
                }
        }

        /**
         * Prepare the article node for display. Clean out any inline styles,
         * iframes, forms, strip extraneous
         * <p>
         * tags, etc.
         *
         * @param Element
         **/
        protected void prepArticle(Element articleContent) {
                cleanStyles(articleContent);
                killBreaks(articleContent);

                /* Clean out junk from the article content */
                clean(articleContent, "form");
                clean(articleContent, "object");
                clean(articleContent, "h1");
                /**
                 * If there is only one h2, they are probably using it as a header and
                 * not a subheader, so remove it since we already have a header.
                 ***/
                if (articleContent.getElementsByTagName("h2").getLength() == 1) {
                        clean(articleContent, "h2");
                }
                clean(articleContent, "iframe");

                cleanHeaders(articleContent);

                /*
                 * Do these last as the previous stuff may have removed junk that will
                 * affect these
                 */
                cleanConditionally(articleContent, "table");
                cleanConditionally(articleContent, "ul");
                cleanConditionally(articleContent, "div");

                /* Remove extra paragraphs */
                final NodeList articleParagraphs = articleContent.getElementsByTagName("p");
                for (int i = articleParagraphs.getLength() - 1; i >= 0; i--)
                {
                        final int imgCount = ((Element) articleParagraphs.item(i)).getElementsByTagName("img").getLength();
                        final int embedCount = ((Element) articleParagraphs.item(i)).getElementsByTagName("embed").getLength();
                        final int objectCount = ((Element) articleParagraphs.item(i)).getElementsByTagName("object").getLength();

                        if (imgCount == 0 && embedCount == 0 && objectCount == 0
                                        && getInnerText((Element) articleParagraphs.item(i), false) == "")
                        {
                                articleParagraphs.item(i).getParentNode().removeChild(articleParagraphs.item(i));
                        }
                }

                // articleContent.innerHTML =
                // articleContent.innerHTML.replace(/<br[^>]*>\s*<p/gi, "<p");
                final Node n = stringToNode(getInnerHTML(articleContent).replaceAll("(?i)<br[^>]*>\\s*<p", "<P"));
                removeChildren(articleContent);
                articleContent.appendChild(n);

                // now remove empty p's and tidy up
                final NodeList nl = articleContent.getElementsByTagName("p");
                for (int i = nl.getLength() - 1; i >= 0; i--) {
                        if (nl.item(i).getTextContent().trim().length() == 0)
                        {
                                nl.item(i).getParentNode().removeChild(nl.item(i));
                        } else if (nl.item(i).getChildNodes().getLength() == 1
                                        && nl.item(i).getChildNodes().item(0).getNodeType() == Node.TEXT_NODE)
                        {
                                nl.item(i).setTextContent("\n" + nl.item(i).getTextContent().trim() + "\n");
                        }
                        else if (((Element) nl.item(i)).getAttribute("class").equals("readability-styled"))
                        {
                                nl.item(i).getParentNode().replaceChild(document.createTextNode(nl.item(i).getTextContent()), nl.item(i));
                        }
                }

        }

        protected void removeChildren(Node n) {
                final NodeList nl = n.getChildNodes();
                final int nn = nl.getLength();
                for (int i = 0; i < nn; i++)
                        n.removeChild(nl.item(0));
        }

        /**
         * Initialize a node with the readability object. Also checks the
         * className/id for special names to add to its score.
         *
         * @param Element
         **/
        protected void initializeNode(Element node) {
                float contentScore = 0;

                if (node.getTagName() == "DIV") {
                        contentScore += 5;
                } else if (node.getTagName() == "PRE" || node.getTagName() == "TD" || node.getTagName() == "BLOCKQUOTE") {
                        contentScore += 3;
                } else if (node.getTagName() == "ADDRESS" || node.getTagName() == "OL" || node.getTagName() == "UL"
                                || node.getTagName() == "DL" || node.getTagName() == "DD" || node.getTagName() == "DT"
                                || node.getTagName() == "LI" || node.getTagName() == "FORM")
                {
                        contentScore -= 3;
                } else if (node.getTagName() == "H1" || node.getTagName() == "H2" || node.getTagName() == "H3"
                                || node.getTagName() == "H4" || node.getTagName() == "H5" || node.getTagName() == "H6"
                                || node.getTagName() == "TH")
                {
                        contentScore -= 5;
                }

                contentScore += getClassWeight(node);
                node.setUserData("readability", contentScore, null);
        }

        /**
         * Get an elements class/id weight. Uses regular expressions to tell if this
         * element looks good or bad.
         *
         * @param Element
         * @return number (Integer)
         **/
        protected int getClassWeight(Element e) {
                if (!flags.contains(Flag.FLAG_WEIGHT_CLASSES)) {
                        return 0;
                }

                int weight = 0;

                /* Look for a special classname */
                if (e.getAttribute("class") != "")
                {
                        if (search(e.getAttribute("class"), Regexps.negativeRe) != -1) {
                                weight -= 25;
                        }

                        if (search(e.getAttribute("class"), Regexps.positiveRe) != -1) {
                                weight += 25;
                        }
                }

                /* Look for a special ID */
                if (e.getAttribute("id") != "")
                {
                        if (search(e.getAttribute("id"), Regexps.negativeRe) != -1) {
                                weight -= 25;
                        }

                        if (search(e.getAttribute("id"), Regexps.positiveRe) != -1) {
                                weight += 25;
                        }
                }

                return weight;
        }

        protected void cleanStyles() {
                cleanStyles((Element) document);
        }

        /**
         * Remove the style attribute on every e and under. TODO: Test if
         * getElementsByTagName(*) is faster.
         *
         * @param Element
         **/
        protected void cleanStyles(Element e) {
                if (e == null)
                        return;
                Node cur = e.getFirstChild();

                // Remove any root styles, if we"re able.
                if (!e.getAttribute("class").equals("readability-styled"))
                        e.removeAttribute("style");

                // Go until there are no more child nodes
                while (cur != null) {
                        if (cur.getNodeType() == Element.ELEMENT_NODE) {
                                // Remove style attribute(s) :
                                if (!((Element) cur).getAttribute("class").equals("readability-styled")) {
                                        ((Element) cur).removeAttribute("style");
                                }
                                cleanStyles((Element) cur);
                        }
                        cur = cur.getNextSibling();
                }
        }

        /**
         * Remove extraneous break tags from a node.
         *
         * @param Element
         **/
        protected void killBreaks(Element e) {
                // e.innerHTML =
                // e.innerHTML.replace(readability.regexps.killBreaksRe,"<br />");

                final Node n = stringToNode(getInnerHTML(e).replaceAll(Regexps.killBreaksRe, "<BR />"));
                removeChildren(e);
                e.appendChild(n);
        }

        /**
         * Clean a node of all elements of type "tag". (Unless it"s a youtube/vimeo
         * video. People love movies.)
         *
         * @param Element
         * @param string
         *            tag to clean
         **/
        protected void clean(Element e, String tag) {
                final NodeList targetList = e.getElementsByTagName(tag);
                final boolean isEmbed = (tag.equals("object") || tag.equals("embed"));

                for (int y = targetList.getLength() - 1; y >= 0; y--) {
                        /*
                         * Allow youtube and vimeo videos through as people usually want to
                         * see those.
                         */
                        if (isEmbed) {
                                String attributeValues = "";
                                for (int i = 0, il = targetList.item(y).getAttributes().getLength(); i < il; i++) {
                                        attributeValues += targetList.item(y).getAttributes().item(i).getNodeValue() + "|";
                                }

                                /*
                                 * First, check the elements attributes to see if any of them
                                 * contain youtube or vimeo
                                 */
                                if (search(attributeValues, Regexps.videoRe) != -1) {
                                        continue;
                                }

                                /* Then check the elements inside this element for the same. */
                                if (search(getInnerHTML(targetList.item(y)), Regexps.videoRe) != -1) {
                                        continue;
                                }
                        }

                        targetList.item(y).getParentNode().removeChild(targetList.item(y));
                }
        }

        /**
         * Clean out spurious headers from an Element. Checks things like classnames
         * and link density.
         *
         * @param Element
         **/
        protected void cleanHeaders(Element e) {
                for (int headerIndex = 1; headerIndex < 7; headerIndex++) {
                        final NodeList headers = e.getElementsByTagName("h" + headerIndex);
                        for (int i = headers.getLength() - 1; i >= 0; i--) {
                                if (getClassWeight((Element) headers.item(i)) < 0
                                                || getLinkDensity((Element) headers.item(i)) > LINK_DENSITY_THRESHOLD)
                                {
                                        headers.item(i).getParentNode().removeChild(headers.item(i));
                                }
                        }
                }
        }

        /**
         * Get the density of links as a percentage of the content This is the
         * amount of text that is inside a link divided by the total text in the
         * node.
         *
         * @param Element
         * @return number (float)
         **/
        protected float getLinkDensity(Element e) {
                final NodeList links = e.getElementsByTagName("a");
                final int textLength = getInnerText(e).length();
                int linkLength = 0;

                for (int i = 0, il = links.getLength(); i < il; i++)
                {
                        linkLength += getInnerText((Element) links.item(i)).length();
                }

                if (linkLength == 0)
                        return 0;

                return (float) linkLength / (float) textLength;
        }

        /**
         * Clean an element of all tags of type "tag" if they look fishy. "Fishy" is
         * an algorithm based on content length, classnames, link density, number of
         * images & embeds, etc.
         **/
        protected void cleanConditionally(Element e, String tag) {
                final NodeList tagsList = e.getElementsByTagName(tag);
                final int curTagsLength = tagsList.getLength();

                /**
                 * Gather counts for other typical elements embedded within. Traverse
                 * backwards so we can remove nodes at the same time without effecting
                 * the traversal.
                 *
                 * Todo: Consider taking into account original contentScore here.
                 **/
                for (int i = curTagsLength - 1; i >= 0; i--) {
                        final int weight = getClassWeight((Element) tagsList.item(i));
                        final float contentScore = (tagsList.item(i).getUserData("readability") != null) ? (Float) (tagsList.item(i)
                                        .getUserData("readability")) : 0;

                        dbg("Cleaning Conditionally "
                                        + tagsList.item(i)
                                        + " ("
                                        + ((Element) tagsList.item(i)).getAttribute("class")
                                        + ":"
                                        + ((Element) tagsList.item(i)).getAttribute("id")
                                        + ")"
                                        + ((tagsList.item(i).getUserData("readability") != null) ? (" with score " + tagsList.item(i)
                                                        .getUserData("readability")) : ""));

                        if (weight + contentScore < 0)
                        {
                                dbg("Removing " + tagsList.item(i) + " (" + ((Element) tagsList.item(i)).getAttribute("class") + ":"
                                                + ((Element) tagsList.item(i)).getAttribute("id") + ")");
                                tagsList.item(i).getParentNode().removeChild(tagsList.item(i));
                        }
                        else if (getCharCount((Element) tagsList.item(i), ",") < 10) {
                                /**
                                 * If there are not very many commas, and the number of
                                 * non-paragraph elements is more than paragraphs or other
                                 * ominous signs, remove the element.
                                 **/
                                final int p = ((Element) tagsList.item(i)).getElementsByTagName("p").getLength();
                                final int img = ((Element) tagsList.item(i)).getElementsByTagName("img").getLength();
                                final int li = ((Element) tagsList.item(i)).getElementsByTagName("li").getLength() - 100;
                                final int input = ((Element) tagsList.item(i)).getElementsByTagName("input").getLength();

                                int embedCount = 0;
                                final NodeList embeds = ((Element) tagsList.item(i)).getElementsByTagName("embed");
                                for (int ei = 0, il = embeds.getLength(); ei < il; ei++) {
                                        if (search(((Element) embeds.item(ei)).getAttribute("src"), Regexps.videoRe) == -1) {
                                                embedCount++;
                                        }
                                }

                                final float linkDensity = getLinkDensity((Element) tagsList.item(i));
                                final int contentLength = getInnerText((Element) tagsList.item(i)).length();
                                boolean toRemove = false;

                                if (img > p) {
                                        toRemove = true;
                                } else if (li > p && tag != "ul" && tag != "ol") {
                                        toRemove = true;
                                } else if (input > Math.floor(p / 3)) {
                                        toRemove = true;
                                } else if (contentLength < 25 && (img == 0 || img > 2)) {
                                        toRemove = true;
                                } else if (weight < 25 && linkDensity > 0.2) {
                                        toRemove = true;
                                } else if (weight >= 25 && linkDensity > 0.5) {
                                        toRemove = true;
                                } else if ((embedCount == 1 && contentLength < 75) || embedCount > 1) {
                                        toRemove = true;
                                }

                                if (img == 1 && p == 0 && contentLength == 0) {
                                        final Element theImg = (Element) ((Element) tagsList.item(i)).getElementsByTagName("img").item(0);

                                        String w = "";
                                        if (theImg.getAttribute("width") != null)
                                                w = theImg.getAttribute("width");

                                        String h = "";
                                        if (theImg.getAttribute("height") != null)
                                                h = theImg.getAttribute("height");

                                        if (!(w.equals("0") || h.equals("0")))
                                                toRemove = false; // special case - it's just an inline
                                        // image
                                }

                                if (toRemove) {
                                        dbg("Removing " + tagsList.item(i) + " (" + ((Element) tagsList.item(i)).getAttribute("class") + ":"
                                                        + ((Element) tagsList.item(i)).getAttribute("id") + ")");
                                        tagsList.item(i).getParentNode().removeChild(tagsList.item(i));
                                }
                        }
                }
        }

        /**
         * Get the number of times a string s appears in the node e.
         *
         * @param Element
         * @param string
         *            - what to split on. Default is ","
         * @return number (integer)
         **/
        protected int getCharCount(Element e, String s) {
                return getInnerText(e).split(s).length - 1;
        }

        protected int getCharCount(Element e) {
                return getCharCount(e, ",");
        }

        /**
         * @return The article title
         */
        public String getArticleTitle() {
                return articleTitle;
        }

        /**
         * @return The content type of the article
         */
        public String getArticleContentType() {
                return article_contentType;
        }

        /***
         * grabArticle - Using a variety of metrics (content score, classname,
         * element types), find the content that is most likely to be the stuff a
         * user wants to read. Then return it wrapped up in a div.
         *
         * @return Element
         **/
        protected Element grabArticle() {
                final boolean stripUnlikelyCandidates = flags.contains(Flag.FLAG_STRIP_UNLIKELYS);

                /**
                 * First, node prepping. Trash nodes that look cruddy (like ones with
                 * the class name "comment", etc), and turn divs into P tags where they
                 * have been used inappropriately (as in, where they contain no other
                 * block level elements.)
                 *
                 * Note: Assignment from index for performance. See
                 * http://www.peachpit.com/articles/article.aspx?p=31567&seqNum=5 Todo:
                 * Shouldn't this be a reverse traversal?
                 **/
                Element node = null;
                final List<Element> nodesToScore = new ArrayList<Element>();
                for (int nodeIndex = 0; (node = (Element) document.getElementsByTagName("*").item(nodeIndex)) != null; nodeIndex++)
                {
                        /* Remove unlikely candidates */
                        if (stripUnlikelyCandidates) {
                                final String unlikelyMatchString = node.getAttribute("class") + node.getAttribute("id");
                                if (search(unlikelyMatchString, Regexps.unlikelyCandidatesRe) != -1 &&
                                                search(unlikelyMatchString, Regexps.okMaybeItsACandidateRe) == -1 &&
                                                !node.getTagName().equals("BODY"))
                                {
                                        dbg("Removing unlikely candidate - " + unlikelyMatchString);
                                        node.getParentNode().removeChild(node);
                                        nodeIndex--;
                                        continue;
                                }
                        }

                        if (node.getTagName().equals("P") || node.getTagName().equals("TD")) {
                                nodesToScore.add(node);
                        }

                        /*
                         * Turn all divs that don't have children block level elements into
                         * p's
                         */
                        if (node.getTagName().equals("DIV")) {

                                if (search(getInnerHTML(node), Regexps.divToPElementsRe) == -1) {
                                        dbg("Altering div to p");
                                        final Element newNode = document.createElement("P");

                                        // newNode.innerHTML = node.innerHTML;
                                        final NodeList nl = node.getChildNodes();
                                        for (int i = 0; i < nl.getLength(); i++)
                                                newNode.appendChild(nl.item(i));

                                        node.getParentNode().replaceChild(newNode, node);
                                        nodeIndex--;
                                }
                                else
                                {
                                        /* EXPERIMENTAL */
                                        for (int i = 0, il = node.getChildNodes().getLength(); i < il; i++) {
                                                final Node childNode = node.getChildNodes().item(i);
                                                if (childNode.getNodeType() == Element.TEXT_NODE) {
                                                        dbg("replacing text node with a p tag with the same content.");
                                                        final Element p = document.createElement("p");
                                                        // p.innerHTML = childNode.nodeValue;
                                                        p.setNodeValue(childNode.getNodeValue());
                                                        p.setTextContent(childNode.getTextContent());
                                                        // p.style.display = "inline";
                                                        p.setAttribute("class", "readability-styled");
                                                        childNode.getParentNode().replaceChild(p, childNode);
                                                }
                                        }
                                }
                        }
                }

                /**
                 * Loop through all paragraphs, and assign a score to them based on how
                 * content-y they look. Then add their score to their parent node.
                 *
                 * A score is determined by things like number of commas, class names,
                 * etc. Maybe eventually link density.
                 **/
                final List<Element> candidates = new ArrayList<Element>();
                for (int pt = 0; pt < nodesToScore.size(); pt++) {
                        final Element parentNode = (Element) nodesToScore.get(pt).getParentNode();
                        final Element grandParentNode = (Element) parentNode.getParentNode();
                        final String innerText = getInnerText(nodesToScore.get(pt));

                        /*
                         * If this paragraph is less than 25 characters, don't even count
                         * it.
                         */
                        if (innerText.length() < 25) {
                                continue;
                        }

                        /* Initialize readability data for the parent. */
                        if (parentNode.getUserData("readability") == null)
                        {
                                initializeNode(parentNode);
                                candidates.add(parentNode);
                        }

                        /* Initialize readability data for the grandparent. */
                        if (grandParentNode.getUserData("readability") == null)
                        {
                                initializeNode(grandParentNode);
                                candidates.add(grandParentNode);
                        }

                        float contentScore = 0;

                        /* Add a point for the paragraph itself as a base. */
                        contentScore++;

                        /* Add points for any commas within this paragraph */
                        contentScore += innerText.split(",").length;

                        /*
                         * For every 100 characters in this paragraph, add another point. Up
                         * to 3 points.
                         */
                        contentScore += Math.min(Math.floor(innerText.length() / 100F), 3F);

                        /* Add the score to the parent. The grandparent gets half. */
                        parentNode.setUserData("readability", ((Float) (parentNode.getUserData("readability")) + contentScore), null);
                        grandParentNode.setUserData("readability", ((Float) (grandParentNode.getUserData("readability")))
                                        + (contentScore / 2F), null);
                }

                /**
                 * After we've calculated scores, loop through all of the possible
                 * candidate nodes we found and find the one with the highest score.
                 **/
                Element topCandidate = null;
                for (int c = 0, cl = candidates.size(); c < cl; c++)
                {
                        /**
                         * Scale the final candidates score based on link density. Good
                         * content should have a relatively small link density (5% or less)
                         * and be mostly unaffected by this operation.
                         **/

                        candidates.get(c).setUserData("readability",
                                        (Float) (candidates.get(c).getUserData("readability")) * (1F - getLinkDensity(candidates.get(c))),
                                        null);

                        dbg("Candidate: " + candidates.get(c) + " (" + candidates.get(c).getAttribute("class") + ":"
                                        + candidates.get(c).getAttribute("id") + ") with score "
                                        + candidates.get(c).getUserData("readability"));

                        if (topCandidate == null
                                        || (Float) (candidates.get(c).getUserData("readability")) > ((Float) topCandidate
                                                        .getUserData("readability")))
                        {
                                topCandidate = candidates.get(c);
                        }
                }

                if (topCandidate != null)
                        dbg("==> TOP Candidate: " + topCandidate + " (" + topCandidate.getAttribute("class") + ":"
                                        + topCandidate.getAttribute("id") + ") with score " + topCandidate.getUserData("readability"));

                /**
                 * If we still have no top candidate, just use the body as a last
                 * resort. We also have to copy the body node so it is something we can
                 * modify.
                 **/
                if (topCandidate == null || topCandidate.getTagName().equals("BODY"))
                {
                        topCandidate = document.createElement("DIV");

                        // topCandidate.innerHTML = document.body.innerHTML;
                        final NodeList nl = getBody().getChildNodes();
                        for (int i = 0; i < nl.getLength(); i++)
                                topCandidate.appendChild(nl.item(i));
                        // document.body.innerHTML = ""; //should be covered by above

                        getBody().appendChild(topCandidate);
                        initializeNode(topCandidate);
                }

                /**
                 * Now that we have the top candidate, look through its siblings for
                 * content that might also be related. Things like preambles, content
                 * split by ads that we removed, etc.
                 **/
                final Element articleContent = document.createElement("DIV");
                articleContent.setAttribute("id", "readability-content");
                final float siblingScoreThreshold = Math.max(10F, (Float) topCandidate.getUserData("readability") * 0.2F);
                final NodeList siblingNodes = topCandidate.getParentNode().getChildNodes();

                for (int s = 0, sl = siblingNodes.getLength(); s < sl; s++)
                {
                        final Node siblingNode = siblingNodes.item(s);
                        boolean append = false;

                        if (siblingNode instanceof Element)
                                dbg("Looking at sibling node: "
                                                + siblingNode
                                                + " ("
                                                + ((Element) siblingNode).getAttribute("class")
                                                + ":"
                                                + ((Element) siblingNode).getAttribute("id")
                                                + ")"
                                                + ((siblingNode.getUserData("readability") != null) ? (" with score " + siblingNode
                                                                .getUserData("readability")) : ""));
                        dbg("Sibling has score "
                                        + (siblingNode.getUserData("readability") != null ? siblingNode.getUserData("readability")
                                                        : "Unknown"));

                        if (siblingNode == topCandidate)
                        {
                                append = true;
                        }

                        float contentBonus = 0;
                        /*
                         * Give a bonus if sibling nodes and top candidates have the example
                         * same classname
                         */
                        if (siblingNode instanceof Element
                                        && ((Element) siblingNode).getAttribute("class").equals(topCandidate.getAttribute("class"))
                                        && !topCandidate.getAttribute("class").equals(""))
                        {
                                contentBonus += (Float) topCandidate.getUserData("readability") * 0.2F;
                        }

                        if (siblingNode.getUserData("readability") != null
                                        && ((Float) siblingNode.getUserData("readability") + contentBonus) >= siblingScoreThreshold)
                        {
                                append = true;
                        }

                        if (siblingNode.getNodeName().equals("P")) {
                                final float linkDensity = getLinkDensity((Element) siblingNode);
                                final String nodeContent = getInnerText((Element) siblingNode);
                                final int nodeLength = nodeContent.length();

                                if (nodeLength > 80 && linkDensity < 0.25)
                                {
                                        append = true;
                                }
                                else if (nodeLength < 80 && linkDensity == 0 && search(nodeContent, "\\.( |$)") != -1)
                                {
                                        append = true;
                                }
                        }

                        if (append)
                        {
                                dbg("Appending node: " + siblingNode);

                                Node nodeToAppend = null;
                                if (!siblingNode.getNodeName().equals("DIV") && !siblingNode.getNodeName().equals("P")) {
                                        /*
                                         * We have a node that isn't a common block level element,
                                         * like a form or td tag. Turn it into a div so it doesn't
                                         * get filtered out later by accident.
                                         */

                                        dbg("Altering siblingNode of " + siblingNode.getNodeName() + " to div.");
                                        nodeToAppend = document.createElement("div");
                                        if (siblingNode instanceof Element)
                                                ((Element) nodeToAppend).setAttribute("id", ((Element) siblingNode).getAttribute("id"));

                                        // nodeToAppend.innerHTML = siblingNode.innerHTML;
                                        final NodeList nl = siblingNode.getChildNodes();
                                        for (int i = 0; i < nl.getLength(); i++)
                                                nodeToAppend.appendChild(nl.item(i));
                                } else {
                                        nodeToAppend = siblingNode;
                                        s--;
                                        sl--;
                                }

                                /*
                                 * To ensure a node does not interfere with readability styles,
                                 * remove its classnames
                                 */
                                if (nodeToAppend instanceof Element)
                                        ((Element) nodeToAppend).setAttribute("class", "");

                                /*
                                 * Append sibling and subtract from our list because it removes
                                 * the node when you append to another node
                                 */
                                articleContent.appendChild(nodeToAppend);
                        }
                }

                /**
                 * So we have all of the content that we need. Now we clean it up for
                 * presentation.
                 **/
                prepArticle(articleContent);

                return articleContent;
        }

        protected String getInnerHTML(Node n) {
                if (n.getNodeType() == Node.TEXT_NODE)
                        return n.getTextContent();

                String result = "";
                final NodeList nl = n.getChildNodes();
                for (int i = 0; i < nl.getLength(); i++) {
                        if (nl.item(i).getNodeType() == Node.TEXT_NODE)
                                result += nl.item(i).getTextContent();
                        else if (nl.item(i).getNodeType() == Node.COMMENT_NODE)
                                result += "<!-- " + nl.item(i).getTextContent() + " -->";
                        else
                                result += nodeToString(nl.item(i));
                }

                return result;
        }

        protected String nodeToString(Node n) {
                return nodeToString(n, false);
        }

        protected static String nodeToString(Node n, boolean pretty) {
                try {
                        final DOMImplementationRegistry registry = DOMImplementationRegistry.newInstance();
                        final DOMImplementationLS impl = (DOMImplementationLS) registry.getDOMImplementation("LS");
                        final LSSerializer writer = impl.createLSSerializer();

                        writer.getDomConfig().setParameter("xml-declaration", false);
                        if (pretty) {
                                writer.getDomConfig().setParameter("format-pretty-print", true);
                        }

                        return writer.writeToString(n);
                } catch (final Exception e) {
                        throw new RuntimeException(e);
                }
        }

        protected Node stringToNode(String str) {
                try {
                        final DOMFragmentParser parser = new DOMFragmentParser();
                        final DocumentFragment fragment = document.createDocumentFragment();
                        parser.parse(new InputSource(new StringReader(str)), fragment);
                        return fragment;

                        // try and return the element itself if possible...
                        // NodeList nl = fragment.getChildNodes();
                        // for (int i=0; i<nl.getLength(); i++) if (nl.item(i).getNodeType()
                        // == Node.ELEMENT_NODE) return nl.item(i);
                        // return fragment;

                } catch (final Exception e) {
                        throw new RuntimeException(e);
                }
        }

        /**
         * Get the inner text of a node - cross browser compatibly. This also strips
         * out any excess whitespace to be found.
         *
         * @param Element
         * @return string
         **/
        protected String getInnerText(Element e, boolean normalizeSpaces) {
                String textContent = "";

                textContent = e.getTextContent().replaceAll(Regexps.trimRe, "");

                if (normalizeSpaces) {
                        return textContent.replaceAll(Regexps.normalizeRe, " ");
                } else {
                        return textContent;
                }
        }

        protected String getInnerTextSep(Node e) {
                if (e.hasChildNodes()) {
                        String s = "";
                        final NodeList nl = e.getChildNodes();
                        for (int i = 0; i < nl.getLength(); i++) {
                                if (!nl.item(i).getNodeName().equalsIgnoreCase("script"))
                                        s += getInnerTextSep(nl.item(i));
                        }
                        return s;
                } else {
                        return e.getTextContent() + " ";
                }
        }

        protected String getInnerText(Element e) {
                return getInnerText(e, true);
        }

        /**
         * @return The article HTML content as a {@link String}.
         */
        public String getArticleHTML() {
                if (articleContent == null)
                        return "";
                return nodeToString(articleContent, true);
        }

        /**
         * @return The articles HTML dom node.
         */
        public Node getArticleHTML_DOM() {
                return articleContent;
        }

        protected String getArticleDateString() {
                return article_date_string;
        }

        /**
         * @return The article date.
         */
        public Date getArticleDate() {
                return article_date;
        }

        /**
         * @return The text of the article.
         */
        public String getArticleText() {
                if (articleContent == null)
                        return "Unable to find article content";
                // return getInnerText(articleContent, false);
                return articleContent.getTextContent().trim().replaceAll("[\r|\n|\r\n]{2,}", "\n\n").replaceAll(" {2,}", " ");
        }

        /**
         * @return Any links in the article.
         */
        public List<Anchor> getArticleLinks() {
                final List<Anchor> anchors = new ArrayList<Anchor>();
                if (articleContent == null)
                        return anchors;

                final NodeList nl = articleContent.getElementsByTagName("a");
                for (int i = 0; i < nl.getLength(); i++) {
                        final Element a = (Element) nl.item(i);

                        final Anchor anchor = new Anchor(getInnerText(a), a.getAttribute("href"));
                        anchors.add(anchor);
                }
                return anchors;
        }

        /**
         * @return Any links in the document.
         */
        public List<Anchor> getAllLinks() {
                final List<Anchor> anchors = new ArrayList<Anchor>();

                final NodeList nl = document.getElementsByTagName("a");
                for (int i = 0; i < nl.getLength(); i++) {
                        final Element a = (Element) nl.item(i);
                        final Anchor anchor = new Anchor(getInnerText(a), a.getAttribute("href"));
                        anchors.add(anchor);
                }
                return anchors;
        }

        /**
         * @return Any images in the article.
         */
        public List<String> getArticleImages() {
                final List<String> images = new ArrayList<String>();
                if (articleContent == null)
                        return images;

                final NodeList nl = articleContent.getElementsByTagName("img");
                for (int i = 0; i < nl.getLength(); i++) {
                        final Element img = (Element) nl.item(i);
                        images.add(img.getAttribute("src"));
                }
                return images;
        }

        /**
         * @return Any subheadings in the article.
         */
        public List<String> getArticleSubheadings() {
                final List<String> subtitles = new ArrayList<String>();
                if (articleContent == null)
                        return subtitles;

                for (int j = 1; j <= 6; j++) {
                        final NodeList nl = articleContent.getElementsByTagName("h" + j);
                        if (nl.getLength() > 0) {
                                for (int i = 0; i < nl.getLength(); i++) {
                                        subtitles.add(nl.item(i).getTextContent());
                                }
                                break;
                        }
                }

                if (subtitles.size() == 0) {
                        // try looking for other likely-looking elements

                        final NodeList nl = articleContent.getElementsByTagName("*");
                        for (int i = 0; i < nl.getLength(); i++) {
                                if (nl.item(i) instanceof Element &&
                                                ((Element) nl.item(i)).getAttribute("class") != null &&
                                                search(((Element) nl.item(i)).getAttribute("class"), Regexps.likelySubheadCandidateRe) != -1)
                                        subtitles.add(nl.item(i).getTextContent());
                        }
                }

                return subtitles;
        }

        protected List<Node> findChildNodesWithName(Node parent, String name) {
                final NodeList children = parent.getChildNodes();
                final List<Node> results = new ArrayList<Node>();

                for (int i = 0; i < children.getLength(); ++i) {
                        final Node child = children.item(i);
                        if (child == null)
                                continue;

                        final String nodeName = child.getNodeName();
                        if (nodeName == null)
                                continue;

                        if (nodeName.equals(name)) {
                                results.add(child);
                        }
                }
                return results;
        }

        protected int findChildNodeIndex(Node parent, Node childToFind)
        {
                for (int index = 0; index < parent.getChildNodes().getLength(); index++)
                        if (parent.getChildNodes().item(index) == childToFind)
                                return index;
                return -1;
        }

        protected void getArticleTextMapping(TreeWalker walker, List<MappingNode> map) throws DOMException {
                final Node parend = walker.getCurrentNode();

                if (parend.getNodeType() == Node.TEXT_NODE && parend.getParentNode().getAttributes().getNamedItem("id") != null)
                {
                        if (parend.getTextContent().trim().length() > 0)
                        {
                                final int index = findChildNodeIndex(parend.getParentNode(), parend);
                                if (index != -1)
                                {
                                        // square brackets are not valid XML/HTML identifier
                                        // characters, so we can use them here
                                        map.add(new MappingNode(
                                                        parend.getParentNode().getAttributes().getNamedItem("id").getNodeValue() + "[" + index + "]",
                                                        parend.getNodeValue()));

                                        // System.out.println(
                                        // "ELEMENT '"+parend.getParentNode().getAttributes().getNamedItem("id").getNodeValue()
                                        // + "["+index+"]"+"'");
                                        // System.out.println( "VALUE:  '"+parend.getNodeValue()+"'"
                                        // );
                                }
                        }
                }

                // traverse children:
                for (Node n = walker.firstChild(); n != null; n = walker.nextSibling()) {
                        getArticleTextMapping(walker, map);
                }

                // return position to the current (level up):
                walker.setCurrentNode(parend);
        }

        protected class MappingNode {
                String id;
                String text;

                public MappingNode(String id, String text) {
                        this.id = id;
                        this.text = text;
                }

                public String getId() {
                        return id;
                }

                public String getText() {
                        return text;
                }

                @Override
                public String toString() {
                        return "MappingNode(" + id + " -> " + text + ")";
                }
        }

        /**
         * Get the mapping between bits of text in the dom & their xpaths
         *
         * @return mapping from xpath to text
         */
        public List<MappingNode> getArticleTextMapping() {
                if (articleContent == null)
                        return null;

                final List<MappingNode> map = new ArrayList<MappingNode>();

                final TreeWalker walker = ((DocumentTraversal) document).createTreeWalker(articleContent, NodeFilter.SHOW_TEXT
                                | NodeFilter.SHOW_ELEMENT, null, true);

                getArticleTextMapping(walker, map);

                return map;
        }

        /**
         * Convenience method to build a {@link Readability} instance from an html
         * string.
         *
         * @param html
         *            The html string
         * @return new {@link Readability} instance.
         * @throws SAXException
         * @throws IOException
         */
        public static Readability getReadability(String html) throws SAXException, IOException {
                return getReadability(html, false);
        }

        /**
         * Convenience method to build a {@link Readability} instance from an html
         * string.
         *
         * @param html
         *            The html string
         * @param addTitle
         *            Should the title be added to the generated article?
         * @return new {@link Readability} instance.
         * @throws SAXException
         * @throws IOException
         */
        public static Readability getReadability(String html, boolean addTitle) throws SAXException, IOException {
                final DOMParser parser = new DOMParser();
                parser.parse(new InputSource(new StringReader(html)));

                return new Readability(parser.getDocument(), false, addTitle);
        }

        /**
         * Testing
         *
         * @param argv
         * @throws Exception
         */
        public static void main(String[] argv) throws Exception {
                // URL input = new
                // URL("file:///home/dd/Programming/Readability4J/t.html");
                // URL input = new
                // URL("http://news.bbc.co.uk/1/hi/politics/10362367.stm");
                final URL input = new URL("http://blog.confluent.io/2015/01/29/making-sense-of-stream-processing/");
                // URL input = new URL("http://euobserver.com/9/30465");
                // URL input = new URL("http://euobserver.com/?aid=23383");
                // URL input = new
                // URL("http://abandoninplace.squarespace.com/blog/2010/6/8/wwdc-monday.html");
                // URL input = new URL("file:///Users/jsh2/Desktop/test.html");
                // URL input = new
                // URL("http://mobile.engadget.com/2010/06/17/htc-aria-review/");
                // URL input = new URL("http://thedailywtf.com/Articles/Benched.aspx");
                // URL input = new
                // URL("http://www.dailymail.co.uk/news/article-1287625/Woman-sparked-150-000-manhunt-slashing-face-crying-rape-faces-jail.html");
                // URL input = new
                // URL("http://mrpaparazzi.com/post/11619/Lindsay-Lohan-Tests-Negative-For-Alcohol-Goes-Clubbing-To-Celebrate.aspx");
                // URL input = new
                // URL("http://www.bbc.co.uk/news/world-middle-east-11415719");
                // URL input = new URL("http://www.thebigproject.co.uk/news/");
                // URL input = new
                // URL("http://blogs.euobserver.com/popescu/2009/12/15/on-euro-optimism-pessimism-and-failures/#more-958");
                // URL input = new
                // URL("http://www.cnn.com/2010/WORLD/meast/09/27/west.bank.settlement.construction/index.html?hpt=T2");

                // URL input = new
                // URL("http://www.huffingtonpost.com/steven-cohen/its-time-to-enact-congest_b_740315.html");
                // URL input = new
                // URL("http://uk.mac.ign.com/articles/573/573319p1.html");
                final DOMParser parser = new DOMParser();
                parser.parse(new InputSource(input.openStream()));

                final Readability r = new Readability(parser.getDocument(), true, true);

                // System.out.println(r.getArticleTitle());
                System.out.println(r.getArticleHTML());
                // System.out.println(r.getAllLinks());
                // System.out.println(r.getArticleText());

                System.out.println();
                System.out.println("***");
                System.out.println();

                for (final MappingNode s : r.getArticleTextMapping())
                        System.out.println(s);

                // PrintStream out = new PrintStream("news-sites");
                // for (Anchor anchor : r.getAllLinks()) {
                // out.println(anchor.getHref() + "\t" + anchor.getText());
                // }
                // out.close();

                System.out.println(r.getArticleImages());
                // System.out.println(r.getArticleSubheadings());
                // System.out.println(r.getArticleHTML());
                // System.out.println(r.getArticleHTML_DOM());

                // System.out.println(r.getArticleDateString());
                // System.out.println(r.getArticleDate());
        }
}