/**
    * Copyright 2010 The University of Southampton, Yahoo Inc., and the
    * individual contributors. All rights reserved.
    *
    * Licensed under the Apache License, Version 2.0 (the "License");
    * you may not use this file except in compliance with the License.
    * You may obtain a copy of the License at
    *
    *    http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  package org.openimaj.web.readability;
  
  import java.io.IOException;
  import java.io.StringReader;
  import java.net.URL;
  import java.text.ParseException;
  import java.text.SimpleDateFormat;
  import java.util.ArrayList;
  import java.util.Date;
  import java.util.EnumSet;
  import java.util.List;
  import java.util.regex.Matcher;
  import java.util.regex.Pattern;
  
  import org.cyberneko.html.parsers.DOMFragmentParser;
  import org.cyberneko.html.parsers.DOMParser;
  import org.pojava.datetime.DateTime;
  import org.w3c.dom.DOMException;
  import org.w3c.dom.Document;
  import org.w3c.dom.DocumentFragment;
  import org.w3c.dom.Element;
  import org.w3c.dom.Node;
  import org.w3c.dom.NodeList;
  import org.w3c.dom.bootstrap.DOMImplementationRegistry;
  import org.w3c.dom.ls.DOMImplementationLS;
  import org.w3c.dom.ls.LSSerializer;
  import org.w3c.dom.traversal.DocumentTraversal;
  import org.w3c.dom.traversal.NodeFilter;
  import org.w3c.dom.traversal.TreeWalker;
  import org.xml.sax.InputSource;
  import org.xml.sax.SAXException;
  
  /**
   * Class for extracting the "content" from web-pages, and ignoring adverts, etc.
   * Based upon readability.js (http://lab.arc90.com/experiments/readability/) and
   * modified to behave better for certain sites (and typically better mimic
   * Safari Reader functionality).
   *
   * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
   * @author Michael Matthews (mikemat@yahoo-inc.com)
   * @author David Dupplaw (dpd@ecs.soton.ac.uk)
   */
  public class Readability
  {
  	/**
  	 * Regular expressions for different types of content
  	 */
  	protected static class Regexps {
  
  		public static String unlikelyCandidatesRe = "(?i)combx|comment|disqus|foot|header|menu|rss|shoutbox|sidebar|sponsor|story-feature|banner"; // caption?
  		public static String okMaybeItsACandidateRe = "(?i)and|comments|article|body|column|main";
  		public static String positiveRe = "(?i)article|body|comments|content|entry|hentry|page|pagination|post|text";
  		public static String negativeRe = "(?i)combx|comment|contact|foot|footer|footnote|link|masthead|media|meta|promo|related|scroll|shoutbox|sponsor|tags|widget|warning";
  		public static String divToPElementsRe = "(?i)(a|blockquote|dl|div|img|ol|p|pre|table|ul)";
  		public static String replaceBrsRe = "(?i)(<br[^>]*>[ \n\r\t]*){2,}";
  		public static String replaceFontsRe = "(?i)<(\\/?)font[^>]*>";
  		public static String trimRe = "^\\s+|\\s+$";
  		public static String normalizeRe = "\\s{2,}";
  		public static String killBreaksRe = "(<br\\s*\\/?>(\\s|&nbsp;?)*){1,}";
  		public static String videoRe = "(?i)http:\\/\\/(www\\.)?(youtube|vimeo)\\.com";
  
  		public static String titleSeparatorRe = "\\|\\-\\/";
  
  		// this is used to try and find elements that represent sub-headings
  		// (that are not h1..h6)
  		public static String likelySubheadCandidateRe = "(?i)cross-head";
  	}
  
  	enum Flag {
  		FLAG_STRIP_UNLIKELYS,
  		FLAG_WEIGHT_CLASSES
  	}
  
  	/**
  	 * Threshold for removing elements with lots of links
  	 */
  	public static float LINK_DENSITY_THRESHOLD = 0.33F;
  
  	// IVARS below
  	protected Document document;
  	private Node bodyCache;
  	protected EnumSet<Flag> flags = EnumSet.allOf(Flag.class);
  
 	protected String articleTitle;
 	protected Element articleContent;
 	protected String article_date_string;
 	protected Date article_date;
 	protected String article_contentType;
 
 	protected boolean debug = false;
 
 	protected boolean addTitle = false;
 
 	/**
 	 * Construct with the given document. Debugging is disabled.
 	 *
 	 * @param document
 	 *            The document.
 	 */
 	public Readability(Document document) {
 		this(document, false);
 	}
 
 	/**
 	 * Construct with the given document. The second argument can be used to
 	 * enable debugging output.
 	 *
 	 * @param document
 	 *            The document.
 	 * @param debug
 	 *            Enable debugging output.
 	 */
 	public Readability(Document document, boolean debug) {
 		this(document, debug, false);
 	}
 
 	/**
 	 * Construct with the given document. The second argument can be used to
 	 * enable debugging output. The third option controls whether the title
 	 * should be included in the output.
 	 *
 	 * @param document
 	 *            The document.
 	 * @param debug
 	 *            Enable debugging output.
 	 * @param addTitle
 	 *            Add title to output.
 	 */
 	public Readability(Document document, boolean debug, boolean addTitle) {
 		this.debug = debug;
 		this.document = document;
 		this.addTitle = addTitle;
 		augmentDocument(document);
 		init();
 	}
 
 	/**
 	 * Iterates through all the ELEMENT nodes in a document and gives them ids
 	 * if they don't already have them.
 	 *
 	 * @param document
 	 */
 	public static void augmentDocument(Document document) {
 		final DocumentTraversal traversal = (DocumentTraversal) document;
 
 		final TreeWalker walker = traversal.createTreeWalker(document, NodeFilter.SHOW_ELEMENT, null, true);
 
 		traverseLevel(walker, 0);
 	}
 
 	private static int traverseLevel(TreeWalker walker, int counter) {
 		// describe current node:
 		final Node parend = walker.getCurrentNode();
 
 		if (parend instanceof Element) {
 			if (((Element) parend).getAttribute("id").length() == 0) {
 				((Element) parend).setAttribute("id", "gen-id-" + counter);
 				counter++;
 			}
 		}
 
 		// traverse children:
 		for (Node n = walker.firstChild(); n != null; n = walker.nextSibling()) {
 			counter = traverseLevel(walker, counter);
 		}
 
 		// return position to the current (level up):
 		walker.setCurrentNode(parend);
 
 		return counter;
 	}
 
 	protected void dbg(String s) {
 		if (debug)
 			System.err.println(s);
 	}
 
 	protected String getTitle() {
 		final NodeList l = document.getElementsByTagName("title");
 
 		if (l.getLength() == 0)
 			return "";
 
 		return l.item(0).getTextContent();
 	}
 
 	/**
 	 * Javascript-like String.match
 	 *
 	 * @param input
 	 * @param regex
 	 * @return
 	 */
 	protected String[] match(String input, String regex) {
 		final Matcher matcher = Pattern.compile(regex).matcher(input);
 		final List<String> matches = new ArrayList<String>();
 
 		while (matcher.find()) {
 			matches.add(matcher.group(0));
 		}
 
 		return matches.toArray(new String[matches.size()]);
 	}
 
 	/**
 	 * @return True if the article has any detected content; false otherwise.
 	 */
 	public boolean hasContent() {
 		return articleContent != null;
 	}
 
 	/**
 	 * Javascript-like String.search
 	 *
 	 * @param input
 	 * @param regex
 	 * @return
 	 */
 	protected int search(String input, String regex) {
 		final Matcher matcher = Pattern.compile(regex).matcher(input);
 
 		if (!matcher.find())
 			return -1;
 		return matcher.start();
 	}
 
 	protected void findArticleEncoding() {
 		final NodeList nl = document.getElementsByTagName("meta");
 		for (int j = 0; j < nl.getLength(); j++) {
 			if (((Element) nl.item(j)).getAttribute("http-equiv").equals("Content-Type")) {
 				article_contentType = ((Element) nl.item(j)).getAttribute("content");
 				return;
 			}
 		}
 
 	}
 
 	protected void findArticleDate() {
 		// <meta name="OriginalPublicationDate" content="2010/07/12 14:08:02"/>
 		// <meta name="DC.date.issued" content="2010-07-12">
 		NodeList nl = document.getElementsByTagName("meta");
 		for (int j = 0; j < nl.getLength(); j++) {
 			if (((Element) nl.item(j)).getAttribute("name").equals("OriginalPublicationDate")) {
 				article_date_string = ((Element) nl.item(j)).getAttribute("content");
 				article_date = DateTime.parse(article_date_string).toDate();
 				return;
 			}
 			if (((Element) nl.item(j)).getAttribute("name").equals("DC.date.issued")) {
 				article_date_string = ((Element) nl.item(j)).getAttribute("content");
 				article_date = DateTime.parse(article_date_string).toDate();
 				return;
 			}
 		}
 
 		// <time datetime="2010-07-12T10:26BST" pubdate>Monday 12 July 2010
 		// 10.26 BST</time>
 		nl = document.getElementsByTagName("time");
 		for (int j = 0; j < nl.getLength(); j++) {
 			if (((Element) nl.item(j)).getAttributeNode("pubdate") != null) {
 				article_date_string = ((Element) nl.item(j)).getAttribute("datetime");
 				article_date = DateTime.parse(article_date_string).toDate();
 				return;
 			}
 		}
 
 		// <span class="date">14:08 GMT, Monday, 12 July 2010 15:08 UK</span>
 		// <p class="date">09.07.2010 @ 17:49 CET</p>
 		// <p class="date">Today @ 09:29 CET</p>
 		nl = document.getElementsByTagName("*");
 		for (int j = 0; j < nl.getLength(); j++) {
 			if ((((Element) nl.item(j)).getAttribute("class").contains("date") ||
 					((Element) nl.item(j)).getAttribute("class").contains("Date")) &&
 					!(((Element) nl.item(j)).getAttribute("class").contains("update") ||
 							((Element) nl.item(j)).getAttribute("class").contains("Update")))
 			{
 				article_date_string = getInnerTextSep(nl.item(j)).trim();
 				parseDate();
 				return;
 			}
 		}
 		for (int j = 0; j < nl.getLength(); j++) {
 			if ((((Element) nl.item(j)).getAttribute("id").contains("date") ||
 					((Element) nl.item(j)).getAttribute("id").contains("Date")) &&
 					!(((Element) nl.item(j)).getAttribute("id").contains("update") ||
 							((Element) nl.item(j)).getAttribute("id").contains("Update")))
 			{
 				article_date_string = getInnerTextSep(nl.item(j)).trim();
 				parseDate();
 				return;
 			}
 		}
 
 		// Last updated at 3:05 PM on 12th July 2010
 		nl = document.getElementsByTagName("*");
 		for (int j = 0; j < nl.getLength(); j++) {
 			final String text = nl.item(j).getTextContent();
 
 			if (text == null)
 				continue;
 
 			final Pattern p = Pattern.compile("Last updated at (\\d+:\\d\\d [AP]M on \\d+[thsndr]+ \\w+ \\d\\d\\d\\d)");
 			final Matcher m = p.matcher(text);
 			if (m.find()) {
 				article_date_string = m.group(1);
 
 				String cpy = article_date_string.replaceAll("th", "");
 				cpy = cpy.replaceAll("st", "");
 				cpy = cpy.replaceAll("nd", "");
 				cpy = cpy.replaceAll("rd", "");
 
 				final SimpleDateFormat sdf = new SimpleDateFormat("h:mm a 'on' dd MMMM yyyy");
 				try {
 					article_date = sdf.parse(cpy);
 				} catch (final ParseException e) {
 				}
 				return;
 			}
 		}
 	}
 
 	@SuppressWarnings("deprecation")
 	protected void parseDate() {
 		if (article_date_string == null || article_date_string.trim().isEmpty())
 			return;
 
 		if (article_date_string.contains("Today")) {
 			try {
 				final SimpleDateFormat sdf = new SimpleDateFormat("'Today @' HH:mm z");
 				article_date = sdf.parse(article_date_string);
 				final Date now = new Date();
 				article_date.setDate(now.getDate());
 				article_date.setMonth(now.getMonth());
 				article_date.setYear(now.getYear());
 			} catch (final ParseException e) {
 			}
 		} else {
 			try {
 				final SimpleDateFormat sdf = new SimpleDateFormat("h:mm z',' E',' dd M yyyy");
 				article_date = sdf.parse(article_date_string);
 			} catch (final ParseException e) {
 				try {
 					final SimpleDateFormat sdf = new SimpleDateFormat("dd.MM.yyyy '@' HH:mm z");
 					article_date = sdf.parse(article_date_string);
 				} catch (final ParseException ee) {
 					try {
 						final SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy");
 						article_date = sdf.parse(article_date_string);
 					} catch (final ParseException eee) {
 						try {
 							article_date = DateTime.parse(article_date_string).toDate();
 						} catch (final IllegalArgumentException ie) {
 						} catch (final java.lang.ArrayIndexOutOfBoundsException ie) {
 							System.out.println(article_date_string);
 						}
 					}
 				}
 			}
 		}
 	}
 
 	/**
 	 * Get the article title.
 	 *
 	 * @return void
 	 **/
 	protected String findArticleTitle() {
 		String curTitle = "", origTitle = "";
 
 		curTitle = origTitle = getTitle();
 
 		//
 		final List<String> potentialTitles = new ArrayList<String>();
 		for (int i = 1; i <= 6; i++) {
 			final NodeList nl = document.getElementsByTagName("h" + i);
 			if (nl.getLength() > 0) {
 				for (int j = 0; j < nl.getLength(); j++)
 					potentialTitles.add(nl.item(j).getTextContent().trim());
 			}
 		}
 
 		String potentialTitle = null;
 		int score = 0;
 		for (final String s : potentialTitles) {
 			if (s.length() > score && curTitle.contains(s)) {
 				potentialTitle = s;
 				score = s.length();
 			}
 		}
 		if (potentialTitle != null)
 			return potentialTitle;
 		//
 
 		if (match(curTitle, " [" + Regexps.titleSeparatorRe + "]+ ").length > 0)
 		{
 			curTitle = origTitle.replaceAll("(.*) [" + Regexps.titleSeparatorRe + "]+ .*", "$1");
 
 			if (curTitle.split(" ").length < 3) {
 				curTitle = origTitle.replaceAll("(?i)[^" + Regexps.titleSeparatorRe + "]*[" + Regexps.titleSeparatorRe
 						+ "]+(.*)", "$1");
 			}
 		}
 		else if (curTitle.indexOf(": ") != -1)
 		{
 			curTitle = origTitle.replaceAll("(?i).*:(.*)", "$1");
 
 			if (curTitle.split(" ").length < 3) {
 				curTitle = origTitle.replaceAll("(?i)[^:]*[:](.*)", "$1");
 			}
 		}
 		else if (curTitle.length() > 150 || curTitle.length() < 15)
 		{
 			final NodeList hOnes = document.getElementsByTagName("h1");
 			if (hOnes.getLength() == 1)
 			{
 				curTitle = getInnerText((Element) hOnes.item(0));
 			}
 		}
 
 		curTitle = curTitle.replaceAll(Regexps.trimRe, "");
 
 		if (curTitle.split(" ").length <= 3) {
 			curTitle = origTitle;
 		}
 
 		return curTitle;
 	}
 
 	/**
 	 * Equivalent to document.body in JS
 	 *
 	 * @return
 	 */
 	protected Element getBody() {
 		final NodeList nl = document.getElementsByTagName("body");
 
 		if (nl.getLength() == 0)
 			return null;
 		else
 			return (Element) nl.item(0);
 	}
 
 	/**
 	 * Runs readability.
 	 *
 	 * Workflow: 1. Prep the document by removing script tags, css, etc. 2.
 	 * Build readability"s DOM tree. 3. Grab the article content from the
 	 * current dom tree. 4. Replace the current DOM tree with the new one. 5.
 	 * Read peacefully.
 	 *
 	 **/
 	protected void init() {
 		if (getBody() != null && bodyCache == null) {
 			bodyCache = getBody().cloneNode(true);
 		}
 
 		findArticleDate(); // must be done before prepDocument()
 
 		findArticleEncoding();
 
 		prepDocument();
 
 		/* Build readability"s DOM tree */
 		articleTitle = findArticleTitle();
 		articleContent = grabArticle();
 
 		/**
 		 * If we attempted to strip unlikely candidates on the first run
 		 * through, and we ended up with no content, that may mean we stripped
 		 * out the actual content so we couldn"t parse it. So re-run init while
 		 * preserving unlikely candidates to have a better shot at getting our
 		 * content out properly.
 		 **/
 		if (getInnerText(articleContent, false).length() < 250)
 		{
 			if (flags.contains(Flag.FLAG_STRIP_UNLIKELYS)) {
 				flags.remove(Flag.FLAG_STRIP_UNLIKELYS);
 				getBody().getParentNode().replaceChild(bodyCache, getBody());
 				init();
 				return;
 			}
 			else if (flags.contains(Flag.FLAG_WEIGHT_CLASSES)) {
 				flags.remove(Flag.FLAG_WEIGHT_CLASSES);
 				getBody().getParentNode().replaceChild(bodyCache, getBody());
 				init();
 				return;
 			}
 			else {
 				articleContent = null;
 			}
 		}
 
 		if (addTitle && articleContent != null) {
 			final Element titleNode = document.createElement("h1");
 			titleNode.setAttribute("id", "title");
 			titleNode.appendChild(document.createTextNode(getArticleTitle()));
 			articleContent.insertBefore(titleNode, articleContent.getFirstChild());
 		}
 	}
 
 	/**
 	 * Prepare the HTML document for readability to scrape it. This includes
 	 * things like stripping javascript, CSS, and handling terrible markup.
 	 *
 	 **/
 	protected void prepDocument() {
 		/**
 		 * In some cases a body element can"t be found (if the HTML is totally
 		 * hosed for example) so we create a new body node and append it to the
 		 * document.
 		 */
 		if (getBody() == null)
 		{
 			final Node body = document.createElement("body");
 			document.appendChild(body);
 		}
 
 		// frames are not supported in this version!
 		// NodeList frames = document.getElementsByTagName("frame");
 		// if(frames.length > 0)
 		// {
 		// Node bestFrame = null;
 		// int bestFrameSize = 0;
 		// for(int frameIndex = 0; frameIndex < frames.getLength();
 		// frameIndex++)
 		// {
 		// int frameSize = frames.item(frameIndex).offsetWidth +
 		// frames[frameIndex].offsetHeight;
 		// var canAccessFrame = false;
 		// try {
 		// frames[frameIndex].contentWindow.document.body;
 		// canAccessFrame = true;
 		// }
 		// catch(eFrames) {
 		// dbg(eFrames);
 		// }
 		//
 		// if(canAccessFrame && frameSize > bestFrameSize)
 		// {
 		// bestFrame = frames[frameIndex];
 		// bestFrameSize = frameSize;
 		// }
 		// }
 		//
 		// if(bestFrame)
 		// {
 		// var newBody = document.createElement("body");
 		// newBody.innerHTML = bestFrame.contentWindow.document.body.innerHTML;
 		// newBody.style.overflow = "scroll";
 		// document.body = newBody;
 		//
 		// var frameset = document.getElementsByTagName("frameset")[0];
 		// if(frameset) {
 		// frameset.parentNode.removeChild(frameset); }
 		//
 		// readability.frameHack = true;
 		// }
 		// }
 
 		/* remove all scripts that are not readability */
 		final NodeList scripts = document.getElementsByTagName("script");
 		for (int i = scripts.getLength() - 1; i >= 0; i--)
 		{
 			scripts.item(i).getParentNode().removeChild(scripts.item(i));
 		}
 
 		/* Remove all style tags in head */
 		final NodeList styleTags = document.getElementsByTagName("style");
 		for (int st = styleTags.getLength() - 1; st >= 0; st--) {
 			styleTags.item(st).getParentNode().removeChild(styleTags.item(st));
 		}
 
 		/* Remove all meta tags */
 		final NodeList metaTags = document.getElementsByTagName("meta");
 		for (int mt = metaTags.getLength() - 1; mt >= 0; mt--) {
 			metaTags.item(mt).getParentNode().removeChild(metaTags.item(mt));
 		}
 
 		/* Turn all double br's into p's */
 		/*
 		 * Note, this is pretty costly as far as processing goes. Maybe optimize
 		 * later.
 		 */
 		// document.body.innerHTML =
 		// document.body.innerHTML.replace(readability.regexps.replaceBrsRe,
 		// '</p><p>').replace(readability.regexps.replaceFontsRe, '<$1span>');
 		final Element body = getBody();
 		// Node rep =
 		// stringToNode(nodeToString(body).replaceAll(Regexps.replaceBrsRe,
 		// "</P><P>").replaceAll(Regexps.replaceFontsRe, "<$1span>"));
 		// body.getParentNode().replaceChild(rep, body);
 
 		// This is slow!
 		final Node frag = stringToNode(getInnerHTML(body).replaceAll(Regexps.replaceBrsRe, "</P><P>").replaceAll(
 				Regexps.replaceFontsRe, "<$1span>"));
 		removeChildren(body);
 		body.appendChild(frag);
 
 		/* Remove all comments */
 		removeComments(document);
 	}
 
 	protected void removeComments(Node n) {
 		if (n.getNodeType() == Node.COMMENT_NODE) {
 			n.getParentNode().removeChild(n);
 		} else {
 			final NodeList nl = n.getChildNodes();
 			for (int i = nl.getLength() - 1; i >= 0; i--)
 				removeComments(nl.item(i));
 		}
 	}
 
 	/**
 	 * Prepare the article node for display. Clean out any inline styles,
 	 * iframes, forms, strip extraneous
 	 * <p>
 	 * tags, etc.
 	 *
 	 * @param Element
 	 **/
 	protected void prepArticle(Element articleContent) {
 		cleanStyles(articleContent);
 		killBreaks(articleContent);
 
 		/* Clean out junk from the article content */
 		clean(articleContent, "form");
 		clean(articleContent, "object");
 		clean(articleContent, "h1");
 		/**
 		 * If there is only one h2, they are probably using it as a header and
 		 * not a subheader, so remove it since we already have a header.
 		 ***/
 		if (articleContent.getElementsByTagName("h2").getLength() == 1) {
 			clean(articleContent, "h2");
 		}
 		clean(articleContent, "iframe");
 
 		cleanHeaders(articleContent);
 
 		/*
 		 * Do these last as the previous stuff may have removed junk that will
 		 * affect these
 		 */
 		cleanConditionally(articleContent, "table");
 		cleanConditionally(articleContent, "ul");
 		cleanConditionally(articleContent, "div");
 
 		/* Remove extra paragraphs */
 		final NodeList articleParagraphs = articleContent.getElementsByTagName("p");
 		for (int i = articleParagraphs.getLength() - 1; i >= 0; i--)
 		{
 			final int imgCount = ((Element) articleParagraphs.item(i)).getElementsByTagName("img").getLength();
 			final int embedCount = ((Element) articleParagraphs.item(i)).getElementsByTagName("embed").getLength();
 			final int objectCount = ((Element) articleParagraphs.item(i)).getElementsByTagName("object").getLength();
 
 			if (imgCount == 0 && embedCount == 0 && objectCount == 0
 					&& getInnerText((Element) articleParagraphs.item(i), false) == "")
 			{
 				articleParagraphs.item(i).getParentNode().removeChild(articleParagraphs.item(i));
 			}
 		}
 
 		// articleContent.innerHTML =
 		// articleContent.innerHTML.replace(/<br[^>]*>\s*<p/gi, "<p");
 		final Node n = stringToNode(getInnerHTML(articleContent).replaceAll("(?i)<br[^>]*>\\s*<p", "<P"));
 		removeChildren(articleContent);
 		articleContent.appendChild(n);
 
 		// now remove empty p's and tidy up
 		final NodeList nl = articleContent.getElementsByTagName("p");
 		for (int i = nl.getLength() - 1; i >= 0; i--) {
 			if (nl.item(i).getTextContent().trim().length() == 0)
 			{
 				nl.item(i).getParentNode().removeChild(nl.item(i));
 			} else if (nl.item(i).getChildNodes().getLength() == 1
 					&& nl.item(i).getChildNodes().item(0).getNodeType() == Node.TEXT_NODE)
 			{
 				nl.item(i).setTextContent("\n" + nl.item(i).getTextContent().trim() + "\n");
 			}
 			else if (((Element) nl.item(i)).getAttribute("class").equals("readability-styled"))
 			{
 				nl.item(i).getParentNode().replaceChild(document.createTextNode(nl.item(i).getTextContent()), nl.item(i));
 			}
 		}
 
 	}
 
 	protected void removeChildren(Node n) {
 		final NodeList nl = n.getChildNodes();
 		final int nn = nl.getLength();
 		for (int i = 0; i < nn; i++)
 			n.removeChild(nl.item(0));
 	}
 
 	/**
 	 * Initialize a node with the readability object. Also checks the
 	 * className/id for special names to add to its score.
 	 *
 	 * @param Element
 	 **/
 	protected void initializeNode(Element node) {
 		float contentScore = 0;
 
 		if (node.getTagName() == "DIV") {
 			contentScore += 5;
 		} else if (node.getTagName() == "PRE" || node.getTagName() == "TD" || node.getTagName() == "BLOCKQUOTE") {
 			contentScore += 3;
 		} else if (node.getTagName() == "ADDRESS" || node.getTagName() == "OL" || node.getTagName() == "UL"
 				|| node.getTagName() == "DL" || node.getTagName() == "DD" || node.getTagName() == "DT"
 				|| node.getTagName() == "LI" || node.getTagName() == "FORM")
 		{
 			contentScore -= 3;
 		} else if (node.getTagName() == "H1" || node.getTagName() == "H2" || node.getTagName() == "H3"
 				|| node.getTagName() == "H4" || node.getTagName() == "H5" || node.getTagName() == "H6"
 				|| node.getTagName() == "TH")
 		{
 			contentScore -= 5;
 		}
 
 		contentScore += getClassWeight(node);
 		node.setUserData("readability", contentScore, null);
 	}
 
 	/**
 	 * Get an elements class/id weight. Uses regular expressions to tell if this
 	 * element looks good or bad.
 	 *
 	 * @param Element
 	 * @return number (Integer)
 	 **/
 	protected int getClassWeight(Element e) {
 		if (!flags.contains(Flag.FLAG_WEIGHT_CLASSES)) {
 			return 0;
 		}
 
 		int weight = 0;
 
 		/* Look for a special classname */
 		if (e.getAttribute("class") != "")
 		{
 			if (search(e.getAttribute("class"), Regexps.negativeRe) != -1) {
 				weight -= 25;
 			}
 
 			if (search(e.getAttribute("class"), Regexps.positiveRe) != -1) {
 				weight += 25;
 			}
 		}
 
 		/* Look for a special ID */
 		if (e.getAttribute("id") != "")
 		{
 			if (search(e.getAttribute("id"), Regexps.negativeRe) != -1) {
 				weight -= 25;
 			}
 
 			if (search(e.getAttribute("id"), Regexps.positiveRe) != -1) {
 				weight += 25;
 			}
 		}
 
 		return weight;
 	}
 
 	protected void cleanStyles() {
 		cleanStyles((Element) document);
 	}
 
 	/**
 	 * Remove the style attribute on every e and under. TODO: Test if
 	 * getElementsByTagName(*) is faster.
 	 *
 	 * @param Element
 	 **/
 	protected void cleanStyles(Element e) {
 		if (e == null)
 			return;
 		Node cur = e.getFirstChild();
 
 		// Remove any root styles, if we"re able.
 		if (!e.getAttribute("class").equals("readability-styled"))
 			e.removeAttribute("style");
 
 		// Go until there are no more child nodes
 		while (cur != null) {
 			if (cur.getNodeType() == Element.ELEMENT_NODE) {
 				// Remove style attribute(s) :
 				if (!((Element) cur).getAttribute("class").equals("readability-styled")) {
 					((Element) cur).removeAttribute("style");
 				}
 				cleanStyles((Element) cur);
 			}
 			cur = cur.getNextSibling();
 		}
 	}
 
 	/**
 	 * Remove extraneous break tags from a node.
 	 *
 	 * @param Element
 	 **/
 	protected void killBreaks(Element e) {
 		// e.innerHTML =
 		// e.innerHTML.replace(readability.regexps.killBreaksRe,"<br />");
 
 		final Node n = stringToNode(getInnerHTML(e).replaceAll(Regexps.killBreaksRe, "<BR />"));
 		removeChildren(e);
 		e.appendChild(n);
 	}
 
 	/**
 	 * Clean a node of all elements of type "tag". (Unless it"s a youtube/vimeo
 	 * video. People love movies.)
 	 *
 	 * @param Element
 	 * @param string
 	 *            tag to clean
 	 **/
 	protected void clean(Element e, String tag) {
 		final NodeList targetList = e.getElementsByTagName(tag);
 		final boolean isEmbed = (tag.equals("object") || tag.equals("embed"));
 
 		for (int y = targetList.getLength() - 1; y >= 0; y--) {
 			/*
 			 * Allow youtube and vimeo videos through as people usually want to
 			 * see those.
 			 */
 			if (isEmbed) {
 				String attributeValues = "";
 				for (int i = 0, il = targetList.item(y).getAttributes().getLength(); i < il; i++) {
 					attributeValues += targetList.item(y).getAttributes().item(i).getNodeValue() + "|";
 				}
 
 				/*
 				 * First, check the elements attributes to see if any of them
 				 * contain youtube or vimeo
 				 */
 				if (search(attributeValues, Regexps.videoRe) != -1) {
 					continue;
 				}
 
 				/* Then check the elements inside this element for the same. */
 				if (search(getInnerHTML(targetList.item(y)), Regexps.videoRe) != -1) {
 					continue;
 				}
 			}
 
 			targetList.item(y).getParentNode().removeChild(targetList.item(y));
 		}
 	}
 
 	/**
 	 * Clean out spurious headers from an Element. Checks things like classnames
 	 * and link density.
 	 *
 	 * @param Element
 	 **/
 	protected void cleanHeaders(Element e) {
 		for (int headerIndex = 1; headerIndex < 7; headerIndex++) {
 			final NodeList headers = e.getElementsByTagName("h" + headerIndex);
 			for (int i = headers.getLength() - 1; i >= 0; i--) {
 				if (getClassWeight((Element) headers.item(i)) < 0
 						|| getLinkDensity((Element) headers.item(i)) > LINK_DENSITY_THRESHOLD)
 				{
 					headers.item(i).getParentNode().removeChild(headers.item(i));
 				}
 			}
 		}
 	}
 
 	/**
 	 * Get the density of links as a percentage of the content This is the
 	 * amount of text that is inside a link divided by the total text in the
 	 * node.
 	 *
 	 * @param Element
 	 * @return number (float)
 	 **/
 	protected float getLinkDensity(Element e) {
 		final NodeList links = e.getElementsByTagName("a");
 		final int textLength = getInnerText(e).length();
 		int linkLength = 0;
 
 		for (int i = 0, il = links.getLength(); i < il; i++)
 		{
 			linkLength += getInnerText((Element) links.item(i)).length();
 		}
 
 		if (linkLength == 0)
 			return 0;
 
 		return (float) linkLength / (float) textLength;
 	}
 
 	/**
 	 * Clean an element of all tags of type "tag" if they look fishy. "Fishy" is
 	 * an algorithm based on content length, classnames, link density, number of
 	 * images & embeds, etc.
 	 **/
 	protected void cleanConditionally(Element e, String tag) {
 		final NodeList tagsList = e.getElementsByTagName(tag);
 		final int curTagsLength = tagsList.getLength();
 
 		/**
 		 * Gather counts for other typical elements embedded within. Traverse
 		 * backwards so we can remove nodes at the same time without effecting
 		 * the traversal.
 		 *
 		 * Todo: Consider taking into account original contentScore here.
 		 **/
 		for (int i = curTagsLength - 1; i >= 0; i--) {
 			final int weight = getClassWeight((Element) tagsList.item(i));
 			final float contentScore = (tagsList.item(i).getUserData("readability") != null) ? (Float) (tagsList.item(i)
 					.getUserData("readability")) : 0;
 
 			dbg("Cleaning Conditionally "
 					+ tagsList.item(i)
 					+ " ("
 					+ ((Element) tagsList.item(i)).getAttribute("class")
 					+ ":"
 					+ ((Element) tagsList.item(i)).getAttribute("id")
 					+ ")"
 					+ ((tagsList.item(i).getUserData("readability") != null) ? (" with score " + tagsList.item(i)
 							.getUserData("readability")) : ""));
 
 			if (weight + contentScore < 0)
 			{
 				dbg("Removing " + tagsList.item(i) + " (" + ((Element) tagsList.item(i)).getAttribute("class") + ":"
 						+ ((Element) tagsList.item(i)).getAttribute("id") + ")");
 				tagsList.item(i).getParentNode().removeChild(tagsList.item(i));
 			}
 			else if (getCharCount((Element) tagsList.item(i), ",") < 10) {
 				/**
 				 * If there are not very many commas, and the number of
 				 * non-paragraph elements is more than paragraphs or other
 				 * ominous signs, remove the element.
 				 **/
 				final int p = ((Element) tagsList.item(i)).getElementsByTagName("p").getLength();
 				final int img = ((Element) tagsList.item(i)).getElementsByTagName("img").getLength();
 				final int li = ((Element) tagsList.item(i)).getElementsByTagName("li").getLength() - 100;
 				final int input = ((Element) tagsList.item(i)).getElementsByTagName("input").getLength();
 
 				int embedCount = 0;
 				final NodeList embeds = ((Element) tagsList.item(i)).getElementsByTagName("embed");
 				for (int ei = 0, il = embeds.getLength(); ei < il; ei++) {
 					if (search(((Element) embeds.item(ei)).getAttribute("src"), Regexps.videoRe) == -1) {
 						embedCount++;
 					}
 				}
 
 				final float linkDensity = getLinkDensity((Element) tagsList.item(i));
 				final int contentLength = getInnerText((Element) tagsList.item(i)).length();
 				boolean toRemove = false;
 
 				if (img > p) {
 					toRemove = true;
 				} else if (li > p && tag != "ul" && tag != "ol") {
 					toRemove = true;
 				} else if (input > Math.floor(p / 3)) {
 					toRemove = true;
 				} else if (contentLength < 25 && (img == 0 || img > 2)) {
 					toRemove = true;
 				} else if (weight < 25 && linkDensity > 0.2) {
 					toRemove = true;
 				} else if (weight >= 25 && linkDensity > 0.5) {
 					toRemove = true;
 				} else if ((embedCount == 1 && contentLength < 75) || embedCount > 1) {
 					toRemove = true;
 				}
 
 				if (img == 1 && p == 0 && contentLength == 0) {
 					final Element theImg = (Element) ((Element) tagsList.item(i)).getElementsByTagName("img").item(0);
 
 					String w = "";
 					if (theImg.getAttribute("width") != null)
 						w = theImg.getAttribute("width");
 
 					String h = "";
 					if (theImg.getAttribute("height") != null)
 						h = theImg.getAttribute("height");
 
 					if (!(w.equals("0") || h.equals("0")))
 						toRemove = false; // special case - it's just an inline
 					// image
 				}
 
 				if (toRemove) {
 					dbg("Removing " + tagsList.item(i) + " (" + ((Element) tagsList.item(i)).getAttribute("class") + ":"
 							+ ((Element) tagsList.item(i)).getAttribute("id") + ")");
 					tagsList.item(i).getParentNode().removeChild(tagsList.item(i));
 				}
 			}
 		}
 	}
 
 	/**
 	 * Get the number of times a string s appears in the node e.
 	 *
 	 * @param Element
 	 * @param string
 	 *            - what to split on. Default is ","
 	 * @return number (integer)
 	 **/
 	protected int getCharCount(Element e, String s) {
 		return getInnerText(e).split(s).length - 1;
 	}
 
 	protected int getCharCount(Element e) {
 		return getCharCount(e, ",");
 	}
 
 	/**
 	 * @return The article title
 	 */
 	public String getArticleTitle() {
 		return articleTitle;
 	}
 
 	/**
 	 * @return The content type of the article
 	 */
 	public String getArticleContentType() {
 		return article_contentType;
 	}
 
 	/***
 	 * grabArticle - Using a variety of metrics (content score, classname,
 	 * element types), find the content that is most likely to be the stuff a
 	 * user wants to read. Then return it wrapped up in a div.
 	 *
 	 * @return Element
 	 **/
 	protected Element grabArticle() {
 		final boolean stripUnlikelyCandidates = flags.contains(Flag.FLAG_STRIP_UNLIKELYS);
 
 		/**
 		 * First, node prepping. Trash nodes that look cruddy (like ones with
 		 * the class name "comment", etc), and turn divs into P tags where they
 		 * have been used inappropriately (as in, where they contain no other
 		 * block level elements.)
 		 *
 		 * Note: Assignment from index for performance. See
 		 * http://www.peachpit.com/articles/article.aspx?p=31567&seqNum=5 Todo:
 		 * Shouldn't this be a reverse traversal?
 		 **/
 		Element node = null;
 		final List<Element> nodesToScore = new ArrayList<Element>();
 		for (int nodeIndex = 0; (node = (Element) document.getElementsByTagName("*").item(nodeIndex)) != null; nodeIndex++)
 		{
 			/* Remove unlikely candidates */
 			if (stripUnlikelyCandidates) {
 				final String unlikelyMatchString = node.getAttribute("class") + node.getAttribute("id");
 				if (search(unlikelyMatchString, Regexps.unlikelyCandidatesRe) != -1 &&
 						search(unlikelyMatchString, Regexps.okMaybeItsACandidateRe) == -1 &&
 						!node.getTagName().equals("BODY"))
 				{
 					dbg("Removing unlikely candidate - " + unlikelyMatchString);
 					node.getParentNode().removeChild(node);
 					nodeIndex--;
 					continue;
 				}
 			}
 
 			if (node.getTagName().equals("P") || node.getTagName().equals("TD")) {
 				nodesToScore.add(node);
 			}
 
 			/*
 			 * Turn all divs that don't have children block level elements into
 			 * p's
 			 */
 			if (node.getTagName().equals("DIV")) {
 
 				if (search(getInnerHTML(node), Regexps.divToPElementsRe) == -1) {
 					dbg("Altering div to p");
 					final Element newNode = document.createElement("P");
 
 					// newNode.innerHTML = node.innerHTML;
 					final NodeList nl = node.getChildNodes();
 					for (int i = 0; i < nl.getLength(); i++)
 						newNode.appendChild(nl.item(i));
 
 					node.getParentNode().replaceChild(newNode, node);
 					nodeIndex--;
 				}
 				else
 				{
 					/* EXPERIMENTAL */
 					for (int i = 0, il = node.getChildNodes().getLength(); i < il; i++) {
 						final Node childNode = node.getChildNodes().item(i);
 						if (childNode.getNodeType() == Element.TEXT_NODE) {
 							dbg("replacing text node with a p tag with the same content.");
 							final Element p = document.createElement("p");
 							// p.innerHTML = childNode.nodeValue;
 							p.setNodeValue(childNode.getNodeValue());
 							p.setTextContent(childNode.getTextContent());
 							// p.style.display = "inline";
 							p.setAttribute("class", "readability-styled");
 							childNode.getParentNode().replaceChild(p, childNode);
 						}
 					}
 				}
 			}
 		}
 
 		/**
 		 * Loop through all paragraphs, and assign a score to them based on how
 		 * content-y they look. Then add their score to their parent node.
 		 *
 		 * A score is determined by things like number of commas, class names,
 		 * etc. Maybe eventually link density.
 		 **/
 		final List<Element> candidates = new ArrayList<Element>();
 		for (int pt = 0; pt < nodesToScore.size(); pt++) {
 			final Element parentNode = (Element) nodesToScore.get(pt).getParentNode();
 			final Element grandParentNode = (Element) parentNode.getParentNode();
 			final String innerText = getInnerText(nodesToScore.get(pt));
 
 			/*
 			 * If this paragraph is less than 25 characters, don't even count
 			 * it.
 			 */
 			if (innerText.length() < 25) {
 				continue;
 			}
 
 			/* Initialize readability data for the parent. */
 			if (parentNode.getUserData("readability") == null)
 			{
 				initializeNode(parentNode);
 				candidates.add(parentNode);
 			}
 
 			/* Initialize readability data for the grandparent. */
 			if (grandParentNode.getUserData("readability") == null)
 			{
 				initializeNode(grandParentNode);
 				candidates.add(grandParentNode);
 			}
 
 			float contentScore = 0;
 
 			/* Add a point for the paragraph itself as a base. */
 			contentScore++;
 
 			/* Add points for any commas within this paragraph */
 			contentScore += innerText.split(",").length;
 
 			/*
 			 * For every 100 characters in this paragraph, add another point. Up
 			 * to 3 points.
 			 */
 			contentScore += Math.min(Math.floor(innerText.length() / 100F), 3F);
 
 			/* Add the score to the parent. The grandparent gets half. */
 			parentNode.setUserData("readability", ((Float) (parentNode.getUserData("readability")) + contentScore), null);
 			grandParentNode.setUserData("readability", ((Float) (grandParentNode.getUserData("readability")))
 					+ (contentScore / 2F), null);
 		}
 
 		/**
 		 * After we've calculated scores, loop through all of the possible
 		 * candidate nodes we found and find the one with the highest score.
 		 **/
 		Element topCandidate = null;
 		for (int c = 0, cl = candidates.size(); c < cl; c++)
 		{
 			/**
 			 * Scale the final candidates score based on link density. Good
 			 * content should have a relatively small link density (5% or less)
 			 * and be mostly unaffected by this operation.
 			 **/
 
 			candidates.get(c).setUserData("readability",
 					(Float) (candidates.get(c).getUserData("readability")) * (1F - getLinkDensity(candidates.get(c))),
 					null);
 
 			dbg("Candidate: " + candidates.get(c) + " (" + candidates.get(c).getAttribute("class") + ":"
 					+ candidates.get(c).getAttribute("id") + ") with score "
 					+ candidates.get(c).getUserData("readability"));
 
 			if (topCandidate == null
 					|| (Float) (candidates.get(c).getUserData("readability")) > ((Float) topCandidate
 							.getUserData("readability")))
 			{
 				topCandidate = candidates.get(c);
 			}
 		}
 
 		if (topCandidate != null)
 			dbg("==> TOP Candidate: " + topCandidate + " (" + topCandidate.getAttribute("class") + ":"
 					+ topCandidate.getAttribute("id") + ") with score " + topCandidate.getUserData("readability"));
 
 		/**
 		 * If we still have no top candidate, just use the body as a last
 		 * resort. We also have to copy the body node so it is something we can
 		 * modify.
 		 **/
 		if (topCandidate == null || topCandidate.getTagName().equals("BODY"))
 		{
 			topCandidate = document.createElement("DIV");
 
 			// topCandidate.innerHTML = document.body.innerHTML;
 			final NodeList nl = getBody().getChildNodes();
 			for (int i = 0; i < nl.getLength(); i++)
 				topCandidate.appendChild(nl.item(i));
 			// document.body.innerHTML = ""; //should be covered by above
 
 			getBody().appendChild(topCandidate);
 			initializeNode(topCandidate);
 		}
 
 		/**
 		 * Now that we have the top candidate, look through its siblings for
 		 * content that might also be related. Things like preambles, content
 		 * split by ads that we removed, etc.
 		 **/
 		final Element articleContent = document.createElement("DIV");
 		articleContent.setAttribute("id", "readability-content");
 		final float siblingScoreThreshold = Math.max(10F, (Float) topCandidate.getUserData("readability") * 0.2F);
 		final NodeList siblingNodes = topCandidate.getParentNode().getChildNodes();
 
 		for (int s = 0, sl = siblingNodes.getLength(); s < sl; s++)
 		{
 			final Node siblingNode = siblingNodes.item(s);
 			boolean append = false;
 
 			if (siblingNode instanceof Element)
 				dbg("Looking at sibling node: "
 						+ siblingNode
 						+ " ("
 						+ ((Element) siblingNode).getAttribute("class")
 						+ ":"
 						+ ((Element) siblingNode).getAttribute("id")
 						+ ")"
 						+ ((siblingNode.getUserData("readability") != null) ? (" with score " + siblingNode
 								.getUserData("readability")) : ""));
 			dbg("Sibling has score "
 					+ (siblingNode.getUserData("readability") != null ? siblingNode.getUserData("readability")
 							: "Unknown"));
 
 			if (siblingNode == topCandidate)
 			{
 				append = true;
 			}
 
 			float contentBonus = 0;
 			/*
 			 * Give a bonus if sibling nodes and top candidates have the example
 			 * same classname
 			 */
 			if (siblingNode instanceof Element
 					&& ((Element) siblingNode).getAttribute("class").equals(topCandidate.getAttribute("class"))
 					&& !topCandidate.getAttribute("class").equals(""))
 			{
 				contentBonus += (Float) topCandidate.getUserData("readability") * 0.2F;
 			}
 
 			if (siblingNode.getUserData("readability") != null
 					&& ((Float) siblingNode.getUserData("readability") + contentBonus) >= siblingScoreThreshold)
 			{
 				append = true;
 			}
 
 			if (siblingNode.getNodeName().equals("P")) {
 				final float linkDensity = getLinkDensity((Element) siblingNode);
 				final String nodeContent = getInnerText((Element) siblingNode);
 				final int nodeLength = nodeContent.length();
 
 				if (nodeLength > 80 && linkDensity < 0.25)
 				{
 					append = true;
 				}
 				else if (nodeLength < 80 && linkDensity == 0 && search(nodeContent, "\\.( |$)") != -1)
 				{
 					append = true;
 				}
 			}
 
 			if (append)
 			{
 				dbg("Appending node: " + siblingNode);
 
 				Node nodeToAppend = null;
 				if (!siblingNode.getNodeName().equals("DIV") && !siblingNode.getNodeName().equals("P")) {
 					/*
 					 * We have a node that isn't a common block level element,
 					 * like a form or td tag. Turn it into a div so it doesn't
 					 * get filtered out later by accident.
 					 */
 
 					dbg("Altering siblingNode of " + siblingNode.getNodeName() + " to div.");
 					nodeToAppend = document.createElement("div");
 					if (siblingNode instanceof Element)
 						((Element) nodeToAppend).setAttribute("id", ((Element) siblingNode).getAttribute("id"));
 
 					// nodeToAppend.innerHTML = siblingNode.innerHTML;
 					final NodeList nl = siblingNode.getChildNodes();
 					for (int i = 0; i < nl.getLength(); i++)
 						nodeToAppend.appendChild(nl.item(i));
 				} else {
 					nodeToAppend = siblingNode;
 					s--;
 					sl--;
 				}
 
 				/*
 				 * To ensure a node does not interfere with readability styles,
 				 * remove its classnames
 				 */
 				if (nodeToAppend instanceof Element)
 					((Element) nodeToAppend).setAttribute("class", "");
 
 				/*
 				 * Append sibling and subtract from our list because it removes
 				 * the node when you append to another node
 				 */
 				articleContent.appendChild(nodeToAppend);
 			}
 		}
 
 		/**
 		 * So we have all of the content that we need. Now we clean it up for
 		 * presentation.
 		 **/
 		prepArticle(articleContent);
 
 		return articleContent;
 	}
 
 	protected String getInnerHTML(Node n) {
 		if (n.getNodeType() == Node.TEXT_NODE)
 			return n.getTextContent();
 
 		String result = "";
 		final NodeList nl = n.getChildNodes();
 		for (int i = 0; i < nl.getLength(); i++) {
 			if (nl.item(i).getNodeType() == Node.TEXT_NODE)
 				result += nl.item(i).getTextContent();
 			else if (nl.item(i).getNodeType() == Node.COMMENT_NODE)
 				result += "<!-- " + nl.item(i).getTextContent() + " -->";
 			else
 				result += nodeToString(nl.item(i));
 		}
 
 		return result;
 	}
 
 	protected String nodeToString(Node n) {
 		return nodeToString(n, false);
 	}
 
 	protected static String nodeToString(Node n, boolean pretty) {
 		try {
 			final DOMImplementationRegistry registry = DOMImplementationRegistry.newInstance();
 			final DOMImplementationLS impl = (DOMImplementationLS) registry.getDOMImplementation("LS");
 			final LSSerializer writer = impl.createLSSerializer();
 
 			writer.getDomConfig().setParameter("xml-declaration", false);
 			if (pretty) {
 				writer.getDomConfig().setParameter("format-pretty-print", true);
 			}
 
 			return writer.writeToString(n);
 		} catch (final Exception e) {
 			throw new RuntimeException(e);
 		}
 	}
 
 	protected Node stringToNode(String str) {
 		try {
 			final DOMFragmentParser parser = new DOMFragmentParser();
 			final DocumentFragment fragment = document.createDocumentFragment();
 			parser.parse(new InputSource(new StringReader(str)), fragment);
 			return fragment;
 
 			// try and return the element itself if possible...
 			// NodeList nl = fragment.getChildNodes();
 			// for (int i=0; i<nl.getLength(); i++) if (nl.item(i).getNodeType()
 			// == Node.ELEMENT_NODE) return nl.item(i);
 			// return fragment;
 
 		} catch (final Exception e) {
 			throw new RuntimeException(e);
 		}
 	}
 
 	/**
 	 * Get the inner text of a node - cross browser compatibly. This also strips
 	 * out any excess whitespace to be found.
 	 *
 	 * @param Element
 	 * @return string
 	 **/
 	protected String getInnerText(Element e, boolean normalizeSpaces) {
 		String textContent = "";
 
 		textContent = e.getTextContent().replaceAll(Regexps.trimRe, "");
 
 		if (normalizeSpaces) {
 			return textContent.replaceAll(Regexps.normalizeRe, " ");
 		} else {
 			return textContent;
 		}
 	}
 
 	protected String getInnerTextSep(Node e) {
 		if (e.hasChildNodes()) {
 			String s = "";
 			final NodeList nl = e.getChildNodes();
 			for (int i = 0; i < nl.getLength(); i++) {
 				if (!nl.item(i).getNodeName().equalsIgnoreCase("script"))
 					s += getInnerTextSep(nl.item(i));
 			}
 			return s;
 		} else {
 			return e.getTextContent() + " ";
 		}
 	}
 
 	protected String getInnerText(Element e) {
 		return getInnerText(e, true);
 	}
 
 	/**
 	 * @return The article HTML content as a {@link String}.
 	 */
 	public String getArticleHTML() {
 		if (articleContent == null)
 			return "";
 		return nodeToString(articleContent, true);
 	}
 
 	/**
 	 * @return The articles HTML dom node.
 	 */
 	public Node getArticleHTML_DOM() {
 		return articleContent;
 	}
 
 	protected String getArticleDateString() {
 		return article_date_string;
 	}
 
 	/**
 	 * @return The article date.
 	 */
 	public Date getArticleDate() {
 		return article_date;
 	}
 
 	/**
 	 * @return The text of the article.
 	 */
 	public String getArticleText() {
 		if (articleContent == null)
 			return "Unable to find article content";
 		// return getInnerText(articleContent, false);
 		return articleContent.getTextContent().trim().replaceAll("[\r|\n|\r\n]{2,}", "\n\n").replaceAll(" {2,}", " ");
 	}
 
 	/**
 	 * @return Any links in the article.
 	 */
 	public List<Anchor> getArticleLinks() {
 		final List<Anchor> anchors = new ArrayList<Anchor>();
 		if (articleContent == null)
 			return anchors;
 
 		final NodeList nl = articleContent.getElementsByTagName("a");
 		for (int i = 0; i < nl.getLength(); i++) {
 			final Element a = (Element) nl.item(i);
 
 			final Anchor anchor = new Anchor(getInnerText(a), a.getAttribute("href"));
 			anchors.add(anchor);
 		}
 		return anchors;
 	}
 
 	/**
 	 * @return Any links in the document.
 	 */
 	public List<Anchor> getAllLinks() {
 		final List<Anchor> anchors = new ArrayList<Anchor>();
 
 		final NodeList nl = document.getElementsByTagName("a");
 		for (int i = 0; i < nl.getLength(); i++) {
 			final Element a = (Element) nl.item(i);
 			final Anchor anchor = new Anchor(getInnerText(a), a.getAttribute("href"));
 			anchors.add(anchor);
 		}
 		return anchors;
 	}
 
 	/**
 	 * @return Any images in the article.
 	 */
 	public List<String> getArticleImages() {
 		final List<String> images = new ArrayList<String>();
 		if (articleContent == null)
 			return images;
 
 		final NodeList nl = articleContent.getElementsByTagName("img");
 		for (int i = 0; i < nl.getLength(); i++) {
 			final Element img = (Element) nl.item(i);
 			images.add(img.getAttribute("src"));
 		}
 		return images;
 	}
 
 	/**
 	 * @return Any subheadings in the article.
 	 */
 	public List<String> getArticleSubheadings() {
 		final List<String> subtitles = new ArrayList<String>();
 		if (articleContent == null)
 			return subtitles;
 
 		for (int j = 1; j <= 6; j++) {
 			final NodeList nl = articleContent.getElementsByTagName("h" + j);
 			if (nl.getLength() > 0) {
 				for (int i = 0; i < nl.getLength(); i++) {
 					subtitles.add(nl.item(i).getTextContent());
 				}
 				break;
 			}
 		}
 
 		if (subtitles.size() == 0) {
 			// try looking for other likely-looking elements
 
 			final NodeList nl = articleContent.getElementsByTagName("*");
 			for (int i = 0; i < nl.getLength(); i++) {
 				if (nl.item(i) instanceof Element &&
 						((Element) nl.item(i)).getAttribute("class") != null &&
 						search(((Element) nl.item(i)).getAttribute("class"), Regexps.likelySubheadCandidateRe) != -1)
 					subtitles.add(nl.item(i).getTextContent());
 			}
 		}
 
 		return subtitles;
 	}
 
 	protected List<Node> findChildNodesWithName(Node parent, String name) {
 		final NodeList children = parent.getChildNodes();
 		final List<Node> results = new ArrayList<Node>();
 
 		for (int i = 0; i < children.getLength(); ++i) {
 			final Node child = children.item(i);
 			if (child == null)
 				continue;
 
 			final String nodeName = child.getNodeName();
 			if (nodeName == null)
 				continue;
 
 			if (nodeName.equals(name)) {
 				results.add(child);
 			}
 		}
 		return results;
 	}
 
 	protected int findChildNodeIndex(Node parent, Node childToFind)
 	{
 		for (int index = 0; index < parent.getChildNodes().getLength(); index++)
 			if (parent.getChildNodes().item(index) == childToFind)
 				return index;
 		return -1;
 	}
 
 	protected void getArticleTextMapping(TreeWalker walker, List<MappingNode> map) throws DOMException {
 		final Node parend = walker.getCurrentNode();
 
 		if (parend.getNodeType() == Node.TEXT_NODE && parend.getParentNode().getAttributes().getNamedItem("id") != null)
 		{
 			if (parend.getTextContent().trim().length() > 0)
 			{
 				final int index = findChildNodeIndex(parend.getParentNode(), parend);
 				if (index != -1)
 				{
 					// square brackets are not valid XML/HTML identifier
 					// characters, so we can use them here
 					map.add(new MappingNode(
 							parend.getParentNode().getAttributes().getNamedItem("id").getNodeValue() + "[" + index + "]",
 							parend.getNodeValue()));
 
 					// System.out.println(
 					// "ELEMENT '"+parend.getParentNode().getAttributes().getNamedItem("id").getNodeValue()
 					// + "["+index+"]"+"'");
 					// System.out.println( "VALUE:  '"+parend.getNodeValue()+"'"
 					// );
 				}
 			}
 		}
 
 		// traverse children:
 		for (Node n = walker.firstChild(); n != null; n = walker.nextSibling()) {
 			getArticleTextMapping(walker, map);
 		}
 
 		// return position to the current (level up):
 		walker.setCurrentNode(parend);
 	}
 
 	protected class MappingNode {
 		String id;
 		String text;
 
 		public MappingNode(String id, String text) {
 			this.id = id;
 			this.text = text;
 		}
 
 		public String getId() {
 			return id;
 		}
 
 		public String getText() {
 			return text;
 		}
 
 		@Override
 		public String toString() {
 			return "MappingNode(" + id + " -> " + text + ")";
 		}
 	}
 
 	/**
 	 * Get the mapping between bits of text in the dom & their xpaths
 	 *
 	 * @return mapping from xpath to text
 	 */
 	public List<MappingNode> getArticleTextMapping() {
 		if (articleContent == null)
 			return null;
 
 		final List<MappingNode> map = new ArrayList<MappingNode>();
 
 		final TreeWalker walker = ((DocumentTraversal) document).createTreeWalker(articleContent, NodeFilter.SHOW_TEXT
 				| NodeFilter.SHOW_ELEMENT, null, true);
 
 		getArticleTextMapping(walker, map);
 
 		return map;
 	}
 
 	/**
 	 * Convenience method to build a {@link Readability} instance from an html
 	 * string.
 	 *
 	 * @param html
 	 *            The html string
 	 * @return new {@link Readability} instance.
 	 * @throws SAXException
 	 * @throws IOException
 	 */
 	public static Readability getReadability(String html) throws SAXException, IOException {
 		return getReadability(html, false);
 	}
 
 	/**
 	 * Convenience method to build a {@link Readability} instance from an html
 	 * string.
 	 *
 	 * @param html
 	 *            The html string
 	 * @param addTitle
 	 *            Should the title be added to the generated article?
 	 * @return new {@link Readability} instance.
 	 * @throws SAXException
 	 * @throws IOException
 	 */
 	public static Readability getReadability(String html, boolean addTitle) throws SAXException, IOException {
 		final DOMParser parser = new DOMParser();
 		parser.parse(new InputSource(new StringReader(html)));
 
 		return new Readability(parser.getDocument(), false, addTitle);
 	}
 
 	/**
 	 * Testing
 	 *
 	 * @param argv
 	 * @throws Exception
 	 */
 	public static void main(String[] argv) throws Exception {
 		// URL input = new
 		// URL("file:///home/dd/Programming/Readability4J/t.html");
 		// URL input = new
 		// URL("http://news.bbc.co.uk/1/hi/politics/10362367.stm");
 		final URL input = new URL("http://blog.confluent.io/2015/01/29/making-sense-of-stream-processing/");
 		// URL input = new URL("http://euobserver.com/9/30465");
 		// URL input = new URL("http://euobserver.com/?aid=23383");
 		// URL input = new
 		// URL("http://abandoninplace.squarespace.com/blog/2010/6/8/wwdc-monday.html");
 		// URL input = new URL("file:///Users/jsh2/Desktop/test.html");
 		// URL input = new
 		// URL("http://mobile.engadget.com/2010/06/17/htc-aria-review/");
 		// URL input = new URL("http://thedailywtf.com/Articles/Benched.aspx");
 		// URL input = new
 		// URL("http://www.dailymail.co.uk/news/article-1287625/Woman-sparked-150-000-manhunt-slashing-face-crying-rape-faces-jail.html");
 		// URL input = new
 		// URL("http://mrpaparazzi.com/post/11619/Lindsay-Lohan-Tests-Negative-For-Alcohol-Goes-Clubbing-To-Celebrate.aspx");
 		// URL input = new
 		// URL("http://www.bbc.co.uk/news/world-middle-east-11415719");
 		// URL input = new URL("http://www.thebigproject.co.uk/news/");
 		// URL input = new
 		// URL("http://blogs.euobserver.com/popescu/2009/12/15/on-euro-optimism-pessimism-and-failures/#more-958");
 		// URL input = new
 		// URL("http://www.cnn.com/2010/WORLD/meast/09/27/west.bank.settlement.construction/index.html?hpt=T2");
 
 		// URL input = new
 		// URL("http://www.huffingtonpost.com/steven-cohen/its-time-to-enact-congest_b_740315.html");
 		// URL input = new
 		// URL("http://uk.mac.ign.com/articles/573/573319p1.html");
 		final DOMParser parser = new DOMParser();
 		parser.parse(new InputSource(input.openStream()));
 
 		final Readability r = new Readability(parser.getDocument(), true, true);
 
 		// System.out.println(r.getArticleTitle());
 		System.out.println(r.getArticleHTML());
 		// System.out.println(r.getAllLinks());
 		// System.out.println(r.getArticleText());
 
 		System.out.println();
 		System.out.println("***");
 		System.out.println();
 
 		for (final MappingNode s : r.getArticleTextMapping())
 			System.out.println(s);
 
 		// PrintStream out = new PrintStream("news-sites");
 		// for (Anchor anchor : r.getAllLinks()) {
 		// out.println(anchor.getHref() + "\t" + anchor.getText());
 		// }
 		// out.close();
 
 		System.out.println(r.getArticleImages());
 		// System.out.println(r.getArticleSubheadings());
 		// System.out.println(r.getArticleHTML());
 		// System.out.println(r.getArticleHTML_DOM());
 
 		// System.out.println(r.getArticleDateString());
 		// System.out.println(r.getArticleDate());
 	}
 }