ImgScroll/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java

package com.rarchives.ripme.ripper.rippers;

import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.ripper.DownloadThreadPool;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

/**
 *
 * @author
 */
public class E621Ripper extends AbstractHTMLRipper {
	public static final int POOL_IMAGES_PER_PAGE = 24;

	private DownloadThreadPool e621ThreadPool = new DownloadThreadPool("e621");

	public E621Ripper(URL url) throws IOException {
		super(url);
	}

	@Override
	public DownloadThreadPool getThreadPool() {
		return e621ThreadPool;
	}

	@Override
	public String getDomain() {
		return "e621.net";
	}

	@Override
	public String getHost() {
		return "e621";
	}

	@Override
	public Document getFirstPage() throws IOException {
		if (url.getPath().startsWith("/pool/show/")) {
			return Http.url("https://e621.net/pool/show/" + getTerm(url)).get();
		} else {
			return Http.url("https://e621.net/post/index/1/" + getTerm(url)).get();
		}
	}

	@Override
	public List<String> getURLsFromPage(Document page) {
		Elements elements = page.select("#post-list .thumb a,#pool-show .thumb a");
		List<String> res = new ArrayList<String>(elements.size());

		if (page.getElementById("pool-show") != null) {
			int index = 0;

			Element e = page.getElementById("paginator");
			if (e != null) {
				e = e.getElementsByClass("current").first();
				if (e != null) {
					index = (Integer.parseInt(e.text()) - 1) * POOL_IMAGES_PER_PAGE;
				}
			}

			for (Element e_ : elements) {
				res.add(e_.absUrl("href") + "#" + ++index);
			}

		} else {
			for (Element e : elements) {
				res.add(e.absUrl("href") + "#" + e.child(0).attr("id").substring(1));
			}
		}

		return res;
	}

	@Override
	public Document getNextPage(Document page) throws IOException {
		for (Element e : page.select("#paginator a")) {
			if (e.attr("rel").equals("next")) {
				return Http.url(e.absUrl("href")).get();
			}
		}

		return null;
	}

	@Override
	public void downloadURL(final URL url, int index) {
		e621ThreadPool.addThread(new Thread(new Runnable() {
			public void run() {
				try {
					Document page = Http.url(url).get();
					Element e = page.getElementById("image");

					if (e != null) {
						addURLToDownload(new URL(e.absUrl("src")), Utils.getConfigBoolean("download.save_order", true) ? url.getRef() + "-" : "");
					} else if ((e = page.select(".content object>param[name=\"movie\"]").first()) != null) {
						addURLToDownload(new URL(e.absUrl("value")), Utils.getConfigBoolean("download.save_order", true) ? url.getRef() + "-" : "");
					} else {
						Logger.getLogger(E621Ripper.class.getName()).log(Level.WARNING, "Unsupported media type - please report to program author: " + url.toString());
					}

				} catch (IOException ex) {
					Logger.getLogger(E621Ripper.class.getName()).log(Level.SEVERE, null, ex);
				}
			}
		}));
	}

	private String getTerm(URL url) throws MalformedURLException {
		String query = url.getQuery();

		if (query != null) {
			return Utils.parseUrlQuery(query, "tags");
		}

		if (query == null) {
			if ((query = url.getPath()).startsWith("/post/index/")) {
				query = query.substring(12);

				int pos = query.indexOf('/');
				if (pos == -1) {
					return null;
				}

				// skip page number
				query = query.substring(pos + 1);

				if (query.endsWith("/")) {
					query = query.substring(0, query.length() - 1);
				}

				try {
					return URLDecoder.decode(query, "UTF-8");
				} catch (UnsupportedEncodingException e) {
					// Shouldn't happen since UTF-8 is required to be supported
					throw new RuntimeException(e);
				}

			} else if (query.startsWith("/pool/show/")) {
				query = query.substring(11);

				if (query.endsWith("/")) {
					query = query.substring(0, query.length() - 1);
				}

				return query;
			}
		}

		return null;
	}

	@Override
	public String getGID(URL url) throws MalformedURLException {
		String prefix = "";
		if (url.getPath().startsWith("/pool/show/")) {
			prefix = "pool_";
		} else {
			prefix = "term_";
		}

		return Utils.filesystemSafe(prefix + getTerm(url));
	}

}