From 5d319355d37a8e4f130933c38697f60a60770a96 Mon Sep 17 00:00:00 2001 From: 4pr0n Date: Sat, 7 Jun 2014 12:10:36 -0700 Subject: [PATCH] 1.0.57 - Nfsfw ripper #8 --- pom.xml | 2 +- .../ripme/ripper/rippers/NfsfwRipper.java | 192 ++++++++++++++++++ .../com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 194 insertions(+), 2 deletions(-) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java diff --git a/pom.xml b/pom.xml index fee9c304..b4cc8d00 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.0.56 + 1.0.57 ripme http://rip.rarchives.com diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java new file mode 100644 index 00000000..4159bafd --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java @@ -0,0 +1,192 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.log4j.Logger; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import com.rarchives.ripme.ripper.AlbumRipper; +import com.rarchives.ripme.ripper.DownloadThreadPool; +import com.rarchives.ripme.ui.RipStatusMessage.STATUS; +import com.rarchives.ripme.utils.Utils; + +public class NfsfwRipper extends AlbumRipper { + + private static final String DOMAIN = "nfsfw.com", + HOST = "nfsfw"; + private static final Logger logger = Logger.getLogger(NfsfwRipper.class); + + private Document albumDoc = null; + + private DownloadThreadPool nfsfwThreadPool; + + public NfsfwRipper(URL url) throws IOException { + super(url); + nfsfwThreadPool = new DownloadThreadPool("NFSFW"); + } + + @Override + public String getHost() { + return HOST; + } + + @Override + public URL sanitizeURL(URL url) throws MalformedURLException { + return url; + } + + @Override + public String getAlbumTitle(URL url) throws MalformedURLException { + try { + // Attempt to use album title as GID + if (albumDoc == null) { + albumDoc = Jsoup.connect(url.toExternalForm()) + .userAgent(USER_AGENT) + .get(); + } + String title = albumDoc.select("h2").first().text().trim(); + return "nfsfw_" + Utils.filesystemSafe(title); + } catch (Exception e) { + // Fall back to default album naming convention + } + return super.getAlbumTitle(url); + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p; Matcher m; + + p = Pattern.compile("https?://[wm.]*nfsfw.com/gallery/v/([a-zA-Z0-9\\-_]+).*"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + + throw new MalformedURLException( + "Expected nfsfw.com gallery format: " + + "nfsfw.com/v/albumname" + + " Got: " + url); + } + + @Override + public void rip() throws IOException { + List subAlbums = new ArrayList(); + int index = 0; + subAlbums.add(new Pair(this.url.toExternalForm(), "")); + while (subAlbums.size() > 0) { + if (isStopped()) { + break; + } + Pair nextAlbum = subAlbums.remove(0); + String nextURL = nextAlbum.first; + String nextSubalbum = nextAlbum.second; + sendUpdate(STATUS.LOADING_RESOURCE, nextURL); + logger.info(" Retrieving " + nextURL); + if (albumDoc == null) { + albumDoc = Jsoup.connect(nextURL) + .userAgent(USER_AGENT) + .get(); + } + // Subalbums + for (Element suba : albumDoc.select("td.IMG > a")) { + String subURL = "http://nfsfw.com" + suba.attr("href"); + String subdir = subURL; + while (subdir.endsWith("/")) { + subdir = subdir.substring(0, subdir.length() - 1); + } + subdir = subdir.substring(subdir.lastIndexOf("/") + 1); + subAlbums.add(new Pair(subURL, subdir)); + } + // Images + for (Element thumb : albumDoc.select("td.giItemCell > div > a")) { + String imagePage = "http://nfsfw.com" + thumb.attr("href"); + try { + NfsfwImageThread t = new NfsfwImageThread(new URL(imagePage), nextSubalbum, ++index); + nfsfwThreadPool.addThread(t); + } catch (MalformedURLException mue) { + logger.warn("Invalid URL: " + imagePage); + } + } + // Get next page + for (Element a : albumDoc.select("a.next")) { + subAlbums.add(0, new Pair("http://nfsfw.com" + a.attr("href"), "")); + break; + } + // Insert next page at the top + albumDoc = null; + // Wait + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + logger.error("Interrupted while waiting to load next page", e); + throw new IOException(e); + } + } + nfsfwThreadPool.waitForThreads(); + waitForThreads(); + } + + public boolean canRip(URL url) { + return url.getHost().endsWith(DOMAIN); + } + + /** + * Helper class to find and download images found on "image" pages + */ + private class NfsfwImageThread extends Thread { + private URL url; + private String subdir; + private int index; + + public NfsfwImageThread(URL url, String subdir, int index) { + super(); + this.url = url; + this.subdir = subdir; + this.index = index; + } + + @Override + public void run() { + try { + Document doc = Jsoup.connect(this.url.toExternalForm()) + .userAgent(USER_AGENT) + .timeout(5000) + .referrer(this.url.toExternalForm()) + .get(); + Elements images = doc.select(".gbBlock img"); + if (images.size() == 0) { + logger.error("Failed to find image at " + this.url); + return; + } + String file = images.first().attr("src"); + if (file.startsWith("/")) { + file = "http://nfsfw.com" + file; + } + String prefix = ""; + if (Utils.getConfigBoolean("download.save_order", true)) { + prefix = String.format("%03d_", index); + } + addURLToDownload(new URL(file), prefix, this.subdir); + } catch (IOException e) { + logger.error("[!] Exception while loading/parsing " + this.url, e); + } + } + } + + private class Pair { + public String first, second; + public Pair(String first, String second) { + this.first = first; + this.second = second; + } + } +} \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 853ee43a..212a379c 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.0.56"; + private static final String DEFAULT_VERSION = "1.0.57"; private static final String updateJsonURL = "http://rarchives.com/ripme.json"; private static final String updateJarURL = "http://rarchives.com/ripme.jar"; private static final String mainFileName = "ripme.jar";