From 0073de17ea1e0462aa22b2650e1d88373c61d089 Mon Sep 17 00:00:00 2001 From: 4pr0n Date: Fri, 30 May 2014 05:32:04 -0700 Subject: [PATCH] 1.0.48 - Imagestash.org /tag/ ripper #8 --- pom.xml | 2 +- .../ripme/ripper/rippers/ImagearnRipper.java | 5 +- .../ripper/rippers/ImagestashRipper.java | 107 ++++++++++++++++++ .../com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 4 files changed, 110 insertions(+), 6 deletions(-) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/ImagestashRipper.java diff --git a/pom.xml b/pom.xml index 71b35946..294671a9 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.0.47 + 1.0.48 ripme http://rip.rarchives.com diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java index b12b0e93..531edade 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java @@ -25,10 +25,7 @@ public class ImagearnRipper extends AlbumRipper { } public boolean canRip(URL url) { - if (!url.getHost().endsWith(DOMAIN)) { - return false; - } - return true; + return url.getHost().endsWith(DOMAIN); } public URL sanitizeURL(URL url) throws MalformedURLException { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagestashRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagestashRipper.java new file mode 100644 index 00000000..f666ce21 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagestashRipper.java @@ -0,0 +1,107 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.log4j.Logger; +import org.json.JSONArray; +import org.json.JSONObject; +import org.jsoup.Connection.Method; +import org.jsoup.Jsoup; + +import com.rarchives.ripme.ripper.AlbumRipper; +import com.rarchives.ripme.ui.RipStatusMessage.STATUS; +import com.rarchives.ripme.utils.Utils; + +public class ImagestashRipper extends AlbumRipper { + + private static final String DOMAIN = "imagestash.org", + HOST = "imagestash"; + private static final Logger logger = Logger.getLogger(ImagestashRipper.class); + + public ImagestashRipper(URL url) throws IOException { + super(url); + } + + public boolean canRip(URL url) { + System.err.println(url.getHost() +"/"+DOMAIN); + return url.getHost().equals(DOMAIN); + } + + public URL sanitizeURL(URL url) throws MalformedURLException { + return url; + } + + @Override + public void rip() throws IOException { + // Given URL: https://imagestash.org/tag/everydayuncensor + // GID: "everydayuncensor" + // JSON URL: https://imagestash.org/images?tags=everydayuncensor&page=1 + String baseURL = "https://imagestash.org/images?tags=" + getGID(this.url); + int page = 0, index = 0; + while (true) { + page++; + String nextURL = baseURL + "&page=" + page; + logger.info("[ ] Retrieving " + nextURL); + sendUpdate(STATUS.LOADING_RESOURCE, nextURL); + String jsonText = Jsoup.connect(nextURL) + .ignoreContentType(true) + .userAgent(USER_AGENT) + .method(Method.GET) + .execute() + .body(); + logger.info(jsonText); + JSONObject json = new JSONObject(jsonText); + JSONArray images = json.getJSONArray("images"); + for (int i = 0; i < images.length(); i++) { + JSONObject image = images.getJSONObject(i); + String imageURL = image.getString("src"); + if (imageURL.startsWith("/")) { + imageURL = "http://imagestash.org" + imageURL; + } + index += 1; + String prefix = ""; + if (Utils.getConfigBoolean("download.save_order", true)) { + prefix = String.format("%03d_", index); + } + addURLToDownload(new URL(imageURL), prefix); + } + // Check if there are more images to fetch + int count = json.getInt("count"), + offset = json.getInt("offset"), + total = json.getInt("total"); + if (count + offset >= total || images.length() == 0) { + break; + } + // Wait a bit + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + logger.error("Interrupted while waiting to load next page", e); + break; + } + } + waitForThreads(); + } + + @Override + public String getHost() { + return HOST; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^.*imagestash.org/tag/([a-zA-Z0-9\\-_]+)$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException( + "Expected imagestash.org tag formats: " + + "imagestash.org/tag/tagname" + + " Got: " + url); + } +} diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index bb988d20..e5086b45 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.0.47"; + private static final String DEFAULT_VERSION = "1.0.48"; private static final String updateJsonURL = "http://rarchives.com/ripme.json"; private static final String updateJarURL = "http://rarchives.com/ripme.jar"; private static final String mainFileName = "ripme.jar";