From c2a9d680b2073001ea0f35fc288cf22222143c19 Mon Sep 17 00:00:00 2001 From: 4pr0n Date: Sun, 25 Jan 2015 21:12:30 -0800 Subject: [PATCH] 1.1.4 - Fix deviantart 403 errors, fixes #159 --- pom.xml | 2 +- .../ripme/ripper/rippers/DeviantartRipper.java | 15 +++++++++++++-- .../java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 71694432..8cc6d732 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.1.3 + 1.1.4 ripme http://rip.rarchives.com diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index f7f1ed93..f11920a6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -28,7 +28,8 @@ import com.rarchives.ripme.utils.Utils; public class DeviantartRipper extends AbstractHTMLRipper { - private static final int SLEEP_TIME = 2000; + private static final int PAGE_SLEEP_TIME = 3000, + IMAGE_SLEEP_TIME = 1500; private Map cookies = new HashMap(); private Set triedURLs = new HashSet(); @@ -52,6 +53,15 @@ public class DeviantartRipper extends AbstractHTMLRipper { @Override public URL sanitizeURL(URL url) throws MalformedURLException { String u = url.toExternalForm(); + + if (u.replace("/", "").endsWith(".deviantart.com")) { + // Root user page, get all albums + if (!u.endsWith("/")) { + u += "/"; + } + u += "gallery/?"; + } + String subdir = "/"; if (u.contains("catpath=scraps")) { subdir = "scraps"; @@ -166,7 +176,7 @@ public class DeviantartRipper extends AbstractHTMLRipper { if (nextPage.startsWith("/")) { nextPage = "http://" + this.url.getHost() + nextPage; } - if (!sleep(SLEEP_TIME)) { + if (!sleep(PAGE_SLEEP_TIME)) { throw new IOException("Interrupted while waiting to load next page: " + nextPage); } logger.info("Found next page: " + nextPage); @@ -178,6 +188,7 @@ public class DeviantartRipper extends AbstractHTMLRipper { @Override public void downloadURL(URL url, int index) { addURLToDownload(url, getPrefix(index), "", this.url.toExternalForm(), cookies); + sleep(IMAGE_SLEEP_TIME); } /** diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index b4dbf64c..ad1017e1 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.1.3"; + private static final String DEFAULT_VERSION = "1.1.4"; private static final String updateJsonURL = "http://rarchives.com/ripme.json"; private static final String updateJarURL = "http://rarchives.com/ripme.jar"; private static final String mainFileName = "ripme.jar";