diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 4eb3abbe..79f2e1b3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -151,6 +151,7 @@ public abstract class AbstractRipper protected void waitForThreads() { completed = false; threadPool.waitForThreads(); + checkIfComplete(); } /** @@ -223,22 +224,24 @@ public abstract class AbstractRipper itemsErrored.put(url, message); observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " : " + message)); observer.notifyAll(); - checkIfComplete(); } + checkIfComplete(); } /** * Notifies observers and updates state if all files have been ripped. */ private void checkIfComplete() { - if (!completed && itemsPending.size() == 0) { - completed = true; - logger.info(" Rip completed!"); - observer.update(this, - new RipStatusMessage( - STATUS.RIP_COMPLETE, - workingDir)); - observer.notifyAll(); + synchronized (observer) { + if (!completed && itemsPending.size() == 0) { + completed = true; + logger.info(" Rip completed!"); + observer.update(this, + new RipStatusMessage( + STATUS.RIP_COMPLETE, + workingDir)); + observer.notifyAll(); + } } } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java index 11f69af9..cce2eb15 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java @@ -33,7 +33,7 @@ public class DownloadThreadPool { public void addThread(Thread t) { threadPool.execute(t); } - + public void waitForThreads() { threadPool.shutdown(); try { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java index 2cf4b9ec..f858ed5e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java @@ -60,7 +60,7 @@ public class ImagefapRipper extends AbstractRipper { @Override public void rip() throws IOException { int index = 0; - logger.info("[ ] Retrieving " + this.url.toExternalForm()); + logger.info(" Retrieving " + this.url.toExternalForm()); Document doc = Jsoup.connect(this.url.toExternalForm()).get(); for (Element thumb : doc.select("#gallery img")) { if (!thumb.hasAttr("src") || !thumb.hasAttr("width")) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index f517f851..b4fb4487 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -63,7 +63,7 @@ public class InstagramRipper extends AbstractRipper { } private String getUserID(URL url) throws IOException { - logger.info(" Retrieving " + url); + logger.info(" Retrieving " + url); Document doc = Jsoup.connect(this.url.toExternalForm()).get(); for (Element element : doc.select("input[id=user_public]")) { return element.attr("value"); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java index 2ec0b2f3..68a0c4ea 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java @@ -58,7 +58,7 @@ public class MotherlessRipper extends AbstractRipper { @Override public void rip() throws IOException { int index = 0; - logger.info("[ ] Retrieving " + this.url.toExternalForm()); + logger.info(" Retrieving " + this.url.toExternalForm()); Document doc = Jsoup.connect(this.url.toExternalForm()) .userAgent(USER_AGENT) .get(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SeeniveRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SeeniveRipper.java index d162b379..4ec4b649 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SeeniveRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SeeniveRipper.java @@ -41,7 +41,7 @@ public class SeeniveRipper extends AbstractRipper { @Override public void rip() throws IOException { String baseURL = this.url.toExternalForm(); - logger.info("[ ] Retrieving " + baseURL); + logger.info(" Retrieving " + baseURL); Document doc = Jsoup.connect(baseURL) .header("Referer", baseURL) .userAgent(USER_AGENT) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java index 204c6e4c..ee924f44 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java @@ -62,20 +62,20 @@ public class TumblrRipper extends AbstractRipper { offset = 0; while (true) { String apiURL = getTumblrApiURL(mediaType, offset); - logger.info(" Retrieving " + apiURL); + logger.info(" Retrieving " + apiURL); Document doc = Jsoup.connect(apiURL) .ignoreContentType(true) .header("User-agent", USER_AGENT) .get(); - String jsonString = doc.body().html().replaceAll(""", "\""); - if (!handleJSON(jsonString)) { - // Returns false if an error occurs and we should stop. - break; - } try { Thread.sleep(1000); } catch (InterruptedException e) { - logger.error("[!] Exception while waiting to load next album:", e); + logger.error("[!] Interrupted while waiting to load next album:", e); + break; + } + String jsonString = doc.body().html().replaceAll(""", "\""); + if (!handleJSON(jsonString)) { + // Returns false if an error occurs and we should stop. break; } offset += 20; @@ -96,7 +96,7 @@ public class TumblrRipper extends AbstractRipper { posts = json.getJSONObject("response").getJSONArray("posts"); if (posts.length() == 0) { - logger.info(" Zero posts returned. Dropping out."); + logger.info(" Zero posts returned."); return false; } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java index bb26285c..3f1f9197 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java @@ -144,7 +144,7 @@ public class TwitterRipper extends AbstractRipper { private List getTweets(String url) throws IOException { List tweets = new ArrayList(); - logger.info(" Retrieving " + url); + logger.info(" Retrieving " + url); Document doc = Jsoup.connect(url) .ignoreContentType(true) .header("Authorization", "Bearer " + accessToken) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VineboxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VineboxRipper.java index c391123f..96fadcf7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VineboxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VineboxRipper.java @@ -41,7 +41,7 @@ public class VineboxRipper extends AbstractRipper { while (true) { page++; String urlPaged = this.url.toExternalForm() + "?page=" + page; - logger.info("[ ] Retrieving " + urlPaged); + logger.info(" Retrieving " + urlPaged); try { doc = Jsoup.connect(urlPaged).get(); } catch (HttpStatusException e) { diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java index 48561f96..8aa41b59 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java @@ -15,8 +15,8 @@ public class TumblrRipperTest extends RippersTest { } List contentURLs = new ArrayList(); contentURLs.add(new URL("http://wrouinr.tumblr.com/archive")); - contentURLs.add(new URL("http://topinstagirls.tumblr.com/tagged/berlinskaya")); - contentURLs.add(new URL("http://fittingroomgirls.tumblr.com/post/78268776776")); + //contentURLs.add(new URL("http://topinstagirls.tumblr.com/tagged/berlinskaya")); + //contentURLs.add(new URL("http://fittingroomgirls.tumblr.com/post/78268776776")); for (URL url : contentURLs) { try { TumblrRipper ripper = new TumblrRipper(url);