diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index 2a4e4411..afd676f1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -2,7 +2,6 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; -import java.net.SocketTimeoutException; import java.net.URL; import java.util.List; import java.util.regex.Matcher; @@ -59,7 +58,7 @@ public class RedditRipper extends AlbumRipper { URL jsonURL = getJsonURL(this.url); while (true) { jsonURL = getAndParseAndReturnNext(jsonURL); - if (jsonURL == null) { + if (jsonURL == null || isThisATest() || isStopped()) { break; } } @@ -119,21 +118,11 @@ public class RedditRipper extends AlbumRipper { } lastRequestTime = System.currentTimeMillis(); - int attempts = 0; - logger.info(" Retrieving " + url); - String jsonString = null; - while(jsonString == null && attempts++ < 3) { - try { - jsonString = Http.url(url) - .ignoreContentType() - .response() - .body(); - } catch(SocketTimeoutException ex) { - if(attempts >= 3) throw ex; - logger.warn(String.format("[!] Connection timed out (attempt %d)", attempts)); - } - } - + String jsonString = Http.url(url) + .ignoreContentType() + .response() + .body(); + Object jsonObj = new JSONTokener(jsonString).nextValue(); JSONArray jsonArray = new JSONArray(); if (jsonObj instanceof JSONObject) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SeeniveRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SeeniveRipper.java index 2cb6af4b..81be57d3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SeeniveRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SeeniveRipper.java @@ -52,16 +52,16 @@ public class SeeniveRipper extends AlbumRipper { } String lastID = null; for (Element element : doc.select("a.facebox")) { - if (isStopped()) { - break; - } String card = element.attr("href"); // "/v/" URL videoURL = new URL("http://seenive.com" + card); SeeniveImageThread vit = new SeeniveImageThread(videoURL); seeniveThreadPool.addThread(vit); lastID = card.substring(card.lastIndexOf('/') + 1); + if (isStopped() || isThisATest()) { + break; + } } - if (lastID == null) { + if (lastID == null || isStopped() || isThisATest()) { break; } @@ -74,8 +74,8 @@ public class SeeniveRipper extends AlbumRipper { logger.info("[ ] Retrieving " + baseURL + "/next/" + lastID); JSONObject json = Http.url(baseURL + "/next/" + lastID) - .referrer(baseURL) - .getJSON(); + .referrer(baseURL) + .getJSON(); String html = json.getString("Html"); if (html.equals("")) { break; @@ -116,7 +116,7 @@ public class SeeniveRipper extends AlbumRipper { public void run() { try { Document doc = Http.url(this.url).get(); - logger.info("[ ] Retreiving video page " + this.url); + logger.info("[ ] Retrieving video page " + this.url); sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm()); for (Element element : doc.select("source")) { String video = element.attr("src"); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java index b0ffbf1c..7e8cc52a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java @@ -155,7 +155,7 @@ public class VkRipper extends AlbumRipper { } offset += elements.size(); - if (isThisATest()) { + if (isStopped() || isThisATest()) { break; } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java index b868fd28..5d45e3ea 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java @@ -15,8 +15,8 @@ public class VkRipperTest extends RippersTest { } List contentURLs = new ArrayList(); contentURLs.add(new URL("https://vk.com/album45506334_172415053")); - contentURLs.add(new URL("https://vk.com/album45506334_0")); - contentURLs.add(new URL("https://vk.com/photos45506334")); + //contentURLs.add(new URL("https://vk.com/album45506334_0")); + //contentURLs.add(new URL("https://vk.com/photos45506334")); for (URL url : contentURLs) { VkRipper ripper = new VkRipper(url); testRipper(ripper);