Added unit test for desuarchive.org

src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java

@@ -1,11 +1,14 @@
 package com.rarchives.ripme.tst.ripper.rippers;
 
 import java.io.IOException;
+import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.List;
 
 import com.rarchives.ripme.ripper.rippers.ChanRipper;
+import com.rarchives.ripme.utils.Http;
+import org.jsoup.nodes.Document;
 
 public class ChanRipperTest extends RippersTest {
 
@@ -42,24 +45,26 @@ public class ChanRipperTest extends RippersTest {
 
     public void testChanRipper() throws IOException {
         List<URL> contentURLs = new ArrayList<>();
-        // URLs that should return more than 1 image
+        contentURLs.add(new URL(getRandomThreadDesuarchive()));
-        //contentURLs.add(new URL("http://desuchan.net/v/res/7034.html"));
-        //contentURLs.add(new URL("http://boards.420chan.org/ana/res/75984.php"));
-        //contentURLs.add(new URL("http://archive.4plebs.org/s4s/thread/3005257/"));
-        //contentURLs.add(new URL("http://drawchan.net/dc/dw/res/114910.html"));
-
-        // Most *chans have volatile threads & can't be trusted for integration testing.
-
-        //contentURLs.add(new URL("http://boards.4chan.org/r/res/12225949"));
-        //contentURLs.add(new URL("http://7chan.org/gif/res/23795.html"));
-        //contentURLs.add(new URL("http://unichan2.org/b/res/518004.html"));
-
-        // xchan has an HTTPS certificaiton error...
-        //contentURLs.add(new URL("http://xchan.pw/porn/res/437.html"));
         for (URL url : contentURLs) {
             ChanRipper ripper = new ChanRipper(url);
-            testRipper(ripper);
+            testChanRipper(ripper);
         }
     }
 
+    /**
+     *
+     * @return String returns a url to a active desuarchive.org tread as a string
+     */
+    public String getRandomThreadDesuarchive() {
+        try {
+            Document doc = Http.url(new URL("https://desuarchive.org/wsg/")).get();
+            System.out.println(doc);
+            return doc.select("div.post_data > a").first().attr("href");
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        return null;
+    }
+
 }

src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java

@@ -2,7 +2,9 @@ package com.rarchives.ripme.tst.ripper.rippers;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.List;
 
+import com.rarchives.ripme.ripper.rippers.ChanRipper;
 import junit.framework.TestCase;
 
 import org.apache.log4j.ConsoleAppender;
@@ -52,6 +54,38 @@ public class RippersTest extends TestCase {
         }
     }
 
+    // We have a special test for chan rippers because we can't assume that content will be downloadable, as content
+    // is often removed within mere hours of it being posted. So instead of trying to download any content we just check
+    // that we found links to it
+    void testChanRipper(ChanRipper ripper) {
+        try {
+            // Turn on Debug logging
+            ((ConsoleAppender)Logger.getRootLogger().getAppender("stdout")).setThreshold(Level.DEBUG);
+
+            // Decrease timeout
+            Utils.setConfigInteger("page.timeout", 20 * 1000);
+
+            ripper.setup();
+            ripper.markAsTest();
+            List<String> foundUrls = ripper.getURLsFromPage(ripper.getFirstPage());
+            assertTrue("Failed to find single url on page " + ripper.getURL(), foundUrls.size() >= 1);
+        } catch (IOException e) {
+            if (e.getMessage().contains("Ripping interrupted")) {
+                // We expect some rips to get interrupted
+            }
+            else {
+                e.printStackTrace();
+                fail("Failed to rip " + ripper.getURL() + " : " + e.getMessage());
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail("Failed to rip " + ripper.getURL() + " : " + e.getMessage());
+        }
+        finally {
+            deleteDir(ripper.getWorkingDir());
+        }
+    }
+
     /** File extensions that are safe to delete. */
     private static final String[] SAFE_EXTENSIONS =
         {"png", "jpg",  "jpeg", "gif",