Attempt to remove transient failure of tests

src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java

@@ -26,7 +26,7 @@ public abstract class AbstractHTMLRipper extends AlbumRipper {
 
     public abstract Document getFirstPage() throws IOException;
     public Document getNextPage(Document doc) throws IOException {
-        throw new IOException("getNextPage not implemented");
+        return null;
     }
     public abstract List<String> getURLsFromPage(Document page);
     public List<String> getDescriptionsFromPage(Document doc) throws IOException {

src/main/java/com/rarchives/ripme/ripper/VideoRipper.java

@@ -58,7 +58,7 @@ public abstract class VideoRipper extends AbstractRipper {
         }
         else {
             if (isThisATest()) {
-                System.err.println("TEST, download url: " + url);
+                this.url = url;
                 return true;
             }
             threadPool.addThread(new DownloadVideoThread(url, saveAs, this));

src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java

@@ -105,24 +105,29 @@ public class ChanRipper extends AbstractHTMLRipper {
         return Http.url(this.url).get();
     }
 
+    private boolean isURLBlacklisted(String url) {
+        for (String blacklist_item : url_piece_blacklist) {
+            if (url.contains(blacklist_item)) {
+                logger.debug("Skipping link that contains '"+blacklist_item+"': " + url);
+                return true;
+            }            
+        }
+        return false;
+    }
     @Override
     public List<String> getURLsFromPage(Document page) {
         List<String> imageURLs = new ArrayList<String>();
         Pattern p; Matcher m;
-        elementloop:
         for (Element link : page.select("a")) {
             if (!link.hasAttr("href")) { 
                 continue;
             }
             String href = link.attr("href").trim();
 
-            //Check all blacklist items
+            if (isURLBlacklisted(href)) {
-            for (String blacklist_item : url_piece_blacklist) {
+                continue;
-                if (href.contains(blacklist_item)) {
-                    logger.debug("Skipping link that contains '"+blacklist_item+"': " + href);
-                    continue elementloop;
-                }            
             }
+            //Check all blacklist items
             Boolean self_hosted = false;
             if (!generalChanSite) {
                 for (String cdnDomain : chanSite.cdnDomains) {
@@ -132,7 +137,7 @@ public class ChanRipper extends AbstractHTMLRipper {
                 }   
             }
 
-            if(self_hosted||generalChanSite){
+            if (self_hosted || generalChanSite){
                 p = Pattern.compile("^.*\\.(jpg|jpeg|png|gif|apng|webp|tif|tiff|webm)$", Pattern.CASE_INSENSITIVE);
                 m = p.matcher(href);
                 if (m.matches()) {
@@ -148,10 +153,17 @@ public class ChanRipper extends AbstractHTMLRipper {
                         continue;
                     }
                     imageURLs.add(href);
+                    if (isThisATest()) {
+                        break;
+                    }
                 }
             } else {
                 //TODO also grab imgur/flickr albums (And all other supported rippers) Maybe add a setting?
             }            
+
+            if (isStopped()) {
+                break;
+            }
         }
         return imageURLs;
     }

src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java

@@ -119,6 +119,9 @@ public class InstagramRipper extends AbstractJSONRipper {
 
     @Override
     public JSONObject getNextPage(JSONObject json) throws IOException {
+        if (isThisATest()) {
+            return null;
+        }
         JSONObject pagination = json.getJSONObject("pagination");
         String nextMaxID = "";
         JSONArray datas = json.getJSONArray("data");
@@ -162,6 +165,9 @@ public class InstagramRipper extends AbstractJSONRipper {
                 continue;
             }
             imageURLs.add(imageURL);
+            if (isThisATest()) {
+                break;
+            }
         }
         return imageURLs;
     }

src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java

@@ -190,6 +190,7 @@ public class TwitterRipper extends AlbumRipper {
             }
         }
 
+        /*
         if (entities.has("urls")) {
             JSONArray urls = entities.getJSONArray("urls");
             JSONObject url;
@@ -202,10 +203,7 @@ public class TwitterRipper extends AlbumRipper {
                 }
             }
         }
-    }
+        */
-    
-    private void handleTweetedURL(String url) {
-        logger.error("[!] Need to handle URL: " + url);
     }
 
     @Override
@@ -223,6 +221,10 @@ public class TwitterRipper extends AlbumRipper {
 
         Long lastMaxID = 0L;
         for (int i = 0; i < MAX_REQUESTS; i++) {
+            if (isStopped()) {
+                break;
+            }
+
             List<JSONObject> tweets = getTweets(getApiURL(lastMaxID - 1));
             if (tweets.size() == 0) {
                 logger.info("   No more tweets found.");

src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java

@@ -154,6 +154,10 @@ public class VkRipper extends AlbumRipper {
                 break;
             }
             offset += elements.size();
+
+            if (isThisATest()) {
+                break;
+            }
         }
         waitForThreads();
     }

src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java

@@ -58,7 +58,7 @@ public class PornhubRipper extends VideoRipper {
     public void rip() throws IOException {
         logger.info("    Retrieving " + this.url.toExternalForm());
         Document doc = Http.url(this.url).get();
-        Pattern p = Pattern.compile("^.*var flashvars = (.*});.*$", Pattern.DOTALL);
+        Pattern p = Pattern.compile("^.*'flashvars' : (.*});.*$", Pattern.DOTALL);
         Matcher m = p.matcher(doc.body().html());
         if (m.matches()) {
             String title = null,
@@ -90,6 +90,9 @@ public class PornhubRipper extends VideoRipper {
                 throw new IOException(e);
             }
         }
+        else {
+            throw new IOException("Failed to download " + this.url + " : could not find 'flashvars'");
+        }
         waitForThreads();
     }
 }

src/main/java/com/rarchives/ripme/utils/AES.java

@@ -1,5 +1,6 @@
 package com.rarchives.ripme.utils;
 
+import java.lang.reflect.Field;
 import java.util.Arrays;
 
 import javax.crypto.Cipher;
@@ -9,6 +10,22 @@ import javax.crypto.spec.SecretKeySpec;
 
 public class AES {
 
+    /**
+     * Hack to get JCE Unlimited Strenght so we can use weird AES encryption stuff.
+     * From http://stackoverflow.com/a/20286961
+     */
+    static {
+        try {
+            Field field = Class.forName("javax.crypto.JceSecurity").getDeclaredField("isRestricted");
+            if (!field.isAccessible()) {
+                field.setAccessible(true);
+                field.set(null, java.lang.Boolean.FALSE);
+            }
+        } catch (Exception ex) {
+            // Assume it's fine.
+        }
+    }
+
     public static String decrypt(String cipherText, String key, int nBits) throws Exception {
         String res = null;
         nBits = nBits / 8;
@@ -31,6 +48,7 @@ public class AES {
             cipher.init(Cipher.ENCRYPT_MODE, secretKey);
             keyBytes = cipher.doFinal(keyBytes);
         } catch (Throwable e1) {
+            e1.printStackTrace();
             return null;
         }
         System.arraycopy(keyBytes, 0, keyBytes, nBits / 2, nBits / 2);

src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java

@@ -5,6 +5,7 @@ import java.net.URL;
 import java.util.ArrayList;
 import java.util.List;
 
+import com.rarchives.ripme.ripper.VideoRipper;
 import com.rarchives.ripme.ripper.rippers.video.BeegRipper;
 import com.rarchives.ripme.ripper.rippers.video.PornhubRipper;
 import com.rarchives.ripme.ripper.rippers.video.VineRipper;
@@ -13,6 +14,24 @@ import com.rarchives.ripme.ripper.rippers.video.YoupornRipper;
 
 public class VideoRippersTest extends RippersTest {
     
+    private void videoTestHelper(VideoRipper ripper) {
+        URL oldURL = ripper.getURL();
+        try {
+            ripper.setup();
+            ripper.markAsTest();
+            ripper.rip();
+            // Video ripper testing is... weird.
+            // If we find the URL to download the video, and it's a test,
+            // then the ripper sets it as the ripper's URL.
+            assertFalse("Failed to find download url for " + oldURL, oldURL.equals(ripper.getURL()));
+        } catch (Exception e) {
+            fail("Error while ripping " + ripper.getURL() + " : " + e);
+            e.printStackTrace();
+        } finally {
+            deleteDir(ripper.getWorkingDir());
+        }
+    }
+
     public void testXvideosRipper() throws IOException {
         if (!DOWNLOAD_CONTENT) {
             return;
@@ -22,7 +41,7 @@ public class VideoRippersTest extends RippersTest {
         contentURLs.add(new URL("http://www.xvideos.com/video7136868/vid-20140205-wa0011"));
         for (URL url : contentURLs) {
             XvideosRipper ripper = new XvideosRipper(url);
-            testRipper(ripper);
+            videoTestHelper(ripper);
         }
     }
     
@@ -34,7 +53,7 @@ public class VideoRippersTest extends RippersTest {
         contentURLs.add(new URL("http://www.pornhub.com/view_video.php?viewkey=993166542"));
         for (URL url : contentURLs) {
             PornhubRipper ripper = new PornhubRipper(url);
-            testRipper(ripper);
+            videoTestHelper(ripper);
         }
     }
 
@@ -46,7 +65,7 @@ public class VideoRippersTest extends RippersTest {
         contentURLs.add(new URL("https://vine.co/v/hiqQrP0eUZx"));
         for (URL url : contentURLs) {
             VineRipper ripper = new VineRipper(url);
-            testRipper(ripper);
+            videoTestHelper(ripper);
         }
     }
 
@@ -58,7 +77,7 @@ public class VideoRippersTest extends RippersTest {
         contentURLs.add(new URL("http://www.youporn.com/watch/7669155/mrs-li-amateur-69-orgasm/?from=categ"));
         for (URL url : contentURLs) {
             YoupornRipper ripper = new YoupornRipper(url);
-            testRipper(ripper);
+            videoTestHelper(ripper);
         }
     }
 
@@ -70,9 +89,7 @@ public class VideoRippersTest extends RippersTest {
         contentURLs.add(new URL("http://beeg.com/4554321"));
         for (URL url : contentURLs) {
             BeegRipper ripper = new BeegRipper(url);
-            testRipper(ripper);
+            videoTestHelper(ripper);
         }
     }
-
-
 }