diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index f5b2531b..729ee7bb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -26,7 +26,7 @@ public abstract class AbstractHTMLRipper extends AlbumRipper { public abstract Document getFirstPage() throws IOException; public Document getNextPage(Document doc) throws IOException { - throw new IOException("getNextPage not implemented"); + return null; } public abstract List getURLsFromPage(Document page); public List getDescriptionsFromPage(Document doc) throws IOException { diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index d15c0f84..ff8bca4c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -58,7 +58,7 @@ public abstract class VideoRipper extends AbstractRipper { } else { if (isThisATest()) { - System.err.println("TEST, download url: " + url); + this.url = url; return true; } threadPool.addThread(new DownloadVideoThread(url, saveAs, this)); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java index f3e3256b..0147e30d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java @@ -105,24 +105,29 @@ public class ChanRipper extends AbstractHTMLRipper { return Http.url(this.url).get(); } + private boolean isURLBlacklisted(String url) { + for (String blacklist_item : url_piece_blacklist) { + if (url.contains(blacklist_item)) { + logger.debug("Skipping link that contains '"+blacklist_item+"': " + url); + return true; + } + } + return false; + } @Override public List getURLsFromPage(Document page) { List imageURLs = new ArrayList(); Pattern p; Matcher m; - elementloop: for (Element link : page.select("a")) { if (!link.hasAttr("href")) { continue; } String href = link.attr("href").trim(); - //Check all blacklist items - for (String blacklist_item : url_piece_blacklist) { - if (href.contains(blacklist_item)) { - logger.debug("Skipping link that contains '"+blacklist_item+"': " + href); - continue elementloop; - } + if (isURLBlacklisted(href)) { + continue; } + //Check all blacklist items Boolean self_hosted = false; if (!generalChanSite) { for (String cdnDomain : chanSite.cdnDomains) { @@ -132,7 +137,7 @@ public class ChanRipper extends AbstractHTMLRipper { } } - if(self_hosted||generalChanSite){ + if (self_hosted || generalChanSite){ p = Pattern.compile("^.*\\.(jpg|jpeg|png|gif|apng|webp|tif|tiff|webm)$", Pattern.CASE_INSENSITIVE); m = p.matcher(href); if (m.matches()) { @@ -148,10 +153,17 @@ public class ChanRipper extends AbstractHTMLRipper { continue; } imageURLs.add(href); + if (isThisATest()) { + break; + } } } else { //TODO also grab imgur/flickr albums (And all other supported rippers) Maybe add a setting? } + + if (isStopped()) { + break; + } } return imageURLs; } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index f92d0582..dc41edee 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -119,6 +119,9 @@ public class InstagramRipper extends AbstractJSONRipper { @Override public JSONObject getNextPage(JSONObject json) throws IOException { + if (isThisATest()) { + return null; + } JSONObject pagination = json.getJSONObject("pagination"); String nextMaxID = ""; JSONArray datas = json.getJSONArray("data"); @@ -162,6 +165,9 @@ public class InstagramRipper extends AbstractJSONRipper { continue; } imageURLs.add(imageURL); + if (isThisATest()) { + break; + } } return imageURLs; } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java index 7b12409e..b3f1b832 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java @@ -190,6 +190,7 @@ public class TwitterRipper extends AlbumRipper { } } + /* if (entities.has("urls")) { JSONArray urls = entities.getJSONArray("urls"); JSONObject url; @@ -202,10 +203,7 @@ public class TwitterRipper extends AlbumRipper { } } } - } - - private void handleTweetedURL(String url) { - logger.error("[!] Need to handle URL: " + url); + */ } @Override @@ -223,6 +221,10 @@ public class TwitterRipper extends AlbumRipper { Long lastMaxID = 0L; for (int i = 0; i < MAX_REQUESTS; i++) { + if (isStopped()) { + break; + } + List tweets = getTweets(getApiURL(lastMaxID - 1)); if (tweets.size() == 0) { logger.info(" No more tweets found."); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java index 724acef8..b0ffbf1c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java @@ -154,6 +154,10 @@ public class VkRipper extends AlbumRipper { break; } offset += elements.size(); + + if (isThisATest()) { + break; + } } waitForThreads(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java index ac522d99..5910d6b3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java @@ -58,7 +58,7 @@ public class PornhubRipper extends VideoRipper { public void rip() throws IOException { logger.info(" Retrieving " + this.url.toExternalForm()); Document doc = Http.url(this.url).get(); - Pattern p = Pattern.compile("^.*var flashvars = (.*});.*$", Pattern.DOTALL); + Pattern p = Pattern.compile("^.*'flashvars' : (.*});.*$", Pattern.DOTALL); Matcher m = p.matcher(doc.body().html()); if (m.matches()) { String title = null, @@ -90,6 +90,9 @@ public class PornhubRipper extends VideoRipper { throw new IOException(e); } } + else { + throw new IOException("Failed to download " + this.url + " : could not find 'flashvars'"); + } waitForThreads(); } } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/utils/AES.java b/src/main/java/com/rarchives/ripme/utils/AES.java index e6bc742c..b0b49ec9 100644 --- a/src/main/java/com/rarchives/ripme/utils/AES.java +++ b/src/main/java/com/rarchives/ripme/utils/AES.java @@ -1,5 +1,6 @@ package com.rarchives.ripme.utils; +import java.lang.reflect.Field; import java.util.Arrays; import javax.crypto.Cipher; @@ -9,6 +10,22 @@ import javax.crypto.spec.SecretKeySpec; public class AES { + /** + * Hack to get JCE Unlimited Strenght so we can use weird AES encryption stuff. + * From http://stackoverflow.com/a/20286961 + */ + static { + try { + Field field = Class.forName("javax.crypto.JceSecurity").getDeclaredField("isRestricted"); + if (!field.isAccessible()) { + field.setAccessible(true); + field.set(null, java.lang.Boolean.FALSE); + } + } catch (Exception ex) { + // Assume it's fine. + } + } + public static String decrypt(String cipherText, String key, int nBits) throws Exception { String res = null; nBits = nBits / 8; @@ -31,6 +48,7 @@ public class AES { cipher.init(Cipher.ENCRYPT_MODE, secretKey); keyBytes = cipher.doFinal(keyBytes); } catch (Throwable e1) { + e1.printStackTrace(); return null; } System.arraycopy(keyBytes, 0, keyBytes, nBits / 2, nBits / 2); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java index 90a9b14a..9057e0d7 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java @@ -5,6 +5,7 @@ import java.net.URL; import java.util.ArrayList; import java.util.List; +import com.rarchives.ripme.ripper.VideoRipper; import com.rarchives.ripme.ripper.rippers.video.BeegRipper; import com.rarchives.ripme.ripper.rippers.video.PornhubRipper; import com.rarchives.ripme.ripper.rippers.video.VineRipper; @@ -13,6 +14,24 @@ import com.rarchives.ripme.ripper.rippers.video.YoupornRipper; public class VideoRippersTest extends RippersTest { + private void videoTestHelper(VideoRipper ripper) { + URL oldURL = ripper.getURL(); + try { + ripper.setup(); + ripper.markAsTest(); + ripper.rip(); + // Video ripper testing is... weird. + // If we find the URL to download the video, and it's a test, + // then the ripper sets it as the ripper's URL. + assertFalse("Failed to find download url for " + oldURL, oldURL.equals(ripper.getURL())); + } catch (Exception e) { + fail("Error while ripping " + ripper.getURL() + " : " + e); + e.printStackTrace(); + } finally { + deleteDir(ripper.getWorkingDir()); + } + } + public void testXvideosRipper() throws IOException { if (!DOWNLOAD_CONTENT) { return; @@ -22,7 +41,7 @@ public class VideoRippersTest extends RippersTest { contentURLs.add(new URL("http://www.xvideos.com/video7136868/vid-20140205-wa0011")); for (URL url : contentURLs) { XvideosRipper ripper = new XvideosRipper(url); - testRipper(ripper); + videoTestHelper(ripper); } } @@ -34,7 +53,7 @@ public class VideoRippersTest extends RippersTest { contentURLs.add(new URL("http://www.pornhub.com/view_video.php?viewkey=993166542")); for (URL url : contentURLs) { PornhubRipper ripper = new PornhubRipper(url); - testRipper(ripper); + videoTestHelper(ripper); } } @@ -46,7 +65,7 @@ public class VideoRippersTest extends RippersTest { contentURLs.add(new URL("https://vine.co/v/hiqQrP0eUZx")); for (URL url : contentURLs) { VineRipper ripper = new VineRipper(url); - testRipper(ripper); + videoTestHelper(ripper); } } @@ -58,7 +77,7 @@ public class VideoRippersTest extends RippersTest { contentURLs.add(new URL("http://www.youporn.com/watch/7669155/mrs-li-amateur-69-orgasm/?from=categ")); for (URL url : contentURLs) { YoupornRipper ripper = new YoupornRipper(url); - testRipper(ripper); + videoTestHelper(ripper); } } @@ -70,9 +89,7 @@ public class VideoRippersTest extends RippersTest { contentURLs.add(new URL("http://beeg.com/4554321")); for (URL url : contentURLs) { BeegRipper ripper = new BeegRipper(url); - testRipper(ripper); + videoTestHelper(ripper); } } - - }