Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Attempt to remove transient failure of tests

Browse Source
This commit is contained in:
4pr0n 2015-02-06 03:01:02 -08:00
parent 9c964e432e
commit c524d20ccb
9 changed files with 84 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java
View File

@ -26,7 +26,7 @@ public abstract class AbstractHTMLRipper extends AlbumRipper {
public abstract Document getFirstPage() throws IOException;
public Document getNextPage(Document doc) throws IOException {
throw new IOException("getNextPage not implemented");
return null;
}
public abstract List<String> getURLsFromPage(Document page);
public List<String> getDescriptionsFromPage(Document doc) throws IOException {

2
src/main/java/com/rarchives/ripme/ripper/VideoRipper.java
View File

@ -58,7 +58,7 @@ public abstract class VideoRipper extends AbstractRipper {
}
else {
if (isThisATest()) {
System.err.println("TEST, download url: " + url);
this.url = url;
return true;
}
threadPool.addThread(new DownloadVideoThread(url, saveAs, this));

28
src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
View File

@ -105,24 +105,29 @@ public class ChanRipper extends AbstractHTMLRipper {
return Http.url(this.url).get();
}
private boolean isURLBlacklisted(String url) {
for (String blacklist_item : url_piece_blacklist) {
if (url.contains(blacklist_item)) {
logger.debug("Skipping link that contains '"+blacklist_item+"': " + url);
return true;
}
}
return false;
}
@Override
public List<String> getURLsFromPage(Document page) {
List<String> imageURLs = new ArrayList<String>();
Pattern p; Matcher m;
elementloop:
for (Element link : page.select("a")) {
if (!link.hasAttr("href")) {
continue;
}
String href = link.attr("href").trim();
//Check all blacklist items
for (String blacklist_item : url_piece_blacklist) {
if (href.contains(blacklist_item)) {
logger.debug("Skipping link that contains '"+blacklist_item+"': " + href);
continue elementloop;
}
if (isURLBlacklisted(href)) {
continue;
}
//Check all blacklist items
Boolean self_hosted = false;
if (!generalChanSite) {
for (String cdnDomain : chanSite.cdnDomains) {
@ -132,7 +137,7 @@ public class ChanRipper extends AbstractHTMLRipper {
}
}
if(self_hosted||generalChanSite){
if (self_hosted || generalChanSite){
p = Pattern.compile("^.*\\.(jpg|jpeg|png|gif|apng|webp|tif|tiff|webm)$", Pattern.CASE_INSENSITIVE);
m = p.matcher(href);
if (m.matches()) {
@ -148,10 +153,17 @@ public class ChanRipper extends AbstractHTMLRipper {
continue;
}
imageURLs.add(href);
if (isThisATest()) {
break;
}
}
} else {
//TODO also grab imgur/flickr albums (And all other supported rippers) Maybe add a setting?
}
if (isStopped()) {
break;
}
}
return imageURLs;
}

6
src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java
View File

@ -119,6 +119,9 @@ public class InstagramRipper extends AbstractJSONRipper {
@Override
public JSONObject getNextPage(JSONObject json) throws IOException {
if (isThisATest()) {
return null;
}
JSONObject pagination = json.getJSONObject("pagination");
String nextMaxID = "";
JSONArray datas = json.getJSONArray("data");
@ -162,6 +165,9 @@ public class InstagramRipper extends AbstractJSONRipper {
continue;
}
imageURLs.add(imageURL);
if (isThisATest()) {
break;
}
}
return imageURLs;
}

10
src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
View File

@ -190,6 +190,7 @@ public class TwitterRipper extends AlbumRipper {
}
}
/*
if (entities.has("urls")) {
JSONArray urls = entities.getJSONArray("urls");
JSONObject url;
@ -202,10 +203,7 @@ public class TwitterRipper extends AlbumRipper {
}
}
}
}
private void handleTweetedURL(String url) {
logger.error("[!] Need to handle URL: " + url);
*/
}
@Override
@ -223,6 +221,10 @@ public class TwitterRipper extends AlbumRipper {
Long lastMaxID = 0L;
for (int i = 0; i < MAX_REQUESTS; i++) {
if (isStopped()) {
break;
}
List<JSONObject> tweets = getTweets(getApiURL(lastMaxID - 1));
if (tweets.size() == 0) {
logger.info(" No more tweets found.");

4
src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java
View File

@ -154,6 +154,10 @@ public class VkRipper extends AlbumRipper {
break;
}
offset += elements.size();
if (isThisATest()) {
break;
}
}
waitForThreads();
}

5
src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java
View File

@ -58,7 +58,7 @@ public class PornhubRipper extends VideoRipper {
public void rip() throws IOException {
logger.info(" Retrieving " + this.url.toExternalForm());
Document doc = Http.url(this.url).get();
Pattern p = Pattern.compile("^.*var flashvars = (.*});.*$", Pattern.DOTALL);
Pattern p = Pattern.compile("^.*'flashvars' : (.*});.*$", Pattern.DOTALL);
Matcher m = p.matcher(doc.body().html());
if (m.matches()) {
String title = null,
@ -90,6 +90,9 @@ public class PornhubRipper extends VideoRipper {
throw new IOException(e);
}
}
else {
throw new IOException("Failed to download " + this.url + " : could not find 'flashvars'");
}
waitForThreads();
}
}

18
src/main/java/com/rarchives/ripme/utils/AES.java
View File

@ -1,5 +1,6 @@
package com.rarchives.ripme.utils;
import java.lang.reflect.Field;
import java.util.Arrays;
import javax.crypto.Cipher;
@ -9,6 +10,22 @@ import javax.crypto.spec.SecretKeySpec;
public class AES {
/**
* Hack to get JCE Unlimited Strenght so we can use weird AES encryption stuff.
* From http://stackoverflow.com/a/20286961
*/
static {
try {
Field field = Class.forName("javax.crypto.JceSecurity").getDeclaredField("isRestricted");
if (!field.isAccessible()) {
field.setAccessible(true);
field.set(null, java.lang.Boolean.FALSE);
}
} catch (Exception ex) {
// Assume it's fine.
}
}
public static String decrypt(String cipherText, String key, int nBits) throws Exception {
String res = null;
nBits = nBits / 8;
@ -31,6 +48,7 @@ public class AES {
cipher.init(Cipher.ENCRYPT_MODE, secretKey);
keyBytes = cipher.doFinal(keyBytes);
} catch (Throwable e1) {
e1.printStackTrace();
return null;
}
System.arraycopy(keyBytes, 0, keyBytes, nBits / 2, nBits / 2);

31
src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java
View File

@ -5,6 +5,7 @@ import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import com.rarchives.ripme.ripper.VideoRipper;
import com.rarchives.ripme.ripper.rippers.video.BeegRipper;
import com.rarchives.ripme.ripper.rippers.video.PornhubRipper;
import com.rarchives.ripme.ripper.rippers.video.VineRipper;
@ -13,6 +14,24 @@ import com.rarchives.ripme.ripper.rippers.video.YoupornRipper;
public class VideoRippersTest extends RippersTest {
private void videoTestHelper(VideoRipper ripper) {
URL oldURL = ripper.getURL();
try {
ripper.setup();
ripper.markAsTest();
ripper.rip();
// Video ripper testing is... weird.
// If we find the URL to download the video, and it's a test,
// then the ripper sets it as the ripper's URL.
assertFalse("Failed to find download url for " + oldURL, oldURL.equals(ripper.getURL()));
} catch (Exception e) {
fail("Error while ripping " + ripper.getURL() + " : " + e);
e.printStackTrace();
} finally {
deleteDir(ripper.getWorkingDir());
}
}
public void testXvideosRipper() throws IOException {
if (!DOWNLOAD_CONTENT) {
return;
@ -22,7 +41,7 @@ public class VideoRippersTest extends RippersTest {
contentURLs.add(new URL("http://www.xvideos.com/video7136868/vid-20140205-wa0011"));
for (URL url : contentURLs) {
XvideosRipper ripper = new XvideosRipper(url);
testRipper(ripper);
videoTestHelper(ripper);
}
}
@ -34,7 +53,7 @@ public class VideoRippersTest extends RippersTest {
contentURLs.add(new URL("http://www.pornhub.com/view_video.php?viewkey=993166542"));
for (URL url : contentURLs) {
PornhubRipper ripper = new PornhubRipper(url);
testRipper(ripper);
videoTestHelper(ripper);
}
}
@ -46,7 +65,7 @@ public class VideoRippersTest extends RippersTest {
contentURLs.add(new URL("https://vine.co/v/hiqQrP0eUZx"));
for (URL url : contentURLs) {
VineRipper ripper = new VineRipper(url);
testRipper(ripper);
videoTestHelper(ripper);
}
}
@ -58,7 +77,7 @@ public class VideoRippersTest extends RippersTest {
contentURLs.add(new URL("http://www.youporn.com/watch/7669155/mrs-li-amateur-69-orgasm/?from=categ"));
for (URL url : contentURLs) {
YoupornRipper ripper = new YoupornRipper(url);
testRipper(ripper);
videoTestHelper(ripper);
}
}
@ -70,9 +89,7 @@ public class VideoRippersTest extends RippersTest {
contentURLs.add(new URL("http://beeg.com/4554321"));
for (URL url : contentURLs) {
BeegRipper ripper = new BeegRipper(url);
testRipper(ripper);
videoTestHelper(ripper);
}
}
}