Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Better integration tests, testing out TravisCI

Browse Source
This commit is contained in:
4pr0n 2015-02-05 23:58:17 -08:00
parent c2a9d680b2
commit 9c964e432e
31 changed files with 229 additions and 292 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.travis.yml Normal file
View File

@ -0,0 +1,4 @@
language: java
jdk:
- oraclejdk8
- oraclejdk7

2
pom.xml
View File

@ -21,7 +21,7 @@
<!-- jsoup HTML parser library @ http://jsoup.org/ -->
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.7.3</version>
<version>1.8.1</version>
</dependency>
<dependency>
<groupId>org.json</groupId>

4
src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java
View File

@ -72,11 +72,11 @@ public abstract class AbstractHTMLRipper extends AlbumRipper {
}
for (String imageURL : imageURLs) {
index += 1;
downloadURL(new URL(imageURL), index);
if (isStopped()) {
break;
}
index += 1;
downloadURL(new URL(imageURL), index);
}
if (hasDescriptionSupport()) {
List<String> textURLs = getDescriptionsFromPage(doc);

12
src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
View File

@ -28,7 +28,7 @@ public abstract class AbstractRipper
protected static final Logger logger = Logger.getLogger(AbstractRipper.class);
public static final String USER_AGENT =
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:34.0) Gecko/20100101 Firefox/34.0";
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:35.0) Gecko/20100101 Firefox/35.0";
protected URL url;
protected File workingDir;
@ -42,6 +42,7 @@ public abstract class AbstractRipper
public abstract String getGID(URL url) throws MalformedURLException;
private boolean shouldStop = false;
private boolean thisIsATest = false;
public void stop() {
shouldStop = true;
@ -370,6 +371,11 @@ public abstract class AbstractRipper
// Do nothing
}
// Thar be overloaded methods afoot
/** Methods for detecting when we're running a test. */
public void markAsTest() {
thisIsATest = true;
}
public boolean isThisATest() {
return thisIsATest;
}
}

6
src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java
View File

@ -39,6 +39,12 @@ public abstract class AlbumRipper extends AbstractRipper {
}
public boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies) {
// Only download one file if this is a test.
if (super.isThisATest() &&
(itemsPending.size() > 0 || itemsCompleted.size() > 0 || itemsErrored.size() > 0)) {
stop();
return false;
}
if (!allowDuplicates()
&& ( itemsPending.containsKey(url)
|| itemsCompleted.containsKey(url)

4
src/main/java/com/rarchives/ripme/ripper/VideoRipper.java
View File

@ -57,6 +57,10 @@ public abstract class VideoRipper extends AbstractRipper {
}
}
else {
if (isThisATest()) {
System.err.println("TEST, download url: " + url);
return true;
}
threadPool.addThread(new DownloadVideoThread(url, saveAs, this));
}
return true;

7
src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
View File

@ -45,7 +45,7 @@ public class ChanRipper extends AbstractHTMLRipper {
}
if (chanSite == null) {
chanSite = new ChanSite(Arrays.asList(url.getHost()));
}
}
}
@Override
@ -61,7 +61,7 @@ public class ChanRipper extends AbstractHTMLRipper {
}
@Override
public boolean canRip(URL url) {
public boolean canRip(URL url) {
for (ChanSite _chanSite : explicit_domains) {
if (_chanSite.domains.contains(url.getHost())) {
return true;
@ -70,6 +70,7 @@ public class ChanRipper extends AbstractHTMLRipper {
return url.toExternalForm().contains("/res/") // Most chans
|| url.toExternalForm().contains("/thread/"); // 4chan, archive.moe
}
/**
* For example the achrives are all known. (Check 4chan-x)
* Should be based on the software the specific chan uses.
@ -79,7 +80,7 @@ public class ChanRipper extends AbstractHTMLRipper {
public String getGID(URL url) throws MalformedURLException {
Pattern p; Matcher m;
String u = url.toExternalForm();
String u = url.toExternalForm();
if (u.contains("/thread/") || u.contains("/res/")) {
p = Pattern.compile("^.*\\.[a-z]{1,3}/[a-zA-Z0-9]+/(thread|res)/([0-9]+)(\\.html|\\.php)?.*$");
m = p.matcher(u);

11
src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
View File

@ -142,6 +142,11 @@ public class DeviantartRipper extends AbstractHTMLRipper {
}
triedURLs.add(fullSize);
imageURLs.add(fullSize);
if (isThisATest()) {
// Only need one image for a test
break;
}
}
return imageURLs;
}
@ -164,6 +169,9 @@ public class DeviantartRipper extends AbstractHTMLRipper {
}
@Override
public Document getNextPage(Document page) throws IOException {
if (isThisATest()) {
return null;
}
Elements nextButtons = page.select("li.next > a");
if (nextButtons.size() == 0) {
throw new IOException("No next page found");
@ -225,6 +233,9 @@ public class DeviantartRipper extends AbstractHTMLRipper {
*/
@Override
public String getDescription(String page) {
if (isThisATest()) {
return null;
}
try {
// Fetch the image page
Response resp = Http.url(page)

1
src/main/java/com/rarchives/ripme/ripper/rippers/FineboxRipper.java
View File

@ -57,6 +57,7 @@ public class FineboxRipper extends AlbumRipper {
if(videourl.substring(0,4)!="http"){
videourl = "http://"+DOMAIN+ videourl;
}
logger.error("URL to download: " + videourl);
if(!addURLToDownload(new URL(videourl))){
hasPagesLeft = false;
break;

31
src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java
View File

@ -17,6 +17,7 @@ import com.rarchives.ripme.utils.Http;
public class ImagefapRipper extends AbstractHTMLRipper {
private Document albumDoc = null;
private boolean isNewAlbumType = false;
public ImagefapRipper(URL url) throws IOException {
super(url);
@ -37,16 +38,25 @@ public class ImagefapRipper extends AbstractHTMLRipper {
@Override
public URL sanitizeURL(URL url) throws MalformedURLException {
String gid = getGID(url);
URL newURL = new URL("http://www.imagefap.com/gallery.php?gid="
+ gid + "&view=2");
String newURL = "http://www.imagefap.com/gallery.php?";
if (isNewAlbumType) {
newURL += "p";
}
newURL += "gid=" + gid + "&view=2";
logger.debug("Changed URL from " + url + " to " + newURL);
return newURL;
return new URL(newURL);
}
@Override
public String getGID(URL url) throws MalformedURLException {
Pattern p; Matcher m;
p = Pattern.compile("^.*imagefap.com/gallery.php\\?pgid=([a-f0-9]+).*$");
m = p.matcher(url.toExternalForm());
if (m.matches()) {
isNewAlbumType = true;
return m.group(1);
}
p = Pattern.compile("^.*imagefap.com/gallery.php\\?gid=([0-9]+).*$");
m = p.matcher(url.toExternalForm());
if (m.matches()) {
@ -58,12 +68,24 @@ public class ImagefapRipper extends AbstractHTMLRipper {
if (m.matches()) {
return m.group(1);
}
p = Pattern.compile("^.*imagefap.com/pictures/([a-f0-9]+).*$");
m = p.matcher(url.toExternalForm());
if (m.matches()) {
isNewAlbumType = true;
return m.group(1);
}
p = Pattern.compile("^.*imagefap.com/gallery/([0-9]+).*$");
m = p.matcher(url.toExternalForm());
if (m.matches()) {
return m.group(1);
}
p = Pattern.compile("^.*imagefap.com/gallery/([a-f0-9]+).*$");
m = p.matcher(url.toExternalForm());
if (m.matches()) {
isNewAlbumType = true;
return m.group(1);
}
throw new MalformedURLException(
"Expected imagefap.com gallery formats: "
@ -108,6 +130,9 @@ public class ImagefapRipper extends AbstractHTMLRipper {
"http://x.*.fap.to/images/thumb/",
"http://fap.to/images/full/");
imageURLs.add(image);
if (isThisATest()) {
break;
}
}
return imageURLs;
}

1
src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java
View File

@ -95,7 +95,6 @@ public class InstagramRipper extends AbstractJSONRipper {
}
private String getUserID(URL url) throws IOException {
logger.info("Retrieving " + url);
this.sendUpdate(STATUS.LOADING_RESOURCE, url.toExternalForm());
Document doc = Http.url(url).get();
for (Element element : doc.select("input[id=user_public]")) {

29
src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java
View File

@ -46,25 +46,6 @@ public class MotherlessRipper extends AlbumRipper {
public URL sanitizeURL(URL url) throws MalformedURLException {
return url;
}
/*
@Override
public Document getFirstPage() throws IOException {
}
@Override
public Document getNextPage(Document doc) throws IOException {
}
@Override
public List<String> getURLsFromPage(Document doc) {
List<String> imageURLs = new ArrayList<String>();
return imageURLs;
}
@Override
public void downloadURL(URL url, int index) {
addURLToDownload(url, getPrefix(index));
}
*/
@Override
public String getGID(URL url) throws MalformedURLException {
@ -115,9 +96,17 @@ public class MotherlessRipper extends AlbumRipper {
url = new URL(thumbURL);
}
index += 1;
// Create thread for finding image at "url" page
MotherlessImageThread mit = new MotherlessImageThread(url, index);
motherlessThreadPool.addThread(mit);
if (isThisATest()) {
break;
}
}
if (isThisATest()) {
break;
}
// Next page
nextURL = null;
@ -146,7 +135,7 @@ public class MotherlessRipper extends AlbumRipper {
@Override
public void run() {
try {
if (isStopped()) {
if (isStopped() && !isThisATest()) {
return;
}
String u = this.url.toExternalForm();

4
src/main/java/com/rarchives/ripme/ripper/rippers/SeeniveRipper.java
View File

@ -56,7 +56,7 @@ public class SeeniveRipper extends AlbumRipper {
break;
}
String card = element.attr("href"); // "/v/<video_id>"
URL videoURL = new URL("https://seenive.com" + card);
URL videoURL = new URL("http://seenive.com" + card);
SeeniveImageThread vit = new SeeniveImageThread(videoURL);
seeniveThreadPool.addThread(vit);
lastID = card.substring(card.lastIndexOf('/') + 1);
@ -96,7 +96,7 @@ public class SeeniveRipper extends AlbumRipper {
Pattern p = Pattern.compile("^https?://(www\\.)?seenive\\.com/u/([a-zA-Z0-9]{1,}).*$");
Matcher m = p.matcher(url.toExternalForm());
if (!m.matches()) {
throw new MalformedURLException("Expected format: https://seenive.com/u/USERID");
throw new MalformedURLException("Expected format: http://seenive.com/u/USERID");
}
return m.group(m.groupCount());
}

1
src/main/java/com/rarchives/ripme/utils/Http.java
View File

@ -53,7 +53,6 @@ public class Http {
connection.method(Method.GET);
connection.timeout(TIMEOUT);
connection.maxBodySize(0);
}
// Setters

39
src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java
View File

@ -35,13 +35,12 @@ public class ChanRipperTest extends RippersTest {
passURLs.add(new URL("http://xchan.pw/porn/res/437.html"));
passURLs.add(new URL("http://archive.moe/c/thread/2295132/"));
for (URL url : passURLs) {
try {
ChanRipper ripper = new ChanRipper(url);
assert(ripper.canRip(url));
deleteDir(ripper.getWorkingDir());
} catch (Exception e) {
fail("Failed to instantiate ripper for " + url + " with message: "+e.toString());
}
ChanRipper ripper = new ChanRipper(url);
ripper.setup();
assert(ripper.canRip(url));
assertNotNull("Ripper for " + url + " did not have a valid working directory.",
ripper.getWorkingDir());
deleteDir(ripper.getWorkingDir());
}
}
@ -52,22 +51,20 @@ public class ChanRipperTest extends RippersTest {
List<URL> contentURLs = new ArrayList<URL>();
// URLs that should return more than 1 image
contentURLs.add(new URL("http://desuchan.net/v/res/7034.html"));
contentURLs.add(new URL("http://boards.4chan.org/r/res/12225949"));
contentURLs.add(new URL("http://boards.420chan.org/ana/res/75984.php"));
contentURLs.add(new URL("http://7chan.org/gif/res/23795.html"));
contentURLs.add(new URL("http://unichan2.org/b/res/518004.html"));
contentURLs.add(new URL("http://xchan.pw/porn/res/437.html"));
contentURLs.add(new URL("http://archive.4plebs.org/hr/thread/2215899/"));
contentURLs.add(new URL("http://archive.4plebs.org/s4s/thread/3005257/"));
// Most *chans have volatile threads & can't be trusted for integration testing.
//contentURLs.add(new URL("http://boards.4chan.org/r/res/12225949"));
//contentURLs.add(new URL("http://7chan.org/gif/res/23795.html"));
//contentURLs.add(new URL("http://unichan2.org/b/res/518004.html"));
// xchan has an HTTPS certificaiton error...
//contentURLs.add(new URL("http://xchan.pw/porn/res/437.html"));
for (URL url : contentURLs) {
try {
ChanRipper ripper = new ChanRipper(url);
ripper.rip();
assert(ripper.getWorkingDir().listFiles().length > 1);
deleteDir(ripper.getWorkingDir());
} catch (Exception e) {
e.printStackTrace();
fail("Error while ripping URL " + url + ": " + e.getMessage());
}
ChanRipper ripper = new ChanRipper(url);
testRipper(ripper);
}
}

11
src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java
View File

@ -23,15 +23,8 @@ public class DeviantartRipperTest extends RippersTest {
contentURLs.add(new URL("http://geekysica.deviantart.com/gallery/35209412"));
for (URL url : contentURLs) {
try {
DeviantartRipper ripper = new DeviantartRipper(url);
ripper.rip();
assert(ripper.getWorkingDir().listFiles().length > 1);
deleteDir(ripper.getWorkingDir());
} catch (Exception e) {
e.printStackTrace();
fail("Error while ripping URL " + url + ": " + e.getMessage());
}
DeviantartRipper ripper = new DeviantartRipper(url);
testRipper(ripper);
}
}

11
src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java
View File

@ -18,15 +18,8 @@ public class EightmusesRipperTest extends RippersTest {
contentURLs.add(new URL("http://www.8muses.com/index/category/jab-hotassneighbor7"));
for (URL url : contentURLs) {
try {
EightmusesRipper ripper = new EightmusesRipper(url);
ripper.rip();
assert(ripper.getWorkingDir().listFiles().length > 1);
deleteDir(ripper.getWorkingDir());
} catch (Exception e) {
e.printStackTrace();
fail("Error while ripping URL " + url + ": " + e.getMessage());
}
EightmusesRipper ripper = new EightmusesRipper(url);
testRipper(ripper);
}
}

17
src/test/java/com/rarchives/ripme/tst/ripper/rippers/FineboxRipperTest.java
View File

@ -5,27 +5,24 @@ import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import com.rarchives.ripme.ripper.rippers.FineboxRipper;
public class FineboxRipperTest extends RippersTest {
public void testVineboxAlbums() throws IOException {
if (DOWNLOAD_CONTENT) {
if (!DOWNLOAD_CONTENT) {
return;
}
Logger.getRootLogger().setLevel(Level.ALL);
List<URL> contentURLs = new ArrayList<URL>();
contentURLs.add(new URL("http://vinebox.co/u/wi57hMjc2Ka"));
contentURLs.add(new URL("http://finebox.co/u/wi57hMjc2Ka"));
for (URL url : contentURLs) {
try {
FineboxRipper ripper = new FineboxRipper(url);
ripper.rip();
assert(ripper.getWorkingDir().listFiles().length > 1);
deleteDir(ripper.getWorkingDir());
} catch (Exception e) {
e.printStackTrace();
fail("Error while ripping URL " + url + ": " + e.getMessage());
}
FineboxRipper ripper = new FineboxRipper(url);
testRipper(ripper);
}
}

13
src/test/java/com/rarchives/ripme/tst/ripper/rippers/GonewildRipperTest.java
View File

@ -14,17 +14,10 @@ public class GonewildRipperTest extends RippersTest {
return;
}
List<URL> contentURLs = new ArrayList<URL>();
contentURLs.add(new URL("http://gonewild.com/u/amle69"));
contentURLs.add(new URL("http://gonewild.com/user/amle69"));
for (URL url : contentURLs) {
try {
GonewildRipper ripper = new GonewildRipper(url);
ripper.rip();
assert(ripper.getWorkingDir().listFiles().length > 1);
deleteDir(ripper.getWorkingDir());
} catch (Exception e) {
e.printStackTrace();
fail("Error while ripping URL " + url + ": " + e.getMessage());
}
GonewildRipper ripper = new GonewildRipper(url);
testRipper(ripper);
}
}

38
src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java
View File

@ -2,43 +2,29 @@ package com.rarchives.ripme.tst.ripper.rippers;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.rarchives.ripme.ripper.rippers.ImagefapRipper;
public class ImagefapRipperTest extends RippersTest {
public void testImagefapGID() throws IOException {
if (!DOWNLOAD_CONTENT) {
return;
}
Map<URL, String> testURLs = new HashMap<URL, String>();
testURLs.put(new URL("http://www.imagefap.com/pictures/4649440/Frozen-%28Elsa-and-Anna%29?view=2"), "Frozen (Elsa and Anna)");
for (URL url : testURLs.keySet()) {
ImagefapRipper ripper = new ImagefapRipper(url);
assertEquals(testURLs.get(url), ripper.getAlbumTitle(ripper.getURL()));
deleteDir(ripper.getWorkingDir());
}
}
public void testImagefapAlbums() throws IOException {
if (!DOWNLOAD_CONTENT) {
return;
}
List<URL> contentURLs = new ArrayList<URL>();
contentURLs.add(new URL("http://www.imagefap.com/pictures/4649440/Frozen-%28Elsa-and-Anna%29?view=2"));
for (URL url : contentURLs) {
try {
ImagefapRipper ripper = new ImagefapRipper(url);
ripper.rip();
assert(ripper.getWorkingDir().listFiles().length > 1);
deleteDir(ripper.getWorkingDir());
} catch (Exception e) {
fail("Error while ripping URL " + url + ": " + e.getMessage());
}
Map<URL, String> testURLs = new HashMap<URL, String>();
// Album with specific title
testURLs.put(new URL("http://www.imagefap.com/pictures/4649440/Frozen-%28Elsa-and-Anna%29?view=2"),
"Frozen (Elsa and Anna)");
// New URL format
testURLs.put(new URL("http://www.imagefap.com/gallery.php?pgid=fffd68f659befa5535cf78f014e348f1"),
"imagefap_fffd68f659befa5535cf78f014e348f1");
for (URL url : testURLs.keySet()) {
ImagefapRipper ripper = new ImagefapRipper(url);
testRipper(ripper);
}
}

43
src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java
View File

@ -30,59 +30,24 @@ public class ImgurRipperTest extends RippersTest {
}
}
public void testImgurURLPasses() throws IOException {
List<URL> passURLs = new ArrayList<URL>();
// Imgur URLs that should work
passURLs.add(new URL("http://imgur.com/a/XPd4F"));
passURLs.add(new URL("http://imgur.com/a/XPd4F/"));
passURLs.add(new URL("http://imgur.com/a/WxG6f/all"));
passURLs.add(new URL("http://imgur.com/a/WxG6f/layout/vertical#0"));
passURLs.add(new URL("http://imgur.com/a/WxG6f/layout/horizontal#0"));
passURLs.add(new URL("http://imgur.com/a/WxG6f/layout/grid#0"));
passURLs.add(new URL("http://imgur.com/YOdjht3,x5VxH9G,5juXjJ2"));
passURLs.add(new URL("http://markedone911.imgur.com"));
passURLs.add(new URL("http://markedone911.imgur.com/"));
passURLs.add(new URL("http://imgur.com/r/nsfw_oc/top/all"));
passURLs.add(new URL("http://imgur.com/r/nsfw_oc/top"));
passURLs.add(new URL("http://imgur.com/r/nsfw_oc/new"));
passURLs.add(new URL("http://imgur.com/r/nsfw_oc"));
for (URL url : passURLs) {
try {
ImgurRipper ripper = new ImgurRipper(url);
assert(ripper.canRip(url));
deleteDir(ripper.getWorkingDir());
} catch (Exception e) {
fail("Failed to instantiate ripper for " + url);
}
}
}
public void testImgurAlbums() throws IOException {
if (!DOWNLOAD_CONTENT) {
return;
}
List<URL> contentURLs = new ArrayList<URL>();
// URLs that should return more than 1 image
/*
contentURLs.add(new URL("http://imgur.com/a/hqJIu")); // Vertical layout
contentURLs.add(new URL("http://imgur.com/a/dS9OQ#0")); // Horizontal layout
contentURLs.add(new URL("http://imgur.com/a/YpsW9#0")); // Grid layout
contentURLs.add(new URL("http://imgur.com/a/WxG6f/layout/vertical#0"));
contentURLs.add(new URL("http://imgur.com/a/WxG6f/layout/horizontal#0"));
contentURLs.add(new URL("http://imgur.com/a/WxG6f/layout/grid#0"));
contentURLs.add(new URL("http://imgur.com/r/nsfw_oc/top/all"));
*/
// Sometimes hangs up
//contentURLs.add(new URL("http://imgur.com/r/nsfw_oc/top/all"));
contentURLs.add(new URL("http://imgur.com/a/bXQpH"));
for (URL url : contentURLs) {
try {
ImgurRipper ripper = new ImgurRipper(url);
ripper.rip();
assert(ripper.getWorkingDir().listFiles().length > 1);
deleteDir(ripper.getWorkingDir());
} catch (Exception e) {
fail("Error while ripping URL " + url + ": " + e.getMessage());
}
ImgurRipper ripper = new ImgurRipper(url);
testRipper(ripper);
}
}

13
src/test/java/com/rarchives/ripme/tst/ripper/rippers/InstagramRipperTest.java
View File

@ -19,6 +19,7 @@ public class InstagramRipperTest extends RippersTest {
testURLs.put(new URL("http://statigr.am/username"), "username");
for (URL url : testURLs.keySet()) {
InstagramRipper ripper = new InstagramRipper(url);
ripper.setup();
assertEquals(testURLs.get(url), ripper.getGID(ripper.getURL()));
deleteDir(ripper.getWorkingDir());
}
@ -29,16 +30,10 @@ public class InstagramRipperTest extends RippersTest {
return;
}
List<URL> contentURLs = new ArrayList<URL>();
contentURLs.add(new URL("http://instagram.com/feelgoodincc#"));
contentURLs.add(new URL("http://instagram.com/tayloralesia/"));
for (URL url : contentURLs) {
try {
InstagramRipper ripper = new InstagramRipper(url);
ripper.rip();
assert(ripper.getWorkingDir().listFiles().length > 1);
deleteDir(ripper.getWorkingDir());
} catch (Exception e) {
fail("Error while ripping URL " + url + ": " + e.getMessage());
}
InstagramRipper ripper = new InstagramRipper(url);
testRipper(ripper);
}
}

14
src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java
View File

@ -18,18 +18,12 @@ public class MotherlessRipperTest extends RippersTest {
// Image album
contentURLs.add(new URL("http://motherless.com/G4DAA18D"));
// Video album
contentURLs.add(new URL("http://motherless.com/GFD0F537"));
// XXX: Commented out because test takes too long to download the file.
// contentURLs.add(new URL("http://motherless.com/GFD0F537"));
for (URL url : contentURLs) {
try {
MotherlessRipper ripper = new MotherlessRipper(url);
ripper.rip();
assert(ripper.getWorkingDir().listFiles().length > 1);
deleteDir(ripper.getWorkingDir());
} catch (Exception e) {
e.printStackTrace();
fail("Error while ripping URL " + url + ": " + e.getMessage());
}
MotherlessRipper ripper = new MotherlessRipper(url);
testRipper(ripper);
}
}

16
src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java
View File

@ -14,20 +14,12 @@ public class RedditRipperTest extends RippersTest {
return;
}
List<URL> contentURLs = new ArrayList<URL>();
//contentURLs.add(new URL("http://www.reddit.com/r/nsfw_oc"));
//contentURLs.add(new URL("http://www.reddit.com/r/nsfw_oc/top?t=all"));
//contentURLs.add(new URL("http://www.reddit.com/u/gingerpuss"));
contentURLs.add(new URL("http://www.reddit.com/r/nsfw_oc"));
contentURLs.add(new URL("http://www.reddit.com/r/nsfw_oc/top?t=all"));
contentURLs.add(new URL("http://www.reddit.com/r/UnrealGirls/comments/1ziuhl/in_class_veronique_popa/"));
for (URL url : contentURLs) {
try {
RedditRipper ripper = new RedditRipper(url);
ripper.rip();
assert(ripper.getWorkingDir().listFiles().length > 1);
deleteDir(ripper.getWorkingDir());
} catch (Exception e) {
e.printStackTrace();
fail("Error while ripping URL " + url + ": " + e.getMessage());
}
RedditRipper ripper = new RedditRipper(url);
testRipper(ripper);
}
}

69
src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java
View File

@ -1,30 +1,87 @@
package com.rarchives.ripme.tst.ripper.rippers;
import java.io.File;
import java.io.IOException;
import junit.framework.TestCase;
import org.apache.log4j.Logger;
import com.rarchives.ripme.ripper.AbstractRipper;
import com.rarchives.ripme.utils.Utils;
/**
* Contains helper methods for testing rippers.
*/
public class RippersTest extends TestCase {
// Flag for avoiding downloading content with every unit test
public final boolean DOWNLOAD_CONTENT = false;
public final boolean DOWNLOAD_CONTENT = true;
public void testNothing() {
// Avoid complaints about no test cases in this file.
public final Logger logger = Logger.getLogger(RippersTest.class);
/** Dummy test to make JUnit not complain */
public void test() {
assert(true);
}
protected void testRipper(AbstractRipper ripper) {
try {
Utils.setConfigInteger("page.timeout", 5 * 1000);
ripper.setup();
ripper.markAsTest();
System.err.println("Sleeping 1000ms");
Thread.sleep(1000);
ripper.rip();
for (File f : ripper.getWorkingDir().listFiles()) {
System.err.println(f.toString());
}
assertTrue("Failed to download files from " + ripper.getURL(), ripper.getWorkingDir().listFiles().length >= 1);
} catch (IOException e) {
if (e.getMessage().contains("Ripping interrupted")) {
// We expect some rips to get interrupted
}
else {
e.printStackTrace();
fail("Failed to rip " + ripper.getURL() + " : " + e.getMessage());
}
} catch (Exception e) {
e.printStackTrace();
fail("Failed to rip " + ripper.getURL() + " : " + e.getMessage());
}
finally {
deleteDir(ripper.getWorkingDir());
}
}
/** File extensions that are safe to delete. */
private static final String[] SAFE_EXTENSIONS =
{"png", "jpg", "jpeg", "gif",
"mp4", "webm", "mov", "mpg", "mpeg",
"txt", "log", "php"};
/** Recursively deletes a directory */
protected void deleteDir(File dir) {
return;
/*
if (!dir.getName().contains("_")) {
// All ripped albums contain an underscore
// Don't delete an album if it doesn't have an underscore
return;
}
for (File f : dir.listFiles()) {
boolean safe = false;
for (String ext : SAFE_EXTENSIONS) {
safe |= f.getAbsolutePath().toLowerCase().endsWith("." + ext);
}
if (!safe) {
// Found a file we shouldn't delete! Stop deleting immediately.
return;
}
if (f.isDirectory()) {
deleteDir(f);
}
f.delete();
}
dir.delete();
//*/
}
}

11
src/test/java/com/rarchives/ripme/tst/ripper/rippers/SeeniveRipperTest.java
View File

@ -16,15 +16,8 @@ public class SeeniveRipperTest extends RippersTest {
List<URL> contentURLs = new ArrayList<URL>();
contentURLs.add(new URL("http://seenive.com/u/946491170220040192"));
for (URL url : contentURLs) {
try {
SeeniveRipper ripper = new SeeniveRipper(url);
ripper.rip();
assert(ripper.getWorkingDir().listFiles().length > 1);
deleteDir(ripper.getWorkingDir());
} catch (Exception e) {
e.printStackTrace();
fail("Error while ripping URL " + url + ": " + e.getMessage());
}
SeeniveRipper ripper = new SeeniveRipper(url);
testRipper(ripper);
}
}

13
src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java
View File

@ -16,17 +16,10 @@ public class TumblrRipperTest extends RippersTest {
List<URL> contentURLs = new ArrayList<URL>();
contentURLs.add(new URL("http://wrouinr.tumblr.com/archive"));
contentURLs.add(new URL("http://topinstagirls.tumblr.com/tagged/berlinskaya"));
contentURLs.add(new URL("http://fittingroomgirls.tumblr.com/post/78268776776"));
contentURLs.add(new URL("http://genekellyclarkson.tumblr.com/post/86100752527/lucyannebrooks-rachaelboden-friends-goodtimes-bed-boobs"));
for (URL url : contentURLs) {
try {
TumblrRipper ripper = new TumblrRipper(url);
ripper.rip();
assert(ripper.getWorkingDir().listFiles().length > 1);
deleteDir(ripper.getWorkingDir());
} catch (Exception e) {
e.printStackTrace();
fail("Error while ripping URL " + url + ": " + e.getMessage());
}
TumblrRipper ripper = new TumblrRipper(url);
testRipper(ripper);
}
}

13
src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwitterRipperTest.java
View File

@ -14,18 +14,11 @@ public class TwitterRipperTest extends RippersTest {
return;
}
List<URL> contentURLs = new ArrayList<URL>();
//contentURLs.add(new URL("https://twitter.com/danngamber01/media"));
contentURLs.add(new URL("https://twitter.com/danngamber01/media"));
contentURLs.add(new URL("https://twitter.com/search?q=from%3Apurrbunny%20filter%3Aimages&src=typd"));
for (URL url : contentURLs) {
try {
TwitterRipper ripper = new TwitterRipper(url);
ripper.rip();
assert(ripper.getWorkingDir().listFiles().length > 1);
deleteDir(ripper.getWorkingDir());
} catch (Exception e) {
e.printStackTrace();
fail("Error while ripping URL " + url + ": " + e.getMessage());
}
TwitterRipper ripper = new TwitterRipper(url);
testRipper(ripper);
}
}
}

57
src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java
View File

@ -21,15 +21,8 @@ public class VideoRippersTest extends RippersTest {
contentURLs.add(new URL("http://www.xvideos.com/video1428195/stephanie_first_time_anal"));
contentURLs.add(new URL("http://www.xvideos.com/video7136868/vid-20140205-wa0011"));
for (URL url : contentURLs) {
try {
XvideosRipper ripper = new XvideosRipper(url);
ripper.rip();
assert(ripper.getWorkingDir().listFiles().length > 1);
deleteDir(ripper.getWorkingDir());
} catch (Exception e) {
e.printStackTrace();
fail("Error while ripping URL " + url + ": " + e.getMessage());
}
XvideosRipper ripper = new XvideosRipper(url);
testRipper(ripper);
}
}
@ -40,18 +33,11 @@ public class VideoRippersTest extends RippersTest {
List<URL> contentURLs = new ArrayList<URL>();
contentURLs.add(new URL("http://www.pornhub.com/view_video.php?viewkey=993166542"));
for (URL url : contentURLs) {
try {
PornhubRipper ripper = new PornhubRipper(url);
ripper.rip();
assert(ripper.getWorkingDir().listFiles().length > 1);
deleteDir(ripper.getWorkingDir());
} catch (Exception e) {
e.printStackTrace();
fail("Error while ripping URL " + url + ": " + e.getMessage());
}
PornhubRipper ripper = new PornhubRipper(url);
testRipper(ripper);
}
}
public void testVineRipper() throws IOException {
if (!DOWNLOAD_CONTENT) {
return;
@ -59,15 +45,8 @@ public class VideoRippersTest extends RippersTest {
List<URL> contentURLs = new ArrayList<URL>();
contentURLs.add(new URL("https://vine.co/v/hiqQrP0eUZx"));
for (URL url : contentURLs) {
try {
VineRipper ripper = new VineRipper(url);
ripper.rip();
assert(ripper.getWorkingDir().listFiles().length > 1);
deleteDir(ripper.getWorkingDir());
} catch (Exception e) {
e.printStackTrace();
fail("Error while ripping URL " + url + ": " + e.getMessage());
}
VineRipper ripper = new VineRipper(url);
testRipper(ripper);
}
}
@ -78,15 +57,8 @@ public class VideoRippersTest extends RippersTest {
List<URL> contentURLs = new ArrayList<URL>();
contentURLs.add(new URL("http://www.youporn.com/watch/7669155/mrs-li-amateur-69-orgasm/?from=categ"));
for (URL url : contentURLs) {
try {
YoupornRipper ripper = new YoupornRipper(url);
ripper.rip();
assert(ripper.getWorkingDir().listFiles().length > 1);
deleteDir(ripper.getWorkingDir());
} catch (Exception e) {
e.printStackTrace();
fail("Error while ripping URL " + url + ": " + e.getMessage());
}
YoupornRipper ripper = new YoupornRipper(url);
testRipper(ripper);
}
}
@ -97,15 +69,8 @@ public class VideoRippersTest extends RippersTest {
List<URL> contentURLs = new ArrayList<URL>();
contentURLs.add(new URL("http://beeg.com/4554321"));
for (URL url : contentURLs) {
try {
BeegRipper ripper = new BeegRipper(url);
ripper.rip();
assert(ripper.getWorkingDir().listFiles().length > 1);
deleteDir(ripper.getWorkingDir());
} catch (Exception e) {
e.printStackTrace();
fail("Error while ripping URL " + url + ": " + e.getMessage());
}
BeegRipper ripper = new BeegRipper(url);
testRipper(ripper);
}
}

15
src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java
View File

@ -15,18 +15,11 @@ public class VkRipperTest extends RippersTest {
}
List<URL> contentURLs = new ArrayList<URL>();
contentURLs.add(new URL("https://vk.com/album45506334_172415053"));
//contentURLs.add(new URL("https://vk.com/album45506334_0"));
//contentURLs.add(new URL("https://vk.com/photos45506334"));
contentURLs.add(new URL("https://vk.com/album45506334_0"));
contentURLs.add(new URL("https://vk.com/photos45506334"));
for (URL url : contentURLs) {
try {
VkRipper ripper = new VkRipper(url);
ripper.rip();
assert(ripper.getWorkingDir().listFiles().length > 1);
deleteDir(ripper.getWorkingDir());
} catch (Exception e) {
e.printStackTrace();
fail("Error while ripping URL " + url + ": " + e.getMessage());
}
VkRipper ripper = new VkRipper(url);
testRipper(ripper);
}
}

11
src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java
View File

@ -17,15 +17,8 @@ public class XhamsterRipperTest extends RippersTest {
contentURLs.add(new URL("http://xhamster.com/photos/gallery/1462237/alyssa_gadson.html"));
contentURLs.add(new URL("http://xhamster.com/photos/gallery/2941201/tableau_d_039_art_ii.html"));
for (URL url : contentURLs) {
try {
XhamsterRipper ripper = new XhamsterRipper(url);
ripper.rip();
assert(ripper.getWorkingDir().listFiles().length > 1);
deleteDir(ripper.getWorkingDir());
} catch (Exception e) {
e.printStackTrace();
fail("Error while ripping URL " + url + ": " + e.getMessage());
}
XhamsterRipper ripper = new XhamsterRipper(url);
testRipper(ripper);
}
}