From c5ea044f7907ea3724dbcb9b5706138d8b84406f Mon Sep 17 00:00:00 2001 From: 4pr0n Date: Mon, 9 Feb 2015 23:29:29 -0800 Subject: [PATCH] Various fixes to tests: Ability to set log level, lots of debugging messages Turn on debug logging during tests, simplified test cases for HTML ripper Fix fusktator ripper, added test Fixed gifyo, added test Added tests for *all* rippers Adding a few album-guessing URLs --- .../ripme/ripper/AbstractHTMLRipper.java | 13 +- .../ripme/ripper/AbstractJSONRipper.java | 10 +- .../ripme/ripper/AbstractRipper.java | 9 + .../rarchives/ripme/ripper/AlbumRipper.java | 1 + .../ripme/ripper/DownloadFileThread.java | 4 + .../ripme/ripper/DownloadVideoThread.java | 3 +- .../rarchives/ripme/ripper/VideoRipper.java | 3 + .../ripper/rippers/FivehundredpxRipper.java | 23 +- .../ripme/ripper/rippers/FlickrRipper.java | 6 + .../ripme/ripper/rippers/FuskatorRipper.java | 4 +- .../ripme/ripper/rippers/GifyoRipper.java | 6 +- .../ripme/ripper/rippers/MinusRipper.java | 9 +- .../ripper/rippers/ModelmayhemRipper.java | 4 + .../ripme/ripper/rippers/NfsfwRipper.java | 8 +- .../ripper/rippers/PhotobucketRipper.java | 2 +- .../ripme/ripper/rippers/PornhubRipper.java | 16 +- .../ripper/rippers/SankakuComplexRipper.java | 8 +- .../ripme/ripper/rippers/SmuttyRipper.java | 7 + .../ripper/rippers/SupertangasRipper.java | 4 + .../ripme/ripper/rippers/TapasticRipper.java | 3 + .../ripper/rippers/TeenplanetRipper.java | 3 + .../ripme/ripper/rippers/VidbleRipper.java | 2 +- .../ripme/ripper/rippers/VidmeRipper.java | 66 ---- .../ripme/ripper/rippers/VineRipper.java | 6 + .../ripme/ripper/rippers/XhamsterRipper.java | 6 + .../ripper/rippers/video/ViddmeRipper.java | 10 +- .../com/rarchives/ripme/ui/MainWindow.java | 39 +- .../com/rarchives/ripme/ui/UpdateUtils.java | 6 +- .../com/rarchives/ripme/utils/RipUtils.java | 12 +- src/main/resources/log4j.file.properties | 2 +- .../tst/ripper/rippers/BasicRippersTest.java | 354 ++++++++++++------ .../ripme/tst/ripper/rippers/RippersTest.java | 26 +- 32 files changed, 456 insertions(+), 219 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/VidmeRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 729ee7bb..c9bb0259 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -66,6 +66,12 @@ public abstract class AbstractHTMLRipper extends AlbumRipper { while (doc != null) { List imageURLs = getURLsFromPage(doc); + // Remove all but 1 image + if (isThisATest()) { + while (imageURLs.size() > 1) { + imageURLs.remove(1); + } + } if (imageURLs.size() == 0) { throw new IOException("No images found at " + doc.location()); @@ -73,12 +79,14 @@ public abstract class AbstractHTMLRipper extends AlbumRipper { for (String imageURL : imageURLs) { index += 1; + logger.debug("Found image url #" + index + ": " + imageURL); downloadURL(new URL(imageURL), index); if (isStopped()) { break; } } if (hasDescriptionSupport()) { + logger.debug("Fetching description(s) from " + doc.location()); List textURLs = getDescriptionsFromPage(doc); if (textURLs.size() > 0) { for (String textURL : textURLs) { @@ -86,15 +94,17 @@ public abstract class AbstractHTMLRipper extends AlbumRipper { break; } textindex += 1; + logger.debug("Getting decription from " + textURL); String tempDesc = getDescription(textURL); if (tempDesc != null) { + logger.debug("Got description: " + tempDesc); saveText(new URL(textURL), "", tempDesc, textindex); } } } } - if (isStopped()) { + if (isStopped() || isThisATest()) { break; } @@ -109,6 +119,7 @@ public abstract class AbstractHTMLRipper extends AlbumRipper { // If they're using a thread pool, wait for it. if (getThreadPool() != null) { + logger.debug("Waiting for threadpool " + getThreadPool().getClass().getName()); getThreadPool().waitForThreads(); } waitForThreads(); diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index 7d9a5640..a89e8974 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -55,6 +55,12 @@ public abstract class AbstractJSONRipper extends AlbumRipper { while (json != null) { List imageURLs = getURLsFromJSON(json); + // Remove all but 1 image + if (isThisATest()) { + while (imageURLs.size() > 1) { + imageURLs.remove(1); + } + } if (imageURLs.size() == 0) { throw new IOException("No images found at " + this.url); @@ -65,10 +71,11 @@ public abstract class AbstractJSONRipper extends AlbumRipper { break; } index += 1; + logger.debug("Found image url #" + index+ ": " + imageURL); downloadURL(new URL(imageURL), index); } - if (isStopped()) { + if (isStopped() || isThisATest()) { break; } @@ -83,6 +90,7 @@ public abstract class AbstractJSONRipper extends AlbumRipper { // If they're using a thread pool, wait for it. if (getThreadPool() != null) { + logger.debug("Waiting for threadpool " + getThreadPool().getClass().getName()); getThreadPool().waitForThreads(); } waitForThreads(); diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index d2a1f9be..c304f6b5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -102,8 +102,10 @@ public abstract class AbstractRipper try { stopCheck(); } catch (IOException e) { + logger.debug("Ripper has been stopped"); return false; } + logger.debug("url: " + url + ", prefix: " + prefix + ", subdirectory" + subdirectory + ", referrer: " + referrer + ", cookies: " + cookies); String saveAs = url.toExternalForm(); saveAs = saveAs.substring(saveAs.lastIndexOf('/')+1); if (saveAs.indexOf('?') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('?')); } @@ -163,6 +165,7 @@ public abstract class AbstractRipper * Waits for downloading threads to complete. */ protected void waitForThreads() { + logger.debug("Waiting for threads to finish"); completed = false; threadPool.waitForThreads(); checkIfComplete(); @@ -212,6 +215,7 @@ public abstract class AbstractRipper */ protected void checkIfComplete() { if (observer == null) { + logger.debug("observer is null"); return; } @@ -226,6 +230,7 @@ public abstract class AbstractRipper Logger rootLogger = Logger.getRootLogger(); FileAppender fa = (FileAppender) rootLogger.getAppender("FILE"); if (fa != null) { + logger.debug("Changing log file back to 'ripme.log'"); fa.setFile("ripme.log"); fa.activateOptions(); } @@ -272,6 +277,7 @@ public abstract class AbstractRipper for (Constructor constructor : getRipperConstructors("com.rarchives.ripme.ripper.rippers")) { try { AlbumRipper ripper = (AlbumRipper) constructor.newInstance(url); + logger.debug("Found album ripper: " + ripper.getClass().getName()); return ripper; } catch (Exception e) { // Incompatible rippers *will* throw exceptions during instantiation. @@ -280,6 +286,7 @@ public abstract class AbstractRipper for (Constructor constructor : getRipperConstructors("com.rarchives.ripme.ripper.rippers.video")) { try { VideoRipper ripper = (VideoRipper) constructor.newInstance(url); + logger.debug("Found video ripper: " + ripper.getClass().getName()); return ripper; } catch (Exception e) { // Incompatible rippers *will* throw exceptions during instantiation. @@ -355,6 +362,7 @@ public abstract class AbstractRipper public boolean sleep(int milliseconds) { try { + logger.debug("Sleeping " + milliseconds + "ms"); Thread.sleep(milliseconds); return true; } catch (InterruptedException e) { @@ -372,6 +380,7 @@ public abstract class AbstractRipper /** Methods for detecting when we're running a test. */ public void markAsTest() { + logger.debug("THIS IS A TEST RIP"); thisIsATest = true; } public boolean isThisATest() { diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 3c859b0a..e8dbc6e3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -175,6 +175,7 @@ public abstract class AlbumRipper extends AbstractRipper { } else { title = super.getAlbumTitle(this.url); } + logger.debug("Using album title '" + title + "'"); title = Utils.filesystemSafe(title); path += title + File.separator; this.workingDir = new File(path); diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 859fa668..8a346fb4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -111,9 +111,11 @@ public class DownloadFileThread extends Thread { cookie += key + "=" + cookies.get(key); } huc.setRequestProperty("Cookie", cookie); + logger.debug("Request properties: " + huc.getRequestProperties()); huc.connect(); int statusCode = huc.getResponseCode(); + logger.debug("Status code: " + statusCode); if (statusCode / 100 == 3) { // 3xx Redirect if (!redirected) { // Don't increment retries on the first redirect @@ -148,12 +150,14 @@ public class DownloadFileThread extends Thread { IOUtils.copy(bis, fos); break; // Download successful: break out of infinite loop } catch (HttpStatusException hse) { + logger.debug("HTTP status exception", hse); logger.error("[!] HTTP status " + hse.getStatusCode() + " while downloading from " + urlToDownload); if (hse.getStatusCode() == 404 && Utils.getConfigBoolean("errors.skip404", false)) { observer.downloadErrored(url, "HTTP status code " + hse.getStatusCode() + " while downloading " + url.toExternalForm()); return; } } catch (IOException e) { + logger.debug("IOException", e); logger.error("[!] Exception while downloading file: " + url + " - " + e.getMessage()); } finally { // Close any open streams diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java index 7675c89e..dc437c87 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java @@ -71,7 +71,7 @@ public class DownloadVideoThread extends Thread { } observer.setBytesTotal(bytesTotal); observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal); - logger.info("Size of file at " + this.url + " = " + bytesTotal + "b"); + logger.debug("Size of file at " + this.url + " = " + bytesTotal + "b"); int tries = 0; // Number of attempts to download do { @@ -95,6 +95,7 @@ public class DownloadVideoThread extends Thread { huc.setRequestProperty("Referer", this.url.toExternalForm()); // Sic huc.setRequestProperty("User-agent", AbstractRipper.USER_AGENT); tries += 1; + logger.debug("Request properties: " + huc.getRequestProperties().toString()); huc.connect(); // Check status code bis = new BufferedInputStream(huc.getInputStream()); diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index ff8bca4c..6cdc06fd 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -58,6 +58,9 @@ public abstract class VideoRipper extends AbstractRipper { } else { if (isThisATest()) { + // Tests shouldn't download the whole video + // Just change this.url to the download URL so the test knows we found it. + logger.debug("Test rip, found URL: " + url); this.url = url; return true; } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java index 7ad2bee0..702b919e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java @@ -1,5 +1,6 @@ package com.rarchives.ripme.ripper.rippers; +import java.io.File; import java.io.IOException; import java.net.HttpURLConnection; import java.net.MalformedURLException; @@ -117,13 +118,15 @@ public class FivehundredpxRipper extends AbstractJSONRipper { @Override public JSONObject getFirstPage() throws IOException { URL apiURL = new URL(baseURL + "&consumer_key=" + CONSUMER_KEY); + logger.debug("apiURL: " + apiURL); JSONObject json = Http.url(apiURL).getJSON(); if (baseURL.contains("/blogs?")) { - // List of stories + // List of stories to return JSONObject result = new JSONObject(); result.put("photos", new JSONArray()); - JSONArray jsonBlogs = json.getJSONArray("blog_posts"); + // Iterate over every story + JSONArray jsonBlogs = json.getJSONArray("blog_posts"); for (int i = 0; i < jsonBlogs.length(); i++) { if (i > 0) { sleep(500); @@ -153,6 +156,9 @@ public class FivehundredpxRipper extends AbstractJSONRipper { @Override public JSONObject getNextPage(JSONObject json) throws IOException { + if (isThisATest()) { + return null; + } // Check previous JSON to see if we hit the last page if (!json.has("current_page") || !json.has("total_pages")) { @@ -191,6 +197,9 @@ public class FivehundredpxRipper extends AbstractJSONRipper { } } imageURLs.add(imageURL); + if (isThisATest()) { + break; + } } return imageURLs; } @@ -208,12 +217,18 @@ public class FivehundredpxRipper extends AbstractJSONRipper { } } + @Override + public boolean keepSortOrder() { + return false; + } + @Override public void downloadURL(URL url, int index) { String u = url.toExternalForm(); String[] fields = u.split("/"); - String prefix = getPrefix(index) + fields[fields.length - 2] + "-"; - addURLToDownload(url, prefix); + String prefix = getPrefix(index) + fields[fields.length - 3]; + File saveAs = new File(getWorkingDir() + File.separator + prefix + ".jpg"); + addURLToDownload(url, saveAs, "", null); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java index 19c72ffe..71d35da1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java @@ -130,6 +130,9 @@ public class FlickrRipper extends AbstractHTMLRipper { @Override public Document getNextPage(Document doc) throws IOException { + if (isThisATest()) { + return null; + } // Find how many pages there are int lastPage = 0; for (Element apage : doc.select("a[data-track^=page-]")) { @@ -185,6 +188,9 @@ public class FlickrRipper extends AbstractHTMLRipper { } attempted.add(imagePage); imageURLs.add(imagePage); + if (isThisATest()) { + break; + } } return imageURLs; } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuskatorRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuskatorRipper.java index 0ac86139..1f5fc35d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuskatorRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuskatorRipper.java @@ -63,7 +63,7 @@ public class FuskatorRipper extends AbstractHTMLRipper { List imageURLs = new ArrayList(); String html = doc.html(); // Get "baseUrl" - String baseUrl = Utils.between(html, "var baseUrl = unescape('", "'").get(0); + String baseUrl = Utils.between(html, "unescape('", "'").get(0); try { baseUrl = URLDecoder.decode(baseUrl, "UTF-8"); } catch (UnsupportedEncodingException e) { @@ -73,7 +73,7 @@ public class FuskatorRipper extends AbstractHTMLRipper { baseUrl = "http:" + baseUrl; } // Iterate over images - for (String filename : Utils.between(html, ".src=baseUrl+'", "'")) { + for (String filename : Utils.between(html, "+'", "'")) { imageURLs.add(baseUrl + filename); } return imageURLs; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GifyoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GifyoRipper.java index 93e22d86..6c27a2c4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GifyoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GifyoRipper.java @@ -93,8 +93,8 @@ public class GifyoRipper extends AbstractHTMLRipper { @Override public List getURLsFromPage(Document doc) { List imageURLs = new ArrayList(); - for (Element image : doc.select("div.gif img")) { - String imageUrl = image.attr("src"); + for (Element image : doc.select("img.profile_gif")) { + String imageUrl = image.attr("data-animated"); if (imageUrl.startsWith("//")) { imageUrl = "http:" + imageUrl; } @@ -102,7 +102,7 @@ public class GifyoRipper extends AbstractHTMLRipper { imageUrl = imageUrl.replace("_s.gif", ".gif"); imageURLs.add(imageUrl); } - logger.info("Found " + imageURLs.size() + " images"); + logger.debug("Found " + imageURLs.size() + " images"); return imageURLs; } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MinusRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MinusRipper.java index 0fc7b971..f51dc584 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MinusRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MinusRipper.java @@ -136,8 +136,12 @@ public class MinusRipper extends AlbumRipper { String title = gallery.getString("name"); String albumUrl = "http://" + user + ".minus.com/m" + gallery.getString("reader_id"); ripAlbum(new URL(albumUrl), Utils.filesystemSafe(title)); + + if (isThisATest()) { + break; + } } - if (page >= json.getInt("total_pages")) { + if (page >= json.getInt("total_pages") || isThisATest()) { break; } page++; @@ -169,6 +173,9 @@ public class MinusRipper extends AlbumRipper { prefix = String.format("%03d_", i + 1); } addURLToDownload(new URL(image), prefix, subdir); + if (isThisATest()) { + break; + } } } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ModelmayhemRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ModelmayhemRipper.java index e4c468ca..58ecded9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ModelmayhemRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ModelmayhemRipper.java @@ -110,6 +110,10 @@ public class ModelmayhemRipper extends AlbumRipper { prefix = String.format("%03d_", i + 1); } addURLToDownload(new URL(bigImage), prefix); + + if (isThisATest()) { + break; + } } waitForThreads(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java index c7d98c15..a51b6d89 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java @@ -92,7 +92,7 @@ public class NfsfwRipper extends AlbumRipper { } // Subalbums for (Element suba : albumDoc.select("td.IMG > a")) { - if (isStopped()) { + if (isStopped() || isThisATest()) { break; } String subURL = "http://nfsfw.com" + suba.attr("href"); @@ -112,10 +112,16 @@ public class NfsfwRipper extends AlbumRipper { try { NfsfwImageThread t = new NfsfwImageThread(new URL(imagePage), nextSubalbum, ++index); nfsfwThreadPool.addThread(t); + if (isThisATest()) { + break; + } } catch (MalformedURLException mue) { logger.warn("Invalid URL: " + imagePage); } } + if (isThisATest()) { + break; + } // Get next page for (Element a : albumDoc.select("a.next")) { subAlbums.add(0, new Pair("http://nfsfw.com" + a.attr("href"), "")); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PhotobucketRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PhotobucketRipper.java index c6c8885c..751dfafc 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PhotobucketRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PhotobucketRipper.java @@ -167,7 +167,7 @@ public class PhotobucketRipper extends AlbumRipper { filesIndex += 1; addURLToDownload(new URL(image), "", - object.getString("location"), + object.getString("location").replaceAll(" ", "_"), albumDoc.location(), pageResponse.cookies()); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java index 997d3cb9..23decd4f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java @@ -81,10 +81,6 @@ public class PornhubRipper extends AlbumRipper { int index = 0; String nextUrl = this.url.toExternalForm(); - if (isStopped()) { - return; - } - if (albumDoc == null) { logger.info(" Retrieving album page " + nextUrl); sendUpdate(STATUS.LOADING_RESOURCE, nextUrl); @@ -92,15 +88,15 @@ public class PornhubRipper extends AlbumRipper { .referrer(this.url) .get(); } - + // Find thumbnails Elements thumbs = albumDoc.select(".photoBlockBox li"); if (thumbs.size() == 0) { - logger.info("albumDoc: " + albumDoc); - logger.info("No images found at " + nextUrl); + logger.debug("albumDoc: " + albumDoc); + logger.debug("No images found at " + nextUrl); return; } - + // Iterate over images on page for (Element thumb : thumbs) { if (isStopped()) { @@ -111,6 +107,9 @@ public class PornhubRipper extends AlbumRipper { URL imagePage = new URL(url, imagePageUrl); PornhubImageThread t = new PornhubImageThread(imagePage, index, this.workingDir); pornhubThreadPool.addThread(t); + if (isThisATest()) { + break; + } try { Thread.sleep(IMAGE_SLEEP_TIME); } catch (InterruptedException e) { @@ -118,6 +117,7 @@ public class PornhubRipper extends AlbumRipper { } } + pornhubThreadPool.waitForThreads(); waitForThreads(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SankakuComplexRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SankakuComplexRipper.java index 4a425775..fb086e86 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SankakuComplexRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SankakuComplexRipper.java @@ -34,12 +34,12 @@ public class SankakuComplexRipper extends AbstractHTMLRipper { @Override public String getDomain() { - return "idol.sankakucomplex.com"; + return "sankakucomplex.com"; } @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://idol\\.sankakucomplex\\.com/.*tags=([^&]+).*$"); + Pattern p = Pattern.compile("^https?://([a-zA-Z0-9]+\\.)?sankakucomplex\\.com/.*tags=([^&]+).*$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { try { @@ -72,8 +72,8 @@ public class SankakuComplexRipper extends AbstractHTMLRipper { String postId = thumbSpan.attr("id").replaceAll("p", ""); Element thumb = thumbSpan.getElementsByTag("img").first(); String image = thumb.attr("abs:src") - .replace("i.sankakucomplex.com/data/preview", - "is.sankakucomplex.com/data") + "?" + postId; + .replace(".sankakucomplex.com/data/preview", + "s.sankakucomplex.com/data") + "?" + postId; imageURLs.add(image); } return imageURLs; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SmuttyRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SmuttyRipper.java index 920ce07b..94670ab2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SmuttyRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SmuttyRipper.java @@ -13,6 +13,13 @@ import com.rarchives.ripme.ripper.AlbumRipper; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; +/** + * Appears to be broken as of 2015-02-11. + * Generating large image from thumbnail requires replacing "/m/" with something else: + * -> Sometimes "/b/" + * -> Sometimes "/p/" + * No way to know without loading the image page. + */ public class SmuttyRipper extends AlbumRipper { private static final String DOMAIN = "smutty.com", diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SupertangasRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SupertangasRipper.java index 86e680aa..8cc20b0a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SupertangasRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SupertangasRipper.java @@ -15,6 +15,10 @@ import com.rarchives.ripme.ripper.AlbumRipper; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; +/** + * Appears to be broken as of 2015-02-11. + * Looks like supertangas changed their site completely. + */ public class SupertangasRipper extends AlbumRipper { private static final String DOMAIN = "supertangas.com", diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TapasticRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TapasticRipper.java index e6756307..d155dae0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TapasticRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TapasticRipper.java @@ -88,6 +88,9 @@ public class TapasticRipper extends AbstractHTMLRipper { prefix.append(episode.filename.replace(" ", "-")); prefix.append("-"); addURLToDownload(new URL(link), prefix.toString()); + if (isThisATest()) { + break; + } } } catch (IOException e) { logger.error("[!] Exception while downloading " + url, e); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TeenplanetRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TeenplanetRipper.java index f3fa3826..b81daa7a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TeenplanetRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TeenplanetRipper.java @@ -88,6 +88,9 @@ public class TeenplanetRipper extends AlbumRipper { prefix = String.format("%03d_", index); } addURLToDownload(new URL(image), prefix); + if (isThisATest()) { + break; + } } waitForThreads(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java index 8cd8b957..9b735ebb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java @@ -59,7 +59,7 @@ public class VidbleRipper extends AbstractHTMLRipper { Elements els = doc.select("#ContentPlaceHolder1_thumbs"); String thumbs = els.first().attr("value"); for (String thumb : thumbs.split(",")) { - if (thumb.trim().equals("")) { + if (thumb.trim().equals("") || thumb.contains("reddit.com")) { continue; } thumb = thumb.replaceAll("_[a-zA-Z]{3,5}", ""); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VidmeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VidmeRipper.java deleted file mode 100644 index 4b466fcc..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VidmeRipper.java +++ /dev/null @@ -1,66 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.LinkedList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.commons.lang.StringEscapeUtils; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; - -public class VidmeRipper extends AbstractHTMLRipper { - - public VidmeRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getDomain() { - return "vid.me"; - } - - @Override - public String getHost() { - return "vid"; - } - - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } - - @Override - public List getURLsFromPage(Document page) { - List result = new LinkedList(); - for(Element elem : page.select("a.js-download-video-link")){ - String link = StringEscapeUtils.unescapeHtml(elem.attr("data-href").toString()); - result.add(link); - } - return result; - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://vid\\.me/([a-zA-Z0-9]+).*$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - // Return the text contained between () in the regex - return m.group(1); - } - throw new MalformedURLException("Expected imgur.com URL format: " - + "imgur.com/a/albumid - got " + url + " instead"); - } - -} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VineRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VineRipper.java index aa453e1a..e37df64f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VineRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VineRipper.java @@ -56,6 +56,12 @@ public class VineRipper extends AlbumRipper { for (int i = 0; i < records.length(); i++) { String videoURL = records.getJSONObject(i).getString("videoUrl"); addURLToDownload(new URL(videoURL)); + if (isThisATest()) { + break; + } + } + if (isThisATest()) { + break; } if (records.length() == 0) { logger.info("Zero records returned"); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index bc7a2ad9..c6f9cab5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -57,6 +57,12 @@ public class XhamsterRipper extends AlbumRipper { prefix = String.format("%03d_", index); } addURLToDownload(new URL(image), prefix); + if (isThisATest()) { + break; + } + } + if (isThisATest()) { + break; } nextURL = null; for (Element element : doc.select("a.last")) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java index 5fab8fe4..f807b496 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java @@ -14,7 +14,7 @@ import com.rarchives.ripme.utils.Http; public class ViddmeRipper extends VideoRipper { - private static final String HOST = "vidd"; + private static final String HOST = "vid"; public ViddmeRipper(URL url) throws IOException { super(url); @@ -27,7 +27,7 @@ public class ViddmeRipper extends VideoRipper { @Override public boolean canRip(URL url) { - Pattern p = Pattern.compile("^https?://[wm.]*vidd\\.me/[a-zA-Z0-9]+.*$"); + Pattern p = Pattern.compile("^https?://[wm.]*vid\\.me/[a-zA-Z0-9]+.*$"); Matcher m = p.matcher(url.toExternalForm()); return m.matches(); } @@ -39,15 +39,15 @@ public class ViddmeRipper extends VideoRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://[wm.]*vidd\\.me/([a-zA-Z0-9]+).*$"); + Pattern p = Pattern.compile("^https?://[wm.]*vid\\.me/([a-zA-Z0-9]+).*$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); } throw new MalformedURLException( - "Expected vidd.me format:" - + "vidd.me/id" + "Expected vid.me format:" + + "vid.me/id" + " Got: " + url); } diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index a1b5557b..a33e8459 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -37,6 +37,7 @@ import javax.swing.DefaultListModel; import javax.swing.ImageIcon; import javax.swing.JButton; import javax.swing.JCheckBox; +import javax.swing.JComboBox; import javax.swing.JFileChooser; import javax.swing.JFrame; import javax.swing.JLabel; @@ -62,6 +63,8 @@ import javax.swing.text.SimpleAttributeSet; import javax.swing.text.StyleConstants; import javax.swing.text.StyledDocument; +import org.apache.log4j.ConsoleAppender; +import org.apache.log4j.FileAppender; import org.apache.log4j.Level; import org.apache.log4j.Logger; @@ -124,6 +127,7 @@ public class MainWindow implements Runnable, RipStatusHandler { private static JButton configSaveDirButton; private static JTextField configRetriesText; private static JCheckBox configAutoupdateCheckbox; + private static JComboBox configLogLevelCombobox; private static JCheckBox configPlaySound; private static JCheckBox configSaveOrderCheckbox; private static JCheckBox configShowPopup; @@ -143,7 +147,6 @@ public class MainWindow implements Runnable, RipStatusHandler { private static AbstractRipper ripper; public MainWindow() { - Logger.getRootLogger().setLevel(Level.ERROR); mainFrame = new JFrame("RipMe v" + UpdateUtils.getThisJarVersion()); mainFrame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); mainFrame.setResizable(false); @@ -196,6 +199,7 @@ public class MainWindow implements Runnable, RipStatusHandler { Utils.setConfigInteger("download.timeout", Integer.parseInt(configTimeoutText.getText())); Utils.setConfigBoolean("clipboard.autorip", ClipboardUtils.getClipboardAutoRip()); Utils.setConfigBoolean("auto.update", configAutoupdateCheckbox.isSelected()); + Utils.setConfigString("log.level", configLogLevelCombobox.getSelectedItem().toString()); Utils.setConfigBoolean("play.sound", configPlaySound.isSelected()); Utils.setConfigBoolean("download.save_order", configSaveOrderCheckbox.isSelected()); Utils.setConfigBoolean("download.show_popup", configShowPopup.isSelected()); @@ -419,6 +423,9 @@ public class MainWindow implements Runnable, RipStatusHandler { configAutoupdateCheckbox = new JCheckBox("Auto-update?", Utils.getConfigBoolean("auto.update", true)); configAutoupdateCheckbox.setHorizontalAlignment(JCheckBox.RIGHT); configAutoupdateCheckbox.setHorizontalTextPosition(JCheckBox.LEFT); + configLogLevelCombobox = new JComboBox(new String[] {"Log level: Error", "Log level: Warn", "Log level: Info", "Log level: Debug"}); + configLogLevelCombobox.setSelectedItem(Utils.getConfigString("log.level", "Log level: Debug")); + setLogLevel(configLogLevelCombobox.getSelectedItem().toString()); configPlaySound = new JCheckBox("Sound when rip completes", Utils.getConfigBoolean("play.sound", false)); configPlaySound.setHorizontalAlignment(JCheckBox.RIGHT); configPlaySound.setHorizontalTextPosition(JCheckBox.LEFT); @@ -451,6 +458,7 @@ public class MainWindow implements Runnable, RipStatusHandler { gbc.gridy = 0; gbc.gridx = 0; configurationPanel.add(configUpdateLabel, gbc); gbc.gridx = 1; configurationPanel.add(configUpdateButton, gbc); gbc.gridy = 1; gbc.gridx = 0; configurationPanel.add(configAutoupdateCheckbox, gbc); + gbc.gridx = 1; configurationPanel.add(configLogLevelCombobox, gbc); gbc.gridy = 2; gbc.gridx = 0; configurationPanel.add(configThreadsLabel, gbc); gbc.gridx = 1; configurationPanel.add(configThreadsText, gbc); gbc.gridy = 3; gbc.gridx = 0; configurationPanel.add(configTimeoutLabel, gbc); @@ -648,6 +656,14 @@ public class MainWindow implements Runnable, RipStatusHandler { t.start(); } }); + configLogLevelCombobox.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent arg0) { + String level = ((JComboBox) arg0.getSource()).getSelectedItem().toString(); + setLogLevel(level); + } + }); + configSaveDirButton.addActionListener(new ActionListener() { @Override public void actionPerformed(ActionEvent arg0) { @@ -729,6 +745,27 @@ public class MainWindow implements Runnable, RipStatusHandler { public void intervalRemoved(ListDataEvent arg0) { } }); } + + private void setLogLevel(String level) { + Level newLevel = Level.ERROR; + level = level.substring(level.lastIndexOf(' ') + 1); + if (level.equals("Debug")) { + newLevel = Level.DEBUG; + } + else if (level.equals("Info")) { + newLevel = Level.INFO; + } + else if (level.equals("Warn")) { + newLevel = Level.WARN; + } + else if (level.equals("Error")) { + newLevel = Level.ERROR; + } + Logger.getRootLogger().setLevel(newLevel); + logger.setLevel(newLevel); + ((ConsoleAppender)Logger.getRootLogger().getAppender("stdout")).setThreshold(newLevel); + ((FileAppender)Logger.getRootLogger().getAppender("FILE")).setThreshold(newLevel); + } private void setupTrayIcon() { mainFrame.addWindowListener(new WindowAdapter() { diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 499eaa76..2260c971 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -41,6 +41,7 @@ public class UpdateUtils { Document doc = null; try { + logger.debug("Retrieving " + UpdateUtils.updateJsonURL); doc = Jsoup.connect(UpdateUtils.updateJsonURL) .timeout(10 * 1000) .ignoreContentType(true) @@ -69,6 +70,7 @@ public class UpdateUtils { String latestVersion = json.getString("latestVersion"); if (UpdateUtils.isNewerVersion(latestVersion)) { + logger.info("Found newer version: " + latestVersion); int result = JOptionPane.showConfirmDialog( null, "New version (" + latestVersion + ") is available!" @@ -95,8 +97,10 @@ public class UpdateUtils { return; } } else { + logger.debug("This version (" + UpdateUtils.getThisJarVersion() + + ") is the same or newer than the website's version (" + latestVersion + ")"); configUpdateLabel.setText("v" + UpdateUtils.getThisJarVersion() + " is the latest version"); - logger.info("Running latest version: " + UpdateUtils.getThisJarVersion()); + logger.debug("Running latest version: " + UpdateUtils.getThisJarVersion()); } } diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index c11f2db1..5a449b64 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -33,8 +33,10 @@ public class RipUtils { if ((url.getHost().endsWith("imgur.com")) && url.toExternalForm().contains("imgur.com/a/")) { try { + logger.debug("Fetching imgur album at " + url); ImgurAlbum imgurAlbum = ImgurRipper.getImgurAlbum(url); for (ImgurImage imgurImage : imgurAlbum.images) { + logger.debug("Got imgur image: " + imgurImage.url); result.add(imgurImage.url); } } catch (IOException e) { @@ -44,7 +46,9 @@ public class RipUtils { } else if (url.getHost().endsWith("gfycat.com")) { try { + logger.debug("Fetching gfycat page " + url); String videoURL = GfycatRipper.getVideoURL(url); + logger.debug("Got gfycat URL: " + videoURL); result.add(new URL(videoURL)); } catch (IOException e) { // Do nothing @@ -54,6 +58,7 @@ public class RipUtils { } else if (url.toExternalForm().contains("vidble.com/album/")) { try { + logger.info("Getting vidble album " + url); result.addAll(VidbleRipper.getURLsFromPage(url)); } catch (IOException e) { // Do nothing @@ -68,6 +73,7 @@ public class RipUtils { if (m.matches()) { try { URL singleURL = new URL(m.group(1)); + logger.debug("Found single URL: " + singleURL); result.add(singleURL); return result; } catch (MalformedURLException e) { @@ -101,7 +107,7 @@ public class RipUtils { public static Pattern getURLRegex() { return Pattern.compile("(https?://[a-zA-Z0-9\\-\\.]+\\.[a-zA-Z]{2,3}(/\\S*))"); } - + public static String urlFromDirectoryName(String dir) { String url = null; if (url == null) url = urlFromImgurDirectoryName(dir); @@ -114,9 +120,10 @@ public class RipUtils { if (url == null) url = urlFromSiteDirectoryName(dir, "datwin", "http://datw.in/", ""); if (url == null) url = urlFromSiteDirectoryName(dir, "drawcrowd", "http://drawcrowd.com/", ""); if (url == null) url = urlFromSiteDirectoryName(dir.replace("-", "/"), "ehentai", "http://g.e-hentai.org/g/", ""); - if (url == null) url = urlFromSiteDirectoryName(dir, "8muses", "http://www.8muses.com/index/category/", ""); if (url == null) url = urlFromSiteDirectoryName(dir, "fapproved", "http://fapproved.com/users/", ""); if (url == null) url = urlFromSiteDirectoryName(dir, "vinebox", "http://finebox.co/u/", ""); + if (url == null) url = urlFromSiteDirectoryName(dir, "imgbox", "http://imgbox.com/g/", ""); + if (url == null) url = urlFromSiteDirectoryName(dir, "modelmayhem", "http://www.modelmayhem.com/", ""); /* if (url == null) url = urlFromSiteDirectoryName(dir, "", "", ""); if (url == null) url = urlFromSiteDirectoryName(dir, "", "", ""); @@ -129,6 +136,7 @@ public class RipUtils { if (url == null) url = urlFromSiteDirectoryName(dir, "", "", ""); if (url == null) url = urlFromSiteDirectoryName(dir, "", "", ""); */ + //if (url == null) url = urlFromSiteDirectoryName(dir, "8muses", "http://www.8muses.com/index/category/", ""); return url; } diff --git a/src/main/resources/log4j.file.properties b/src/main/resources/log4j.file.properties index 3bb9d65d..c45bd574 100644 --- a/src/main/resources/log4j.file.properties +++ b/src/main/resources/log4j.file.properties @@ -5,7 +5,7 @@ log4j.appender.FILE.ImmediateFlush = true log4j.appender.FILE.Threshold = warn log4j.appender.FILE.maxFileSize = 20MB log4j.appender.FILE.layout = org.apache.log4j.PatternLayout -log4j.appender.FILE.layout.ConversionPattern = %d %-5p %c{2} %x - %m%n +log4j.appender.FILE.layout.ConversionPattern = %d %-5p %c{2} %x.%M() %m%n # define the console appender log4j.appender.stdout = org.apache.log4j.ConsoleAppender diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java index 5adf4f71..4ac52345 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java @@ -2,21 +2,42 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; import java.net.URL; -import java.util.ArrayList; -import java.util.List; - -import org.apache.log4j.Level; -import org.apache.log4j.Logger; +import com.rarchives.ripme.ripper.AbstractRipper; import com.rarchives.ripme.ripper.rippers.DeviantartRipper; import com.rarchives.ripme.ripper.rippers.EightmusesRipper; import com.rarchives.ripme.ripper.rippers.FineboxRipper; +import com.rarchives.ripme.ripper.rippers.FivehundredpxRipper; +import com.rarchives.ripme.ripper.rippers.FlickrRipper; +import com.rarchives.ripme.ripper.rippers.FuraffinityRipper; +import com.rarchives.ripme.ripper.rippers.FuskatorRipper; +import com.rarchives.ripme.ripper.rippers.GifyoRipper; +import com.rarchives.ripme.ripper.rippers.GirlsOfDesireRipper; import com.rarchives.ripme.ripper.rippers.GonewildRipper; +import com.rarchives.ripme.ripper.rippers.HentaifoundryRipper; +import com.rarchives.ripme.ripper.rippers.ImagearnRipper; +import com.rarchives.ripme.ripper.rippers.ImagebamRipper; +import com.rarchives.ripme.ripper.rippers.ImagestashRipper; +import com.rarchives.ripme.ripper.rippers.ImagevenueRipper; +import com.rarchives.ripme.ripper.rippers.ImgboxRipper; +import com.rarchives.ripme.ripper.rippers.IrarchivesRipper; +import com.rarchives.ripme.ripper.rippers.MinusRipper; +import com.rarchives.ripme.ripper.rippers.ModelmayhemRipper; import com.rarchives.ripme.ripper.rippers.MotherlessRipper; +import com.rarchives.ripme.ripper.rippers.NfsfwRipper; +import com.rarchives.ripme.ripper.rippers.PhotobucketRipper; +import com.rarchives.ripme.ripper.rippers.PornhubRipper; import com.rarchives.ripme.ripper.rippers.RedditRipper; +import com.rarchives.ripme.ripper.rippers.SankakuComplexRipper; import com.rarchives.ripme.ripper.rippers.SeeniveRipper; +import com.rarchives.ripme.ripper.rippers.ShesFreakyRipper; +import com.rarchives.ripme.ripper.rippers.TapasticRipper; +import com.rarchives.ripme.ripper.rippers.TeenplanetRipper; import com.rarchives.ripme.ripper.rippers.TumblrRipper; import com.rarchives.ripme.ripper.rippers.TwitterRipper; +import com.rarchives.ripme.ripper.rippers.TwodgalleriesRipper; +import com.rarchives.ripme.ripper.rippers.VidbleRipper; +import com.rarchives.ripme.ripper.rippers.VineRipper; import com.rarchives.ripme.ripper.rippers.VkRipper; import com.rarchives.ripme.ripper.rippers.XhamsterRipper; @@ -26,127 +47,240 @@ import com.rarchives.ripme.ripper.rippers.XhamsterRipper; */ public class BasicRippersTest extends RippersTest { - public void testMotherlessAlbums() throws IOException { - List contentURLs = new ArrayList(); - - // Image album - contentURLs.add(new URL("http://motherless.com/G4DAA18D")); - // Video album - // XXX: Commented out because test takes too long to download the file. - // contentURLs.add(new URL("http://motherless.com/GFD0F537")); - - for (URL url : contentURLs) { - MotherlessRipper ripper = new MotherlessRipper(url); - testRipper(ripper); - } + public void testDeviantartAlbum() throws IOException { + DeviantartRipper ripper = new DeviantartRipper(new URL("http://airgee.deviantart.com/gallery/")); + testRipper(ripper); } - - public void testDeviantartAlbums() throws IOException { - List contentURLs = new ArrayList(); - - // Small gallery - contentURLs.add(new URL("http://airgee.deviantart.com/gallery/")); + public void testDeviantartNSFWAlbum() throws IOException { // NSFW gallery - contentURLs.add(new URL("http://faterkcx.deviantart.com/gallery/")); - // Multi-page NSFW - contentURLs.add(new URL("http://geekysica.deviantart.com/gallery/35209412")); - - for (URL url : contentURLs) { - DeviantartRipper ripper = new DeviantartRipper(url); - testRipper(ripper); - } + DeviantartRipper ripper = new DeviantartRipper(new URL("http://faterkcx.deviantart.com/gallery/")); + testRipper(ripper); } - public void testEightmusesAlbums() throws IOException { - List contentURLs = new ArrayList(); - - contentURLs.add(new URL("http://www.8muses.com/index/category/jab-hotassneighbor7")); - - for (URL url : contentURLs) { - EightmusesRipper ripper = new EightmusesRipper(url); - testRipper(ripper); - } + public void testEightmusesAlbum() throws IOException { + EightmusesRipper ripper = new EightmusesRipper(new URL("http://www.8muses.com/index/category/jab-hotassneighbor7")); + testRipper(ripper); } - public void testVineboxAlbums() throws IOException { - Logger.getRootLogger().setLevel(Level.ALL); - List contentURLs = new ArrayList(); - contentURLs.add(new URL("http://vinebox.co/u/wi57hMjc2Ka")); - contentURLs.add(new URL("http://finebox.co/u/wi57hMjc2Ka")); - for (URL url : contentURLs) { - FineboxRipper ripper = new FineboxRipper(url); - testRipper(ripper); - } + public void testVineboxAlbum() throws IOException { + FineboxRipper ripper = new FineboxRipper(new URL("http://vinebox.co/u/wi57hMjc2Ka")); + testRipper(ripper); + } + public void testFineboxAlbum() throws IOException { + FineboxRipper ripper = new FineboxRipper(new URL("http://finebox.co/u/wi57hMjc2Ka")); + testRipper(ripper); + } + + public void testRedditSubredditRip() throws IOException { + RedditRipper ripper = new RedditRipper(new URL("http://www.reddit.com/r/nsfw_oc")); + testRipper(ripper); + } + public void testRedditSubredditTopRip() throws IOException { + RedditRipper ripper = new RedditRipper(new URL("http://www.reddit.com/r/nsfw_oc/top?t=all")); + testRipper(ripper); + } + public void testRedditPostRip() throws IOException { + RedditRipper ripper = new RedditRipper(new URL("http://www.reddit.com/r/UnrealGirls/comments/1ziuhl/in_class_veronique_popa/")); + testRipper(ripper); } - public void testXhamsterAlbums() throws IOException { - List contentURLs = new ArrayList(); - contentURLs.add(new URL("http://xhamster.com/photos/gallery/1462237/alyssa_gadson.html")); - contentURLs.add(new URL("http://xhamster.com/photos/gallery/2941201/tableau_d_039_art_ii.html")); - for (URL url : contentURLs) { - XhamsterRipper ripper = new XhamsterRipper(url); - testRipper(ripper); - } + public void testSeeniveAlbum() throws IOException { + SeeniveRipper ripper = new SeeniveRipper(new URL("http://seenive.com/u/946491170220040192")); + testRipper(ripper); + } + + public void testTumblrFullRip() throws IOException { + TumblrRipper ripper = new TumblrRipper(new URL("http://wrouinr.tumblr.com/archive")); + testRipper(ripper); + } + public void testTumblrTagRip() throws IOException { + TumblrRipper ripper = new TumblrRipper(new URL("http://topinstagirls.tumblr.com/tagged/berlinskaya")); + testRipper(ripper); + } + public void testTumblrPostRip() throws IOException { + TumblrRipper ripper = new TumblrRipper(new URL("http://genekellyclarkson.tumblr.com/post/86100752527/lucyannebrooks-rachaelboden-friends-goodtimes-bed-boobs")); + testRipper(ripper); + } + + public void testTwitterUserRip() throws IOException { + TwitterRipper ripper = new TwitterRipper(new URL("https://twitter.com/danngamber01/media")); + testRipper(ripper); + } + public void testTwitterSearchRip() throws IOException { + TwitterRipper ripper = new TwitterRipper(new URL("https://twitter.com/search?q=from%3Apurrbunny%20filter%3Aimages&src=typd")); + testRipper(ripper); + } + + public void test500pxAlbum() throws IOException { + FivehundredpxRipper ripper = new FivehundredpxRipper(new URL("https://prime.500px.com/alexander_hurman")); + testRipper(ripper); + } + + public void testFlickrAlbum() throws IOException { + FlickrRipper ripper = new FlickrRipper(new URL("https://www.flickr.com/photos/leavingallbehind/sets/72157621895942720/")); + testRipper(ripper); + } + + public void testFuraffinityAlbum() throws IOException { + FuraffinityRipper ripper = new FuraffinityRipper(new URL("https://www.furaffinity.net/gallery/mustardgas/")); + testRipper(ripper); + } + + public void testFuskatorAlbum() throws IOException { + FuskatorRipper ripper = new FuskatorRipper(new URL("http://fuskator.com/full/emJa1U6cqbi/index.html")); + testRipper(ripper); + } + + public void testGifyoAlbum() throws IOException { + GifyoRipper ripper = new GifyoRipper(new URL("http://gifyo.com/PieSecrets/")); + testRipper(ripper); + } + + public void testGirlsofdesireAlbum() throws IOException { + GirlsOfDesireRipper ripper = new GirlsOfDesireRipper(new URL("http://www.girlsofdesire.org/galleries/krillia/")); + testRipper(ripper); } public void testGonewildAlbums() throws IOException { - List contentURLs = new ArrayList(); - contentURLs.add(new URL("http://gonewild.com/user/amle69")); - for (URL url : contentURLs) { - GonewildRipper ripper = new GonewildRipper(url); - testRipper(ripper); - } + GonewildRipper ripper = new GonewildRipper(new URL("http://gonewild.com/user/amle69")); + testRipper(ripper); } - public void testRedditAlbums() throws IOException { - List contentURLs = new ArrayList(); - contentURLs.add(new URL("http://www.reddit.com/r/nsfw_oc")); - contentURLs.add(new URL("http://www.reddit.com/r/nsfw_oc/top?t=all")); - contentURLs.add(new URL("http://www.reddit.com/r/UnrealGirls/comments/1ziuhl/in_class_veronique_popa/")); - for (URL url : contentURLs) { - RedditRipper ripper = new RedditRipper(url); - testRipper(ripper); - } + public void testHentaifoundryRip() throws IOException { + HentaifoundryRipper ripper = new HentaifoundryRipper(new URL("http://www.hentai-foundry.com/pictures/user/personalami")); + testRipper(ripper); + } + + public void testImagearnRip() throws IOException { + AbstractRipper ripper = new ImagearnRipper(new URL("http://imagearn.com//gallery.php?id=578682")); + testRipper(ripper); + } + + public void testImagebamRip() throws IOException { + AbstractRipper ripper = new ImagebamRipper(new URL("http://www.imagebam.com/gallery/488cc796sllyf7o5srds8kpaz1t4m78i")); + testRipper(ripper); + } + + public void testImagestashRip() throws IOException { + AbstractRipper ripper = new ImagestashRipper(new URL("https://imagestash.org/tag/everydayuncensor")); + testRipper(ripper); + } + + public void testImagevenueRip() throws IOException { + AbstractRipper ripper = new ImagevenueRipper(new URL("http://img120.imagevenue.com/galshow.php?gal=gallery_1373818527696_191lo")); + testRipper(ripper); + } + + public void testImgboxRip() throws IOException { + AbstractRipper ripper = new ImgboxRipper(new URL("http://imgbox.com/g/sEMHfsqx4w")); + testRipper(ripper); + } + + public void testIrarchivesRip() throws IOException { + AbstractRipper ripper = new IrarchivesRipper(new URL("http://i.rarchives.com/?url=user%3Agingerpuss")); + testRipper(ripper); + } + + public void testMinusUserRip() throws IOException { + AbstractRipper ripper = new MinusRipper(new URL("http://vampyr3.minus.com/")); + testRipper(ripper); + deleteSubdirs(ripper.getWorkingDir()); + deleteDir(ripper.getWorkingDir()); + } + public void testMinusUserAlbumRip() throws IOException { + AbstractRipper ripper = new MinusRipper(new URL("http://vampyr3.minus.com/mw7ztQ6xzP7ae")); + testRipper(ripper); + } + public void testMinusUserUploadsRip() throws IOException { + AbstractRipper ripper = new MinusRipper(new URL("http://vampyr3.minus.com/uploads")); + testRipper(ripper); + } + public void testMinusAlbumRip() throws IOException { + AbstractRipper ripper = new MinusRipper(new URL("http://minus.com/mw7ztQ6xzP7ae")); + testRipper(ripper); + } + + public void testModelmayhemRip() throws IOException { + AbstractRipper ripper = new ModelmayhemRipper(new URL("http://www.modelmayhem.com/portfolio/520206/viewall")); + testRipper(ripper); + } + + public void testMotherlessAlbumRip() throws IOException { + MotherlessRipper ripper = new MotherlessRipper(new URL("http://motherless.com/G4DAA18D")); + testRipper(ripper); + } + + public void testNfsfwRip() throws IOException { + AbstractRipper ripper = new NfsfwRipper(new URL("http://nfsfw.com/gallery/v/Kitten/")); + testRipper(ripper); + } + + public void testPhotobucketRip() throws IOException { + AbstractRipper ripper = new PhotobucketRipper(new URL("http://s844.photobucket.com/user/SpazzySpizzy/library/Album%20Covers?sort=3&page=1")); + testRipper(ripper); + deleteSubdirs(ripper.getWorkingDir()); + deleteDir(ripper.getWorkingDir()); + } + + public void testPornhubRip() throws IOException { + AbstractRipper ripper = new PornhubRipper(new URL("http://www.pornhub.com/album/428351")); + testRipper(ripper); + } + + public void testSankakuChanRip() throws IOException { + AbstractRipper ripper = new SankakuComplexRipper(new URL("https://chan.sankakucomplex.com/?tags=blue_necktie")); + testRipper(ripper); + } + public void testSankakuIdolRip() throws IOException { + AbstractRipper ripper = new SankakuComplexRipper(new URL("https://idol.sankakucomplex.com/?tags=meme_%28me%21me%21me%21%29_%28cosplay%29")); + testRipper(ripper); + } + + public void testShesFreakyRip() throws IOException { + AbstractRipper ripper = new ShesFreakyRipper(new URL("http://www.shesfreaky.com/gallery/nicee-snow-bunny-579NbPjUcYa.html")); + testRipper(ripper); + } + + public void testTapasticRip() throws IOException { + AbstractRipper ripper = new TapasticRipper(new URL("http://tapastic.com/episode/2139")); + testRipper(ripper); + } + + public void testTeenplanetRip() throws IOException { + AbstractRipper ripper = new TeenplanetRipper(new URL("http://teenplanet.org/galleries/the-perfect-side-of-me-6588.html")); + testRipper(ripper); + } + + public void testTwodgalleriesRip() throws IOException { + AbstractRipper ripper = new TwodgalleriesRipper(new URL("http://www.2dgalleries.com/artist/regis-loisel-6477")); + testRipper(ripper); + } + + public void testVidbleRip() throws IOException { + AbstractRipper ripper = new VidbleRipper(new URL("http://www.vidble.com/album/y1oyh3zd")); + testRipper(ripper); + } + + public void testVineRip() throws IOException { + AbstractRipper ripper = new VineRipper(new URL("https://vine.co/u/954440445776334848")); + testRipper(ripper); } - public void testSeeniveAlbums() throws IOException { - List contentURLs = new ArrayList(); - contentURLs.add(new URL("http://seenive.com/u/946491170220040192")); - for (URL url : contentURLs) { - SeeniveRipper ripper = new SeeniveRipper(url); - testRipper(ripper); - } + public void testVkSubalbumRip() throws IOException { + VkRipper ripper = new VkRipper(new URL("https://vk.com/album45506334_172415053")); + testRipper(ripper); + } + public void testVkRootAlbumRip() throws IOException { + VkRipper ripper = new VkRipper(new URL("https://vk.com/album45506334_0")); + testRipper(ripper); + } + public void testVkPhotosRip() throws IOException { + VkRipper ripper = new VkRipper(new URL("https://vk.com/photos45506334")); + testRipper(ripper); } - public void testTumblrAlbums() throws IOException { - List contentURLs = new ArrayList(); - contentURLs.add(new URL("http://wrouinr.tumblr.com/archive")); - contentURLs.add(new URL("http://topinstagirls.tumblr.com/tagged/berlinskaya")); - contentURLs.add(new URL("http://genekellyclarkson.tumblr.com/post/86100752527/lucyannebrooks-rachaelboden-friends-goodtimes-bed-boobs")); - for (URL url : contentURLs) { - TumblrRipper ripper = new TumblrRipper(url); - testRipper(ripper); - } - } - - public void testTwitterAlbums() throws IOException { - List contentURLs = new ArrayList(); - contentURLs.add(new URL("https://twitter.com/danngamber01/media")); - contentURLs.add(new URL("https://twitter.com/search?q=from%3Apurrbunny%20filter%3Aimages&src=typd")); - for (URL url : contentURLs) { - TwitterRipper ripper = new TwitterRipper(url); - testRipper(ripper); - } - } - - public void testVkAlbum() throws IOException { - List contentURLs = new ArrayList(); - contentURLs.add(new URL("https://vk.com/album45506334_172415053")); - contentURLs.add(new URL("https://vk.com/album45506334_0")); - contentURLs.add(new URL("https://vk.com/photos45506334")); - for (URL url : contentURLs) { - VkRipper ripper = new VkRipper(url); - testRipper(ripper); - } + public void testXhamsterAlbums() throws IOException { + XhamsterRipper ripper = new XhamsterRipper(new URL("http://xhamster.com/photos/gallery/1462237/alyssa_gadson.html")); + testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java index 024c2215..13992ad7 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java @@ -5,6 +5,8 @@ import java.io.IOException; import junit.framework.TestCase; +import org.apache.log4j.ConsoleAppender; +import org.apache.log4j.Level; import org.apache.log4j.Logger; import com.rarchives.ripme.ripper.AbstractRipper; @@ -24,14 +26,16 @@ public class RippersTest extends TestCase { protected void testRipper(AbstractRipper ripper) { try { - Utils.setConfigInteger("page.timeout", 5 * 1000); + // Turn on Debug logging + ((ConsoleAppender)Logger.getRootLogger().getAppender("stdout")).setThreshold(Level.DEBUG); + + // Decrease timeout + Utils.setConfigInteger("page.timeout", 10 * 1000); + ripper.setup(); ripper.markAsTest(); ripper.rip(); - for (File f : ripper.getWorkingDir().listFiles()) { - System.err.println(f.toString()); - } - assertTrue("Failed to download files from " + ripper.getURL(), ripper.getWorkingDir().listFiles().length >= 1); + assertTrue("Failed to download a single file from " + ripper.getURL(), ripper.getWorkingDir().listFiles().length >= 1); } catch (IOException e) { if (e.getMessage().contains("Ripping interrupted")) { // We expect some rips to get interrupted @@ -78,5 +82,17 @@ public class RippersTest extends TestCase { } dir.delete(); } + protected void deleteSubdirs(File workingDir) { + for (File f : workingDir.listFiles()) { + if (f.isDirectory()) { + for (File sf : f.listFiles()) { + logger.debug("Deleting " + sf); + sf.delete(); + } + logger.debug("Deleting " + f); + f.delete(); + } + } + } }