Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Using new wrapper for HTTP requests, started abstract classes to simplify rippers

Browse Source
This commit is contained in:
4pr0n 2014-06-21 17:08:42 -07:00
parent 1b20e98f8f
commit b7397cd31e
52 changed files with 690 additions and 505 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
src/main/java/com/rarchives/ripme/ripper/AbstractMultiPageRipper.java Normal file
View File

@ -0,0 +1,80 @@
package com.rarchives.ripme.ripper;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.List;
import org.jsoup.nodes.Document;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Utils;
public abstract class AbstractMultiPageRipper extends AlbumRipper {
public AbstractMultiPageRipper(URL url) throws IOException {
super(url);
}
public abstract String getDomain();
public abstract String getHost();
public abstract Document getFirstPage() throws IOException;
public abstract Document getNextPage(Document doc) throws IOException;
public abstract List<String> getURLsFromPage(Document page);
public abstract void downloadURL(URL url, int index);
public boolean keepSortOrder() {
return true;
}
@Override
public boolean canRip(URL url) {
return url.getHost().endsWith(getDomain());
}
@Override
public URL sanitizeURL(URL url) throws MalformedURLException {
return url;
}
@Override
public void rip() throws IOException {
int index = 0;
logger.info("Retrieving " + this.url);
sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm());
Document doc = getFirstPage();
while (doc != null) {
List<String> imageURLs = getURLsFromPage(doc);
if (imageURLs.size() == 0) {
throw new IOException("No images found at " + this.url);
}
for (String imageURL : imageURLs) {
if (isStopped()) {
logger.info("Interrupted");
break;
}
index += 1;
downloadURL(new URL(imageURL), index);
}
try {
doc = getNextPage(doc);
} catch (IOException e) {
logger.info("Can't get next page: " + e.getMessage());
break;
}
}
waitForThreads();
}
public String getPrefix(int index) {
String prefix = "";
if (keepSortOrder() && Utils.getConfigBoolean("download.save_order", true)) {
prefix = String.format("%03d_", index);
}
return prefix;
}
}

98
src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
View File

@ -12,12 +12,7 @@ import java.util.Observable;
import org.apache.log4j.FileAppender;
import org.apache.log4j.Logger;
import org.jsoup.Connection;
import org.jsoup.Connection.Method;
import org.jsoup.Connection.Response;
import org.jsoup.HttpStatusException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import com.rarchives.ripme.ui.RipStatusHandler;
import com.rarchives.ripme.ui.RipStatusMessage;
@ -33,8 +28,6 @@ public abstract class AbstractRipper
public static final String USER_AGENT =
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:29.0) Gecko/20100101 Firefox/29.0";
public static final int TIMEOUT = Utils.getConfigInteger("page.timeout", 5 * 1000);
protected URL url;
protected File workingDir;
protected DownloadThreadPool threadPool;
@ -61,10 +54,6 @@ public abstract class AbstractRipper
}
}
protected int getTimeout() {
return TIMEOUT;
}
/**
* Ensures inheriting ripper can rip this URL, raises exception if not.
* Otherwise initializes working directory and thread pool.
@ -340,6 +329,16 @@ public abstract class AbstractRipper
}
}
public boolean sleep(int milliseconds) {
try {
Thread.sleep(milliseconds);
return true;
} catch (InterruptedException e) {
logger.error("Interrupted while waiting to load next page", e);
return false;
}
}
public void setBytesTotal(int bytes) {
// Do nothing
}
@ -348,82 +347,5 @@ public abstract class AbstractRipper
}
// Thar be overloaded methods afoot
public Document getDocument(URL url) throws IOException {
return getDocument(url.toExternalForm());
}
public Document getDocument(String url) throws IOException {
return getResponse(url).parse();
}
public Document getDocument(String url, boolean ignoreContentType) throws IOException {
return getResponse(url, ignoreContentType).parse();
}
public Document getDocument(String url, Map<String,String> cookies) throws IOException {
return getResponse(url, cookies).parse();
}
public Document getDocument(String url, String referrer, Map<String,String> cookies) throws IOException {
return getResponse(url, Method.GET, referrer, cookies).parse();
}
public Response getResponse(String url) throws IOException {
return getResponse(url, Method.GET, USER_AGENT, null, null, false);
}
public Response getResponse(URL url) throws IOException {
return getResponse(url.toExternalForm());
}
public Response getResponse(String url, String referrer) throws IOException {
return getResponse(url, Method.GET, USER_AGENT, referrer, null, false);
}
public Response getResponse(URL url, boolean ignoreContentType) throws IOException {
return getResponse(url.toExternalForm(), Method.GET, USER_AGENT, null, null, ignoreContentType);
}
public Response getResponse(String url, Map<String, String> cookies) throws IOException {
return getResponse(url, Method.GET, USER_AGENT, cookies);
}
public Response getResponse(String url, String referrer, Map<String, String> cookies) throws IOException {
return getResponse(url, Method.GET, referrer, cookies);
}
public Response getResponse(String url, Method method) throws IOException {
return getResponse(url, method, USER_AGENT, null, null, false);
}
public Response getResponse(String url, Method method, String referrer, Map<String,String> cookies) throws IOException {
return getResponse(url, method, USER_AGENT, referrer, cookies, false);
}
public Response getResponse(String url, boolean ignoreContentType) throws IOException {
return getResponse(url, Method.GET, USER_AGENT, null, null, ignoreContentType);
}
public Response getResponse(String url, Method method, boolean ignoreContentType) throws IOException {
return getResponse(url, method, USER_AGENT, null, null, false);
}
public Response getResponse(String url,
Method method,
String userAgent,
String referrer,
Map<String,String> cookies,
boolean ignoreContentType)
throws IOException {
Connection connection = Jsoup.connect(url);
connection.method( (method == null) ? Method.GET : method);
connection.userAgent( (userAgent == null) ? USER_AGENT : userAgent);
connection.ignoreContentType(ignoreContentType);
connection.timeout(getTimeout());
connection.maxBodySize(0);
if (cookies != null) { connection.cookies(cookies); }
if (referrer != null) { connection.referrer(referrer); }
Response response = null;
int retries = Utils.getConfigInteger("download.retries", 1);;
while (retries >= 0) {
retries--;
try {
response = connection.execute();
} catch (IOException e) {
logger.warn("Error while loading " + url, e);
continue;
}
}
return response;
}
}

70
src/main/java/com/rarchives/ripme/ripper/AbstractSinglePageRipper.java Normal file
View File

@ -0,0 +1,70 @@
package com.rarchives.ripme.ripper;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.List;
import org.jsoup.nodes.Document;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Utils;
public abstract class AbstractSinglePageRipper extends AlbumRipper {
public AbstractSinglePageRipper(URL url) throws IOException {
super(url);
}
public abstract String getDomain();
public abstract String getHost();
public abstract Document getFirstPage() throws IOException;
public abstract List<String> getURLsFromPage(Document page);
public abstract void downloadURL(URL url, int index);
public boolean keepSortOrder() {
return false;
}
@Override
public boolean canRip(URL url) {
return url.getHost().endsWith(getDomain());
}
@Override
public URL sanitizeURL(URL url) throws MalformedURLException {
return url;
}
@Override
public void rip() throws IOException {
int index = 0;
logger.info("Retrieving " + this.url);
sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm());
Document doc = getFirstPage();
List<String> imageURLs = getURLsFromPage(doc);
if (imageURLs.size() == 0) {
throw new IOException("No images found at " + this.url);
}
for (String imageURL : imageURLs) {
if (isStopped()) {
logger.info("Interrupted");
break;
}
index += 1;
downloadURL(new URL(imageURL), index);
}
waitForThreads();
}
public String getPrefix(int index) {
String prefix = "";
if (keepSortOrder() && Utils.getConfigBoolean("download.save_order", true)) {
prefix = String.format("%03d_", index);
}
return prefix;
}
}

54
src/main/java/com/rarchives/ripme/ripper/rippers/ButttoucherRipper.java
View File

@ -3,22 +3,18 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.utils.Utils;
import com.rarchives.ripme.ripper.AbstractSinglePageRipper;
import com.rarchives.ripme.utils.Http;
public class ButttoucherRipper extends AlbumRipper {
private static final String DOMAIN = "butttoucher.com",
HOST = "butttoucher";
private Document albumDoc = null;
public class ButttoucherRipper extends AbstractSinglePageRipper {
public ButttoucherRipper(URL url) throws IOException {
super(url);
@ -26,7 +22,11 @@ public class ButttoucherRipper extends AlbumRipper {
@Override
public String getHost() {
return HOST;
return "butttoucher";
}
@Override
public String getDomain() {
return "butttoucher.com";
}
@Override
@ -45,38 +45,26 @@ public class ButttoucherRipper extends AlbumRipper {
}
@Override
public void rip() throws IOException {
logger.info("Retrieving " + this.url);
if (albumDoc == null) {
albumDoc = getDocument(this.url);
public Document getFirstPage() throws IOException {
return Http.url(this.url).get();
}
int index = 0;
for (Element thumb : albumDoc.select("div.image-gallery > a > img")) {
@Override
public List<String> getURLsFromPage(Document page) {
List<String> thumbs = new ArrayList<String>();
for (Element thumb : page.select("div.image-gallery > a > img")) {
if (!thumb.hasAttr("src")) {
continue;
}
String smallImage = thumb.attr("src");
String image = smallImage.replace("m.", ".");
index += 1;
String prefix = "";
if (Utils.getConfigBoolean("download.save_order", true)) {
prefix = String.format("%03d_", index);
thumbs.add(smallImage.replace("m.", "."));
}
addURLToDownload(new URL(image), prefix);
}
waitForThreads();
}
public boolean canRip(URL url) {
if (!url.getHost().endsWith(DOMAIN)) {
return false;
}
return true;
return thumbs;
}
@Override
public URL sanitizeURL(URL url) throws MalformedURLException {
return url;
public void downloadURL(URL url, int index) {
addURLToDownload(url, getPrefix(index));
}
}

51
src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
View File

@ -3,18 +3,18 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashSet;
import java.util.Set;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.utils.Utils;
import com.rarchives.ripme.ripper.AbstractSinglePageRipper;
import com.rarchives.ripme.utils.Http;
public class ChanRipper extends AlbumRipper {
public class ChanRipper extends AbstractSinglePageRipper {
public ChanRipper(URL url) throws IOException {
super(url);
@ -43,10 +43,6 @@ public class ChanRipper extends AlbumRipper {
|| url.toExternalForm().contains("/thread/")); // 4chan
}
public URL sanitizeURL(URL url) throws MalformedURLException {
return url;
}
@Override
public String getGID(URL url) throws MalformedURLException {
Pattern p; Matcher m;
@ -74,13 +70,20 @@ public class ChanRipper extends AlbumRipper {
}
@Override
public void rip() throws IOException {
Set<String> attempted = new HashSet<String>();
int index = 0;
public String getDomain() {
return this.url.getHost();
}
@Override
public Document getFirstPage() throws IOException {
return Http.url(this.url).get();
}
@Override
public List<String> getURLsFromPage(Document page) {
List<String> imageURLs = new ArrayList<String>();
Pattern p; Matcher m;
logger.info("Retrieving " + this.url);
Document doc = getDocument(this.url);
for (Element link : doc.select("a")) {
for (Element link : page.select("a")) {
if (!link.hasAttr("href")) {
continue;
}
@ -104,20 +107,20 @@ public class ChanRipper extends AlbumRipper {
if (image.startsWith("/")) {
image = "http://" + this.url.getHost() + image;
}
if (attempted.contains(image)) {
// Don't download the same URL twice
if (imageURLs.contains(image)) {
logger.debug("Already attempted: " + image);
continue;
}
index += 1;
String prefix = "";
if (Utils.getConfigBoolean("download.save_order", true)) {
prefix = String.format("%03d_", index);
}
addURLToDownload(new URL(image), prefix);
attempted.add(image);
imageURLs.add(image);
}
}
waitForThreads();
return imageURLs;
}
@Override
public void downloadURL(URL url, int index) {
addURLToDownload(url, getPrefix(index));
}
}

210
src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
View File

@ -6,69 +6,96 @@ import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.jsoup.Connection.Method;
import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.ripper.AbstractMultiPageRipper;
import com.rarchives.ripme.utils.Base64;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class DeviantartRipper extends AlbumRipper {
private static final String DOMAIN = "deviantart.com",
HOST = "deviantart";
public class DeviantartRipper extends AbstractMultiPageRipper {
private static final int SLEEP_TIME = 2000;
private Map<String,String> cookies = new HashMap<String,String>();
private Set<String> triedURLs = new HashSet<String>();
public DeviantartRipper(URL url) throws IOException {
super(url);
}
@Override
public boolean canRip(URL url) {
return url.getHost().endsWith(DOMAIN);
public String getHost() {
return "deviantart";
}
@Override
public String getDomain() {
return "deviantart.com";
}
@Override
public URL sanitizeURL(URL url) throws MalformedURLException {
String u = url.toExternalForm();
u = u.replaceAll("\\?.*", "");
String subdir = "/";
if (u.contains("catpath=scraps")) {
subdir = "scraps";
}
u = u.replaceAll("\\?.*", "?catpath=" + subdir);
return new URL(u);
}
@Override
public void rip() throws IOException {
int index = 0;
String nextURL = this.url.toExternalForm();
public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^https?://([a-zA-Z0-9\\-]+)\\.deviantart\\.com(/gallery)?/?(\\?.*)?$");
Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) {
// Root gallery
if (url.toExternalForm().contains("catpath=scraps")) {
return m.group(1) + "_scraps";
}
else {
return m.group(1);
}
}
p = Pattern.compile("^https?://([a-zA-Z0-9\\-]{1,})\\.deviantart\\.com/gallery/([0-9]{1,}).*$");
m = p.matcher(url.toExternalForm());
if (m.matches()) {
// Subgallery
return m.group(1) + "_" + m.group(2);
}
throw new MalformedURLException("Expected URL format: http://username.deviantart.com/[/gallery/#####], got: " + url);
}
@Override
public Document getFirstPage() throws IOException {
// Login
try {
cookies = loginToDeviantart();
} catch (Exception e) {
logger.warn("Failed to login: ", e);
}
return Http.url(this.url)
.cookies(cookies)
.get();
}
// Iterate over every page
while (nextURL != null) {
logger.info(" Retrieving " + nextURL);
sendUpdate(STATUS.LOADING_RESOURCE, "Retrieving " + nextURL);
Document doc = getDocument(nextURL, cookies);
@Override
public List<String> getURLsFromPage(Document page) {
List<String> imageURLs = new ArrayList<String>();
// Iterate over all thumbnails
for (Element thumb : doc.select("div.zones-container a.thumb")) {
for (Element thumb : page.select("div.zones-container a.thumb")) {
if (isStopped()) {
break;
}
@ -77,77 +104,54 @@ public class DeviantartRipper extends AlbumRipper {
continue; // a.thumbs to other albums are invisible
}
index++;
// Get full-sized image via helper methods
String fullSize = null;
try {
fullSize = thumbToFull(img.attr("src"), true);
} catch (Exception e) {
logger.info("Attempting to get full size image from " + thumb.attr("href"));
fullSize = smallToFull(img.attr("src"), thumb.attr("href"));
}
if (fullSize == null) {
continue;
}
}
try {
URL fullsizeURL = new URL(fullSize);
String imageId = fullSize.substring(fullSize.lastIndexOf('-') + 1);
imageId = imageId.substring(0, imageId.indexOf('.'));
long imageIdLong = alphaToLong(imageId);
addURLToDownload(fullsizeURL, String.format("%010d_", imageIdLong));
} catch (MalformedURLException e) {
logger.error("[!] Invalid thumbnail image: " + fullSize);
if (triedURLs.contains(fullSize)) {
logger.warn("Already tried to download " + fullSize);
continue;
}
triedURLs.add(fullSize);
imageURLs.add(fullSize);
}
return imageURLs;
}
try {
Thread.sleep(SLEEP_TIME);
} catch (InterruptedException e) {
logger.error("[!] Interrupted while waiting for page to load", e);
break;
@Override
public Document getNextPage(Document page) throws IOException {
Elements nextButtons = page.select("li.next > a");
if (nextButtons.size() == 0) {
throw new IOException("No next page found");
}
Element a = nextButtons.first();
if (a.hasClass("disabled")) {
throw new IOException("Hit end of pages");
}
String nextPage = a.attr("href");
if (nextPage.startsWith("/")) {
nextPage = "http://" + this.url.getHost() + nextPage;
}
if (!sleep(SLEEP_TIME)) {
throw new IOException("Interrupted while waiting to load next page: " + nextPage);
}
logger.info("Found next page: " + nextPage);
return Http.url(nextPage)
.cookies(cookies)
.get();
}
// Find the next page
nextURL = null;
for (Element nextButton : doc.select("a.away")) {
if (nextButton.attr("href").contains("offset=" + index)) {
nextURL = this.url.toExternalForm() + "?offset=" + index;
}
}
if (nextURL == null) {
logger.info("No next button found");
}
}
waitForThreads();
}
/**
* Convert alpha-numeric string into a corresponding number
* @param alpha String to convert
* @return Numeric representation of 'alpha'
*/
public static long alphaToLong(String alpha) {
long result = 0;
for (int i = 0; i < alpha.length(); i++) {
result += charToInt(alpha, i);
}
return result;
}
/**
* Convert character at index in a string 'text' to numeric form (base-36)
* @param text Text to retrieve the character from
* @param index Index of the desired character
* @return Number representing character at text[index]
*/
private static int charToInt(String text, int index) {
char c = text.charAt(text.length() - index - 1);
c = Character.toLowerCase(c);
int number = "0123456789abcdefghijklmnopqrstuvwxyz".indexOf(c);
number *= Math.pow(36, index);
return number;
@Override
public void downloadURL(URL url, int index) {
addURLToDownload(url, getPrefix(index), "", this.url.toExternalForm(), cookies);
}
/**
@ -163,7 +167,6 @@ public class DeviantartRipper extends AlbumRipper {
fields.remove(4);
if (!fields.get(4).equals("f") && throwException) {
// Not a full-size image
logger.warn("Can't get full size image from " + thumb);
throw new Exception("Can't get full size image from " + thumb);
}
StringBuilder result = new StringBuilder();
@ -187,27 +190,20 @@ public class DeviantartRipper extends AlbumRipper {
public String smallToFull(String thumb, String page) {
try {
// Fetch the image page
Response resp = getResponse(page, Method.GET, USER_AGENT, this.url.toExternalForm(), cookies, false);
Map<String,String> cookies = resp.cookies();
cookies.putAll(this.cookies);
Response resp = Http.url(page)
.referrer(this.url)
.cookies(cookies)
.response();
cookies.putAll(resp.cookies());
// Try to find the "Download" box
Elements els = resp.parse().select("a.dev-page-download");
if (els.size() == 0) {
throw new IOException("no download page found");
throw new IOException("No download page found");
}
// Full-size image
String fsimage = els.get(0).attr("href");
String prefix = "";
if (Utils.getConfigBoolean("download.save_order", true)) {
String imageId = fsimage.substring(fsimage.lastIndexOf('-') + 1);
imageId = imageId.substring(0, imageId.indexOf('.'));
prefix = String.format("%010d_", alphaToLong(imageId));
}
// Download it
addURLToDownload(new URL(fsimage), prefix, "", page, cookies);
return null;
return fsimage;
} catch (IOException ioe) {
try {
logger.info("Failed to get full size download image at " + page + " : '" + ioe.getMessage() + "'");
@ -220,28 +216,6 @@ public class DeviantartRipper extends AlbumRipper {
}
}
@Override
public String getHost() {
return HOST;
}
@Override
public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^https?://([a-zA-Z0-9\\-]{1,})\\.deviantart\\.com(/gallery)?/?$");
Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) {
// Root gallery
return m.group(1);
}
p = Pattern.compile("^https?://([a-zA-Z0-9\\-]{1,})\\.deviantart\\.com/gallery/([0-9]{1,}).*$");
m = p.matcher(url.toExternalForm());
if (m.matches()) {
// Subgallery
return m.group(1) + "_" + m.group(2);
}
throw new MalformedURLException("Expected URL format: http://username.deviantart.com/[/gallery/#####], got: " + url);
}
/**
* Logs into deviant art. Required to rip full-size NSFW content.
* @return Map of cookies containing session data.
@ -254,7 +228,8 @@ public class DeviantartRipper extends AlbumRipper {
if (username == null || password == null) {
throw new IOException("could not find username or password in config");
}
Response resp = getResponse("http://www.deviantart.com/");
Response resp = Http.url("http://www.deviantart.com/")
.response();
for (Element input : resp.parse().select("form#form-login input[type=hidden]")) {
postData.put(input.attr("name"), input.attr("value"));
}
@ -263,17 +238,17 @@ public class DeviantartRipper extends AlbumRipper {
postData.put("remember_me", "1");
// Send login request
resp = Jsoup.connect("https://www.deviantart.com/users/login")
resp = Http.url("https://www.deviantart.com/users/login")
.userAgent(USER_AGENT)
.data(postData)
.cookies(resp.cookies())
.method(Method.POST)
.execute();
.response();
// Assert we are logged in
if (resp.hasHeader("Location") && resp.header("Location").contains("password")) {
// Wrong password
throw new IOException("Wrong pasword");
throw new IOException("Wrong password");
}
if (resp.url().toExternalForm().contains("bad_form")) {
throw new IOException("Login form was incorrectly submitted");
@ -285,5 +260,4 @@ public class DeviantartRipper extends AlbumRipper {
// We are logged in, save the cookies
return resp.cookies();
}
}

80
src/main/java/com/rarchives/ripme/ripper/rippers/DrawcrowdRipper.java
View File

@ -3,6 +3,8 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -10,14 +12,10 @@ import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Utils;
import com.rarchives.ripme.ripper.AbstractMultiPageRipper;
import com.rarchives.ripme.utils.Http;
public class DrawcrowdRipper extends AlbumRipper {
private static final String DOMAIN = "drawcrowd.com",
HOST = "drawcrowd";
public class DrawcrowdRipper extends AbstractMultiPageRipper {
public DrawcrowdRipper(URL url) throws IOException {
super(url);
@ -25,14 +23,11 @@ public class DrawcrowdRipper extends AlbumRipper {
@Override
public String getHost() {
return HOST;
return "drawcrowd";
}
/**
* Reformat given URL into the desired format (all images on single page)
*/
public URL sanitizeURL(URL url) throws MalformedURLException {
return url;
@Override
public String getDomain() {
return "drawcrowd.com";
}
@Override
@ -58,46 +53,39 @@ public class DrawcrowdRipper extends AlbumRipper {
}
@Override
public void rip() throws IOException {
int index = 0;
sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm());
logger.info("Retrieving " + this.url);
Document albumDoc = getDocument(this.url);
while (true) {
if (isStopped()) {
break;
public Document getFirstPage() throws IOException {
return Http.url(this.url).get();
}
for (Element thumb : albumDoc.select("div.item.asset img")) {
@Override
public Document getNextPage(Document doc) throws IOException {
Elements loadMore = doc.select("a#load-more");
if (loadMore.size() == 0) {
throw new IOException("No next page found");
}
if (!sleep(1000)) {
throw new IOException("Interrupted while waiting for next page");
}
String nextPage = "http://drawcrowd.com" + loadMore.get(0).attr("href");
return Http.url(nextPage).get();
}
@Override
public List<String> getURLsFromPage(Document page) {
List<String> imageURLs = new ArrayList<String>();
for (Element thumb : page.select("div.item.asset img")) {
String image = thumb.attr("src");
image = image
.replaceAll("/medium/", "/large/")
.replaceAll("/small/", "/large/");
index++;
String prefix = "";
if (Utils.getConfigBoolean("download.save_order", true)) {
prefix = String.format("%03d_", index);
imageURLs.add(image);
}
addURLToDownload(new URL(image), prefix);
}
Elements loadMore = albumDoc.select("a#load-more");
if (loadMore.size() == 0) {
break;
}
String nextURL = "http://drawcrowd.com" + loadMore.get(0).attr("href");
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
logger.error("Interrupted while waiting to load next page", e);
throw new IOException(e);
}
sendUpdate(STATUS.LOADING_RESOURCE, nextURL);
albumDoc = getDocument(nextURL);
}
waitForThreads();
return imageURLs;
}
public boolean canRip(URL url) {
return url.getHost().endsWith(DOMAIN);
@Override
public void downloadURL(URL url, int index) {
addURLToDownload(url, getPrefix(index));
}
}

16
src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java
View File

@ -16,6 +16,7 @@ import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ripper.DownloadThreadPool;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class EHentaiRipper extends AlbumRipper {
@ -57,7 +58,9 @@ public class EHentaiRipper extends AlbumRipper {
if (albumDoc == null) {
sendUpdate(STATUS.LOADING_RESOURCE, url.toString());
logger.info("Retrieving " + url);
albumDoc = getDocument(url.toExternalForm(), cookies);
albumDoc = Http.url(url)
.cookies(cookies)
.get();
}
Elements elems = albumDoc.select("#gn");
return HOST + "_" + elems.get(0).text();
@ -96,7 +99,10 @@ public class EHentaiRipper extends AlbumRipper {
if (albumDoc == null) {
logger.info(" Retrieving album page " + nextUrl);
sendUpdate(STATUS.LOADING_RESOURCE, nextUrl);
albumDoc = getDocument(nextUrl, this.url.toExternalForm(), cookies);
albumDoc = Http.url(nextUrl)
.referrer(this.url)
.cookies(cookies)
.get();
}
// Check for rate limiting
if (albumDoc.toString().contains("IP address will be automatically banned")) {
@ -197,8 +203,10 @@ public class EHentaiRipper extends AlbumRipper {
private void fetchImage() {
try {
String u = this.url.toExternalForm();
Document doc = getDocument(u, u, cookies);
Document doc = Http.url(this.url)
.referrer(this.url)
.cookies(cookies)
.get();
// Check for rate limit
if (doc.toString().contains("IP address will be automatically banned")) {
if (this.retries == 0) {

5
src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java
View File

@ -12,6 +12,7 @@ import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class EightmusesRipper extends AlbumRipper {
@ -40,7 +41,7 @@ public class EightmusesRipper extends AlbumRipper {
try {
// Attempt to use album title as GID
if (albumDoc == null) {
albumDoc = getDocument(url);
albumDoc = Http.url(url).get();
}
Element titleElement = albumDoc.select("meta[name=description]").first();
String title = titleElement.attr("content");
@ -63,7 +64,7 @@ public class EightmusesRipper extends AlbumRipper {
logger.info(" Retrieving " + url);
sendUpdate(STATUS.LOADING_RESOURCE, url);
if (albumDoc == null) {
albumDoc = getDocument(url);
albumDoc = Http.url(url).get();
}
int index = 0; // Both album index and image index

5
src/main/java/com/rarchives/ripme/ripper/rippers/FapprovedRipper.java
View File

@ -11,6 +11,7 @@ import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class FapprovedRipper extends AlbumRipper {
@ -46,7 +47,9 @@ public class FapprovedRipper extends AlbumRipper {
url = "http://fapproved.com/users/" + user + "/images?page=" + page;
this.sendUpdate(STATUS.LOADING_RESOURCE, url);
logger.info(" Retrieving " + url);
Document doc = getDocument(url, true);
Document doc = Http.url(url)
.ignoreContentType()
.get();
for (Element image : doc.select("div.actual-image img")) {
String imageUrl = image.attr("src");
if (imageUrl.startsWith("//")) {

9
src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java
View File

@ -20,6 +20,7 @@ import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ripper.DownloadThreadPool;
import com.rarchives.ripme.utils.Base64;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class FlickrRipper extends AlbumRipper {
@ -61,7 +62,7 @@ public class FlickrRipper extends AlbumRipper {
try {
// Attempt to use album title as GID
if (albumDoc == null) {
albumDoc = getDocument(url);
albumDoc = Http.url(url).get();
}
String user = url.toExternalForm();
user = user.substring(user.indexOf("/photos/") + "/photos/".length());
@ -124,7 +125,7 @@ public class FlickrRipper extends AlbumRipper {
}
logger.info(" Retrieving " + nextURL);
if (albumDoc == null) {
albumDoc = getDocument(nextURL);
albumDoc = Http.url(nextURL).get();
}
for (Element thumb : albumDoc.select("a[data-track=photo-click]")) {
String imageTitle = null;
@ -259,7 +260,7 @@ public class FlickrRipper extends AlbumRipper {
private Document getLargestImagePageDocument(URL url) throws IOException {
// Get current page
Document doc = getDocument(url);
Document doc = Http.url(url).get();
// Look for larger image page
String largestImagePage = this.url.toExternalForm();
for (Element olSize : doc.select("ol.sizes-list > li > ol > li")) {
@ -277,7 +278,7 @@ public class FlickrRipper extends AlbumRipper {
}
if (!largestImagePage.equals(this.url.toExternalForm())) {
// Found larger image page, get it.
doc = getDocument(largestImagePage);
doc = Http.url(largestImagePage).get();
}
return doc;
}

13
src/main/java/com/rarchives/ripme/ripper/rippers/GifyoRipper.java
View File

@ -10,13 +10,13 @@ import java.util.regex.Pattern;
import org.jsoup.Connection.Method;
import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
public class GifyoRipper extends AlbumRipper {
@ -50,7 +50,9 @@ public class GifyoRipper extends AlbumRipper {
logger.info(" Retrieving " + this.url + "(page #" + page + ")");
Response resp = null;
if (page == 0) {
resp = getResponse(this.url, true);
resp = Http.url(this.url)
.ignoreContentType()
.response();
cookies = resp.cookies();
}
else {
@ -59,13 +61,12 @@ public class GifyoRipper extends AlbumRipper {
postData.put("view", "gif");
postData.put("layout", "grid");
postData.put("page", Integer.toString(page));
resp = Jsoup.connect(this.url.toExternalForm())
.ignoreContentType(true)
.userAgent(USER_AGENT)
resp = Http.url(this.url)
.ignoreContentType()
.data(postData)
.cookies(cookies)
.method(Method.POST)
.execute();
.response();
cookies.putAll(resp.cookies());
}
Document doc = resp.parse();

5
src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java
View File

@ -12,6 +12,7 @@ import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
public class GirlsOfDesireRipper extends AlbumRipper {
// All sleep times are in milliseconds
@ -41,7 +42,7 @@ public class GirlsOfDesireRipper extends AlbumRipper {
if (albumDoc == null) {
logger.info(" Retrieving " + url.toExternalForm());
sendUpdate(STATUS.LOADING_RESOURCE, url.toString());
albumDoc = getDocument(url);
albumDoc = Http.url(url).get();
}
Elements elems = albumDoc.select(".albumName");
return HOST + "_" + elems.first().text();
@ -76,7 +77,7 @@ public class GirlsOfDesireRipper extends AlbumRipper {
if (albumDoc == null) {
logger.info(" Retrieving album page " + nextUrl);
sendUpdate(STATUS.LOADING_RESOURCE, nextUrl);
albumDoc = getDocument(nextUrl);
albumDoc = Http.url(nextUrl).get();
}
// Find thumbnails

8
src/main/java/com/rarchives/ripme/ripper/rippers/GonewildRipper.java
View File

@ -8,9 +8,9 @@ import java.util.regex.Pattern;
import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class GonewildRipper extends AlbumRipper {
@ -49,7 +49,7 @@ public class GonewildRipper extends AlbumRipper {
+ "?method=get_user"
+ "&user=" + username
+ "&count=" + count;
String gwURL, jsonString, imagePath;
String gwURL, imagePath;
JSONArray posts, images;
JSONObject json, post, image;
while (true) {
@ -57,8 +57,8 @@ public class GonewildRipper extends AlbumRipper {
gwURL = baseGwURL
+ "&start=" + start;
start += count;
jsonString = getResponse(gwURL, true).body();
json = new JSONObject(jsonString);
json = Http.url(gwURL)
.getJSON();
if (json.has("error")) {
logger.error("Error while retrieving user posts:" + json.getString("error"));
break;

15
src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java
View File

@ -14,6 +14,7 @@ import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class HentaifoundryRipper extends AlbumRipper {
@ -40,11 +41,12 @@ public class HentaifoundryRipper extends AlbumRipper {
int index = 0;
// Get cookies
Response resp = getResponse("http://www.hentai-foundry.com/");
Response resp = Http.url("http://www.hentai-foundry.com/").response();
Map<String,String> cookies = resp.cookies();
resp = getResponse("http://www.hentai-foundry.com/?enterAgree=1&size=1500",
"http://www.hentai-foundry.com/",
cookies);
resp = Http.url("http://www.hentai-foundry.com/?enterAgree=1&size=1500")
.referrer("http://www.hentai-foundry.com/")
.cookies(cookies)
.response();
cookies = resp.cookies();
logger.info("cookies: " + cookies);
@ -54,7 +56,10 @@ public class HentaifoundryRipper extends AlbumRipper {
break;
}
sendUpdate(STATUS.LOADING_RESOURCE, nextURL);
Document doc = getDocument(nextURL, this.url.toExternalForm(), cookies);
Document doc = Http.url(nextURL)
.referrer(this.url)
.cookies(cookies)
.get();
for (Element thumb : doc.select("td > a:first-child")) {
if (isStopped()) {
break;

5
src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java
View File

@ -11,6 +11,7 @@ import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class ImagearnRipper extends AlbumRipper {
@ -41,7 +42,7 @@ public class ImagearnRipper extends AlbumRipper {
}
private URL getGalleryFromImage(URL url) throws IOException {
Document doc = getDocument(url);
Document doc = Http.url(url).get();
for (Element link : doc.select("a[href~=^gallery\\.php.*$]")) {
logger.info("LINK: " + link.toString());
if (link.hasAttr("href")
@ -59,7 +60,7 @@ public class ImagearnRipper extends AlbumRipper {
int index = 0;
logger.info("Retrieving " + this.url.toExternalForm());
sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm());
Document doc = getDocument(this.url);
Document doc = Http.url(this.url).get();
for (Element thumb : doc.select("img.border")) {
if (isStopped()) {
break;

9
src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java
View File

@ -13,6 +13,7 @@ import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ripper.DownloadThreadPool;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class ImagebamRipper extends AlbumRipper {
@ -47,7 +48,7 @@ public class ImagebamRipper extends AlbumRipper {
if (albumDoc == null) {
logger.info(" Retrieving " + url.toExternalForm());
sendUpdate(STATUS.LOADING_RESOURCE, url.toString());
albumDoc = getDocument(url);
albumDoc = Http.url(url).get();
}
Elements elems = albumDoc.select("legend");
String title = elems.first().text();
@ -94,7 +95,9 @@ public class ImagebamRipper extends AlbumRipper {
if (albumDoc == null) {
logger.info(" Retrieving album page " + nextUrl);
sendUpdate(STATUS.LOADING_RESOURCE, nextUrl);
albumDoc = getDocument(nextUrl, this.url.toExternalForm(), null);
albumDoc = Http.url(nextUrl)
.referrer(this.url)
.get();
}
// Find thumbnails
Elements thumbs = albumDoc.select("div > a[target=_blank]:not(.footera)");
@ -171,7 +174,7 @@ public class ImagebamRipper extends AlbumRipper {
private void fetchImage() {
try {
Document doc = getDocument(url);
Document doc = Http.url(url).get();
// Find image
Elements images = doc.select("td > img");
if (images.size() == 0) {

5
src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java
View File

@ -12,6 +12,7 @@ import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class ImagefapRipper extends AlbumRipper {
@ -45,7 +46,7 @@ public class ImagefapRipper extends AlbumRipper {
try {
// Attempt to use album title as GID
if (albumDoc == null) {
albumDoc = getDocument(url);
albumDoc = Http.url(url).get();
}
String title = albumDoc.title();
Pattern p = Pattern.compile("^Porn pics of (.*) \\(Page 1\\)$");
@ -94,7 +95,7 @@ public class ImagefapRipper extends AlbumRipper {
sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm());
logger.info("Retrieving " + this.url);
if (albumDoc == null) {
albumDoc = getDocument(this.url);
albumDoc = Http.url(this.url).get();
}
while (true) {
if (isStopped()) {

5
src/main/java/com/rarchives/ripme/ripper/rippers/ImagestashRipper.java
View File

@ -11,6 +11,7 @@ import org.json.JSONObject;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class ImagestashRipper extends AlbumRipper {
@ -45,9 +46,7 @@ public class ImagestashRipper extends AlbumRipper {
String nextURL = baseURL + "&page=" + page;
logger.info("[ ] Retrieving " + nextURL);
sendUpdate(STATUS.LOADING_RESOURCE, nextURL);
String jsonText = getResponse(nextURL, true).body();
logger.info(jsonText);
JSONObject json = new JSONObject(jsonText);
JSONObject json = Http.url(nextURL).getJSON();
JSONArray images = json.getJSONArray("images");
for (int i = 0; i < images.length(); i++) {
if (isStopped()) {

5
src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java
View File

@ -13,6 +13,7 @@ import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ripper.DownloadThreadPool;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class ImagevenueRipper extends AlbumRipper {
@ -60,7 +61,7 @@ public class ImagevenueRipper extends AlbumRipper {
String nextUrl = this.url.toExternalForm();
logger.info(" Retrieving album page " + nextUrl);
sendUpdate(STATUS.LOADING_RESOURCE, nextUrl);
Document albumDoc = getDocument(nextUrl);
Document albumDoc = Http.url(nextUrl).get();
// Find thumbnails
Elements thumbs = albumDoc.select("a[target=_blank]");
if (thumbs.size() == 0) {
@ -115,7 +116,7 @@ public class ImagevenueRipper extends AlbumRipper {
private void fetchImage() {
try {
sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm());
Document doc = getDocument(this.url);
Document doc = Http.url(this.url).get();
// Find image
Elements images = doc.select("a > img");
if (images.size() == 0) {

3
src/main/java/com/rarchives/ripme/ripper/rippers/ImgboxRipper.java
View File

@ -12,6 +12,7 @@ import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class ImgboxRipper extends AlbumRipper {
@ -36,7 +37,7 @@ public class ImgboxRipper extends AlbumRipper {
@Override
public void rip() throws IOException {
sendUpdate(STATUS.LOADING_RESOURCE, url.toExternalForm());
Document doc = getDocument(this.url);
Document doc = Http.url(this.url).get();
Elements images = doc.select("div.boxed-content > a > img");
if (images.size() == 0) {
logger.error("No images found at " + this.url);

10
src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java
View File

@ -19,6 +19,7 @@ import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class ImgurRipper extends AlbumRipper {
@ -83,7 +84,7 @@ public class ImgurRipper extends AlbumRipper {
try {
// Attempt to use album title as GID
if (albumDoc == null) {
albumDoc = getDocument(url);
albumDoc = Http.url(url).get();
}
String title = albumDoc.title();
if (!title.contains(" - Imgur")
@ -259,7 +260,7 @@ public class ImgurRipper extends AlbumRipper {
private void ripUserAccount(URL url) throws IOException {
logger.info("Retrieving " + url);
sendUpdate(STATUS.LOADING_RESOURCE, url.toExternalForm());
Document doc = getDocument(url);
Document doc = Http.url(url).get();
for (Element album : doc.select("div.cover a")) {
stopCheck();
if (!album.hasAttr("href")
@ -289,8 +290,7 @@ public class ImgurRipper extends AlbumRipper {
try {
page++;
String jsonUrlWithParams = jsonUrl + "?sort=0&order=1&album=0&page=" + page + "&perPage=60";
String jsonString = getResponse(jsonUrlWithParams, true).body();
JSONObject json = new JSONObject(jsonString);
JSONObject json = Http.url(jsonUrlWithParams).getJSON();
JSONObject jsonData = json.getJSONObject("data");
if (jsonData.has("count")) {
imagesTotal = jsonData.getInt("count");
@ -327,7 +327,7 @@ public class ImgurRipper extends AlbumRipper {
}
pageURL += "page/" + page + "/miss?scrolled";
logger.info(" Retrieving " + pageURL);
Document doc = getDocument(pageURL);
Document doc = Http.url(pageURL).get();
Elements imgs = doc.select(".post img");
for (Element img : imgs) {
String image = img.attr("src");

8
src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java
View File

@ -13,6 +13,7 @@ import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
public class InstagramRipper extends AlbumRipper {
@ -62,7 +63,7 @@ public class InstagramRipper extends AlbumRipper {
}
private URL getUserPageFromImage(URL url) throws IOException {
Document doc = getDocument(url);
Document doc = Http.url(url).get();
for (Element element : doc.select("meta[property='og:description']")) {
String content = element.attr("content");
if (content.endsWith("'s photo on Instagram")) {
@ -75,7 +76,7 @@ public class InstagramRipper extends AlbumRipper {
private String getUserID(URL url) throws IOException {
logger.info("Retrieving " + url);
this.sendUpdate(STATUS.LOADING_RESOURCE, url.toExternalForm());
Document doc = getDocument(url);
Document doc = Http.url(url).get();
for (Element element : doc.select("input[id=user_public]")) {
return element.attr("value");
}
@ -91,8 +92,7 @@ public class InstagramRipper extends AlbumRipper {
String url = baseURL + params;
this.sendUpdate(STATUS.LOADING_RESOURCE, url);
logger.info(" Retrieving " + url);
String jsonString = getResponse(url, true).body();
JSONObject json = new JSONObject(jsonString);
JSONObject json = Http.url(url).getJSON();
JSONArray datas = json.getJSONArray("data");
String nextMaxID = "";
if (datas.length() == 0) {

11
src/main/java/com/rarchives/ripme/ripper/rippers/IrarchivesRipper.java
View File

@ -11,6 +11,7 @@ import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ripper.rippers.ImgurRipper.ImgurAlbum;
import com.rarchives.ripme.ripper.rippers.ImgurRipper.ImgurImage;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class IrarchivesRipper extends AlbumRipper {
@ -22,11 +23,6 @@ public class IrarchivesRipper extends AlbumRipper {
super(url);
}
@Override
public int getTimeout() {
return 60 * 1000;
}
@Override
public boolean canRip(URL url) {
return url.getHost().endsWith(DOMAIN);
@ -48,8 +44,9 @@ public class IrarchivesRipper extends AlbumRipper {
@Override
public void rip() throws IOException {
logger.info(" Retrieving " + this.url);
String jsonString = getResponse(url, true).body();
JSONObject json = new JSONObject(jsonString);
JSONObject json = Http.url(url)
.timeout(60 * 1000)
.getJSON();
JSONArray posts = json.getJSONArray("posts");
if (posts.length() == 0) {
logger.error("No posts found at " + this.url);

6
src/main/java/com/rarchives/ripme/ripper/rippers/MediacrushRipper.java
View File

@ -18,6 +18,7 @@ import org.json.JSONObject;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class MediacrushRipper extends AlbumRipper {
@ -67,9 +68,9 @@ public class MediacrushRipper extends AlbumRipper {
String url = this.url.toExternalForm();
logger.info(" Retrieving " + url);
sendUpdate(STATUS.LOADING_RESOURCE, url);
String jsonString = null;
JSONObject json = null;
try {
jsonString = getResponse(url, true).body();
json = Http.url(url).getJSON();
} catch (Exception re) {
// Check for >1024 bit encryption but in older versions of Java
if (re.getCause().getCause() instanceof InvalidAlgorithmParameterException) {
@ -96,7 +97,6 @@ public class MediacrushRipper extends AlbumRipper {
}
// Convert to JSON
JSONObject json = new JSONObject(jsonString);
if (!json.has("files")) {
sendUpdate(STATUS.RIP_ERRORED, "No files found at " + url);
throw new IOException("Could not find any files at " + url);

9
src/main/java/com/rarchives/ripme/ripper/rippers/MinusRipper.java
View File

@ -8,11 +8,11 @@ import java.util.regex.Pattern;
import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.Connection.Response;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class MinusRipper extends AlbumRipper {
@ -47,7 +47,7 @@ public class MinusRipper extends AlbumRipper {
try {
// Attempt to use album title as GID
if (albumDoc == null) {
albumDoc = getDocument(url);
albumDoc = Http.url(url).get();
}
Elements titles = albumDoc.select("meta[property=og:title]");
if (titles.size() > 0) {
@ -129,8 +129,7 @@ public class MinusRipper extends AlbumRipper {
+ user + "/shares.json/"
+ page;
logger.info(" Retrieving " + jsonUrl);
Response resp = getResponse(jsonUrl, true);
JSONObject json = new JSONObject(resp.body());
JSONObject json = Http.url(jsonUrl).getJSON();
JSONArray galleries = json.getJSONArray("galleries");
for (int i = 0; i < galleries.length(); i++) {
JSONObject gallery = galleries.getJSONObject(i);
@ -151,7 +150,7 @@ public class MinusRipper extends AlbumRipper {
private void ripAlbum(URL url, String subdir) throws IOException {
logger.info(" Retrieving " + url.toExternalForm());
if (albumDoc == null || !subdir.equals("")) {
albumDoc = getDocument(url);
albumDoc = Http.url(url).get();
}
Pattern p = Pattern.compile("^.*var gallerydata = (\\{.*\\});.*$", Pattern.DOTALL);
Matcher m = p.matcher(albumDoc.data());

9
src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java
View File

@ -12,6 +12,7 @@ import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ripper.DownloadThreadPool;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class MotherlessRipper extends AlbumRipper {
@ -66,7 +67,9 @@ public class MotherlessRipper extends AlbumRipper {
}
logger.info("Retrieving " + nextURL);
sendUpdate(STATUS.LOADING_RESOURCE, nextURL);
Document doc = getDocument(nextURL, "http://motherless.com", null);
Document doc = Http.url(nextURL)
.referrer("http://motherless.com")
.get();
for (Element thumb : doc.select("div.thumb a.img-container")) {
if (isStopped()) {
break;
@ -118,7 +121,9 @@ public class MotherlessRipper extends AlbumRipper {
return;
}
String u = this.url.toExternalForm();
Document doc = getDocument(u, u, null);
Document doc = Http.url(u)
.referrer(u)
.get();
Pattern p = Pattern.compile("^.*__fileurl = '([^']{1,})';.*$", Pattern.DOTALL);
Matcher m = p.matcher(doc.outerHtml());
if (m.matches()) {

10
src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java
View File

@ -15,6 +15,7 @@ import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ripper.DownloadThreadPool;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class NfsfwRipper extends AlbumRipper {
@ -46,7 +47,7 @@ public class NfsfwRipper extends AlbumRipper {
try {
// Attempt to use album title as GID
if (albumDoc == null) {
albumDoc = getDocument(url);
albumDoc = Http.url(url).get();
}
String title = albumDoc.select("h2").first().text().trim();
return "nfsfw_" + Utils.filesystemSafe(title);
@ -87,7 +88,7 @@ public class NfsfwRipper extends AlbumRipper {
sendUpdate(STATUS.LOADING_RESOURCE, nextURL);
logger.info(" Retrieving " + nextURL);
if (albumDoc == null) {
albumDoc = getDocument(nextURL);
albumDoc = Http.url(nextURL).get();
}
// Subalbums
for (Element suba : albumDoc.select("td.IMG > a")) {
@ -156,8 +157,9 @@ public class NfsfwRipper extends AlbumRipper {
@Override
public void run() {
try {
String u = this.url.toExternalForm();
Document doc = getDocument(u, u, null);
Document doc = Http.url(this.url)
.referrer(this.url)
.get();
Elements images = doc.select(".gbBlock img");
if (images.size() == 0) {
logger.error("Failed to find image at " + this.url);

9
src/main/java/com/rarchives/ripme/ripper/rippers/PhotobucketRipper.java
View File

@ -16,6 +16,7 @@ import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.utils.Http;
public class PhotobucketRipper extends AlbumRipper {
@ -49,7 +50,7 @@ public class PhotobucketRipper extends AlbumRipper {
try {
// Attempt to use album title as GID
if (pageResponse == null) {
pageResponse = getResponse(url);
pageResponse = Http.url(url).response();
}
Document albumDoc = pageResponse.parse();
Elements els = albumDoc.select("div.libraryTitle > h1");
@ -131,7 +132,7 @@ public class PhotobucketRipper extends AlbumRipper {
if (pageIndex > 1 || pageResponse == null) {
url = theUrl + String.format("?sort=3&page=", pageIndex);
logger.info(" Retrieving " + url);
pageResponse = getResponse(url);
pageResponse = Http.url(url).response();
}
Document albumDoc = pageResponse.parse();
// Retrieve JSON from request
@ -189,9 +190,7 @@ public class PhotobucketRipper extends AlbumRipper {
+ "&json=1";
try {
logger.info("Loading " + apiUrl);
Document doc = getDocument(apiUrl, true);
String jsonString = doc.body().html().replace("&quot;", "\"");
JSONObject json = new JSONObject(jsonString);
JSONObject json = Http.url(apiUrl).getJSON();
JSONArray subalbums = json.getJSONObject("body").getJSONArray("subAlbums");
for (int i = 0; i < subalbums.length(); i++) {
String suburl =

14
src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java
View File

@ -7,7 +7,6 @@ import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
@ -15,12 +14,12 @@ import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ripper.DownloadThreadPool;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class PornhubRipper extends AlbumRipper {
// All sleep times are in milliseconds
private static final int IMAGE_SLEEP_TIME = 1 * 1000;
private static final int TIMEOUT = 5 * 1000;
private static final String DOMAIN = "pornhub.com", HOST = "Pornhub";
@ -49,7 +48,7 @@ public class PornhubRipper extends AlbumRipper {
if (albumDoc == null) {
logger.info(" Retrieving " + url.toExternalForm());
sendUpdate(STATUS.LOADING_RESOURCE, url.toString());
albumDoc = getDocument(url);
albumDoc = Http.url(url).get();
}
Elements elems = albumDoc.select(".photoAlbumTitleV2");
return HOST + "_" + elems.get(0).text();
@ -89,7 +88,9 @@ public class PornhubRipper extends AlbumRipper {
if (albumDoc == null) {
logger.info(" Retrieving album page " + nextUrl);
sendUpdate(STATUS.LOADING_RESOURCE, nextUrl);
albumDoc = getDocument(nextUrl, this.url.toExternalForm(), null);
albumDoc = Http.url(nextUrl)
.referrer(this.url)
.get();
}
// Find thumbnails
@ -146,8 +147,9 @@ public class PornhubRipper extends AlbumRipper {
private void fetchImage() {
try {
String u = this.url.toExternalForm();
Document doc = getDocument(u, u, null);
Document doc = Http.url(this.url)
.referrer(this.url)
.get();
// Find image
Elements images = doc.select("#photoImageSection img");

14
src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java
View File

@ -11,9 +11,9 @@ import java.util.regex.Pattern;
import org.json.JSONArray;
import org.json.JSONObject;
import org.json.JSONTokener;
import org.jsoup.nodes.Document;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.RipUtils;
import com.rarchives.ripme.utils.Utils;
@ -113,27 +113,25 @@ public class RedditRipper extends AlbumRipper {
lastRequestTime = System.currentTimeMillis();
int attempts = 0;
Document doc = null;
logger.info(" Retrieving " + url);
while(doc == null && attempts++ < 3) {
JSONObject json = null;
while(json == null && attempts++ < 3) {
try {
doc = getResponse(url, true).parse();
json = Http.url(url).getJSON();
} catch(SocketTimeoutException ex) {
if(attempts >= 3) throw ex;
logger.warn(String.format("[!] Connection timed out (attempt %d)", attempts));
}
}
String jsonString = doc.body().html().replaceAll("&quot;", "\"");
Object jsonObj = new JSONTokener(jsonString).nextValue();
Object jsonObj = new JSONTokener(json.toString()).nextValue();
JSONArray jsonArray = new JSONArray();
if (jsonObj instanceof JSONObject) {
jsonArray.put( (JSONObject) jsonObj);
} else if (jsonObj instanceof JSONArray){
jsonArray = (JSONArray) jsonObj;
} else {
logger.warn("[!] Unable to parse child: " + jsonString);
logger.warn("[!] Unable to parse child: " + json.toString());
}
return jsonArray;
}

13
src/main/java/com/rarchives/ripme/ripper/rippers/SeeniveRipper.java
View File

@ -7,7 +7,6 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.json.JSONObject;
import org.jsoup.Connection.Method;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
@ -15,6 +14,7 @@ import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ripper.DownloadThreadPool;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
public class SeeniveRipper extends AlbumRipper {
@ -42,7 +42,9 @@ public class SeeniveRipper extends AlbumRipper {
public void rip() throws IOException {
String baseURL = this.url.toExternalForm();
logger.info(" Retrieving " + baseURL);
Document doc = getDocument(baseURL, baseURL, null);
Document doc = Http.url(baseURL)
.referrer(baseURL)
.get();
while (true) {
if (isStopped()) {
break;
@ -70,8 +72,9 @@ public class SeeniveRipper extends AlbumRipper {
}
logger.info("[ ] Retrieving " + baseURL + "/next/" + lastID);
String jsonString = getResponse(baseURL + "/next/" + lastID, Method.GET, USER_AGENT, baseURL, null, true).body();
JSONObject json = new JSONObject(jsonString);
JSONObject json = Http.url(baseURL + "/next/" + lastID)
.referrer(baseURL)
.getJSON();
String html = json.getString("Html");
if (html.equals("")) {
break;
@ -111,7 +114,7 @@ public class SeeniveRipper extends AlbumRipper {
@Override
public void run() {
try {
Document doc = getDocument(this.url);
Document doc = Http.url(this.url).get();
logger.info("[ ] Retreiving video page " + this.url);
sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm());
for (Element element : doc.select("source")) {

5
src/main/java/com/rarchives/ripme/ripper/rippers/SmuttyRipper.java
View File

@ -11,6 +11,7 @@ import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
public class SmuttyRipper extends AlbumRipper {
@ -46,7 +47,9 @@ public class SmuttyRipper extends AlbumRipper {
logger.info(" Retrieving " + url);
Document doc;
try {
doc = getResponse(url, true).parse();
doc = Http.url(url)
.ignoreContentType()
.get();
} catch (IOException e) {
if (e.toString().contains("Status=404")) {
logger.info("No more pages to load");

3
src/main/java/com/rarchives/ripme/ripper/rippers/SupertangasRipper.java
View File

@ -13,6 +13,7 @@ import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
public class SupertangasRipper extends AlbumRipper {
@ -47,7 +48,7 @@ public class SupertangasRipper extends AlbumRipper {
try {
logger.info(" Retrieving " + theURL);
sendUpdate(STATUS.LOADING_RESOURCE, theURL);
doc = getDocument(theURL);
doc = Http.url(theURL).get();
} catch (HttpStatusException e) {
logger.debug("Hit end of pages at page " + page, e);
break;

5
src/main/java/com/rarchives/ripme/ripper/rippers/TeenplanetRipper.java
View File

@ -12,6 +12,7 @@ import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class TeenplanetRipper extends AlbumRipper {
@ -38,7 +39,7 @@ public class TeenplanetRipper extends AlbumRipper {
try {
// Attempt to use album title as GID
if (albumDoc == null) {
albumDoc = getDocument(url);
albumDoc = Http.url(url).get();
}
Elements elems = albumDoc.select("div.header > h2");
return HOST + "_" + elems.get(0).text();
@ -71,7 +72,7 @@ public class TeenplanetRipper extends AlbumRipper {
logger.info("Retrieving " + this.url);
sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm());
if (albumDoc == null) {
albumDoc = getDocument(url);
albumDoc = Http.url(url).get();
}
for (Element thumb : albumDoc.select("#galleryImages > a > img")) {
if (!thumb.hasAttr("src")) {

19
src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java
View File

@ -9,10 +9,10 @@ import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.nodes.Document;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class TumblrRipper extends AlbumRipper {
@ -66,9 +66,8 @@ public class TumblrRipper extends AlbumRipper {
checkURL += url.getHost();
checkURL += "/info?api_key=" + API_KEY;
try {
Document doc = getResponse(checkURL, true).parse();
String jsonString = doc.body().html().replaceAll("&quot;", "\"");
JSONObject json = new JSONObject(jsonString);
JSONObject json = Http.url(checkURL)
.getJSON();
int status = json.getJSONObject("meta").getInt("status");
return status == 200;
} catch (IOException e) {
@ -98,15 +97,14 @@ public class TumblrRipper extends AlbumRipper {
String apiURL = getTumblrApiURL(mediaType, offset);
logger.info("Retrieving " + apiURL);
sendUpdate(STATUS.LOADING_RESOURCE, apiURL);
Document doc = getResponse(apiURL, true).parse();
JSONObject json = Http.url(apiURL).getJSON();
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
logger.error("[!] Interrupted while waiting to load next album:", e);
break;
}
String jsonString = doc.body().html().replaceAll("&quot;", "\"");
if (!handleJSON(jsonString)) {
if (!handleJSON(json)) {
// Returns false if an error occurs and we should stop.
break;
}
@ -119,12 +117,7 @@ public class TumblrRipper extends AlbumRipper {
waitForThreads();
}
private boolean handleJSON(String jsonString) {
JSONObject json = new JSONObject(jsonString);
if (json == null || !json.has("response")) {
logger.error("[!] JSON response from tumblr was invalid: " + jsonString);
return false;
}
private boolean handleJSON(JSONObject json) {
JSONArray posts, photos;
JSONObject post, photo;
URL fileURL;

14
src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
View File

@ -12,10 +12,10 @@ import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONTokener;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class TwitterRipper extends AlbumRipper {
@ -71,8 +71,8 @@ public class TwitterRipper extends AlbumRipper {
}
private void getAccessToken() throws IOException {
Document doc = Jsoup.connect("https://api.twitter.com/oauth2/token")
.ignoreContentType(true)
Document doc = Http.url("https://api.twitter.com/oauth2/token")
.ignoreContentType()
.header("Authorization", "Basic " + authKey)
.header("Content-Type", "application/x-www-form-urlencoded;charset=UTF-8")
.header("User-agent", "ripe and zipe")
@ -90,8 +90,8 @@ public class TwitterRipper extends AlbumRipper {
}
private void checkRateLimits(String resource, String api) throws IOException {
Document doc = Jsoup.connect("https://api.twitter.com/1.1/application/rate_limit_status.json?resources=" + resource)
.ignoreContentType(true)
Document doc = Http.url("https://api.twitter.com/1.1/application/rate_limit_status.json?resources=" + resource)
.ignoreContentType()
.header("Authorization", "Bearer " + accessToken)
.header("Content-Type", "application/x-www-form-urlencoded;charset=UTF-8")
.header("User-agent", "ripe and zipe")
@ -143,8 +143,8 @@ public class TwitterRipper extends AlbumRipper {
private List<JSONObject> getTweets(String url) throws IOException {
List<JSONObject> tweets = new ArrayList<JSONObject>();
logger.info(" Retrieving " + url);
Document doc = Jsoup.connect(url)
.ignoreContentType(true)
Document doc = Http.url(url)
.ignoreContentType()
.header("Authorization", "Bearer " + accessToken)
.header("Content-Type", "application/x-www-form-urlencoded;charset=UTF-8")
.header("User-agent", "ripe and zipe")

3
src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java
View File

@ -11,6 +11,7 @@ import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class VidbleRipper extends AlbumRipper {
@ -49,7 +50,7 @@ public class VidbleRipper extends AlbumRipper {
logger.info("Retrieving " + this.url);
sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm());
if (albumDoc == null) {
albumDoc = getDocument(this.url);
albumDoc = Http.url(this.url).get();
}
Elements els = albumDoc.select("#ContentPlaceHolder1_thumbs");
if (els.size() == 0) {

9
src/main/java/com/rarchives/ripme/ripper/rippers/VineRipper.java
View File

@ -9,10 +9,10 @@ import java.util.regex.Pattern;
import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.HttpStatusException;
import org.jsoup.nodes.Document;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
public class VineRipper extends AlbumRipper {
@ -37,7 +37,7 @@ public class VineRipper extends AlbumRipper {
public void rip() throws IOException {
int page = 0;
String baseURL = "https://vine.co/api/timelines/users/" + getGID(this.url);
Document doc;
JSONObject json = null;
while (true) {
page++;
String theURL = baseURL;
@ -47,14 +47,11 @@ public class VineRipper extends AlbumRipper {
try {
logger.info(" Retrieving " + theURL);
sendUpdate(STATUS.LOADING_RESOURCE, theURL);
doc = getResponse(theURL, true).parse();
json = Http.url(theURL).getJSON();
} catch (HttpStatusException e) {
logger.debug("Hit end of pages at page " + page, e);
break;
}
String jsonString = doc.body().html();
jsonString = jsonString.replace("&quot;", "\"");
JSONObject json = new JSONObject(jsonString);
JSONArray records = json.getJSONObject("data").getJSONArray("records");
for (int i = 0; i < records.length(); i++) {
String videoURL = records.getJSONObject(i).getString("videoUrl");

3
src/main/java/com/rarchives/ripme/ripper/rippers/VineboxRipper.java
View File

@ -12,6 +12,7 @@ import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
public class VineboxRipper extends AlbumRipper {
@ -42,7 +43,7 @@ public class VineboxRipper extends AlbumRipper {
logger.info("Retrieving " + urlPaged);
sendUpdate(STATUS.LOADING_RESOURCE, urlPaged);
try {
doc = getDocument(this.url);;
doc = Http.url(this.url).get();
} catch (HttpStatusException e) {
logger.debug("Hit end of pages at page " + page, e);
break;

17
src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java
View File

@ -18,6 +18,7 @@ import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class VkRipper extends AlbumRipper {
@ -66,11 +67,9 @@ public class VkRipper extends AlbumRipper {
postData.put("act", "load_videos_silent");
postData.put("offset", "0");
postData.put("oid", oid);
Document doc = Jsoup.connect(u)
.header("Referer", this.url.toExternalForm())
.ignoreContentType(true)
.userAgent(USER_AGENT)
.timeout(5000)
Document doc = Http.url(u)
.referrer(this.url)
.ignoreContentType()
.data(postData)
.post();
String[] jsonStrings = doc.toString().split("<!>");
@ -108,11 +107,9 @@ public class VkRipper extends AlbumRipper {
postData.put("al", "1");
postData.put("offset", Integer.toString(offset));
postData.put("part", "1");
Document doc = Jsoup.connect(this.url.toExternalForm())
.header("Referer", this.url.toExternalForm())
.ignoreContentType(true)
.userAgent(USER_AGENT)
.timeout(5000)
Document doc = Http.url(this.url)
.referrer(this.url)
.ignoreContentType()
.data(postData)
.post();

3
src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java
View File

@ -10,6 +10,7 @@ import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class XhamsterRipper extends AlbumRipper {
@ -37,7 +38,7 @@ public class XhamsterRipper extends AlbumRipper {
String nextURL = this.url.toExternalForm();
while (nextURL != null) {
logger.info(" Retrieving " + nextURL);
Document doc = getDocument(nextURL);
Document doc = Http.url(nextURL).get();
for (Element thumb : doc.select("table.iListing div.img img")) {
if (!thumb.hasAttr("src")) {
continue;

3
src/main/java/com/rarchives/ripme/ripper/rippers/video/BeegRipper.java
View File

@ -11,6 +11,7 @@ import org.json.JSONObject;
import org.jsoup.nodes.Document;
import com.rarchives.ripme.ripper.VideoRipper;
import com.rarchives.ripme.utils.Http;
public class BeegRipper extends VideoRipper {
@ -54,7 +55,7 @@ public class BeegRipper extends VideoRipper {
@Override
public void rip() throws IOException {
logger.info(" Retrieving " + this.url);
Document doc = getDocument(this.url);
Document doc = Http.url(this.url).get();
Pattern p = Pattern.compile("^.*var qualityArr = (.*});.*$", Pattern.DOTALL);
Matcher m = p.matcher(doc.html());
if (m.matches()) {

3
src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java
View File

@ -10,6 +10,7 @@ import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.VideoRipper;
import com.rarchives.ripme.utils.Http;
public class GfycatRipper extends VideoRipper {
@ -51,7 +52,7 @@ public class GfycatRipper extends VideoRipper {
@Override
public void rip() throws IOException {
logger.info(" Retrieving " + this.url.toExternalForm());
Document doc = getDocument(this.url);
Document doc = Http.url(this.url).get();
Elements videos = doc.select("source#mp4source");
if (videos.size() == 0) {
throw new IOException("Could not find source#mp4source at " + url);

4
src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java
View File

@ -9,11 +9,11 @@ import java.util.regex.Pattern;
import org.json.JSONException;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import com.rarchives.ripme.ripper.VideoRipper;
import com.rarchives.ripme.utils.AES;
import com.rarchives.ripme.utils.Http;
public class PornhubRipper extends VideoRipper {
@ -57,7 +57,7 @@ public class PornhubRipper extends VideoRipper {
@Override
public void rip() throws IOException {
logger.info(" Retrieving " + this.url.toExternalForm());
Document doc = getDocument(this.url);
Document doc = Http.url(this.url).get();
Pattern p = Pattern.compile("^.*var flashvars = (.*});.*$", Pattern.DOTALL);
Matcher m = p.matcher(doc.body().html());
if (m.matches()) {

3
src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java
View File

@ -10,6 +10,7 @@ import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.VideoRipper;
import com.rarchives.ripme.utils.Http;
public class ViddmeRipper extends VideoRipper {
@ -53,7 +54,7 @@ public class ViddmeRipper extends VideoRipper {
@Override
public void rip() throws IOException {
logger.info(" Retrieving " + this.url.toExternalForm());
Document doc = getDocument(this.url);
Document doc = Http.url(this.url).get();
Elements videos = doc.select("meta[name=twitter:player:stream]");
if (videos.size() == 0) {
throw new IOException("Could not find twitter:player:stream at " + url);

3
src/main/java/com/rarchives/ripme/ripper/rippers/video/VineRipper.java
View File

@ -10,6 +10,7 @@ import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.VideoRipper;
import com.rarchives.ripme.utils.Http;
public class VineRipper extends VideoRipper {
@ -54,7 +55,7 @@ public class VineRipper extends VideoRipper {
@Override
public void rip() throws IOException {
logger.info(" Retrieving " + this.url.toExternalForm());
Document doc = getDocument(this.url);
Document doc = Http.url(this.url).get();
Elements props = doc.select("meta[property=twitter:player:stream]");
if (props.size() == 0) {
throw new IOException("Could not find meta property 'twitter:player:stream' at " + url);

4
src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java
View File

@ -6,10 +6,10 @@ import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import com.rarchives.ripme.ripper.VideoRipper;
import com.rarchives.ripme.utils.Http;
public class VkRipper extends VideoRipper {
@ -59,7 +59,7 @@ public class VkRipper extends VideoRipper {
}
public static String getVideoURLAtPage(String url) throws IOException {
Document doc = Jsoup.connect(url)
Document doc = Http.url(url)
.userAgent(USER_AGENT)
.get();
String html = doc.outerHtml();

3
src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java
View File

@ -12,6 +12,7 @@ import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.VideoRipper;
import com.rarchives.ripme.utils.Http;
public class XvideosRipper extends VideoRipper {
@ -55,7 +56,7 @@ public class XvideosRipper extends VideoRipper {
@Override
public void rip() throws IOException {
logger.info(" Retrieving " + this.url);
Document doc = getDocument(this.url);
Document doc = Http.url(this.url).get();
Elements embeds = doc.select("embed");
if (embeds.size() == 0) {
throw new IOException("Could not find Embed code at " + url);

4
src/main/java/com/rarchives/ripme/ripper/rippers/video/YoupornRipper.java
View File

@ -6,12 +6,12 @@ import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.VideoRipper;
import com.rarchives.ripme.utils.Http;
public class YoupornRipper extends VideoRipper {
@ -55,7 +55,7 @@ public class YoupornRipper extends VideoRipper {
@Override
public void rip() throws IOException {
logger.info(" Retrieving " + this.url);
Document doc = getDocument(this.url);
Document doc = Http.url(this.url).get();
Elements videos = doc.select("video");
if (videos.size() == 0) {
throw new IOException("Could not find Embed code at " + url);

136
src/main/java/com/rarchives/ripme/utils/Http.java Normal file
View File

@ -0,0 +1,136 @@
package com.rarchives.ripme.utils;
import java.io.IOException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import org.apache.log4j.Logger;
import org.json.JSONObject;
import org.jsoup.Connection;
import org.jsoup.Connection.Method;
import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import com.rarchives.ripme.ripper.AbstractRipper;
/**
* Wrapper around the Jsoup connection methods.
*
* Benefit is retry logic.
*/
public class Http {
public static final int TIMEOUT = Utils.getConfigInteger("page.timeout", 5 * 1000);
private static final Logger logger = Logger.getLogger(AbstractRipper.class);
private int retries;
private String url;
private Connection connection;
// Constructors
public Http(String url) {
this.url = url;
defaultSettings();
}
public Http(URL url) {
this.url = url.toExternalForm();
defaultSettings();
}
public static Http url(String url) {
return new Http(url);
}
public static Http url(URL url) {
return new Http(url);
}
private void defaultSettings() {
this.retries = Utils.getConfigInteger("download.retries", 1);
connection = Jsoup.connect(this.url);
connection.userAgent(AbstractRipper.USER_AGENT);
connection.method(Method.GET);
connection.timeout(TIMEOUT);
connection.maxBodySize(0);
}
// Setters
public Http timeout(int timeout) {
connection.timeout(timeout);
return this;
}
public Http ignoreContentType() {
connection.ignoreContentType(true);
return this;
}
public Http referrer(String ref) {
connection.referrer(ref);
return this;
}
public Http referrer(URL ref) {
return referrer(ref.toExternalForm());
}
public Http userAgent(String ua) {
connection.userAgent(ua);
return this;
}
public Http retries(int tries) {
this.retries = tries;
return this;
}
public Http header(String name, String value) {
connection.header(name, value);
return this;
}
public Http cookies(Map<String,String> cookies) {
connection.cookies(cookies);
return this;
}
public Http data(Map<String,String> data) {
connection.data(data);
return this;
}
public Http data(String name, String value) {
Map<String,String> data = new HashMap<String,String>();
data.put(name, value);
return data(data);
}
public Http method(Method method) {
connection.method(method);
return this;
}
// Getters
public Document get() throws IOException {
connection.method(Method.GET);
return response().parse();
}
public Document post() throws IOException {
connection.method(Method.POST);
return response().parse();
}
public JSONObject getJSON() throws IOException {
ignoreContentType();
String jsonString = response().body().replace("&quot;", "\"");
return new JSONObject(jsonString);
}
public Response response() throws IOException {
Response response = null;
int retries = this.retries;
while (--retries >= 0) {
try {
response = connection.execute();
return response;
} catch (IOException e) {
logger.warn("Error while loading " + url, e);
continue;
}
}
throw new IOException("Failed to load " + url + " after " + this.retries + " attempts");
}
}

7
src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java
View File

@ -9,13 +9,6 @@ import com.rarchives.ripme.ripper.rippers.DeviantartRipper;
public class DeviantartRipperTest extends RippersTest {
public void testAlphaSorting() {
String[] strings = new String[]{"a", "aa", "aaa", "d6hg2dz", "d6fspba", "d6fcvvr"};
for (String string : strings) {
System.err.println(string + ": " + DeviantartRipper.alphaToLong(string));
}
}
public void testDeviantartAlbums() throws IOException {
if (!DOWNLOAD_CONTENT) {
return;