From a8728ece9ff9aad0fa1b9a8b1d6b08237aede21d Mon Sep 17 00:00:00 2001 From: 4pr0n Date: Tue, 22 Apr 2014 20:48:41 -0700 Subject: [PATCH] Allowing rippers to pass referrer and cookies to download threads --- .../ripme/ripper/AbstractRipper.java | 64 +++++++++---------- .../rarchives/ripme/ripper/AlbumRipper.java | 28 +++++++- .../ripme/ripper/DownloadFileThread.java | 14 ++++ .../rarchives/ripme/ripper/VideoRipper.java | 6 ++ 4 files changed, 74 insertions(+), 38 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index cff22079..1d617ef2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -7,6 +7,7 @@ import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Observable; import org.apache.log4j.Logger; @@ -72,30 +73,6 @@ public abstract class AbstractRipper public void setObserver(RipStatusHandler obs) { this.observer = obs; } - - /** - * Queues image to be downloaded and saved. - * Uses filename from URL to decide filename. - * @param url - * URL to download - */ - public void addURLToDownload(URL url) { - // Use empty prefix and empty subdirectory - addURLToDownload(url, "", ""); - } - - /** - * Queues image to be downloaded and saved. - * Uses filename from URL (and 'prefix') to decide filename. - * @param url - * URL to download - * @param prefix - * Text to append to saved filename. - */ - public void addURLToDownload(URL url, String prefix) { - // Use empty subdirectory - addURLToDownload(url, prefix, ""); - } /** * Queues image to be downloaded and saved. @@ -105,17 +82,9 @@ public abstract class AbstractRipper * Path of the local file to save the content to. */ public abstract void addURLToDownload(URL url, File saveAs); + public abstract void addURLToDownload(URL url, File saveAs, String referrer, Map cookies); - /** - * Queues file to be downloaded and saved. With options. - * @param url - * URL to download. - * @param prefix - * Prefix to prepend to the saved filename. - * @param subdirectory - * Sub-directory of the working directory to save the images to. - */ - public void addURLToDownload(URL url, String prefix, String subdirectory) { + public void addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies) { try { stopCheck(); } catch (IOException e) { @@ -147,9 +116,34 @@ public abstract class AbstractRipper logger.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent())); saveFileAs.getParentFile().mkdirs(); } - addURLToDownload(url, saveFileAs); + addURLToDownload(url, saveFileAs, referrer, cookies); } + /** + * Queues file to be downloaded and saved. With options. + * @param url + * URL to download. + * @param prefix + * Prefix to prepend to the saved filename. + * @param subdirectory + * Sub-directory of the working directory to save the images to. + */ + public void addURLToDownload(URL url, String prefix, String subdirectory) { + addURLToDownload(url, prefix, subdirectory, null, null); + } + + /** + * Queues image to be downloaded and saved. + * Uses filename from URL (and 'prefix') to decide filename. + * @param url + * URL to download + * @param prefix + * Text to append to saved filename. + */ + public void addURLToDownload(URL url, String prefix) { + // Use empty subdirectory + addURLToDownload(url, prefix, ""); + } /** * Waits for downloading threads to complete. */ diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index ca456737..8161d7c0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -28,8 +28,7 @@ public abstract class AlbumRipper extends AbstractRipper { public abstract String getHost(); public abstract String getGID(URL url) throws MalformedURLException; - @Override - public void addURLToDownload(URL url, File saveAs) { + public void addURLToDownload(URL url, File saveAs, String referrer, Map cookies) { if (itemsPending.containsKey(url) || itemsCompleted.containsKey(url) || itemsErrored.containsKey(url)) { @@ -38,7 +37,30 @@ public abstract class AlbumRipper extends AbstractRipper { return; } itemsPending.put(url, saveAs); - threadPool.addThread(new DownloadFileThread(url, saveAs, this)); + DownloadFileThread dft = new DownloadFileThread(url, saveAs, this); + if (referrer != null) { + dft.setReferrer(referrer); + } + if (cookies != null) { + dft.setCookies(cookies); + } + threadPool.addThread(dft); + } + + @Override + public void addURLToDownload(URL url, File saveAs) { + addURLToDownload(url, saveAs, null, null); + } + + /** + * Queues image to be downloaded and saved. + * Uses filename from URL to decide filename. + * @param url + * URL to download + */ + public void addURLToDownload(URL url) { + // Use empty prefix and empty subdirectory + addURLToDownload(url, "", ""); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 8fdc2133..18a8c14e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -4,6 +4,8 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.net.URL; +import java.util.HashMap; +import java.util.Map; import org.apache.log4j.Logger; import org.jsoup.Connection.Response; @@ -20,6 +22,9 @@ public class DownloadFileThread extends Thread { private static final Logger logger = Logger.getLogger(DownloadFileThread.class); + private String referrer = ""; + private Map cookies = new HashMap(); + private URL url; private File saveAs; private String prettySaveAs; @@ -40,6 +45,13 @@ public class DownloadFileThread extends Thread { this.MAX_BODY_SIZE = Utils.getConfigInteger("download.max_bytes", 1024 * 1024 * 100); } + public void setReferrer(String referrer) { + this.referrer = referrer; + } + public void setCookies(Map cookies) { + this.cookies = cookies; + } + /** * Attempts to download the file. Retries as needed. * Notifies observers upon completion/error/warn. @@ -74,6 +86,8 @@ public class DownloadFileThread extends Thread { .userAgent(AbstractRipper.USER_AGENT) .timeout(TIMEOUT) .maxBodySize(MAX_BODY_SIZE) + .cookies(cookies) + .referrer(referrer) .execute(); if (response.statusCode() != 200) { logger.error("[!] Non-OK status code " + response.statusCode() + " while downloading from " + url); diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 3b4e06a8..b80339c4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -4,6 +4,7 @@ import java.io.File; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.util.Map; import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; @@ -42,6 +43,11 @@ public abstract class VideoRipper extends AbstractRipper { threadPool.addThread(new DownloadVideoThread(url, saveAs, this)); } + @Override + public void addURLToDownload(URL url, File saveAs, String referrer, Map cookies) { + addURLToDownload(url, saveAs); + } + @Override public void setWorkingDir(URL url) throws IOException { String path = Utils.getWorkingDirectory().getCanonicalPath();