Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Allowing rippers to pass referrer and cookies to download threads

Browse Source
This commit is contained in:
4pr0n 2014-04-22 20:48:41 -07:00
parent b0d13d51b1
commit a8728ece9f
4 changed files with 74 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
View File

@ -7,6 +7,7 @@ import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Observable;
import org.apache.log4j.Logger;
@ -72,30 +73,6 @@ public abstract class AbstractRipper
public void setObserver(RipStatusHandler obs) {
this.observer = obs;
}
/**
* Queues image to be downloaded and saved.
* Uses filename from URL to decide filename.
* @param url
* URL to download
*/
public void addURLToDownload(URL url) {
// Use empty prefix and empty subdirectory
addURLToDownload(url, "", "");
}
/**
* Queues image to be downloaded and saved.
* Uses filename from URL (and 'prefix') to decide filename.
* @param url
* URL to download
* @param prefix
* Text to append to saved filename.
*/
public void addURLToDownload(URL url, String prefix) {
// Use empty subdirectory
addURLToDownload(url, prefix, "");
}
/**
* Queues image to be downloaded and saved.
@ -105,17 +82,9 @@ public abstract class AbstractRipper
* Path of the local file to save the content to.
*/
public abstract void addURLToDownload(URL url, File saveAs);
public abstract void addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies);
/**
* Queues file to be downloaded and saved. With options.
* @param url
* URL to download.
* @param prefix
* Prefix to prepend to the saved filename.
* @param subdirectory
* Sub-directory of the working directory to save the images to.
*/
public void addURLToDownload(URL url, String prefix, String subdirectory) {
public void addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map<String,String> cookies) {
try {
stopCheck();
} catch (IOException e) {
@ -147,9 +116,34 @@ public abstract class AbstractRipper
logger.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent()));
saveFileAs.getParentFile().mkdirs();
}
addURLToDownload(url, saveFileAs);
addURLToDownload(url, saveFileAs, referrer, cookies);
}
/**
* Queues file to be downloaded and saved. With options.
* @param url
* URL to download.
* @param prefix
* Prefix to prepend to the saved filename.
* @param subdirectory
* Sub-directory of the working directory to save the images to.
*/
public void addURLToDownload(URL url, String prefix, String subdirectory) {
addURLToDownload(url, prefix, subdirectory, null, null);
}
/**
* Queues image to be downloaded and saved.
* Uses filename from URL (and 'prefix') to decide filename.
* @param url
* URL to download
* @param prefix
* Text to append to saved filename.
*/
public void addURLToDownload(URL url, String prefix) {
// Use empty subdirectory
addURLToDownload(url, prefix, "");
}
/**
* Waits for downloading threads to complete.
*/

28
src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java
View File

@ -28,8 +28,7 @@ public abstract class AlbumRipper extends AbstractRipper {
public abstract String getHost();
public abstract String getGID(URL url) throws MalformedURLException;
@Override
public void addURLToDownload(URL url, File saveAs) {
public void addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies) {
if (itemsPending.containsKey(url)
|| itemsCompleted.containsKey(url)
|| itemsErrored.containsKey(url)) {
@ -38,7 +37,30 @@ public abstract class AlbumRipper extends AbstractRipper {
return;
}
itemsPending.put(url, saveAs);
threadPool.addThread(new DownloadFileThread(url, saveAs, this));
DownloadFileThread dft = new DownloadFileThread(url, saveAs, this);
if (referrer != null) {
dft.setReferrer(referrer);
}
if (cookies != null) {
dft.setCookies(cookies);
}
threadPool.addThread(dft);
}
@Override
public void addURLToDownload(URL url, File saveAs) {
addURLToDownload(url, saveAs, null, null);
}
/**
* Queues image to be downloaded and saved.
* Uses filename from URL to decide filename.
* @param url
* URL to download
*/
public void addURLToDownload(URL url) {
// Use empty prefix and empty subdirectory
addURLToDownload(url, "", "");
}
@Override

14
src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
View File

@ -4,6 +4,8 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import org.apache.log4j.Logger;
import org.jsoup.Connection.Response;
@ -20,6 +22,9 @@ public class DownloadFileThread extends Thread {
private static final Logger logger = Logger.getLogger(DownloadFileThread.class);
private String referrer = "";
private Map<String,String> cookies = new HashMap<String,String>();
private URL url;
private File saveAs;
private String prettySaveAs;
@ -40,6 +45,13 @@ public class DownloadFileThread extends Thread {
this.MAX_BODY_SIZE = Utils.getConfigInteger("download.max_bytes", 1024 * 1024 * 100);
}
public void setReferrer(String referrer) {
this.referrer = referrer;
}
public void setCookies(Map<String,String> cookies) {
this.cookies = cookies;
}
/**
* Attempts to download the file. Retries as needed.
* Notifies observers upon completion/error/warn.
@ -74,6 +86,8 @@ public class DownloadFileThread extends Thread {
.userAgent(AbstractRipper.USER_AGENT)
.timeout(TIMEOUT)
.maxBodySize(MAX_BODY_SIZE)
.cookies(cookies)
.referrer(referrer)
.execute();
if (response.statusCode() != 200) {
logger.error("[!] Non-OK status code " + response.statusCode() + " while downloading from " + url);

6
src/main/java/com/rarchives/ripme/ripper/VideoRipper.java
View File

@ -4,6 +4,7 @@ import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Map;
import com.rarchives.ripme.ui.RipStatusMessage;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
@ -42,6 +43,11 @@ public abstract class VideoRipper extends AbstractRipper {
threadPool.addThread(new DownloadVideoThread(url, saveAs, this));
}
@Override
public void addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies) {
addURLToDownload(url, saveAs);
}
@Override
public void setWorkingDir(URL url) throws IOException {
String path = Utils.getWorkingDirectory().getCanonicalPath();