Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added timeouts and increased maximum file download sizes

Browse Source
This commit is contained in:
4pr0n 2014-03-12 19:29:11 -07:00
parent 1cdec765cc
commit 9576896117
2 changed files with 23 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
View File

@ -26,6 +26,9 @@ public class DownloadFileThread extends Thread {
private AbstractRipper observer; private AbstractRipper observer;
private int retries; private int retries;
private final int TIMEOUT;
private final int MAX_BODY_SIZE;
public DownloadFileThread(URL url, File saveAs, AbstractRipper observer) { public DownloadFileThread(URL url, File saveAs, AbstractRipper observer) {
super(); super();
this.url = url; this.url = url;
@ -33,6 +36,8 @@ public class DownloadFileThread extends Thread {
this.prettySaveAs = Utils.removeCWD(saveAs); this.prettySaveAs = Utils.removeCWD(saveAs);
this.observer = observer; this.observer = observer;
this.retries = Utils.getConfigInteger("download.retries", 1); this.retries = Utils.getConfigInteger("download.retries", 1);
this.TIMEOUT = Utils.getConfigInteger("download.timeout", 60000);
this.MAX_BODY_SIZE = Utils.getConfigInteger("download.max_bytes", 1024 * 1024 * 100);
} }
/** /**
@ -60,13 +65,16 @@ public class DownloadFileThread extends Thread {
Response response; Response response;
response = Jsoup.connect(url.toExternalForm()) response = Jsoup.connect(url.toExternalForm())
.ignoreContentType(true) .ignoreContentType(true)
.userAgent(AbstractRipper.USER_AGENT)
.timeout(TIMEOUT)
.maxBodySize(MAX_BODY_SIZE)
.execute(); .execute();
FileOutputStream out = (new FileOutputStream(saveAs)); FileOutputStream out = (new FileOutputStream(saveAs));
out.write(response.bodyAsBytes()); out.write(response.bodyAsBytes());
out.close(); out.close();
break; // Download successful: break out of infinite loop break; // Download successful: break out of infinite loop
} catch (IOException e) { } catch (IOException e) {
logger.error("[!] Exception while downloading file: " + url + " - " + e.getMessage()); logger.error("[!] Exception while downloading file: " + url + " - " + e.getMessage(), e);
} }
if (tries > this.retries) { if (tries > this.retries) {
logger.error("[!] Exceeded maximum retries (" + this.retries + ") for URL " + url); logger.error("[!] Exceeded maximum retries (" + this.retries + ") for URL " + url);

15
src/main/resources/rip.properties
View File

@ -1,6 +1,19 @@
# Download threads to use per ripper
threads.size = 5 threads.size = 5
# Overwrite existing files
file.overwrite = false file.overwrite = false
download.retries = 3
# Number of retries on failed downloads
download.retries = 1
# File download timeout (in milliseconds)
download.timeout = 60000
# Maximum size of downloaded files in bytes (required)
download.max_size = 104857600
# API creds
twitter.auth = VW9Ybjdjb1pkd2J0U3kwTUh2VXVnOm9GTzVQVzNqM29LQU1xVGhnS3pFZzhKbGVqbXU0c2lHQ3JrUFNNZm8= twitter.auth = VW9Ybjdjb1pkd2J0U3kwTUh2VXVnOm9GTzVQVzNqM29LQU1xVGhnS3pFZzhKbGVqbXU0c2lHQ3JrUFNNZm8=
tumblr.auth = v5kUqGQXUtmF7K0itri1DGtgTs0VQpbSEbh1jxYgj9d2Sq18F8 tumblr.auth = v5kUqGQXUtmF7K0itri1DGtgTs0VQpbSEbh1jxYgj9d2Sq18F8
gw.api = gonewild gw.api = gonewild