From 66739e4c7cba2b049dd01b554947ac6c705a8fca Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 25 Feb 2018 18:10:30 -0500 Subject: [PATCH 1/3] Implemented a last seen feature; Added comments --- src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index edddea78..edcff83d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -44,7 +44,8 @@ public abstract class AbstractRipper public abstract String getHost(); public abstract String getGID(URL url) throws MalformedURLException; public boolean hasASAPRipping() { return false; } - + // Everytime addUrlToDownload skips a already downloaded url this increases by 1 + public int alreadyDownloadedUrls = 0; private boolean shouldStop = false; private boolean thisIsATest = false; @@ -194,9 +195,11 @@ public abstract class AbstractRipper * False if failed to download */ protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies) { + // Don't re-add the url if it was downloaded in a previous rip if (Utils.getConfigBoolean("remember.url_history", true) && !isThisATest()) { if (hasDownloadedURL(url.toExternalForm())) { sendUpdate(STATUS.DOWNLOAD_WARN, "Already downloaded " + url.toExternalForm()); + alreadyDownloadedUrls += 1; return false; } } From 8c86377a7faccbb941b82802e295a983b925251d Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 25 Feb 2018 18:10:38 -0500 Subject: [PATCH 2/3] Implemented a last seen feature; Added comments --- .../com/rarchives/ripme/ripper/AbstractHTMLRipper.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index e71eb389..d49c3292 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -68,7 +68,14 @@ public abstract class AbstractHTMLRipper extends AlbumRipper { Document doc = getFirstPage(); while (doc != null) { + logger.debug("alreadyDownloadedUrls is " + alreadyDownloadedUrls); + if (alreadyDownloadedUrls >= Utils.getConfigInteger("skip_after_already_seen", -1) && !isThisATest()) { + sendUpdate(STATUS.DOWNLOAD_COMPLETE, "Already seen the last " + alreadyDownloadedUrls + " images ending rip"); + break; + } List imageURLs = getURLsFromPage(doc); + // If hasASAPRipping() returns true then the ripper will handle downloading the files + // if not it's done in the following block of code if (!hasASAPRipping()) { // Remove all but 1 image if (isThisATest()) { From c0a0b9dce809c40efda7aa5f0f3c5cf559a68c1b Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 25 Feb 2018 19:51:28 -0500 Subject: [PATCH 3/3] Removed debug statement; Changed option name --- .../java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index d49c3292..49a0fc7e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -68,8 +68,7 @@ public abstract class AbstractHTMLRipper extends AlbumRipper { Document doc = getFirstPage(); while (doc != null) { - logger.debug("alreadyDownloadedUrls is " + alreadyDownloadedUrls); - if (alreadyDownloadedUrls >= Utils.getConfigInteger("skip_after_already_seen", -1) && !isThisATest()) { + if (alreadyDownloadedUrls >= Utils.getConfigInteger("history.end_rip_after_already_seen", -1) && !isThisATest()) { sendUpdate(STATUS.DOWNLOAD_COMPLETE, "Already seen the last " + alreadyDownloadedUrls + " images ending rip"); break; }