From 96a8801c190f1b396373d642dad1278eab5c2101 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 26 Jun 2018 00:54:18 -0400 Subject: [PATCH 01/11] Rippers can now choose to use a byte progess bar --- .../ripme/ripper/AbstractRipper.java | 3 ++ .../ripme/ripper/DownloadFileThread.java | 43 ++++++++++++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 87d8bd46..cbc86548 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -613,4 +613,7 @@ public abstract class AbstractRipper protected boolean isThisATest() { return thisIsATest; } + + // If true ripme uses a byte progress bar + protected boolean useByteProgessBar() { return false;} } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 42dedffe..768e1c12 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -59,6 +59,16 @@ class DownloadFileThread extends Thread { this.cookies = cookies; } + private int getTotalBytes(URL url) throws IOException { + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("HEAD"); + conn.setRequestProperty("accept", "*/*"); + conn.setRequestProperty("Referer", this.url.toExternalForm()); // Sic + conn.setRequestProperty("User-agent", AbstractRipper.USER_AGENT); + return conn.getContentLength(); + } + + /** * Attempts to download the file. Retries as needed. * Notifies observers upon completion/error/warn. @@ -81,6 +91,21 @@ class DownloadFileThread extends Thread { } } + int bytesTotal, bytesDownloaded = 0; + if (observer.useByteProgessBar()) { + try { + bytesTotal = getTotalBytes(this.url); + } catch (IOException e) { + logger.error("Failed to get file size at " + this.url, e); + observer.downloadErrored(this.url, "Failed to get file size of " + this.url); + return; + } + + observer.setBytesTotal(bytesTotal); + observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal); + logger.debug("Size of file at " + this.url + " = " + bytesTotal + "b"); + } + URL urlToDownload = this.url; boolean redirected = false; int tries = 0; // Number of attempts to download @@ -156,7 +181,23 @@ class DownloadFileThread extends Thread { } fos = new FileOutputStream(saveAs); - IOUtils.copy(bis, fos); + byte[] data = new byte[1024 * 256]; int bytesRead; + while ( (bytesRead = bis.read(data)) != -1) { + try { + observer.stopCheck(); + } catch (IOException e) { + observer.downloadErrored(url, "Download interrupted"); + return; + } + fos.write(data, 0, bytesRead); + if (observer.useByteProgessBar()) { + bytesDownloaded += bytesRead; + observer.setBytesCompleted(bytesDownloaded); + observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded); + } + } + bis.close(); + fos.close(); break; // Download successful: break out of infinite loop } catch (HttpStatusException hse) { logger.debug("HTTP status exception", hse); From 9b70350b33684ccf87cf756a439619626e0f757e Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 26 Jun 2018 01:36:23 -0400 Subject: [PATCH 02/11] GfycatporntubeRipper now uses byte progess bar --- .../ripper/rippers/GfycatporntubeRipper.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java index 504b89d6..d8f629ae 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java @@ -8,6 +8,7 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.rarchives.ripme.utils.Utils; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -58,4 +59,34 @@ public class GfycatporntubeRipper extends AbstractHTMLRipper { public void downloadURL(URL url, int index) { addURLToDownload(url, getPrefix(index)); } + + private int bytesTotal = 1; + private int bytesCompleted = 1; + + @Override + public String getStatusText() { + return String.valueOf(getCompletionPercentage()) + + "% - " + + Utils.bytesToHumanReadable(bytesCompleted) + + " / " + + Utils.bytesToHumanReadable(bytesTotal); + } + + @Override + public int getCompletionPercentage() { + return (int) (100 * (bytesCompleted / (float) bytesTotal)); + } + + @Override + public void setBytesTotal(int bytes) { + this.bytesTotal = bytes; + } + + @Override + public void setBytesCompleted(int bytes) { + this.bytesCompleted = bytes; + } + + @Override + public boolean useByteProgessBar() {return true;} } From e1635f39e17476b1e02df150d0aac070b67b351e Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 26 Jun 2018 02:28:14 -0400 Subject: [PATCH 03/11] GfycatRipper now uses AbstractHTMLRipper --- .../rippers/{video => }/GfycatRipper.java | 75 ++++++++++++++++--- .../com/rarchives/ripme/utils/RipUtils.java | 2 +- 2 files changed, 66 insertions(+), 11 deletions(-) rename src/main/java/com/rarchives/ripme/ripper/rippers/{video => }/GfycatRipper.java (54%) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java similarity index 54% rename from src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java rename to src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java index 75577597..78a0af19 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java @@ -1,18 +1,23 @@ -package com.rarchives.ripme.ripper.rippers.video; +package com.rarchives.ripme.ripper.rippers; + import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.util.ArrayList; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Utils; import org.jsoup.nodes.Document; import org.jsoup.select.Elements; -import com.rarchives.ripme.ripper.VideoRipper; import com.rarchives.ripme.utils.Http; -public class GfycatRipper extends VideoRipper { + +public class GfycatRipper extends AbstractHTMLRipper { private static final String HOST = "gfycat.com"; @@ -20,9 +25,14 @@ public class GfycatRipper extends VideoRipper { super(url); } + @Override + public String getDomain() { + return "gfycat.com"; + } + @Override public String getHost() { - return HOST; + return "gfycat"; } @Override @@ -37,6 +47,16 @@ public class GfycatRipper extends VideoRipper { return url; } + @Override + public Document getFirstPage() throws IOException { + return Http.url(url).get(); + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } + @Override public String getGID(URL url) throws MalformedURLException { Pattern p = Pattern.compile("^https?://[wm.]*gfycat\\.com/([a-zA-Z0-9]+).*$"); @@ -52,10 +72,15 @@ public class GfycatRipper extends VideoRipper { } @Override - public void rip() throws IOException { - String vidUrl = getVideoURL(this.url); - addURLToDownload(new URL(vidUrl), "gfycat_" + getGID(this.url)); - waitForThreads(); + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + Elements videos = doc.select("source#mp4Source"); + String vidUrl = videos.first().attr("src"); + if (vidUrl.startsWith("//")) { + vidUrl = "http:" + vidUrl; + } + result.add(vidUrl); + return result; } /** @@ -66,10 +91,10 @@ public class GfycatRipper extends VideoRipper { */ public static String getVideoURL(URL url) throws IOException { LOGGER.info("Retrieving " + url.toExternalForm()); - + //Sanitize the URL first url = new URL(url.toExternalForm().replace("/gifs/detail", "")); - + Document doc = Http.url(url).get(); Elements videos = doc.select("source#mp4Source"); if (videos.isEmpty()) { @@ -81,4 +106,34 @@ public class GfycatRipper extends VideoRipper { } return vidUrl; } + + private int bytesTotal = 1; + private int bytesCompleted = 1; + + @Override + public String getStatusText() { + return String.valueOf(getCompletionPercentage()) + + "% - " + + Utils.bytesToHumanReadable(bytesCompleted) + + " / " + + Utils.bytesToHumanReadable(bytesTotal); + } + + @Override + public int getCompletionPercentage() { + return (int) (100 * (bytesCompleted / (float) bytesTotal)); + } + + @Override + public void setBytesTotal(int bytes) { + this.bytesTotal = bytes; + } + + @Override + public void setBytesCompleted(int bytes) { + this.bytesCompleted = bytes; + } + + @Override + public boolean useByteProgessBar() {return true;} } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index 34081852..9845145c 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -12,7 +12,7 @@ import com.rarchives.ripme.ripper.rippers.EroShareRipper; import com.rarchives.ripme.ripper.rippers.EromeRipper; import com.rarchives.ripme.ripper.rippers.ImgurRipper; import com.rarchives.ripme.ripper.rippers.VidbleRipper; -import com.rarchives.ripme.ripper.rippers.video.GfycatRipper; +import com.rarchives.ripme.ripper.rippers.GfycatRipper; import org.apache.commons.lang.math.NumberUtils; import org.apache.log4j.Logger; import org.jsoup.Jsoup; From 1428b2cd6b57c057493c4f000064310aaa2038f9 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 26 Jun 2018 02:49:18 -0400 Subject: [PATCH 04/11] Fixed some unit tests that broke when the GfycatRipper was moved --- .../rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java | 2 +- .../rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java index ca73f138..d3d011be 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java @@ -1,6 +1,6 @@ package com.rarchives.ripme.tst.ripper.rippers; -import com.rarchives.ripme.ripper.rippers.video.GfycatRipper; +import com.rarchives.ripme.ripper.rippers.GfycatRipper; import java.io.IOException; import java.net.URL; diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java index fdd61cf9..efe23a01 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java @@ -5,7 +5,6 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.RedditRipper; -import com.rarchives.ripme.ripper.rippers.video.GfycatRipper; public class RedditRipperTest extends RippersTest { // https://github.com/RipMeApp/ripme/issues/253 - Disabled tests: RedditRipperTest#testRedditSubreddit*Rip is flaky From b1718965c5e97ecb5e8df353138a5618a245f8df Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 26 Jun 2018 05:12:48 -0400 Subject: [PATCH 05/11] Removed ripper for dead site --- .../ripper/rippers/video/VineRipper.java | 67 ------------------- 1 file changed, 67 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/video/VineRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VineRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VineRipper.java deleted file mode 100644 index 1ca59676..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VineRipper.java +++ /dev/null @@ -1,67 +0,0 @@ -package com.rarchives.ripme.ripper.rippers.video; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.jsoup.nodes.Document; -import org.jsoup.select.Elements; - -import com.rarchives.ripme.ripper.VideoRipper; -import com.rarchives.ripme.utils.Http; - -public class VineRipper extends VideoRipper { - - private static final String HOST = "vine"; - - public VineRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return HOST; - } - - @Override - public boolean canRip(URL url) { - // https://vine.co/v/hiqQrP0eUZx - Pattern p = Pattern.compile("^https?://[wm.]*vine\\.co/v/[a-zA-Z0-9]+.*$"); - Matcher m = p.matcher(url.toExternalForm()); - return m.matches(); - } - - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return url; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://[wm.]*vine\\.co/v/([a-zA-Z0-9]+).*$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - - throw new MalformedURLException( - "Expected vine format:" - + "vine.co/v/####" - + " Got: " + url); - } - - @Override - public void rip() throws IOException { - LOGGER.info(" Retrieving " + this.url.toExternalForm()); - Document doc = Http.url(this.url).get(); - Elements props = doc.select("meta[property=twitter:player:stream]"); - if (props.isEmpty()) { - throw new IOException("Could not find meta property 'twitter:player:stream' at " + url); - } - String vidUrl = props.get(0).attr("content"); - addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url)); - waitForThreads(); - } -} \ No newline at end of file From 04ca5a2054a51495fbd4f3e3049626c99fd60f04 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 27 Jun 2018 00:51:28 -0400 Subject: [PATCH 06/11] Added support for resuming downloads --- .../ripme/ripper/AbstractRipper.java | 2 + .../ripme/ripper/DownloadFileThread.java | 59 ++++++++++--------- 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index cbc86548..94ac1ec9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -616,4 +616,6 @@ public abstract class AbstractRipper // If true ripme uses a byte progress bar protected boolean useByteProgessBar() { return false;} + // If true ripme will try to resume a broken download for this ripper + protected boolean tryResumeDownload() { return false;} } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 768e1c12..45c72c6e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -59,28 +59,25 @@ class DownloadFileThread extends Thread { this.cookies = cookies; } - private int getTotalBytes(URL url) throws IOException { - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn.setRequestMethod("HEAD"); - conn.setRequestProperty("accept", "*/*"); - conn.setRequestProperty("Referer", this.url.toExternalForm()); // Sic - conn.setRequestProperty("User-agent", AbstractRipper.USER_AGENT); - return conn.getContentLength(); - } - /** * Attempts to download the file. Retries as needed. * Notifies observers upon completion/error/warn. */ public void run() { + long fileSize = 0; + int bytesTotal = 0; + int bytesDownloaded = 0; + if (saveAs.exists() && observer.tryResumeDownload()) { + fileSize = saveAs.length(); + } try { observer.stopCheck(); } catch (IOException e) { observer.downloadErrored(url, "Download interrupted"); return; } - if (saveAs.exists()) { + if (saveAs.exists() && !observer.tryResumeDownload()) { if (Utils.getConfigBoolean("file.overwrite", false)) { logger.info("[!] Deleting existing file" + prettySaveAs); saveAs.delete(); @@ -90,22 +87,6 @@ class DownloadFileThread extends Thread { return; } } - - int bytesTotal, bytesDownloaded = 0; - if (observer.useByteProgessBar()) { - try { - bytesTotal = getTotalBytes(this.url); - } catch (IOException e) { - logger.error("Failed to get file size at " + this.url, e); - observer.downloadErrored(this.url, "Failed to get file size of " + this.url); - return; - } - - observer.setBytesTotal(bytesTotal); - observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal); - logger.debug("Size of file at " + this.url + " = " + bytesTotal + "b"); - } - URL urlToDownload = this.url; boolean redirected = false; int tries = 0; // Number of attempts to download @@ -139,11 +120,20 @@ class DownloadFileThread extends Thread { cookie += key + "=" + cookies.get(key); } huc.setRequestProperty("Cookie", cookie); + if (observer.tryResumeDownload()) { + if (fileSize != 0) { + huc.setRequestProperty("Range", "bytes=" + fileSize + "-"); + } + } logger.debug("Request properties: " + huc.getRequestProperties()); huc.connect(); int statusCode = huc.getResponseCode(); logger.debug("Status code: " + statusCode); + if (statusCode != 206 && observer.tryResumeDownload()) { + // TODO find a better way to handle servers that don't support resuming downloads then just erroring out + throw new IOException("Server doesn't support resuming downloads"); + } if (statusCode / 100 == 3) { // 3xx Redirect if (!redirected) { // Don't increment retries on the first redirect @@ -171,6 +161,15 @@ class DownloadFileThread extends Thread { observer.downloadErrored(url, "Imgur image is 404: " + url.toExternalForm()); return; } + + // If the ripper is using the bytes progress bar set bytesTotal to huc.getContentLength() + if (observer.useByteProgessBar()) { + bytesTotal = huc.getContentLength(); + observer.setBytesTotal(bytesTotal); + observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal); + logger.debug("Size of file at " + this.url + " = " + bytesTotal + "b"); + } + // Save file bis = new BufferedInputStream(huc.getInputStream()); @@ -179,8 +178,12 @@ class DownloadFileThread extends Thread { String fileExt = URLConnection.guessContentTypeFromStream(bis).replaceAll("image/", ""); saveAs = new File(saveAs.toString() + "." + fileExt); } - - fos = new FileOutputStream(saveAs); + // If we're resuming a download we append data to the existing file + if (statusCode == 206) { + fos = new FileOutputStream(saveAs, true); + } else { + fos = new FileOutputStream(saveAs); + } byte[] data = new byte[1024 * 256]; int bytesRead; while ( (bytesRead = bis.read(data)) != -1) { try { From 2c08be83f69b466baf8dbe389d0a85fa8f1830d0 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 27 Jun 2018 00:58:55 -0400 Subject: [PATCH 07/11] Removed unit test for vine ripper has the ripper was removed --- .../ripme/tst/ripper/rippers/VideoRippersTest.java | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java index 6a7df184..05d7514e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java @@ -8,7 +8,6 @@ import java.util.List; import com.rarchives.ripme.ripper.VideoRipper; import com.rarchives.ripme.ripper.rippers.video.PornhubRipper; import com.rarchives.ripme.ripper.rippers.video.TwitchVideoRipper; -import com.rarchives.ripme.ripper.rippers.video.VineRipper; import com.rarchives.ripme.ripper.rippers.video.XhamsterRipper; import com.rarchives.ripme.ripper.rippers.video.XvideosRipper; import com.rarchives.ripme.ripper.rippers.video.YoupornRipper; @@ -78,18 +77,6 @@ public class VideoRippersTest extends RippersTest { } } - // https://github.com/RipMeApp/ripme/issues/186 - /* - public void testVineRipper() throws IOException { - List contentURLs = new ArrayList<>(); - contentURLs.add(new URL("https://vine.co/v/hiqQrP0eUZx")); - for (URL url : contentURLs) { - VineRipper ripper = new VineRipper(url); - videoTestHelper(ripper); - } - } - */ - public void testYoupornRipper() throws IOException { List contentURLs = new ArrayList<>(); contentURLs.add(new URL("http://www.youporn.com/watch/7669155/mrs-li-amateur-69-orgasm/?from=categ")); From ed7df108e5c974d36fc32b43397f560a0a44b72c Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 27 Jun 2018 01:02:23 -0400 Subject: [PATCH 08/11] Minor code style clean up --- .../java/com/rarchives/ripme/ripper/DownloadFileThread.java | 3 ++- .../java/com/rarchives/ripme/ripper/DownloadVideoThread.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 45c72c6e..d4439a74 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -184,7 +184,8 @@ class DownloadFileThread extends Thread { } else { fos = new FileOutputStream(saveAs); } - byte[] data = new byte[1024 * 256]; int bytesRead; + byte[] data = new byte[1024 * 256]; + int bytesRead; while ( (bytesRead = bis.read(data)) != -1) { try { observer.stopCheck(); diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java index 437f18d0..ef55e54e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java @@ -76,7 +76,8 @@ class DownloadVideoThread extends Thread { int tries = 0; // Number of attempts to download do { InputStream bis = null; OutputStream fos = null; - byte[] data = new byte[1024 * 256]; int bytesRead; + byte[] data = new byte[1024 * 256]; + int bytesRead; try { logger.info(" Downloading file: " + url + (tries > 0 ? " Retry #" + tries : "")); observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm()); From 1f8e9184c7a1522f37e4910be44dfbee7c2d3e06 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 27 Jun 2018 01:24:04 -0400 Subject: [PATCH 09/11] Added the helper func getByteStatusText to cut down on copy pasted code --- .../ripme/ripper/rippers/GfycatRipper.java | 6 +----- .../ripper/rippers/GfycatporntubeRipper.java | 6 +----- .../java/com/rarchives/ripme/utils/Utils.java | 16 ++++++++++++++++ 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java index 78a0af19..97afef94 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java @@ -112,11 +112,7 @@ public class GfycatRipper extends AbstractHTMLRipper { @Override public String getStatusText() { - return String.valueOf(getCompletionPercentage()) + - "% - " + - Utils.bytesToHumanReadable(bytesCompleted) + - " / " + - Utils.bytesToHumanReadable(bytesTotal); + return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java index d8f629ae..9010eb0e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java @@ -65,11 +65,7 @@ public class GfycatporntubeRipper extends AbstractHTMLRipper { @Override public String getStatusText() { - return String.valueOf(getCompletionPercentage()) + - "% - " + - Utils.bytesToHumanReadable(bytesCompleted) + - " / " + - Utils.bytesToHumanReadable(bytesTotal); + return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal); } @Override diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 5821e9f3..e57acf77 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -715,4 +715,20 @@ public class Utils { } } + /** + * Formats and reuturns the status text for rippers using the byte progress bar + * + * @param completionPercentage An int between 0 and 100 which repersents how close the download is to complete + * @param bytesCompleted How many bytes have been downloaded + * @param bytesTotal The total size of the file that is being downloaded + * @return Returns the formatted status text for rippers using the byte progress bar + */ + public static String getByteStatusText(int completionPercentage, int bytesCompleted, int bytesTotal) { + return String.valueOf(completionPercentage) + + "% - " + + Utils.bytesToHumanReadable(bytesCompleted) + + " / " + + Utils.bytesToHumanReadable(bytesTotal); + } + } \ No newline at end of file From af064ee435f60365d56c2a94cb8567bd1c26d8f4 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 27 Jun 2018 01:25:58 -0400 Subject: [PATCH 10/11] Minor code refactoring --- .../com/rarchives/ripme/ripper/rippers/GfycatRipper.java | 6 +++--- .../ripme/ripper/rippers/GfycatporntubeRipper.java | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java index 97afef94..a09d68ab 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java @@ -19,6 +19,9 @@ import com.rarchives.ripme.utils.Http; public class GfycatRipper extends AbstractHTMLRipper { + private int bytesTotal = 1; + private int bytesCompleted = 1; + private static final String HOST = "gfycat.com"; public GfycatRipper(URL url) throws IOException { @@ -107,9 +110,6 @@ public class GfycatRipper extends AbstractHTMLRipper { return vidUrl; } - private int bytesTotal = 1; - private int bytesCompleted = 1; - @Override public String getStatusText() { return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java index 9010eb0e..55150d9e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java @@ -17,6 +17,9 @@ import com.rarchives.ripme.utils.Http; public class GfycatporntubeRipper extends AbstractHTMLRipper { + private int bytesTotal = 1; + private int bytesCompleted = 1; + public GfycatporntubeRipper(URL url) throws IOException { super(url); } @@ -60,9 +63,6 @@ public class GfycatporntubeRipper extends AbstractHTMLRipper { addURLToDownload(url, getPrefix(index)); } - private int bytesTotal = 1; - private int bytesCompleted = 1; - @Override public String getStatusText() { return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal); From ffbda6032d681502864fc208d845b3cd3ac89927 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 27 Jun 2018 02:05:36 -0400 Subject: [PATCH 11/11] Fixed bug that caused downloads from sites that support download resuming to fail with 'Server doesn't support resuming downloads' --- .../java/com/rarchives/ripme/ripper/DownloadFileThread.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index d4439a74..10e4462c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -130,7 +130,7 @@ class DownloadFileThread extends Thread { int statusCode = huc.getResponseCode(); logger.debug("Status code: " + statusCode); - if (statusCode != 206 && observer.tryResumeDownload()) { + if (statusCode != 206 && observer.tryResumeDownload() && saveAs.exists()) { // TODO find a better way to handle servers that don't support resuming downloads then just erroring out throw new IOException("Server doesn't support resuming downloads"); }