diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 87d8bd46..94ac1ec9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -613,4 +613,9 @@ public abstract class AbstractRipper protected boolean isThisATest() { return thisIsATest; } + + // If true ripme uses a byte progress bar + protected boolean useByteProgessBar() { return false;} + // If true ripme will try to resume a broken download for this ripper + protected boolean tryResumeDownload() { return false;} } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 42dedffe..10e4462c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -59,18 +59,25 @@ class DownloadFileThread extends Thread { this.cookies = cookies; } + /** * Attempts to download the file. Retries as needed. * Notifies observers upon completion/error/warn. */ public void run() { + long fileSize = 0; + int bytesTotal = 0; + int bytesDownloaded = 0; + if (saveAs.exists() && observer.tryResumeDownload()) { + fileSize = saveAs.length(); + } try { observer.stopCheck(); } catch (IOException e) { observer.downloadErrored(url, "Download interrupted"); return; } - if (saveAs.exists()) { + if (saveAs.exists() && !observer.tryResumeDownload()) { if (Utils.getConfigBoolean("file.overwrite", false)) { logger.info("[!] Deleting existing file" + prettySaveAs); saveAs.delete(); @@ -80,7 +87,6 @@ class DownloadFileThread extends Thread { return; } } - URL urlToDownload = this.url; boolean redirected = false; int tries = 0; // Number of attempts to download @@ -114,11 +120,20 @@ class DownloadFileThread extends Thread { cookie += key + "=" + cookies.get(key); } huc.setRequestProperty("Cookie", cookie); + if (observer.tryResumeDownload()) { + if (fileSize != 0) { + huc.setRequestProperty("Range", "bytes=" + fileSize + "-"); + } + } logger.debug("Request properties: " + huc.getRequestProperties()); huc.connect(); int statusCode = huc.getResponseCode(); logger.debug("Status code: " + statusCode); + if (statusCode != 206 && observer.tryResumeDownload() && saveAs.exists()) { + // TODO find a better way to handle servers that don't support resuming downloads then just erroring out + throw new IOException("Server doesn't support resuming downloads"); + } if (statusCode / 100 == 3) { // 3xx Redirect if (!redirected) { // Don't increment retries on the first redirect @@ -146,6 +161,15 @@ class DownloadFileThread extends Thread { observer.downloadErrored(url, "Imgur image is 404: " + url.toExternalForm()); return; } + + // If the ripper is using the bytes progress bar set bytesTotal to huc.getContentLength() + if (observer.useByteProgessBar()) { + bytesTotal = huc.getContentLength(); + observer.setBytesTotal(bytesTotal); + observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal); + logger.debug("Size of file at " + this.url + " = " + bytesTotal + "b"); + } + // Save file bis = new BufferedInputStream(huc.getInputStream()); @@ -154,9 +178,30 @@ class DownloadFileThread extends Thread { String fileExt = URLConnection.guessContentTypeFromStream(bis).replaceAll("image/", ""); saveAs = new File(saveAs.toString() + "." + fileExt); } - - fos = new FileOutputStream(saveAs); - IOUtils.copy(bis, fos); + // If we're resuming a download we append data to the existing file + if (statusCode == 206) { + fos = new FileOutputStream(saveAs, true); + } else { + fos = new FileOutputStream(saveAs); + } + byte[] data = new byte[1024 * 256]; + int bytesRead; + while ( (bytesRead = bis.read(data)) != -1) { + try { + observer.stopCheck(); + } catch (IOException e) { + observer.downloadErrored(url, "Download interrupted"); + return; + } + fos.write(data, 0, bytesRead); + if (observer.useByteProgessBar()) { + bytesDownloaded += bytesRead; + observer.setBytesCompleted(bytesDownloaded); + observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded); + } + } + bis.close(); + fos.close(); break; // Download successful: break out of infinite loop } catch (HttpStatusException hse) { logger.debug("HTTP status exception", hse); diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java index 437f18d0..ef55e54e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java @@ -76,7 +76,8 @@ class DownloadVideoThread extends Thread { int tries = 0; // Number of attempts to download do { InputStream bis = null; OutputStream fos = null; - byte[] data = new byte[1024 * 256]; int bytesRead; + byte[] data = new byte[1024 * 256]; + int bytesRead; try { logger.info(" Downloading file: " + url + (tries > 0 ? " Retry #" + tries : "")); observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm()); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java similarity index 56% rename from src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java rename to src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java index 75577597..a09d68ab 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java @@ -1,18 +1,26 @@ -package com.rarchives.ripme.ripper.rippers.video; +package com.rarchives.ripme.ripper.rippers; + import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.util.ArrayList; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Utils; import org.jsoup.nodes.Document; import org.jsoup.select.Elements; -import com.rarchives.ripme.ripper.VideoRipper; import com.rarchives.ripme.utils.Http; -public class GfycatRipper extends VideoRipper { + +public class GfycatRipper extends AbstractHTMLRipper { + + private int bytesTotal = 1; + private int bytesCompleted = 1; private static final String HOST = "gfycat.com"; @@ -20,9 +28,14 @@ public class GfycatRipper extends VideoRipper { super(url); } + @Override + public String getDomain() { + return "gfycat.com"; + } + @Override public String getHost() { - return HOST; + return "gfycat"; } @Override @@ -37,6 +50,16 @@ public class GfycatRipper extends VideoRipper { return url; } + @Override + public Document getFirstPage() throws IOException { + return Http.url(url).get(); + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } + @Override public String getGID(URL url) throws MalformedURLException { Pattern p = Pattern.compile("^https?://[wm.]*gfycat\\.com/([a-zA-Z0-9]+).*$"); @@ -52,10 +75,15 @@ public class GfycatRipper extends VideoRipper { } @Override - public void rip() throws IOException { - String vidUrl = getVideoURL(this.url); - addURLToDownload(new URL(vidUrl), "gfycat_" + getGID(this.url)); - waitForThreads(); + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + Elements videos = doc.select("source#mp4Source"); + String vidUrl = videos.first().attr("src"); + if (vidUrl.startsWith("//")) { + vidUrl = "http:" + vidUrl; + } + result.add(vidUrl); + return result; } /** @@ -66,10 +94,10 @@ public class GfycatRipper extends VideoRipper { */ public static String getVideoURL(URL url) throws IOException { LOGGER.info("Retrieving " + url.toExternalForm()); - + //Sanitize the URL first url = new URL(url.toExternalForm().replace("/gifs/detail", "")); - + Document doc = Http.url(url).get(); Elements videos = doc.select("source#mp4Source"); if (videos.isEmpty()) { @@ -81,4 +109,27 @@ public class GfycatRipper extends VideoRipper { } return vidUrl; } + + @Override + public String getStatusText() { + return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal); + } + + @Override + public int getCompletionPercentage() { + return (int) (100 * (bytesCompleted / (float) bytesTotal)); + } + + @Override + public void setBytesTotal(int bytes) { + this.bytesTotal = bytes; + } + + @Override + public void setBytesCompleted(int bytes) { + this.bytesCompleted = bytes; + } + + @Override + public boolean useByteProgessBar() {return true;} } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java index 504b89d6..55150d9e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java @@ -8,6 +8,7 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.rarchives.ripme.utils.Utils; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -16,6 +17,9 @@ import com.rarchives.ripme.utils.Http; public class GfycatporntubeRipper extends AbstractHTMLRipper { + private int bytesTotal = 1; + private int bytesCompleted = 1; + public GfycatporntubeRipper(URL url) throws IOException { super(url); } @@ -58,4 +62,27 @@ public class GfycatporntubeRipper extends AbstractHTMLRipper { public void downloadURL(URL url, int index) { addURLToDownload(url, getPrefix(index)); } + + @Override + public String getStatusText() { + return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal); + } + + @Override + public int getCompletionPercentage() { + return (int) (100 * (bytesCompleted / (float) bytesTotal)); + } + + @Override + public void setBytesTotal(int bytes) { + this.bytesTotal = bytes; + } + + @Override + public void setBytesCompleted(int bytes) { + this.bytesCompleted = bytes; + } + + @Override + public boolean useByteProgessBar() {return true;} } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VineRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VineRipper.java deleted file mode 100644 index 1ca59676..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VineRipper.java +++ /dev/null @@ -1,67 +0,0 @@ -package com.rarchives.ripme.ripper.rippers.video; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.jsoup.nodes.Document; -import org.jsoup.select.Elements; - -import com.rarchives.ripme.ripper.VideoRipper; -import com.rarchives.ripme.utils.Http; - -public class VineRipper extends VideoRipper { - - private static final String HOST = "vine"; - - public VineRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return HOST; - } - - @Override - public boolean canRip(URL url) { - // https://vine.co/v/hiqQrP0eUZx - Pattern p = Pattern.compile("^https?://[wm.]*vine\\.co/v/[a-zA-Z0-9]+.*$"); - Matcher m = p.matcher(url.toExternalForm()); - return m.matches(); - } - - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return url; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://[wm.]*vine\\.co/v/([a-zA-Z0-9]+).*$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - - throw new MalformedURLException( - "Expected vine format:" - + "vine.co/v/####" - + " Got: " + url); - } - - @Override - public void rip() throws IOException { - LOGGER.info(" Retrieving " + this.url.toExternalForm()); - Document doc = Http.url(this.url).get(); - Elements props = doc.select("meta[property=twitter:player:stream]"); - if (props.isEmpty()) { - throw new IOException("Could not find meta property 'twitter:player:stream' at " + url); - } - String vidUrl = props.get(0).attr("content"); - addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url)); - waitForThreads(); - } -} \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index 34081852..9845145c 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -12,7 +12,7 @@ import com.rarchives.ripme.ripper.rippers.EroShareRipper; import com.rarchives.ripme.ripper.rippers.EromeRipper; import com.rarchives.ripme.ripper.rippers.ImgurRipper; import com.rarchives.ripme.ripper.rippers.VidbleRipper; -import com.rarchives.ripme.ripper.rippers.video.GfycatRipper; +import com.rarchives.ripme.ripper.rippers.GfycatRipper; import org.apache.commons.lang.math.NumberUtils; import org.apache.log4j.Logger; import org.jsoup.Jsoup; diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 5821e9f3..e57acf77 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -715,4 +715,20 @@ public class Utils { } } + /** + * Formats and reuturns the status text for rippers using the byte progress bar + * + * @param completionPercentage An int between 0 and 100 which repersents how close the download is to complete + * @param bytesCompleted How many bytes have been downloaded + * @param bytesTotal The total size of the file that is being downloaded + * @return Returns the formatted status text for rippers using the byte progress bar + */ + public static String getByteStatusText(int completionPercentage, int bytesCompleted, int bytesTotal) { + return String.valueOf(completionPercentage) + + "% - " + + Utils.bytesToHumanReadable(bytesCompleted) + + " / " + + Utils.bytesToHumanReadable(bytesTotal); + } + } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java index ca73f138..d3d011be 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java @@ -1,6 +1,6 @@ package com.rarchives.ripme.tst.ripper.rippers; -import com.rarchives.ripme.ripper.rippers.video.GfycatRipper; +import com.rarchives.ripme.ripper.rippers.GfycatRipper; import java.io.IOException; import java.net.URL; diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java index fdd61cf9..efe23a01 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java @@ -5,7 +5,6 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.RedditRipper; -import com.rarchives.ripme.ripper.rippers.video.GfycatRipper; public class RedditRipperTest extends RippersTest { // https://github.com/RipMeApp/ripme/issues/253 - Disabled tests: RedditRipperTest#testRedditSubreddit*Rip is flaky diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java index 6a7df184..05d7514e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java @@ -8,7 +8,6 @@ import java.util.List; import com.rarchives.ripme.ripper.VideoRipper; import com.rarchives.ripme.ripper.rippers.video.PornhubRipper; import com.rarchives.ripme.ripper.rippers.video.TwitchVideoRipper; -import com.rarchives.ripme.ripper.rippers.video.VineRipper; import com.rarchives.ripme.ripper.rippers.video.XhamsterRipper; import com.rarchives.ripme.ripper.rippers.video.XvideosRipper; import com.rarchives.ripme.ripper.rippers.video.YoupornRipper; @@ -78,18 +77,6 @@ public class VideoRippersTest extends RippersTest { } } - // https://github.com/RipMeApp/ripme/issues/186 - /* - public void testVineRipper() throws IOException { - List contentURLs = new ArrayList<>(); - contentURLs.add(new URL("https://vine.co/v/hiqQrP0eUZx")); - for (URL url : contentURLs) { - VineRipper ripper = new VineRipper(url); - videoTestHelper(ripper); - } - } - */ - public void testYoupornRipper() throws IOException { List contentURLs = new ArrayList<>(); contentURLs.add(new URL("http://www.youporn.com/watch/7669155/mrs-li-amateur-69-orgasm/?from=categ"));