Merge pull request #746 from cyian-1756/byteProgressBar

All rippers can now use a byte progress bar, added the ability to resume some file downloads

src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java

@@ -613,4 +613,9 @@ public abstract class AbstractRipper
     protected boolean isThisATest() {
         return thisIsATest;
     }
+
+    // If true ripme uses a byte progress bar
+    protected boolean useByteProgessBar() { return false;}
+    // If true ripme will try to resume a broken download for this ripper
+    protected boolean tryResumeDownload() { return false;}
 }

src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java

@@ -59,18 +59,25 @@ class DownloadFileThread extends Thread {
         this.cookies = cookies;
     }
 
+
     /**
      * Attempts to download the file. Retries as needed.
      * Notifies observers upon completion/error/warn.
      */
     public void run() {
+        long fileSize = 0;
+        int bytesTotal = 0;
+        int bytesDownloaded = 0;
+        if (saveAs.exists() && observer.tryResumeDownload()) {
+            fileSize = saveAs.length();
+        }
         try {
             observer.stopCheck();
         } catch (IOException e) {
             observer.downloadErrored(url, "Download interrupted");
             return;
         }
-        if (saveAs.exists()) {
+        if (saveAs.exists() && !observer.tryResumeDownload()) {
             if (Utils.getConfigBoolean("file.overwrite", false)) {
                 logger.info("[!] Deleting existing file" + prettySaveAs);
                 saveAs.delete();
@@ -80,7 +87,6 @@ class DownloadFileThread extends Thread {
                 return;
             }
         }
-
         URL urlToDownload = this.url;
         boolean redirected = false;
         int tries = 0; // Number of attempts to download
@@ -114,11 +120,20 @@ class DownloadFileThread extends Thread {
                     cookie += key + "=" + cookies.get(key);
                 }
                 huc.setRequestProperty("Cookie", cookie);
+                if (observer.tryResumeDownload()) {
+                    if (fileSize != 0) {
+                        huc.setRequestProperty("Range", "bytes=" + fileSize + "-");
+                    }
+                }
                 logger.debug("Request properties: " + huc.getRequestProperties());
                 huc.connect();
 
                 int statusCode = huc.getResponseCode();
                 logger.debug("Status code: " + statusCode);
+                if (statusCode != 206 && observer.tryResumeDownload() && saveAs.exists()) {
+                    // TODO find a better way to handle servers that don't support resuming downloads then just erroring out
+                    throw new IOException("Server doesn't support resuming downloads");
+                }
                 if (statusCode  / 100 == 3) { // 3xx Redirect
                     if (!redirected) {
                         // Don't increment retries on the first redirect
@@ -146,6 +161,15 @@ class DownloadFileThread extends Thread {
                     observer.downloadErrored(url, "Imgur image is 404: " + url.toExternalForm());
                     return;
                 }
+
+                // If the ripper is using the bytes progress bar set bytesTotal to huc.getContentLength()
+                if (observer.useByteProgessBar()) {
+                    bytesTotal = huc.getContentLength();
+                    observer.setBytesTotal(bytesTotal);
+                    observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal);
+                    logger.debug("Size of file at " + this.url + " = " + bytesTotal + "b");
+                }
+
                 // Save file
                 bis = new BufferedInputStream(huc.getInputStream());
 
@@ -154,9 +178,30 @@ class DownloadFileThread extends Thread {
                     String fileExt = URLConnection.guessContentTypeFromStream(bis).replaceAll("image/", "");
                     saveAs = new File(saveAs.toString() + "." + fileExt);
                 }
-
+                // If we're resuming a download we append data to the existing file
-                fos = new FileOutputStream(saveAs);
+                if (statusCode == 206) {
-                IOUtils.copy(bis, fos);
+                    fos = new FileOutputStream(saveAs, true);
+                } else {
+                    fos = new FileOutputStream(saveAs);
+                }
+                byte[] data = new byte[1024 * 256];
+                int bytesRead;
+                while ( (bytesRead = bis.read(data)) != -1) {
+                    try {
+                        observer.stopCheck();
+                    } catch (IOException e) {
+                        observer.downloadErrored(url, "Download interrupted");
+                        return;
+                    }
+                    fos.write(data, 0, bytesRead);
+                    if (observer.useByteProgessBar()) {
+                        bytesDownloaded += bytesRead;
+                        observer.setBytesCompleted(bytesDownloaded);
+                        observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded);
+                    }
+                }
+                bis.close();
+                fos.close();
                 break; // Download successful: break out of infinite loop
             } catch (HttpStatusException hse) {
                 logger.debug("HTTP status exception", hse);

src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java

@@ -76,7 +76,8 @@ class DownloadVideoThread extends Thread {
         int tries = 0; // Number of attempts to download
         do {
             InputStream bis = null; OutputStream fos = null;
-            byte[] data = new byte[1024 * 256]; int bytesRead;
+            byte[] data = new byte[1024 * 256];
+            int bytesRead;
             try {
                 logger.info("    Downloading file: " + url + (tries > 0 ? " Retry #" + tries : ""));
                 observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm());

src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java

@@ -1,18 +1,26 @@
-package com.rarchives.ripme.ripper.rippers.video;
+package com.rarchives.ripme.ripper.rippers;
+
 
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import com.rarchives.ripme.ripper.AbstractHTMLRipper;
+import com.rarchives.ripme.utils.Utils;
 import org.jsoup.nodes.Document;
 import org.jsoup.select.Elements;
 
-import com.rarchives.ripme.ripper.VideoRipper;
 import com.rarchives.ripme.utils.Http;
 
-public class GfycatRipper extends VideoRipper {
+
+public class GfycatRipper extends AbstractHTMLRipper {
+
+    private int bytesTotal = 1;
+    private int bytesCompleted = 1;
 
     private static final String HOST = "gfycat.com";
 
@@ -20,9 +28,14 @@ public class GfycatRipper extends VideoRipper {
         super(url);
     }
 
+    @Override
+    public String getDomain() {
+        return "gfycat.com";
+    }
+
     @Override
     public String getHost() {
-        return HOST;
+        return "gfycat";
     }
 
     @Override
@@ -37,6 +50,16 @@ public class GfycatRipper extends VideoRipper {
         return url;
     }
 
+    @Override
+    public Document getFirstPage() throws IOException {
+        return Http.url(url).get();
+    }
+
+    @Override
+    public void downloadURL(URL url, int index) {
+        addURLToDownload(url, getPrefix(index));
+    }
+
     @Override
     public String getGID(URL url) throws MalformedURLException {
         Pattern p = Pattern.compile("^https?://[wm.]*gfycat\\.com/([a-zA-Z0-9]+).*$");
@@ -52,10 +75,15 @@ public class GfycatRipper extends VideoRipper {
     }
 
     @Override
-    public void rip() throws IOException {
+    public List<String> getURLsFromPage(Document doc) {
-        String vidUrl = getVideoURL(this.url);
+        List<String> result = new ArrayList<>();
-        addURLToDownload(new URL(vidUrl), "gfycat_" + getGID(this.url));
+        Elements videos = doc.select("source#mp4Source");
-        waitForThreads();
+        String vidUrl = videos.first().attr("src");
+        if (vidUrl.startsWith("//")) {
+            vidUrl = "http:" + vidUrl;
+        }
+        result.add(vidUrl);
+        return result;
     }
 
     /**
@@ -81,4 +109,27 @@ public class GfycatRipper extends VideoRipper {
         }
         return vidUrl;
     }
+
+    @Override
+    public String getStatusText() {
+        return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal);
+    }
+
+    @Override
+    public int getCompletionPercentage() {
+        return (int) (100 * (bytesCompleted / (float) bytesTotal));
+    }
+
+    @Override
+    public void setBytesTotal(int bytes) {
+        this.bytesTotal = bytes;
+    }
+
+    @Override
+    public void setBytesCompleted(int bytes) {
+        this.bytesCompleted = bytes;
+    }
+
+    @Override
+    public boolean useByteProgessBar() {return true;}
 }

src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java

@@ -8,6 +8,7 @@ import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import com.rarchives.ripme.utils.Utils;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 
@@ -16,6 +17,9 @@ import com.rarchives.ripme.utils.Http;
 
 public class GfycatporntubeRipper extends AbstractHTMLRipper {
 
+    private int bytesTotal = 1;
+    private int bytesCompleted = 1;
+
     public GfycatporntubeRipper(URL url) throws IOException {
         super(url);
     }
@@ -58,4 +62,27 @@ public class GfycatporntubeRipper extends AbstractHTMLRipper {
     public void downloadURL(URL url, int index) {
         addURLToDownload(url, getPrefix(index));
     }
+
+    @Override
+    public String getStatusText() {
+        return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal);
+    }
+
+    @Override
+    public int getCompletionPercentage() {
+        return (int) (100 * (bytesCompleted / (float) bytesTotal));
+    }
+
+    @Override
+    public void setBytesTotal(int bytes) {
+        this.bytesTotal = bytes;
+    }
+
+    @Override
+    public void setBytesCompleted(int bytes) {
+        this.bytesCompleted = bytes;
+    }
+
+    @Override
+    public boolean useByteProgessBar() {return true;}
 }

src/main/java/com/rarchives/ripme/ripper/rippers/video/VineRipper.java

@@ -1,67 +0,0 @@
-package com.rarchives.ripme.ripper.rippers.video;
-
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.jsoup.nodes.Document;
-import org.jsoup.select.Elements;
-
-import com.rarchives.ripme.ripper.VideoRipper;
-import com.rarchives.ripme.utils.Http;
-
-public class VineRipper extends VideoRipper {
-
-    private static final String HOST = "vine";
-
-    public VineRipper(URL url) throws IOException {
-        super(url);
-    }
-
-    @Override
-    public String getHost() {
-        return HOST;
-    }
-
-    @Override
-    public boolean canRip(URL url) {
-        // https://vine.co/v/hiqQrP0eUZx
-        Pattern p = Pattern.compile("^https?://[wm.]*vine\\.co/v/[a-zA-Z0-9]+.*$");
-        Matcher m = p.matcher(url.toExternalForm());
-        return m.matches();
-    }
-
-    @Override
-    public URL sanitizeURL(URL url) throws MalformedURLException {
-        return url;
-    }
-
-    @Override
-    public String getGID(URL url) throws MalformedURLException {
-        Pattern p = Pattern.compile("^https?://[wm.]*vine\\.co/v/([a-zA-Z0-9]+).*$");
-        Matcher m = p.matcher(url.toExternalForm());
-        if (m.matches()) {
-            return m.group(1);
-        }
-
-        throw new MalformedURLException(
-                "Expected vine format:"
-                        + "vine.co/v/####"
-                        + " Got: " + url);
-    }
-
-    @Override
-    public void rip() throws IOException {
-        LOGGER.info("    Retrieving " + this.url.toExternalForm());
-        Document doc = Http.url(this.url).get();
-        Elements props = doc.select("meta[property=twitter:player:stream]");
-        if (props.isEmpty()) {
-            throw new IOException("Could not find meta property 'twitter:player:stream' at " + url);
-        }
-        String vidUrl = props.get(0).attr("content");
-        addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url));
-        waitForThreads();
-    }
-}

src/main/java/com/rarchives/ripme/utils/RipUtils.java

@@ -12,7 +12,7 @@ import com.rarchives.ripme.ripper.rippers.EroShareRipper;
 import com.rarchives.ripme.ripper.rippers.EromeRipper;
 import com.rarchives.ripme.ripper.rippers.ImgurRipper;
 import com.rarchives.ripme.ripper.rippers.VidbleRipper;
-import com.rarchives.ripme.ripper.rippers.video.GfycatRipper;
+import com.rarchives.ripme.ripper.rippers.GfycatRipper;
 import org.apache.commons.lang.math.NumberUtils;
 import org.apache.log4j.Logger;
 import org.jsoup.Jsoup;

src/main/java/com/rarchives/ripme/utils/Utils.java

@@ -715,4 +715,20 @@ public class Utils {
         }
     }
 
+    /**
+     * Formats and reuturns the status text for rippers using the byte progress bar
+     *
+     * @param completionPercentage An int between 0 and 100 which repersents how close the download is to complete
+     * @param bytesCompleted How many bytes have been downloaded
+     * @param bytesTotal The total size of the file that is being downloaded
+     * @return Returns the formatted status text for rippers using the byte progress bar
+     */
+    public static String getByteStatusText(int completionPercentage, int bytesCompleted, int bytesTotal) {
+        return String.valueOf(completionPercentage) +
+                "%  - " +
+                Utils.bytesToHumanReadable(bytesCompleted) +
+                " / " +
+                Utils.bytesToHumanReadable(bytesTotal);
+    }
+
 }

src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java

@@ -1,6 +1,6 @@
 package com.rarchives.ripme.tst.ripper.rippers;
 
-import com.rarchives.ripme.ripper.rippers.video.GfycatRipper;
+import com.rarchives.ripme.ripper.rippers.GfycatRipper;
 import java.io.IOException;
 import java.net.URL;
 

src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java

@@ -5,7 +5,6 @@ import java.io.IOException;
 import java.net.URL;
 
 import com.rarchives.ripme.ripper.rippers.RedditRipper;
-import com.rarchives.ripme.ripper.rippers.video.GfycatRipper;
 
 public class RedditRipperTest extends RippersTest {
     // https://github.com/RipMeApp/ripme/issues/253 - Disabled tests: RedditRipperTest#testRedditSubreddit*Rip is flaky

src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java

@@ -8,7 +8,6 @@ import java.util.List;
 import com.rarchives.ripme.ripper.VideoRipper;
 import com.rarchives.ripme.ripper.rippers.video.PornhubRipper;
 import com.rarchives.ripme.ripper.rippers.video.TwitchVideoRipper;
-import com.rarchives.ripme.ripper.rippers.video.VineRipper;
 import com.rarchives.ripme.ripper.rippers.video.XhamsterRipper;
 import com.rarchives.ripme.ripper.rippers.video.XvideosRipper;
 import com.rarchives.ripme.ripper.rippers.video.YoupornRipper;
@@ -78,18 +77,6 @@ public class VideoRippersTest extends RippersTest {
         }
     }
 
-    // https://github.com/RipMeApp/ripme/issues/186
-    /*
-    public void testVineRipper() throws IOException {
-        List<URL> contentURLs = new ArrayList<>();
-        contentURLs.add(new URL("https://vine.co/v/hiqQrP0eUZx"));
-        for (URL url : contentURLs) {
-            VineRipper ripper = new VineRipper(url);
-            videoTestHelper(ripper);
-        }
-    }
-    */
-
     public void testYoupornRipper() throws IOException {
         List<URL> contentURLs = new ArrayList<>();
         contentURLs.add(new URL("http://www.youporn.com/watch/7669155/mrs-li-amateur-69-orgasm/?from=categ"));