Added video ripper support, xvideos for now #18

pom.xml

@@ -4,7 +4,7 @@
   <groupId>com.rarchives.ripme</groupId>
   <artifactId>ripme</artifactId>
   <packaging>jar</packaging>
-  <version>1.0.19</version>
+  <version>1.0.20</version>
   <name>ripme</name>
   <url>http://rip.rarchives.com</url>
   <properties>

src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java

@@ -6,10 +6,7 @@ import java.lang.reflect.Constructor;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Observable;
 
 import org.apache.log4j.Logger;
@@ -23,7 +20,7 @@ public abstract class AbstractRipper
                 extends Observable
                 implements RipperInterface, Runnable {
 
-    private static final Logger logger = Logger.getLogger(AbstractRipper.class);
+    protected static final Logger logger = Logger.getLogger(AbstractRipper.class);
 
     protected static final String USER_AGENT = 
             "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:27.0) Gecko/20100101 Firefox/27.0";
@@ -33,9 +30,6 @@ public abstract class AbstractRipper
     protected DownloadThreadPool threadPool;
     protected RipStatusHandler observer = null;
 
-    protected Map<URL, File> itemsPending = Collections.synchronizedMap(new HashMap<URL, File>());
-    protected Map<URL, File> itemsCompleted = Collections.synchronizedMap(new HashMap<URL, File>());
-    protected Map<URL, String> itemsErrored = Collections.synchronizedMap(new HashMap<URL, String>());
     protected boolean completed = true;
 
     public abstract void rip() throws IOException;
@@ -102,7 +96,7 @@ public abstract class AbstractRipper
         // Use empty subdirectory
         addURLToDownload(url, prefix, "");
     }
-
+    
     /**
      * Queues image to be downloaded and saved.
      * @param url
@@ -110,17 +104,7 @@ public abstract class AbstractRipper
      * @param saveAs
      *      Path of the local file to save the content to.
      */
-    public void addURLToDownload(URL url, File saveAs) {
+    public abstract void addURLToDownload(URL url, File saveAs);
-        if (itemsPending.containsKey(url)
-                || itemsCompleted.containsKey(url)
-                || itemsErrored.containsKey(url)) {
-            // Item is already downloaded/downloading, skip it.
-            logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs));
-            return;
-        }
-        itemsPending.put(url, saveAs);
-        threadPool.addThread(new DownloadFileThread(url, saveAs, this));
-    }
 
     /**
      * Queues file to be downloaded and saved. With options.
@@ -192,67 +176,30 @@ public abstract class AbstractRipper
      * @param saveAs
      *      Where the downloaded file is stored.
      */
-    public void downloadCompleted(URL url, File saveAs) {
+    public abstract void downloadCompleted(URL url, File saveAs);
-        if (observer == null) {
-            return;
-        }
-        try {
-            String path = Utils.removeCWD(saveAs);
-            RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path);
-            itemsPending.remove(url);
-            itemsCompleted.put(url, saveAs);
-            observer.update(this, msg);
-
-            checkIfComplete();
-        } catch (Exception e) {
-            logger.error("Exception while updating observer: ", e);
-        }
-    }
-
     /**
      * Notifies observers that a file could not be downloaded (includes a reason).
      * @param url
      * @param reason
      */
-    public void downloadErrored(URL url, String reason) {
+    public abstract void downloadErrored(URL url, String reason);
-        if (observer == null) {
-            return;
-        }
-        itemsPending.remove(url);
-        itemsErrored.put(url, reason);
-        observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason));
-
-        checkIfComplete();
-    }
-
     /**
      * Notify observers that a download could not be completed,
      * but was not technically an "error".
      * @param url
      * @param message
      */
-    public void downloadProblem(URL url, String message) {
+    public abstract void downloadProblem(URL url, String message);
-        if (observer == null) {
-            return;
-        }
-        
-        itemsPending.remove(url);
-        itemsErrored.put(url, message);
-        observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " : " + message));
-            
-        
-        checkIfComplete();
-    }
 
     /**
      * Notifies observers and updates state if all files have been ripped.
      */
-    private void checkIfComplete() {
+    protected void checkIfComplete() {
         if (observer == null) {
             return;
         }
         
-        if (!completed && itemsPending.isEmpty()) {
+        if (!completed) {
             completed = true;
             logger.info("   Rip completed!");
             
@@ -274,26 +221,7 @@ public abstract class AbstractRipper
         return workingDir;
     }
 
-    /**
+    public abstract void setWorkingDir(URL url) throws IOException;
-     * Sets directory to save all ripped files to.
-     * @param url
-     *      URL to define how the workin directory should be saved.
-     */
-    public void setWorkingDir(URL url) throws IOException {
-        String path = Utils.getWorkingDirectory().getCanonicalPath();
-        if (!path.endsWith(File.separator)) {
-            path += File.separator;
-        }
-        String title = getAlbumTitle(this.url);
-        title = Utils.filesystemSafe(title);
-        path += title + File.separator;
-        this.workingDir = new File(path);
-        if (!this.workingDir.exists()) {
-            logger.info("[+] Creating directory: " + Utils.removeCWD(this.workingDir));
-            this.workingDir.mkdirs();
-        }
-        logger.debug("Set working directory to: " + this.workingDir);
-    }
     
     public String getAlbumTitle(URL url) throws MalformedURLException {
         return getHost() + "_" + getGID(url);
@@ -309,9 +237,17 @@ public abstract class AbstractRipper
      *      If no compatible rippers can be found.
      */
     public static AbstractRipper getRipper(URL url) throws Exception {
-        for (Constructor<?> constructor : getRipperConstructors()) {
+        for (Constructor<?> constructor : getRipperConstructors("com.rarchives.ripme.ripper.rippers")) {
             try {
-                AbstractRipper ripper = (AbstractRipper) constructor.newInstance(url);
+                AlbumRipper ripper = (AlbumRipper) constructor.newInstance(url);
+                return ripper;
+            } catch (Exception e) {
+                // Incompatible rippers *will* throw exceptions during instantiation.
+            }
+        }
+        for (Constructor<?> constructor : getRipperConstructors("com.rarchives.ripme.ripper.rippers.video")) {
+            try {
+                VideoRipper ripper = (VideoRipper) constructor.newInstance(url);
                 return ripper;
             } catch (Exception e) {
                 // Incompatible rippers *will* throw exceptions during instantiation.
@@ -325,9 +261,9 @@ public abstract class AbstractRipper
      *      List of constructors for all eligible Rippers.
      * @throws Exception
      */
-    private static List<Constructor<?>> getRipperConstructors() throws Exception {
+    private static List<Constructor<?>> getRipperConstructors(String pkg) throws Exception {
         List<Constructor<?>> constructors = new ArrayList<Constructor<?>>();
-        for (Class<?> clazz : Utils.getClassesForPackage("com.rarchives.ripme.ripper.rippers")) {
+        for (Class<?> clazz : Utils.getClassesForPackage(pkg)) {
             if (AbstractRipper.class.isAssignableFrom(clazz)) {
                 constructors.add( (Constructor<?>) clazz.getConstructor(URL.class) );
             }
@@ -347,28 +283,9 @@ public abstract class AbstractRipper
         observer.update(this, new RipStatusMessage(status, message));
     }
     
-    /**
+    public abstract int getCompletionPercentage();
-     * @return
-     *      Integer between 0 and 100 defining the progress of the album rip.
-     */
-    public int getCompletionPercentage() {
-        double total = itemsPending.size()  + itemsErrored.size() + itemsCompleted.size();
-        return (int) (100 * ( (total - itemsPending.size()) / total));
-    }
     
-    /**
+    public abstract String getStatusText();
-     * @return
-     *      Human-readable information on the status of the current rip.
-     */
-    public String getStatusText() {
-        StringBuilder sb = new StringBuilder();
-        sb.append(getCompletionPercentage())
-          .append("% ")
-          .append("- Pending: "  ).append(itemsPending.size())
-          .append(", Completed: ").append(itemsCompleted.size())
-          .append(", Errored: "  ).append(itemsErrored.size());
-        return sb.toString();
-    }
 
     /**
      * Rips the album when the thread is invoked.
@@ -382,4 +299,10 @@ public abstract class AbstractRipper
         }
     }
 
+    public void setBytesTotal(int bytes) {
+        // Do nothing
+    }
+    public void setBytesCompleted(int bytes) {
+        // Do nothing
+    }
 }

src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java

@@ -0,0 +1,146 @@
+package com.rarchives.ripme.ripper;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.rarchives.ripme.ui.RipStatusMessage;
+import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
+import com.rarchives.ripme.utils.Utils;
+
+public abstract class AlbumRipper extends AbstractRipper {
+
+    protected Map<URL, File> itemsPending = Collections.synchronizedMap(new HashMap<URL, File>());
+    protected Map<URL, File> itemsCompleted = Collections.synchronizedMap(new HashMap<URL, File>());
+    protected Map<URL, String> itemsErrored = Collections.synchronizedMap(new HashMap<URL, String>());
+
+    public AlbumRipper(URL url) throws IOException {
+        super(url);
+    }
+
+    public abstract boolean canRip(URL url);
+    public abstract URL sanitizeURL(URL url) throws MalformedURLException;
+    public abstract void rip() throws IOException;
+    public abstract String getHost();
+    public abstract String getGID(URL url) throws MalformedURLException;
+
+    @Override
+    public void addURLToDownload(URL url, File saveAs) {
+        if (itemsPending.containsKey(url)
+                || itemsCompleted.containsKey(url)
+                || itemsErrored.containsKey(url)) {
+            // Item is already downloaded/downloading, skip it.
+            logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs));
+            return;
+        }
+        itemsPending.put(url, saveAs);
+        threadPool.addThread(new DownloadFileThread(url, saveAs, this));
+    }
+
+    @Override
+    public void downloadCompleted(URL url, File saveAs) {
+        if (observer == null) {
+            return;
+        }
+        try {
+            String path = Utils.removeCWD(saveAs);
+            RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path);
+            itemsPending.remove(url);
+            itemsCompleted.put(url, saveAs);
+            observer.update(this, msg);
+
+            checkIfComplete();
+        } catch (Exception e) {
+            logger.error("Exception while updating observer: ", e);
+        }
+    }
+
+    @Override
+    public void downloadErrored(URL url, String reason) {
+        if (observer == null) {
+            return;
+        }
+        itemsPending.remove(url);
+        itemsErrored.put(url, reason);
+        observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason));
+
+        checkIfComplete();
+    }
+
+    @Override
+    public void downloadProblem(URL url, String message) {
+        if (observer == null) {
+            return;
+        }
+        
+        itemsPending.remove(url);
+        itemsErrored.put(url, message);
+        observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " : " + message));
+            
+        checkIfComplete();
+    }
+
+    /**
+     * Notifies observers and updates state if all files have been ripped.
+     */
+    @Override
+    protected void checkIfComplete() {
+        if (observer == null) {
+            return;
+        }
+        if (itemsPending.isEmpty()) {
+            super.checkIfComplete();
+        }
+    }
+
+    /**
+     * Sets directory to save all ripped files to.
+     * @param url
+     *      URL to define how the workin directory should be saved.
+     */
+    @Override
+    public void setWorkingDir(URL url) throws IOException {
+        String path = Utils.getWorkingDirectory().getCanonicalPath();
+        if (!path.endsWith(File.separator)) {
+            path += File.separator;
+        }
+        String title = getAlbumTitle(this.url);
+        title = Utils.filesystemSafe(title);
+        path += title + File.separator;
+        this.workingDir = new File(path);
+        if (!this.workingDir.exists()) {
+            logger.info("[+] Creating directory: " + Utils.removeCWD(this.workingDir));
+            this.workingDir.mkdirs();
+        }
+        logger.debug("Set working directory to: " + this.workingDir);
+    }
+
+    /**
+     * @return
+     *      Integer between 0 and 100 defining the progress of the album rip.
+     */
+    @Override
+    public int getCompletionPercentage() {
+        double total = itemsPending.size()  + itemsErrored.size() + itemsCompleted.size();
+        return (int) (100 * ( (total - itemsPending.size()) / total));
+    }
+
+    /**
+     * @return
+     *      Human-readable information on the status of the current rip.
+     */
+    @Override
+    public String getStatusText() {
+        StringBuilder sb = new StringBuilder();
+        sb.append(getCompletionPercentage())
+          .append("% ")
+          .append("- Pending: "  ).append(itemsPending.size())
+          .append(", Completed: ").append(itemsCompleted.size())
+          .append(", Errored: "  ).append(itemsErrored.size());
+        return sb.toString();
+    }
+}

src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java

@@ -0,0 +1,126 @@
+package com.rarchives.ripme.ripper;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+import org.apache.log4j.Logger;
+
+import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
+import com.rarchives.ripme.utils.Utils;
+
+/**
+ * Thread for downloading files.
+ * Includes retry logic, observer notifications, and other goodies.
+ */
+public class DownloadVideoThread extends Thread {
+
+    private static final Logger logger = Logger.getLogger(DownloadVideoThread.class);
+
+    private URL url;
+    private File saveAs;
+    private String prettySaveAs;
+    private AbstractRipper observer;
+    private int retries;
+
+    public DownloadVideoThread(URL url, File saveAs, AbstractRipper observer) {
+        super();
+        this.url = url;
+        this.saveAs = saveAs;
+        this.prettySaveAs = Utils.removeCWD(saveAs);
+        this.observer = observer;
+        this.retries = Utils.getConfigInteger("download.retries", 1);
+    }
+
+    /**
+     * Attempts to download the file. Retries as needed.
+     * Notifies observers upon completion/error/warn.
+     */
+    public void run() {
+        try {
+            observer.stopCheck();
+        } catch (IOException e) {
+            observer.downloadErrored(url, "Download interrupted");
+            return;
+        }
+        if (saveAs.exists()) {
+            if (Utils.getConfigBoolean("file.overwrite", false)) {
+                logger.info("[!] Deleting existing file" + prettySaveAs);
+                saveAs.delete();
+            } else {
+                logger.info("[!] Skipping " + url + " -- file already exists: " + prettySaveAs);
+                observer.downloadProblem(url, "File already exists: " + prettySaveAs);
+                return;
+            }
+        }
+
+        int bytesTotal, bytesDownloaded = 0;
+        try {
+            bytesTotal = getTotalBytes(this.url);
+        } catch (IOException e) {
+            logger.error("Failed to get file size at " + this.url, e);
+            observer.downloadErrored(this.url, "Failed to get file size of " + this.url);
+            return;
+        }
+        observer.setBytesTotal(bytesTotal);
+        observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal);
+        logger.info("Size of file at " + this.url + " = " + bytesTotal + "b");
+
+        int tries = 0; // Number of attempts to download
+        do {
+            InputStream bis = null; OutputStream fos = null;
+            byte[] data = new byte[1024]; int bytesRead;
+            try {
+                logger.info("    Downloading file: " + url + (tries > 0 ? " Retry #" + tries : ""));
+                observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm());
+                tries += 1;
+                bis = new BufferedInputStream(this.url.openStream());
+                fos = new FileOutputStream(saveAs);
+                while ( (bytesRead = bis.read(data)) != -1) {
+                    try {
+                        observer.stopCheck();
+                    } catch (IOException e) {
+                        observer.downloadErrored(url, "Download interrupted");
+                        return;
+                    }
+                    fos.write(data, 0, bytesRead);
+                    bytesDownloaded += bytesRead;
+                    observer.setBytesCompleted(bytesDownloaded);
+                    observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded);
+                }
+                bis.close();
+                fos.close();
+                break; // Download successful: break out of infinite loop
+            } catch (IOException e) {
+                logger.error("[!] Exception while downloading file: " + url + " - " + e.getMessage(), e);
+            } finally {
+                // Close any open streams
+                try {
+                    if (bis != null) { bis.close(); }
+                } catch (IOException e) { }
+                try {
+                    if (fos != null) { fos.close(); }
+                } catch (IOException e) { }
+            }
+            if (tries > this.retries) {
+                logger.error("[!] Exceeded maximum retries (" + this.retries + ") for URL " + url);
+                observer.downloadErrored(url, "Failed to download " + url.toExternalForm());
+                return;
+            }
+        } while (true);
+        observer.downloadCompleted(url, saveAs);
+        logger.info("[+] Saved " + url + " as " + this.prettySaveAs);
+    }
+    
+    private int getTotalBytes(URL url) throws IOException {
+        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+        conn.setRequestMethod("HEAD");
+        return conn.getContentLength();
+    }
+
+}

src/main/java/com/rarchives/ripme/ripper/VideoRipper.java

@@ -0,0 +1,121 @@
+package com.rarchives.ripme.ripper;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+
+import com.rarchives.ripme.ui.RipStatusMessage;
+import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
+import com.rarchives.ripme.utils.Utils;
+
+public abstract class VideoRipper extends AbstractRipper {
+
+    private int bytesTotal = 1,
+                 bytesCompleted = 1;
+
+    public VideoRipper(URL url) throws IOException {
+        super(url);
+    }
+
+    public abstract boolean canRip(URL url);
+    public abstract void rip() throws IOException;
+    public abstract String getHost();
+    public abstract String getGID(URL url) throws MalformedURLException;
+
+    @Override
+    public void setBytesTotal(int bytes) {
+        this.bytesTotal = bytes;
+    }
+    @Override
+    public void setBytesCompleted(int bytes) {
+        this.bytesCompleted = bytes;
+    }
+
+    @Override
+    public void addURLToDownload(URL url, File saveAs) {
+        threadPool.addThread(new DownloadVideoThread(url, saveAs, this));
+    }
+
+    @Override
+    public void setWorkingDir(URL url) throws IOException {
+        String path = Utils.getWorkingDirectory().getCanonicalPath();
+        if (!path.endsWith(File.separator)) {
+            path += File.separator;
+        }
+        path += "videos" + File.separator;
+        this.workingDir = new File(path);
+        if (!this.workingDir.exists()) {
+            logger.info("[+] Creating directory: " + Utils.removeCWD(this.workingDir));
+            this.workingDir.mkdirs();
+        }
+        logger.debug("Set working directory to: " + this.workingDir);
+    }
+
+    @Override
+    public int getCompletionPercentage() {
+        return (int) (100 * (bytesCompleted / (float) bytesTotal));
+    }
+
+    @Override
+    public void downloadCompleted(URL url, File saveAs) {
+        if (observer == null) {
+            return;
+        }
+        try {
+            String path = Utils.removeCWD(saveAs);
+            RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path);
+            observer.update(this, msg);
+
+            checkIfComplete();
+        } catch (Exception e) {
+            logger.error("Exception while updating observer: ", e);
+        }
+    }
+    @Override
+    public void downloadErrored(URL url, String reason) {
+        if (observer == null) {
+            return;
+        }
+        observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason));
+        checkIfComplete();
+    }
+    @Override
+    public void downloadProblem(URL url, String message) {
+        if (observer == null) {
+            return;
+        }
+        observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " : " + message));
+        checkIfComplete();
+    }
+
+    @Override
+    public String getStatusText() {
+        StringBuilder sb = new StringBuilder();
+        sb.append(getCompletionPercentage())
+          .append("% ")
+          .append(" - ")
+          .append(Utils.bytesToHumanReadable(bytesCompleted))
+          .append(" / ")
+          .append(Utils.bytesToHumanReadable(bytesTotal));
+        return sb.toString();
+    }
+
+    @Override
+    public URL sanitizeURL(URL url) throws MalformedURLException {
+        return url;
+    }
+
+    /**
+     * Notifies observers and updates state if all files have been ripped.
+     */
+    @Override
+    protected void checkIfComplete() {
+        if (observer == null) {
+            return;
+        }
+        if (bytesCompleted >= bytesTotal) {
+            super.checkIfComplete();
+        }
+    }
+}

src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java

@@ -13,9 +13,9 @@ import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
 
-public class ChanRipper extends AbstractRipper {
+public class ChanRipper extends AlbumRipper {
 
     private static final Logger logger = Logger.getLogger(ChanRipper.class);
 

src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java

@@ -18,10 +18,10 @@ import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
 import com.rarchives.ripme.utils.Utils;
 
-public class DeviantartRipper extends AbstractRipper {
+public class DeviantartRipper extends AlbumRipper {
 
     private static final String DOMAIN = "deviantart.com",
                                 HOST   = "deviantart";

src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java

@@ -12,9 +12,9 @@ import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
 
-public class EightmusesRipper extends AbstractRipper {
+public class EightmusesRipper extends AlbumRipper {
 
     private static final String DOMAIN = "8muses.com",
                                 HOST   = "8muses";

src/main/java/com/rarchives/ripme/ripper/rippers/GonewildRipper.java

@@ -11,10 +11,10 @@ import org.json.JSONArray;
 import org.json.JSONObject;
 import org.jsoup.Jsoup;
 
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
 import com.rarchives.ripme.utils.Utils;
 
-public class GonewildRipper extends AbstractRipper {
+public class GonewildRipper extends AlbumRipper {
 
     private static final String HOST   = "gonewild";
     private static final Logger logger = Logger.getLogger(GonewildRipper.class);

src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java

@@ -11,9 +11,9 @@ import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
 
-public class ImagearnRipper extends AbstractRipper {
+public class ImagearnRipper extends AlbumRipper {
 
     private static final String DOMAIN = "imagearn.com",
                                 HOST   = "imagearn";

src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java

@@ -11,9 +11,9 @@ import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
 
-public class ImagefapRipper extends AbstractRipper {
+public class ImagefapRipper extends AlbumRipper {
 
     private static final String DOMAIN = "imagefap.com",
                                 HOST   = "imagefap";

src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java

@@ -18,11 +18,11 @@ import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
 
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
 import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
 import com.rarchives.ripme.utils.Utils;
 
-public class ImgurRipper extends AbstractRipper {
+public class ImgurRipper extends AlbumRipper {
 
     private static final String DOMAIN = "imgur.com",
                                 HOST   = "imgur";

src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java

@@ -13,9 +13,9 @@ import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
 
-public class InstagramRipper extends AbstractRipper {
+public class InstagramRipper extends AlbumRipper {
 
     private static final String DOMAIN = "instagram.com",
                                 HOST   = "instagram";

src/main/java/com/rarchives/ripme/ripper/rippers/KinkyshareRipper.java

@@ -11,9 +11,9 @@ import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
 
-public class KinkyshareRipper extends AbstractRipper {
+public class KinkyshareRipper extends AlbumRipper {
 
     private static final String HOST   = "kinkyshare";
     private static final Logger logger = Logger.getLogger(KinkyshareRipper.class);

src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java

@@ -11,10 +11,10 @@ import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
 import com.rarchives.ripme.ripper.DownloadThreadPool;
 
-public class MotherlessRipper extends AbstractRipper {
+public class MotherlessRipper extends AlbumRipper {
 
     private static final String DOMAIN = "motherless.com",
                                 HOST   = "motherless";

src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java

@@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers;
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.SocketTimeoutException;
 import java.net.URL;
 import java.util.List;
 import java.util.regex.Matcher;
@@ -14,12 +15,11 @@ import org.json.JSONTokener;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
 import com.rarchives.ripme.utils.RipUtils;
 import com.rarchives.ripme.utils.Utils;
-import java.net.SocketTimeoutException;
 
-public class RedditRipper extends AbstractRipper {
+public class RedditRipper extends AlbumRipper {
 
     public RedditRipper(URL url) throws IOException {
         super(url);

src/main/java/com/rarchives/ripme/ripper/rippers/SeeniveRipper.java

@@ -12,10 +12,10 @@ import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
 import com.rarchives.ripme.ripper.DownloadThreadPool;
 
-public class SeeniveRipper extends AbstractRipper {
+public class SeeniveRipper extends AlbumRipper {
 
     private static final String DOMAIN = "seenive.com",
                                 HOST   = "seenive";

src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java

@@ -12,10 +12,10 @@ import org.json.JSONObject;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
 import com.rarchives.ripme.utils.Utils;
 
-public class TumblrRipper extends AbstractRipper {
+public class TumblrRipper extends AlbumRipper {
 
     private static final String DOMAIN = "tumblr.com",
                                 HOST   = "tumblr";

src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java

@@ -16,10 +16,10 @@ import org.json.JSONTokener;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
 import com.rarchives.ripme.utils.Utils;
 
-public class TwitterRipper extends AbstractRipper {
+public class TwitterRipper extends AlbumRipper {
 
     private static final String DOMAIN = "twitter.com",
                                 HOST   = "twitter";

src/main/java/com/rarchives/ripme/ripper/rippers/VineboxRipper.java

@@ -12,9 +12,9 @@ import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
 
-public class VineboxRipper extends AbstractRipper {
+public class VineboxRipper extends AlbumRipper {
 
     private static final String DOMAIN = "vinebox.co",
                                 HOST   = "vinebox";

src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java

@@ -18,9 +18,9 @@ import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
 
-public class VkRipper extends AbstractRipper {
+public class VkRipper extends AlbumRipper {
 
     private static final String DOMAIN = "vk.com",
                                 HOST   = "vk";

src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java

@@ -11,9 +11,9 @@ import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
 
-public class XhamsterRipper extends AbstractRipper {
+public class XhamsterRipper extends AlbumRipper {
 
     private static final String DOMAIN = "xhamster.com",
                                 HOST   = "xhamster";

src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java

@@ -0,0 +1,84 @@
+package com.rarchives.ripme.ripper.rippers.video;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.log4j.Logger;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+
+import com.rarchives.ripme.ripper.VideoRipper;
+
+public class XvideosRipper extends VideoRipper {
+
+    private static final String HOST = "xvideos";
+    private static final Logger logger = Logger.getLogger(XvideosRipper.class);
+
+    public XvideosRipper(URL url) throws IOException {
+        super(url);
+    }
+
+    @Override
+    public String getHost() {
+        return HOST;
+    }
+
+    @Override
+    public boolean canRip(URL url) {
+        Pattern p = Pattern.compile("^https?://[wm.]*xvideos\\.com/video[0-9]+.*$");
+        Matcher m = p.matcher(url.toExternalForm());
+        return m.matches();
+    }
+    
+    @Override
+    public String getAlbumTitle(URL url) {
+        return "videos";
+    }
+
+    @Override
+    public URL sanitizeURL(URL url) throws MalformedURLException {
+        return url;
+    }
+
+    @Override
+    public String getGID(URL url) throws MalformedURLException {
+        Pattern p = Pattern.compile("^https?://[wm.]*xvideos\\.com/video([0-9]+).*$");
+        Matcher m = p.matcher(url.toExternalForm());
+        if (m.matches()) {
+            return m.group(1);
+        }
+
+        throw new MalformedURLException(
+                "Expected xvideo format:"
+                        + "xvideos.com/video####"
+                        + " Got: " + url);
+    }
+
+    @Override
+    public void rip() throws IOException {
+        logger.info("    Retrieving " + this.url.toExternalForm());
+        Document doc = Jsoup.connect(this.url.toExternalForm())
+                            .userAgent(USER_AGENT)
+                            .get();
+        Elements embeds = doc.select("embed");
+        if (embeds.size() == 0) {
+            throw new IOException("Could not find Embed code at " + url);
+        }
+        Element embed = embeds.get(0);
+        String vars = embed.attr("flashvars");
+        for (String var : vars.split("&")) {
+            if (var.startsWith("flv_url=")) {
+                String vidUrl = var.substring("flv_url=".length());
+                vidUrl = URLDecoder.decode(vidUrl, "UTF-8");
+                addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url));
+            }
+        }
+        waitForThreads();
+    }
+}

src/main/java/com/rarchives/ripme/ui/MainWindow.java

@@ -721,6 +721,13 @@ public class MainWindow implements Runnable, RipStatusHandler {
                 }
             });
             mainFrame.pack();
+            break;
+        case COMPLETED_BYTES:
+            // Update completed bytes
+            break;
+        case TOTAL_BYTES:
+            // Update total bytes
+            break;
         }
     }
 

src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java

@@ -11,7 +11,9 @@ public class RipStatusMessage {
         DOWNLOAD_COMPLETE("Download Complete"),
         DOWNLOAD_ERRORED("Download Errored"),
         RIP_COMPLETE("Rip Complete"), 
-        DOWNLOAD_WARN("Download problem");
+        DOWNLOAD_WARN("Download problem"),
+        TOTAL_BYTES("Total bytes"),
+        COMPLETED_BYTES("Completed bytes");
 
         String value;
         STATUS(String value) {

src/main/java/com/rarchives/ripme/ui/UpdateUtils.java

@@ -19,7 +19,7 @@ import org.jsoup.nodes.Document;
 public class UpdateUtils {
 
     private static final Logger logger = Logger.getLogger(UpdateUtils.class);
-    private static final String DEFAULT_VERSION = "1.0.19";
+    private static final String DEFAULT_VERSION = "1.0.20";
     private static final String updateJsonURL = "http://rarchives.com/ripme.json";
     private static final String updateJarURL = "http://rarchives.com/ripme.jar";
     private static final String mainFileName = "ripme.jar";

src/main/java/com/rarchives/ripme/utils/Utils.java

@@ -247,4 +247,15 @@ public class Utils {
                    .replaceAll("__", "_")
                    .replaceAll("_+$", "");
     }
+    
+    public static String bytesToHumanReadable(int bytes) {
+        float fbytes = (float) bytes;
+        String[] mags = new String[] {"", "k", "m", "g", "t"};
+        int magIndex = 0;
+        while (fbytes >= 1024) {
+            fbytes /= 1024;
+            magIndex++;
+        }
+        return String.format("%.2f%sb", fbytes, mags[magIndex]);
+    }
 }

src/test/java/com/rarchives/ripme/tst/AppTest.java

@@ -4,6 +4,8 @@ import junit.framework.Test;
 import junit.framework.TestCase;
 import junit.framework.TestSuite;
 
+import com.rarchives.ripme.utils.Utils;
+
 public class AppTest extends TestCase {
     /**
      * Create the test case
@@ -25,6 +27,7 @@ public class AppTest extends TestCase {
      * Rigourous Test :-)
      */
     public void testApp() {
+        System.err.println(Utils.bytesToHumanReadable(1023 * 5000));
         assertTrue( true );
     }
 }

src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java

@@ -0,0 +1,32 @@
+package com.rarchives.ripme.tst.ripper.rippers;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.rarchives.ripme.ripper.rippers.video.XvideosRipper;
+
+public class VideoRippersTest extends RippersTest {
+    
+    public void testXvideosRipper() throws IOException {
+        if (false && !DOWNLOAD_CONTENT) {
+            return;
+        }
+        List<URL> contentURLs = new ArrayList<URL>();
+        contentURLs.add(new URL("http://www.xvideos.com/video1428195/stephanie_first_time_anal"));
+        contentURLs.add(new URL("http://www.xvideos.com/video7136868/vid-20140205-wa0011"));
+        for (URL url : contentURLs) {
+            try {
+                XvideosRipper ripper = new XvideosRipper(url);
+                ripper.rip();
+                assert(ripper.getWorkingDir().listFiles().length > 1);
+                deleteDir(ripper.getWorkingDir());
+            } catch (Exception e) {
+                e.printStackTrace();
+                fail("Error while ripping URL " + url + ": " + e.getMessage());
+            }
+        }
+    }
+
+}