From b7c4a25f7e3584a5baa47f8799ca0d2aa38f9496 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Wed, 6 Jun 2018 11:34:16 -0400
Subject: [PATCH 001/132] Removed questionable code which caused ripme to
 assume all sites with '/thread/ and '/res/' in the url are chans

---
 .../java/com/rarchives/ripme/ripper/rippers/ChanRipper.java    | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
index ace305c1..3a137123 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
@@ -85,8 +85,7 @@ public class ChanRipper extends AbstractHTMLRipper {
                 return true;
             }
         }
-        return  url.toExternalForm().contains("/res/")     // Most chans
-             || url.toExternalForm().contains("/thread/"); // 4chan, archive.moe
+        return false;
     }
 
     /**

From 8ea588a4a5f98e4722791c225a56a6a0197cbfe0 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Wed, 6 Jun 2018 11:35:39 -0400
Subject: [PATCH 002/132] Added support for new 4chan cdn domain

---
 .../java/com/rarchives/ripme/ripper/rippers/ChanRipper.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
index 3a137123..be0a4cf5 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
@@ -19,7 +19,7 @@ import com.rarchives.ripme.utils.RipUtils;
 
 public class ChanRipper extends AbstractHTMLRipper {
     private static List<ChanSite> explicit_domains = Arrays.asList(
-        new ChanSite(Arrays.asList("boards.4chan.org"),   Arrays.asList("4cdn.org", "is.4chan.org", "is2.4chan.org")),
+        new ChanSite(Arrays.asList("boards.4chan.org"),   Arrays.asList("4cdn.org", "is.4chan.org", "is2.4chan.org", "is3.4chan.org")),
         new ChanSite(Arrays.asList("4archive.org"),       Arrays.asList("imgur.com")),
         new ChanSite(Arrays.asList("archive.4plebs.org"), Arrays.asList("img.4plebs.org"))
         );

From dad016e5be63041d4680027c3c9e4e8053f87301 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Thu, 7 Jun 2018 05:08:09 -0400
Subject: [PATCH 003/132] Readded support desuchan.net

---
 .../java/com/rarchives/ripme/ripper/rippers/ChanRipper.java    | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
index be0a4cf5..33725f39 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
@@ -85,6 +85,9 @@ public class ChanRipper extends AbstractHTMLRipper {
                 return true;
             }
         }
+        if (url.toExternalForm().contains("desuchan.net") && url.toExternalForm().contains("/res/")) {
+            return true;
+        }
         return false;
     }
 

From 1b5b58ebb4d7575feeafadc98c1b7d03d6759c5f Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Fri, 8 Jun 2018 16:52:33 -0400
Subject: [PATCH 004/132] Removed shared account due to privacy concerns

---
 .../ripme/ripper/rippers/FuraffinityRipper.java    | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java
index 440e9ae5..ed42ffcc 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java
@@ -29,8 +29,11 @@ public class FuraffinityRipper extends AbstractHTMLRipper {
     private static final String urlBase = "https://www.furaffinity.net";
     private static Map<String,String> cookies = new HashMap<>();
     static {
-        cookies.put("b", "bd5ccac8-51dc-4265-8ae1-7eac685ad667");
-        cookies.put("a", "7c41b782-d01d-4b0e-b45b-62a4f0b2a369");
+        if (Utils.getConfigString("furaffinity.cookie.a", "") != "" && Utils.getConfigString("furaffinity.cookie.b", "") != "") {
+            LOGGER.info("Logging in using cookies");
+            cookies.put("b", Utils.getConfigString("furaffinity.cookie.b", ""));
+            cookies.put("a", Utils.getConfigString("furaffinity.cookie.a", ""));
+        }
     }
 
     // Thread pool for finding direct image links from "image" pages (html)
@@ -91,9 +94,12 @@ public class FuraffinityRipper extends AbstractHTMLRipper {
     @Override
     public List<String> getURLsFromPage(Document page) {
         List<String> urls = new ArrayList<>();
-        Elements urlElements = page.select("figure.t-image > b > u > a");
+        Elements urlElements = page.select("figure > b > u > a");
         for (Element e : urlElements) {
-            urls.add(getImageFromPost(urlBase + e.select("a").first().attr("href")));
+            String urlToAdd = getImageFromPost(urlBase + e.select("a").first().attr("href"));
+            if (urlToAdd.startsWith("http")) {
+                urls.add(urlToAdd);
+            }
         }
         return urls;
     }

From cbd1372ba4c02ae221f35978189547402f12e6fd Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sat, 9 Jun 2018 02:46:46 -0400
Subject: [PATCH 005/132] Readded shared account cookies; added option to not
 login to Furaffinity when ripping

---
 .../ripper/rippers/FuraffinityRipper.java     | 19 ++++---------------
 1 file changed, 4 insertions(+), 15 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java
index ed42ffcc..2131567b 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java
@@ -29,10 +29,10 @@ public class FuraffinityRipper extends AbstractHTMLRipper {
     private static final String urlBase = "https://www.furaffinity.net";
     private static Map<String,String> cookies = new HashMap<>();
     static {
-        if (Utils.getConfigString("furaffinity.cookie.a", "") != "" && Utils.getConfigString("furaffinity.cookie.b", "") != "") {
+        if (Utils.getConfigBoolean("furaffinity.login", true)) {
             LOGGER.info("Logging in using cookies");
-            cookies.put("b", Utils.getConfigString("furaffinity.cookie.b", ""));
-            cookies.put("a", Utils.getConfigString("furaffinity.cookie.a", ""));
+            cookies.put("a", Utils.getConfigString("furaffinity.cookie.a", "897bc45b-1f87-49f1-8a85-9412bc103e7a"));
+            cookies.put("b", Utils.getConfigString("furaffinity.cookie.b", "c8807f36-7a85-4caf-80ca-01c2a2368267"));
         }
     }
 
@@ -97,7 +97,7 @@ public class FuraffinityRipper extends AbstractHTMLRipper {
         Elements urlElements = page.select("figure > b > u > a");
         for (Element e : urlElements) {
             String urlToAdd = getImageFromPost(urlBase + e.select("a").first().attr("href"));
-            if (urlToAdd.startsWith("http")) {
+            if (urlToAdd.startsWith("http") && urlToAdd.contains("/view/")) {
                 urls.add(urlToAdd);
             }
         }
@@ -206,16 +206,5 @@ public class FuraffinityRipper extends AbstractHTMLRipper {
                 + " instead");
     }
 
-    private class FuraffinityDocumentThread extends Thread {
-        private URL url;
-
-        FuraffinityDocumentThread(URL url) {
-            super();
-            this.url = url;
-        }
-
-
-    }
-
 
 }
\ No newline at end of file

From a753b016f622999926f4fbf4c5c90f62d4563a59 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sat, 9 Jun 2018 03:01:32 -0400
Subject: [PATCH 006/132] Ripme now warns users about using the shared
 Furaffinity account

---
 .../ripper/rippers/FuraffinityRipper.java     | 21 +++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java
index 2131567b..eecde5dd 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java
@@ -12,6 +12,7 @@ import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import com.rarchives.ripme.ui.RipStatusMessage;
 import com.rarchives.ripme.utils.Utils;
 import org.jsoup.Connection.Response;
 import org.jsoup.Jsoup;
@@ -27,12 +28,23 @@ import com.rarchives.ripme.utils.Http;
 public class FuraffinityRipper extends AbstractHTMLRipper {
 
     private static final String urlBase = "https://www.furaffinity.net";
-    private static Map<String,String> cookies = new HashMap<>();
-    static {
+    private  Map<String,String> cookies = new HashMap<>();
+
+    private void setCookies() {
         if (Utils.getConfigBoolean("furaffinity.login", true)) {
             LOGGER.info("Logging in using cookies");
-            cookies.put("a", Utils.getConfigString("furaffinity.cookie.a", "897bc45b-1f87-49f1-8a85-9412bc103e7a"));
-            cookies.put("b", Utils.getConfigString("furaffinity.cookie.b", "c8807f36-7a85-4caf-80ca-01c2a2368267"));
+            String faACookie = Utils.getConfigString("furaffinity.cookie.a", "897bc45b-1f87-49f1-8a85-9412bc103e7a");
+            String faBCookie = Utils.getConfigString("furaffinity.cookie.b", "c8807f36-7a85-4caf-80ca-01c2a2368267");
+            warnAboutSharedAccount(faACookie, faBCookie);
+            cookies.put("a", faACookie);
+            cookies.put("b", faBCookie);
+        }
+    }
+
+    private void warnAboutSharedAccount(String a, String b) {
+        if (a.equals("897bc45b-1f87-49f1-8a85-9412bc103e7a") && b.equals("c8807f36-7a85-4caf-80ca-01c2a2368267")) {
+            sendUpdate(RipStatusMessage.STATUS.DOWNLOAD_ERRORED,
+                    "WARNING: Using the shared furaffinity account exposes both your IP and how many items you downloaded to the other users of the share account");
         }
     }
 
@@ -64,6 +76,7 @@ public class FuraffinityRipper extends AbstractHTMLRipper {
     }
     @Override
     public Document getFirstPage() throws IOException {
+        setCookies();
         return Http.url(url).cookies(cookies).get();
     }
 

From c77593570b16a11af26ccd3ce119bbfb26960a95 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sat, 9 Jun 2018 15:05:21 -0400
Subject: [PATCH 007/132] Started work on getting DA login working again

---
 .../ripme/ripper/rippers/DeviantartRipper.java  | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
index 99942adb..e23e7478 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
@@ -437,15 +437,19 @@ public class DeviantartRipper extends AbstractHTMLRipper {
         if (username == null || password == null) {
             throw new IOException("could not find username or password in config");
         }
-        Response resp = Http.url("http://www.deviantart.com/")
+        Response resp = Http.url("http://www.deviantart.com/users/login")
                             .response();
-        for (Element input : resp.parse().select("form#form-login input[type=hidden]")) {
-            postData.put(input.attr("name"), input.attr("value"));
-        }
+//        for (Element input : resp.parse().select("form#form-login input[type=hidden]")) {
+//            postData.put(input.attr("name"), input.attr("value"));
+//        }
+        Document r = resp.parse();
+        LOGGER.info("R: " + r.html());
         postData.put("username", username);
         postData.put("password", password);
-        postData.put("remember_me", "1");
-
+        postData.put("validate_token", "829439d1f0b53e20f2f5");
+        postData.put("validate_key", "1528530690");
+//        postData.put("remember_me", "1");
+        LOGGER.info(postData);
         // Send login request
         resp = Http.url("https://www.deviantart.com/users/login")
                     .userAgent(USER_AGENT)
@@ -453,6 +457,7 @@ public class DeviantartRipper extends AbstractHTMLRipper {
                     .cookies(resp.cookies())
                     .method(Method.POST)
                     .response();
+//        LOGGER.info("RESP: " + resp.parse().html());
 
         // Assert we are logged in
         if (resp.hasHeader("Location") && resp.header("Location").contains("password")) {

From 8df9c70d6c60523016f1042d5444866623ab78a7 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sat, 9 Jun 2018 15:12:39 -0400
Subject: [PATCH 008/132] Fixed failing tests on chanripper

---
 .../com/rarchives/ripme/ripper/rippers/ChanRipper.java   | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
index f5401d35..0ab86244 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
@@ -88,6 +88,15 @@ public class ChanRipper extends AbstractHTMLRipper {
         if (url.toExternalForm().contains("desuchan.net") && url.toExternalForm().contains("/res/")) {
             return true;
         }
+        if (url.toExternalForm().contains("boards.420chan.org") && url.toExternalForm().contains("/res/")) {
+            return true;
+        }
+        if (url.toExternalForm().contains("7chan.org") && url.toExternalForm().contains("/res/")) {
+            return true;
+        }
+        if (url.toExternalForm().contains("xchan.pw") && url.toExternalForm().contains("/board/")) {
+            return true;
+        }
         return false;
     }
 

From 2c5a9fdac37bb4ca9f4f8d82a445444cfc78207b Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sun, 10 Jun 2018 12:12:29 -0400
Subject: [PATCH 009/132] Added getCookiesFromString func to turn a cookie
 string into a hashMap

---
 .../java/com/rarchives/ripme/utils/RipUtils.java | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java
index 01d20e7c..84b17ca1 100644
--- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java
+++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java
@@ -3,9 +3,7 @@ package com.rarchives.ripme.utils;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
+import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -279,4 +277,16 @@ public class RipUtils {
         }
         return url;
     }
+    /**
+     * Reads a cookie string (Key1=value1;key2=value2) from the config file and turns it into a hashmap
+     * @return Map of cookies containing session data.
+     */
+    public static Map<String, String> getCookiesFromString(String line) {
+        Map<String,String> cookies = new HashMap<>();
+        for (String pair : line.split(";")) {
+            String[] kv = pair.split("=");
+            cookies.put(kv[0], kv[1]);
+        }
+        return cookies;
+    }
 }

From 312e7e5c3316fd6e55e56a4705326ab33bf531bb Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sun, 10 Jun 2018 12:12:51 -0400
Subject: [PATCH 010/132] DeviantartRipper now logs in using cookies

---
 .../ripper/rippers/DeviantartRipper.java      | 81 +++++--------------
 1 file changed, 20 insertions(+), 61 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
index e23e7478..cb1c9614 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
@@ -3,6 +3,7 @@ package com.rarchives.ripme.ripper.rippers;
 import com.rarchives.ripme.ripper.AbstractHTMLRipper;
 import com.rarchives.ripme.utils.Base64;
 import com.rarchives.ripme.utils.Http;
+import com.rarchives.ripme.utils.RipUtils;
 import com.rarchives.ripme.utils.Utils;
 import java.io.IOException;
 import java.net.HttpURLConnection;
@@ -37,6 +38,10 @@ public class DeviantartRipper extends AbstractHTMLRipper {
         super(url);
     }
 
+    String loginCookies = "auth=__0f9158aaec09f417b235%3B%221ff79836392a515d154216d919eae573%22;" +
+            "auth_secure=__41d14dd0da101f411bb0%3B%2281cf2cf9477776162a1172543aae85ce%22;" +
+            "userinfo=__bf84ac233bfa8ae642e8%3B%7B%22username%22%3A%22grabpy%22%2C%22uniqueid%22%3A%22a0a876aa37dbd4b30e1c80406ee9c280%22%2C%22vd%22%3A%22BbHUXZ%2CBbHUXZ%2CA%2CU%2CA%2C%2CB%2CA%2CB%2CBbHUXZ%2CBbHUdj%2CL%2CL%2CA%2CBbHUdj%2C13%2CA%2CB%2CA%2C%2CA%2CA%2CB%2CA%2CA%2C%2CA%22%2C%22attr%22%3A56%7D";
+
     @Override
     public String getHost() {
         return "deviantart";
@@ -117,24 +122,20 @@ public class DeviantartRipper extends AbstractHTMLRipper {
     @Override
     public Document getFirstPage() throws IOException {
         
-        //Test to see if there is a login:
-        String username = Utils.getConfigString("deviantart.username", new String(Base64.decode("Z3JhYnB5")));
-        String password = Utils.getConfigString("deviantart.password", new String(Base64.decode("ZmFrZXJz")));
-        
-        if (username == null || password == null) {
-            LOGGER.debug("No DeviantArt login provided.");
-            cookies.put("agegate_state","1"); // Bypasses the age gate
-        } else {
-            // Attempt Login
-            try {
-                cookies = loginToDeviantart();
-            } catch (IOException e) {
-                LOGGER.warn("Failed to login: ", e);
+        // Base64 da login
+        // username: Z3JhYnB5
+        // password: ZmFrZXJz
+
+
+        cookies = getDACookies();
+            if (cookies.isEmpty()) {
+                LOGGER.warn("Failed to get login cookies");
                 cookies.put("agegate_state","1"); // Bypasses the age gate
             }
-        }
-        
-        
+
+        LOGGER.info(Http.url(this.url)
+                .cookies(cookies)
+                .get());
         return Http.url(this.url)
                    .cookies(cookies)
                    .get();
@@ -426,52 +427,10 @@ public class DeviantartRipper extends AbstractHTMLRipper {
     }
 
     /**
-     * Logs into deviant art. Required to rip full-size NSFW content.
+     * Returns DA cookies.
      * @return Map of cookies containing session data.
      */
-    private Map<String, String> loginToDeviantart() throws IOException {
-        // Populate postData fields
-        Map<String,String> postData = new HashMap<>();
-        String username = Utils.getConfigString("deviantart.username", new String(Base64.decode("Z3JhYnB5")));
-        String password = Utils.getConfigString("deviantart.password", new String(Base64.decode("ZmFrZXJz")));
-        if (username == null || password == null) {
-            throw new IOException("could not find username or password in config");
-        }
-        Response resp = Http.url("http://www.deviantart.com/users/login")
-                            .response();
-//        for (Element input : resp.parse().select("form#form-login input[type=hidden]")) {
-//            postData.put(input.attr("name"), input.attr("value"));
-//        }
-        Document r = resp.parse();
-        LOGGER.info("R: " + r.html());
-        postData.put("username", username);
-        postData.put("password", password);
-        postData.put("validate_token", "829439d1f0b53e20f2f5");
-        postData.put("validate_key", "1528530690");
-//        postData.put("remember_me", "1");
-        LOGGER.info(postData);
-        // Send login request
-        resp = Http.url("https://www.deviantart.com/users/login")
-                    .userAgent(USER_AGENT)
-                    .data(postData)
-                    .cookies(resp.cookies())
-                    .method(Method.POST)
-                    .response();
-//        LOGGER.info("RESP: " + resp.parse().html());
-
-        // Assert we are logged in
-        if (resp.hasHeader("Location") && resp.header("Location").contains("password")) {
-            // Wrong password
-            throw new IOException("Wrong password");
-        }
-        if (resp.url().toExternalForm().contains("bad_form")) {
-            throw new IOException("Login form was incorrectly submitted");
-        }
-        if (resp.cookie("auth_secure") == null ||
-            resp.cookie("auth") == null) {
-            throw new IOException("No auth_secure or auth cookies received");
-        }
-        // We are logged in, save the cookies
-        return resp.cookies();
+    private Map<String, String> getDACookies() {
+        return RipUtils.getCookiesFromString(Utils.getConfigString("deviantart.cookies", loginCookies));
     }
 }
\ No newline at end of file

From 00f1b8a5e9c0fafbc38f495dffc831afe20306d6 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Mon, 11 Jun 2018 08:31:47 -0400
Subject: [PATCH 011/132] FurAffinityRipper now downloads fullsized images and
 NSFW images

---
 .../ripper/rippers/FuraffinityRipper.java     | 25 ++++++++++++++-----
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java
index eecde5dd..7dd5b731 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java
@@ -15,6 +15,7 @@ import java.util.regex.Pattern;
 import com.rarchives.ripme.ui.RipStatusMessage;
 import com.rarchives.ripme.utils.Utils;
 import org.jsoup.Connection.Response;
+import org.jsoup.HttpStatusException;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
@@ -77,6 +78,7 @@ public class FuraffinityRipper extends AbstractHTMLRipper {
     @Override
     public Document getFirstPage() throws IOException {
         setCookies();
+        LOGGER.info(Http.url(url).cookies(cookies).get().html());
         return Http.url(url).cookies(cookies).get();
     }
 
@@ -96,22 +98,33 @@ public class FuraffinityRipper extends AbstractHTMLRipper {
     }
 
     private String getImageFromPost(String url) {
+        sleep(1000);
+        Document d = null;
         try {
-            LOGGER.info("found url " + Http.url(url).cookies(cookies).get().select("meta[property=og:image]").attr("content"));
-            return Http.url(url).cookies(cookies).get().select("meta[property=og:image]").attr("content");
+            d = Http.url(url).cookies(cookies).get();
+            Elements links = d.getElementsByTag("a");
+            for (Element link : links) {
+                if (link.text().equals("Download")) {
+                    LOGGER.info("Found image " + link.attr("href"));
+                   return "https:" + link.attr("href");
+                }
+            }
         } catch (IOException e) {
-            return "";
+            return null;
         }
+        return null;
     }
 
     @Override
     public List<String> getURLsFromPage(Document page) {
         List<String> urls = new ArrayList<>();
-        Elements urlElements = page.select("figure > b > u > a");
+        Elements urlElements = page.select("figure.t-image > b > u > a");
         for (Element e : urlElements) {
             String urlToAdd = getImageFromPost(urlBase + e.select("a").first().attr("href"));
-            if (urlToAdd.startsWith("http") && urlToAdd.contains("/view/")) {
-                urls.add(urlToAdd);
+            if (url != null) {
+                if (urlToAdd.startsWith("http")) {
+                    urls.add(urlToAdd);
+                }
             }
         }
         return urls;

From 7d20b3927d8a45e012bef64025b798c6c53f7d64 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Mon, 11 Jun 2018 09:22:44 -0400
Subject: [PATCH 012/132] release script now makes sure update file is named
 ripme.jar; added --test option for dry runs

---
 release.py      | 34 ++++++++++++++++++++++++----------
 releaseTests.py |  0
 2 files changed, 24 insertions(+), 10 deletions(-)
 create mode 100644 releaseTests.py

diff --git a/release.py b/release.py
index 3dcd3210..9b199bf6 100644
--- a/release.py
+++ b/release.py
@@ -15,9 +15,11 @@ parser.add_argument("-f", "--file", help="Path to the version of ripme to releas
 parser.add_argument("-t", "--token", help="Your github personal access token")
 parser.add_argument("-d", "--debug", help="Run in debug mode", action="store_true")
 parser.add_argument("-n", "--non-interactive", help="Do not ask for any input from the user", action="store_true")
+parser.add_argument("--test", help="Perform a dry run", action="store_true")
 args = parser.parse_args()
 
 try:
+    # This binds input to raw_input on python2, we do this because input acts like eval on python2
     input = raw_input
 except NameError:
     pass
@@ -38,9 +40,20 @@ def isValidCommitMessage(message):
     pattern = re.compile("^\d+\.\d+\.\d+:")
     return re.match(pattern, message)
 
+# Checks if the update has the name ripme.jar, if not it renames the file
+# Returns the update file path
+def renameFile(path):
+    if not path.endswith("ripme.jar"):
+        print("Specified file is not named ripme.jar, renaming")
+        # os.sep is the path separator for the os this is being run on
+        os.rename(path, path.split(os.sep)[0] + os.sep + "ripme.jar")
+        return path.split(os.sep)[0] + os.sep + "ripme.jar"
+    return path
+
+
 
 ripmeJson = json.loads(open("ripme.json").read())
-fileToUploadPath = args.file
+fileToUploadPath = renameFile(args.file)
 InNoninteractiveMode = args.non_interactive
 commitMessage = ripmeJson.get("changeList")[0]
 releaseVersion = ripmeJson.get("latestVersion")
@@ -63,11 +76,12 @@ if not isValidCommitMessage(commitMessage):
 
 ripmeUpdate = open(fileToUploadPath, mode='rb').read()
 
+# The actual hash of the file on disk
+actualHash = sha256(ripmeUpdate).hexdigest()
+
 # The hash that we expect the update to have
 expectedHash = ripmeJson.get("currentHash")
 
-# The actual hash of the file on disk
-actualHash = sha256(ripmeUpdate).hexdigest()
 
 # Make sure that the hash of the file we're uploading matches the hash in ripme.json. These hashes not matching will
 # cause ripme to refuse to install the update for all users who haven't disabled update hash checking
@@ -85,12 +99,12 @@ if not InNoninteractiveMode:
     print("Repo: {}/{}".format(repoOwner, repoName))
     input("\nPlease review the information above and ensure it is correct and then press enter")
 
-print("Accessing github using token")
-g = Github(accessToken)
+if args.test:
+    print("Accessing github using token")
+    g = Github(accessToken)
 
+    print("Creating release")
+    release = g.get_user(repoOwner).get_repo(repoName).create_git_release(releaseVersion, commitMessage, "")
 
-print("Creating release")
-release = g.get_user(repoOwner).get_repo(repoName).create_git_release(releaseVersion, commitMessage, "")
-
-print("Uploading file")
-release.upload_asset(fileToUploadPath, "ripme.jar")
+    print("Uploading file")
+    release.upload_asset(fileToUploadPath, "ripme.jar")
diff --git a/releaseTests.py b/releaseTests.py
new file mode 100644
index 00000000..e69de29b

From 80e63aa291592b30baf31522d8f2fb7b87f412d9 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Mon, 11 Jun 2018 19:10:09 -0400
Subject: [PATCH 013/132] [release.py] Fixed bug that caused releases to be
 uploaded only during dry runs; Added --skip-hash-check flag to skip the hash
 check; Minor formatting improvements

---
 release.py | 40 ++++++++++++++++++++++++----------------
 1 file changed, 24 insertions(+), 16 deletions(-)

diff --git a/release.py b/release.py
index 9b199bf6..04ad7e08 100644
--- a/release.py
+++ b/release.py
@@ -15,7 +15,8 @@ parser.add_argument("-f", "--file", help="Path to the version of ripme to releas
 parser.add_argument("-t", "--token", help="Your github personal access token")
 parser.add_argument("-d", "--debug", help="Run in debug mode", action="store_true")
 parser.add_argument("-n", "--non-interactive", help="Do not ask for any input from the user", action="store_true")
-parser.add_argument("--test", help="Perform a dry run", action="store_true")
+parser.add_argument("--test", help="Perform a dry run (Do everything but upload new release)", action="store_true")
+parser.add_argument("--skip-hash-check", help="Skip hash check (This should only be used for testing)", action="store_true")
 args = parser.parse_args()
 
 try:
@@ -40,8 +41,10 @@ def isValidCommitMessage(message):
     pattern = re.compile("^\d+\.\d+\.\d+:")
     return re.match(pattern, message)
 
+
 # Checks if the update has the name ripme.jar, if not it renames the file
 # Returns the update file path
+# TODO handle files that aren't in sub dirs
 def renameFile(path):
     if not path.endswith("ripme.jar"):
         print("Specified file is not named ripme.jar, renaming")
@@ -51,7 +54,6 @@ def renameFile(path):
     return path
 
 
-
 ripmeJson = json.loads(open("ripme.json").read())
 fileToUploadPath = renameFile(args.file)
 InNoninteractiveMode = args.non_interactive
@@ -74,23 +76,27 @@ if not isValidCommitMessage(commitMessage):
     print("[!] Error: {} is not a valid commit message as it does not start with a version".format(fileToUploadPath))
     sys.exit(1)
 
-ripmeUpdate = open(fileToUploadPath, mode='rb').read()
 
-# The actual hash of the file on disk
-actualHash = sha256(ripmeUpdate).hexdigest()
+if not args.skip_hash_check:
+    if debug:
+        print("Reading file {}".format(fileToUploadPath))
+    ripmeUpdate = open(fileToUploadPath, mode='rb').read()
 
-# The hash that we expect the update to have
-expectedHash = ripmeJson.get("currentHash")
+    # The actual hash of the file on disk
+    actualHash = sha256(ripmeUpdate).hexdigest()
 
+    # The hash that we expect the update to have
+    expectedHash = ripmeJson.get("currentHash")
 
-# Make sure that the hash of the file we're uploading matches the hash in ripme.json. These hashes not matching will
-# cause ripme to refuse to install the update for all users who haven't disabled update hash checking
-if expectedHash != actualHash:
-    print("[!] Error: expected hash of file and actual hash differ")
-    print("[!] Expected hash is {}".format(expectedHash))
-    print("[!] Actual hash is {}".format(actualHash))
-    sys.exit(1)
-
+    # Make sure that the hash of the file we're uploading matches the hash in ripme.json. These hashes not matching will
+    # cause ripme to refuse to install the update for all users who haven't disabled update hash checking
+    if expectedHash != actualHash:
+        print("[!] Error: expected hash of file and actual hash differ")
+        print("[!] Expected hash is {}".format(expectedHash))
+        print("[!] Actual hash is {}".format(actualHash))
+        sys.exit(1)
+else:
+    print("[*] WARNING: SKIPPING HASH CHECK")
 # Ask the user to review the information before we precede
 # This only runs in we're in interactive mode
 if not InNoninteractiveMode:
@@ -99,7 +105,7 @@ if not InNoninteractiveMode:
     print("Repo: {}/{}".format(repoOwner, repoName))
     input("\nPlease review the information above and ensure it is correct and then press enter")
 
-if args.test:
+if not args.test:
     print("Accessing github using token")
     g = Github(accessToken)
 
@@ -108,3 +114,5 @@ if args.test:
 
     print("Uploading file")
     release.upload_asset(fileToUploadPath, "ripme.jar")
+else:
+    print("Not uploading release being script was run with --test flag")

From d2d59e73cdac1d1c15135dad5b3931f2478ba6ec Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Mon, 11 Jun 2018 19:10:36 -0400
Subject: [PATCH 014/132] Remove empty file

---
 releaseTests.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 releaseTests.py

diff --git a/releaseTests.py b/releaseTests.py
deleted file mode 100644
index e69de29b..00000000

From 529347a5536ab2657b81d775e988df8d3584cc96 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Tue, 12 Jun 2018 16:26:23 -0400
Subject: [PATCH 015/132] Tumblr ripper no longer tries to download images in
 fullsize if the image doesn't support it

---
 .../com/rarchives/ripme/ripper/rippers/TumblrRipper.java  | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java
index b7a437b2..2760ab62 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java
@@ -235,8 +235,12 @@ public class TumblrRipper extends AlbumRipper {
                 for (int j = 0; j < photos.length(); j++) {
                     photo = photos.getJSONObject(j);
                     try {
-                        if (Utils.getConfigBoolean("tumblr.get_raw_image", false)) {
-                            String urlString = photo.getJSONObject("original_size").getString("url").replaceAll("https", "http");
+                        String imageUrl = photo.getJSONObject("original_size").getString("url");
+                        // If the url is shorter than 65 chars long we skip it because it's those images don't support grabbing them in fullsize
+                        if (Utils.getConfigBoolean("tumblr.get_raw_image", false) &&
+                                imageUrl.replaceAll("https", "http").length() > 65) {
+                            // We have to change the link to http because tumblr uses an invalid cert for data.tumblr.com
+                            String urlString = imageUrl.replaceAll("https", "http");
                             urlString = urlString.replaceAll("https?://[a-sA-Z0-9_\\-\\.]*\\.tumblr", "http://data.tumblr");
                             urlString = urlString.replaceAll("_\\d+\\.", "_raw.");
                             fileURL = new URL(urlString);

From 723473180fe0a9b76ce5aabffece5951dab306e6 Mon Sep 17 00:00:00 2001
From: kravenor <baldelli.simone@gmail.com>
Date: Wed, 13 Jun 2018 15:05:22 +0200
Subject: [PATCH 016/132] Create LabelsBundle_it_IT.properties

This is the italian translation
---
 .../resources/LabelsBundle_it_IT.properties   | 37 +++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100644 src/main/resources/LabelsBundle_it_IT.properties

diff --git a/src/main/resources/LabelsBundle_it_IT.properties b/src/main/resources/LabelsBundle_it_IT.properties
new file mode 100644
index 00000000..57953c71
--- /dev/null
+++ b/src/main/resources/LabelsBundle_it_IT.properties
@@ -0,0 +1,37 @@
+Log = Log
+History = Cronologia
+created = creato
+modified = modificato
+Queue = Coda
+Configuration = Configurazione
+
+# Keys for the Configuration menu
+
+current.version = Versione Corrente
+check.for.updates = Controlla Aggiornamenti
+auto.update = Aggiornamento automatico?
+max.download.threads = Thread massini
+timeout.mill = Timeout (in millisecondi):
+retry.download.count = Tentativi di download
+overwrite.existing.files = Sovrascrivi file esistenti?
+sound.when.rip.completes = Suono al terminie
+preserve.order = Conserva ordine
+save.logs = Salva log
+notification.when.rip.starts = Notifica inizio
+save.urls.only = Salva solo URL
+save.album.titles = Salva titoli album
+autorip.from.clipboard = Scarica da appunti
+save.descriptions = Salva descrizioni
+prefer.mp4.over.gif = Preferisci MP4 a GIF
+restore.window.position = Ripristina posizione della finestra
+remember.url.history = Riscorda la cronologia degli URL
+loading.history.from = Carica cronologia da
+
+# Misc UI keys
+
+loading.history.from.configuration = Caricamento cronologia da configurazione
+interrupted.while.waiting.to.rip.next.album = Interrotto mentre scaricavo album successivo
+inactive = Inattivo
+re-rip.checked = Re-rip selezionato
+remove = Rimuovi
+clear = Pulisci

From 135e55fb0e4fe0a6da56e9c6ef4f541af3f90fd8 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Thu, 14 Jun 2018 20:25:23 -0400
Subject: [PATCH 017/132] E621 ripper now accepts urls with order:Score at the
 end

---
 .../ripme/ripper/rippers/E621Ripper.java      | 30 +++++++++----------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java
index 2f47a0be..d7f4d961 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java
@@ -96,40 +96,40 @@ public class E621Ripper extends AbstractHTMLRipper{
 
 	private String getTerm(URL url) throws MalformedURLException{
 		if(gidPattern==null)
-			gidPattern=Pattern.compile("^https?://(www\\.)?e621\\.net/post/index/[^/]+/([a-zA-Z0-9$_.+!*'(),%-]+)(/.*)?(#.*)?$");
+			gidPattern=Pattern.compile("^https?://(www\\.)?e621\\.net/post/index/[^/]+/([a-zA-Z0-9$_.+!*'():,%\\-]+)(/.*)?(#.*)?$");
 		if(gidPatternPool==null)
-			gidPatternPool=Pattern.compile("^https?://(www\\.)?e621\\.net/pool/show/([a-zA-Z0-9$_.+!*'(),%-]+)(\\?.*)?(/.*)?(#.*)?$");
+			gidPatternPool=Pattern.compile("^https?://(www\\.)?e621\\.net/pool/show/([a-zA-Z0-9$_.+!*'(),%:\\-]+)(\\?.*)?(/.*)?(#.*)?$");
 
 		Matcher m = gidPattern.matcher(url.toExternalForm());
-		if(m.matches())
-			return m.group(2);
+		if(m.matches()) {
+            LOGGER.info(m.group(2));
+            return m.group(2);
+        }
 
 		m = gidPatternPool.matcher(url.toExternalForm());
-		if(m.matches())
-			return m.group(2);
+		if(m.matches()) {
+            return m.group(2);
+        }
 
 		throw new MalformedURLException("Expected e621.net URL format: e621.net/post/index/1/searchterm - got "+url+" instead");
 	}
 
 	@Override
 	public String getGID(URL url) throws MalformedURLException {
-		try {
+
 			String prefix="";
-			if(url.getPath().startsWith("/pool/show/"))
-				prefix="pool_";
+			if (url.getPath().startsWith("/pool/show/")) {
+                prefix = "pool_";
+            }
 
-			return Utils.filesystemSafe(prefix+new URI(getTerm(url)).getPath());
-		} catch (URISyntaxException ex) {
-			logger.error(ex);
-		}
+			return Utils.filesystemSafe(prefix+getTerm(url));
 
-		throw new MalformedURLException("Expected e621.net URL format: e621.net/post/index/1/searchterm - got "+url+" instead");
 	}
 
 	@Override
 	public URL sanitizeURL(URL url) throws MalformedURLException {
 		if(gidPattern2==null)
-			gidPattern2=Pattern.compile("^https?://(www\\.)?e621\\.net/post/search\\?tags=([a-zA-Z0-9$_.+!*'(),%-]+)(/.*)?(#.*)?$");
+			gidPattern2=Pattern.compile("^https?://(www\\.)?e621\\.net/post/search\\?tags=([a-zA-Z0-9$_.+!*'():,%-]+)(/.*)?(#.*)?$");
 
 		Matcher m = gidPattern2.matcher(url.toExternalForm());
 		if(m.matches())

From e6f30670155819eaa0f9fc8f0e0549449c84d43b Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Thu, 14 Jun 2018 22:18:17 -0400
Subject: [PATCH 018/132] XhamsterRipper now accepts all countries subdomains

---
 .../java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java
index 385f6d1f..b4529f9b 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java
@@ -62,7 +62,7 @@ public class XhamsterRipper extends AbstractHTMLRipper {
 
     @Override
     public boolean canRip(URL url) {
-        Pattern p = Pattern.compile("^https?://[wmde.]*xhamster\\.com/photos/gallery/.*?(\\d+)$");
+        Pattern p = Pattern.compile("^https?://[\\w\\w]*\\.xhamster\\.com/photos/gallery/.*?(\\d+)$");
         Matcher m = p.matcher(url.toExternalForm());
         return m.matches();
     }

From d7f39d9f9bb8b04a0beb1dd25bd8ac7e34e6f283 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Thu, 14 Jun 2018 22:23:27 -0400
Subject: [PATCH 019/132] Fixed regression with xhamster ripper that causes it
 to not accept links to the english version of xhamster.com

---
 .../java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java
index b4529f9b..026a6a30 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java
@@ -62,7 +62,7 @@ public class XhamsterRipper extends AbstractHTMLRipper {
 
     @Override
     public boolean canRip(URL url) {
-        Pattern p = Pattern.compile("^https?://[\\w\\w]*\\.xhamster\\.com/photos/gallery/.*?(\\d+)$");
+        Pattern p = Pattern.compile("^https?://([\\w\\w]*\\.)?xhamster\\.com/photos/gallery/.*?(\\d+)$");
         Matcher m = p.matcher(url.toExternalForm());
         return m.matches();
     }

From 340c225dc601cf6f9c34761d1dc1ceb4520145e2 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Thu, 14 Jun 2018 22:25:41 -0400
Subject: [PATCH 020/132] Added unit test for non-english xhamster url

---
 .../ripme/tst/ripper/rippers/XhamsterRipperTest.java         | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java
index 54b22eb0..0a8e7f3c 100644
--- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java
+++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java
@@ -17,6 +17,11 @@ public class XhamsterRipperTest extends RippersTest {
         testRipper(ripper);
     }
 
+    public void testBrazilianXhamster() throws IOException {
+        XhamsterRipper ripper = new XhamsterRipper(new URL("https://pt.xhamster.com/photos/gallery/silvana-7105696"));
+        testRipper(ripper);
+    }
+
     public void testGetGID() throws IOException {
         URL url = new URL("https://xhamster.com/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664");
         XhamsterRipper ripper = new XhamsterRipper(url);

From 416972ca7bc48e3320f2bb72b114040e47ec010c Mon Sep 17 00:00:00 2001
From: Gaboso <gabrielsantiago2@gmail.com>
Date: Thu, 14 Jun 2018 23:08:29 -0400
Subject: [PATCH 021/132] Refactoring in Utils class

---
 .../java/com/rarchives/ripme/utils/Utils.java | 325 +++++++++---------
 1 file changed, 167 insertions(+), 158 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java
index e773a305..5821e9f3 100644
--- a/src/main/java/com/rarchives/ripme/utils/Utils.java
+++ b/src/main/java/com/rarchives/ripme/utils/Utils.java
@@ -1,87 +1,102 @@
 package com.rarchives.ripme.utils;
 
-import java.io.*;
-import java.lang.reflect.Constructor;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.net.URLDecoder;
-import java.util.*;
-import java.util.jar.JarEntry;
-import java.util.jar.JarFile;
-
-import javax.sound.sampled.AudioSystem;
-import javax.sound.sampled.Clip;
-import javax.sound.sampled.Line;
-import javax.sound.sampled.LineEvent;
-
+import com.rarchives.ripme.ripper.AbstractRipper;
 import org.apache.commons.configuration.ConfigurationException;
 import org.apache.commons.configuration.PropertiesConfiguration;
 import org.apache.log4j.LogManager;
 import org.apache.log4j.Logger;
 import org.apache.log4j.PropertyConfigurator;
 
-import com.rarchives.ripme.ripper.AbstractRipper;
+import javax.sound.sampled.AudioSystem;
+import javax.sound.sampled.Clip;
+import javax.sound.sampled.Line;
+import javax.sound.sampled.LineEvent;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Constructor;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.MissingResourceException;
+import java.util.ResourceBundle;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
 
 /**
  * Common utility functions used in various places throughout the project.
  */
 public class Utils {
+
     private static final String RIP_DIRECTORY = "rips";
-    private static final String configFile = "rip.properties";
+    private static final String CONFIG_FILE = "rip.properties";
     private static final String OS = System.getProperty("os.name").toLowerCase();
-    private static final Logger logger = Logger.getLogger(Utils.class);
+    private static final Logger LOGGER = Logger.getLogger(Utils.class);
+    private static final int SHORTENED_PATH_LENGTH = 12;
 
     private static PropertiesConfiguration config;
+    private static HashMap<String, HashMap<String, String>> cookieCache;
+
     static {
+        cookieCache = new HashMap<>();
+
         try {
             String configPath = getConfigFilePath();
-            File f = new File(configPath);
-            if (!f.exists()) {
+            File file = new File(configPath);
+
+            if (!file.exists()) {
                 // Use default bundled with .jar
-                configPath = configFile;
+                configPath = CONFIG_FILE;
             }
+
             config = new PropertiesConfiguration(configPath);
-            logger.info("Loaded " + config.getPath());
-            if (f.exists()) {
+            LOGGER.info("Loaded " + config.getPath());
+
+            if (file.exists()) {
                 // Config was loaded from file
-                if ( !config.containsKey("twitter.auth")
-                  || !config.containsKey("twitter.max_requests")
-                  || !config.containsKey("tumblr.auth")
-                  || !config.containsKey("error.skip404")
-                  || !config.containsKey("gw.api")
-                  || !config.containsKey("page.timeout")
-                  || !config.containsKey("download.max_size")
-                  ) {
+                if (!config.containsKey("twitter.auth") || !config.containsKey("twitter.max_requests")
+                        || !config.containsKey("tumblr.auth") || !config.containsKey("error.skip404")
+                        || !config.containsKey("gw.api") || !config.containsKey("page.timeout")
+                        || !config.containsKey("download.max_size")) {
                     // Config is missing key fields
                     // Need to reload the default config
                     // See https://github.com/4pr0n/ripme/issues/158
-                    logger.warn("Config does not contain key fields, deleting old config");
-                    f.delete();
-                    config = new PropertiesConfiguration(configFile);
-                    logger.info("Loaded " + config.getPath());
+                    LOGGER.warn("Config does not contain key fields, deleting old config");
+                    file.delete();
+                    config = new PropertiesConfiguration(CONFIG_FILE);
+                    LOGGER.info("Loaded " + config.getPath());
                 }
             }
         } catch (Exception e) {
-            logger.error("[!] Failed to load properties file from " + configFile, e);
+            LOGGER.error("[!] Failed to load properties file from " + CONFIG_FILE, e);
         }
     }
 
     /**
      * Get the root rips directory.
-     * @return
-     *      Root directory to save rips to.
-     * @throws IOException
+     *
+     * @return Root directory to save rips to.
      */
     public static File getWorkingDirectory() {
         String currentDir = ".";
         try {
             currentDir = new File(".").getCanonicalPath() + File.separator + RIP_DIRECTORY + File.separator;
         } catch (IOException e) {
-            logger.error("Error while finding working dir: ", e);
+            LOGGER.error("Error while finding working dir: ", e);
         }
+
         if (config != null) {
             currentDir = getConfigString("rips.directory", currentDir);
         }
+
         File workingDir = new File(currentDir);
         if (!workingDir.exists()) {
             workingDir.mkdirs();
@@ -91,8 +106,8 @@ public class Utils {
 
     /**
      * Gets the value of a specific config key.
-     * 
-     * @param key The name of the config parameter you want to find.
+     *
+     * @param key          The name of the config parameter you want to find.
      * @param defaultValue What the default value would be.
      */
     public static String getConfigString(String key, String defaultValue) {
@@ -100,36 +115,46 @@ public class Utils {
     }
 
     public static String[] getConfigStringArray(String key) {
-        String[] s = config.getStringArray(key);
-        if (s.length == 0) {
-            return null;
-        } else {
-            return s;
-        }
+        String[] configStringArray = config.getStringArray(key);
+
+        return configStringArray.length == 0 ? null : configStringArray;
     }
 
     public static int getConfigInteger(String key, int defaultValue) {
         return config.getInt(key, defaultValue);
     }
+
     public static boolean getConfigBoolean(String key, boolean defaultValue) {
         return config.getBoolean(key, defaultValue);
     }
+
     public static List<String> getConfigList(String key) {
         List<String> result = new ArrayList<>();
         for (Object obj : config.getList(key, new ArrayList<String>())) {
             if (obj instanceof String) {
-                result.add( (String) obj);
+                result.add((String) obj);
             }
         }
         return result;
     }
-    public static void setConfigBoolean(String key, boolean value)  { config.setProperty(key, value); }
-    public static void setConfigString(String key, String value)    { config.setProperty(key, value); }
-    public static void setConfigInteger(String key, int value)      { config.setProperty(key, value); }
+
+    public static void setConfigBoolean(String key, boolean value) {
+        config.setProperty(key, value);
+    }
+
+    public static void setConfigString(String key, String value) {
+        config.setProperty(key, value);
+    }
+
+    public static void setConfigInteger(String key, int value) {
+        config.setProperty(key, value);
+    }
+
     public static void setConfigList(String key, List<Object> list) {
         config.clearProperty(key);
         config.addProperty(key, list);
     }
+
     public static void setConfigList(String key, Enumeration<Object> enumeration) {
         config.clearProperty(key);
         List<Object> list = new ArrayList<>();
@@ -142,9 +167,9 @@ public class Utils {
     public static void saveConfig() {
         try {
             config.save(getConfigFilePath());
-            logger.info("Saved configuration to " + getConfigFilePath());
+            LOGGER.info("Saved configuration to " + getConfigFilePath());
         } catch (ConfigurationException e) {
-            logger.error("Error while saving configuration: ", e);
+            LOGGER.error("Error while saving configuration: ", e);
         }
     }
 
@@ -176,14 +201,13 @@ public class Utils {
         return System.getenv("LOCALAPPDATA") + File.separator + "ripme";
     }
 
-    
     /**
      * Gets the directory of where the config file is stored on a UNIX machine.
      */
     private static String getUnixConfigDir() {
         return System.getProperty("user.home") + File.separator + ".config" + File.separator + "ripme";
     }
-    
+
     /**
      * Gets the directory of where the config file is stored on a Mac machine.
      */
@@ -197,13 +221,14 @@ public class Utils {
      */
     private static boolean portableMode() {
         try {
-            File f = new File(new File(".").getCanonicalPath() + File.separator + configFile);
-            if(f.exists() && !f.isDirectory()) {
+            File file = new File(new File(".").getCanonicalPath() + File.separator + CONFIG_FILE);
+            if (file.exists() && !file.isDirectory()) {
                 return true;
             }
         } catch (IOException e) {
             return false;
         }
+
         return false;
     }
 
@@ -222,13 +247,14 @@ public class Utils {
         if (isWindows()) return getWindowsConfigDir();
         if (isMacOS()) return getMacOSConfigDir();
         if (isUnix()) return getUnixConfigDir();
-        
+
         try {
             return new File(".").getCanonicalPath();
         } catch (Exception e) {
             return ".";
         }
     }
+
     /**
      * Delete the url history file
      */
@@ -238,7 +264,7 @@ public class Utils {
     }
 
     /**
-     *  Return the path of the url history file
+     * Return the path of the url history file
      */
     public static String getURLHistoryFile() {
         return getConfigDir() + File.separator + "url_history.txt";
@@ -248,26 +274,23 @@ public class Utils {
      * Gets the path to the configuration file.
      */
     private static String getConfigFilePath() {
-        return getConfigDir() + File.separator + configFile;
+        return getConfigDir() + File.separator + CONFIG_FILE;
     }
 
     /**
      * Removes the current working directory (CWD) from a File.
-     * @param saveAs
-     *      The File path
-     * @return
-     *      saveAs in relation to the CWD
+     *
+     * @param saveAs The File path
+     * @return saveAs in relation to the CWD
      */
     public static String removeCWD(File saveAs) {
         String prettySaveAs = saveAs.toString();
         try {
             prettySaveAs = saveAs.getCanonicalPath();
             String cwd = new File(".").getCanonicalPath() + File.separator;
-            prettySaveAs = prettySaveAs.replace(
-                    cwd,
-                    "." + File.separator);
+            prettySaveAs = prettySaveAs.replace(cwd, "." + File.separator);
         } catch (Exception e) {
-            logger.error("Exception: ", e);
+            LOGGER.error("Exception: ", e);
         }
         return prettySaveAs;
     }
@@ -275,10 +298,9 @@ public class Utils {
     /**
      * Strips away URL parameters, which usually appear at the end of URLs.
      * E.g. the ?query on PHP
-     * 
-     * @param url The URL to filter/strip
+     *
+     * @param url       The URL to filter/strip
      * @param parameter The parameter to strip
-     * 
      * @return The stripped URL
      */
     public static String stripURLParameter(String url, String parameter) {
@@ -290,13 +312,13 @@ public class Utils {
         }
 
         if (paramIndex > 0) {
-            int nextParam = url.indexOf("&", paramIndex+1);
+            int nextParam = url.indexOf('&', paramIndex + 1);
             if (nextParam != -1) {
                 String c = "&";
                 if (wasFirstParam) {
                     c = "?";
                 }
-                url = url.substring(0, paramIndex) + c + url.substring(nextParam+1, url.length());
+                url = url.substring(0, paramIndex) + c + url.substring(nextParam + 1, url.length());
             } else {
                 url = url.substring(0, paramIndex);
             }
@@ -307,10 +329,9 @@ public class Utils {
 
     /**
      * Removes the current working directory from a given filename
-     * @param file
-     *      Path to the file
-     * @return
-     *      'file' without the leading current working directory
+     *
+     * @param file Path to the file
+     * @return 'file' without the leading current working directory
      */
     public static String removeCWD(String file) {
         return removeCWD(new File(file));
@@ -320,12 +341,11 @@ public class Utils {
      * Get a list of all Classes within a package.
      * Works with file system projects and jar files!
      * Borrowed from StackOverflow, but I don't have a link :[
-     * @param pkgname
-     *      The name of the package
-     * @return
-     *      List of classes within the package
+     *
+     * @param pkgname The name of the package
+     * @return List of classes within the package
      */
-    public static ArrayList<Class<?>> getClassesForPackage(String pkgname) {
+    public static List<Class<?>> getClassesForPackage(String pkgname) {
         ArrayList<Class<?>> classes = new ArrayList<>();
         String relPath = pkgname.replace('.', '/');
         URL resource = ClassLoader.getSystemClassLoader().getResource(relPath);
@@ -334,7 +354,8 @@ public class Utils {
         }
 
         String fullPath = resource.getFile();
-        File directory = null;
+        File directory;
+
         try {
             directory = new File(resource.toURI());
         } catch (URISyntaxException e) {
@@ -356,8 +377,7 @@ public class Utils {
                     }
                 }
             }
-        }
-        else {
+        } else {
             // Load from JAR
             try {
                 String jarPath = fullPath
@@ -376,7 +396,7 @@ public class Utils {
                         try {
                             classes.add(Class.forName(className));
                         } catch (ClassNotFoundException e) {
-                            logger.error("ClassNotFoundException loading " + className);
+                            LOGGER.error("ClassNotFoundException loading " + className);
                             jarFile.close(); // Resource leak fix?
                             throw new RuntimeException("ClassNotFoundException loading " + className);
                         }
@@ -384,20 +404,18 @@ public class Utils {
                 }
                 jarFile.close(); // Eclipse said not closing it would have a resource leak
             } catch (IOException e) {
-                logger.error("Error while loading jar file:", e);
+                LOGGER.error("Error while loading jar file:", e);
                 throw new RuntimeException(pkgname + " (" + directory + ") does not appear to be a valid package", e);
             }
         }
         return classes;
     }
 
-    private static final int SHORTENED_PATH_LENGTH = 12;
     /**
      * Shortens the path to a file
-     * @param path
-     *      String of the path to the file
-     * @return
-     *      The simplified path to the file.
+     *
+     * @param path String of the path to the file
+     * @return The simplified path to the file.
      */
     public static String shortenPath(String path) {
         return shortenPath(new File(path));
@@ -405,10 +423,9 @@ public class Utils {
 
     /**
      * Shortens the path to a file
-     * @param file
-     *      File object that you want the shortened path of.
-     * @return 
-     *      The simplified path to the file.
+     *
+     * @param file File object that you want the shortened path of.
+     * @return The simplified path to the file.
      */
     public static String shortenPath(File file) {
         String path = removeCWD(file);
@@ -422,10 +439,9 @@ public class Utils {
 
     /**
      * Sanitizes a string so that a filesystem can handle it
-     * @param text
-     *      The text to be sanitized.
-     * @return
-     *      The sanitized text.
+     *
+     * @param text The text to be sanitized.
+     * @return The sanitized text.
      */
     public static String filesystemSanitized(String text) {
         text = text.replaceAll("[^a-zA-Z0-9.-]", "_");
@@ -434,8 +450,8 @@ public class Utils {
 
     public static String filesystemSafe(String text) {
         text = text.replaceAll("[^a-zA-Z0-9.-]", "_")
-                   .replaceAll("__", "_")
-                   .replaceAll("_+$", "");
+                .replaceAll("__", "_")
+                .replaceAll("_+$", "");
         if (text.length() > 100) {
             text = text.substring(0, 99);
         }
@@ -451,7 +467,7 @@ public class Utils {
     public static String getOriginalDirectory(String path) {
 
         int index;
-        if(isUnix() || isMacOS()) {
+        if (isUnix() || isMacOS()) {
             index = path.lastIndexOf('/');
         } else {
             // current OS is windows - nothing to do here
@@ -459,17 +475,17 @@ public class Utils {
         }
 
         String original = path;                                         // needs to be checked if lowercase exists
-        String lastPart = original.substring(index+1).toLowerCase();    // setting lowercase to check if it exists
+        String lastPart = original.substring(index + 1).toLowerCase();    // setting lowercase to check if it exists
 
         // Get a List of all Directories and check its lowercase
         // if file exists return it
-        File f = new File(path.substring(0, index));
-        ArrayList<String> names = new ArrayList<String>(Arrays.asList(f.list()));
+        File file = new File(path.substring(0, index));
+        ArrayList<String> names = new ArrayList<>(Arrays.asList(file.list()));
 
-        for (String s : names) {
-            if(s.toLowerCase().equals(lastPart)) {
+        for (String name : names) {
+            if (name.toLowerCase().equals(lastPart)) {
                 // Building Path of existing file
-                return path.substring(0, index) + File.separator + s;
+                return path.substring(0, index) + File.separator + name;
             }
         }
 
@@ -478,14 +494,13 @@ public class Utils {
 
     /**
      * Converts an integer into a human readable string
-     * @param bytes
-     *      Non-human readable integer.
-     * @return
-     *      Human readable interpretation of a byte.
+     *
+     * @param bytes Non-human readable integer.
+     * @return Human readable interpretation of a byte.
      */
     public static String bytesToHumanReadable(int bytes) {
         float fbytes = (float) bytes;
-        String[] mags = new String[] {"", "K", "M", "G", "T"};
+        String[] mags = new String[]{"", "K", "M", "G", "T"};
         int magIndex = 0;
         while (fbytes >= 1024) {
             fbytes /= 1024;
@@ -496,6 +511,7 @@ public class Utils {
 
     /**
      * Gets and returns a list of all the album rippers present in the "com.rarchives.ripme.ripper.rippers" package.
+     *
      * @return List<String> of all album rippers present.
      */
     public static List<String> getListOfAlbumRippers() throws Exception {
@@ -508,6 +524,7 @@ public class Utils {
 
     /**
      * Gets and returns a list of all video rippers present in the "com.rarchives.rime.rippers.video" package
+     *
      * @return List<String> of all the video rippers.
      */
     public static List<String> getListOfVideoRippers() throws Exception {
@@ -520,8 +537,8 @@ public class Utils {
 
     /**
      * Plays a sound from a file.
-     * @param filename 
-     *      Path to the sound file
+     *
+     * @param filename Path to the sound file
      */
     public static void playSound(String filename) {
         URL resource = ClassLoader.getSystemClassLoader().getResource(filename);
@@ -535,7 +552,7 @@ public class Utils {
             clip.open(AudioSystem.getAudioInputStream(resource));
             clip.start();
         } catch (Exception e) {
-            logger.error("Failed to play sound " + filename, e);
+            LOGGER.error("Failed to play sound " + filename, e);
         }
     }
 
@@ -544,57 +561,55 @@ public class Utils {
      */
     public static void configureLogger() {
         LogManager.shutdown();
-        String logFile;
-        if (getConfigBoolean("log.save", false)) {
-            logFile = "log4j.file.properties";
+        String logFile = getConfigBoolean("log.save", false) ? "log4j.file.properties" : "log4j.properties";
+
+        try (InputStream stream = Utils.class.getClassLoader().getResourceAsStream(logFile)) {
+            if (stream == null) {
+                PropertyConfigurator.configure("src/main/resources/" + logFile);
+            } else {
+                PropertyConfigurator.configure(stream);
+            }
+
+            LOGGER.info("Loaded " + logFile);
+        } catch (IOException e) {
+            LOGGER.error(e.getMessage(), e);
         }
-        else {
-            logFile = "log4j.properties";
-        }
-        InputStream stream = Utils.class.getClassLoader().getResourceAsStream(logFile);
-        if (stream == null) {
-            PropertyConfigurator.configure("src/main/resources/" + logFile);
-        } else {
-            PropertyConfigurator.configure(stream);
-        }
-        logger.info("Loaded " + logFile);
-        try {
-            stream.close();
-        } catch (IOException e) { }
+
     }
 
     /**
      * Gets list of strings between two strings.
+     *
      * @param fullText Text to retrieve from.
-     * @param start String that precedes the desired text
-     * @param finish String that follows the desired text
+     * @param start    String that precedes the desired text
+     * @param finish   String that follows the desired text
      * @return List of all strings that are between 'start' and 'finish'
      */
     public static List<String> between(String fullText, String start, String finish) {
         List<String> result = new ArrayList<>();
-        int i, j;
-        i = fullText.indexOf(start);
+        int i = fullText.indexOf(start);
+
         while (i >= 0) {
             i += start.length();
-            j = fullText.indexOf(finish, i);
+            int j = fullText.indexOf(finish, i);
             if (j < 0) {
                 break;
             }
             result.add(fullText.substring(i, j));
             i = fullText.indexOf(start, j + finish.length());
         }
+
         return result;
     }
 
     /**
      * Parses an URL query
      *
-     * @param query
-     *          The query part of an URL
+     * @param query The query part of an URL
      * @return The map of all query parameters
      */
-    public static Map<String,String> parseUrlQuery(String query) {
-        Map<String,String> res = new HashMap<>();
+    public static Map<String, String> parseUrlQuery(String query) {
+        Map<String, String> res = new HashMap<>();
 
         if (query.equals("")) {
             return res;
@@ -622,10 +637,8 @@ public class Utils {
     /**
      * Parses an URL query and returns the requested parameter's value
      *
-     * @param query
-     *          The query part of an URL
-     * @param key
-     *          The key whose value is requested
+     * @param query The query part of an URL
+     * @param key   The key whose value is requested
      * @return The associated value or null if key wasn't found
      */
     public static String parseUrlQuery(String query, String key) {
@@ -655,18 +668,13 @@ public class Utils {
         return null;
     }
 
-    private static HashMap<String, HashMap<String, String>> cookieCache;
-    static {
-        cookieCache = new HashMap<String, HashMap<String, String>>();
-    }
-
     /**
      * Gets all the cookies from a certain host
      */
     public static Map<String, String> getCookies(String host) {
         HashMap<String, String> domainCookies = cookieCache.get(host);
         if (domainCookies == null) {
-            domainCookies = new HashMap<String, String>();
+            domainCookies = new HashMap<>();
             String cookiesConfig = getConfigString("cookies." + host, "");
             for (String pair : cookiesConfig.split(" ")) {
                 pair = pair.trim();
@@ -683,27 +691,28 @@ public class Utils {
     /**
      * Gets the ResourceBundle AKA language package.
      * Used for choosing the language of the UI.
-     * 
+     *
      * @return Returns the default resource bundle using the language specified in the config file.
      */
     public static ResourceBundle getResourceBundle(String langSelect) {
         if (langSelect == null) {
             if (!getConfigString("lang", "").equals("")) {
                 String[] langCode = getConfigString("lang", "").split("_");
-                logger.info("Setting locale to " + getConfigString("lang", ""));
+                LOGGER.info("Setting locale to " + getConfigString("lang", ""));
                 return ResourceBundle.getBundle("LabelsBundle", new Locale(langCode[0], langCode[1]), new UTF8Control());
             }
         } else {
             String[] langCode = langSelect.split("_");
-            logger.info("Setting locale to " + langSelect);
+            LOGGER.info("Setting locale to " + langSelect);
             return ResourceBundle.getBundle("LabelsBundle", new Locale(langCode[0], langCode[1]), new UTF8Control());
         }
         try {
-            logger.info("Setting locale to default");
+            LOGGER.info("Setting locale to default");
             return ResourceBundle.getBundle("LabelsBundle", Locale.getDefault(), new UTF8Control());
         } catch (MissingResourceException e) {
-            logger.info("Setting locale to root");
+            LOGGER.info("Setting locale to root");
             return ResourceBundle.getBundle("LabelsBundle", Locale.ROOT);
         }
     }
-}
+
+}
\ No newline at end of file

From 79aaabf58023d20191bf7aafb0787347212fdb1a Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Fri, 15 Jun 2018 15:13:13 -0400
Subject: [PATCH 022/132] Removed unneeded logging line

---
 .../com/rarchives/ripme/ripper/rippers/DeviantartRipper.java | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
index cb1c9614..2afae2dc 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
@@ -132,10 +132,7 @@ public class DeviantartRipper extends AbstractHTMLRipper {
                 LOGGER.warn("Failed to get login cookies");
                 cookies.put("agegate_state","1"); // Bypasses the age gate
             }
-
-        LOGGER.info(Http.url(this.url)
-                .cookies(cookies)
-                .get());
+            
         return Http.url(this.url)
                    .cookies(cookies)
                    .get();

From a8dd989d3739bf1aa4ef9376aad519ca5592e6cd Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Fri, 15 Jun 2018 16:03:02 -0400
Subject: [PATCH 023/132] Furaffinity ripper now uses
 RipUtils.getCookiesFromString to get cookies from config file

---
 .../ripme/ripper/rippers/FuraffinityRipper.java    | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java
index 7dd5b731..54fb3292 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java
@@ -26,6 +26,8 @@ import com.rarchives.ripme.ripper.AbstractHTMLRipper;
 import com.rarchives.ripme.ripper.DownloadThreadPool;
 import com.rarchives.ripme.utils.Http;
 
+import static com.rarchives.ripme.utils.RipUtils.getCookiesFromString;
+
 public class FuraffinityRipper extends AbstractHTMLRipper {
 
     private static final String urlBase = "https://www.furaffinity.net";
@@ -34,16 +36,14 @@ public class FuraffinityRipper extends AbstractHTMLRipper {
     private void setCookies() {
         if (Utils.getConfigBoolean("furaffinity.login", true)) {
             LOGGER.info("Logging in using cookies");
-            String faACookie = Utils.getConfigString("furaffinity.cookie.a", "897bc45b-1f87-49f1-8a85-9412bc103e7a");
-            String faBCookie = Utils.getConfigString("furaffinity.cookie.b", "c8807f36-7a85-4caf-80ca-01c2a2368267");
-            warnAboutSharedAccount(faACookie, faBCookie);
-            cookies.put("a", faACookie);
-            cookies.put("b", faBCookie);
+            String faCookies = Utils.getConfigString("furaffinity.cookies", "a=897bc45b-1f87-49f1-8a85-9412bc103e7a;b=c8807f36-7a85-4caf-80ca-01c2a2368267");
+            warnAboutSharedAccount(faCookies);
+            cookies = getCookiesFromString(faCookies);
         }
     }
 
-    private void warnAboutSharedAccount(String a, String b) {
-        if (a.equals("897bc45b-1f87-49f1-8a85-9412bc103e7a") && b.equals("c8807f36-7a85-4caf-80ca-01c2a2368267")) {
+    private void warnAboutSharedAccount(String loginCookies) {
+        if (loginCookies.equals("a=897bc45b-1f87-49f1-8a85-9412bc103e7a;b=c8807f36-7a85-4caf-80ca-01c2a2368267")) {
             sendUpdate(RipStatusMessage.STATUS.DOWNLOAD_ERRORED,
                     "WARNING: Using the shared furaffinity account exposes both your IP and how many items you downloaded to the other users of the share account");
         }

From 963ae7460abd309e7c2069d9754956f404e586bf Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Fri, 15 Jun 2018 16:03:31 -0400
Subject: [PATCH 024/132] Added unittest for logging in to Furaffinity

---
 .../ripme/tst/ripper/rippers/FuraffinityRipperTest.java   | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java
index 90d66ecd..4982b40f 100644
--- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java
+++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java
@@ -16,4 +16,12 @@ public class FuraffinityRipperTest extends RippersTest {
         FuraffinityRipper ripper = new FuraffinityRipper(url);
         assertEquals("mustardgas", ripper.getGID(url));
     }
+
+    public void testLogin() throws IOException {
+        URL url = new URL("https://www.furaffinity.net/gallery/mustardgas/");
+        FuraffinityRipper ripper = new FuraffinityRipper(url);
+        // Check if the first page contain the username of ripmes shared account
+        Boolean containsUsername = ripper.getFirstPage().html().contains("ripmethrowaway");
+        assert containsUsername;
+    }
 }

From 6cae35ccfb1340caa9a7016f92bdc04f77be7228 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Fri, 15 Jun 2018 16:23:10 -0400
Subject: [PATCH 025/132] 1.7.52: Added warning about using furaffinty shared
 account; Refactoring in Utils class; XhamsterRipper now accepts all countries
 subdomains; E621 ripper now accepts urls with order:Score at the end;
 release.py imrpovements; DeviantartRipper now logs in using cookies; patch.py
 imrpovements

---
 pom.xml                                               | 2 +-
 ripme.json                                            | 7 ++++---
 src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/pom.xml b/pom.xml
index 81b772ea..00b0493a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
   <groupId>com.rarchives.ripme</groupId>
   <artifactId>ripme</artifactId>
   <packaging>jar</packaging>
-  <version>1.7.51</version>
+  <version>1.7.52</version>
   <name>ripme</name>
   <url>http://rip.rarchives.com</url>
   <properties>
diff --git a/ripme.json b/ripme.json
index 90bde0ce..e0c6fed6 100644
--- a/ripme.json
+++ b/ripme.json
@@ -1,6 +1,7 @@
 {
-    "currentHash": "aadb71bf5cdf46fe92e270b50a55c8d8d7200a6dd304a4c2ac9f68cddc687d7e",
+    "latestVersion": "1.7.52",
     "changeList": [
+        "1.7.52: Added warning about using furaffinty shared account; Refactoring in Utils class; XhamsterRipper now accepts all countries subdomains; E621 ripper now accepts urls with order:Score at the end; release.py imrpovements; DeviantartRipper now logs in using cookies; patch.py imrpovements",
         "1.7.51: Fixed instagram ripper; Added the ability to rip from vsco profiles; Fixed TheyiffgalleryRipper; Can now update ripme using the -j flag; added script to automate releases; Code style fixes",
         "1.7.50: Ripme now checks file hash before running update; fixed update bug which cased ripme to report every update as new",
         "1.7.49: Fixed -n flag; Added ability to change locale at runtime and from gui; Update kr_KR translation; Removed support for tnbtu.com; No longer writes url to url_history file is save urls only is checked",
@@ -223,5 +224,5 @@
         "1.0.3: Added VK.com ripper",
         "1.0.1: Added auto-update functionality"
     ],
-    "latestVersion": "1.7.51"
-}
+    "currentHash": "a384d44f2b0c7f517bef3d87ebabc73aaa0887622da8ac3b61bd9d8b29ada48d"
+}
\ No newline at end of file
diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
index 82e1144a..f64046c3 100644
--- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
+++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
@@ -20,7 +20,7 @@ import com.rarchives.ripme.utils.Utils;
 public class UpdateUtils {
 
     private static final Logger logger = Logger.getLogger(UpdateUtils.class);
-    private static final String DEFAULT_VERSION = "1.7.51";
+    private static final String DEFAULT_VERSION = "1.7.52";
     private static final String REPO_NAME = "ripmeapp/ripme";
     private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json";
     private static final String mainFileName = "ripme.jar";

From 5cfd2d7b6ab6d120e083855e130f0f1dee53cd74 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Fri, 15 Jun 2018 16:45:45 -0400
Subject: [PATCH 026/132] Fixed ZikiRipper getAlbumTitle

---
 .../java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java
index d0ca82b2..efa3e25f 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java
@@ -49,9 +49,8 @@ public class ZizkiRipper extends AbstractHTMLRipper {
     public String getAlbumTitle(URL url) throws MalformedURLException {
         try {
             // Attempt to use album title as GID
-            Element titleElement = getFirstPage().select("meta[name=description]").first();
-            String title = titleElement.attr("content");
-            title = title.substring(title.lastIndexOf('/') + 1);
+            Element titleElement = getFirstPage().select("h1.title").first();
+            String title = titleElement.text();
 
             Element authorSpan = getFirstPage().select("span[class=creator]").first();
             String author = authorSpan.select("a").first().text();

From c2938af568e4b7bfc6740a49e93bcf3921a9e9b0 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Fri, 15 Jun 2018 22:23:29 -0400
Subject: [PATCH 027/132] Reddit ripper can now rip videos from v.redd.it

---
 .../ripme/ripper/rippers/RedditRipper.java    | 35 +++++++++++++++++++
 .../com/rarchives/ripme/utils/RipUtils.java   |  3 ++
 2 files changed, 38 insertions(+)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java
index bb60d616..5a967068 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java
@@ -17,6 +17,10 @@ import com.rarchives.ripme.ui.UpdateUtils;
 import com.rarchives.ripme.utils.Http;
 import com.rarchives.ripme.utils.RipUtils;
 import com.rarchives.ripme.utils.Utils;
+import org.jsoup.Jsoup;
+
+import javax.swing.text.Document;
+import javax.swing.text.Element;
 
 public class RedditRipper extends AlbumRipper {
 
@@ -179,6 +183,32 @@ public class RedditRipper extends AlbumRipper {
         }
     }
 
+    private URL parseRedditVideoMPD(String vidURL) {
+        org.jsoup.nodes.Document doc = null;
+        try {
+            doc = Http.url(vidURL + "/DASHPlaylist.mpd").ignoreContentType().get();
+            int largestHeight = 0;
+            String baseURL = null;
+            // Loops over all the videos and finds the one with the largest height and sets baseURL to the base url of that video
+            for (org.jsoup.nodes.Element e : doc.select("MPD > Period > AdaptationSet > Representation")) {
+                String height = e.attr("height");
+                if (height.equals("")) {
+                    height = "0";
+                }
+                if (largestHeight < Integer.parseInt(height)) {
+                    largestHeight = Integer.parseInt(height);
+                    baseURL = doc.select("MPD > Period > AdaptationSet > Representation[height=" + height + "]").select("BaseURL").text();
+                }
+                LOGGER.info("H " + e.attr("height") + " V " + e.attr("width"));
+            }
+            return new URL(vidURL + "/" + baseURL);
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        return null;
+
+    }
+
     private void handleURL(String theUrl, String id) {
         URL originalURL;
         try {
@@ -198,6 +228,11 @@ public class RedditRipper extends AlbumRipper {
                 savePath += id + "-" + m.group(1) + ".jpg";
                 addURLToDownload(urls.get(0), new File(savePath));
             }
+            if (url.contains("v.redd.it")) {
+                String savePath = this.workingDir + File.separator;
+                savePath += id + "-" + url.split("/")[3] + ".mp4";
+                addURLToDownload(parseRedditVideoMPD(urls.get(0).toExternalForm()), new File(savePath));
+            }
             else {
                 addURLToDownload(urls.get(0), id + "-", "", theUrl, null);
             }
diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java
index 84b17ca1..34081852 100644
--- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java
+++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java
@@ -86,6 +86,9 @@ public class RipUtils {
                 logger.warn("Exception while retrieving eroshare page:", e);
             }
             return result;
+        } else if (url.toExternalForm().contains("v.redd.it")) {
+            result.add(url);
+            return result;
         }
 
         else if (url.toExternalForm().contains("erome.com")) {

From df3475270cc3e77f44d58ec80141be6757fdcab8 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Fri, 15 Jun 2018 22:37:36 -0400
Subject: [PATCH 028/132] patch.py now reminds users to run git push before
 release.py

---
 patch.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/patch.py b/patch.py
index 0232ba7d..99b719eb 100644
--- a/patch.py
+++ b/patch.py
@@ -8,8 +8,8 @@ from hashlib import sha256
 # - update version in a few places
 # - insert new line in ripme.json with message
 # - build ripme
-# - add the hash of the lastest binary to ripme.json
-
+# - add the hash of the latest binary to ripme.json
+# - commit all changes
 message = input('message: ')
 
 
@@ -89,3 +89,4 @@ update_hash(file_hash)
 subprocess.call(['git', 'add', '-u'])
 subprocess.call(['git', 'commit', '-m', commitMessage])
 subprocess.call(['git', 'tag', nextVersion])
+print("Remember to run `git push origin master` before release.py")

From ab4fb97eac3f976f98996aac1b19736e8e8de09e Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sat, 16 Jun 2018 12:18:42 -0400
Subject: [PATCH 029/132] Twitter ripper now get retweets when ripping a
 profile and gets largest video when downloading a video

---
 .../ripme/ripper/rippers/TwitterRipper.java        | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
index cd5cf582..4187f5d1 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
@@ -123,7 +123,7 @@ public class TwitterRipper extends AlbumRipper {
                         .append("&include_entities=true")
                         .append("&exclude_replies=true")
                         .append("&trim_user=true")
-                        .append("&include_rts=false")
+                        .append("&include_rts=true")
                         .append("&count=" + 200);
                 break;
             case SEARCH:
@@ -187,14 +187,18 @@ public class TwitterRipper extends AlbumRipper {
                 url = media.getString("media_url");
                 if (media.getString("type").equals("video")) {
                     JSONArray variants = media.getJSONObject("video_info").getJSONArray("variants");
+                    int largestBitrate = 0;
+                    String urlToDownload = null;
+                    // Loop over all the video options and find the biggest video
                     for (int j = 0; j < medias.length(); j++) {
                         JSONObject variant = (JSONObject) variants.get(i);
-                        if (variant.has("bitrate") && variant.getInt("bitrate") == 832000) {
-                            addURLToDownload(new URL(variant.getString("url")));
-                            parsedCount++;
-                            break;
+                        if (variant.getInt("bitrate") > largestBitrate) {
+                            largestBitrate = variant.getInt("bitrate");
+                            urlToDownload = variant.getString("url");
                         }
                     }
+                    addURLToDownload(new URL(urlToDownload));
+                    parsedCount++;
                 } else if (media.getString("type").equals("photo")) {
                     if (url.contains(".twimg.com/")) {
                         url += ":orig";

From c4d6f6d9fd9a628f9146c62b0251875b50489de3 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sat, 16 Jun 2018 12:29:27 -0400
Subject: [PATCH 030/132] twitter ripper now prfixes filenames

---
 .../ripme/ripper/rippers/TwitterRipper.java      | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
index 4187f5d1..94ab41a3 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
@@ -20,6 +20,8 @@ import com.rarchives.ripme.utils.Utils;
 
 public class TwitterRipper extends AlbumRipper {
 
+    int downloadUrls = 1;
+
     private static final String DOMAIN = "twitter.com",
             HOST = "twitter";
 
@@ -197,12 +199,18 @@ public class TwitterRipper extends AlbumRipper {
                             urlToDownload = variant.getString("url");
                         }
                     }
-                    addURLToDownload(new URL(urlToDownload));
+                    if (urlToDownload != null) {
+                        addURLToDownload(new URL(urlToDownload), getPrefix(downloadUrls));
+                        downloadUrls++;
+                    } else {
+                        LOGGER.error("URLToDownload was null");
+                    }
                     parsedCount++;
                 } else if (media.getString("type").equals("photo")) {
                     if (url.contains(".twimg.com/")) {
                         url += ":orig";
-                        addURLToDownload(new URL(url));
+                        addURLToDownload(new URL(url), getPrefix(downloadUrls));
+                        downloadUrls++;
                         parsedCount++;
                     } else {
                         LOGGER.debug("Unexpected media_url: " + url);
@@ -215,6 +223,10 @@ public class TwitterRipper extends AlbumRipper {
         return parsedCount;
     }
 
+    public String getPrefix(int index) {
+        return String.format("%03d_", index);
+    }
+
     @Override
     public void rip() throws IOException {
         getAccessToken();

From 21a1af7d31d0459032241f63376806cfa3e24a2b Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sat, 16 Jun 2018 12:37:31 -0400
Subject: [PATCH 031/132] Twitter ripper no longer errors out on some videos

---
 .../rarchives/ripme/ripper/rippers/TwitterRipper.java  | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
index 94ab41a3..52c7e16f 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
@@ -194,9 +194,13 @@ public class TwitterRipper extends AlbumRipper {
                     // Loop over all the video options and find the biggest video
                     for (int j = 0; j < medias.length(); j++) {
                         JSONObject variant = (JSONObject) variants.get(i);
-                        if (variant.getInt("bitrate") > largestBitrate) {
-                            largestBitrate = variant.getInt("bitrate");
-                            urlToDownload = variant.getString("url");
+                        LOGGER.info(variant);
+                        // If the video doesn't have a bitrate it's a m3u8 file we can't download
+                        if (variant.has("bitrate")) {
+                            if (variant.getInt("bitrate") > largestBitrate) {
+                                largestBitrate = variant.getInt("bitrate");
+                                urlToDownload = variant.getString("url");
+                            }
                         }
                     }
                     if (urlToDownload != null) {

From 3ebb7d1874e61f6f06c9e670b9106f07e8f265ac Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sun, 17 Jun 2018 15:55:57 -0400
Subject: [PATCH 032/132] Started work on Picstatio ripper

---
 .../ripme/ripper/rippers/PicstatioRipper.java | 82 +++++++++++++++++++
 1 file changed, 82 insertions(+)
 create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/PicstatioRipper.java

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PicstatioRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PicstatioRipper.java
new file mode 100644
index 00000000..29c71a08
--- /dev/null
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PicstatioRipper.java
@@ -0,0 +1,82 @@
+package com.rarchives.ripme.ripper.rippers;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+
+import com.rarchives.ripme.ripper.AbstractHTMLRipper;
+import com.rarchives.ripme.utils.Http;
+
+public class PicstatioRipper extends AbstractHTMLRipper {
+
+    public PicstatioRipper(URL url) throws IOException {
+        super(url);
+    }
+
+    private String getFullSizedImageFromURL(String fileName) {
+        try {
+            return Http.url("https://www.picstatio.com/wallpaper/" + fileName + "/download").get().select("p.text-center > a").attr("href");
+        } catch (IOException e) {
+            e.printStackTrace();
+            return null;
+        }
+    }
+
+    @Override
+    public String getHost() {
+        return "picstatio";
+    }
+
+    @Override
+    public String getDomain() {
+        return "picstatio.com";
+    }
+
+    @Override
+    public String getGID(URL url) throws MalformedURLException {
+        Pattern p = Pattern.compile("https?://www.picstatio.com/([a-zA-Z1-9_-]*)/?$");
+        Matcher m = p.matcher(url.toExternalForm());
+        if (m.matches()) {
+            return m.group(1);
+        }
+        throw new MalformedURLException("Expected picstatio URL format: " +
+                "www.picstatio.com//ID - got " + url + " instead");
+    }
+
+    @Override
+    public Document getFirstPage() throws IOException {
+        // "url" is an instance field of the superclass
+        return Http.url(url).get();
+    }
+
+    @Override
+    public Document getNextPage(Document doc) throws IOException {
+        if (doc.select("a.next_page") != null) {
+            return Http.url("https://www.picstatio.com" + doc.select("a.next_page").attr("href")).get();
+        }
+        throw new IOException("No more pages");
+    }
+
+    @Override
+    public List<String> getURLsFromPage(Document doc) {
+        List<String> result = new ArrayList<>();
+        for (Element e : doc.select("img.img")) {
+            String imageName = e.parent().attr("href");
+            LOGGER.info(getFullSizedImageFromURL(imageName.split("/")[2]));
+            result.add(getFullSizedImageFromURL(imageName.split("/")[2]));
+        }
+        return result;
+    }
+
+    @Override
+    public void downloadURL(URL url, int index) {
+        addURLToDownload(url, getPrefix(index));
+    }
+}
\ No newline at end of file

From f34b1273f5a38fa494c1bd9b9ec6119a80aa676a Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Mon, 18 Jun 2018 11:29:10 -0400
Subject: [PATCH 033/132] Got PicstatioRipper working

---
 .../com/rarchives/ripme/ripper/rippers/PicstatioRipper.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PicstatioRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PicstatioRipper.java
index 29c71a08..1bd103b5 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/PicstatioRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PicstatioRipper.java
@@ -22,7 +22,8 @@ public class PicstatioRipper extends AbstractHTMLRipper {
 
     private String getFullSizedImageFromURL(String fileName) {
         try {
-            return Http.url("https://www.picstatio.com/wallpaper/" + fileName + "/download").get().select("p.text-center > a").attr("href");
+            LOGGER.info("https://www.picstatio.com/wallpaper/" + fileName + "/download");
+            return Http.url("https://www.picstatio.com/wallpaper/" + fileName + "/download").get().select("p.text-center > span > a").attr("href");
         } catch (IOException e) {
             e.printStackTrace();
             return null;

From 7102bdec6f478c21521c84ddf6bacbf5fd265c13 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Mon, 18 Jun 2018 11:37:24 -0400
Subject: [PATCH 034/132] Added PicstatioRipper unit test

---
 .../ripper/rippers/PicstatioRipperTest.java   | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/PicstatioRipperTest.java

diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PicstatioRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PicstatioRipperTest.java
new file mode 100644
index 00000000..606cd569
--- /dev/null
+++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PicstatioRipperTest.java
@@ -0,0 +1,19 @@
+package com.rarchives.ripme.tst.ripper.rippers;
+
+import java.io.IOException;
+import java.net.URL;
+
+import com.rarchives.ripme.ripper.rippers.PicstatioRipper;
+
+public class PicstatioRipperTest extends RippersTest {
+
+    public void testRip() throws IOException {
+        PicstatioRipper ripper = new PicstatioRipper(new URL("https://www.picstatio.com/aerial-view-wallpapers"));
+        testRipper(ripper);
+    }
+
+    public void testGID() throws IOException {
+        PicstatioRipper ripper = new PicstatioRipper(new URL("https://www.picstatio.com/aerial-view-wallpapers"));
+        assertEquals("aerial-view-wallpapers", ripper.getGID(new URL("https://www.picstatio.com/aerial-view-wallpapers")));
+    }
+}
\ No newline at end of file

From d0b61d8fb3b8108060649e49b8d5495de017c92b Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Mon, 18 Jun 2018 20:36:06 -0400
Subject: [PATCH 035/132] Fixed instagram ripper

---
 .../rarchives/ripme/ripper/rippers/InstagramRipper.java   | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java
index 63a07e91..cb410fb3 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java
@@ -246,11 +246,12 @@ public class InstagramRipper extends AbstractJSONRipper {
                 try {
                     JSONArray profilePage = json.getJSONObject("entry_data").getJSONArray("ProfilePage");
                     userID = profilePage.getJSONObject(0).getString("logging_page_id").replaceAll("profilePage_", "");
-                    datas = profilePage.getJSONObject(0).getJSONObject("graphql").getJSONObject("user")
+                    datas = json.getJSONObject("entry_data").getJSONArray("ProfilePage").getJSONObject(0)
+                            .getJSONObject("graphql").getJSONObject("user")
                             .getJSONObject("edge_owner_to_timeline_media").getJSONArray("edges");
                 } catch (JSONException e) {
                     datas = json.getJSONObject("data").getJSONObject("user")
-                            .getJSONObject("edge_owner_to_timeline_media").getJSONArray("edges");
+                            .getJSONObject("edge_user_to_photos_of_you").getJSONArray("edges");
                 }
             } else {
                 try {
@@ -391,8 +392,9 @@ public class InstagramRipper extends AbstractJSONRipper {
     }
 
     private boolean pageHasImages(JSONObject json) {
+        LOGGER.info(json);
         int numberOfImages = json.getJSONObject("data").getJSONObject("user")
-                .getJSONObject("edge_owner_to_timeline_media").getJSONArray("edges").length();
+                .getJSONObject("edge_user_to_photos_of_you").getJSONArray("edges").length();
         if (numberOfImages == 0) {
             return false;
         }

From 53684ca169080f88699fbedc68d925015a223146 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Mon, 18 Jun 2018 21:00:52 -0400
Subject: [PATCH 036/132] 1.7.53: Added Picstatio ripper; Fixed instagram
 ripper; Reddit ripper now gets videos from v.redd.it; Fixed ZikiRipper
 getAlbumTitle; fixed twitter ripper

---
 pom.xml                                               | 2 +-
 ripme.json                                            | 5 +++--
 src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/pom.xml b/pom.xml
index 00b0493a..b1d82bbb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
   <groupId>com.rarchives.ripme</groupId>
   <artifactId>ripme</artifactId>
   <packaging>jar</packaging>
-  <version>1.7.52</version>
+  <version>1.7.53</version>
   <name>ripme</name>
   <url>http://rip.rarchives.com</url>
   <properties>
diff --git a/ripme.json b/ripme.json
index e0c6fed6..b6be1995 100644
--- a/ripme.json
+++ b/ripme.json
@@ -1,6 +1,6 @@
 {
-    "latestVersion": "1.7.52",
     "changeList": [
+        "1.7.53: Added Picstatio ripper; Fixed instagram ripper; Reddit ripper now gets videos from v.redd.it; Fixed ZikiRipper getAlbumTitle; fixed twitter ripper",
         "1.7.52: Added warning about using furaffinty shared account; Refactoring in Utils class; XhamsterRipper now accepts all countries subdomains; E621 ripper now accepts urls with order:Score at the end; release.py imrpovements; DeviantartRipper now logs in using cookies; patch.py imrpovements",
         "1.7.51: Fixed instagram ripper; Added the ability to rip from vsco profiles; Fixed TheyiffgalleryRipper; Can now update ripme using the -j flag; added script to automate releases; Code style fixes",
         "1.7.50: Ripme now checks file hash before running update; fixed update bug which cased ripme to report every update as new",
@@ -224,5 +224,6 @@
         "1.0.3: Added VK.com ripper",
         "1.0.1: Added auto-update functionality"
     ],
-    "currentHash": "a384d44f2b0c7f517bef3d87ebabc73aaa0887622da8ac3b61bd9d8b29ada48d"
+    "latestVersion": "1.7.53",
+    "currentHash": "3447ec65444b979d8c28bdbfdbc3a675166c8dce695b9c5fb4716ff1b5195e35"
 }
\ No newline at end of file
diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
index f64046c3..7fd6a41f 100644
--- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
+++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
@@ -20,7 +20,7 @@ import com.rarchives.ripme.utils.Utils;
 public class UpdateUtils {
 
     private static final Logger logger = Logger.getLogger(UpdateUtils.class);
-    private static final String DEFAULT_VERSION = "1.7.52";
+    private static final String DEFAULT_VERSION = "1.7.53";
     private static final String REPO_NAME = "ripmeapp/ripme";
     private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json";
     private static final String mainFileName = "ripme.jar";

From 16cfe8a9ddbc145133c11cffcd37b834e9a4ca65 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Tue, 19 Jun 2018 16:06:19 -0400
Subject: [PATCH 037/132] AbstractJSONRipper now respects hasASAPRipping

---
 .../java/com/rarchives/ripme/ripper/AbstractJSONRipper.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java
index 291dd7df..4455270e 100644
--- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java
@@ -69,7 +69,7 @@ public abstract class AbstractJSONRipper extends AlbumRipper {
                 }
             }
 
-            if (imageURLs.isEmpty()) {
+            if (imageURLs.isEmpty() && !hasASAPRipping()) {
                 throw new IOException("No images found at " + this.url);
             }
 

From 867af57713655886305ecf47468dd889ef8a973e Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Tue, 19 Jun 2018 16:06:39 -0400
Subject: [PATCH 038/132] Got InstagramRipper working again

---
 .../ripme/ripper/rippers/InstagramRipper.java | 43 ++++++++++---------
 1 file changed, 22 insertions(+), 21 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java
index cb410fb3..b38d37ea 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java
@@ -76,6 +76,10 @@ public class InstagramRipper extends AbstractJSONRipper {
         return url.replaceAll("/[A-Z0-9]{8}/", "/");
     }
 
+    @Override public boolean hasASAPRipping() {
+        return true;
+    }
+
     private List<String> getPostsFromSinglePage(JSONObject json) {
         List<String> imageURLs = new ArrayList<>();
         JSONArray datas;
@@ -231,9 +235,21 @@ public class InstagramRipper extends AbstractJSONRipper {
         return imageURL;
     }
 
+    public String getAfter(JSONObject json) {
+        try {
+            return json.getJSONObject("entry_data").getJSONArray("ProfilePage").getJSONObject(0)
+                    .getJSONObject("graphql").getJSONObject("user")
+                    .getJSONObject("edge_owner_to_timeline_media").getJSONObject("page_info").getString("end_cursor");
+        } catch (JSONException e) {
+            return json.getJSONObject("data").getJSONObject("user")
+                    .getJSONObject("edge_owner_to_timeline_media").getJSONObject("page_info").getString("end_cursor");
+        }
+    }
+
     @Override
     public List<String> getURLsFromJSON(JSONObject json) {
         List<String> imageURLs = new ArrayList<>();
+        nextPageID = getAfter(json);
 
         // get the rhx_gis value so we can get the next page later on
         if (rhx_gis == null) {
@@ -251,7 +267,7 @@ public class InstagramRipper extends AbstractJSONRipper {
                             .getJSONObject("edge_owner_to_timeline_media").getJSONArray("edges");
                 } catch (JSONException e) {
                     datas = json.getJSONObject("data").getJSONObject("user")
-                            .getJSONObject("edge_user_to_photos_of_you").getJSONArray("edges");
+                            .getJSONObject("edge_owner_to_timeline_media").getJSONArray("edges");
                 }
             } else {
                 try {
@@ -301,11 +317,10 @@ public class InstagramRipper extends AbstractJSONRipper {
                         }
                     }
                 } catch (MalformedURLException e) {
+                    LOGGER.info("Got MalformedURLException");
                     return imageURLs;
                 }
 
-                nextPageID = data.getString("id");
-
                 if (isThisATest()) {
                     break;
                 }
@@ -369,10 +384,11 @@ public class InstagramRipper extends AbstractJSONRipper {
             try {
                 // Sleep for a while to avoid a ban
                 sleep(2500);
-                String vars = "{\"id\":\"" + userID + "\",\"first\":50,\"after\":\"" + nextPageID + "\"}";
+                String vars = "{\"id\":\"" + userID + "\",\"first\":12,\"after\":\"" + nextPageID + "\"}";
                 String ig_gis = getIGGis(vars);
                 LOGGER.info(ig_gis);
 
+                LOGGER.info("https://www.instagram.com/graphql/query/?query_hash=" + qHash + "&variables=" + vars);
                 toreturn = getPage("https://www.instagram.com/graphql/query/?query_hash=" + qHash + "&variables=" + vars, ig_gis);
                 if (!pageHasImages(toreturn)) {
                     throw new IOException("No more pages");
@@ -394,7 +410,7 @@ public class InstagramRipper extends AbstractJSONRipper {
     private boolean pageHasImages(JSONObject json) {
         LOGGER.info(json);
         int numberOfImages = json.getJSONObject("data").getJSONObject("user")
-                .getJSONObject("edge_user_to_photos_of_you").getJSONArray("edges").length();
+                .getJSONObject("edge_owner_to_timeline_media").getJSONArray("edges").length();
         if (numberOfImages == 0) {
             return false;
         }
@@ -453,26 +469,11 @@ public class InstagramRipper extends AbstractJSONRipper {
             return null;
         }
         if (!rippingTag) {
-            Pattern jsP = Pattern.compile("o},queryId:.([a-zA-Z0-9]+).");
+            Pattern jsP = Pattern.compile("m=\"9ca88e465c3f866a76f7adee3871bdd8\",g=Object\\(c.b\\)\\(\\{pageSize:p.a,pagesToPreload:0,getState:function\\(e,t\\)\\{var o;return null===\\(o=e.profilePosts.byUserId.get\\(t\\)\\)\\|\\|void 0===o\\?void 0:o\\.pagination},queryId:.([a-zA-Z0-9]+)");
             Matcher m = jsP.matcher(sb.toString());
             if (m.find()) {
                 return m.group(1);
             }
-            jsP = Pattern.compile("n.pagination:n},queryId:.([a-zA-Z0-9]+).");
-            m = jsP.matcher(sb.toString());
-            if (m.find()) {
-                return m.group(1);
-            }
-            jsP = Pattern.compile("0:n.pagination},queryId:.([a-zA-Z0-9]+).");
-            m = jsP.matcher(sb.toString());
-            if (m.find()) {
-                return m.group(1);
-            }
-            jsP = Pattern.compile("o.pagination},queryId:.([a-zA-Z0-9]+).");
-            m = jsP.matcher(sb.toString());
-            if (m.find()) {
-                return m.group(1);
-            }
 
         } else {
             Pattern jsP = Pattern.compile("return e.tagMedia.byTagName.get\\(t\\).pagination},queryId:.([a-zA-Z0-9]+).");

From 98d4c57f93cd35e3c65d04efff8fa0450d3f84eb Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Tue, 19 Jun 2018 16:22:22 -0400
Subject: [PATCH 039/132] Fixed ripping from single instagram page

---
 .../com/rarchives/ripme/ripper/rippers/InstagramRipper.java   | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java
index b38d37ea..3077df94 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java
@@ -249,7 +249,9 @@ public class InstagramRipper extends AbstractJSONRipper {
     @Override
     public List<String> getURLsFromJSON(JSONObject json) {
         List<String> imageURLs = new ArrayList<>();
-        nextPageID = getAfter(json);
+        if (!url.toExternalForm().contains("/p/")) {
+            nextPageID = getAfter(json);
+        }
 
         // get the rhx_gis value so we can get the next page later on
         if (rhx_gis == null) {

From c445ab422fce7155b1c70dc5729d30fc1587c9cb Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Tue, 19 Jun 2018 16:29:48 -0400
Subject: [PATCH 040/132] 1.7.54: Fixed twitter ripper video downloading; fixed
 instagram ripper

---
 pom.xml                                               | 2 +-
 ripme.json                                            | 7 ++++---
 src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/pom.xml b/pom.xml
index b1d82bbb..cfe6cd9a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
   <groupId>com.rarchives.ripme</groupId>
   <artifactId>ripme</artifactId>
   <packaging>jar</packaging>
-  <version>1.7.53</version>
+  <version>1.7.54</version>
   <name>ripme</name>
   <url>http://rip.rarchives.com</url>
   <properties>
diff --git a/ripme.json b/ripme.json
index b6be1995..6a0587ab 100644
--- a/ripme.json
+++ b/ripme.json
@@ -1,6 +1,9 @@
 {
+    "currentHash": "f52212a8bd77f38cef7e362429c6a43e5a105a4567e02c69315da31f952c9b35",
+    "latestVersion": "1.7.54",
     "changeList": [
         "1.7.53: Added Picstatio ripper; Fixed instagram ripper; Reddit ripper now gets videos from v.redd.it; Fixed ZikiRipper getAlbumTitle; fixed twitter ripper",
+        "1.7.54: Fixed twitter ripper video downloading; fixed instagram ripper",
         "1.7.52: Added warning about using furaffinty shared account; Refactoring in Utils class; XhamsterRipper now accepts all countries subdomains; E621 ripper now accepts urls with order:Score at the end; release.py imrpovements; DeviantartRipper now logs in using cookies; patch.py imrpovements",
         "1.7.51: Fixed instagram ripper; Added the ability to rip from vsco profiles; Fixed TheyiffgalleryRipper; Can now update ripme using the -j flag; added script to automate releases; Code style fixes",
         "1.7.50: Ripme now checks file hash before running update; fixed update bug which cased ripme to report every update as new",
@@ -223,7 +226,5 @@
         "1.0.4: Fixed spaces-in-directory bug",
         "1.0.3: Added VK.com ripper",
         "1.0.1: Added auto-update functionality"
-    ],
-    "latestVersion": "1.7.53",
-    "currentHash": "3447ec65444b979d8c28bdbfdbc3a675166c8dce695b9c5fb4716ff1b5195e35"
+    ]
 }
\ No newline at end of file
diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
index 7fd6a41f..3e26f2f0 100644
--- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
+++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
@@ -20,7 +20,7 @@ import com.rarchives.ripme.utils.Utils;
 public class UpdateUtils {
 
     private static final Logger logger = Logger.getLogger(UpdateUtils.class);
-    private static final String DEFAULT_VERSION = "1.7.53";
+    private static final String DEFAULT_VERSION = "1.7.54";
     private static final String REPO_NAME = "ripmeapp/ripme";
     private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json";
     private static final String mainFileName = "ripme.jar";

From 3350a78656a2b3ed8521415e4e1924e32a2a0abc Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Tue, 19 Jun 2018 16:32:40 -0400
Subject: [PATCH 041/132] Fixed change list order

---
 ripme.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ripme.json b/ripme.json
index 6a0587ab..626a3d9d 100644
--- a/ripme.json
+++ b/ripme.json
@@ -2,8 +2,8 @@
     "currentHash": "f52212a8bd77f38cef7e362429c6a43e5a105a4567e02c69315da31f952c9b35",
     "latestVersion": "1.7.54",
     "changeList": [
-        "1.7.53: Added Picstatio ripper; Fixed instagram ripper; Reddit ripper now gets videos from v.redd.it; Fixed ZikiRipper getAlbumTitle; fixed twitter ripper",
         "1.7.54: Fixed twitter ripper video downloading; fixed instagram ripper",
+        "1.7.53: Added Picstatio ripper; Fixed instagram ripper; Reddit ripper now gets videos from v.redd.it; Fixed ZikiRipper getAlbumTitle; fixed twitter ripper",
         "1.7.52: Added warning about using furaffinty shared account; Refactoring in Utils class; XhamsterRipper now accepts all countries subdomains; E621 ripper now accepts urls with order:Score at the end; release.py imrpovements; DeviantartRipper now logs in using cookies; patch.py imrpovements",
         "1.7.51: Fixed instagram ripper; Added the ability to rip from vsco profiles; Fixed TheyiffgalleryRipper; Can now update ripme using the -j flag; added script to automate releases; Code style fixes",
         "1.7.50: Ripme now checks file hash before running update; fixed update bug which cased ripme to report every update as new",

From c549010c6aaf71de5ccb9ab7ec8d07d85a859c83 Mon Sep 17 00:00:00 2001
From: "Mateus B. Cassiano" <mbc07@live.com>
Date: Wed, 20 Jun 2018 18:47:44 -0400
Subject: [PATCH 042/132] Added ArtStation ripper

---
 .../ripper/rippers/ArtStationRipper.java      | 258 ++++++++++++++++++
 1 file changed, 258 insertions(+)
 create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java
new file mode 100644
index 00000000..aeb1991d
--- /dev/null
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java
@@ -0,0 +1,258 @@
+package com.rarchives.ripme.ripper.rippers;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.rarchives.ripme.ripper.AbstractJSONRipper;
+import com.rarchives.ripme.utils.Http;
+
+import org.json.JSONObject;
+
+public class ArtStationRipper extends AbstractJSONRipper {
+    enum URL_TYPE {
+        SINGLE_PROJECT, USER_PORTFOLIO, UNKNOWN
+    }
+
+    private ParsedURL albumURL;
+    private String projectName;
+    private Integer projectIndex;
+
+    public ArtStationRipper(URL url) throws IOException {
+        super(url);
+    }
+
+    @Override
+    protected String getDomain() {
+        return "artstation.com";
+    }
+
+    @Override
+    public String getHost() {
+        return "ArtStation";
+    }
+
+    @Override
+    public String getGID(URL url) throws MalformedURLException {
+        JSONObject groupData;
+
+        // Parse URL and store for later use
+        albumURL = parseURL(url);
+
+        if (albumURL.getType() == URL_TYPE.SINGLE_PROJECT) {
+            // URL points to single project, use project title as GID
+            try {
+                groupData = Http.url(albumURL.getLocation()).getJSON();
+            } catch (IOException e) {
+                throw new MalformedURLException("Couldn't load JSON from " + albumURL.getLocation());
+            }
+            return groupData.getString("title");
+        }
+
+        if (albumURL.getType() == URL_TYPE.USER_PORTFOLIO) {
+            // URL points to user portfolio, use user's full name as GID
+            String userInfoURL = "https://www.artstation.com/users/" + albumURL.getID() + "/quick.json";
+            try {
+                groupData = Http.url(userInfoURL).getJSON();
+            } catch (IOException e) {
+                throw new MalformedURLException("Couldn't load JSON from " + userInfoURL);
+            }
+            return groupData.getString("full_name");
+        }
+
+        // No JSON found in the URL entered, can't rip
+        throw new MalformedURLException(
+                "Expected URL to an ArtStation project or user profile - got " + url + " instead");
+    }
+
+    @Override
+    protected JSONObject getFirstPage() throws IOException {
+        if (albumURL.getType() == URL_TYPE.SINGLE_PROJECT) {
+            // URL points to JSON of a single project, just return it
+            return Http.url(albumURL.getLocation()).getJSON();
+        }
+
+        if (albumURL.getType() == URL_TYPE.USER_PORTFOLIO) {
+            // URL points to JSON of a list of projects, load it to parse individual
+            // projects
+            JSONObject albumContent = Http.url(albumURL.getLocation()).getJSON();
+
+            if (albumContent.getInt("total_count") > 0) {
+                // Get JSON of the first project and return it
+                JSONObject projectInfo = albumContent.getJSONArray("data").getJSONObject(0);
+                ParsedURL projectURL = parseURL(new URL(projectInfo.getString("permalink")));
+                return Http.url(projectURL.getLocation()).getJSON();
+            }
+        }
+
+        throw new IOException("URL specified points to an user with empty portfolio");
+    }
+
+    @Override
+    protected JSONObject getNextPage(JSONObject doc) throws IOException {
+        if (albumURL.getType() == URL_TYPE.USER_PORTFOLIO) {
+            // Initialize the index if it hasn't been initialized already
+            if (projectIndex == null) {
+                projectIndex = 1;
+            }
+
+            JSONObject albumContent = Http.url(albumURL.getLocation()).getJSON();
+
+            if (albumContent.getInt("total_count") > projectIndex) {
+                // Get JSON of the next project and return it
+                JSONObject projectInfo = albumContent.getJSONArray("data").getJSONObject(projectIndex);
+                ParsedURL projectURL = parseURL(new URL(projectInfo.getString("permalink")));
+                projectIndex++;
+                return Http.url(projectURL.getLocation()).getJSON();
+            }
+
+            throw new IOException("No more projects");
+        }
+
+        throw new IOException("Downloading a single project");
+    }
+
+    @Override
+    protected List<String> getURLsFromJSON(JSONObject json) {
+        List<String> assetURLs = new ArrayList<>();
+        JSONObject currentObject;
+
+        // Update project name variable from JSON data. Used by downloadURL() to create
+        // subfolders when input URL is URL_TYPE.USER_PORTFOLIO
+        projectName = json.getString("title");
+
+        for (int i = 0; i < json.getJSONArray("assets").length(); i++) {
+            currentObject = json.getJSONArray("assets").getJSONObject(i);
+
+            if (!currentObject.getString("image_url").isEmpty()) {
+                // TODO: Find a way to rip external content.
+                // ArtStation hosts only image content, everything else (videos, 3D Models, etc)
+                // is hosted in other websites and displayed through embedded HTML5 players
+                assetURLs.add(currentObject.getString("image_url"));
+            }
+        }
+
+        return assetURLs;
+    }
+
+    @Override
+    protected void downloadURL(URL url, int index) {
+        if (albumURL.getType() == URL_TYPE.USER_PORTFOLIO) {
+            // Replace not allowed characters with underlines
+            String folderName = projectName.replaceAll("[\\\\/:*?\"<>|]", "_");
+
+            // Folder name also can't end with dots or spaces, strip them
+            folderName = folderName.replaceAll("\\s+$", "");
+            folderName = folderName.replaceAll("\\.+$", "");
+
+            // Downloading multiple projects, separate each one in subfolders
+            addURLToDownload(url, "", folderName);
+        } else {
+            addURLToDownload(url);
+        }
+    }
+
+    @Override
+    public String normalizeUrl(String url) {
+        // Strip URL parameters
+        return url.replaceAll("\\?\\w+$", "");
+    }
+
+    private static class ParsedURL {
+        URL_TYPE urlType;
+        String jsonURL, urlID;
+
+        /**
+         * Construct a new ParsedURL object.
+         * 
+         * @param urlType URL_TYPE enum containing the URL type
+         * @param jsonURL String containing the JSON URL location
+         * @param urlID   String containing the ID of this URL
+         * 
+         */
+        ParsedURL(URL_TYPE urlType, String jsonURL, String urlID) {
+            this.urlType = urlType;
+            this.jsonURL = jsonURL;
+            this.urlID = urlID;
+        }
+
+        /**
+         * Get URL Type of this ParsedURL object.
+         * 
+         * @return URL_TYPE enum containing this object type
+         * 
+         */
+        URL_TYPE getType() {
+            return this.urlType;
+        }
+
+        /**
+         * Get JSON location of this ParsedURL object.
+         * 
+         * @return String containing the JSON URL
+         * 
+         */
+        String getLocation() {
+            return this.jsonURL;
+        }
+
+        /**
+         * Get ID of this ParsedURL object.
+         * 
+         * @return For URL_TYPE.SINGLE_PROJECT, returns the project hash. For
+         *         URL_TYPE.USER_PORTFOLIO, returns the account name
+         */
+        String getID() {
+            return this.urlID;
+        }
+    }
+
+    /**
+     * Parses an ArtStation URL.
+     * 
+     * @param url URL to an ArtStation user profile
+     *            (https://www.artstation.com/username) or single project
+     *            (https://www.artstation.com/artwork/projectid)
+     * @return ParsedURL object containing URL type, JSON location and ID (stores
+     *         account name or project hash, depending of the URL type identified)
+     * 
+     */
+    private ParsedURL parseURL(URL url) {
+        String htmlSource;
+        ParsedURL parsedURL;
+
+        // Load HTML Source of the specified URL
+        try {
+            htmlSource = Http.url(url).get().html();
+        } catch (IOException e) {
+            htmlSource = "";
+        }
+
+        // Check if HTML Source of the specified URL references a project
+        Pattern p = Pattern.compile("'/projects/(\\w+)\\.json'");
+        Matcher m = p.matcher(htmlSource);
+        if (m.find()) {
+            parsedURL = new ParsedURL(URL_TYPE.SINGLE_PROJECT,
+                    "https://www.artstation.com/projects/" + m.group(1) + ".json", m.group(1));
+            return parsedURL;
+        }
+
+        // Check if HTML Source of the specified URL references a user profile
+        p = Pattern.compile("'/users/([\\w-]+)/quick\\.json'");
+        m = p.matcher(htmlSource);
+        if (m.find()) {
+            parsedURL = new ParsedURL(URL_TYPE.USER_PORTFOLIO,
+                    "https://www.artstation.com/users/" + m.group(1) + "/projects.json", m.group(1));
+            return parsedURL;
+        }
+
+        // HTML Source of the specified URL doesn't reference a user profile or project
+        parsedURL = new ParsedURL(URL_TYPE.UNKNOWN, null, null);
+        return parsedURL;
+    }
+
+}

From d3dde5eebea89eefbcdb53479b8bf13d3127bef7 Mon Sep 17 00:00:00 2001
From: "Mateus B. Cassiano" <mbc07@live.com>
Date: Wed, 20 Jun 2018 18:48:46 -0400
Subject: [PATCH 043/132] Added ArtStationRipper unit test

---
 .../ripper/rippers/ArtStationRipperTest.java  | 32 +++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java

diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java
new file mode 100644
index 00000000..381c007f
--- /dev/null
+++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java
@@ -0,0 +1,32 @@
+package com.rarchives.ripme.tst.ripper.rippers;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.rarchives.ripme.ripper.rippers.ArtStationRipper;
+
+public class ArtStationRipperTest extends RippersTest {
+
+    public void testArtStationProjects() throws IOException {
+        List<URL> contentURLs = new ArrayList<>();
+        contentURLs.add(new URL("https://www.artstation.com/artwork/the-dwarf-mortar"));
+        contentURLs.add(new URL("https://www.artstation.com/artwork/K36GR"));
+        contentURLs.add(new URL("http://artstation.com/artwork/5JJQw"));
+        for (URL url : contentURLs) {
+            ArtStationRipper ripper = new ArtStationRipper(url);
+            testRipper(ripper);
+        }
+    }
+
+    public void testArtStationUserProfiles() throws IOException {
+        List<URL> contentURLs = new ArrayList<>();
+        contentURLs.add(new URL("https://www.artstation.com/heitoramatsu"));
+        contentURLs.add(new URL("http://artstation.com/givemeapiggy"));
+        for (URL url : contentURLs) {
+            ArtStationRipper ripper = new ArtStationRipper(url);
+            testRipper(ripper);
+        }
+    }
+}

From 9c1c1a6b838fdd88d7f11cb268345e0fd4239a51 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Thu, 21 Jun 2018 16:22:00 -0400
Subject: [PATCH 044/132] Disabled test for Bcfakes.com as the site has been
 down for many days

---
 .../ripme/tst/ripper/rippers/BcfakesRipperTest.java      | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BcfakesRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BcfakesRipperTest.java
index 6b8e2315..5453a81f 100644
--- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BcfakesRipperTest.java
+++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BcfakesRipperTest.java
@@ -6,8 +6,9 @@ import java.net.URL;
 import com.rarchives.ripme.ripper.rippers.BcfakesRipper;
 
 public class BcfakesRipperTest extends RippersTest {
-    public void testRip() throws IOException {
-        BcfakesRipper ripper = new BcfakesRipper(new URL("http://www.bcfakes.com/celebritylist/olivia-wilde/"));
-        testRipper(ripper);
-    }
+    // 21/06/2018 This test was disbaled as the site has experienced notable downtime
+//    public void testRip() throws IOException {
+//        BcfakesRipper ripper = new BcfakesRipper(new URL("http://www.bcfakes.com/celebritylist/olivia-wilde/"));
+//        testRipper(ripper);
+//    }
 }

From 68b13cbbad1b4c7d8a060d29dab4b082758f187b Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Fri, 22 Jun 2018 17:03:04 -0400
Subject: [PATCH 045/132] Added a rip method status for when a download
 completes due to the history.end_rip_after_already_seen config setting

---
 src/main/java/com/rarchives/ripme/ui/MainWindow.java       | 6 ++++++
 src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java | 1 +
 2 files changed, 7 insertions(+)

diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java
index 0193aa08..9f4ff519 100644
--- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java
+++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java
@@ -1206,6 +1206,12 @@ public final class MainWindow implements Runnable, RipStatusHandler {
                 appendLog("Downloaded " + msg.getObject(), Color.GREEN);
             }
             break;
+        case DOWNLOAD_COMPLETE_HISTORY:
+            if (LOGGER.isEnabledFor(Level.INFO)) {
+                appendLog("" + msg.getObject(), Color.GREEN);
+            }
+            break;
+
         case DOWNLOAD_ERRORED:
             if (LOGGER.isEnabledFor(Level.ERROR)) {
                 appendLog((String) msg.getObject(), Color.RED);
diff --git a/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java b/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java
index bdd68e3b..462e00ac 100644
--- a/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java
+++ b/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java
@@ -10,6 +10,7 @@ public class RipStatusMessage {
         DOWNLOAD_STARTED("Download Started"),
         DOWNLOAD_COMPLETE("Download Complete"),
         DOWNLOAD_ERRORED("Download Errored"),
+        DOWNLOAD_COMPLETE_HISTORY("Download Complete History"),
         RIP_COMPLETE("Rip Complete"),
         DOWNLOAD_WARN("Download problem"),
         TOTAL_BYTES("Total bytes"),

From 0918ab461625d7417a66a44bb02ea450e5faee29 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Fri, 22 Jun 2018 17:04:01 -0400
Subject: [PATCH 046/132] now uses DOWNLOAD_COMPLETE_HISTORY when sending a
 message about the rip ending due to history.end_rip_after_already_seen config
 setting

---
 .../java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java
index b28b0ba4..01444642 100644
--- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java
@@ -97,7 +97,7 @@ public abstract class AbstractHTMLRipper extends AlbumRipper {
 
         while (doc != null) {
             if (alreadyDownloadedUrls >= Utils.getConfigInteger("history.end_rip_after_already_seen", 1000000000) && !isThisATest()) {
-                sendUpdate(STATUS.DOWNLOAD_COMPLETE, "Already seen the last " + alreadyDownloadedUrls + " images ending rip");
+                sendUpdate(STATUS.DOWNLOAD_COMPLETE_HISTORY, "Already seen the last " + alreadyDownloadedUrls + " images ending rip");
                 break;
             }
             List<String> imageURLs = getURLsFromPage(doc);

From f5bdada31b6047c0f5aa4db5431d0fe5a9f3e6e4 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Fri, 22 Jun 2018 17:04:28 -0400
Subject: [PATCH 047/132] Reddit ripper now respects
 history.end_rip_after_already_seen

---
 .../com/rarchives/ripme/ripper/rippers/RedditRipper.java | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java
index 5a967068..08229d72 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java
@@ -8,6 +8,7 @@ import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import com.rarchives.ripme.ui.RipStatusMessage;
 import org.json.JSONArray;
 import org.json.JSONObject;
 import org.json.JSONTokener;
@@ -39,6 +40,10 @@ public class RedditRipper extends AlbumRipper {
 
     private long lastRequestTime = 0;
 
+    private Boolean shouldAddURL() {
+        return (alreadyDownloadedUrls >= Utils.getConfigInteger("history.end_rip_after_already_seen", 1000000000) && !isThisATest());
+    }
+
     @Override
     public boolean canRip(URL url) {
         return url.getHost().endsWith(DOMAIN);
@@ -65,6 +70,10 @@ public class RedditRipper extends AlbumRipper {
     public void rip() throws IOException {
         URL jsonURL = getJsonURL(this.url);
         while (true) {
+            if (shouldAddURL()) {
+                sendUpdate(RipStatusMessage.STATUS.DOWNLOAD_COMPLETE_HISTORY, "Already seen the last " + alreadyDownloadedUrls + " images ending rip");
+                break;
+            }
             jsonURL = getAndParseAndReturnNext(jsonURL);
             if (jsonURL == null || isThisATest() || isStopped()) {
                 break;

From 76b3565f7fbdd748c0004a9650d5cc88911a09e3 Mon Sep 17 00:00:00 2001
From: Aman <amanraoverma@gmail.com>
Date: Fri, 22 Jun 2018 18:25:28 -0400
Subject: [PATCH 048/132] Fix handling of files that are not in subdirs. Use
 raw strings for regex. Should fix #672.

---
 release.py | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/release.py b/release.py
index 04ad7e08..8496c4f8 100644
--- a/release.py
+++ b/release.py
@@ -37,25 +37,24 @@ def isJar(filename):
 # false if not
 def isValidCommitMessage(message):
     if debug:
-        print("Checking if {} matchs pattern ^\d+\.\d+\.\d+:".format(message))
-    pattern = re.compile("^\d+\.\d+\.\d+:")
+        print(r"Checking if {} matches pattern ^\d+\.\d+\.\d+:".format(message))
+    pattern = re.compile(r"^\d+\.\d+\.\d+:")
     return re.match(pattern, message)
 
 
 # Checks if the update has the name ripme.jar, if not it renames the file
-# Returns the update file path
-# TODO handle files that aren't in sub dirs
-def renameFile(path):
+def checkAndRenameFile(path):
+    """Check if path (a string) points to a ripme.jar. Returns the possibly renamed file path"""
     if not path.endswith("ripme.jar"):
         print("Specified file is not named ripme.jar, renaming")
-        # os.sep is the path separator for the os this is being run on
-        os.rename(path, path.split(os.sep)[0] + os.sep + "ripme.jar")
-        return path.split(os.sep)[0] + os.sep + "ripme.jar"
+        new_path = os.path.join(os.path.dirname(path), "ripme.jar")
+        os.rename(path, new_path) 
+        return new_path 
     return path
 
 
 ripmeJson = json.loads(open("ripme.json").read())
-fileToUploadPath = renameFile(args.file)
+fileToUploadPath = checkAndRenameFile(args.file)
 InNoninteractiveMode = args.non_interactive
 commitMessage = ripmeJson.get("changeList")[0]
 releaseVersion = ripmeJson.get("latestVersion")

From 08c11fa122b03c6507a4b65d922b892b1d733c54 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sat, 23 Jun 2018 15:10:15 -0400
Subject: [PATCH 049/132] Instagram ripper can now get query hash again

---
 .../com/rarchives/ripme/ripper/rippers/InstagramRipper.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java
index 3077df94..c20a5696 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java
@@ -471,7 +471,7 @@ public class InstagramRipper extends AbstractJSONRipper {
             return null;
         }
         if (!rippingTag) {
-            Pattern jsP = Pattern.compile("m=\"9ca88e465c3f866a76f7adee3871bdd8\",g=Object\\(c.b\\)\\(\\{pageSize:p.a,pagesToPreload:0,getState:function\\(e,t\\)\\{var o;return null===\\(o=e.profilePosts.byUserId.get\\(t\\)\\)\\|\\|void 0===o\\?void 0:o\\.pagination},queryId:.([a-zA-Z0-9]+)");
+            Pattern jsP = Pattern.compile("byUserId\\.get\\(t\\)\\)\\|\\|void 0===r\\?void 0:r\\.pagination},queryId:.([a-zA-Z0-9]+)");
             Matcher m = jsP.matcher(sb.toString());
             if (m.find()) {
                 return m.group(1);

From a5ba72df33ee7e3a0dc5665ca96f6b96aff3bee9 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sat, 23 Jun 2018 18:24:10 -0400
Subject: [PATCH 050/132] Now uses update_change_list to update change list
 inststead of just inserting a string at a hardcoded line number

---
 patch.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/patch.py b/patch.py
index 99b719eb..dd8bbea5 100644
--- a/patch.py
+++ b/patch.py
@@ -65,15 +65,8 @@ subprocess.call(['git', 'grep', '<version>' + nextVersion + '</version>', 'pom.x
 commitMessage = nextVersion + ': ' + message
 changeLogLine = '        \"' + commitMessage + '\",\n'
 
-dataFile = open("ripme.json", "r")
-ripmeJsonLines = dataFile.readlines()
-ripmeJsonLines.insert(3, changeLogLine)
-outputContent = ''.join(ripmeJsonLines)
-dataFile.close()
+update_change_list(changeLogLine)
 
-dataFile = open("ripme.json", "w")
-dataFile.write(outputContent)
-dataFile.close()
 
 print("Building ripme")
 subprocess.call(["mvn", "clean", "compile", "assembly:single"])

From 8aebc3671de52d5578152a8450fbc436dfc916aa Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sat, 23 Jun 2018 18:50:46 -0400
Subject: [PATCH 051/132] Fixed patch.py

---
 patch.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/patch.py b/patch.py
index dd8bbea5..c31e58cb 100644
--- a/patch.py
+++ b/patch.py
@@ -30,8 +30,9 @@ def update_hash(current_hash):
 def update_change_list(message):
     ripmeJson = get_ripme_json()
     with open('ripme.json', 'w') as dataFile:
-        ripmeJson["changeList"] = ripmeJson["changeList"].insert(0, message)
-        print(ripmeJson["currentHash"])
+        new_cl = ripmeJson["changeList"].insert(0, message)
+        ripmeJson["changeList"] = new_cl
+        print(ripmeJson["changeList"])
         json.dump(ripmeJson, dataFile, indent=4)
 
 

From 54cc6661f13fbe44ea58db93721254d3efd8fe7f Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sat, 23 Jun 2018 18:52:31 -0400
Subject: [PATCH 052/132] Fixed patch.py

---
 patch.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/patch.py b/patch.py
index c31e58cb..ba468d23 100644
--- a/patch.py
+++ b/patch.py
@@ -30,9 +30,8 @@ def update_hash(current_hash):
 def update_change_list(message):
     ripmeJson = get_ripme_json()
     with open('ripme.json', 'w') as dataFile:
-        new_cl = ripmeJson["changeList"].insert(0, message)
-        ripmeJson["changeList"] = new_cl
         print(ripmeJson["changeList"])
+        ripmeJson["changeList"].insert(0, message)
         json.dump(ripmeJson, dataFile, indent=4)
 
 

From aaf31dc06548980b3895ceb81cfa1a878f318024 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sat, 23 Jun 2018 18:54:16 -0400
Subject: [PATCH 053/132] Fixed json formating

---
 patch.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/patch.py b/patch.py
index ba468d23..6c2408d9 100644
--- a/patch.py
+++ b/patch.py
@@ -63,9 +63,8 @@ subprocess.call(['sed', '-i', '-e', substrExpr, 'pom.xml'])
 subprocess.call(['git', 'grep', '<version>' + nextVersion + '</version>', 'pom.xml'])
 
 commitMessage = nextVersion + ': ' + message
-changeLogLine = '        \"' + commitMessage + '\",\n'
 
-update_change_list(changeLogLine)
+update_change_list(commitMessage)
 
 
 print("Building ripme")

From a536738c1a4bf7ef82b199e87c92a97417ab1c09 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sat, 23 Jun 2018 18:59:42 -0400
Subject: [PATCH 054/132] No longer prints change list

---
 patch.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/patch.py b/patch.py
index 6c2408d9..5ed45626 100644
--- a/patch.py
+++ b/patch.py
@@ -30,7 +30,6 @@ def update_hash(current_hash):
 def update_change_list(message):
     ripmeJson = get_ripme_json()
     with open('ripme.json', 'w') as dataFile:
-        print(ripmeJson["changeList"])
         ripmeJson["changeList"].insert(0, message)
         json.dump(ripmeJson, dataFile, indent=4)
 

From a855957ab1417b28b08b1c751081d532b7f7d4e1 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sat, 23 Jun 2018 19:03:46 -0400
Subject: [PATCH 055/132] 1.7.55: Fixed instagram ripper; Reddit ripper now
 respects history.end_rip_after_already_seen; Improvements to patch.py and
 release.py

---
 pom.xml                                               | 2 +-
 ripme.json                                            | 7 ++++---
 src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/pom.xml b/pom.xml
index cfe6cd9a..b9c412ff 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
   <groupId>com.rarchives.ripme</groupId>
   <artifactId>ripme</artifactId>
   <packaging>jar</packaging>
-  <version>1.7.54</version>
+  <version>1.7.55</version>
   <name>ripme</name>
   <url>http://rip.rarchives.com</url>
   <properties>
diff --git a/ripme.json b/ripme.json
index 626a3d9d..2c6982c3 100644
--- a/ripme.json
+++ b/ripme.json
@@ -1,7 +1,7 @@
 {
-    "currentHash": "f52212a8bd77f38cef7e362429c6a43e5a105a4567e02c69315da31f952c9b35",
-    "latestVersion": "1.7.54",
+    "latestVersion": "1.7.55",
     "changeList": [
+        "1.7.55: Fixed instagram ripper; Reddit ripper now respects history.end_rip_after_already_seen; Improvements to patch.py and release.py",
         "1.7.54: Fixed twitter ripper video downloading; fixed instagram ripper",
         "1.7.53: Added Picstatio ripper; Fixed instagram ripper; Reddit ripper now gets videos from v.redd.it; Fixed ZikiRipper getAlbumTitle; fixed twitter ripper",
         "1.7.52: Added warning about using furaffinty shared account; Refactoring in Utils class; XhamsterRipper now accepts all countries subdomains; E621 ripper now accepts urls with order:Score at the end; release.py imrpovements; DeviantartRipper now logs in using cookies; patch.py imrpovements",
@@ -226,5 +226,6 @@
         "1.0.4: Fixed spaces-in-directory bug",
         "1.0.3: Added VK.com ripper",
         "1.0.1: Added auto-update functionality"
-    ]
+    ],
+    "currentHash": "1df13a792f17e2e36a2c3a62527d0e97a3edbee14d1bdb0cd822c7d2a8ce3cf4"
 }
\ No newline at end of file
diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
index 3e26f2f0..2ca91ba8 100644
--- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
+++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
@@ -20,7 +20,7 @@ import com.rarchives.ripme.utils.Utils;
 public class UpdateUtils {
 
     private static final Logger logger = Logger.getLogger(UpdateUtils.class);
-    private static final String DEFAULT_VERSION = "1.7.54";
+    private static final String DEFAULT_VERSION = "1.7.55";
     private static final String REPO_NAME = "ripmeapp/ripme";
     private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json";
     private static final String mainFileName = "ripme.jar";

From ca11f35df893e4fb25f281e79e72e1ffde4fe9f8 Mon Sep 17 00:00:00 2001
From: Aman <amanraoverma@gmail.com>
Date: Sun, 24 Jun 2018 21:29:20 -0400
Subject: [PATCH 056/132] Fix shebang for release.py and make it executable

---
 release.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 mode change 100644 => 100755 release.py

diff --git a/release.py b/release.py
old mode 100644
new mode 100755
index 8496c4f8..ad099bad
--- a/release.py
+++ b/release.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import re
 

From 5dabdd16edce443bdc0f37e8b1ece4654e101a8d Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Mon, 25 Jun 2018 04:43:12 -0400
Subject: [PATCH 057/132] ModelmayhemRipper can now donwload nsfw images again

---
 .../rarchives/ripme/ripper/rippers/ModelmayhemRipper.java | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ModelmayhemRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ModelmayhemRipper.java
index dac7ccb3..aa6ac521 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/ModelmayhemRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ModelmayhemRipper.java
@@ -4,7 +4,9 @@ import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -16,6 +18,8 @@ import com.rarchives.ripme.utils.Http;
 
 public class ModelmayhemRipper extends AbstractHTMLRipper {
 
+    private Map<String,String> cookies = new HashMap<>();
+
     public ModelmayhemRipper(URL url) throws IOException {
         super(url);
     }
@@ -43,8 +47,10 @@ public class ModelmayhemRipper extends AbstractHTMLRipper {
 
     @Override
     public Document getFirstPage() throws IOException {
+        // Bypass NSFW filter
+        cookies.put("worksafe", "0");
         // "url" is an instance field of the superclass
-        return Http.url(url).get();
+        return Http.url(url).cookies(cookies).get();
     }
 
     @Override

From c4f2d52370b9a92d306f7ec4715431e31d3cdbaa Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Mon, 25 Jun 2018 05:32:37 -0400
Subject: [PATCH 058/132] Fixed erome ripper and added quickQueue support

---
 .../ripme/ripper/rippers/EromeRipper.java     | 78 +++++++++----------
 1 file changed, 35 insertions(+), 43 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java
index a01c134a..2711cd1d 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java
@@ -22,6 +22,8 @@ import com.rarchives.ripme.utils.Http;
  */
 public class EromeRipper extends AbstractHTMLRipper {
 
+    boolean rippingProfile;
+
 
     public EromeRipper (URL url) throws IOException {
         super(url);
@@ -42,6 +44,27 @@ public class EromeRipper extends AbstractHTMLRipper {
         addURLToDownload(url, getPrefix(index));
     }
 
+    @Override
+    public boolean hasQueueSupport() {
+        return true;
+    }
+
+    @Override
+    public boolean pageContainsAlbums(URL url) {
+        Pattern pa = Pattern.compile("https?://www.erome.com/([a-zA-Z0-9_-]*)/?");
+        Matcher ma = pa.matcher(url.toExternalForm());
+        return ma.matches();
+    }
+
+    @Override
+    public List<String> getAlbumsToQueue(Document doc) {
+        List<String> urlsToAddToQueue = new ArrayList<>();
+        for (Element elem : doc.select("div#albums > div.album > a")) {
+            urlsToAddToQueue.add(elem.attr("href"));
+        }
+        return urlsToAddToQueue;
+    }
+
     @Override
     public String getAlbumTitle(URL url) throws MalformedURLException {
             try {
@@ -53,6 +76,8 @@ public class EromeRipper extends AbstractHTMLRipper {
             } catch (IOException e) {
                 // Fall back to default album naming convention
                 LOGGER.info("Unable to find title at " + url);
+            } catch (NullPointerException e) {
+                return getHost() + "_" + getGID(url);
             }
             return super.getAlbumTitle(url);
     }
@@ -66,21 +91,7 @@ public class EromeRipper extends AbstractHTMLRipper {
     @Override
     public List<String> getURLsFromPage(Document doc) {
         List<String> URLs = new ArrayList<>();
-        //Pictures
-        Elements imgs = doc.select("div.img > img.img-front");
-        for (Element img : imgs) {
-            String imageURL = img.attr("src");
-            imageURL = "https:" + imageURL;
-            URLs.add(imageURL);
-        }
-        //Videos
-        Elements vids = doc.select("div.video > video > source");
-        for (Element vid : vids) {
-            String videoURL = vid.attr("src");
-            URLs.add("https:" + videoURL);
-        }
-
-        return URLs;
+        return getMediaFromPage(doc);
     }
 
     @Override
@@ -100,7 +111,7 @@ public class EromeRipper extends AbstractHTMLRipper {
             return m.group(1);
         }
 
-        p = Pattern.compile("^https?://erome.com/a/([a-zA-Z0-9]*)/?$");
+        p = Pattern.compile("^https?://www.erome.com/([a-zA-Z0-9_-]+)/?$");
         m = p.matcher(url.toExternalForm());
 
         if (m.matches()) {
@@ -110,34 +121,15 @@ public class EromeRipper extends AbstractHTMLRipper {
         throw new MalformedURLException("erome album not found in " + url + ", expected https://www.erome.com/album");
     }
 
-    public static List<URL> getURLs(URL url) throws IOException{
-
-        Response resp = Http.url(url)
-                            .ignoreContentType()
-                            .response();
-
-        Document doc = resp.parse();
-
-        List<URL> URLs = new ArrayList<>();
-        //Pictures
-        Elements imgs = doc.getElementsByTag("img");
-        for (Element img : imgs) {
-            if (img.hasClass("album-image")) {
-                String imageURL = img.attr("src");
-                imageURL = "https:" + imageURL;
-                URLs.add(new URL(imageURL));
-            }
+    private List<String> getMediaFromPage(Document doc) {
+        List<String> results = new ArrayList<>();
+        for (Element el : doc.select("img.img-front")) {
+            results.add("https:" + el.attr("src"));
         }
-        //Videos
-        Elements vids = doc.getElementsByTag("video");
-        for (Element vid : vids) {
-            if (vid.hasClass("album-video")) {
-                Elements source = vid.getElementsByTag("source");
-                String videoURL = source.first().attr("src");
-                URLs.add(new URL(videoURL));
-            }
+        for (Element el : doc.select("source[label=HD]")) {
+            results.add("https:" + el.attr("src"));
         }
-
-        return URLs;
+        return results;
     }
+
 }

From b63f8f16b9474e2023e23c9d51a296373daeebed Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Mon, 25 Jun 2018 06:03:10 -0400
Subject: [PATCH 059/132] Added unit test for erome

---
 .../tst/ripper/rippers/EromeRipperTest.java   | 40 +++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/EromeRipperTest.java

diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EromeRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EromeRipperTest.java
new file mode 100644
index 00000000..c07c41ac
--- /dev/null
+++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EromeRipperTest.java
@@ -0,0 +1,40 @@
+package com.rarchives.ripme.tst.ripper.rippers;
+
+import java.io.IOException;
+import java.net.URL;
+
+import com.rarchives.ripme.ripper.rippers.EromeRipper;
+
+public class EromeRipperTest extends RippersTest {
+
+    public void testGetGIDProfilePage() throws IOException {
+        URL url = new URL("https://www.erome.com/Jay-Jenna");
+        EromeRipper ripper = new EromeRipper(url);
+        assertEquals("Jay-Jenna", ripper.getGID(url));
+    }
+
+    public void testGetGIDAlbum() throws IOException {
+        URL url = new URL("https://www.erome.com/a/KbDAM1XT");
+        EromeRipper ripper = new EromeRipper(url);
+        assertEquals("KbDAM1XT", ripper.getGID(url));
+    }
+
+    public void testGetAlbumsToQueue() throws IOException {
+        URL url = new URL("https://www.erome.com/Jay-Jenna");
+        EromeRipper ripper = new EromeRipper(url);
+        assert(2 >= ripper.getAlbumsToQueue(ripper.getFirstPage()).size());
+    }
+
+    public void testPageContainsAlbums() throws IOException {
+        URL url = new URL("https://www.erome.com/Jay-Jenna");
+        EromeRipper ripper = new EromeRipper(url);
+        assert(ripper.pageContainsAlbums(url));
+        assert(!ripper.pageContainsAlbums(new URL("https://www.erome.com/a/KbDAM1XT")));
+    }
+
+    public void testRip() throws IOException {
+        URL url = new URL("https://www.erome.com/a/4FqeUxil");
+        EromeRipper ripper = new EromeRipper(url);
+        testRipper(ripper);
+    }
+}

From da198cf0215d33cac76d9fee2063f28652cb75f0 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Mon, 25 Jun 2018 17:27:32 -0400
Subject: [PATCH 060/132] Added unit test for Myhentaicomics quickQueue

---
 .../ripper/rippers/MyhentaicomicsRipperTest.java   | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java
index b4d01032..5e6cfc7e 100644
--- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java
+++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java
@@ -21,4 +21,18 @@ public class MyhentaicomicsRipperTest extends RippersTest {
         // Test a tag
         assertEquals("2409", ripper.getGID(new URL("http://myhentaicomics.com/index.php/tag/2409/")));
     }
+
+    public void testGetAlbumsToQueue() throws IOException {
+        URL url = new URL("https://myhentaicomics.com/index.php/tag/3167/");
+        MyhentaicomicsRipper ripper = new MyhentaicomicsRipper(url);
+        assertEquals(15, ripper.getAlbumsToQueue(ripper.getFirstPage()).size());
+    }
+
+    public void testPageContainsAlbums() throws IOException {
+        URL url = new URL("https://myhentaicomics.com/index.php/tag/3167/");
+        URL url2 = new URL("https://myhentaicomics.com/index.php/search?q=test");
+        MyhentaicomicsRipper ripper = new MyhentaicomicsRipper(url);
+        assertTrue(ripper.pageContainsAlbums(url));
+        assertTrue(ripper.pageContainsAlbums(url2));
+    }
 }
\ No newline at end of file

From 96a8801c190f1b396373d642dad1278eab5c2101 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Tue, 26 Jun 2018 00:54:18 -0400
Subject: [PATCH 061/132] Rippers can now choose to use a byte progess bar

---
 .../ripme/ripper/AbstractRipper.java          |  3 ++
 .../ripme/ripper/DownloadFileThread.java      | 43 ++++++++++++++++++-
 2 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
index 87d8bd46..cbc86548 100644
--- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
@@ -613,4 +613,7 @@ public abstract class AbstractRipper
     protected boolean isThisATest() {
         return thisIsATest;
     }
+
+    // If true ripme uses a byte progress bar
+    protected boolean useByteProgessBar() { return false;}
 }
diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
index 42dedffe..768e1c12 100644
--- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
+++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
@@ -59,6 +59,16 @@ class DownloadFileThread extends Thread {
         this.cookies = cookies;
     }
 
+    private int getTotalBytes(URL url) throws IOException {
+        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+        conn.setRequestMethod("HEAD");
+        conn.setRequestProperty("accept",  "*/*");
+        conn.setRequestProperty("Referer", this.url.toExternalForm()); // Sic
+        conn.setRequestProperty("User-agent", AbstractRipper.USER_AGENT);
+        return conn.getContentLength();
+    }
+
+
     /**
      * Attempts to download the file. Retries as needed.
      * Notifies observers upon completion/error/warn.
@@ -81,6 +91,21 @@ class DownloadFileThread extends Thread {
             }
         }
 
+        int bytesTotal, bytesDownloaded = 0;
+        if (observer.useByteProgessBar()) {
+            try {
+                bytesTotal = getTotalBytes(this.url);
+            } catch (IOException e) {
+                logger.error("Failed to get file size at " + this.url, e);
+                observer.downloadErrored(this.url, "Failed to get file size of " + this.url);
+                return;
+            }
+
+            observer.setBytesTotal(bytesTotal);
+            observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal);
+            logger.debug("Size of file at " + this.url + " = " + bytesTotal + "b");
+        }
+
         URL urlToDownload = this.url;
         boolean redirected = false;
         int tries = 0; // Number of attempts to download
@@ -156,7 +181,23 @@ class DownloadFileThread extends Thread {
                 }
 
                 fos = new FileOutputStream(saveAs);
-                IOUtils.copy(bis, fos);
+                byte[] data = new byte[1024 * 256]; int bytesRead;
+                while ( (bytesRead = bis.read(data)) != -1) {
+                    try {
+                        observer.stopCheck();
+                    } catch (IOException e) {
+                        observer.downloadErrored(url, "Download interrupted");
+                        return;
+                    }
+                    fos.write(data, 0, bytesRead);
+                    if (observer.useByteProgessBar()) {
+                        bytesDownloaded += bytesRead;
+                        observer.setBytesCompleted(bytesDownloaded);
+                        observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded);
+                    }
+                }
+                bis.close();
+                fos.close();
                 break; // Download successful: break out of infinite loop
             } catch (HttpStatusException hse) {
                 logger.debug("HTTP status exception", hse);

From 9b70350b33684ccf87cf756a439619626e0f757e Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Tue, 26 Jun 2018 01:36:23 -0400
Subject: [PATCH 062/132] GfycatporntubeRipper now uses byte progess bar

---
 .../ripper/rippers/GfycatporntubeRipper.java  | 31 +++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java
index 504b89d6..d8f629ae 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java
@@ -8,6 +8,7 @@ import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import com.rarchives.ripme.utils.Utils;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 
@@ -58,4 +59,34 @@ public class GfycatporntubeRipper extends AbstractHTMLRipper {
     public void downloadURL(URL url, int index) {
         addURLToDownload(url, getPrefix(index));
     }
+
+    private int bytesTotal = 1;
+    private int bytesCompleted = 1;
+
+    @Override
+    public String getStatusText() {
+        return String.valueOf(getCompletionPercentage()) +
+                "%  - " +
+                Utils.bytesToHumanReadable(bytesCompleted) +
+                " / " +
+                Utils.bytesToHumanReadable(bytesTotal);
+    }
+
+    @Override
+    public int getCompletionPercentage() {
+        return (int) (100 * (bytesCompleted / (float) bytesTotal));
+    }
+
+    @Override
+    public void setBytesTotal(int bytes) {
+        this.bytesTotal = bytes;
+    }
+
+    @Override
+    public void setBytesCompleted(int bytes) {
+        this.bytesCompleted = bytes;
+    }
+
+    @Override
+    public boolean useByteProgessBar() {return true;}
 }

From e1635f39e17476b1e02df150d0aac070b67b351e Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Tue, 26 Jun 2018 02:28:14 -0400
Subject: [PATCH 063/132] GfycatRipper now uses AbstractHTMLRipper

---
 .../rippers/{video => }/GfycatRipper.java     | 75 ++++++++++++++++---
 .../com/rarchives/ripme/utils/RipUtils.java   |  2 +-
 2 files changed, 66 insertions(+), 11 deletions(-)
 rename src/main/java/com/rarchives/ripme/ripper/rippers/{video => }/GfycatRipper.java (54%)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java
similarity index 54%
rename from src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java
rename to src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java
index 75577597..78a0af19 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java
@@ -1,18 +1,23 @@
-package com.rarchives.ripme.ripper.rippers.video;
+package com.rarchives.ripme.ripper.rippers;
+
 
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import com.rarchives.ripme.ripper.AbstractHTMLRipper;
+import com.rarchives.ripme.utils.Utils;
 import org.jsoup.nodes.Document;
 import org.jsoup.select.Elements;
 
-import com.rarchives.ripme.ripper.VideoRipper;
 import com.rarchives.ripme.utils.Http;
 
-public class GfycatRipper extends VideoRipper {
+
+public class GfycatRipper extends AbstractHTMLRipper {
 
     private static final String HOST = "gfycat.com";
 
@@ -20,9 +25,14 @@ public class GfycatRipper extends VideoRipper {
         super(url);
     }
 
+    @Override
+    public String getDomain() {
+        return "gfycat.com";
+    }
+
     @Override
     public String getHost() {
-        return HOST;
+        return "gfycat";
     }
 
     @Override
@@ -37,6 +47,16 @@ public class GfycatRipper extends VideoRipper {
         return url;
     }
 
+    @Override
+    public Document getFirstPage() throws IOException {
+        return Http.url(url).get();
+    }
+
+    @Override
+    public void downloadURL(URL url, int index) {
+        addURLToDownload(url, getPrefix(index));
+    }
+
     @Override
     public String getGID(URL url) throws MalformedURLException {
         Pattern p = Pattern.compile("^https?://[wm.]*gfycat\\.com/([a-zA-Z0-9]+).*$");
@@ -52,10 +72,15 @@ public class GfycatRipper extends VideoRipper {
     }
 
     @Override
-    public void rip() throws IOException {
-        String vidUrl = getVideoURL(this.url);
-        addURLToDownload(new URL(vidUrl), "gfycat_" + getGID(this.url));
-        waitForThreads();
+    public List<String> getURLsFromPage(Document doc) {
+        List<String> result = new ArrayList<>();
+        Elements videos = doc.select("source#mp4Source");
+        String vidUrl = videos.first().attr("src");
+        if (vidUrl.startsWith("//")) {
+            vidUrl = "http:" + vidUrl;
+        }
+        result.add(vidUrl);
+        return result;
     }
 
     /**
@@ -66,10 +91,10 @@ public class GfycatRipper extends VideoRipper {
      */
     public static String getVideoURL(URL url) throws IOException {
         LOGGER.info("Retrieving " + url.toExternalForm());
-        
+
         //Sanitize the URL first
         url = new URL(url.toExternalForm().replace("/gifs/detail", ""));
-        
+
         Document doc = Http.url(url).get();
         Elements videos = doc.select("source#mp4Source");
         if (videos.isEmpty()) {
@@ -81,4 +106,34 @@ public class GfycatRipper extends VideoRipper {
         }
         return vidUrl;
     }
+
+    private int bytesTotal = 1;
+    private int bytesCompleted = 1;
+
+    @Override
+    public String getStatusText() {
+        return String.valueOf(getCompletionPercentage()) +
+                "%  - " +
+                Utils.bytesToHumanReadable(bytesCompleted) +
+                " / " +
+                Utils.bytesToHumanReadable(bytesTotal);
+    }
+
+    @Override
+    public int getCompletionPercentage() {
+        return (int) (100 * (bytesCompleted / (float) bytesTotal));
+    }
+
+    @Override
+    public void setBytesTotal(int bytes) {
+        this.bytesTotal = bytes;
+    }
+
+    @Override
+    public void setBytesCompleted(int bytes) {
+        this.bytesCompleted = bytes;
+    }
+
+    @Override
+    public boolean useByteProgessBar() {return true;}
 }
\ No newline at end of file
diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java
index 34081852..9845145c 100644
--- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java
+++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java
@@ -12,7 +12,7 @@ import com.rarchives.ripme.ripper.rippers.EroShareRipper;
 import com.rarchives.ripme.ripper.rippers.EromeRipper;
 import com.rarchives.ripme.ripper.rippers.ImgurRipper;
 import com.rarchives.ripme.ripper.rippers.VidbleRipper;
-import com.rarchives.ripme.ripper.rippers.video.GfycatRipper;
+import com.rarchives.ripme.ripper.rippers.GfycatRipper;
 import org.apache.commons.lang.math.NumberUtils;
 import org.apache.log4j.Logger;
 import org.jsoup.Jsoup;

From 1428b2cd6b57c057493c4f000064310aaa2038f9 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Tue, 26 Jun 2018 02:49:18 -0400
Subject: [PATCH 064/132] Fixed some unit tests that broke when the
 GfycatRipper was moved

---
 .../rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java    | 2 +-
 .../rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java    | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java
index ca73f138..d3d011be 100644
--- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java
+++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java
@@ -1,6 +1,6 @@
 package com.rarchives.ripme.tst.ripper.rippers;
 
-import com.rarchives.ripme.ripper.rippers.video.GfycatRipper;
+import com.rarchives.ripme.ripper.rippers.GfycatRipper;
 import java.io.IOException;
 import java.net.URL;
 
diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java
index fdd61cf9..efe23a01 100644
--- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java
+++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java
@@ -5,7 +5,6 @@ import java.io.IOException;
 import java.net.URL;
 
 import com.rarchives.ripme.ripper.rippers.RedditRipper;
-import com.rarchives.ripme.ripper.rippers.video.GfycatRipper;
 
 public class RedditRipperTest extends RippersTest {
     // https://github.com/RipMeApp/ripme/issues/253 - Disabled tests: RedditRipperTest#testRedditSubreddit*Rip is flaky

From b1718965c5e97ecb5e8df353138a5618a245f8df Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Tue, 26 Jun 2018 05:12:48 -0400
Subject: [PATCH 065/132] Removed ripper for dead site

---
 .../ripper/rippers/video/VineRipper.java      | 67 -------------------
 1 file changed, 67 deletions(-)
 delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/video/VineRipper.java

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VineRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VineRipper.java
deleted file mode 100644
index 1ca59676..00000000
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VineRipper.java
+++ /dev/null
@@ -1,67 +0,0 @@
-package com.rarchives.ripme.ripper.rippers.video;
-
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.jsoup.nodes.Document;
-import org.jsoup.select.Elements;
-
-import com.rarchives.ripme.ripper.VideoRipper;
-import com.rarchives.ripme.utils.Http;
-
-public class VineRipper extends VideoRipper {
-
-    private static final String HOST = "vine";
-
-    public VineRipper(URL url) throws IOException {
-        super(url);
-    }
-
-    @Override
-    public String getHost() {
-        return HOST;
-    }
-
-    @Override
-    public boolean canRip(URL url) {
-        // https://vine.co/v/hiqQrP0eUZx
-        Pattern p = Pattern.compile("^https?://[wm.]*vine\\.co/v/[a-zA-Z0-9]+.*$");
-        Matcher m = p.matcher(url.toExternalForm());
-        return m.matches();
-    }
-
-    @Override
-    public URL sanitizeURL(URL url) throws MalformedURLException {
-        return url;
-    }
-
-    @Override
-    public String getGID(URL url) throws MalformedURLException {
-        Pattern p = Pattern.compile("^https?://[wm.]*vine\\.co/v/([a-zA-Z0-9]+).*$");
-        Matcher m = p.matcher(url.toExternalForm());
-        if (m.matches()) {
-            return m.group(1);
-        }
-
-        throw new MalformedURLException(
-                "Expected vine format:"
-                        + "vine.co/v/####"
-                        + " Got: " + url);
-    }
-
-    @Override
-    public void rip() throws IOException {
-        LOGGER.info("    Retrieving " + this.url.toExternalForm());
-        Document doc = Http.url(this.url).get();
-        Elements props = doc.select("meta[property=twitter:player:stream]");
-        if (props.isEmpty()) {
-            throw new IOException("Could not find meta property 'twitter:player:stream' at " + url);
-        }
-        String vidUrl = props.get(0).attr("content");
-        addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url));
-        waitForThreads();
-    }
-}
\ No newline at end of file

From 04ca5a2054a51495fbd4f3e3049626c99fd60f04 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Wed, 27 Jun 2018 00:51:28 -0400
Subject: [PATCH 066/132] Added support for resuming downloads

---
 .../ripme/ripper/AbstractRipper.java          |  2 +
 .../ripme/ripper/DownloadFileThread.java      | 59 ++++++++++---------
 2 files changed, 33 insertions(+), 28 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
index cbc86548..94ac1ec9 100644
--- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
@@ -616,4 +616,6 @@ public abstract class AbstractRipper
 
     // If true ripme uses a byte progress bar
     protected boolean useByteProgessBar() { return false;}
+    // If true ripme will try to resume a broken download for this ripper
+    protected boolean tryResumeDownload() { return false;}
 }
diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
index 768e1c12..45c72c6e 100644
--- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
+++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
@@ -59,28 +59,25 @@ class DownloadFileThread extends Thread {
         this.cookies = cookies;
     }
 
-    private int getTotalBytes(URL url) throws IOException {
-        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-        conn.setRequestMethod("HEAD");
-        conn.setRequestProperty("accept",  "*/*");
-        conn.setRequestProperty("Referer", this.url.toExternalForm()); // Sic
-        conn.setRequestProperty("User-agent", AbstractRipper.USER_AGENT);
-        return conn.getContentLength();
-    }
-
 
     /**
      * Attempts to download the file. Retries as needed.
      * Notifies observers upon completion/error/warn.
      */
     public void run() {
+        long fileSize = 0;
+        int bytesTotal = 0;
+        int bytesDownloaded = 0;
+        if (saveAs.exists() && observer.tryResumeDownload()) {
+            fileSize = saveAs.length();
+        }
         try {
             observer.stopCheck();
         } catch (IOException e) {
             observer.downloadErrored(url, "Download interrupted");
             return;
         }
-        if (saveAs.exists()) {
+        if (saveAs.exists() && !observer.tryResumeDownload()) {
             if (Utils.getConfigBoolean("file.overwrite", false)) {
                 logger.info("[!] Deleting existing file" + prettySaveAs);
                 saveAs.delete();
@@ -90,22 +87,6 @@ class DownloadFileThread extends Thread {
                 return;
             }
         }
-
-        int bytesTotal, bytesDownloaded = 0;
-        if (observer.useByteProgessBar()) {
-            try {
-                bytesTotal = getTotalBytes(this.url);
-            } catch (IOException e) {
-                logger.error("Failed to get file size at " + this.url, e);
-                observer.downloadErrored(this.url, "Failed to get file size of " + this.url);
-                return;
-            }
-
-            observer.setBytesTotal(bytesTotal);
-            observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal);
-            logger.debug("Size of file at " + this.url + " = " + bytesTotal + "b");
-        }
-
         URL urlToDownload = this.url;
         boolean redirected = false;
         int tries = 0; // Number of attempts to download
@@ -139,11 +120,20 @@ class DownloadFileThread extends Thread {
                     cookie += key + "=" + cookies.get(key);
                 }
                 huc.setRequestProperty("Cookie", cookie);
+                if (observer.tryResumeDownload()) {
+                    if (fileSize != 0) {
+                        huc.setRequestProperty("Range", "bytes=" + fileSize + "-");
+                    }
+                }
                 logger.debug("Request properties: " + huc.getRequestProperties());
                 huc.connect();
 
                 int statusCode = huc.getResponseCode();
                 logger.debug("Status code: " + statusCode);
+                if (statusCode != 206 && observer.tryResumeDownload()) {
+                    // TODO find a better way to handle servers that don't support resuming downloads then just erroring out
+                    throw new IOException("Server doesn't support resuming downloads");
+                }
                 if (statusCode  / 100 == 3) { // 3xx Redirect
                     if (!redirected) {
                         // Don't increment retries on the first redirect
@@ -171,6 +161,15 @@ class DownloadFileThread extends Thread {
                     observer.downloadErrored(url, "Imgur image is 404: " + url.toExternalForm());
                     return;
                 }
+
+                // If the ripper is using the bytes progress bar set bytesTotal to huc.getContentLength()
+                if (observer.useByteProgessBar()) {
+                    bytesTotal = huc.getContentLength();
+                    observer.setBytesTotal(bytesTotal);
+                    observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal);
+                    logger.debug("Size of file at " + this.url + " = " + bytesTotal + "b");
+                }
+
                 // Save file
                 bis = new BufferedInputStream(huc.getInputStream());
 
@@ -179,8 +178,12 @@ class DownloadFileThread extends Thread {
                     String fileExt = URLConnection.guessContentTypeFromStream(bis).replaceAll("image/", "");
                     saveAs = new File(saveAs.toString() + "." + fileExt);
                 }
-
-                fos = new FileOutputStream(saveAs);
+                // If we're resuming a download we append data to the existing file
+                if (statusCode == 206) {
+                    fos = new FileOutputStream(saveAs, true);
+                } else {
+                    fos = new FileOutputStream(saveAs);
+                }
                 byte[] data = new byte[1024 * 256]; int bytesRead;
                 while ( (bytesRead = bis.read(data)) != -1) {
                     try {

From 2c08be83f69b466baf8dbe389d0a85fa8f1830d0 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Wed, 27 Jun 2018 00:58:55 -0400
Subject: [PATCH 067/132] Removed unit test for vine ripper has the ripper was
 removed

---
 .../ripme/tst/ripper/rippers/VideoRippersTest.java  | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java
index 6a7df184..05d7514e 100644
--- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java
+++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java
@@ -8,7 +8,6 @@ import java.util.List;
 import com.rarchives.ripme.ripper.VideoRipper;
 import com.rarchives.ripme.ripper.rippers.video.PornhubRipper;
 import com.rarchives.ripme.ripper.rippers.video.TwitchVideoRipper;
-import com.rarchives.ripme.ripper.rippers.video.VineRipper;
 import com.rarchives.ripme.ripper.rippers.video.XhamsterRipper;
 import com.rarchives.ripme.ripper.rippers.video.XvideosRipper;
 import com.rarchives.ripme.ripper.rippers.video.YoupornRipper;
@@ -78,18 +77,6 @@ public class VideoRippersTest extends RippersTest {
         }
     }
 
-    // https://github.com/RipMeApp/ripme/issues/186
-    /*
-    public void testVineRipper() throws IOException {
-        List<URL> contentURLs = new ArrayList<>();
-        contentURLs.add(new URL("https://vine.co/v/hiqQrP0eUZx"));
-        for (URL url : contentURLs) {
-            VineRipper ripper = new VineRipper(url);
-            videoTestHelper(ripper);
-        }
-    }
-    */
-
     public void testYoupornRipper() throws IOException {
         List<URL> contentURLs = new ArrayList<>();
         contentURLs.add(new URL("http://www.youporn.com/watch/7669155/mrs-li-amateur-69-orgasm/?from=categ"));

From ed7df108e5c974d36fc32b43397f560a0a44b72c Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Wed, 27 Jun 2018 01:02:23 -0400
Subject: [PATCH 068/132] Minor code style clean up

---
 .../java/com/rarchives/ripme/ripper/DownloadFileThread.java    | 3 ++-
 .../java/com/rarchives/ripme/ripper/DownloadVideoThread.java   | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
index 45c72c6e..d4439a74 100644
--- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
+++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
@@ -184,7 +184,8 @@ class DownloadFileThread extends Thread {
                 } else {
                     fos = new FileOutputStream(saveAs);
                 }
-                byte[] data = new byte[1024 * 256]; int bytesRead;
+                byte[] data = new byte[1024 * 256];
+                int bytesRead;
                 while ( (bytesRead = bis.read(data)) != -1) {
                     try {
                         observer.stopCheck();
diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java
index 437f18d0..ef55e54e 100644
--- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java
+++ b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java
@@ -76,7 +76,8 @@ class DownloadVideoThread extends Thread {
         int tries = 0; // Number of attempts to download
         do {
             InputStream bis = null; OutputStream fos = null;
-            byte[] data = new byte[1024 * 256]; int bytesRead;
+            byte[] data = new byte[1024 * 256];
+            int bytesRead;
             try {
                 logger.info("    Downloading file: " + url + (tries > 0 ? " Retry #" + tries : ""));
                 observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm());

From 1f8e9184c7a1522f37e4910be44dfbee7c2d3e06 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Wed, 27 Jun 2018 01:24:04 -0400
Subject: [PATCH 069/132] Added the helper func getByteStatusText to cut down
 on copy pasted code

---
 .../ripme/ripper/rippers/GfycatRipper.java       |  6 +-----
 .../ripper/rippers/GfycatporntubeRipper.java     |  6 +-----
 .../java/com/rarchives/ripme/utils/Utils.java    | 16 ++++++++++++++++
 3 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java
index 78a0af19..97afef94 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java
@@ -112,11 +112,7 @@ public class GfycatRipper extends AbstractHTMLRipper {
 
     @Override
     public String getStatusText() {
-        return String.valueOf(getCompletionPercentage()) +
-                "%  - " +
-                Utils.bytesToHumanReadable(bytesCompleted) +
-                " / " +
-                Utils.bytesToHumanReadable(bytesTotal);
+        return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal);
     }
 
     @Override
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java
index d8f629ae..9010eb0e 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java
@@ -65,11 +65,7 @@ public class GfycatporntubeRipper extends AbstractHTMLRipper {
 
     @Override
     public String getStatusText() {
-        return String.valueOf(getCompletionPercentage()) +
-                "%  - " +
-                Utils.bytesToHumanReadable(bytesCompleted) +
-                " / " +
-                Utils.bytesToHumanReadable(bytesTotal);
+        return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal);
     }
 
     @Override
diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java
index 5821e9f3..e57acf77 100644
--- a/src/main/java/com/rarchives/ripme/utils/Utils.java
+++ b/src/main/java/com/rarchives/ripme/utils/Utils.java
@@ -715,4 +715,20 @@ public class Utils {
         }
     }
 
+    /**
+     * Formats and reuturns the status text for rippers using the byte progress bar
+     *
+     * @param completionPercentage An int between 0 and 100 which repersents how close the download is to complete
+     * @param bytesCompleted How many bytes have been downloaded
+     * @param bytesTotal The total size of the file that is being downloaded
+     * @return Returns the formatted status text for rippers using the byte progress bar
+     */
+    public static String getByteStatusText(int completionPercentage, int bytesCompleted, int bytesTotal) {
+        return String.valueOf(completionPercentage) +
+                "%  - " +
+                Utils.bytesToHumanReadable(bytesCompleted) +
+                " / " +
+                Utils.bytesToHumanReadable(bytesTotal);
+    }
+
 }
\ No newline at end of file

From af064ee435f60365d56c2a94cb8567bd1c26d8f4 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Wed, 27 Jun 2018 01:25:58 -0400
Subject: [PATCH 070/132] Minor code refactoring

---
 .../com/rarchives/ripme/ripper/rippers/GfycatRipper.java    | 6 +++---
 .../ripme/ripper/rippers/GfycatporntubeRipper.java          | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java
index 97afef94..a09d68ab 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java
@@ -19,6 +19,9 @@ import com.rarchives.ripme.utils.Http;
 
 public class GfycatRipper extends AbstractHTMLRipper {
 
+    private int bytesTotal = 1;
+    private int bytesCompleted = 1;
+
     private static final String HOST = "gfycat.com";
 
     public GfycatRipper(URL url) throws IOException {
@@ -107,9 +110,6 @@ public class GfycatRipper extends AbstractHTMLRipper {
         return vidUrl;
     }
 
-    private int bytesTotal = 1;
-    private int bytesCompleted = 1;
-
     @Override
     public String getStatusText() {
         return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal);
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java
index 9010eb0e..55150d9e 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java
@@ -17,6 +17,9 @@ import com.rarchives.ripme.utils.Http;
 
 public class GfycatporntubeRipper extends AbstractHTMLRipper {
 
+    private int bytesTotal = 1;
+    private int bytesCompleted = 1;
+
     public GfycatporntubeRipper(URL url) throws IOException {
         super(url);
     }
@@ -60,9 +63,6 @@ public class GfycatporntubeRipper extends AbstractHTMLRipper {
         addURLToDownload(url, getPrefix(index));
     }
 
-    private int bytesTotal = 1;
-    private int bytesCompleted = 1;
-
     @Override
     public String getStatusText() {
         return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal);

From ffbda6032d681502864fc208d845b3cd3ac89927 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Wed, 27 Jun 2018 02:05:36 -0400
Subject: [PATCH 071/132] Fixed bug that caused downloads from sites that
 support download resuming to fail with 'Server doesn't support resuming
 downloads'

---
 .../java/com/rarchives/ripme/ripper/DownloadFileThread.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
index d4439a74..10e4462c 100644
--- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
+++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
@@ -130,7 +130,7 @@ class DownloadFileThread extends Thread {
 
                 int statusCode = huc.getResponseCode();
                 logger.debug("Status code: " + statusCode);
-                if (statusCode != 206 && observer.tryResumeDownload()) {
+                if (statusCode != 206 && observer.tryResumeDownload() && saveAs.exists()) {
                     // TODO find a better way to handle servers that don't support resuming downloads then just erroring out
                     throw new IOException("Server doesn't support resuming downloads");
                 }

From e83032906d36275d51ae52fdbf55836124230595 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Thu, 28 Jun 2018 21:04:05 -0400
Subject: [PATCH 072/132] Started work on fixing da ripper

---
 .../ripper/rippers/DeviantartRipper.java      | 45 ++++++++-----------
 1 file changed, 19 insertions(+), 26 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
index 2afae2dc..28ab9e5e 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
@@ -66,7 +66,7 @@ public class DeviantartRipper extends AbstractHTMLRipper {
             u += "gallery/?";
         }
 
-        Pattern p = Pattern.compile("^https?://([a-zA-Z0-9\\-]+)\\.deviantart\\.com/favou?rites/([0-9]+)/*?$");
+        Pattern p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/favou?rites/([0-9]+)/*?$");
         Matcher m = p.matcher(url.toExternalForm());
         if (!m.matches()) {
             String subdir = "/";
@@ -80,7 +80,7 @@ public class DeviantartRipper extends AbstractHTMLRipper {
 
     @Override
     public String getGID(URL url) throws MalformedURLException {
-        Pattern p = Pattern.compile("^https?://([a-zA-Z0-9\\-]+)\\.deviantart\\.com(/gallery)?/?(\\?.*)?$");
+        Pattern p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)(/gallery)?/?(\\?.*)?$");
         Matcher m = p.matcher(url.toExternalForm());
         if (m.matches()) {
             // Root gallery
@@ -91,24 +91,24 @@ public class DeviantartRipper extends AbstractHTMLRipper {
                 return m.group(1);
             }
         }
-        p = Pattern.compile("^https?://([a-zA-Z0-9\\-]+)\\.deviantart\\.com/gallery/([0-9]+).*$");
+        p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/gallery/([0-9]+).*$");
         m = p.matcher(url.toExternalForm());
         if (m.matches()) {
             // Subgallery
             return m.group(1) + "_" + m.group(2);
         }
-        p = Pattern.compile("^https?://([a-zA-Z0-9\\-]+)\\.deviantart\\.com/favou?rites/([0-9]+)/.*?$");
+        p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/favou?rites/([0-9]+)/.*?$");
         m = p.matcher(url.toExternalForm());
         if (m.matches()) {
             return m.group(1) + "_faves_" + m.group(2);
         }
-        p = Pattern.compile("^https?://([a-zA-Z0-9\\-]+)\\.deviantart\\.com/favou?rites/?$");
+        p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/favou?rites/?$");
         m = p.matcher(url.toExternalForm());
         if (m.matches()) {
             // Subgallery
             return m.group(1) + "_faves";
         }
-        throw new MalformedURLException("Expected URL format: http://username.deviantart.com/[/gallery/#####], got: " + url);
+        throw new MalformedURLException("Expected URL format: http://www.deviantart.com/username[/gallery/#####], got: " + url);
     }
 
     /**
@@ -238,26 +238,19 @@ public class DeviantartRipper extends AbstractHTMLRipper {
         if (isThisATest()) {
             return null;
         }
-        Elements nextButtons = page.select("link[rel=\"next\"]");
-        if (nextButtons.isEmpty()) {
-            if (page.select("link[rel=\"prev\"]").isEmpty()) {
-                throw new IOException("No next page found");
-            } else {
-                throw new IOException("Hit end of pages");
-            }
-        }
-        Element a = nextButtons.first();
-        String nextPage = a.attr("href");
-        if (nextPage.startsWith("/")) {
-            nextPage = "http://" + this.url.getHost() + nextPage;
-        }
-        if (!sleep(PAGE_SLEEP_TIME)) {
-            throw new IOException("Interrupted while waiting to load next page: " + nextPage);
-        }
-        LOGGER.info("Found next page: " + nextPage);
-        return Http.url(nextPage)
-                   .cookies(cookies)
-                   .get();
+        String baseURL = "https://www.deviantart.com/dapi/v1/gallery/";
+        String id = page.select("div[gmi-name=gallery]").first().attr("gmi-itemid");
+        baseURL = baseURL + id;
+        String requestID = getRequestID(page);
+        Document d = Http.url(baseURL).data("idd", requestID).post();
+        LOGGER.info(d.html());
+        return d;
+    }
+
+    private String getRequestID(Document doc) {
+        Pattern p = Pattern.compile("requestid\":\"([a-zA-Z0-9]+)\"");
+        Matcher m = p.matcher(doc.html());
+        return "590m257da2ea3eea661e272dde2948081c4d";
     }
 
     @Override

From 17a3e4eb5094d03961e5ffb9ca06dd2b19810ebb Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Fri, 29 Jun 2018 21:09:06 -0400
Subject: [PATCH 073/132] DeviantartRipper can now rip more than one page again

---
 .../ripper/rippers/DeviantartRipper.java      | 303 +++++++++---------
 1 file changed, 145 insertions(+), 158 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
index 28ab9e5e..1d7fb17c 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
@@ -1,6 +1,6 @@
 package com.rarchives.ripme.ripper.rippers;
 
-import com.rarchives.ripme.ripper.AbstractHTMLRipper;
+import com.rarchives.ripme.ripper.AbstractJSONRipper;
 import com.rarchives.ripme.utils.Base64;
 import com.rarchives.ripme.utils.Http;
 import com.rarchives.ripme.utils.RipUtils;
@@ -18,15 +18,23 @@ import java.util.Map;
 import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import org.jsoup.Connection.Method;
+
+import org.json.JSONArray;
+import org.json.JSONObject;
 import org.jsoup.Connection.Response;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
-import org.jsoup.safety.Whitelist;
 import org.jsoup.select.Elements;
 
-public class DeviantartRipper extends AbstractHTMLRipper {
+
+public class DeviantartRipper extends AbstractJSONRipper {
+    String requestID;
+    String galleryID;
+    String username;
+    String baseApiUrl = "https://www.deviantart.com/dapi/v1/gallery/";
+    String csrf;
+    Map<String, String> pageCookies = new HashMap<>();
 
     private static final int PAGE_SLEEP_TIME  = 3000,
                              IMAGE_SLEEP_TIME = 2000;
@@ -50,10 +58,10 @@ public class DeviantartRipper extends AbstractHTMLRipper {
     public String getDomain() {
         return "deviantart.com";
     }
-    @Override
-    public boolean hasDescriptionSupport() {
-        return true;
-    }
+//    @Override
+//    public boolean hasDescriptionSupport() {
+//        return true;
+//    }
     @Override
     public URL sanitizeURL(URL url) throws MalformedURLException {
         String u = url.toExternalForm();
@@ -120,7 +128,7 @@ public class DeviantartRipper extends AbstractHTMLRipper {
      * @throws IOException 
      */
     @Override
-    public Document getFirstPage() throws IOException {
+    public JSONObject getFirstPage() throws IOException {
         
         // Base64 da login
         // username: Z3JhYnB5
@@ -133,124 +141,103 @@ public class DeviantartRipper extends AbstractHTMLRipper {
                 cookies.put("agegate_state","1"); // Bypasses the age gate
             }
             
-        return Http.url(this.url)
+        Response res = Http.url(this.url)
                    .cookies(cookies)
-                   .get();
+                   .response();
+        Document page = res.parse();
+
+        JSONObject firstPageJSON = getFirstPageJSON(page);
+        requestID = firstPageJSON.getJSONObject("dapx").getString("requestid");
+        galleryID = page.select("input[name=set]").attr("value");
+        username = page.select("div.tt-tv150").attr("username");
+        csrf = firstPageJSON.getString("csrf");
+        pageCookies = res.cookies();
+
+        return requestPage(0, galleryID, username, requestID, csrf, pageCookies);
     }
-    
-    /**
-     * 
-     * @param page
-     * @param id
-     * @return 
-     */
-    private String jsonToImage(Document page, String id) {
-        Elements js = page.select("script[type=\"text/javascript\"]");
-        for (Element tag : js) {
-            if (tag.html().contains("window.__pageload")) {
-                try {
-                    String script = tag.html();
-                    script = script.substring(script.indexOf("window.__pageload"));
-                    if (!script.contains(id)) {
-                        continue;
-                    }
-                    script = script.substring(script.indexOf(id));
-                    // first },"src":"url" after id
-                    script = script.substring(script.indexOf("},\"src\":\"") + 9, script.indexOf("\",\"type\""));
-                    return script.replace("\\/", "/");
-                } catch (StringIndexOutOfBoundsException e) {
-                    LOGGER.debug("Unable to get json link from " + page.location());
-                }
+
+    private JSONObject requestPage(int offset, String galleryID, String username, String requestID, String csfr, Map<String, String> c) {
+        LOGGER.debug("offset: " + Integer.toString(offset));
+        LOGGER.debug("galleryID: " + galleryID);
+        LOGGER.debug("username: " + username);
+        LOGGER.debug("requestID: " + requestID);
+        String url = baseApiUrl + galleryID + "?iid=" + requestID;
+        try {
+            Document doc = Http.url(url).cookies(c).data("username", username).data("offset", Integer.toString(offset))
+                    .data("limit", "24").data("_csrf", csfr).data("id", requestID)
+                    .ignoreContentType().post();
+            return new JSONObject(doc.body().text());
+        } catch (IOException e) {
+            LOGGER.error("Got error trying to get page: " + e.getMessage());
+            e.printStackTrace();
+            return null;
+        }
+
+
+    }
+
+    private JSONObject getFirstPageJSON(Document doc) {
+        for (Element js : doc.select("script")) {
+            LOGGER.info(js.html());
+            if (js.html().contains("requestid")) {
+                String json = js.html().replaceAll("window.__initial_body_data=", "").replaceAll("\\);", "")
+                        .replaceAll(";__wake\\(.+", "");
+                LOGGER.info("json: " + json);
+                JSONObject j = new JSONObject(json);
+                return j;
             }
         }
         return null;
     }
+    
+
     @Override
-    public List<String> getURLsFromPage(Document page) {
+    public List<String> getURLsFromJSON(JSONObject json) {
         List<String> imageURLs = new ArrayList<>();
+        LOGGER.info(json);
+        JSONArray results = json.getJSONObject("content").getJSONArray("results");
+        for (int i = 0; i < results.length(); i++) {
+            LOGGER.info(results.getJSONObject(i).toString());
+            Document doc = Jsoup.parseBodyFragment(results.getJSONObject(i).getString("html"));
+            try {
+                String imageURL = doc.select("span").first().attr("data-super-full-img");
+                if (!imageURL.isEmpty()) {
+                    imageURLs.add(imageURL);
+                }
+            } catch (NullPointerException e) {
+               LOGGER.info(i + " does not contain any images");
+            }
 
-        // Iterate over all thumbnails
-        for (Element thumb : page.select("div.zones-container span.thumb")) {
-            if (isStopped()) {
-                break;
-            }
-            Element img = thumb.select("img").get(0);
-            if (img.attr("transparent").equals("false")) {
-                continue; // a.thumbs to other albums are invisible
-            }
-            // Get full-sized image via helper methods
-            String fullSize = null;
-            if (thumb.attr("data-super-full-img").contains("//orig")) {
-                fullSize = thumb.attr("data-super-full-img");
-            } else {
-                String spanUrl = thumb.attr("href");
-                String fullSize1 = jsonToImage(page,spanUrl.substring(spanUrl.lastIndexOf('-') + 1));
-                if (fullSize1 == null || !fullSize1.contains("//orig")) {
-                    fullSize = smallToFull(img.attr("src"), spanUrl);
-                }
-                if (fullSize == null && fullSize1 != null) {
-                    fullSize = fullSize1;
-                }
-            }
-            if (fullSize == null) {
-                if (thumb.attr("data-super-full-img") != null) {
-                    fullSize = thumb.attr("data-super-full-img");
-                } else if (thumb.attr("data-super-img") != null) {
-                    fullSize = thumb.attr("data-super-img");
-                } else {
-                    continue;
-                }
-            }
-            if (triedURLs.contains(fullSize)) {
-                LOGGER.warn("Already tried to download " + fullSize);
-                continue;
-            }
-            triedURLs.add(fullSize);
-            imageURLs.add(fullSize);
-
-            if (isThisATest()) {
-                // Only need one image for a test
-                break;
-            }
         }
         return imageURLs;
     }
-    @Override
-    public List<String> getDescriptionsFromPage(Document page) {
-        List<String> textURLs = new ArrayList<>();
-        // Iterate over all thumbnails
-        for (Element thumb : page.select("div.zones-container span.thumb")) {
-            LOGGER.info(thumb.attr("href"));
-            if (isStopped()) {
-                break;
-            }
-            Element img = thumb.select("img").get(0);
-            if (img.attr("transparent").equals("false")) {
-                continue; // a.thumbs to other albums are invisible
-            }
-            textURLs.add(thumb.attr("href"));
+//    @Override
+//    public List<String> getDescriptionsFromPage(Document page) {
+//        List<String> textURLs = new ArrayList<>();
+//        // Iterate over all thumbnails
+//        for (Element thumb : page.select("div.zones-container span.thumb")) {
+//            LOGGER.info(thumb.attr("href"));
+//            if (isStopped()) {
+//                break;
+//            }
+//            Element img = thumb.select("img").get(0);
+//            if (img.attr("transparent").equals("false")) {
+//                continue; // a.thumbs to other albums are invisible
+//            }
+//            textURLs.add(thumb.attr("href"));
+//
+//        }
+//        return textURLs;
+//    }
 
-        }
-        return textURLs;
-    }
     @Override
-    public Document getNextPage(Document page) throws IOException {
-        if (isThisATest()) {
-            return null;
+    public JSONObject getNextPage(JSONObject page) throws IOException {
+        boolean hasMore = page.getJSONObject("content").getBoolean("has_more");
+        if (hasMore) {
+            return requestPage(page.getJSONObject("content").getInt("next_offset"), galleryID, username, requestID, csrf, pageCookies);
         }
-        String baseURL = "https://www.deviantart.com/dapi/v1/gallery/";
-        String id = page.select("div[gmi-name=gallery]").first().attr("gmi-itemid");
-        baseURL = baseURL + id;
-        String requestID = getRequestID(page);
-        Document d = Http.url(baseURL).data("idd", requestID).post();
-        LOGGER.info(d.html());
-        return d;
-    }
 
-    private String getRequestID(Document doc) {
-        Pattern p = Pattern.compile("requestid\":\"([a-zA-Z0-9]+)\"");
-        Matcher m = p.matcher(doc.html());
-        return "590m257da2ea3eea661e272dde2948081c4d";
+        throw new IOException("No more pages");
     }
 
     @Override
@@ -299,53 +286,53 @@ public class DeviantartRipper extends AbstractHTMLRipper {
      * @param page The gallery page the URL was found on
      * @return A String[] with first object being the description, and the second object being image file name if found.
      */
-    @Override
-    public String[] getDescription(String url,Document page) {
-        if (isThisATest()) {
-            return null;
-        }
-        try {
-            // Fetch the image page
-            Response resp = Http.url(url)
-                                .referrer(this.url)
-                                .cookies(cookies)
-                                .response();
-            cookies.putAll(resp.cookies());
-
-            // Try to find the description
-            Document documentz = resp.parse();
-            Element ele = documentz.select("div.dev-description").first();
-            if (ele == null) {
-                throw new IOException("No description found");
-            }
-            documentz.outputSettings(new Document.OutputSettings().prettyPrint(false));
-            ele.select("br").append("\\n");
-            ele.select("p").prepend("\\n\\n");
-            String fullSize = null;
-            Element thumb = page.select("div.zones-container span.thumb[href=\"" + url + "\"]").get(0);
-            if (!thumb.attr("data-super-full-img").isEmpty()) {
-                fullSize = thumb.attr("data-super-full-img");
-                String[] split = fullSize.split("/");
-                fullSize = split[split.length - 1];
-            } else {
-                String spanUrl = thumb.attr("href");
-                fullSize = jsonToImage(page,spanUrl.substring(spanUrl.lastIndexOf('-') + 1));
-                if (fullSize != null) {
-                    String[] split = fullSize.split("/");
-                    fullSize = split[split.length - 1];
-                }
-            }
-            if (fullSize == null) {
-                return new String[] {Jsoup.clean(ele.html().replaceAll("\\\\n", System.getProperty("line.separator")), "", Whitelist.none(), new Document.OutputSettings().prettyPrint(false))};
-            }
-            fullSize = fullSize.substring(0, fullSize.lastIndexOf("."));
-            return new String[] {Jsoup.clean(ele.html().replaceAll("\\\\n", System.getProperty("line.separator")), "", Whitelist.none(), new Document.OutputSettings().prettyPrint(false)),fullSize};
-            // TODO Make this not make a newline if someone just types \n into the description.
-        } catch (IOException ioe) {
-                LOGGER.info("Failed to get description at " + url + ": '" + ioe.getMessage() + "'");
-                return null;
-        }
-    }
+//    @Override
+//    public String[] getDescription(String url,Document page) {
+//        if (isThisATest()) {
+//            return null;
+//        }
+//        try {
+//            // Fetch the image page
+//            Response resp = Http.url(url)
+//                                .referrer(this.url)
+//                                .cookies(cookies)
+//                                .response();
+//            cookies.putAll(resp.cookies());
+//
+//            // Try to find the description
+//            Document documentz = resp.parse();
+//            Element ele = documentz.select("div.dev-description").first();
+//            if (ele == null) {
+//                throw new IOException("No description found");
+//            }
+//            documentz.outputSettings(new Document.OutputSettings().prettyPrint(false));
+//            ele.select("br").append("\\n");
+//            ele.select("p").prepend("\\n\\n");
+//            String fullSize = null;
+//            Element thumb = page.select("div.zones-container span.thumb[href=\"" + url + "\"]").get(0);
+//            if (!thumb.attr("data-super-full-img").isEmpty()) {
+//                fullSize = thumb.attr("data-super-full-img");
+//                String[] split = fullSize.split("/");
+//                fullSize = split[split.length - 1];
+//            } else {
+//                String spanUrl = thumb.attr("href");
+//                fullSize = jsonToImage(page,spanUrl.substring(spanUrl.lastIndexOf('-') + 1));
+//                if (fullSize != null) {
+//                    String[] split = fullSize.split("/");
+//                    fullSize = split[split.length - 1];
+//                }
+//            }
+//            if (fullSize == null) {
+//                return new String[] {Jsoup.clean(ele.html().replaceAll("\\\\n", System.getProperty("line.separator")), "", Whitelist.none(), new Document.OutputSettings().prettyPrint(false))};
+//            }
+//            fullSize = fullSize.substring(0, fullSize.lastIndexOf("."));
+//            return new String[] {Jsoup.clean(ele.html().replaceAll("\\\\n", System.getProperty("line.separator")), "", Whitelist.none(), new Document.OutputSettings().prettyPrint(false)),fullSize};
+//            // TODO Make this not make a newline if someone just types \n into the description.
+//        } catch (IOException ioe) {
+//                LOGGER.info("Failed to get description at " + url + ": '" + ioe.getMessage() + "'");
+//                return null;
+//        }
+//    }
 
     /**
      * If largest resolution for image at 'thumb' is found, starts downloading

From c6aa3a2af9caf6b31c5901ea43a6dc37c9da310f Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Fri, 29 Jun 2018 23:04:13 -0400
Subject: [PATCH 074/132] Fixed DeviantartRipper unit tests

---
 .../ripme/tst/ripper/rippers/DeviantartRipperTest.java      | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java
index a3e6a9c8..f68d1db5 100644
--- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java
+++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java
@@ -7,18 +7,18 @@ import com.rarchives.ripme.ripper.rippers.DeviantartRipper;
 
 public class DeviantartRipperTest extends RippersTest {
     public void testDeviantartAlbum() throws IOException {
-        DeviantartRipper ripper = new DeviantartRipper(new URL("http://airgee.deviantart.com/gallery/"));
+        DeviantartRipper ripper = new DeviantartRipper(new URL("https://www.deviantart.com/airgee/gallery/"));
         testRipper(ripper);
     }
 
     public void testDeviantartNSFWAlbum() throws IOException {
         // NSFW gallery
-        DeviantartRipper ripper = new DeviantartRipper(new URL("http://faterkcx.deviantart.com/gallery/"));
+        DeviantartRipper ripper = new DeviantartRipper(new URL("https://www.deviantart.com/faterkcx/gallery/"));
         testRipper(ripper);
     }
 
     public void testGetGID() throws IOException {
-        URL url = new URL("http://airgee.deviantart.com/gallery/");
+        URL url = new URL("https://www.deviantart.com/airgee/gallery/");
         DeviantartRipper ripper = new DeviantartRipper(url);
         assertEquals("airgee", ripper.getGID(url));
     }

From ef207890fc15258c8d375b59781b65283f86df0b Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sat, 30 Jun 2018 18:35:13 -0400
Subject: [PATCH 075/132] 1.7.56: Fixed DeviantartRipper ripper; Added support
 for resuming file downloads; Fixed erome ripper; Fixed ModelmayhemRipper NSFW
 image downloading

---
 pom.xml                                               | 2 +-
 ripme.json                                            | 5 +++--
 src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/pom.xml b/pom.xml
index b9c412ff..4bf06c19 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
   <groupId>com.rarchives.ripme</groupId>
   <artifactId>ripme</artifactId>
   <packaging>jar</packaging>
-  <version>1.7.55</version>
+  <version>1.7.56</version>
   <name>ripme</name>
   <url>http://rip.rarchives.com</url>
   <properties>
diff --git a/ripme.json b/ripme.json
index 2c6982c3..42f0b62a 100644
--- a/ripme.json
+++ b/ripme.json
@@ -1,6 +1,6 @@
 {
-    "latestVersion": "1.7.55",
     "changeList": [
+        "1.7.56: Fixed DeviantartRipper ripper; Added support for resuming file downloads; Fixed erome ripper; Fixed ModelmayhemRipper NSFW image downloading",
         "1.7.55: Fixed instagram ripper; Reddit ripper now respects history.end_rip_after_already_seen; Improvements to patch.py and release.py",
         "1.7.54: Fixed twitter ripper video downloading; fixed instagram ripper",
         "1.7.53: Added Picstatio ripper; Fixed instagram ripper; Reddit ripper now gets videos from v.redd.it; Fixed ZikiRipper getAlbumTitle; fixed twitter ripper",
@@ -227,5 +227,6 @@
         "1.0.3: Added VK.com ripper",
         "1.0.1: Added auto-update functionality"
     ],
-    "currentHash": "1df13a792f17e2e36a2c3a62527d0e97a3edbee14d1bdb0cd822c7d2a8ce3cf4"
+    "currentHash": "1a6b86e2127b69db0461b7085dcd4d2209b3d344834cd700b5d94154684a5d74",
+    "latestVersion": "1.7.56"
 }
\ No newline at end of file
diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
index 2ca91ba8..c474b6c9 100644
--- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
+++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
@@ -20,7 +20,7 @@ import com.rarchives.ripme.utils.Utils;
 public class UpdateUtils {
 
     private static final Logger logger = Logger.getLogger(UpdateUtils.class);
-    private static final String DEFAULT_VERSION = "1.7.55";
+    private static final String DEFAULT_VERSION = "1.7.56";
     private static final String REPO_NAME = "ripmeapp/ripme";
     private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json";
     private static final String mainFileName = "ripme.jar";

From 33a692be24a47fd0637c6d6e2da8b57390baafbb Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sat, 30 Jun 2018 18:38:53 -0400
Subject: [PATCH 076/132] Updated hash because wrong version of ripme was built

---
 ripme.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ripme.json b/ripme.json
index 42f0b62a..d98db2d6 100644
--- a/ripme.json
+++ b/ripme.json
@@ -227,6 +227,6 @@
         "1.0.3: Added VK.com ripper",
         "1.0.1: Added auto-update functionality"
     ],
-    "currentHash": "1a6b86e2127b69db0461b7085dcd4d2209b3d344834cd700b5d94154684a5d74",
+    "currentHash": "9898b7882749cd4ee6616f3be0215247a2933db9e76257b6ad506a01253e42c0",
     "latestVersion": "1.7.56"
 }
\ No newline at end of file

From dab771f2da740516fccceb2cc3cfd069201ebccd Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sun, 1 Jul 2018 14:55:48 -0400
Subject: [PATCH 077/132] Got DeviantartRipper working again

---
 .../ripper/rippers/DeviantartRipper.java      | 36 ++++++++++++++-----
 1 file changed, 27 insertions(+), 9 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
index 1d7fb17c..a3428ae6 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
@@ -148,8 +148,8 @@ public class DeviantartRipper extends AbstractJSONRipper {
 
         JSONObject firstPageJSON = getFirstPageJSON(page);
         requestID = firstPageJSON.getJSONObject("dapx").getString("requestid");
-        galleryID = page.select("input[name=set]").attr("value");
-        username = page.select("div.tt-tv150").attr("username");
+        galleryID = getGalleryID(page);
+        username = getUsername(page);
         csrf = firstPageJSON.getString("csrf");
         pageCookies = res.cookies();
 
@@ -189,6 +189,25 @@ public class DeviantartRipper extends AbstractJSONRipper {
         }
         return null;
     }
+
+    private String getGalleryID(Document doc) {
+        for (Element el : doc.select("input[name=set]")) {
+            try {
+                String galleryID = el.attr("value");
+                if (galleryID.length() == 8) {
+                    return galleryID;
+                }
+            } catch (NullPointerException e) {
+                continue;
+            }
+        }
+        LOGGER.error("Could not find gallery ID");
+        return null;
+    }
+
+    private String getUsername(Document doc) {
+        return doc.select("meta[property=og:title]").attr("content").replaceAll("'s DeviantArt gallery", "");
+    }
     
 
     @Override
@@ -197,7 +216,6 @@ public class DeviantartRipper extends AbstractJSONRipper {
         LOGGER.info(json);
         JSONArray results = json.getJSONObject("content").getJSONArray("results");
         for (int i = 0; i < results.length(); i++) {
-            LOGGER.info(results.getJSONObject(i).toString());
             Document doc = Jsoup.parseBodyFragment(results.getJSONObject(i).getString("html"));
             try {
                 String imageURL = doc.select("span").first().attr("data-super-full-img");
@@ -240,12 +258,12 @@ public class DeviantartRipper extends AbstractJSONRipper {
         throw new IOException("No more pages");
     }
 
-    @Override
-    public boolean keepSortOrder() {
-         // Don't keep sort order (do not add prefixes).
-         // Causes file duplication, as outlined in https://github.com/4pr0n/ripme/issues/113
-        return false;
-    }
+//    @Override
+//    public boolean keepSortOrder() {
+//         // Don't keep sort order (do not add prefixes).
+//         // Causes file duplication, as outlined in https://github.com/4pr0n/ripme/issues/113
+//        return false;
+//    }
 
     @Override
     public void downloadURL(URL url, int index) {

From ae6d09c6e85578c301f789b7c7e113a5204cf0fc Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sun, 1 Jul 2018 15:14:09 -0400
Subject: [PATCH 078/132] Added some unit tests for DeviantartRipper

---
 .../ripme/ripper/rippers/DeviantartRipper.java         |  8 +++-----
 .../ripme/tst/ripper/rippers/DeviantartRipperTest.java | 10 ++++++++++
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
index a3428ae6..8d9f3da2 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
@@ -190,13 +190,11 @@ public class DeviantartRipper extends AbstractJSONRipper {
         return null;
     }
 
-    private String getGalleryID(Document doc) {
+    public String getGalleryID(Document doc) {
         for (Element el : doc.select("input[name=set]")) {
             try {
                 String galleryID = el.attr("value");
-                if (galleryID.length() == 8) {
-                    return galleryID;
-                }
+                return galleryID;
             } catch (NullPointerException e) {
                 continue;
             }
@@ -205,7 +203,7 @@ public class DeviantartRipper extends AbstractJSONRipper {
         return null;
     }
 
-    private String getUsername(Document doc) {
+    public String getUsername(Document doc) {
         return doc.select("meta[property=og:title]").attr("content").replaceAll("'s DeviantArt gallery", "");
     }
     
diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java
index f68d1db5..52b6bc59 100644
--- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java
+++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java
@@ -4,6 +4,8 @@ import java.io.IOException;
 import java.net.URL;
 
 import com.rarchives.ripme.ripper.rippers.DeviantartRipper;
+import com.rarchives.ripme.utils.Http;
+import org.jsoup.nodes.Document;
 
 public class DeviantartRipperTest extends RippersTest {
     public void testDeviantartAlbum() throws IOException {
@@ -22,4 +24,12 @@ public class DeviantartRipperTest extends RippersTest {
         DeviantartRipper ripper = new DeviantartRipper(url);
         assertEquals("airgee", ripper.getGID(url));
     }
+
+    public void testGetGalleryIDAndUsername() throws IOException {
+        URL url = new URL("https://www.deviantart.com/airgee/gallery/");
+        DeviantartRipper ripper = new DeviantartRipper(url);
+        Document doc = Http.url(url).get();
+        assertEquals("airgee", ripper.getUsername(doc));
+        assertEquals("714589", ripper.getGalleryID(doc));
+    }
 }

From 0ed2a7ddac29db06536cd999481b28749f1fe6b7 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sun, 1 Jul 2018 16:42:11 -0400
Subject: [PATCH 079/132] DeviantartRipper can now rip nsfw images

---
 .../ripper/rippers/DeviantartRipper.java      | 45 ++++++++++++++-----
 1 file changed, 34 insertions(+), 11 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
index 8d9f3da2..279d82d1 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
@@ -54,26 +54,23 @@ public class DeviantartRipper extends AbstractJSONRipper {
     public String getHost() {
         return "deviantart";
     }
+
     @Override
     public String getDomain() {
         return "deviantart.com";
     }
-//    @Override
-//    public boolean hasDescriptionSupport() {
-//        return true;
-//    }
+
     @Override
     public URL sanitizeURL(URL url) throws MalformedURLException {
         String u = url.toExternalForm();
 
-        if (u.replace("/", "").endsWith(".deviantart.com")) {
-            // Root user page, get all albums
-            if (!u.endsWith("/")) {
-                u += "/";
+        if (!u.endsWith("/gallery/")) {
+            if (!u.endsWith("/gallery")) {
+                u += "gallery/";
             }
-            u += "gallery/?";
         }
 
+
         Pattern p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/favou?rites/([0-9]+)/*?$");
         Matcher m = p.matcher(url.toExternalForm());
         if (!m.matches()) {
@@ -119,6 +116,25 @@ public class DeviantartRipper extends AbstractJSONRipper {
         throw new MalformedURLException("Expected URL format: http://www.deviantart.com/username[/gallery/#####], got: " + url);
     }
 
+    private String getFullsizedNSFWImage(String pageURL) {
+        try {
+            Document doc = Http.url(pageURL).cookies(cookies).get();
+            String imageToReturn = "";
+            String[] d = doc.select("img").attr("srcset").split(",");
+
+            String s = d[d.length -1].split(" ")[0];
+            LOGGER.info("2:" + s);
+
+            if (s == null || s.equals("")) {
+                LOGGER.error("Could not find full sized image at " + pageURL);
+            }
+            return s;
+        } catch (IOException e) {
+            LOGGER.error("Could not find full sized image at " + pageURL);
+            return null;
+        }
+    }
+
     /**
      * Gets first page.
      * Will determine if login is supplied,
@@ -178,7 +194,6 @@ public class DeviantartRipper extends AbstractJSONRipper {
 
     private JSONObject getFirstPageJSON(Document doc) {
         for (Element js : doc.select("script")) {
-            LOGGER.info(js.html());
             if (js.html().contains("requestid")) {
                 String json = js.html().replaceAll("window.__initial_body_data=", "").replaceAll("\\);", "")
                         .replaceAll(";__wake\\(.+", "");
@@ -204,7 +219,8 @@ public class DeviantartRipper extends AbstractJSONRipper {
     }
 
     public String getUsername(Document doc) {
-        return doc.select("meta[property=og:title]").attr("content").replaceAll("'s DeviantArt gallery", "");
+        return doc.select("meta[property=og:title]").attr("content")
+                .replaceAll("'s DeviantArt gallery", "").replaceAll("'s DeviantArt Gallery", "");
     }
     
 
@@ -215,6 +231,13 @@ public class DeviantartRipper extends AbstractJSONRipper {
         JSONArray results = json.getJSONObject("content").getJSONArray("results");
         for (int i = 0; i < results.length(); i++) {
             Document doc = Jsoup.parseBodyFragment(results.getJSONObject(i).getString("html"));
+            if (doc.html().contains("ismature")) {
+                LOGGER.info("Downloading nsfw image");
+                String nsfwImage = getFullsizedNSFWImage(doc.select("span").attr("href"));
+                if (nsfwImage != null) {
+                    imageURLs.add(nsfwImage);
+                }
+            }
             try {
                 String imageURL = doc.select("span").first().attr("data-super-full-img");
                 if (!imageURL.isEmpty()) {

From d0960fad75d5f3f79055281e3d963f4a63582a10 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sun, 1 Jul 2018 19:25:32 -0400
Subject: [PATCH 080/132] Added better url sanitization for DeviantartRipper
 and some unit tests

---
 .../ripme/ripper/rippers/DeviantartRipper.java     |  6 ++++--
 .../tst/ripper/rippers/DeviantartRipperTest.java   | 14 ++++++++++++++
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
index 279d82d1..cb3e7595 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
@@ -64,8 +64,10 @@ public class DeviantartRipper extends AbstractJSONRipper {
     public URL sanitizeURL(URL url) throws MalformedURLException {
         String u = url.toExternalForm();
 
-        if (!u.endsWith("/gallery/")) {
-            if (!u.endsWith("/gallery")) {
+        if (!u.endsWith("/gallery/") && !u.endsWith("/gallery")) {
+            if (!u.endsWith("/")) {
+                u += "/gallery/";
+            } else {
                 u += "gallery/";
             }
         }
diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java
index 52b6bc59..6a1a054b 100644
--- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java
+++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java
@@ -2,6 +2,8 @@ package com.rarchives.ripme.tst.ripper.rippers;
 
 import java.io.IOException;
 import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
 
 import com.rarchives.ripme.ripper.rippers.DeviantartRipper;
 import com.rarchives.ripme.utils.Http;
@@ -32,4 +34,16 @@ public class DeviantartRipperTest extends RippersTest {
         assertEquals("airgee", ripper.getUsername(doc));
         assertEquals("714589", ripper.getGalleryID(doc));
     }
+
+    public void testSanitizeURL() throws IOException {
+        List<URL> urls = new ArrayList<URL>();
+        urls.add(new URL("https://www.deviantart.com/airgee/"));
+        urls.add(new URL("https://www.deviantart.com/airgee"));
+        urls.add(new URL("https://www.deviantart.com/airgee/gallery/"));
+
+        for (URL url : urls) {
+            DeviantartRipper ripper = new DeviantartRipper(url);
+            assertEquals("https://www.deviantart.com/airgee/gallery/", ripper.sanitizeURL(url).toExternalForm());
+        }
+    }
 }

From de0dda0d9fb86125b4e761cd7f68ee38a421830b Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sun, 1 Jul 2018 21:49:00 -0400
Subject: [PATCH 081/132] Added ripper for loveroms.com

---
 .../ripme/ripper/rippers/LoveromRipper.java   | 111 ++++++++++++++++++
 1 file changed, 111 insertions(+)
 create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java
new file mode 100644
index 00000000..cfc264a9
--- /dev/null
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java
@@ -0,0 +1,111 @@
+package com.rarchives.ripme.ripper.rippers;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.rarchives.ripme.utils.Utils;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+
+import com.rarchives.ripme.ripper.AbstractHTMLRipper;
+import com.rarchives.ripme.utils.Http;
+
+public class LoveromRipper extends AbstractHTMLRipper {
+
+    public LoveromRipper(URL url) throws IOException {
+        super(url);
+    }
+
+    private int bytesTotal = 1;
+    private int bytesCompleted = 1;
+
+    @Override
+    public String getHost() {
+        return "loveroms";
+    }
+
+    @Override
+    public String getDomain() {
+        return "loveroms.com";
+    }
+
+    @Override
+    public String getGID(URL url) throws MalformedURLException {
+        Pattern p = Pattern.compile("https://www.loveroms.com/download/([a-zA-Z0-9_-]+)/([a-zA-Z0-9_-]+)/\\d+");
+        Matcher m = p.matcher(url.toExternalForm());
+        if (!m.matches()) {
+            throw new MalformedURLException("Expected URL format: https://www.loveroms.com/download/CONSOLE/GAME, got: " + url);
+        }
+        return m.group(1) + "_" + m.group(2);
+    }
+
+    @Override
+    public Document getFirstPage() throws IOException {
+        // "url" is an instance field of the superclass
+        return Http.url(url).get();
+    }
+
+
+    @Override
+    public List<String> getURLsFromPage(Document doc) {
+        List<String> result = new ArrayList<>();
+        String downloadLink = doc.select("a#download_link").attr("href");
+        if (downloadLink != null && !downloadLink.isEmpty()) {
+            result.add(downloadLink);
+        } else {
+            for (Element el : doc.select("a.multi-file-btn")) {
+                result.add(el.attr("href"));
+            }
+        }
+        return result;
+    }
+
+    @Override
+    public void downloadURL(URL url, int index) {
+        addURLToDownload(url, getPrefix(index));
+    }
+
+    @Override
+    public String getStatusText() {
+        return String.valueOf(getCompletionPercentage()) +
+                "%  - " +
+                Utils.bytesToHumanReadable(bytesCompleted) +
+                " / " +
+                Utils.bytesToHumanReadable(bytesTotal);
+    }
+
+//    @Override
+//    public int getCompletionPercentage() {
+//        return (int) (100 * (bytesCompleted / (float) bytesTotal));
+//    }
+//
+//    @Override
+//    public void setBytesTotal(int bytes) {
+//        this.bytesTotal = bytes;
+//    }
+//
+//    @Override
+//    public void setBytesCompleted(int bytes) {
+//        this.bytesCompleted = bytes;
+//    }
+
+//    @Override
+//    public boolean useByteProgessBar() {return true;}
+//
+//    @Override
+//    public boolean tryResumeDownload() {return true;}
+
+    @Override
+    public String getPrefix(int index) {
+        String prefix = "";
+        if (keepSortOrder() && Utils.getConfigBoolean("download.save_order", true)) {
+            prefix = String.format("%03d_", index);
+        }
+        return prefix;
+    }
+}

From 7581d5d36634f0c138a25fed04bbab3b1b1d9768 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sun, 1 Jul 2018 22:26:12 -0400
Subject: [PATCH 082/132] Xvideo ripper now uses AbstractHTMLRipper

---
 .../rippers/{video => }/XvideosRipper.java    | 66 ++++++++++++++-----
 1 file changed, 51 insertions(+), 15 deletions(-)
 rename src/main/java/com/rarchives/ripme/ripper/rippers/{video => }/XvideosRipper.java (57%)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java
similarity index 57%
rename from src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java
rename to src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java
index ef71a4bb..a94e7a21 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java
@@ -1,31 +1,47 @@
-package com.rarchives.ripme.ripper.rippers.video;
+package com.rarchives.ripme.ripper.rippers;
 
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import com.rarchives.ripme.ripper.AbstractHTMLRipper;
+import com.rarchives.ripme.utils.Utils;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
 
-import com.rarchives.ripme.ripper.VideoRipper;
 import com.rarchives.ripme.utils.Http;
 
-public class XvideosRipper extends VideoRipper {
+public class XvideosRipper extends AbstractHTMLRipper {
 
     private static final String HOST = "xvideos";
 
+    private int bytesTotal = 1;
+    private int bytesCompleted = 1;
+
     public XvideosRipper(URL url) throws IOException {
         super(url);
     }
 
+    @Override
+    public Document getFirstPage() throws IOException {
+        return Http.url(this.url).get();
+    }
+
     @Override
     public String getHost() {
         return HOST;
     }
 
+    @Override
+    public String getDomain() {
+        return HOST + ".com";
+    }
+
     @Override
     public boolean canRip(URL url) {
         Pattern p = Pattern.compile("^https?://[wm.]*xvideos\\.com/video[0-9]+.*$");
@@ -33,11 +49,6 @@ public class XvideosRipper extends VideoRipper {
         return m.matches();
     }
 
-    @Override
-    public URL sanitizeURL(URL url) throws MalformedURLException {
-        return url;
-    }
-
     @Override
     public String getGID(URL url) throws MalformedURLException {
         Pattern p = Pattern.compile("^https?://[wm.]*xvideos\\.com/video([0-9]+).*$");
@@ -53,9 +64,8 @@ public class XvideosRipper extends VideoRipper {
     }
 
     @Override
-    public void rip() throws IOException {
-        LOGGER.info("    Retrieving " + this.url);
-        Document doc = Http.url(this.url).get();
+    public List<String> getURLsFromPage(Document doc) {
+        List<String> results = new ArrayList<>();
         Elements scripts = doc.select("script");
         for (Element e : scripts) {
             if (e.html().contains("html5player.setVideoUrlHigh")) {
@@ -64,13 +74,39 @@ public class XvideosRipper extends VideoRipper {
                 for (String line: lines) {
                     if (line.contains("html5player.setVideoUrlHigh")) {
                         String videoURL = line.replaceAll("\t", "").replaceAll("html5player.setVideoUrlHigh\\(", "").replaceAll("\'", "").replaceAll("\\);", "");
-                        addURLToDownload(new URL(videoURL), HOST + "_" + getGID(this.url));
-                        waitForThreads();
-                        return;
+                        results.add(videoURL);
                     }
                 }
             }
         }
-        throw new IOException("Unable to find video url at " + this.url.toExternalForm());
+        return results;
     }
+
+    @Override
+    public void downloadURL(URL url, int index) {
+        addURLToDownload(url, getPrefix(index));
+    }
+
+    @Override
+    public String getStatusText() {
+        return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal);
+    }
+
+    @Override
+    public int getCompletionPercentage() {
+        return (int) (100 * (bytesCompleted / (float) bytesTotal));
+    }
+
+    @Override
+    public void setBytesTotal(int bytes) {
+        this.bytesTotal = bytes;
+    }
+
+    @Override
+    public void setBytesCompleted(int bytes) {
+        this.bytesCompleted = bytes;
+    }
+
+    @Override
+    public boolean useByteProgessBar() {return true;}
 }
\ No newline at end of file

From f6bb268b4cb9be7a6c3e1202994a2d9365a1c2b7 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sun, 1 Jul 2018 22:32:44 -0400
Subject: [PATCH 083/132] Added a unittest for the XvideosRipper

---
 .../tst/ripper/rippers/VideoRippersTest.java     | 15 ++-------------
 .../tst/ripper/rippers/XvideosRipperTest.java    | 16 ++++++++++++++++
 2 files changed, 18 insertions(+), 13 deletions(-)
 create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/XvideosRipperTest.java

diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java
index 05d7514e..02696755 100644
--- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java
+++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java
@@ -7,9 +7,8 @@ import java.util.List;
 
 import com.rarchives.ripme.ripper.VideoRipper;
 import com.rarchives.ripme.ripper.rippers.video.PornhubRipper;
-import com.rarchives.ripme.ripper.rippers.video.TwitchVideoRipper;
 import com.rarchives.ripme.ripper.rippers.video.XhamsterRipper;
-import com.rarchives.ripme.ripper.rippers.video.XvideosRipper;
+import com.rarchives.ripme.ripper.rippers.XvideosRipper;
 import com.rarchives.ripme.ripper.rippers.video.YoupornRipper;
 import com.rarchives.ripme.ripper.rippers.video.YuvutuRipper;
 
@@ -57,17 +56,7 @@ public class VideoRippersTest extends RippersTest {
             videoTestHelper(ripper);
         }
     }
-
-    public void testXvideosRipper() throws IOException {
-        List<URL> contentURLs = new ArrayList<>();
-        contentURLs.add(new URL("https://www.xvideos.com/video19719109/ziggy_star_ultra_hard_anal_pounding"));
-        contentURLs.add(new URL("https://www.xvideos.com/video23515878/dee_s_pool_toys"));
-        for (URL url : contentURLs) {
-            XvideosRipper ripper = new XvideosRipper(url);
-            videoTestHelper(ripper);
-        }
-    }
-
+    
     public void testPornhubRipper() throws IOException {
         List<URL> contentURLs = new ArrayList<>();
         contentURLs.add(new URL("https://www.pornhub.com/view_video.php?viewkey=ph5a329fa707269"));
diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XvideosRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XvideosRipperTest.java
new file mode 100644
index 00000000..543ad8ca
--- /dev/null
+++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XvideosRipperTest.java
@@ -0,0 +1,16 @@
+package com.rarchives.ripme.tst.ripper.rippers;
+
+import java.io.IOException;
+import java.net.URL;
+
+import com.rarchives.ripme.ripper.rippers.XvideosRipper;
+import com.rarchives.ripme.tst.ripper.rippers.RippersTest;
+
+public class XvideosRipperTest extends RippersTest {
+
+    public void testXhamsterAlbum1() throws IOException {
+        XvideosRipper ripper = new XvideosRipper(new URL("https://www.xvideos.com/video23515878/dee_s_pool_toys"));
+        testRipper(ripper);
+    }
+
+}

From 55c782dbabc05cd28d73a0de88fc59f674d3cdcb Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sun, 1 Jul 2018 22:45:35 -0400
Subject: [PATCH 084/132] Imagefap ripper now downloads full sized images

---
 .../ripme/ripper/rippers/ImagefapRipper.java      | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java
index 1a658c59..07a6e529 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java
@@ -125,11 +125,7 @@ public class ImagefapRipper extends AbstractHTMLRipper {
             if (!thumb.hasAttr("src") || !thumb.hasAttr("width")) {
                 continue;
             }
-            String image = thumb.attr("src");
-            image = image.replaceAll(
-                    "http://x.*.fap.to/images/thumb/",
-                    "http://fap.to/images/full/");
-            image = image.replaceAll("w[0-9]+-h[0-9]+/", "");
+            String image = getFullSizedImage("https://www.imagefap.com" + thumb.parent().attr("href"));
             imageURLs.add(image);
             if (isThisATest()) {
                 break;
@@ -160,4 +156,13 @@ public class ImagefapRipper extends AbstractHTMLRipper {
         return super.getAlbumTitle(url);
     }
 
+    private String getFullSizedImage(String pageURL) {
+        try {
+            Document doc = Http.url(pageURL).get();
+            return doc.select("img#mainPhoto").attr("src");
+        } catch (IOException e) {
+            return null;
+        }
+    }
+
 }

From 31482fa0d8375ef3da76e572ad7415012fdf3f28 Mon Sep 17 00:00:00 2001
From: Jicksaw <jicksaw@protonmail.com>
Date: Tue, 3 Jul 2018 01:21:50 +0300
Subject: [PATCH 085/132] Use paremeter tweet_mode=extended

---
 .../com/rarchives/ripme/ripper/rippers/TwitterRipper.java   | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
index 52c7e16f..8e9f0ab4 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
@@ -126,14 +126,16 @@ public class TwitterRipper extends AlbumRipper {
                         .append("&exclude_replies=true")
                         .append("&trim_user=true")
                         .append("&include_rts=true")
-                        .append("&count=" + 200);
+                        .append("&count=" + 200)
+                        .append("&tweet_mode=extended");
                 break;
             case SEARCH:
                 req.append("https://api.twitter.com/1.1/search/tweets.json")
                         .append("?q=" + this.searchText)
                         .append("&include_entities=true")
                         .append("&result_type=recent")
-                        .append("&count=100");
+                        .append("&count=100")
+                        .append("&tweet_mode=extended");
                 break;
         }
         if (maxID > 0) {

From aac4fa48f5c64eae335d41ab51928dfaab6764a2 Mon Sep 17 00:00:00 2001
From: rephormat <rephormat@users.noreply.github.com>
Date: Mon, 2 Jul 2018 19:49:47 -0500
Subject: [PATCH 086/132] Use descriptive naming when Save album titles option
 is enabled.  Format is xhamster_<user>_<gallery url>.

---
 .../ripme/ripper/rippers/XhamsterRipper.java  | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java
index 026a6a30..96f20ec3 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java
@@ -100,4 +100,23 @@ public class XhamsterRipper extends AbstractHTMLRipper {
     public void downloadURL(URL url, int index) {
         addURLToDownload(url, getPrefix(index));
     }
+    
+    @Override
+    public String getAlbumTitle(URL url) throws MalformedURLException {
+        try {
+            // Attempt to use album title and username as GID
+            Document doc = getFirstPage();
+            Element user = doc.select("a.author").first();
+            String username = user.text();
+            String path = url.getPath();
+            Pattern p = Pattern.compile("^/photos/gallery/(.*)$");
+            Matcher m = p.matcher(path);
+            if (m.matches() && !username.isEmpty()) {
+                return getHost() + "_" + username + "_" + m.group(1);
+            }
+        } catch (IOException e) {
+            // Fall back to default album naming convention
+        }
+        return super.getAlbumTitle(url);
+    }
 }
\ No newline at end of file

From 4b7bd12313467fafef8a225ba903a155afeefd5a Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Tue, 3 Jul 2018 03:52:53 -0400
Subject: [PATCH 087/132] Uncommented out some code

---
 .../ripme/ripper/rippers/LoveromRipper.java   | 38 +++++++++----------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java
index cfc264a9..88c9c0f5 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java
@@ -79,26 +79,26 @@ public class LoveromRipper extends AbstractHTMLRipper {
                 Utils.bytesToHumanReadable(bytesTotal);
     }
 
-//    @Override
-//    public int getCompletionPercentage() {
-//        return (int) (100 * (bytesCompleted / (float) bytesTotal));
-//    }
-//
-//    @Override
-//    public void setBytesTotal(int bytes) {
-//        this.bytesTotal = bytes;
-//    }
-//
-//    @Override
-//    public void setBytesCompleted(int bytes) {
-//        this.bytesCompleted = bytes;
-//    }
+    @Override
+    public int getCompletionPercentage() {
+        return (int) (100 * (bytesCompleted / (float) bytesTotal));
+    }
 
-//    @Override
-//    public boolean useByteProgessBar() {return true;}
-//
-//    @Override
-//    public boolean tryResumeDownload() {return true;}
+    @Override
+    public void setBytesTotal(int bytes) {
+        this.bytesTotal = bytes;
+    }
+
+    @Override
+    public void setBytesCompleted(int bytes) {
+        this.bytesCompleted = bytes;
+    }
+
+    @Override
+    public boolean useByteProgessBar() {return true;}
+
+    @Override
+    public boolean tryResumeDownload() {return true;}
 
     @Override
     public String getPrefix(int index) {

From c3b506e7cae199c0edcf27e40e17ae7ab44ab45a Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Tue, 3 Jul 2018 03:57:32 -0400
Subject: [PATCH 088/132] Added nl_NL translation

---
 .../com/rarchives/ripme/ui/MainWindow.java    |  2 +-
 .../resources/LabelsBundle_nl_NL.properties   | 37 +++++++++++++++++++
 2 files changed, 38 insertions(+), 1 deletion(-)
 create mode 100644 src/main/resources/LabelsBundle_nl_NL.properties

diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java
index 9f4ff519..e0114fea 100644
--- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java
+++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java
@@ -496,7 +496,7 @@ public final class MainWindow implements Runnable, RipStatusHandler {
         configURLHistoryCheckbox = addNewCheckbox(rb.getString("remember.url.history"), "remember.url_history", true);
 
         configLogLevelCombobox = new JComboBox<>(new String[] {"Log level: Error", "Log level: Warn", "Log level: Info", "Log level: Debug"});
-        configSelectLangComboBox = new JComboBox<>(new String[] {"en_US", "de_DE", "es_ES", "fr_CH", "kr_KR", "pt_PT", "fi_FI", "in_ID", "porrisavvo_FI"});
+        configSelectLangComboBox = new JComboBox<>(new String[] {"en_US", "de_DE", "es_ES", "fr_CH", "kr_KR", "pt_PT", "fi_FI", "in_ID", "nl_NL", "porrisavvo_FI"});
         configLogLevelCombobox.setSelectedItem(Utils.getConfigString("log.level", "Log level: Debug"));
         setLogLevel(configLogLevelCombobox.getSelectedItem().toString());
         configSaveDirLabel = new JLabel();
diff --git a/src/main/resources/LabelsBundle_nl_NL.properties b/src/main/resources/LabelsBundle_nl_NL.properties
new file mode 100644
index 00000000..5590906d
--- /dev/null
+++ b/src/main/resources/LabelsBundle_nl_NL.properties
@@ -0,0 +1,37 @@
+Log = Logboek
+History = Geschiedenis
+created = gemaakt
+modified = aangepast
+Queue = Wachtrij
+Configuration = Configuratie
+
+# Keys for the Configuration menu
+
+current.version = Huidige versie
+check.for.updates = Controleer op updates
+auto.update = Auto-update?
+max.download.threads = Maximale downloadthreads
+timeout.mill = Timeout (in milliseconden):
+retry.download.count = Aantal keren opnieuw proberen te downloaden
+overwrite.existing.files = Bestaande bestanden overschrijven?
+sound.when.rip.completes = Geluid wanneer rip klaar is
+preserve.order = Volgorde behouden
+save.logs = Logbestanden opslaan
+notification.when.rip.starts = Notificatie wanneer rip start
+save.urls.only = Alleen URLs opslaan
+save.album.titles = Album titels opslaan
+autorip.from.clipboard = Rip automatisch van klembord
+save.descriptions = Beschrijvingen opslaan
+prefer.mp4.over.gif = Geef de voorkeur aan MP4 over GIF
+restore.window.position = Vensterpositie herstellen
+remember.url.history = URL geschiedenis onthouden
+loading.history.from = Geschiedenis laden van
+
+# Misc UI keys
+
+loading.history.from.configuration = Geschiedenis laden van configuratie
+interrupted.while.waiting.to.rip.next.album = Onderbroken tijdens het wachten om volgend album te rippen
+inactive = Inactief
+re-rip.checked = Re-rip Gecheckt
+remove = Verwijderen
+clear = Opruimen

From 6ac6449815e39a6cee541e06661f8c64b2951c4c Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Wed, 4 Jul 2018 00:10:30 -0400
Subject: [PATCH 089/132] Twitter ripper no longer rips retweets by default

---
 .../java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java   | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
index 8e9f0ab4..d28e4d79 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
@@ -125,7 +125,6 @@ public class TwitterRipper extends AlbumRipper {
                         .append("&include_entities=true")
                         .append("&exclude_replies=true")
                         .append("&trim_user=true")
-                        .append("&include_rts=true")
                         .append("&count=" + 200)
                         .append("&tweet_mode=extended");
                 break;

From 9a29448e73b5a6b87970ce4a69cbddbbf6c8a2a6 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Wed, 4 Jul 2018 00:36:36 -0400
Subject: [PATCH 090/132] 1.7.57: Got DeviantartRipper working again; Imagefap
 ripper now downloads full sized images; Twitter ripper can now rip extended
 tweets; Added nl_NL translation

---
 pom.xml                                               | 2 +-
 ripme.json                                            | 5 +++--
 src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/pom.xml b/pom.xml
index 4bf06c19..e5438d70 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
   <groupId>com.rarchives.ripme</groupId>
   <artifactId>ripme</artifactId>
   <packaging>jar</packaging>
-  <version>1.7.56</version>
+  <version>1.7.57</version>
   <name>ripme</name>
   <url>http://rip.rarchives.com</url>
   <properties>
diff --git a/ripme.json b/ripme.json
index d98db2d6..566c369b 100644
--- a/ripme.json
+++ b/ripme.json
@@ -1,5 +1,6 @@
 {
     "changeList": [
+        "1.7.57: Got DeviantartRipper working again; Imagefap ripper now downloads full sized images; Twitter ripper can now rip extended tweets; Added nl_NL translation",
         "1.7.56: Fixed DeviantartRipper ripper; Added support for resuming file downloads; Fixed erome ripper; Fixed ModelmayhemRipper NSFW image downloading",
         "1.7.55: Fixed instagram ripper; Reddit ripper now respects history.end_rip_after_already_seen; Improvements to patch.py and release.py",
         "1.7.54: Fixed twitter ripper video downloading; fixed instagram ripper",
@@ -227,6 +228,6 @@
         "1.0.3: Added VK.com ripper",
         "1.0.1: Added auto-update functionality"
     ],
-    "currentHash": "9898b7882749cd4ee6616f3be0215247a2933db9e76257b6ad506a01253e42c0",
-    "latestVersion": "1.7.56"
+    "latestVersion": "1.7.57",
+    "currentHash": "cd84b64c5d85c2d14b79b6d18b25cee6e843ba3ba0771a5b43c5b3db15229218"
 }
\ No newline at end of file
diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
index c474b6c9..ec623538 100644
--- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
+++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
@@ -20,7 +20,7 @@ import com.rarchives.ripme.utils.Utils;
 public class UpdateUtils {
 
     private static final Logger logger = Logger.getLogger(UpdateUtils.class);
-    private static final String DEFAULT_VERSION = "1.7.56";
+    private static final String DEFAULT_VERSION = "1.7.57";
     private static final String REPO_NAME = "ripmeapp/ripme";
     private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json";
     private static final String mainFileName = "ripme.jar";

From e8f91d1794a3a25170e722dabcb2a02e1ed2d98b Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Thu, 5 Jul 2018 04:35:03 -0400
Subject: [PATCH 091/132] DA ripper fixes

---
 .../ripper/rippers/DeviantartRipper.java      | 113 +++++-------------
 1 file changed, 33 insertions(+), 80 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
index cb3e7595..be1cf3ec 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
@@ -63,6 +63,9 @@ public class DeviantartRipper extends AbstractJSONRipper {
     @Override
     public URL sanitizeURL(URL url) throws MalformedURLException {
         String u = url.toExternalForm();
+        if (u.contains("/gallery/")) {
+            return url;
+        }
 
         if (!u.endsWith("/gallery/") && !u.endsWith("/gallery")) {
             if (!u.endsWith("/")) {
@@ -118,6 +121,16 @@ public class DeviantartRipper extends AbstractJSONRipper {
         throw new MalformedURLException("Expected URL format: http://www.deviantart.com/username[/gallery/#####], got: " + url);
     }
 
+    private String getUsernameFromURL(String u) {
+        Pattern p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/gallery/\\S+");
+        Matcher m = p.matcher(url.toExternalForm());
+        if (m.matches()) {
+            return m.group(1);
+        }
+        return null;
+
+    }
+
     private String getFullsizedNSFWImage(String pageURL) {
         try {
             Document doc = Http.url(pageURL).cookies(cookies).get();
@@ -158,6 +171,7 @@ public class DeviantartRipper extends AbstractJSONRipper {
                 LOGGER.warn("Failed to get login cookies");
                 cookies.put("agegate_state","1"); // Bypasses the age gate
             }
+        cookies.put("agegate_state", "1");
             
         Response res = Http.url(this.url)
                    .cookies(cookies)
@@ -167,7 +181,7 @@ public class DeviantartRipper extends AbstractJSONRipper {
         JSONObject firstPageJSON = getFirstPageJSON(page);
         requestID = firstPageJSON.getJSONObject("dapx").getString("requestid");
         galleryID = getGalleryID(page);
-        username = getUsername(page);
+        username = getUsernameFromURL(url.toExternalForm());
         csrf = firstPageJSON.getString("csrf");
         pageCookies = res.cookies();
 
@@ -208,6 +222,16 @@ public class DeviantartRipper extends AbstractJSONRipper {
     }
 
     public String getGalleryID(Document doc) {
+        // If the url contains catpath we return 0 as the DA api will provide all galery images if you sent the
+        // gallery id to 0
+        if (url.toExternalForm().contains("catpath=")) {
+            return "0";
+        }
+        Pattern p = Pattern.compile("^https?://www\\.deviantart\\.com/[a-zA-Z0-9\\-]+/gallery/([0-9]+)/?\\S+");
+        Matcher m = p.matcher(url.toExternalForm());
+        if (m.matches()) {
+            return m.group(1);
+        }
         for (Element el : doc.select("input[name=set]")) {
             try {
                 String galleryID = el.attr("value");
@@ -252,24 +276,7 @@ public class DeviantartRipper extends AbstractJSONRipper {
         }
         return imageURLs;
     }
-//    @Override
-//    public List<String> getDescriptionsFromPage(Document page) {
-//        List<String> textURLs = new ArrayList<>();
-//        // Iterate over all thumbnails
-//        for (Element thumb : page.select("div.zones-container span.thumb")) {
-//            LOGGER.info(thumb.attr("href"));
-//            if (isStopped()) {
-//                break;
-//            }
-//            Element img = thumb.select("img").get(0);
-//            if (img.attr("transparent").equals("false")) {
-//                continue; // a.thumbs to other albums are invisible
-//            }
-//            textURLs.add(thumb.attr("href"));
-//
-//        }
-//        return textURLs;
-//    }
+
 
     @Override
     public JSONObject getNextPage(JSONObject page) throws IOException {
@@ -281,12 +288,12 @@ public class DeviantartRipper extends AbstractJSONRipper {
         throw new IOException("No more pages");
     }
 
-//    @Override
-//    public boolean keepSortOrder() {
-//         // Don't keep sort order (do not add prefixes).
-//         // Causes file duplication, as outlined in https://github.com/4pr0n/ripme/issues/113
-//        return false;
-//    }
+    @Override
+    public boolean keepSortOrder() {
+         // Don't keep sort order (do not add prefixes).
+         // Causes file duplication, as outlined in https://github.com/4pr0n/ripme/issues/113
+        return false;
+    }
 
     @Override
     public void downloadURL(URL url, int index) {
@@ -319,61 +326,7 @@ public class DeviantartRipper extends AbstractJSONRipper {
         return result.toString();
     }
 
-    /**
-     * Attempts to download description for image.
-     * Comes in handy when people put entire stories in their description.
-     * If no description was found, returns null.
-     * @param url The URL the description will be retrieved from
-     * @param page The gallery page the URL was found on
-     * @return A String[] with first object being the description, and the second object being image file name if found.
-     */
-//    @Override
-//    public String[] getDescription(String url,Document page) {
-//        if (isThisATest()) {
-//            return null;
-//        }
-//        try {
-//            // Fetch the image page
-//            Response resp = Http.url(url)
-//                                .referrer(this.url)
-//                                .cookies(cookies)
-//                                .response();
-//            cookies.putAll(resp.cookies());
-//
-//            // Try to find the description
-//            Document documentz = resp.parse();
-//            Element ele = documentz.select("div.dev-description").first();
-//            if (ele == null) {
-//                throw new IOException("No description found");
-//            }
-//            documentz.outputSettings(new Document.OutputSettings().prettyPrint(false));
-//            ele.select("br").append("\\n");
-//            ele.select("p").prepend("\\n\\n");
-//            String fullSize = null;
-//            Element thumb = page.select("div.zones-container span.thumb[href=\"" + url + "\"]").get(0);
-//            if (!thumb.attr("data-super-full-img").isEmpty()) {
-//                fullSize = thumb.attr("data-super-full-img");
-//                String[] split = fullSize.split("/");
-//                fullSize = split[split.length - 1];
-//            } else {
-//                String spanUrl = thumb.attr("href");
-//                fullSize = jsonToImage(page,spanUrl.substring(spanUrl.lastIndexOf('-') + 1));
-//                if (fullSize != null) {
-//                    String[] split = fullSize.split("/");
-//                    fullSize = split[split.length - 1];
-//                }
-//            }
-//            if (fullSize == null) {
-//                return new String[] {Jsoup.clean(ele.html().replaceAll("\\\\n", System.getProperty("line.separator")), "", Whitelist.none(), new Document.OutputSettings().prettyPrint(false))};
-//            }
-//            fullSize = fullSize.substring(0, fullSize.lastIndexOf("."));
-//            return new String[] {Jsoup.clean(ele.html().replaceAll("\\\\n", System.getProperty("line.separator")), "", Whitelist.none(), new Document.OutputSettings().prettyPrint(false)),fullSize};
-//            // TODO Make this not make a newline if someone just types \n into the description.
-//        } catch (IOException ioe) {
-//                LOGGER.info("Failed to get description at " + url + ": '" + ioe.getMessage() + "'");
-//                return null;
-//        }
-//    }
+
 
     /**
      * If largest resolution for image at 'thumb' is found, starts downloading

From 07670c98f8baab878b8c8b58d34292736606ae14 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Fri, 6 Jul 2018 01:32:28 -0400
Subject: [PATCH 092/132] Fixed Deviantart getUsernameFromURL

---
 .../com/rarchives/ripme/ripper/rippers/DeviantartRipper.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
index be1cf3ec..10ef233a 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
@@ -122,7 +122,7 @@ public class DeviantartRipper extends AbstractJSONRipper {
     }
 
     private String getUsernameFromURL(String u) {
-        Pattern p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/gallery/\\S+");
+        Pattern p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/gallery/?(\\S+)?");
         Matcher m = p.matcher(url.toExternalForm());
         if (m.matches()) {
             return m.group(1);

From c8e8139cf617a646f8a3d529913b69f5a8948d32 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Fri, 6 Jul 2018 02:11:43 -0400
Subject: [PATCH 093/132] fixed HitomiRipper

---
 .../java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java
index c5f0bbd4..53389a21 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java
@@ -47,7 +47,7 @@ public class HitomiRipper extends AbstractHTMLRipper {
     @Override
     public Document getFirstPage() throws IOException {
         // if we go to /GALLERYID.js we get a nice json array of all images in the gallery
-        return Http.url(new URL(url.toExternalForm().replaceAll(".html", ".js"))).ignoreContentType().get();
+        return Http.url(new URL(url.toExternalForm().replaceAll("hitomi", "ltn.hitomi").replaceAll(".html", ".js"))).ignoreContentType().get();
     }
 
 

From 871e8984d8f22964275f1306d39fab9b34f96ab9 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sat, 7 Jul 2018 04:03:59 -0400
Subject: [PATCH 094/132] Added getAlbumTitle for HitomiRipper

---
 .../ripme/ripper/rippers/HitomiRipper.java       | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java
index 53389a21..fd080d35 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java
@@ -13,10 +13,11 @@ import org.jsoup.nodes.Document;
 
 import com.rarchives.ripme.ripper.AbstractHTMLRipper;
 import com.rarchives.ripme.utils.Http;
+import org.jsoup.nodes.Element;
 
 public class HitomiRipper extends AbstractHTMLRipper {
 
-    String galleryId = "";
+    private String galleryId = "";
 
     public HitomiRipper(URL url) throws IOException {
         super(url);
@@ -64,6 +65,19 @@ public class HitomiRipper extends AbstractHTMLRipper {
         return result;
     }
 
+    @Override
+    public String getAlbumTitle(URL url) throws MalformedURLException {
+        try {
+            // Attempt to use album title and username as GID
+            Document doc = Http.url(url).get();
+            return getHost() + "_" + getGID(url) + "_" +
+                    doc.select("title").text().replaceAll(" - Read Online - hentai artistcg \\| Hitomi.la", "");
+        } catch (IOException e) {
+            LOGGER.info("Falling back");
+        }
+        return super.getAlbumTitle(url);
+    }
+
     @Override
     public void downloadURL(URL url, int index) {
         addURLToDownload(url, getPrefix(index));

From 265e1310453d5767dca3bff6e7b7db8a33ed068f Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sun, 8 Jul 2018 06:17:51 -0400
Subject: [PATCH 095/132] Fixed ManganeloRipper

---
 .../com/rarchives/ripme/ripper/rippers/ManganeloRipper.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java
index cabb4188..79e670d1 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java
@@ -99,7 +99,7 @@ public class ManganeloRipper extends AbstractHTMLRipper {
             Collections.reverse(urlsToGrab);
 
             for (String url : urlsToGrab) {
-                result.addAll(getURLsFromChap(url));
+                result.addAll(getURLsFromChap("https:" + url));
             }
         } else if (url.toExternalForm().contains("/chapter/")) {
             result.addAll(getURLsFromChap(doc));

From 42c5a7be3fd48a738ccb060f3a271b975fe7cae3 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sun, 8 Jul 2018 06:18:23 -0400
Subject: [PATCH 096/132] Changed ManganeloRipper unittest url to a smaller
 album, added test got getGid

---
 .../ripme/tst/ripper/rippers/ManganeloRipperTest.java     | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ManganeloRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ManganeloRipperTest.java
index ca355a2c..7eb1b8e9 100644
--- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ManganeloRipperTest.java
+++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ManganeloRipperTest.java
@@ -7,7 +7,13 @@ import com.rarchives.ripme.ripper.rippers.ManganeloRipper;
 
 public class ManganeloRipperTest extends RippersTest {
     public void testRip() throws IOException {
-        ManganeloRipper ripper = new ManganeloRipper(new URL("http://manganelo.com/manga/black_clover"));
+        ManganeloRipper ripper = new ManganeloRipper(new URL("https://manganelo.com/manga/demonic_housekeeper"));
         testRipper(ripper);
     }
+
+    public void testGetGID() throws IOException {
+        URL url = new URL("https://manganelo.com/manga/demonic_housekeeper");
+        ManganeloRipper ripper = new ManganeloRipper(url);
+        assertEquals("demonic_housekeeper", ripper.getGID(url));
+    }
 }

From 474252a7cb22cf85a0a5b2d2794fb0407396eba0 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sun, 8 Jul 2018 06:44:09 -0400
Subject: [PATCH 097/132] Fixed update box formating

---
 src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
index ec623538..484d23f3 100644
--- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
+++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
@@ -128,9 +128,9 @@ public class UpdateUtils {
             logger.info("Found newer version: " + latestVersion);
             int result = JOptionPane.showConfirmDialog(
                     null,
-                    "<html><font color=\"green\">New version (" + latestVersion + ") is available!</font>"
-                    + "<br><br>Recent changes:" + changeList
-                    + "<br><br>Do you want to download and run the newest version?</html>",
+                    String.format("<html><font color=\"green\">New version (%s) is available!</font>"
+                            + "<br><br>Recent changes: %s"
+                            + "<br><br>Do you want to download and run the newest version?</html>", latestVersion, changeList.replaceAll("\n", "")),
                     "RipMe Updater",
                     JOptionPane.YES_NO_OPTION);
             if (result != JOptionPane.YES_OPTION) {

From ea00d90a072f9f41712163e1657d959f0dbbc923 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sun, 8 Jul 2018 07:05:46 -0400
Subject: [PATCH 098/132] 1.7.58: Fixed Deviantart ripper; fixed HitomiRipper;
 Fixed ManganeloRipper; Fixed update box formating

---
 pom.xml                                               | 2 +-
 ripme.json                                            | 7 ++++---
 src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/pom.xml b/pom.xml
index e5438d70..9de7c444 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
   <groupId>com.rarchives.ripme</groupId>
   <artifactId>ripme</artifactId>
   <packaging>jar</packaging>
-  <version>1.7.57</version>
+  <version>1.7.58</version>
   <name>ripme</name>
   <url>http://rip.rarchives.com</url>
   <properties>
diff --git a/ripme.json b/ripme.json
index 566c369b..d835ecfe 100644
--- a/ripme.json
+++ b/ripme.json
@@ -1,5 +1,8 @@
 {
+    "latestVersion": "1.7.58",
+    "currentHash": "d94ae3f80ae38b5bcd7abe996cf70ed3de610bba794665be01518892a11c70c5",
     "changeList": [
+        "1.7.58: Fixed Deviantart ripper; fixed HitomiRipper; Fixed ManganeloRipper; Fixed update box formating",
         "1.7.57: Got DeviantartRipper working again; Imagefap ripper now downloads full sized images; Twitter ripper can now rip extended tweets; Added nl_NL translation",
         "1.7.56: Fixed DeviantartRipper ripper; Added support for resuming file downloads; Fixed erome ripper; Fixed ModelmayhemRipper NSFW image downloading",
         "1.7.55: Fixed instagram ripper; Reddit ripper now respects history.end_rip_after_already_seen; Improvements to patch.py and release.py",
@@ -227,7 +230,5 @@
         "1.0.4: Fixed spaces-in-directory bug",
         "1.0.3: Added VK.com ripper",
         "1.0.1: Added auto-update functionality"
-    ],
-    "latestVersion": "1.7.57",
-    "currentHash": "cd84b64c5d85c2d14b79b6d18b25cee6e843ba3ba0771a5b43c5b3db15229218"
+    ]
 }
\ No newline at end of file
diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
index 484d23f3..77ccc1b1 100644
--- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
+++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
@@ -20,7 +20,7 @@ import com.rarchives.ripme.utils.Utils;
 public class UpdateUtils {
 
     private static final Logger logger = Logger.getLogger(UpdateUtils.class);
-    private static final String DEFAULT_VERSION = "1.7.57";
+    private static final String DEFAULT_VERSION = "1.7.58";
     private static final String REPO_NAME = "ripmeapp/ripme";
     private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json";
     private static final String mainFileName = "ripme.jar";

From 13dd2694b45ec21ead0f2ed2e7a1878645a44cf3 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Mon, 9 Jul 2018 11:19:15 -0400
Subject: [PATCH 099/132] Fixed multipart file downloading

---
 .../com/rarchives/ripme/ripper/rippers/LoveromRipper.java | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java
index 88c9c0f5..bddecf64 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java
@@ -23,6 +23,7 @@ public class LoveromRipper extends AbstractHTMLRipper {
 
     private int bytesTotal = 1;
     private int bytesCompleted = 1;
+    boolean multipart = false;
 
     @Override
     public String getHost() {
@@ -58,6 +59,7 @@ public class LoveromRipper extends AbstractHTMLRipper {
         if (downloadLink != null && !downloadLink.isEmpty()) {
             result.add(downloadLink);
         } else {
+            multipart = true;
             for (Element el : doc.select("a.multi-file-btn")) {
                 result.add(el.attr("href"));
             }
@@ -72,6 +74,9 @@ public class LoveromRipper extends AbstractHTMLRipper {
 
     @Override
     public String getStatusText() {
+        if (multipart) {
+            return super.getStatusText();
+        }
         return String.valueOf(getCompletionPercentage()) +
                 "%  - " +
                 Utils.bytesToHumanReadable(bytesCompleted) +
@@ -81,6 +86,9 @@ public class LoveromRipper extends AbstractHTMLRipper {
 
     @Override
     public int getCompletionPercentage() {
+        if (multipart) {
+            return super.getCompletionPercentage();
+        }
         return (int) (100 * (bytesCompleted / (float) bytesTotal));
     }
 

From 6c64f73daa11c69ecf086428e57bcf61a2d7dc0d Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Mon, 9 Jul 2018 11:38:14 -0400
Subject: [PATCH 100/132] Fixed minor bug with DeviantartRipper

---
 .../com/rarchives/ripme/ripper/rippers/DeviantartRipper.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
index 10ef233a..04a1766f 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
@@ -260,13 +260,13 @@ public class DeviantartRipper extends AbstractJSONRipper {
             if (doc.html().contains("ismature")) {
                 LOGGER.info("Downloading nsfw image");
                 String nsfwImage = getFullsizedNSFWImage(doc.select("span").attr("href"));
-                if (nsfwImage != null) {
+                if (nsfwImage != null && nsfwImage.startsWith("http")) {
                     imageURLs.add(nsfwImage);
                 }
             }
             try {
                 String imageURL = doc.select("span").first().attr("data-super-full-img");
-                if (!imageURL.isEmpty()) {
+                if (!imageURL.isEmpty() && imageURL.startsWith("http")) {
                     imageURLs.add(imageURL);
                 }
             } catch (NullPointerException e) {

From 59dd904c22d3a42f3b67666611ba4fd8ddfcdab1 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Tue, 10 Jul 2018 12:53:02 -0400
Subject: [PATCH 101/132] Now prints error when ripme can't get a files type
 from the stream

---
 .../com/rarchives/ripme/ripper/DownloadFileThread.java   | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
index 10e4462c..4f189c3c 100644
--- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
+++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
@@ -175,8 +175,13 @@ class DownloadFileThread extends Thread {
 
                 // Check if we should get the file ext from the MIME type
                 if (getFileExtFromMIME) {
-                    String fileExt = URLConnection.guessContentTypeFromStream(bis).replaceAll("image/", "");
-                    saveAs = new File(saveAs.toString() + "." + fileExt);
+                    String fileExt = URLConnection.guessContentTypeFromStream(bis);
+                    if (fileExt != null) {
+                        fileExt = fileExt.replaceAll("image/", "");
+                        saveAs = new File(saveAs.toString() + "." + fileExt);
+                    } else {
+                        logger.error("Was unable to get content type from stream");
+                    }
                 }
                 // If we're resuming a download we append data to the existing file
                 if (statusCode == 206) {

From 67e2451744af30b3ec2c221a7239dfba12e5c606 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Tue, 10 Jul 2018 13:24:56 -0400
Subject: [PATCH 102/132] Now properly names multi-part files; Updated to work
 with recent changes to site

---
 .../rarchives/ripme/ripper/rippers/LoveromRipper.java  | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java
index bddecf64..83af405f 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java
@@ -55,7 +55,7 @@ public class LoveromRipper extends AbstractHTMLRipper {
     @Override
     public List<String> getURLsFromPage(Document doc) {
         List<String> result = new ArrayList<>();
-        String downloadLink = doc.select("a#download_link").attr("href");
+        String downloadLink = doc.select("a#start_download_link").attr("href");
         if (downloadLink != null && !downloadLink.isEmpty()) {
             result.add(downloadLink);
         } else {
@@ -69,7 +69,11 @@ public class LoveromRipper extends AbstractHTMLRipper {
 
     @Override
     public void downloadURL(URL url, int index) {
-        addURLToDownload(url, getPrefix(index));
+        if (multipart) {
+            addURLToDownload(url, "", "", "", null, null, getPrefix(index));
+        } else {
+            addURLToDownload(url);
+        }
     }
 
     @Override
@@ -112,7 +116,7 @@ public class LoveromRipper extends AbstractHTMLRipper {
     public String getPrefix(int index) {
         String prefix = "";
         if (keepSortOrder() && Utils.getConfigBoolean("download.save_order", true)) {
-            prefix = String.format("%03d_", index);
+            prefix = String.format("7z.%03d", index);
         }
         return prefix;
     }

From fc52da3497afadb745250ba39a1b297dccbbef88 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Wed, 11 Jul 2018 13:40:09 -0400
Subject: [PATCH 103/132] No longer tries to send download compete message
 every time ripme saves a url to a text file

---
 src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java
index 977de15d..97943b33 100644
--- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java
@@ -71,10 +71,7 @@ public abstract class AlbumRipper extends AbstractRipper {
             try (FileWriter fw = new FileWriter(urlFile, true)) {
                 fw.write(url.toExternalForm());
                 fw.write("\n");
-
-                RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, urlFile);
                 itemsCompleted.put(url, new File(urlFile));
-                observer.update(this, msg);
             } catch (IOException e) {
                 LOGGER.error("Error while writing to " + urlFile, e);
             }

From f7a1255bab7d2bbfc1225a461b598adf930b459d Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Wed, 11 Jul 2018 13:52:29 -0400
Subject: [PATCH 104/132] Added loverom unit test

---
 .../ripme/tst/ripper/rippers/LoveromRipperTest.java | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/LoveromRipperTest.java

diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LoveromRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LoveromRipperTest.java
new file mode 100644
index 00000000..377b2212
--- /dev/null
+++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LoveromRipperTest.java
@@ -0,0 +1,13 @@
+package com.rarchives.ripme.tst.ripper.rippers;
+
+import com.rarchives.ripme.ripper.rippers.LoveromRipper;
+
+import java.io.IOException;
+import java.net.URL;
+
+public class LoveromRipperTest extends RippersTest{
+    public void testRip() throws IOException {
+        LoveromRipper ripper = new LoveromRipper(new URL("https://www.loveroms.com/download/nintendo/adventures-of-tom-sawyer-u/107165"));
+        testRipper(ripper);
+    }
+}

From 3553118b54e84968c8bf5530dc548001227de2f7 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Thu, 12 Jul 2018 13:44:45 -0400
Subject: [PATCH 105/132] Added support for Desuarchive.org

---
 .../java/com/rarchives/ripme/ripper/rippers/ChanRipper.java    | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
index 0ab86244..85528e8d 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
@@ -97,6 +97,9 @@ public class ChanRipper extends AbstractHTMLRipper {
         if (url.toExternalForm().contains("xchan.pw") && url.toExternalForm().contains("/board/")) {
             return true;
         }
+        if (url.toExternalForm().contains("desuarchive.org")) {
+            return true;
+        }
         return false;
     }
 

From a65ad6f75616b8f022a5dee0fd773353248e1783 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Fri, 13 Jul 2018 18:14:36 -0400
Subject: [PATCH 106/132] Fixed ManganeloRipper

---
 .../com/rarchives/ripme/ripper/rippers/ManganeloRipper.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java
index 79e670d1..cabb4188 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java
@@ -99,7 +99,7 @@ public class ManganeloRipper extends AbstractHTMLRipper {
             Collections.reverse(urlsToGrab);
 
             for (String url : urlsToGrab) {
-                result.addAll(getURLsFromChap("https:" + url));
+                result.addAll(getURLsFromChap(url));
             }
         } else if (url.toExternalForm().contains("/chapter/")) {
             result.addAll(getURLsFromChap(doc));

From 3287b8c59afdb02f55def83aae3c3ff4a8f20d51 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Fri, 13 Jul 2018 19:01:55 -0400
Subject: [PATCH 107/132] Fixed erome video downloading

---
 .../java/com/rarchives/ripme/ripper/rippers/EromeRipper.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java
index 2711cd1d..27e0e0be 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java
@@ -105,7 +105,7 @@ public class EromeRipper extends AbstractHTMLRipper {
 
     @Override
     public String getGID(URL url) throws MalformedURLException {
-        Pattern p = Pattern.compile("^https?://www.erome.com/a/([a-zA-Z0-9]*)/?$");
+        Pattern p = Pattern.compile("^https?://www.erome.com/[ai]/([a-zA-Z0-9]*)/?$");
         Matcher m = p.matcher(url.toExternalForm());
         if (m.matches()) {
             return m.group(1);

From 3f18d8a9909a0e8578c6d6b2b3f16e26dc367297 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Fri, 13 Jul 2018 20:06:54 -0400
Subject: [PATCH 108/132] Readded Imagearn ripper and test

---
 .../ripme/ripper/rippers/ImagearnRipper.java  | 112 ++++++++++++++++++
 .../ripper/rippers/ImagearnRipperTest.java    |  13 ++
 2 files changed, 125 insertions(+)
 create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java
 create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagearnRipperTest.java

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java
new file mode 100644
index 00000000..062217b2
--- /dev/null
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java
@@ -0,0 +1,112 @@
+package com.rarchives.ripme.ripper.rippers;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+
+import com.rarchives.ripme.ripper.AbstractHTMLRipper;
+import com.rarchives.ripme.utils.Http;
+
+public class ImagearnRipper extends AbstractHTMLRipper {
+
+    public ImagearnRipper(URL url) throws IOException {
+        super(url);
+    }
+
+    @Override
+    public String getHost() {
+        return "imagearn";
+    }
+    @Override
+    public String getDomain() {
+        return "imagearn.com";
+    }
+
+    @Override
+    public String getGID(URL url) throws MalformedURLException {
+        Pattern p = Pattern.compile("^.*imagearn.com/+gallery.php\\?id=([0-9]+).*$");
+        Matcher m = p.matcher(url.toExternalForm());
+        if (m.matches()) {
+            return m.group(1);
+        }
+        throw new MalformedURLException(
+                "Expected imagearn.com gallery formats: "
+                        + "imagearn.com/gallery.php?id=####..."
+                        + " Got: " + url);
+    }
+
+    public URL sanitizeURL(URL url) throws MalformedURLException {
+        Pattern p = Pattern.compile("^.*imagearn.com/+image.php\\?id=[0-9]+.*$");
+        Matcher m = p.matcher(url.toExternalForm());
+        if (m.matches()) {
+            // URL points to imagearn *image*, not gallery
+            try {
+                url = getGalleryFromImage(url);
+            } catch (Exception e) {
+                LOGGER.error("[!] " + e.getMessage(), e);
+            }
+        }
+        return url;
+    }
+
+    private URL getGalleryFromImage(URL url) throws IOException {
+        Document doc = Http.url(url).get();
+        for (Element link : doc.select("a[href~=^gallery\\.php.*$]")) {
+            LOGGER.info("LINK: " + link.toString());
+            if (link.hasAttr("href")
+                    && link.attr("href").contains("gallery.php")) {
+                url = new URL("http://imagearn.com/" + link.attr("href"));
+                LOGGER.info("[!] Found gallery from given link: " + url);
+                return url;
+            }
+        }
+        throw new IOException("Failed to find gallery at URL " + url);
+    }
+
+    @Override
+    public Document getFirstPage() throws IOException {
+        return Http.url(url).get();
+    }
+
+    @Override
+    public String getAlbumTitle(URL url) throws MalformedURLException {
+        try {
+            Document doc = getFirstPage();
+            String title = doc.select("h3 > strong").first().text(); // profile name
+            return getHost() + "_" + title + "_" + getGID(url);
+        } catch (Exception e) {
+            // Fall back to default album naming convention
+            LOGGER.warn("Failed to get album title from " + url, e);
+        }
+        return super.getAlbumTitle(url);
+    }
+
+    @Override
+    public List<String> getURLsFromPage(Document doc) {
+        List<String> imageURLs = new ArrayList<>();
+        for (Element thumb : doc.select("div#gallery > div > a")) {
+            String imageURL = thumb.attr("href");
+            try {
+                Document imagedoc = new Http("http://imagearn.com/" + imageURL).get();
+                String image = imagedoc.select("a.thickbox").first().attr("href");
+                imageURLs.add(image);
+            } catch (IOException e) {
+                LOGGER.warn("Was unable to download page: " + imageURL);
+            }
+        }
+        return imageURLs;
+    }
+
+    @Override
+    public void downloadURL(URL url, int index) {
+        addURLToDownload(url, getPrefix(index));
+        sleep(1000);
+    }
+}
diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagearnRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagearnRipperTest.java
new file mode 100644
index 00000000..b9710635
--- /dev/null
+++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagearnRipperTest.java
@@ -0,0 +1,13 @@
+package com.rarchives.ripme.tst.ripper.rippers;
+
+import java.io.IOException;
+import java.net.URL;
+
+import com.rarchives.ripme.ripper.rippers.ImagearnRipper;
+
+public class ImagearnRipperTest extends RippersTest {
+    public void testImagearnRip() throws IOException {
+        ImagearnRipper ripper = new ImagearnRipper(new URL("http://imagearn.com//gallery.php?id=578682"));
+        testRipper(ripper);
+    }
+}

From cbfd3638425282eeb4cadc15747ff04315486400 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sat, 14 Jul 2018 14:56:45 -0400
Subject: [PATCH 109/132] 1.7.59: Added Loverom ripper; added Imagearn ripper;
 Added support for Desuarchive.org; Fixed erome ripper

---
 pom.xml                                               | 2 +-
 ripme.json                                            | 7 ++++---
 src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/pom.xml b/pom.xml
index 9de7c444..1010bdac 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
   <groupId>com.rarchives.ripme</groupId>
   <artifactId>ripme</artifactId>
   <packaging>jar</packaging>
-  <version>1.7.58</version>
+  <version>1.7.59</version>
   <name>ripme</name>
   <url>http://rip.rarchives.com</url>
   <properties>
diff --git a/ripme.json b/ripme.json
index d835ecfe..f4a73d8b 100644
--- a/ripme.json
+++ b/ripme.json
@@ -1,7 +1,7 @@
 {
-    "latestVersion": "1.7.58",
-    "currentHash": "d94ae3f80ae38b5bcd7abe996cf70ed3de610bba794665be01518892a11c70c5",
+    "currentHash": "8861d48484c0b029249408f81ccc8bd70c7d4de9343d867e9615969cc67c9fd7",
     "changeList": [
+        "1.7.59: Added Loverom ripper; added Imagearn ripper; Added support for Desuarchive.org; Fixed erome ripper",
         "1.7.58: Fixed Deviantart ripper; fixed HitomiRipper; Fixed ManganeloRipper; Fixed update box formating",
         "1.7.57: Got DeviantartRipper working again; Imagefap ripper now downloads full sized images; Twitter ripper can now rip extended tweets; Added nl_NL translation",
         "1.7.56: Fixed DeviantartRipper ripper; Added support for resuming file downloads; Fixed erome ripper; Fixed ModelmayhemRipper NSFW image downloading",
@@ -230,5 +230,6 @@
         "1.0.4: Fixed spaces-in-directory bug",
         "1.0.3: Added VK.com ripper",
         "1.0.1: Added auto-update functionality"
-    ]
+    ],
+    "latestVersion": "1.7.59"
 }
\ No newline at end of file
diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
index 77ccc1b1..188a5dc0 100644
--- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
+++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
@@ -20,7 +20,7 @@ import com.rarchives.ripme.utils.Utils;
 public class UpdateUtils {
 
     private static final Logger logger = Logger.getLogger(UpdateUtils.class);
-    private static final String DEFAULT_VERSION = "1.7.58";
+    private static final String DEFAULT_VERSION = "1.7.59";
     private static final String REPO_NAME = "ripmeapp/ripme";
     private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json";
     private static final String mainFileName = "ripme.jar";

From 7cf3a013791554bdc32ec84a3af3023662c7831e Mon Sep 17 00:00:00 2001
From: 0x1f595 <0x1f595@users.noreply.github.com>
Date: Sat, 14 Jul 2018 13:06:29 -0600
Subject: [PATCH 110/132] Adding Jab Archives ripper

This currently only supports ripping bottom-level folders. If a folder you rip contains another folder, that subfolder will not be fully ripped, only the source image from its thumbnail will be included in the rip.
---
 .../ripper/rippers/JabArchivesRipper.java     | 78 +++++++++++++++++++
 1 file changed, 78 insertions(+)
 create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/JabArchivesRipper.java

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/JabArchivesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/JabArchivesRipper.java
new file mode 100644
index 00000000..eeba986a
--- /dev/null
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/JabArchivesRipper.java
@@ -0,0 +1,78 @@
+package com.rarchives.ripme.ripper.rippers;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.rarchives.ripme.ripper.AbstractHTMLRipper;
+import com.rarchives.ripme.utils.Http;
+
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+
+public class JabArchivesRipper extends AbstractHTMLRipper {
+
+    public JabArchivesRipper(URL url) throws IOException {
+        super(url);
+    }
+
+    @Override
+    public String getHost() {
+        return "jabarchives";
+    }
+
+    @Override
+    public String getDomain() {
+        return "jabarchives.com";
+    }
+
+    @Override
+    public String getGID(URL url) throws MalformedURLException {
+        Pattern p = Pattern.compile("^https?://jabarchives.com/main/view/([a-zA-Z0-9_]+).*$");
+        Matcher m = p.matcher(url.toExternalForm());
+        if (m.matches()) {
+            // Return the text contained between () in the regex
+            return m.group(1);
+        }
+        throw new MalformedURLException(
+                "Expected javarchives.com URL format: " +
+                "jabarchives.com/main/view/albumname - got " + url + " instead");
+    }
+
+    @Override
+    public Document getFirstPage() throws IOException {
+        // "url" is an instance field of the superclass
+        return Http.url(url).get();
+    }
+
+    @Override
+    public Document getNextPage(Document doc) throws IOException {
+        // Find next page
+        Elements hrefs = doc.select("a[title=\"Next page\"]");
+        if (hrefs.isEmpty()) {
+            throw new IOException("No more pages");
+        }
+        String nextUrl = "http://jabarchives.com" + hrefs.first().attr("href");
+        sleep(500);
+        return Http.url(nextUrl).get();
+    }
+
+    @Override
+    public List<String> getURLsFromPage(Document doc) {
+        List<String> result = new ArrayList<String>();
+        for (Element el : doc.select("#contentMain img")) {
+            result.add("http://jabarchives.com" + el.attr("src").replace("thumb", "large"));
+        }
+        return result;
+    }
+
+    @Override
+    public void downloadURL(URL url, int index) {
+        addURLToDownload(url, getPrefix(index));
+    }
+}

From daae0747137726cb68a8ff948fb6f30cb1326198 Mon Sep 17 00:00:00 2001
From: 0x1f595 <0x1f595@users.noreply.github.com>
Date: Sat, 14 Jul 2018 13:10:53 -0600
Subject: [PATCH 111/132] Use SSL by default on Jav Archives

---
 .../com/rarchives/ripme/ripper/rippers/JabArchivesRipper.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/JabArchivesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/JabArchivesRipper.java
index eeba986a..1ad0b2f7 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/JabArchivesRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/JabArchivesRipper.java
@@ -57,7 +57,7 @@ public class JabArchivesRipper extends AbstractHTMLRipper {
         if (hrefs.isEmpty()) {
             throw new IOException("No more pages");
         }
-        String nextUrl = "http://jabarchives.com" + hrefs.first().attr("href");
+        String nextUrl = "https://jabarchives.com" + hrefs.first().attr("href");
         sleep(500);
         return Http.url(nextUrl).get();
     }
@@ -66,7 +66,7 @@ public class JabArchivesRipper extends AbstractHTMLRipper {
     public List<String> getURLsFromPage(Document doc) {
         List<String> result = new ArrayList<String>();
         for (Element el : doc.select("#contentMain img")) {
-            result.add("http://jabarchives.com" + el.attr("src").replace("thumb", "large"));
+            result.add("https://jabarchives.com" + el.attr("src").replace("thumb", "large"));
         }
         return result;
     }

From 1cf4aba2abd0324b3305b43eaf89e609cbb161a7 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sun, 15 Jul 2018 14:24:26 -0400
Subject: [PATCH 112/132] loveroms ripper now properly names files

---
 .../com/rarchives/ripme/ripper/rippers/LoveromRipper.java     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java
index 83af405f..6d98a690 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java
@@ -70,9 +70,9 @@ public class LoveromRipper extends AbstractHTMLRipper {
     @Override
     public void downloadURL(URL url, int index) {
         if (multipart) {
-            addURLToDownload(url, "", "", "", null, null, getPrefix(index));
+            addURLToDownload(url, "", "", "", null, null, "7z." + getPrefix(index));
         } else {
-            addURLToDownload(url);
+            addURLToDownload(url, "", "", "", null, null, "7z");
         }
     }
 

From 3f7b6711e3ea1c9f29333cb18867c196670b0228 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sun, 15 Jul 2018 14:39:14 -0400
Subject: [PATCH 113/132] getFilesFromURL now handles imgur gifvs

---
 src/main/java/com/rarchives/ripme/utils/RipUtils.java | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java
index 9845145c..00db501a 100644
--- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java
+++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java
@@ -54,6 +54,15 @@ public class RipUtils {
             } catch (IOException e) {
                 logger.error("[!] Exception while loading album " + url, e);
             }
+        }  else if (url.getHost().endsWith("i.imgur.com") && url.toExternalForm().contains("gifv")) {
+            // links to imgur gifvs
+            try {
+                result.add(new URL(Http.url(url).get().select("meta[itemprop=contentURL]").attr("content")));
+            } catch (IOException e) {
+                logger.info("Couldn't get gifv from " + url);
+            }
+            return result;
+
         }
         else if (url.getHost().endsWith("gfycat.com")) {
             try {

From 726e303a08e2f890adf0f1613b4c02c23cfaacce Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sun, 15 Jul 2018 15:03:56 -0400
Subject: [PATCH 114/132] improved how riputils gets imgur mp4s

---
 src/main/java/com/rarchives/ripme/utils/RipUtils.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java
index 00db501a..e7ffce65 100644
--- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java
+++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java
@@ -57,7 +57,7 @@ public class RipUtils {
         }  else if (url.getHost().endsWith("i.imgur.com") && url.toExternalForm().contains("gifv")) {
             // links to imgur gifvs
             try {
-                result.add(new URL(Http.url(url).get().select("meta[itemprop=contentURL]").attr("content")));
+                result.add(new URL(url.toExternalForm().replaceAll(".gifv", ".mp4")));
             } catch (IOException e) {
                 logger.info("Couldn't get gifv from " + url);
             }

From 47cab788435793ff3ad59ce9d852000a023823bb Mon Sep 17 00:00:00 2001
From: "Mateus B. Cassiano" <mbc07@live.com>
Date: Mon, 16 Jul 2018 02:08:03 -0400
Subject: [PATCH 115/132] ArtStationRipper: fix ripping from user pages with
 more than 50 projects

---
 .../ripper/rippers/ArtStationRipper.java      | 22 ++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java
index aeb1991d..611d9be6 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java
@@ -21,6 +21,7 @@ public class ArtStationRipper extends AbstractJSONRipper {
     private ParsedURL albumURL;
     private String projectName;
     private Integer projectIndex;
+    private Integer projectPageNumber;
 
     public ArtStationRipper(URL url) throws IOException {
         super(url);
@@ -95,14 +96,25 @@ public class ArtStationRipper extends AbstractJSONRipper {
     @Override
     protected JSONObject getNextPage(JSONObject doc) throws IOException {
         if (albumURL.getType() == URL_TYPE.USER_PORTFOLIO) {
-            // Initialize the index if it hasn't been initialized already
-            if (projectIndex == null) {
-                projectIndex = 1;
+            // Initialize the page number if it hasn't been initialized already
+            if (projectPageNumber == null) {
+                projectPageNumber = 1;
             }
 
-            JSONObject albumContent = Http.url(albumURL.getLocation()).getJSON();
+            // Each page holds a maximum of 50 projects. Initialize the index if it hasn't
+            // been initialized already or increment page number and reset the index if all
+            // projects of the current page were already processed
+            if (projectIndex == null) {
+                projectIndex = 0;
+            } else if (projectIndex > 49) {
+                projectPageNumber++;
+                projectIndex = 0;
+            }
 
-            if (albumContent.getInt("total_count") > projectIndex) {
+            Integer currentProject = ((projectPageNumber - 1) * 50) + (projectIndex + 1);
+            JSONObject albumContent = Http.url(albumURL.getLocation() + "?page=" + projectPageNumber).getJSON();
+
+            if (albumContent.getInt("total_count") > currentProject) {
                 // Get JSON of the next project and return it
                 JSONObject projectInfo = albumContent.getJSONArray("data").getJSONObject(projectIndex);
                 ParsedURL projectURL = parseURL(new URL(projectInfo.getString("permalink")));

From 9bb28cf4c7f40718835f144902892450f27be0a0 Mon Sep 17 00:00:00 2001
From: "Mateus B. Cassiano" <mbc07@live.com>
Date: Mon, 16 Jul 2018 02:09:06 -0400
Subject: [PATCH 116/132] ArtStationRipper: update unit test

---
 .../rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java
index 381c007f..fe72c046 100644
--- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java
+++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java
@@ -23,6 +23,7 @@ public class ArtStationRipperTest extends RippersTest {
     public void testArtStationUserProfiles() throws IOException {
         List<URL> contentURLs = new ArrayList<>();
         contentURLs.add(new URL("https://www.artstation.com/heitoramatsu"));
+        contentURLs.add(new URL("https://artstation.com/kuvshinov_ilya"));
         contentURLs.add(new URL("http://artstation.com/givemeapiggy"));
         for (URL url : contentURLs) {
             ArtStationRipper ripper = new ArtStationRipper(url);

From 0c4ec95ca4943bb0c067ca33a944a9623913d822 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Thu, 19 Jul 2018 00:21:39 -0400
Subject: [PATCH 117/132] Fixed xhamster getNextPage

---
 .../rarchives/ripme/ripper/rippers/XhamsterRipper.java | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java
index 96f20ec3..86bb81b3 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java
@@ -69,11 +69,13 @@ public class XhamsterRipper extends AbstractHTMLRipper {
 
     @Override
     public Document getNextPage(Document doc) throws IOException {
-        if (!doc.select("a.next").first().attr("href").equals("")) {
-            return Http.url(doc.select("a.next").first().attr("href")).get();
-        } else {
-            throw new IOException("No more pages");
+        if (doc.select("a.next").first() != null) {
+            if (doc.select("a.next").first().attr("href").startsWith("http")) {
+                return Http.url(doc.select("a.next").first().attr("href")).get();
+            }
         }
+        throw new IOException("No more pages");
+
     }
 
     @Override

From 368646145d31b8e6d6647965e93c955c83b29e69 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sun, 22 Jul 2018 19:02:06 -0400
Subject: [PATCH 118/132] Can now get file type from magic number

---
 .../rarchives/ripme/ripper/DownloadFileThread.java  | 13 +++++++++++++
 src/main/java/com/rarchives/ripme/utils/Utils.java  | 10 ++++++++++
 2 files changed, 23 insertions(+)

diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
index 4f189c3c..c699522e 100644
--- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
+++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
@@ -6,9 +6,11 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.lang.reflect.Array;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.net.URLConnection;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -181,6 +183,17 @@ class DownloadFileThread extends Thread {
                         saveAs = new File(saveAs.toString() + "." + fileExt);
                     } else {
                         logger.error("Was unable to get content type from stream");
+                        // Try to get the file type from the magic number
+                        byte[] magicBytes = new byte[8];
+                        bis.read(magicBytes,0, 5);
+                        bis.reset();
+                        fileExt = Utils.getEXTFromMagic(magicBytes);
+                        if (fileExt != null) {
+                            saveAs = new File(saveAs.toString() + "." + fileExt);
+                        } else {
+                            logger.error("Was unable to get content type using magic number");
+                            logger.error("Magic number was: " + Arrays.toString(magicBytes));
+                        }
                     }
                 }
                 // If we're resuming a download we append data to the existing file
diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java
index e57acf77..77a062d0 100644
--- a/src/main/java/com/rarchives/ripme/utils/Utils.java
+++ b/src/main/java/com/rarchives/ripme/utils/Utils.java
@@ -15,6 +15,7 @@ import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Array;
 import java.lang.reflect.Constructor;
 import java.net.URISyntaxException;
 import java.net.URL;
@@ -731,4 +732,13 @@ public class Utils {
                 Utils.bytesToHumanReadable(bytesTotal);
     }
 
+    public static String getEXTFromMagic(byte[] magic) {
+        if (Arrays.equals(magic, new byte[]{-1, -40, -1, -37, 0, 0, 0, 0})) {
+            return "jpeg";
+        } else {
+            LOGGER.info("Unknown magic number " + Arrays.toString(magic));
+        }
+        return null;
+    }
+
 }
\ No newline at end of file

From 1d60bf695667a297e495ec86804abbf9a0686fdc Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sun, 22 Jul 2018 19:31:11 -0400
Subject: [PATCH 119/132] Added a fuzzyExist func to check if a file exist when
 we don't know it's extension (For use with rippers that use
 getFileExtFromMIME)

---
 .../ripme/ripper/DownloadFileThread.java      |  2 +-
 .../java/com/rarchives/ripme/utils/Utils.java | 22 +++++++++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
index c699522e..f0d78ddb 100644
--- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
+++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
@@ -79,7 +79,7 @@ class DownloadFileThread extends Thread {
             observer.downloadErrored(url, "Download interrupted");
             return;
         }
-        if (saveAs.exists() && !observer.tryResumeDownload()) {
+        if (saveAs.exists() && !observer.tryResumeDownload() || Utils.fuzzyExists(new File(saveAs.getParent()), saveAs.getName())) {
             if (Utils.getConfigBoolean("file.overwrite", false)) {
                 logger.info("[!] Deleting existing file" + prettySaveAs);
                 saveAs.delete();
diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java
index 77a062d0..31af93a0 100644
--- a/src/main/java/com/rarchives/ripme/utils/Utils.java
+++ b/src/main/java/com/rarchives/ripme/utils/Utils.java
@@ -741,4 +741,26 @@ public class Utils {
         return null;
     }
 
+    // Checks if a file exists ignoring it's extension.
+    // Code from: https://stackoverflow.com/a/17698068
+    public static boolean fuzzyExists(File folder, String fileName) {
+        if (!folder.exists()) {
+            return false;
+        }
+        File[] listOfFiles = folder.listFiles();
+        if (listOfFiles == null) {
+            return false;
+        }
+
+        for (File file : listOfFiles) {
+            if (file.isFile()) {
+                String[] filename = file.getName().split("\\.(?=[^\\.]+$)"); //split filename from it's extension
+                if(filename[0].equalsIgnoreCase(fileName)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
 }
\ No newline at end of file

From cb1fec52f551c1091c2c5940fff2ee2117f163cd Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Mon, 23 Jul 2018 03:12:03 -0400
Subject: [PATCH 120/132] saveAs.exists() imrpovements

---
 .../java/com/rarchives/ripme/ripper/DownloadFileThread.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
index f0d78ddb..55b8ffde 100644
--- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
+++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
@@ -79,7 +79,8 @@ class DownloadFileThread extends Thread {
             observer.downloadErrored(url, "Download interrupted");
             return;
         }
-        if (saveAs.exists() && !observer.tryResumeDownload() || Utils.fuzzyExists(new File(saveAs.getParent()), saveAs.getName())) {
+        if (saveAs.exists() && !observer.tryResumeDownload() && !getFileExtFromMIME ||
+                Utils.fuzzyExists(new File(saveAs.getParent()), saveAs.getName()) && getFileExtFromMIME && !observer.tryResumeDownload()) {
             if (Utils.getConfigBoolean("file.overwrite", false)) {
                 logger.info("[!] Deleting existing file" + prettySaveAs);
                 saveAs.delete();

From 58cdea1ee22ffff0166117a8625e80ed3404564d Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Mon, 23 Jul 2018 05:02:25 -0400
Subject: [PATCH 121/132] Added some unit tests

---
 .../com/rarchives/ripme/tst/UtilsTest.java    | 34 +++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 src/test/java/com/rarchives/ripme/tst/UtilsTest.java

diff --git a/src/test/java/com/rarchives/ripme/tst/UtilsTest.java b/src/test/java/com/rarchives/ripme/tst/UtilsTest.java
new file mode 100644
index 00000000..d0789b39
--- /dev/null
+++ b/src/test/java/com/rarchives/ripme/tst/UtilsTest.java
@@ -0,0 +1,34 @@
+package com.rarchives.ripme.tst;
+
+import junit.framework.TestCase;
+import com.rarchives.ripme.utils.Utils;
+
+public class UtilsTest extends TestCase {
+
+    public void testGetEXTFromMagic() {
+        assertEquals("jpeg", Utils.getEXTFromMagic(new byte[]{-1, -40, -1, -37, 0, 0, 0, 0}));
+    }
+
+    public void testStripURLParameter() {
+        assertEquals("http://example.tld/image.ext",
+                Utils.stripURLParameter("http://example.tld/image.ext?param", "param"));
+    }
+
+    public void testShortenPath() {
+        String path = "/test/test/test/test/test/test/test/test/";
+        assertEquals("/test/test1", Utils.shortenPath("/test/test1"));
+        assertEquals("/test/test/t...st/test/test", Utils.shortenPath(path));
+    }
+
+    public void testBytesToHumanReadable() {
+        assertEquals("10.00iB", Utils.bytesToHumanReadable(10));
+        assertEquals("1.00KiB", Utils.bytesToHumanReadable(1024));
+        assertEquals("1.00MiB", Utils.bytesToHumanReadable(1024 * 1024));
+        assertEquals("1.00GiB", Utils.bytesToHumanReadable(1024 * 1024 * 1024));
+    }
+
+    public void testGetListOfAlbumRippers() throws Exception{
+        assert(!Utils.getListOfAlbumRippers().isEmpty());
+    }
+
+}

From af33869e521853f10ee2816f4116e9cb5ec52f68 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Tue, 24 Jul 2018 23:49:53 -0400
Subject: [PATCH 122/132] Fixed EightmusesRipper

---
 .../ripper/rippers/EightmusesRipper.java      | 58 +++++++++++++------
 1 file changed, 41 insertions(+), 17 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java
index 13a22213..9477b41f 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java
@@ -11,6 +11,7 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import com.rarchives.ripme.utils.Utils;
+import org.json.JSONObject;
 import org.jsoup.Connection.Response;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
@@ -116,25 +117,24 @@ public class EightmusesRipper extends AbstractHTMLRipper {
                     image = thumb.attr("data-cfsrc");
                 }
                 else {
-                    String imageHref = thumb.attr("href");
-                    if (imageHref.equals("")) continue;
-                    if (imageHref.startsWith("/")) {
-                        imageHref = "https://www.8muses.com" + imageHref;
-                    }
+                    // Deobfustace the json data
+                    String rawJson = deobfuscateJSON(page.select("script#ractive-public").html()
+                            .replaceAll("&gt;", ">").replaceAll("&lt;", "<").replace("&amp;", "&"));
+                    JSONObject json = new JSONObject(rawJson);
                     try {
-                        LOGGER.info("Retrieving full-size image location from " + imageHref);
-                        image = getFullSizeImage(imageHref);
-                        URL imageUrl = new URL(image);
-                        if (Utils.getConfigBoolean("8muses.use_short_names", false)) {
-                            addURLToDownload(imageUrl, getPrefixShort(x), getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies, "", null, true);
-                        } else {
-                            addURLToDownload(imageUrl, getPrefixLong(x), getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies, "", null, true);
+                        for (int i = 0; i != json.getJSONArray("pictures").length(); i++) {
+                            image = "https://www.8muses.com/image/fm/" + json.getJSONArray("pictures").getJSONObject(i).getString("publicUri");
+                            URL imageUrl = new URL(image);
+                            if (Utils.getConfigBoolean("8muses.use_short_names", false)) {
+                                addURLToDownload(imageUrl, getPrefixShort(x), getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies, "", null, true);
+                            } else {
+                                addURLToDownload(imageUrl, getPrefixLong(x), getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies, "", null, true);
+                            }
+                            // X is our page index
+                            x++;
                         }
-                        // X is our page index
-                        x++;
 
                     } catch (IOException e) {
-                        LOGGER.error("Failed to get full-size image from " + imageHref);
                         continue;
                     }
                 }
@@ -154,7 +154,7 @@ public class EightmusesRipper extends AbstractHTMLRipper {
         sendUpdate(STATUS.LOADING_RESOURCE, imageUrl);
         LOGGER.info("Getting full sized image from " + imageUrl);
         Document doc = new Http(imageUrl).get(); // Retrieve the webpage  of the image URL
-        String imageName = doc.select("input[id=imageName]").attr("value"); // Select the "input" element from the page
+        String imageName = doc.select("div.photo > a > img").attr("src");
         return "https://www.8muses.com/image/fm/" + imageName;
     }
 
@@ -189,4 +189,28 @@ public class EightmusesRipper extends AbstractHTMLRipper {
     public String getPrefixShort(int index) {
         return String.format("%03d", index);
     }
-}
+
+    private String deobfuscateJSON(String obfuscatedString) {
+        LOGGER.info("obfuscatedString: " + obfuscatedString);
+        StringBuilder deobfuscatedString = new StringBuilder();
+        // The first char in one of 8muses obfuscated strings is always ! so we replace it
+        for (char ch : obfuscatedString.replaceFirst("!", "").toCharArray()){
+            LOGGER.info(ch + ":" + deobfuscateChar(ch));
+            LOGGER.info((int) ch + ":" + (int) deobfuscateChar(ch).charAt(0));
+            deobfuscatedString.append(deobfuscateChar(ch));
+        }
+        return deobfuscatedString.toString();
+    }
+
+    private String deobfuscateChar(char c) {
+        if ((int) c == 32) {
+            return fromCharCode(32);
+        }
+        return fromCharCode(33 + (c + 14) % 94);
+
+    }
+
+    private static String fromCharCode(int... codePoints) {
+        return new String(codePoints, 0, codePoints.length);
+    }
+}
\ No newline at end of file

From 0f3148f47fc40f23d05364ba8893b2a6935bca44 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Wed, 25 Jul 2018 00:03:28 -0400
Subject: [PATCH 123/132] Removed some debugging statments

---
 .../com/rarchives/ripme/ripper/rippers/EightmusesRipper.java   | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java
index 9477b41f..35ee4218 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java
@@ -191,12 +191,9 @@ public class EightmusesRipper extends AbstractHTMLRipper {
     }
 
     private String deobfuscateJSON(String obfuscatedString) {
-        LOGGER.info("obfuscatedString: " + obfuscatedString);
         StringBuilder deobfuscatedString = new StringBuilder();
         // The first char in one of 8muses obfuscated strings is always ! so we replace it
         for (char ch : obfuscatedString.replaceFirst("!", "").toCharArray()){
-            LOGGER.info(ch + ":" + deobfuscateChar(ch));
-            LOGGER.info((int) ch + ":" + (int) deobfuscateChar(ch).charAt(0));
             deobfuscatedString.append(deobfuscateChar(ch));
         }
         return deobfuscatedString.toString();

From 24b1c7175e368688603d274829ede0b698f1f22e Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Wed, 25 Jul 2018 00:28:39 -0400
Subject: [PATCH 124/132] 1.7.60: Fixed EightmusesRipper; added Jab Archives
 ripper; loveroms ripper now properly names files; Fixed ArtStationRipper

---
 pom.xml                                               | 2 +-
 ripme.json                                            | 5 +++--
 src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/pom.xml b/pom.xml
index 1010bdac..f6f08811 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
   <groupId>com.rarchives.ripme</groupId>
   <artifactId>ripme</artifactId>
   <packaging>jar</packaging>
-  <version>1.7.59</version>
+  <version>1.7.60</version>
   <name>ripme</name>
   <url>http://rip.rarchives.com</url>
   <properties>
diff --git a/ripme.json b/ripme.json
index f4a73d8b..a94e75fc 100644
--- a/ripme.json
+++ b/ripme.json
@@ -1,6 +1,6 @@
 {
-    "currentHash": "8861d48484c0b029249408f81ccc8bd70c7d4de9343d867e9615969cc67c9fd7",
     "changeList": [
+        "1.7.60: Fixed EightmusesRipper; added Jab Archives ripper; loveroms ripper now properly names files; Fixed ArtStationRipper",
         "1.7.59: Added Loverom ripper; added Imagearn ripper; Added support for Desuarchive.org; Fixed erome ripper",
         "1.7.58: Fixed Deviantart ripper; fixed HitomiRipper; Fixed ManganeloRipper; Fixed update box formating",
         "1.7.57: Got DeviantartRipper working again; Imagefap ripper now downloads full sized images; Twitter ripper can now rip extended tweets; Added nl_NL translation",
@@ -231,5 +231,6 @@
         "1.0.3: Added VK.com ripper",
         "1.0.1: Added auto-update functionality"
     ],
-    "latestVersion": "1.7.59"
+    "latestVersion": "1.7.60",
+    "currentHash": "f206e478822134328763fc41676f438ee5cc795f31984613619952dab8402301"
 }
\ No newline at end of file
diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
index 188a5dc0..4749fa88 100644
--- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
+++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
@@ -20,7 +20,7 @@ import com.rarchives.ripme.utils.Utils;
 public class UpdateUtils {
 
     private static final Logger logger = Logger.getLogger(UpdateUtils.class);
-    private static final String DEFAULT_VERSION = "1.7.59";
+    private static final String DEFAULT_VERSION = "1.7.60";
     private static final String REPO_NAME = "ripmeapp/ripme";
     private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json";
     private static final String mainFileName = "ripme.jar";

From c83d93a2e375862311c24fecc4fda76b561f8927 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Wed, 25 Jul 2018 03:42:50 -0400
Subject: [PATCH 125/132] Added some more unit tests

---
 src/test/java/com/rarchives/ripme/tst/UtilsTest.java | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/test/java/com/rarchives/ripme/tst/UtilsTest.java b/src/test/java/com/rarchives/ripme/tst/UtilsTest.java
index d0789b39..f39604c2 100644
--- a/src/test/java/com/rarchives/ripme/tst/UtilsTest.java
+++ b/src/test/java/com/rarchives/ripme/tst/UtilsTest.java
@@ -3,6 +3,9 @@ package com.rarchives.ripme.tst;
 import junit.framework.TestCase;
 import com.rarchives.ripme.utils.Utils;
 
+import java.util.ArrayList;
+import java.util.Arrays;
+
 public class UtilsTest extends TestCase {
 
     public void testGetEXTFromMagic() {
@@ -31,4 +34,12 @@ public class UtilsTest extends TestCase {
         assert(!Utils.getListOfAlbumRippers().isEmpty());
     }
 
+    public void testGetByteStatusText() {
+        assertEquals("5%  - 500.00iB / 97.66KiB", Utils.getByteStatusText(5, 500, 100000));
+    }
+
+    public void testBetween() {
+        assertEquals(Arrays.asList(" is a "), Utils.between("This is a test", "This", "test"));
+    }
+
 }

From 92005bfb318996982636e194e660c5e561fc40f0 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Wed, 25 Jul 2018 08:51:04 -0400
Subject: [PATCH 126/132] Added russian translation

---
 src/main/java/com/rarchives/ripme/ui/MainWindow.java | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java
index e0114fea..f3c88f0a 100644
--- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java
+++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java
@@ -10,6 +10,7 @@ import java.awt.event.WindowAdapter;
 import java.awt.event.WindowEvent;
 import java.io.File;
 import java.io.IOException;
+import java.lang.reflect.Array;
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URL;
@@ -142,6 +143,10 @@ public final class MainWindow implements Runnable, RipStatusHandler {
 
     private ResourceBundle rb = Utils.getResourceBundle(null);
 
+    // All the langs ripme has been translated into
+    private static String[] supportedLanges = new String[] {"en_US", "de_DE", "es_ES", "fr_CH", "kr_KR", "pt_PT",
+            "fi_FI", "in_ID", "nl_NL", "porrisavvo_FI", "ru_RU"};
+
     private void updateQueueLabel() {
         if (queueListModel.size() > 0) {
             optionQueue.setText(rb.getString("Queue") + " (" + queueListModel.size() + ")");
@@ -496,7 +501,7 @@ public final class MainWindow implements Runnable, RipStatusHandler {
         configURLHistoryCheckbox = addNewCheckbox(rb.getString("remember.url.history"), "remember.url_history", true);
 
         configLogLevelCombobox = new JComboBox<>(new String[] {"Log level: Error", "Log level: Warn", "Log level: Info", "Log level: Debug"});
-        configSelectLangComboBox = new JComboBox<>(new String[] {"en_US", "de_DE", "es_ES", "fr_CH", "kr_KR", "pt_PT", "fi_FI", "in_ID", "nl_NL", "porrisavvo_FI"});
+        configSelectLangComboBox = new JComboBox<>(supportedLanges);
         configLogLevelCombobox.setSelectedItem(Utils.getConfigString("log.level", "Log level: Debug"));
         setLogLevel(configLogLevelCombobox.getSelectedItem().toString());
         configSaveDirLabel = new JLabel();

From 7a86e10f499230d96e96c3239db9b266c30d9317 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Wed, 25 Jul 2018 21:29:43 -0400
Subject: [PATCH 127/132] Added russian translation

---
 .../resources/LabelsBundle_ru_RU.properties   | 37 +++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100644 src/main/resources/LabelsBundle_ru_RU.properties

diff --git a/src/main/resources/LabelsBundle_ru_RU.properties b/src/main/resources/LabelsBundle_ru_RU.properties
new file mode 100644
index 00000000..affcda26
--- /dev/null
+++ b/src/main/resources/LabelsBundle_ru_RU.properties
@@ -0,0 +1,37 @@
+Log = Лог
+History = История
+created = создано
+modified = изменено
+Queue = Очередь
+Configuration = Настройки
+
+# Keys for the Configuration menu
+
+current.version = Текущая версия
+check.for.updates = Проверить обновления
+auto.update = Автообновление?
+max.download.threads = Максимальное число потоков:
+timeout.mill = Задержка (в миллисекундах):
+retry.download.count = Число повторов
+overwrite.existing.files = Перезаписать существующие файлы?
+sound.when.rip.completes = Звук при завершении
+preserve.order = Сохранять порядок
+save.logs = Сохранять логи
+notification.when.rip.starts = Уведомление при запуске
+save.urls.only = Сохранять только ссылки
+save.album.titles = Сохранять названия альбомов
+autorip.from.clipboard = Автоскачивание из буфера
+save.descriptions = Сохранять описания
+prefer.mp4.over.gif = Предпочесть MP4 вместо GIF
+restore.window.position = Восстановить положение окна
+remember.url.history = Запоминать историю запросов
+loading.history.from = Загрузить историю из
+
+# Misc UI keys
+
+loading.history.from.configuration = Загрузить историю из настроек
+interrupted.while.waiting.to.rip.next.album = Прервано во время ожидания скачивания следующего альбома
+inactive = Неактивно
+re-rip.checked = Перекачать выбранное
+remove = Удалить
+clear = Очистить
\ No newline at end of file

From a3678561c3eb493bed01a5b79faefaf131bd62d8 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Wed, 25 Jul 2018 21:32:29 -0400
Subject: [PATCH 128/132] Removed failing unit test

---
 .../ripme/tst/ripper/rippers/LoveromRipperTest.java          | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LoveromRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LoveromRipperTest.java
index 377b2212..be72bc85 100644
--- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LoveromRipperTest.java
+++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LoveromRipperTest.java
@@ -6,8 +6,5 @@ import java.io.IOException;
 import java.net.URL;
 
 public class LoveromRipperTest extends RippersTest{
-    public void testRip() throws IOException {
-        LoveromRipper ripper = new LoveromRipper(new URL("https://www.loveroms.com/download/nintendo/adventures-of-tom-sawyer-u/107165"));
-        testRipper(ripper);
-    }
+
 }

From 596be7b839a95ee6001a67fce31f3274138edef4 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Thu, 26 Jul 2018 11:09:04 -0400
Subject: [PATCH 129/132] Added an Abstraction for rippers that only ever
 download a single file and want to use the bytes progress bar

---
 .../ripper/AbstractSingleFileRipper.java      | 43 +++++++++++++++++++
 .../ripme/ripper/rippers/GfycatRipper.java    | 31 +------------
 .../ripper/rippers/GfycatporntubeRipper.java  | 32 +-------------
 .../ripme/ripper/rippers/XvideosRipper.java   | 32 ++------------
 4 files changed, 50 insertions(+), 88 deletions(-)
 create mode 100644 src/main/java/com/rarchives/ripme/ripper/AbstractSingleFileRipper.java

diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractSingleFileRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractSingleFileRipper.java
new file mode 100644
index 00000000..f1f8be41
--- /dev/null
+++ b/src/main/java/com/rarchives/ripme/ripper/AbstractSingleFileRipper.java
@@ -0,0 +1,43 @@
+package com.rarchives.ripme.ripper;
+
+import com.rarchives.ripme.utils.Utils;
+
+import java.io.IOException;
+import java.net.URL;
+
+
+/**
+ * This is just an extension of AbstractHTMLRipper that auto overrides a few things
+ * to help cut down on copy pasted code
+ */
+public abstract class AbstractSingleFileRipper extends AbstractHTMLRipper {
+    private int bytesTotal = 1;
+    private int bytesCompleted = 1;
+
+    protected AbstractSingleFileRipper(URL url) throws IOException {
+        super(url);
+    }
+
+    @Override
+    public String getStatusText() {
+        return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal);
+    }
+
+    @Override
+    public int getCompletionPercentage() {
+        return (int) (100 * (bytesCompleted / (float) bytesTotal));
+    }
+
+    @Override
+    public void setBytesTotal(int bytes) {
+        this.bytesTotal = bytes;
+    }
+
+    @Override
+    public void setBytesCompleted(int bytes) {
+        this.bytesCompleted = bytes;
+    }
+
+    @Override
+    public boolean useByteProgessBar() {return true;}
+}
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java
index a09d68ab..44ff857d 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java
@@ -9,18 +9,14 @@ import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import com.rarchives.ripme.ripper.AbstractHTMLRipper;
-import com.rarchives.ripme.utils.Utils;
+import com.rarchives.ripme.ripper.AbstractSingleFileRipper;
 import org.jsoup.nodes.Document;
 import org.jsoup.select.Elements;
 
 import com.rarchives.ripme.utils.Http;
 
 
-public class GfycatRipper extends AbstractHTMLRipper {
-
-    private int bytesTotal = 1;
-    private int bytesCompleted = 1;
+public class GfycatRipper extends AbstractSingleFileRipper {
 
     private static final String HOST = "gfycat.com";
 
@@ -109,27 +105,4 @@ public class GfycatRipper extends AbstractHTMLRipper {
         }
         return vidUrl;
     }
-
-    @Override
-    public String getStatusText() {
-        return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal);
-    }
-
-    @Override
-    public int getCompletionPercentage() {
-        return (int) (100 * (bytesCompleted / (float) bytesTotal));
-    }
-
-    @Override
-    public void setBytesTotal(int bytes) {
-        this.bytesTotal = bytes;
-    }
-
-    @Override
-    public void setBytesCompleted(int bytes) {
-        this.bytesCompleted = bytes;
-    }
-
-    @Override
-    public boolean useByteProgessBar() {return true;}
 }
\ No newline at end of file
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java
index 55150d9e..fd8c292a 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java
@@ -8,17 +8,12 @@ import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import com.rarchives.ripme.utils.Utils;
+import com.rarchives.ripme.ripper.AbstractSingleFileRipper;
 import org.jsoup.nodes.Document;
-import org.jsoup.nodes.Element;
 
-import com.rarchives.ripme.ripper.AbstractHTMLRipper;
 import com.rarchives.ripme.utils.Http;
 
-public class GfycatporntubeRipper extends AbstractHTMLRipper {
-
-    private int bytesTotal = 1;
-    private int bytesCompleted = 1;
+public class GfycatporntubeRipper extends AbstractSingleFileRipper {
 
     public GfycatporntubeRipper(URL url) throws IOException {
         super(url);
@@ -62,27 +57,4 @@ public class GfycatporntubeRipper extends AbstractHTMLRipper {
     public void downloadURL(URL url, int index) {
         addURLToDownload(url, getPrefix(index));
     }
-
-    @Override
-    public String getStatusText() {
-        return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal);
-    }
-
-    @Override
-    public int getCompletionPercentage() {
-        return (int) (100 * (bytesCompleted / (float) bytesTotal));
-    }
-
-    @Override
-    public void setBytesTotal(int bytes) {
-        this.bytesTotal = bytes;
-    }
-
-    @Override
-    public void setBytesCompleted(int bytes) {
-        this.bytesCompleted = bytes;
-    }
-
-    @Override
-    public boolean useByteProgessBar() {return true;}
 }
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java
index a94e7a21..ff38b347 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java
@@ -8,21 +8,18 @@ import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import com.rarchives.ripme.ripper.AbstractHTMLRipper;
-import com.rarchives.ripme.utils.Utils;
+
+import com.rarchives.ripme.ripper.AbstractSingleFileRipper;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
 
 import com.rarchives.ripme.utils.Http;
 
-public class XvideosRipper extends AbstractHTMLRipper {
+public class XvideosRipper extends AbstractSingleFileRipper {
 
     private static final String HOST = "xvideos";
 
-    private int bytesTotal = 1;
-    private int bytesCompleted = 1;
-
     public XvideosRipper(URL url) throws IOException {
         super(url);
     }
@@ -86,27 +83,4 @@ public class XvideosRipper extends AbstractHTMLRipper {
     public void downloadURL(URL url, int index) {
         addURLToDownload(url, getPrefix(index));
     }
-
-    @Override
-    public String getStatusText() {
-        return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal);
-    }
-
-    @Override
-    public int getCompletionPercentage() {
-        return (int) (100 * (bytesCompleted / (float) bytesTotal));
-    }
-
-    @Override
-    public void setBytesTotal(int bytes) {
-        this.bytesTotal = bytes;
-    }
-
-    @Override
-    public void setBytesCompleted(int bytes) {
-        this.bytesCompleted = bytes;
-    }
-
-    @Override
-    public boolean useByteProgessBar() {return true;}
 }
\ No newline at end of file

From 0a0b6d299a334c2027c0d13697f42b98024ee6ba Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sun, 29 Jul 2018 10:37:15 -0400
Subject: [PATCH 130/132] Added kenzato.uk to CheveretoRippers

---
 .../ripme/ripper/rippers/CheveretoRipper.java          | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java
index cb0b765f..005ba5c7 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java
@@ -28,7 +28,7 @@ public class CheveretoRipper extends AbstractHTMLRipper {
         super(url);
     }
 
-    private static List<String> explicit_domains_1 = Arrays.asList("tag-fox.com");
+    private static List<String> explicit_domains = Arrays.asList("tag-fox.com", "kenzato.uk");
 
     @Override
     public String getHost() {
@@ -43,12 +43,8 @@ public class CheveretoRipper extends AbstractHTMLRipper {
     @Override
     public boolean canRip(URL url) {
         String url_name = url.toExternalForm();
-        if (explicit_domains_1.contains(url_name.split("/")[2])) {
-            Pattern pa = Pattern.compile("(?:https?://)?(?:www\\.)?[a-z1-9-]*\\.[a-z1-9]*/album/([a-zA-Z1-9]*)/?$");
-            Matcher ma = pa.matcher(url.toExternalForm());
-            if (ma.matches()) {
+        if (explicit_domains.contains(url_name.split("/")[2])) {
                 return true;
-            }
         }
         return false;
     }
@@ -70,7 +66,7 @@ public class CheveretoRipper extends AbstractHTMLRipper {
 
     @Override
     public String getGID(URL url) throws MalformedURLException {
-        Pattern p = Pattern.compile("(?:https?://)?(?:www\\.)?[a-z1-9-]*\\.[a-z1-9]*/album/([a-zA-Z1-9]*)/?$");
+        Pattern p = Pattern.compile("(?:https?://)?(?:www\\.)?[a-z1-9-]*\\.[a-z1-9]*(?:[a-zA-Z1-9]*)/album/([a-zA-Z1-9]*)/?$");
         Matcher m = p.matcher(url.toExternalForm());
         if (m.matches()) {
             return m.group(1);

From 7196c594535181bc113c0425c9d7b68c87d591cb Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sun, 29 Jul 2018 10:40:22 -0400
Subject: [PATCH 131/132] Added unit test for kenzato.uk

---
 .../ripme/tst/ripper/rippers/CheveretoRipperTest.java        | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java
index 4f456b8d..6fd20060 100644
--- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java
+++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java
@@ -10,4 +10,9 @@ public class CheveretoRipperTest extends RippersTest {
         CheveretoRipper ripper = new CheveretoRipper(new URL("http://tag-fox.com/album/Thjb"));
         testRipper(ripper);
     }
+
+    public void testSubdirAlbum() throws IOException {
+        CheveretoRipper ripper = new CheveretoRipper(new URL("https://kenzato.uk/booru/album/TnEc"));
+        testRipper(ripper);
+    }
 }

From 771159de5689d7ee5e2ed1dababcfb4733da051c Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sun, 29 Jul 2018 10:48:46 -0400
Subject: [PATCH 132/132] Removed outdated todo

---
 src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java
index 01444642..b24017f7 100644
--- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java
@@ -224,7 +224,6 @@ public abstract class AbstractHTMLRipper extends AlbumRipper {
             if (!subdirectory.equals("")) { // Not sure about this part
                 subdirectory = File.separator + subdirectory;
             }
-            // TODO Get prefix working again, probably requires reworking a lot of stuff! (Might be fixed now)
             saveFileAs = new File(
                     workingDir.getCanonicalPath()
                     + subdirectory