Merge pull request #3 from RipMeApp/master

Merging with master

pom.xml

@@ -4,7 +4,7 @@
   <groupId>com.rarchives.ripme</groupId>
   <artifactId>ripme</artifactId>
   <packaging>jar</packaging>
-  <version>1.7.35</version>
+  <version>1.7.37</version>
   <name>ripme</name>
   <url>http://rip.rarchives.com</url>
   <properties>

ripme.json

@@ -1,6 +1,8 @@
 {
-    "latestVersion": "1.7.35",
+    "latestVersion": "1.7.37",
     "changeList": [
+        "1.7.37: MInor code clean up; Added socks proxy support; Added support for 8muses.download; Hentaifoundry no longer errors when there are no more pages; Fix bug that causes tumblr to replace https with httpss when downloading resized images",
+        "1.7.36: Fixed Instagram ripper; Fixed hentai2read ripper test; Fixed tnbtu.com ripper",
         "1.7.35: Fixed instagram ripper; hentai2read ripper now properly names folders",
         "1.7.34: Added Blackbrickroadofoz Ripper; Fixed webtoons regex",
         "1.7.33: Instagram ripper no longer errors out when downloading from more than 1 page",

src/main/java/com/rarchives/ripme/ripper/rippers/BlackbrickroadofozRipper.java

@@ -55,14 +55,8 @@ public class BlackbrickroadofozRipper extends AbstractHTMLRipper {
             throw new IOException("No more pages");
         }
         String nextPage = elem.attr("href");
-        // Some times this returns a empty string
+        return Http.url(nextPage).get();
-        // This for stops that
+
-        if (nextPage == "") {
-            throw new IOException("No more pages");
-        }
-        else {
-            return Http.url(nextPage).get();
-        }
     }
 
     @Override

src/main/java/com/rarchives/ripme/ripper/rippers/CfakeRipper.java

@@ -59,7 +59,7 @@ public class CfakeRipper extends AbstractHTMLRipper {
                 String nextPage = elem.attr("href");
                 // Some times this returns a empty string
                 // This for stops that
-                if (nextPage == "") {
+                if (nextPage.equals("")) {
                     return null;
                 }
                 else {

src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java

@@ -4,7 +4,6 @@ import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;

src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java

@@ -86,7 +86,7 @@ public class EroShareRipper extends AbstractHTMLRipper {
             throw new IOException("No more pages");
         }
             nextUrl = elem.attr("href");
-            if (nextUrl == "") {
+            if (nextUrl.equals("")) {
                 throw new IOException("No more pages");
             }
             return Http.url("eroshae.com" + nextUrl).get();

src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java

@@ -13,7 +13,6 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import com.rarchives.ripme.utils.Utils;
-import org.jsoup.Connection.Method;
 import org.jsoup.Connection.Response;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
@@ -23,7 +22,6 @@ import org.jsoup.select.Elements;
 
 import com.rarchives.ripme.ripper.AbstractHTMLRipper;
 import com.rarchives.ripme.ripper.DownloadThreadPool;
-import com.rarchives.ripme.utils.Base64;
 import com.rarchives.ripme.utils.Http;
 
 public class FuraffinityRipper extends AbstractHTMLRipper {
@@ -162,10 +160,6 @@ public class FuraffinityRipper extends AbstractHTMLRipper {
             if (!subdirectory.equals("")) {
                 subdirectory = File.separator + subdirectory;
             }
-            int o = url.toString().lastIndexOf('/')-1;
-            String test = url.toString().substring(url.toString().lastIndexOf('/',o)+1);
-            test = test.replace("/",""); // This is probably not the best way to do this.
-            test = test.replace("\\",""); // CLOSE ENOUGH!
             saveFileAs = new File(
                     workingDir.getCanonicalPath()
                             + subdirectory

src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java

@@ -33,7 +33,7 @@ public class Hentai2readRipper extends AbstractHTMLRipper {
 
         @Override
         public String getGID(URL url) throws MalformedURLException {
-            Pattern p = Pattern.compile("https?://hentai2read\\.com/([a-zA-Z0-9_-]*)/\\d/?");
+            Pattern p = Pattern.compile("https?://hentai2read\\.com/([a-zA-Z0-9_-]*)/(\\d)?/?");
             Matcher m = p.matcher(url.toExternalForm());
             if (m.matches()) {
                 return m.group(1);

src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java

@@ -52,16 +52,16 @@ public class HentaifoundryRipper extends AbstractHTMLRipper {
         Response resp = Http.url("http://www.hentai-foundry.com/").response();
         cookies = resp.cookies();
         resp = Http.url("http://www.hentai-foundry.com/?enterAgree=1&size=1500")
-                   .referrer("http://www.hentai-foundry.com/")
+                .referrer("http://www.hentai-foundry.com/")
-                   .cookies(cookies)
+                .cookies(cookies)
-                   .response();
+                .response();
         // The only cookie that seems to matter in getting around the age wall is the phpsession cookie
         cookies.putAll(resp.cookies());
         sleep(500);
         resp = Http.url(url)
-                   .referrer("http://www.hentai-foundry.com/")
+                .referrer("http://www.hentai-foundry.com/")
-                   .cookies(cookies)
+                .cookies(cookies)
-                   .response();
+                .response();
         cookies.putAll(resp.cookies());
         return resp.parse();
     }
@@ -74,12 +74,16 @@ public class HentaifoundryRipper extends AbstractHTMLRipper {
         }
         Elements els = doc.select("li.next > a");
         Element first = els.first();
-        String nextURL = first.attr("href");
+        try {
-        nextURL = "http://www.hentai-foundry.com" + nextURL;
+            String nextURL = first.attr("href");
-        return Http.url(nextURL)
+            nextURL = "http://www.hentai-foundry.com" + nextURL;
-                   .referrer(url)
+            return Http.url(nextURL)
-                   .cookies(cookies)
+                    .referrer(url)
-                   .get();
+                    .cookies(cookies)
+                    .get();
+        } catch (NullPointerException e) {
+            throw new IOException("No more pages");
+        }
     }
 
     @Override
@@ -97,13 +101,6 @@ public class HentaifoundryRipper extends AbstractHTMLRipper {
             }
             Document imagePage;
             try {
-                Response resp = Http.url("http://www.hentai-foundry.com/").response();
-                cookies = resp.cookies();
-                resp = Http.url("http://www.hentai-foundry.com/?enterAgree=1&size=1500")
-                           .referrer("http://www.hentai-foundry.com/")
-                           .cookies(cookies)
-                           .response();
-                cookies.putAll(resp.cookies());
 
                 logger.info("grabbing " + "http://www.hentai-foundry.com" + thumb.attr("href"));
                 imagePage = Http.url("http://www.hentai-foundry.com" + thumb.attr("href")).cookies(cookies).get();

src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java

@@ -374,7 +374,7 @@ public class InstagramRipper extends AbstractHTMLRipper {
             try {
                 // Sleep for a while to avoid a ban
                 sleep(2500);
-                String vars = "{\"id\":\"" + userID + "\",\"first\":100,\"after\":\"" + nextPageID + "\"}";
+                String vars = "{\"id\":\"" + userID + "\",\"first\":50,\"after\":\"" + nextPageID + "\"}";
                 String ig_gis = getIGGis(vars);
                 logger.info(ig_gis);
                 toreturn = Http.url("https://www.instagram.com/graphql/query/?query_hash=" + qHash + "&variables=" + vars

src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java

@@ -50,7 +50,7 @@ public class LusciousRipper extends AbstractHTMLRipper {
         // This is here for pages with mp4s instead of images
         String video_image = "";
         video_image = page.select("div > video > source").attr("src");
-        if (video_image != "") {
+        if (!video_image.equals("")) {
             urls.add(video_image);
         }
         return urls;

src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java

@@ -73,7 +73,7 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper {
             if (m.matches()) {
                 nextUrl = "http://myhentaicomics.com" + m.group(0);
                 }
-            if (nextUrl == "") {
+            if (nextUrl.equals("")) {
                 throw new IOException("No more pages");
             }
             // Sleep for half a sec to avoid getting IP banned
@@ -100,7 +100,7 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper {
                 Element elem = nextAlbumPage.select("a.ui-icon-right").first();
                 String nextPage = elem.attr("href");
                 pageNumber = pageNumber + 1;
-                if (nextPage == "") {
+                if (nextPage.equals("")) {
                     logger.info("Got " + pageNumber + " pages");
                     break;
                 }
@@ -220,7 +220,7 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper {
         // If true the page is a page of albums
         if (doc.toString().contains("class=\"g-item g-album\"")) {
             // This if checks that there is more than 1 page
-            if (doc.select("a.ui-icon-right").last().attr("href") != "") {
+            if (!doc.select("a.ui-icon-right").last().attr("href").equals("")) {
                 // There is more than one page so we call getListOfPages
                 List<String> pagesToRip = getListOfPages(doc);
                 logger.debug("Pages to rip = " + pagesToRip);

src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java

@@ -57,7 +57,7 @@ public class SinfestRipper extends AbstractHTMLRipper {
         String nextPage = elem.parent().attr("href");
         // Some times this returns a empty string
         // This for stops that
-        if (nextPage == "") {
+        if (nextPage.equals("")) {
             return null;
         }
         else {

src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java

@@ -230,7 +230,7 @@ public class TumblrRipper extends AlbumRipper {
                             urlString = urlString.replaceAll("_\\d+\\.", "_raw.");
                             fileURL = new URL(urlString);
                         } else {
-                            fileURL = new URL(photo.getJSONObject("original_size").getString("url").replaceAll("http", "https"));
+                            fileURL = new URL(photo.getJSONObject("original_size").getString("url").replaceAll("http:", "https:"));
                         }
                         m = p.matcher(fileURL.toString());
                         if (m.matches()) {

src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java

@@ -4,14 +4,12 @@ import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
-import org.jsoup.select.Elements;
 
 import com.rarchives.ripme.ripper.AbstractHTMLRipper;
 import com.rarchives.ripme.utils.Http;

src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java

@@ -44,7 +44,8 @@ public class WordpressComicRipper extends AbstractHTMLRipper {
         "freeadultcomix.com",
         "thisis.delvecomic.com",
         "tnbtu.com",
-        "shipinbottle.pepsaga.com"
+        "shipinbottle.pepsaga.com",
+            "8muses.download"
     );
 
     @Override
@@ -135,6 +136,12 @@ public class WordpressComicRipper extends AbstractHTMLRipper {
             if (shipinbottleMat.matches()) {
                 return true;
             }
+
+            Pattern eight_musesPat = Pattern.compile("https?://8muses.download/([a-zA-Z0-9_-]+)/?$");
+            Matcher eight_musesMat = eight_musesPat.matcher(url.toExternalForm());
+            if (eight_musesMat.matches()) {
+                return true;
+            }
         }
 
 
@@ -209,6 +216,11 @@ public class WordpressComicRipper extends AbstractHTMLRipper {
             return getHost() + "_" + "Ship_in_bottle";
         }
 
+        Pattern eight_musesPat = Pattern.compile("https?://8muses.download/([a-zA-Z0-9_-]+)/?$");
+        Matcher eight_musesMat = eight_musesPat.matcher(url.toExternalForm());
+        if (eight_musesMat.matches()) {
+            return getHost() + "_" + eight_musesMat.group(1);
+        }
         return super.getAlbumTitle(url);
     }
 
@@ -247,7 +259,7 @@ public class WordpressComicRipper extends AbstractHTMLRipper {
             nextPage = elem.attr("href");
         }
 
-        if (nextPage == "") {
+        if (nextPage.equals("")) {
             throw new IOException("No more pages");
         } else {
             return Http.url(nextPage).get();
@@ -315,6 +327,12 @@ public class WordpressComicRipper extends AbstractHTMLRipper {
             }
         }
 
+        if (url.toExternalForm().contains("8muses.download")) {
+            for (Element elem : doc.select("div.popup-gallery > figure > a")) {
+                result.add(elem.attr("href"));
+            }
+        }
+
         return result;
     }
 
@@ -327,8 +345,14 @@ public class WordpressComicRipper extends AbstractHTMLRipper {
                 || getHost().contains("themonsterunderthebed.net")) {
             addURLToDownload(url, pageTitle + "_");
         }
-        // If we're ripping a site where we can't get the page number/title we just rip normally
+        if (getHost().contains("tnbtu.com")) {
-        addURLToDownload(url, getPrefix(index));
+            // We need to set the referrer header for tnbtu
+            addURLToDownload(url, getPrefix(index), "","http://www.tnbtu.com/comic", null);
+        } else {
+            // If we're ripping a site where we can't get the page number/title we just rip normally
+            addURLToDownload(url, getPrefix(index));
+        }
+
     }
 
     @Override

src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java

@@ -72,7 +72,7 @@ public class XhamsterRipper extends AbstractHTMLRipper {
 
     @Override
     public Document getNextPage(Document doc) throws IOException {
-        if (doc.select("a.next").first().attr("href") != "") {
+        if (!doc.select("a.next").first().attr("href").equals("")) {
             return Http.url(doc.select("a.next").first().attr("href")).get();
         } else {
             throw new IOException("No more pages");

src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java

@@ -3,7 +3,6 @@ package com.rarchives.ripme.ripper.rippers.video;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
-import java.net.URLDecoder;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 

src/main/java/com/rarchives/ripme/ui/UpdateUtils.java

@@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils;
 public class UpdateUtils {
 
     private static final Logger logger = Logger.getLogger(UpdateUtils.class);
-    private static final String DEFAULT_VERSION = "1.7.35";
+    private static final String DEFAULT_VERSION = "1.7.37";
     private static final String REPO_NAME = "ripmeapp/ripme";
     private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json";
     private static final String mainFileName = "ripme.jar";

src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java

@@ -3,7 +3,7 @@ package com.rarchives.ripme.tst.ripper.rippers;
 import java.io.IOException;
 import java.net.URL;
 
-import com.rarchives.ripme.ripper.rippers.AerisdiesRipper;;
+import com.rarchives.ripme.ripper.rippers.AerisdiesRipper;
 
 public class AerisdiesRipperTest extends RippersTest {
     public void testAlbum() throws IOException {

src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java

@@ -9,5 +9,7 @@ public class Hentai2readRipperTest extends RippersTest {
     public void testHentai2readAlbum() throws IOException {
         Hentai2readRipper ripper = new Hentai2readRipper(new URL("https://hentai2read.com/sm_school_memorial/"));
         testRipper(ripper);
+        ripper = new Hentai2readRipper(new URL("https://hentai2read.com/sm_school_memorial/1/"));
+        testRipper(ripper);
     }
 }

src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java

@@ -83,6 +83,12 @@ public class WordpressComicRipperTest extends RippersTest {
                 new URL("http://tnbtu.com/comic/01-00/"));
         testRipper(ripper);
     }
+
+    public void test_Eightmuses_download() throws IOException {
+        WordpressComicRipper ripper = new WordpressComicRipper(
+                new URL("https://8muses.download/lustomic-playkittens-josh-samuel-porn-comics-8-muses/"));
+        testRipper(ripper);
+    }
     // https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI
 //    public void test_pepsaga() throws IOException {
 //        WordpressComicRipper ripper = new WordpressComicRipper(