From 9251ee0a255908224207e8227f01a702253a3334 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 03:42:19 -0400 Subject: [PATCH 01/12] Instagram ripper no longer throws a 400 error when getting new pages --- .../com/rarchives/ripme/ripper/rippers/InstagramRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index 9e0fdac5..2b63a44f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -374,7 +374,7 @@ public class InstagramRipper extends AbstractHTMLRipper { try { // Sleep for a while to avoid a ban sleep(2500); - String vars = "{\"id\":\"" + userID + "\",\"first\":100,\"after\":\"" + nextPageID + "\"}"; + String vars = "{\"id\":\"" + userID + "\",\"first\":50,\"after\":\"" + nextPageID + "\"}"; String ig_gis = getIGGis(vars); logger.info(ig_gis); toreturn = Http.url("https://www.instagram.com/graphql/query/?query_hash=" + qHash + "&variables=" + vars From fdf3e7c68deecd08b4fc0432c026f219571ecbc9 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 04:53:34 -0400 Subject: [PATCH 02/12] tnbtu ripper now sets referrer header --- .../ripme/ripper/rippers/WordpressComicRipper.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java index dbc44585..19fd4352 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java @@ -327,8 +327,14 @@ public class WordpressComicRipper extends AbstractHTMLRipper { || getHost().contains("themonsterunderthebed.net")) { addURLToDownload(url, pageTitle + "_"); } - // If we're ripping a site where we can't get the page number/title we just rip normally - addURLToDownload(url, getPrefix(index)); + if (getHost().contains("tnbtu.com")) { + // We need to set the referrer header for tnbtu + addURLToDownload(url, getPrefix(index), "","http://www.tnbtu.com/comic", null); + } else { + // If we're ripping a site where we can't get the page number/title we just rip normally + addURLToDownload(url, getPrefix(index)); + } + } @Override From 1e3b309f4b321ff63839a3be722f870eea54da30 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 04:58:53 -0400 Subject: [PATCH 03/12] Fixed hentai2read regex --- .../com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java index db4cbd97..405d0563 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java @@ -33,7 +33,7 @@ public class Hentai2readRipper extends AbstractHTMLRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("https?://hentai2read\\.com/([a-zA-Z0-9_-]*)/\\d/?"); + Pattern p = Pattern.compile("https?://hentai2read\\.com/([a-zA-Z0-9_-]*)/(\\d)?/?"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); From 6cd6e2ee151ad4d4f4dd78caf174ac6489d572cb Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 04:59:07 -0400 Subject: [PATCH 04/12] Add another unit test --- .../ripme/tst/ripper/rippers/Hentai2readRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java index 144606fc..b5765047 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java @@ -9,5 +9,7 @@ public class Hentai2readRipperTest extends RippersTest { public void testHentai2readAlbum() throws IOException { Hentai2readRipper ripper = new Hentai2readRipper(new URL("https://hentai2read.com/sm_school_memorial/")); testRipper(ripper); + ripper = new Hentai2readRipper(new URL("https://hentai2read.com/sm_school_memorial/1/")); + testRipper(ripper); } } \ No newline at end of file From 1bc696504ad0e09b111c816db1c3d3688492988e Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 05:58:50 -0400 Subject: [PATCH 05/12] 1.7.36: Fixed Instagram ripper; Fixed hentai2read ripper test; Fixed tnbtu.com ripper --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index d00e1746..ed4efbc6 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.35 + 1.7.36 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index af817532..7c31ed69 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.35", + "latestVersion": "1.7.36", "changeList": [ + "1.7.36: Fixed Instagram ripper; Fixed hentai2read ripper test; Fixed tnbtu.com ripper", "1.7.35: Fixed instagram ripper; hentai2read ripper now properly names folders", "1.7.34: Added Blackbrickroadofoz Ripper; Fixed webtoons regex", "1.7.33: Instagram ripper no longer errors out when downloading from more than 1 page", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 48f1f941..608e0c58 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.35"; + private static final String DEFAULT_VERSION = "1.7.36"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 034225cf1ceaef838e3be367474bfb606d41d50a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Thu, 19 Apr 2018 15:14:24 +0200 Subject: [PATCH 06/12] [tumblr] avoid 'httpss' in image URLs fixes #523 replacing 'http' with 'https' in an URL already starting with 'https://...' changes its scheme/protocol to 'httpss' --- .../rarchives/ripme/ripper/rippers/TumblrRipper.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java index dc57c48f..89884854 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java @@ -44,7 +44,7 @@ public class TumblrRipper extends AlbumRipper { private static final String API_KEY = APIKEYS.get(genNum); // Select random API key from APIKEYS /** - * Gets the API key. + * Gets the API key. * Chooses between default/included keys & user specified ones (from the config file). * @return Tumblr API key */ @@ -57,7 +57,7 @@ public class TumblrRipper extends AlbumRipper { logger.info("Using user tumblr.auth api key: " + userDefinedAPIKey); return userDefinedAPIKey; } - + } public TumblrRipper(URL url) throws IOException { @@ -71,12 +71,12 @@ public class TumblrRipper extends AlbumRipper { public boolean canRip(URL url) { return url.getHost().endsWith(DOMAIN); } - + /** * Sanitizes URL. * @param url URL to be sanitized. * @return Sanitized URL - * @throws MalformedURLException + * @throws MalformedURLException */ @Override public URL sanitizeURL(URL url) throws MalformedURLException { @@ -230,7 +230,7 @@ public class TumblrRipper extends AlbumRipper { urlString = urlString.replaceAll("_\\d+\\.", "_raw."); fileURL = new URL(urlString); } else { - fileURL = new URL(photo.getJSONObject("original_size").getString("url").replaceAll("http", "https")); + fileURL = new URL(photo.getJSONObject("original_size").getString("url").replaceAll("http:", "https:")); } m = p.matcher(fileURL.toString()); if (m.matches()) { From be24df4d48a88d114da94205ec1e4359ab1064f1 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 11:35:36 -0400 Subject: [PATCH 07/12] Some minor code clean up --- .../ripme/ripper/rippers/BlackbrickroadofozRipper.java | 10 ++-------- .../rarchives/ripme/ripper/rippers/CfakeRipper.java | 2 +- .../ripme/ripper/rippers/EightmusesRipper.java | 1 - .../rarchives/ripme/ripper/rippers/EroShareRipper.java | 2 +- .../ripme/ripper/rippers/FuraffinityRipper.java | 6 ------ .../rarchives/ripme/ripper/rippers/LusciousRipper.java | 2 +- .../ripme/ripper/rippers/MyhentaicomicsRipper.java | 6 +++--- .../rarchives/ripme/ripper/rippers/SinfestRipper.java | 2 +- .../ripme/ripper/rippers/ViewcomicRipper.java | 2 -- .../ripme/ripper/rippers/WordpressComicRipper.java | 2 +- .../rarchives/ripme/ripper/rippers/XhamsterRipper.java | 2 +- .../ripme/ripper/rippers/video/XvideosRipper.java | 1 - .../ripme/tst/ripper/rippers/AerisdiesRipperTest.java | 2 +- 13 files changed, 12 insertions(+), 28 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/BlackbrickroadofozRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/BlackbrickroadofozRipper.java index 76340cf9..cb5d4b14 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/BlackbrickroadofozRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/BlackbrickroadofozRipper.java @@ -55,14 +55,8 @@ public class BlackbrickroadofozRipper extends AbstractHTMLRipper { throw new IOException("No more pages"); } String nextPage = elem.attr("href"); - // Some times this returns a empty string - // This for stops that - if (nextPage == "") { - throw new IOException("No more pages"); - } - else { - return Http.url(nextPage).get(); - } + return Http.url(nextPage).get(); + } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/CfakeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/CfakeRipper.java index fb29171b..7ccf558c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/CfakeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/CfakeRipper.java @@ -59,7 +59,7 @@ public class CfakeRipper extends AbstractHTMLRipper { String nextPage = elem.attr("href"); // Some times this returns a empty string // This for stops that - if (nextPage == "") { + if (nextPage.equals("")) { return null; } else { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java index 77ca9102..ca9c24e3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java @@ -4,7 +4,6 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java index e7019178..737b8092 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java @@ -86,7 +86,7 @@ public class EroShareRipper extends AbstractHTMLRipper { throw new IOException("No more pages"); } nextUrl = elem.attr("href"); - if (nextUrl == "") { + if (nextUrl.equals("")) { throw new IOException("No more pages"); } return Http.url("eroshae.com" + nextUrl).get(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java index 973796cf..68aa950a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java @@ -13,7 +13,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import com.rarchives.ripme.utils.Utils; -import org.jsoup.Connection.Method; import org.jsoup.Connection.Response; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -23,7 +22,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ripper.DownloadThreadPool; -import com.rarchives.ripme.utils.Base64; import com.rarchives.ripme.utils.Http; public class FuraffinityRipper extends AbstractHTMLRipper { @@ -162,10 +160,6 @@ public class FuraffinityRipper extends AbstractHTMLRipper { if (!subdirectory.equals("")) { subdirectory = File.separator + subdirectory; } - int o = url.toString().lastIndexOf('/')-1; - String test = url.toString().substring(url.toString().lastIndexOf('/',o)+1); - test = test.replace("/",""); // This is probably not the best way to do this. - test = test.replace("\\",""); // CLOSE ENOUGH! saveFileAs = new File( workingDir.getCanonicalPath() + subdirectory diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java index c98dac0c..376d1292 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java @@ -50,7 +50,7 @@ public class LusciousRipper extends AbstractHTMLRipper { // This is here for pages with mp4s instead of images String video_image = ""; video_image = page.select("div > video > source").attr("src"); - if (video_image != "") { + if (!video_image.equals("")) { urls.add(video_image); } return urls; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java index 21942a47..5b60c4f2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java @@ -73,7 +73,7 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper { if (m.matches()) { nextUrl = "http://myhentaicomics.com" + m.group(0); } - if (nextUrl == "") { + if (nextUrl.equals("")) { throw new IOException("No more pages"); } // Sleep for half a sec to avoid getting IP banned @@ -100,7 +100,7 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper { Element elem = nextAlbumPage.select("a.ui-icon-right").first(); String nextPage = elem.attr("href"); pageNumber = pageNumber + 1; - if (nextPage == "") { + if (nextPage.equals("")) { logger.info("Got " + pageNumber + " pages"); break; } @@ -220,7 +220,7 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper { // If true the page is a page of albums if (doc.toString().contains("class=\"g-item g-album\"")) { // This if checks that there is more than 1 page - if (doc.select("a.ui-icon-right").last().attr("href") != "") { + if (!doc.select("a.ui-icon-right").last().attr("href").equals("")) { // There is more than one page so we call getListOfPages List pagesToRip = getListOfPages(doc); logger.debug("Pages to rip = " + pagesToRip); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java index d30e9b63..9de3d2ae 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java @@ -57,7 +57,7 @@ public class SinfestRipper extends AbstractHTMLRipper { String nextPage = elem.parent().attr("href"); // Some times this returns a empty string // This for stops that - if (nextPage == "") { + if (nextPage.equals("")) { return null; } else { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java index ab34620c..abdb0320 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java @@ -4,14 +4,12 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java index dbc44585..e5014e92 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java @@ -247,7 +247,7 @@ public class WordpressComicRipper extends AbstractHTMLRipper { nextPage = elem.attr("href"); } - if (nextPage == "") { + if (nextPage.equals("")) { throw new IOException("No more pages"); } else { return Http.url(nextPage).get(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index 35fe56ff..b92aa9dd 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -72,7 +72,7 @@ public class XhamsterRipper extends AbstractHTMLRipper { @Override public Document getNextPage(Document doc) throws IOException { - if (doc.select("a.next").first().attr("href") != "") { + if (!doc.select("a.next").first().attr("href").equals("")) { return Http.url(doc.select("a.next").first().attr("href")).get(); } else { throw new IOException("No more pages"); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java index 4f2bac97..6dde798d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java @@ -3,7 +3,6 @@ package com.rarchives.ripme.ripper.rippers.video; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; -import java.net.URLDecoder; import java.util.regex.Matcher; import java.util.regex.Pattern; diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java index fb991ec7..503db2c3 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java @@ -3,7 +3,7 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; import java.net.URL; -import com.rarchives.ripme.ripper.rippers.AerisdiesRipper;; +import com.rarchives.ripme.ripper.rippers.AerisdiesRipper; public class AerisdiesRipperTest extends RippersTest { public void testAlbum() throws IOException { From 7d3c7f2b2007355e7f008b16885b5f2bc02998c6 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 15:07:26 -0400 Subject: [PATCH 08/12] Hentaifoundry ripper no long throws a null error when getting the next page; No longer makes unneeded requests --- .../ripper/rippers/HentaifoundryRipper.java | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java index 561c4249..f15d8b01 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java @@ -52,16 +52,16 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { Response resp = Http.url("http://www.hentai-foundry.com/").response(); cookies = resp.cookies(); resp = Http.url("http://www.hentai-foundry.com/?enterAgree=1&size=1500") - .referrer("http://www.hentai-foundry.com/") - .cookies(cookies) - .response(); + .referrer("http://www.hentai-foundry.com/") + .cookies(cookies) + .response(); // The only cookie that seems to matter in getting around the age wall is the phpsession cookie cookies.putAll(resp.cookies()); sleep(500); resp = Http.url(url) - .referrer("http://www.hentai-foundry.com/") - .cookies(cookies) - .response(); + .referrer("http://www.hentai-foundry.com/") + .cookies(cookies) + .response(); cookies.putAll(resp.cookies()); return resp.parse(); } @@ -74,12 +74,16 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { } Elements els = doc.select("li.next > a"); Element first = els.first(); - String nextURL = first.attr("href"); - nextURL = "http://www.hentai-foundry.com" + nextURL; - return Http.url(nextURL) - .referrer(url) - .cookies(cookies) - .get(); + try { + String nextURL = first.attr("href"); + nextURL = "http://www.hentai-foundry.com" + nextURL; + return Http.url(nextURL) + .referrer(url) + .cookies(cookies) + .get(); + } catch (NullPointerException e) { + throw new IOException("No more pages"); + } } @Override @@ -100,9 +104,9 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { Response resp = Http.url("http://www.hentai-foundry.com/").response(); cookies = resp.cookies(); resp = Http.url("http://www.hentai-foundry.com/?enterAgree=1&size=1500") - .referrer("http://www.hentai-foundry.com/") - .cookies(cookies) - .response(); + .referrer("http://www.hentai-foundry.com/") + .cookies(cookies) + .response(); cookies.putAll(resp.cookies()); logger.info("grabbing " + "http://www.hentai-foundry.com" + thumb.attr("href")); From 4d8f4095d257e996a3c5f94a37cab833f2adc060 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 15:10:37 -0400 Subject: [PATCH 09/12] Hentaifoundry ripper no long throws a null error when getting the next page; No longer makes unneeded requests --- .../ripme/ripper/rippers/HentaifoundryRipper.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java index f15d8b01..d09331a9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java @@ -101,13 +101,6 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { } Document imagePage; try { - Response resp = Http.url("http://www.hentai-foundry.com/").response(); - cookies = resp.cookies(); - resp = Http.url("http://www.hentai-foundry.com/?enterAgree=1&size=1500") - .referrer("http://www.hentai-foundry.com/") - .cookies(cookies) - .response(); - cookies.putAll(resp.cookies()); logger.info("grabbing " + "http://www.hentai-foundry.com" + thumb.attr("href")); imagePage = Http.url("http://www.hentai-foundry.com" + thumb.attr("href")).cookies(cookies).get(); From d6535cd2d3c5805f020c4e26e41a9d77c0aa5092 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 16:00:07 -0400 Subject: [PATCH 10/12] Added support for 8muses.download --- .../ripper/rippers/WordpressComicRipper.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java index dbc44585..6de586c8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java @@ -44,7 +44,8 @@ public class WordpressComicRipper extends AbstractHTMLRipper { "freeadultcomix.com", "thisis.delvecomic.com", "tnbtu.com", - "shipinbottle.pepsaga.com" + "shipinbottle.pepsaga.com", + "8muses.download" ); @Override @@ -135,6 +136,12 @@ public class WordpressComicRipper extends AbstractHTMLRipper { if (shipinbottleMat.matches()) { return true; } + + Pattern eight_musesPat = Pattern.compile("https?://8muses.download/([a-zA-Z0-9_-]+)/?$"); + Matcher eight_musesMat = eight_musesPat.matcher(url.toExternalForm()); + if (eight_musesMat.matches()) { + return true; + } } @@ -209,6 +216,11 @@ public class WordpressComicRipper extends AbstractHTMLRipper { return getHost() + "_" + "Ship_in_bottle"; } + Pattern eight_musesPat = Pattern.compile("https?://8muses.download/([a-zA-Z0-9_-]+)/?$"); + Matcher eight_musesMat = eight_musesPat.matcher(url.toExternalForm()); + if (eight_musesMat.matches()) { + return getHost() + "_" + eight_musesMat.group(1); + } return super.getAlbumTitle(url); } @@ -315,6 +327,12 @@ public class WordpressComicRipper extends AbstractHTMLRipper { } } + if (url.toExternalForm().contains("8muses.download")) { + for (Element elem : doc.select("div.popup-gallery > figure > a")) { + result.add(elem.attr("href")); + } + } + return result; } From 2a4bd6c67af4a9f339b2a18e212870fa4734f1ef Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 16:03:48 -0400 Subject: [PATCH 11/12] Added unit test for 8muses.download --- .../ripme/tst/ripper/rippers/WordpressComicRipperTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java index 2f7dbcf9..ec8db0b0 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java @@ -83,6 +83,12 @@ public class WordpressComicRipperTest extends RippersTest { new URL("http://tnbtu.com/comic/01-00/")); testRipper(ripper); } + + public void test_Eightmuses_download() throws IOException { + WordpressComicRipper ripper = new WordpressComicRipper( + new URL("https://8muses.download/lustomic-playkittens-josh-samuel-porn-comics-8-muses/")); + testRipper(ripper); + } // https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI // public void test_pepsaga() throws IOException { // WordpressComicRipper ripper = new WordpressComicRipper( From a52bbeb6d8ba46e6f47199783665c2335e4a6c83 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 16:23:34 -0400 Subject: [PATCH 12/12] 1.7.37: MInor code clean up; Added socks proxy support; Added support for 8muses.download; Hentaifoundry no longer errors when there are no more pages; Fix bug that causes tumblr to replace https with httpss when downloading resized images --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index ed4efbc6..c8de11b2 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.36 + 1.7.37 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 7c31ed69..a3783f9d 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.36", + "latestVersion": "1.7.37", "changeList": [ + "1.7.37: MInor code clean up; Added socks proxy support; Added support for 8muses.download; Hentaifoundry no longer errors when there are no more pages; Fix bug that causes tumblr to replace https with httpss when downloading resized images", "1.7.36: Fixed Instagram ripper; Fixed hentai2read ripper test; Fixed tnbtu.com ripper", "1.7.35: Fixed instagram ripper; hentai2read ripper now properly names folders", "1.7.34: Added Blackbrickroadofoz Ripper; Fixed webtoons regex", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 608e0c58..3e0d5e5d 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.36"; + private static final String DEFAULT_VERSION = "1.7.37"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar";