From 36f85282aada106a1c487eaaa49914023c782008 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 17 Apr 2018 03:56:34 -0400 Subject: [PATCH 01/57] Fixed instagram ripper --- .../com/rarchives/ripme/ripper/rippers/InstagramRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index 12842aa8..9e0fdac5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -328,7 +328,7 @@ public class InstagramRipper extends AbstractHTMLRipper { } private String getIGGis(String variables) { - String stringToMD5 = rhx_gis + ":" + csrftoken + ":" + variables; + String stringToMD5 = rhx_gis + ":" + variables; logger.debug("String to md5 is \"" + stringToMD5 + "\""); try { byte[] bytesOfMessage = stringToMD5.getBytes("UTF-8"); From f8ae2212c0af05470c05d95cb35378efee4a18df Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 17 Apr 2018 04:11:13 -0400 Subject: [PATCH 02/57] Fixed Hentai2Read folder naming --- .../rarchives/ripme/ripper/rippers/Hentai2readRipper.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java index 6e76bbc6..db4cbd97 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java @@ -33,7 +33,7 @@ public class Hentai2readRipper extends AbstractHTMLRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("https://hentai2read\\.com/([a-zA-Z0-9_-]*)/?"); + Pattern p = Pattern.compile("https?://hentai2read\\.com/([a-zA-Z0-9_-]*)/\\d/?"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); @@ -63,9 +63,7 @@ public class Hentai2readRipper extends AbstractHTMLRipper { @Override public String getAlbumTitle(URL url) throws MalformedURLException { try { - Document doc = getFirstPage(); - String title = doc.select("span[itemprop=title]").text(); - return getHost() + "_" + title; + return getHost() + "_" + getGID(url); } catch (Exception e) { // Fall back to default album naming convention logger.warn("Failed to get album title from " + url, e); From 1dadd26e0584c95fd74debff39294bb68de90a20 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 17 Apr 2018 04:23:05 -0400 Subject: [PATCH 03/57] 1.7.35: Fixed instagram ripper; hentai2read ripper now properly names folders --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 8ce4b5ae..d00e1746 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.34 + 1.7.35 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 3a07f1b7..af817532 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.34", + "latestVersion": "1.7.35", "changeList": [ + "1.7.35: Fixed instagram ripper; hentai2read ripper now properly names folders", "1.7.34: Added Blackbrickroadofoz Ripper; Fixed webtoons regex", "1.7.33: Instagram ripper no longer errors out when downloading from more than 1 page", "1.7.32: Instagram ripper update to use new enpoints", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 01b6c7eb..48f1f941 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.34"; + private static final String DEFAULT_VERSION = "1.7.35"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From bf056a75d0ed081123c188dd2dc9b04f24a568f0 Mon Sep 17 00:00:00 2001 From: randomcommitter <38479873+randomcommitter@users.noreply.github.com> Date: Wed, 18 Apr 2018 16:33:09 +0100 Subject: [PATCH 04/57] Added support for SOCKS v5 Server CLI only, the switch is "-s". Accepted formats: user:password@host user:password@host:port host host:port --- src/main/java/com/rarchives/ripme/App.java | 28 ++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index 67c44ab1..508401d2 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -7,7 +7,9 @@ import java.io.BufferedReader; import java.io.FileReader; import java.io.FileNotFoundException; +import java.net.Authenticator; import java.net.MalformedURLException; +import java.net.PasswordAuthentication; import java.net.URL; import java.util.Arrays; import java.util.List; @@ -94,6 +96,30 @@ public class App { if (cl.hasOption('w')) { Utils.setConfigBoolean("file.overwrite", true); } + + if (cl.hasOption('s')) { + String sservfull = cl.getOptionValue('s').trim(); + if (sservfull.lastIndexOf("@") != -1) { + int sservli = sservfull.lastIndexOf("@"); + String userpw = sservfull.substring(0, sservli); + String[] usersplit = userpw.split(":"); + String user = usersplit[0]; + String password = usersplit[1]; + Authenticator.setDefault(new Authenticator(){ + protected PasswordAuthentication getPasswordAuthentication(){ + PasswordAuthentication p = new PasswordAuthentication(user, password.toCharArray()); + return p; + } + }); + + sservfull = sservfull.substring(sservli + 1); + } + String[] servsplit = sservfull.split(":"); + if (servsplit.length == 2) { + System.setProperty("socksProxyPort", servsplit[1]); + } + System.setProperty("socksProxyHost", servsplit[0]); + } if (cl.hasOption('t')) { Utils.setConfigInteger("threads.size", Integer.parseInt(cl.getOptionValue('t'))); @@ -195,6 +221,7 @@ public class App { String url = cl.getOptionValue('u').trim(); ripURL(url, cl.hasOption("n")); } + } /** @@ -242,6 +269,7 @@ public class App { opts.addOption("n", "no-prop-file", false, "Do not create properties file."); opts.addOption("f", "urls-file", true, "Rip URLs from a file."); opts.addOption("v", "version", false, "Show current version"); + opts.addOption("s", "socks-server", true, "Use socks server ([user:password]@host[:port]"); return opts; } From 9251ee0a255908224207e8227f01a702253a3334 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 03:42:19 -0400 Subject: [PATCH 05/57] Instagram ripper no longer throws a 400 error when getting new pages --- .../com/rarchives/ripme/ripper/rippers/InstagramRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index 9e0fdac5..2b63a44f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -374,7 +374,7 @@ public class InstagramRipper extends AbstractHTMLRipper { try { // Sleep for a while to avoid a ban sleep(2500); - String vars = "{\"id\":\"" + userID + "\",\"first\":100,\"after\":\"" + nextPageID + "\"}"; + String vars = "{\"id\":\"" + userID + "\",\"first\":50,\"after\":\"" + nextPageID + "\"}"; String ig_gis = getIGGis(vars); logger.info(ig_gis); toreturn = Http.url("https://www.instagram.com/graphql/query/?query_hash=" + qHash + "&variables=" + vars From fdf3e7c68deecd08b4fc0432c026f219571ecbc9 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 04:53:34 -0400 Subject: [PATCH 06/57] tnbtu ripper now sets referrer header --- .../ripme/ripper/rippers/WordpressComicRipper.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java index dbc44585..19fd4352 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java @@ -327,8 +327,14 @@ public class WordpressComicRipper extends AbstractHTMLRipper { || getHost().contains("themonsterunderthebed.net")) { addURLToDownload(url, pageTitle + "_"); } - // If we're ripping a site where we can't get the page number/title we just rip normally - addURLToDownload(url, getPrefix(index)); + if (getHost().contains("tnbtu.com")) { + // We need to set the referrer header for tnbtu + addURLToDownload(url, getPrefix(index), "","http://www.tnbtu.com/comic", null); + } else { + // If we're ripping a site where we can't get the page number/title we just rip normally + addURLToDownload(url, getPrefix(index)); + } + } @Override From 1e3b309f4b321ff63839a3be722f870eea54da30 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 04:58:53 -0400 Subject: [PATCH 07/57] Fixed hentai2read regex --- .../com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java index db4cbd97..405d0563 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java @@ -33,7 +33,7 @@ public class Hentai2readRipper extends AbstractHTMLRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("https?://hentai2read\\.com/([a-zA-Z0-9_-]*)/\\d/?"); + Pattern p = Pattern.compile("https?://hentai2read\\.com/([a-zA-Z0-9_-]*)/(\\d)?/?"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); From 6cd6e2ee151ad4d4f4dd78caf174ac6489d572cb Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 04:59:07 -0400 Subject: [PATCH 08/57] Add another unit test --- .../ripme/tst/ripper/rippers/Hentai2readRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java index 144606fc..b5765047 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java @@ -9,5 +9,7 @@ public class Hentai2readRipperTest extends RippersTest { public void testHentai2readAlbum() throws IOException { Hentai2readRipper ripper = new Hentai2readRipper(new URL("https://hentai2read.com/sm_school_memorial/")); testRipper(ripper); + ripper = new Hentai2readRipper(new URL("https://hentai2read.com/sm_school_memorial/1/")); + testRipper(ripper); } } \ No newline at end of file From 1bc696504ad0e09b111c816db1c3d3688492988e Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 05:58:50 -0400 Subject: [PATCH 09/57] 1.7.36: Fixed Instagram ripper; Fixed hentai2read ripper test; Fixed tnbtu.com ripper --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index d00e1746..ed4efbc6 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.35 + 1.7.36 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index af817532..7c31ed69 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.35", + "latestVersion": "1.7.36", "changeList": [ + "1.7.36: Fixed Instagram ripper; Fixed hentai2read ripper test; Fixed tnbtu.com ripper", "1.7.35: Fixed instagram ripper; hentai2read ripper now properly names folders", "1.7.34: Added Blackbrickroadofoz Ripper; Fixed webtoons regex", "1.7.33: Instagram ripper no longer errors out when downloading from more than 1 page", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 48f1f941..608e0c58 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.35"; + private static final String DEFAULT_VERSION = "1.7.36"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 034225cf1ceaef838e3be367474bfb606d41d50a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Thu, 19 Apr 2018 15:14:24 +0200 Subject: [PATCH 10/57] [tumblr] avoid 'httpss' in image URLs fixes #523 replacing 'http' with 'https' in an URL already starting with 'https://...' changes its scheme/protocol to 'httpss' --- .../rarchives/ripme/ripper/rippers/TumblrRipper.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java index dc57c48f..89884854 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java @@ -44,7 +44,7 @@ public class TumblrRipper extends AlbumRipper { private static final String API_KEY = APIKEYS.get(genNum); // Select random API key from APIKEYS /** - * Gets the API key. + * Gets the API key. * Chooses between default/included keys & user specified ones (from the config file). * @return Tumblr API key */ @@ -57,7 +57,7 @@ public class TumblrRipper extends AlbumRipper { logger.info("Using user tumblr.auth api key: " + userDefinedAPIKey); return userDefinedAPIKey; } - + } public TumblrRipper(URL url) throws IOException { @@ -71,12 +71,12 @@ public class TumblrRipper extends AlbumRipper { public boolean canRip(URL url) { return url.getHost().endsWith(DOMAIN); } - + /** * Sanitizes URL. * @param url URL to be sanitized. * @return Sanitized URL - * @throws MalformedURLException + * @throws MalformedURLException */ @Override public URL sanitizeURL(URL url) throws MalformedURLException { @@ -230,7 +230,7 @@ public class TumblrRipper extends AlbumRipper { urlString = urlString.replaceAll("_\\d+\\.", "_raw."); fileURL = new URL(urlString); } else { - fileURL = new URL(photo.getJSONObject("original_size").getString("url").replaceAll("http", "https")); + fileURL = new URL(photo.getJSONObject("original_size").getString("url").replaceAll("http:", "https:")); } m = p.matcher(fileURL.toString()); if (m.matches()) { From be24df4d48a88d114da94205ec1e4359ab1064f1 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 11:35:36 -0400 Subject: [PATCH 11/57] Some minor code clean up --- .../ripme/ripper/rippers/BlackbrickroadofozRipper.java | 10 ++-------- .../rarchives/ripme/ripper/rippers/CfakeRipper.java | 2 +- .../ripme/ripper/rippers/EightmusesRipper.java | 1 - .../rarchives/ripme/ripper/rippers/EroShareRipper.java | 2 +- .../ripme/ripper/rippers/FuraffinityRipper.java | 6 ------ .../rarchives/ripme/ripper/rippers/LusciousRipper.java | 2 +- .../ripme/ripper/rippers/MyhentaicomicsRipper.java | 6 +++--- .../rarchives/ripme/ripper/rippers/SinfestRipper.java | 2 +- .../ripme/ripper/rippers/ViewcomicRipper.java | 2 -- .../ripme/ripper/rippers/WordpressComicRipper.java | 2 +- .../rarchives/ripme/ripper/rippers/XhamsterRipper.java | 2 +- .../ripme/ripper/rippers/video/XvideosRipper.java | 1 - .../ripme/tst/ripper/rippers/AerisdiesRipperTest.java | 2 +- 13 files changed, 12 insertions(+), 28 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/BlackbrickroadofozRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/BlackbrickroadofozRipper.java index 76340cf9..cb5d4b14 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/BlackbrickroadofozRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/BlackbrickroadofozRipper.java @@ -55,14 +55,8 @@ public class BlackbrickroadofozRipper extends AbstractHTMLRipper { throw new IOException("No more pages"); } String nextPage = elem.attr("href"); - // Some times this returns a empty string - // This for stops that - if (nextPage == "") { - throw new IOException("No more pages"); - } - else { - return Http.url(nextPage).get(); - } + return Http.url(nextPage).get(); + } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/CfakeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/CfakeRipper.java index fb29171b..7ccf558c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/CfakeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/CfakeRipper.java @@ -59,7 +59,7 @@ public class CfakeRipper extends AbstractHTMLRipper { String nextPage = elem.attr("href"); // Some times this returns a empty string // This for stops that - if (nextPage == "") { + if (nextPage.equals("")) { return null; } else { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java index 77ca9102..ca9c24e3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java @@ -4,7 +4,6 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java index e7019178..737b8092 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java @@ -86,7 +86,7 @@ public class EroShareRipper extends AbstractHTMLRipper { throw new IOException("No more pages"); } nextUrl = elem.attr("href"); - if (nextUrl == "") { + if (nextUrl.equals("")) { throw new IOException("No more pages"); } return Http.url("eroshae.com" + nextUrl).get(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java index 973796cf..68aa950a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java @@ -13,7 +13,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import com.rarchives.ripme.utils.Utils; -import org.jsoup.Connection.Method; import org.jsoup.Connection.Response; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -23,7 +22,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ripper.DownloadThreadPool; -import com.rarchives.ripme.utils.Base64; import com.rarchives.ripme.utils.Http; public class FuraffinityRipper extends AbstractHTMLRipper { @@ -162,10 +160,6 @@ public class FuraffinityRipper extends AbstractHTMLRipper { if (!subdirectory.equals("")) { subdirectory = File.separator + subdirectory; } - int o = url.toString().lastIndexOf('/')-1; - String test = url.toString().substring(url.toString().lastIndexOf('/',o)+1); - test = test.replace("/",""); // This is probably not the best way to do this. - test = test.replace("\\",""); // CLOSE ENOUGH! saveFileAs = new File( workingDir.getCanonicalPath() + subdirectory diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java index c98dac0c..376d1292 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java @@ -50,7 +50,7 @@ public class LusciousRipper extends AbstractHTMLRipper { // This is here for pages with mp4s instead of images String video_image = ""; video_image = page.select("div > video > source").attr("src"); - if (video_image != "") { + if (!video_image.equals("")) { urls.add(video_image); } return urls; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java index 21942a47..5b60c4f2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java @@ -73,7 +73,7 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper { if (m.matches()) { nextUrl = "http://myhentaicomics.com" + m.group(0); } - if (nextUrl == "") { + if (nextUrl.equals("")) { throw new IOException("No more pages"); } // Sleep for half a sec to avoid getting IP banned @@ -100,7 +100,7 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper { Element elem = nextAlbumPage.select("a.ui-icon-right").first(); String nextPage = elem.attr("href"); pageNumber = pageNumber + 1; - if (nextPage == "") { + if (nextPage.equals("")) { logger.info("Got " + pageNumber + " pages"); break; } @@ -220,7 +220,7 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper { // If true the page is a page of albums if (doc.toString().contains("class=\"g-item g-album\"")) { // This if checks that there is more than 1 page - if (doc.select("a.ui-icon-right").last().attr("href") != "") { + if (!doc.select("a.ui-icon-right").last().attr("href").equals("")) { // There is more than one page so we call getListOfPages List pagesToRip = getListOfPages(doc); logger.debug("Pages to rip = " + pagesToRip); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java index d30e9b63..9de3d2ae 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java @@ -57,7 +57,7 @@ public class SinfestRipper extends AbstractHTMLRipper { String nextPage = elem.parent().attr("href"); // Some times this returns a empty string // This for stops that - if (nextPage == "") { + if (nextPage.equals("")) { return null; } else { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java index ab34620c..abdb0320 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java @@ -4,14 +4,12 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java index dbc44585..e5014e92 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java @@ -247,7 +247,7 @@ public class WordpressComicRipper extends AbstractHTMLRipper { nextPage = elem.attr("href"); } - if (nextPage == "") { + if (nextPage.equals("")) { throw new IOException("No more pages"); } else { return Http.url(nextPage).get(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index 35fe56ff..b92aa9dd 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -72,7 +72,7 @@ public class XhamsterRipper extends AbstractHTMLRipper { @Override public Document getNextPage(Document doc) throws IOException { - if (doc.select("a.next").first().attr("href") != "") { + if (!doc.select("a.next").first().attr("href").equals("")) { return Http.url(doc.select("a.next").first().attr("href")).get(); } else { throw new IOException("No more pages"); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java index 4f2bac97..6dde798d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java @@ -3,7 +3,6 @@ package com.rarchives.ripme.ripper.rippers.video; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; -import java.net.URLDecoder; import java.util.regex.Matcher; import java.util.regex.Pattern; diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java index fb991ec7..503db2c3 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java @@ -3,7 +3,7 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; import java.net.URL; -import com.rarchives.ripme.ripper.rippers.AerisdiesRipper;; +import com.rarchives.ripme.ripper.rippers.AerisdiesRipper; public class AerisdiesRipperTest extends RippersTest { public void testAlbum() throws IOException { From 7d3c7f2b2007355e7f008b16885b5f2bc02998c6 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 15:07:26 -0400 Subject: [PATCH 12/57] Hentaifoundry ripper no long throws a null error when getting the next page; No longer makes unneeded requests --- .../ripper/rippers/HentaifoundryRipper.java | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java index 561c4249..f15d8b01 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java @@ -52,16 +52,16 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { Response resp = Http.url("http://www.hentai-foundry.com/").response(); cookies = resp.cookies(); resp = Http.url("http://www.hentai-foundry.com/?enterAgree=1&size=1500") - .referrer("http://www.hentai-foundry.com/") - .cookies(cookies) - .response(); + .referrer("http://www.hentai-foundry.com/") + .cookies(cookies) + .response(); // The only cookie that seems to matter in getting around the age wall is the phpsession cookie cookies.putAll(resp.cookies()); sleep(500); resp = Http.url(url) - .referrer("http://www.hentai-foundry.com/") - .cookies(cookies) - .response(); + .referrer("http://www.hentai-foundry.com/") + .cookies(cookies) + .response(); cookies.putAll(resp.cookies()); return resp.parse(); } @@ -74,12 +74,16 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { } Elements els = doc.select("li.next > a"); Element first = els.first(); - String nextURL = first.attr("href"); - nextURL = "http://www.hentai-foundry.com" + nextURL; - return Http.url(nextURL) - .referrer(url) - .cookies(cookies) - .get(); + try { + String nextURL = first.attr("href"); + nextURL = "http://www.hentai-foundry.com" + nextURL; + return Http.url(nextURL) + .referrer(url) + .cookies(cookies) + .get(); + } catch (NullPointerException e) { + throw new IOException("No more pages"); + } } @Override @@ -100,9 +104,9 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { Response resp = Http.url("http://www.hentai-foundry.com/").response(); cookies = resp.cookies(); resp = Http.url("http://www.hentai-foundry.com/?enterAgree=1&size=1500") - .referrer("http://www.hentai-foundry.com/") - .cookies(cookies) - .response(); + .referrer("http://www.hentai-foundry.com/") + .cookies(cookies) + .response(); cookies.putAll(resp.cookies()); logger.info("grabbing " + "http://www.hentai-foundry.com" + thumb.attr("href")); From 4d8f4095d257e996a3c5f94a37cab833f2adc060 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 15:10:37 -0400 Subject: [PATCH 13/57] Hentaifoundry ripper no long throws a null error when getting the next page; No longer makes unneeded requests --- .../ripme/ripper/rippers/HentaifoundryRipper.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java index f15d8b01..d09331a9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java @@ -101,13 +101,6 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { } Document imagePage; try { - Response resp = Http.url("http://www.hentai-foundry.com/").response(); - cookies = resp.cookies(); - resp = Http.url("http://www.hentai-foundry.com/?enterAgree=1&size=1500") - .referrer("http://www.hentai-foundry.com/") - .cookies(cookies) - .response(); - cookies.putAll(resp.cookies()); logger.info("grabbing " + "http://www.hentai-foundry.com" + thumb.attr("href")); imagePage = Http.url("http://www.hentai-foundry.com" + thumb.attr("href")).cookies(cookies).get(); From d6535cd2d3c5805f020c4e26e41a9d77c0aa5092 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 16:00:07 -0400 Subject: [PATCH 14/57] Added support for 8muses.download --- .../ripper/rippers/WordpressComicRipper.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java index dbc44585..6de586c8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java @@ -44,7 +44,8 @@ public class WordpressComicRipper extends AbstractHTMLRipper { "freeadultcomix.com", "thisis.delvecomic.com", "tnbtu.com", - "shipinbottle.pepsaga.com" + "shipinbottle.pepsaga.com", + "8muses.download" ); @Override @@ -135,6 +136,12 @@ public class WordpressComicRipper extends AbstractHTMLRipper { if (shipinbottleMat.matches()) { return true; } + + Pattern eight_musesPat = Pattern.compile("https?://8muses.download/([a-zA-Z0-9_-]+)/?$"); + Matcher eight_musesMat = eight_musesPat.matcher(url.toExternalForm()); + if (eight_musesMat.matches()) { + return true; + } } @@ -209,6 +216,11 @@ public class WordpressComicRipper extends AbstractHTMLRipper { return getHost() + "_" + "Ship_in_bottle"; } + Pattern eight_musesPat = Pattern.compile("https?://8muses.download/([a-zA-Z0-9_-]+)/?$"); + Matcher eight_musesMat = eight_musesPat.matcher(url.toExternalForm()); + if (eight_musesMat.matches()) { + return getHost() + "_" + eight_musesMat.group(1); + } return super.getAlbumTitle(url); } @@ -315,6 +327,12 @@ public class WordpressComicRipper extends AbstractHTMLRipper { } } + if (url.toExternalForm().contains("8muses.download")) { + for (Element elem : doc.select("div.popup-gallery > figure > a")) { + result.add(elem.attr("href")); + } + } + return result; } From 2a4bd6c67af4a9f339b2a18e212870fa4734f1ef Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 16:03:48 -0400 Subject: [PATCH 15/57] Added unit test for 8muses.download --- .../ripme/tst/ripper/rippers/WordpressComicRipperTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java index 2f7dbcf9..ec8db0b0 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java @@ -83,6 +83,12 @@ public class WordpressComicRipperTest extends RippersTest { new URL("http://tnbtu.com/comic/01-00/")); testRipper(ripper); } + + public void test_Eightmuses_download() throws IOException { + WordpressComicRipper ripper = new WordpressComicRipper( + new URL("https://8muses.download/lustomic-playkittens-josh-samuel-porn-comics-8-muses/")); + testRipper(ripper); + } // https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI // public void test_pepsaga() throws IOException { // WordpressComicRipper ripper = new WordpressComicRipper( From a52bbeb6d8ba46e6f47199783665c2335e4a6c83 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 16:23:34 -0400 Subject: [PATCH 16/57] 1.7.37: MInor code clean up; Added socks proxy support; Added support for 8muses.download; Hentaifoundry no longer errors when there are no more pages; Fix bug that causes tumblr to replace https with httpss when downloading resized images --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index ed4efbc6..c8de11b2 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.36 + 1.7.37 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 7c31ed69..a3783f9d 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.36", + "latestVersion": "1.7.37", "changeList": [ + "1.7.37: MInor code clean up; Added socks proxy support; Added support for 8muses.download; Hentaifoundry no longer errors when there are no more pages; Fix bug that causes tumblr to replace https with httpss when downloading resized images", "1.7.36: Fixed Instagram ripper; Fixed hentai2read ripper test; Fixed tnbtu.com ripper", "1.7.35: Fixed instagram ripper; hentai2read ripper now properly names folders", "1.7.34: Added Blackbrickroadofoz Ripper; Fixed webtoons regex", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 608e0c58..3e0d5e5d 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.36"; + private static final String DEFAULT_VERSION = "1.7.37"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From b73e943b9cf39e89b5bcf1fe08723bc9e77081d5 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 18:52:36 -0400 Subject: [PATCH 17/57] Added HitomiRipper --- .../ripme/ripper/rippers/HitomiRipper.java | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java new file mode 100644 index 00000000..3e92cc61 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java @@ -0,0 +1,73 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class HitomiRipper extends AbstractHTMLRipper { + + String galleryId = ""; + + public HitomiRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "hitomi"; + } + + @Override + public String getDomain() { + return "hitomi.la"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https://hitomi.la/galleries/([\\d]+).html"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + galleryId = m.group(1); + return m.group(1); + } + throw new MalformedURLException("Expected hitomi URL format: " + + "https://hitomi.la/galleries/ID.html - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { + // if we go to /GALLERYID.js we get a nice json array of all images in the gallery + return Http.url(new URL(url.toExternalForm().replaceAll(".html", ".js"))).ignoreContentType().get(); + } + + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + String json = doc.text().replaceAll("var galleryinfo =", ""); + logger.info(json); + JSONArray json_data = new JSONArray(json); + for (int i = 0; i < json_data.length(); i++) { + result.add("https://0a.hitomi.la/galleries/" + galleryId + "/" + json_data.getJSONObject(i).getString("name")); + } + + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} From a7445ec4ac3d4ccab090fd829706f8ad2ff68fc0 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 18:54:54 -0400 Subject: [PATCH 18/57] Added Hitomi Ripper unit test --- .../ripme/tst/ripper/rippers/HitomiRipperTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java new file mode 100644 index 00000000..78627a83 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.HitomiRipper; + +public class HitomiRipperTest extends RippersTest { + public void testRip() throws IOException { + HitomiRipper ripper = new HitomiRipper(new URL("https://hitomi.la/galleries/975973.html")); + testRipper(ripper); + } +} From f8a8fdb9dc74c54064bd021aa774166fff0a7f9e Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 18:58:47 -0400 Subject: [PATCH 19/57] Extended HitomiRipper unit test --- .../com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java index 78627a83..13c2798d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java @@ -9,5 +9,6 @@ public class HitomiRipperTest extends RippersTest { public void testRip() throws IOException { HitomiRipper ripper = new HitomiRipper(new URL("https://hitomi.la/galleries/975973.html")); testRipper(ripper); + assertTrue(ripper.getGID(new URL("https://hitomi.la/galleries/975973.html")).equals("975973")); } } From 7b71a7d3f5f80d35fd2c2bb0f674e257519c9a50 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 19:16:15 -0400 Subject: [PATCH 20/57] Extended some unit tests to include getGid --- .../tst/ripper/rippers/EightmusesRipperTest.java | 5 +++++ .../tst/ripper/rippers/ModelmayhemRipperTest.java | 5 +++++ .../tst/ripper/rippers/MyhentaicomicsRipperTest.java | 11 +++++++++++ 3 files changed, 21 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java index 4a6c3539..e29c9ece 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java @@ -17,4 +17,9 @@ public class EightmusesRipperTest extends RippersTest { ripper = new EightmusesRipper(new URL("https://www.8muses.com/comix/album/Blacknwhitecomics_com-Comix/BlacknWhiteComics/The-Mayor")); testRipper(ripper); } + + public void testGID() throws IOException { + EightmusesRipper ripper = new EightmusesRipper(new URL("https://www.8muses.com/comix/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore")); + assertEquals("Affect3D-Comics", ripper.getGID(new URL("https://www.8muses.com/comics/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore"))); + } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelmayhemRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelmayhemRipperTest.java index 621d77c3..d2a9581b 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelmayhemRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelmayhemRipperTest.java @@ -10,4 +10,9 @@ public class ModelmayhemRipperTest extends RippersTest { ModelmayhemRipper ripper = new ModelmayhemRipper(new URL("https://www.modelmayhem.com/portfolio/520206/viewall")); testRipper(ripper); } + + public void testGetGID() throws IOException { + ModelmayhemRipper ripper = new ModelmayhemRipper(new URL("https://www.modelmayhem.com/portfolio/520206/viewall")); + assertEquals("520206", ripper.getGID(new URL("https://www.modelmayhem.com/portfolio/520206/viewall"))); + } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java index 6714195d..b4d01032 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java @@ -10,4 +10,15 @@ public class MyhentaicomicsRipperTest extends RippersTest { MyhentaicomicsRipper ripper = new MyhentaicomicsRipper(new URL("http://myhentaicomics.com/index.php/Nienna-Lost-Tales")); testRipper(ripper); } + + public void testGetGID() throws IOException { + URL url = new URL("http://myhentaicomics.com/index.php/Nienna-Lost-Tales"); + MyhentaicomicsRipper ripper = new MyhentaicomicsRipper(url); + // Test a comic + assertEquals("Nienna-Lost-Tales", ripper.getGID(url)); + // Test a search + assertEquals("test", ripper.getGID(new URL("http://myhentaicomics.com/index.php/search?q=test"))); + // Test a tag + assertEquals("2409", ripper.getGID(new URL("http://myhentaicomics.com/index.php/tag/2409/"))); + } } \ No newline at end of file From 34ee3a6f60d3d274cc19dedeba4ceaa3495b4bc8 Mon Sep 17 00:00:00 2001 From: Random Committer Date: Fri, 20 Apr 2018 07:51:40 -0700 Subject: [PATCH 21/57] Removed Socks Proxy setup from App.java, added new class Proxy under com.rarchives.ripme.utils which allows to set both HTTP Proxy and SOCKS Proxy --- src/main/java/com/rarchives/ripme/App.java | 40 +++----- .../java/com/rarchives/ripme/utils/Proxy.java | 99 +++++++++++++++++++ 2 files changed, 112 insertions(+), 27 deletions(-) create mode 100644 src/main/java/com/rarchives/ripme/utils/Proxy.java diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index 508401d2..f9357e62 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -7,9 +7,7 @@ import java.io.BufferedReader; import java.io.FileReader; import java.io.FileNotFoundException; -import java.net.Authenticator; import java.net.MalformedURLException; -import java.net.PasswordAuthentication; import java.net.URL; import java.util.Arrays; import java.util.List; @@ -29,6 +27,7 @@ import com.rarchives.ripme.ui.History; import com.rarchives.ripme.ui.HistoryEntry; import com.rarchives.ripme.ui.MainWindow; import com.rarchives.ripme.ui.UpdateUtils; +import com.rarchives.ripme.utils.Proxy; import com.rarchives.ripme.utils.RipUtils; import com.rarchives.ripme.utils.Utils; @@ -69,7 +68,7 @@ public class App { /** * Creates an abstract ripper and instructs it to rip. * @param url URL to be ripped - * @throws Exception + * @throws Exception */ private static void rip(URL url) throws Exception { AbstractRipper ripper = AbstractRipper.getRipper(url); @@ -96,29 +95,15 @@ public class App { if (cl.hasOption('w')) { Utils.setConfigBoolean("file.overwrite", true); } - + if (cl.hasOption('s')) { String sservfull = cl.getOptionValue('s').trim(); - if (sservfull.lastIndexOf("@") != -1) { - int sservli = sservfull.lastIndexOf("@"); - String userpw = sservfull.substring(0, sservli); - String[] usersplit = userpw.split(":"); - String user = usersplit[0]; - String password = usersplit[1]; - Authenticator.setDefault(new Authenticator(){ - protected PasswordAuthentication getPasswordAuthentication(){ - PasswordAuthentication p = new PasswordAuthentication(user, password.toCharArray()); - return p; - } - }); - - sservfull = sservfull.substring(sservli + 1); - } - String[] servsplit = sservfull.split(":"); - if (servsplit.length == 2) { - System.setProperty("socksProxyPort", servsplit[1]); - } - System.setProperty("socksProxyHost", servsplit[0]); + Proxy.setSocks(sservfull); + } + + if (cl.hasOption('p')) { + String proxyserverfull = cl.getOptionValue('p').trim(); + Proxy.setHTTPProxy(proxyserverfull); } if (cl.hasOption('t')) { @@ -221,7 +206,7 @@ public class App { String url = cl.getOptionValue('u').trim(); ripURL(url, cl.hasOption("n")); } - + } /** @@ -269,7 +254,8 @@ public class App { opts.addOption("n", "no-prop-file", false, "Do not create properties file."); opts.addOption("f", "urls-file", true, "Rip URLs from a file."); opts.addOption("v", "version", false, "Show current version"); - opts.addOption("s", "socks-server", true, "Use socks server ([user:password]@host[:port]"); + opts.addOption("s", "socks-server", true, "Use socks server ([user:password]@host[:port])"); + opts.addOption("p", "proxy-server", true, "Use HTTP Proxy server ([user:password]@host[:port])"); return opts; } @@ -288,7 +274,7 @@ public class App { return null; } } - + /** * Loads history from history file into memory. */ diff --git a/src/main/java/com/rarchives/ripme/utils/Proxy.java b/src/main/java/com/rarchives/ripme/utils/Proxy.java new file mode 100644 index 00000000..2f657b88 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/utils/Proxy.java @@ -0,0 +1,99 @@ +package com.rarchives.ripme.utils; + +import java.net.Authenticator; +import java.net.PasswordAuthentication; +import java.util.Map; +import java.util.HashMap; + +/** + * Proxy/Socks setter + */ +public class Proxy { + private Proxy() { + } + + /** + * Parse the proxy server settings from string, using the format + * [user:password]@host[:port]. + * + * @param fullproxy the string to parse + * @return HashMap containing proxy server, port, user and password + */ + private static Map parseServer(String fullproxy) { + Map proxy = new HashMap(); + + if (fullproxy.lastIndexOf("@") != -1) { + int sservli = fullproxy.lastIndexOf("@"); + String userpw = fullproxy.substring(0, sservli); + String[] usersplit = userpw.split(":"); + proxy.put("user", usersplit[0]); + proxy.put("password", usersplit[1]); + fullproxy = fullproxy.substring(sservli + 1); + } + String[] servsplit = fullproxy.split(":"); + if (servsplit.length == 2) { + proxy.put("port", servsplit[1]); + } + proxy.put("server", servsplit[0]); + return proxy; + } + + /** + * Set a HTTP Proxy. + * WARNING: Authenticated HTTP Proxy won't work from jdk1.8.111 unless + * passing the flag -Djdk.http.auth.tunneling.disabledSchemes="" to java + * see https://stackoverflow.com/q/41505219 + * + * @param fullproxy the proxy, using format [user:password]@host[:port] + */ + public static void setHTTPProxy(String fullproxy) { + Map proxyServer = parseServer(fullproxy); + + if (proxyServer.get("user") != null && proxyServer.get("password") != null) { + Authenticator.setDefault(new Authenticator(){ + protected PasswordAuthentication getPasswordAuthentication(){ + PasswordAuthentication p = new PasswordAuthentication(proxyServer.get("user"), proxyServer.get("password").toCharArray()); + return p; + } + }); + System.setProperty("http.proxyUser", proxyServer.get("user")); + System.setProperty("http.proxyPassword", proxyServer.get("password")); + System.setProperty("https.proxyUser", proxyServer.get("user")); + System.setProperty("https.proxyPassword", proxyServer.get("password")); + } + + if (proxyServer.get("port") != null) { + System.setProperty("http.proxyPort", proxyServer.get("port")); + System.setProperty("https.proxyPort", proxyServer.get("port")); + } + + System.setProperty("http.proxyHost", proxyServer.get("server")); + System.setProperty("https.proxyHost", proxyServer.get("server")); + } + + /** + * Set a Socks Proxy Server (globally). + * + * @param fullsocks the socks server, using format [user:password]@host[:port] + */ + public static void setSocks(String fullsocks) { + + Map socksServer = parseServer(fullsocks); + if (socksServer.get("user") != null && socksServer.get("password") != null) { + Authenticator.setDefault(new Authenticator(){ + protected PasswordAuthentication getPasswordAuthentication(){ + PasswordAuthentication p = new PasswordAuthentication(socksServer.get("user"), socksServer.get("password").toCharArray()); + return p; + } + }); + System.setProperty("java.net.socks.username", socksServer.get("user")); + System.setProperty("java.net.socks.password", socksServer.get("password")); + } + if (socksServer.get("port") != null) { + System.setProperty("socksProxyPort", socksServer.get("port")); + } + + System.setProperty("socksProxyHost", socksServer.get("server")); + } + +} \ No newline at end of file From f493b1ed80384349511ba31f7099e4576b0a92d3 Mon Sep 17 00:00:00 2001 From: Random Committer Date: Fri, 20 Apr 2018 08:03:45 -0700 Subject: [PATCH 22/57] Added test for class Proxy --- .../com/rarchives/ripme/tst/proxyTest.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/proxyTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/proxyTest.java b/src/test/java/com/rarchives/ripme/tst/proxyTest.java new file mode 100644 index 00000000..32b7958c --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/proxyTest.java @@ -0,0 +1,42 @@ +package com.rarchives.ripme.tst; + +import java.io.IOException; +import java.net.URL; +import com.rarchives.ripme.utils.Proxy; +import com.rarchives.ripme.utils.Utils; +import junit.framework.TestCase; +import com.rarchives.ripme.utils.Http; + + +public class proxyTest extends TestCase { + // This test will only run on machines where the user has added a entry for socks.server + public void testSocksProxy() throws IOException { + URL url = new URL("https://icanhazip.com"); + String proxyConfig = Utils.getConfigString("socks.server", ""); + if (!proxyConfig.equals("")) { + String ip1 = Http.url(url).ignoreContentType().get().text(); + Proxy.setSocks(Utils.getConfigString("socks.server", "")); + String ip2 = Http.url(url).ignoreContentType().get().text(); + assertFalse(ip1.equals(ip2)); + } else { + System.out.println("Skipping testSocksProxy"); + assert(true); + } + } + + // This test will only run on machines where the user has added a entry for proxy.server + public void testHTTPProxy() throws IOException { + URL url = new URL("https://icanhazip.com"); + String proxyConfig = Utils.getConfigString("proxy.server", ""); + if (!proxyConfig.equals("")) { + String ip1 = Http.url(url).ignoreContentType().get().text(); + Proxy.setHTTPProxy(Utils.getConfigString("proxy.server", "")); + String ip2 = Http.url(url).ignoreContentType().get().text(); + assertFalse(ip1.equals(ip2)); + } else { + System.out.println("Skipping testHTTPProxy"); + assert(true); + } + } + +} From 95d1e0a12046ff9ca1673a8df5ad5508630e2270 Mon Sep 17 00:00:00 2001 From: Random Committer Date: Fri, 20 Apr 2018 09:15:39 -0700 Subject: [PATCH 23/57] Changed socks and proxy config values to proxy.socks and proxy.http; Checking config on start for these values and set proxy/socks accordingly. --- src/main/java/com/rarchives/ripme/App.java | 6 ++++++ src/main/java/com/rarchives/ripme/utils/Proxy.java | 2 +- src/test/java/com/rarchives/ripme/tst/proxyTest.java | 12 ++++++------ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index f9357e62..6f650d56 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -48,6 +48,12 @@ public class App { System.exit(0); } + if (Utils.getConfigString("proxy.http", null) != null) { + Proxy.setHTTPProxy(Utils.getConfigString("proxy.http", null)); + } else if (Utils.getConfigString("proxy.socks", null) != null) { + Proxy.setSocks(Utils.getConfigString("proxy.socks", null)); + } + if (GraphicsEnvironment.isHeadless() || args.length > 0) { handleArguments(args); } else { diff --git a/src/main/java/com/rarchives/ripme/utils/Proxy.java b/src/main/java/com/rarchives/ripme/utils/Proxy.java index 2f657b88..be3c3b7e 100644 --- a/src/main/java/com/rarchives/ripme/utils/Proxy.java +++ b/src/main/java/com/rarchives/ripme/utils/Proxy.java @@ -96,4 +96,4 @@ public class Proxy { System.setProperty("socksProxyHost", socksServer.get("server")); } -} \ No newline at end of file +} diff --git a/src/test/java/com/rarchives/ripme/tst/proxyTest.java b/src/test/java/com/rarchives/ripme/tst/proxyTest.java index 32b7958c..1d702c6b 100644 --- a/src/test/java/com/rarchives/ripme/tst/proxyTest.java +++ b/src/test/java/com/rarchives/ripme/tst/proxyTest.java @@ -9,13 +9,13 @@ import com.rarchives.ripme.utils.Http; public class proxyTest extends TestCase { - // This test will only run on machines where the user has added a entry for socks.server + // This test will only run on machines where the user has added a entry for proxy.socks public void testSocksProxy() throws IOException { URL url = new URL("https://icanhazip.com"); - String proxyConfig = Utils.getConfigString("socks.server", ""); + String proxyConfig = Utils.getConfigString("proxy.socks", ""); if (!proxyConfig.equals("")) { String ip1 = Http.url(url).ignoreContentType().get().text(); - Proxy.setSocks(Utils.getConfigString("socks.server", "")); + Proxy.setSocks(Utils.getConfigString("proxy.socks", "")); String ip2 = Http.url(url).ignoreContentType().get().text(); assertFalse(ip1.equals(ip2)); } else { @@ -24,13 +24,13 @@ public class proxyTest extends TestCase { } } - // This test will only run on machines where the user has added a entry for proxy.server + // This test will only run on machines where the user has added a entry for proxy.http public void testHTTPProxy() throws IOException { URL url = new URL("https://icanhazip.com"); - String proxyConfig = Utils.getConfigString("proxy.server", ""); + String proxyConfig = Utils.getConfigString("proxy.http", ""); if (!proxyConfig.equals("")) { String ip1 = Http.url(url).ignoreContentType().get().text(); - Proxy.setHTTPProxy(Utils.getConfigString("proxy.server", "")); + Proxy.setHTTPProxy(Utils.getConfigString("proxy.http", "")); String ip2 = Http.url(url).ignoreContentType().get().text(); assertFalse(ip1.equals(ip2)); } else { From 34699bf8c66296f8eabb03e9658646ceac19c88b Mon Sep 17 00:00:00 2001 From: Random Committer Date: Fri, 20 Apr 2018 09:53:49 -0700 Subject: [PATCH 24/57] proxyTest.java: unset proxy/socks before testing it --- src/test/java/com/rarchives/ripme/tst/proxyTest.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/proxyTest.java b/src/test/java/com/rarchives/ripme/tst/proxyTest.java index 1d702c6b..36ea2f55 100644 --- a/src/test/java/com/rarchives/ripme/tst/proxyTest.java +++ b/src/test/java/com/rarchives/ripme/tst/proxyTest.java @@ -9,8 +9,14 @@ import com.rarchives.ripme.utils.Http; public class proxyTest extends TestCase { + + // This test will only run on machines where the user has added a entry for proxy.socks public void testSocksProxy() throws IOException { + // Unset proxy before testing + System.setProperty("http.proxyHost", ""); + System.setProperty("https.proxyHost", ""); + System.setProperty("socksProxyHost", ""); URL url = new URL("https://icanhazip.com"); String proxyConfig = Utils.getConfigString("proxy.socks", ""); if (!proxyConfig.equals("")) { @@ -26,6 +32,10 @@ public class proxyTest extends TestCase { // This test will only run on machines where the user has added a entry for proxy.http public void testHTTPProxy() throws IOException { + // Unset proxy before testing + System.setProperty("http.proxyHost", ""); + System.setProperty("https.proxyHost", ""); + System.setProperty("socksProxyHost", ""); URL url = new URL("https://icanhazip.com"); String proxyConfig = Utils.getConfigString("proxy.http", ""); if (!proxyConfig.equals("")) { From a587ec77faf0cb989dd8f15bc5813fdc644855b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sat, 21 Apr 2018 10:46:16 +0200 Subject: [PATCH 25/57] [hentaifoundry] set content filter --- .../ripper/rippers/HentaifoundryRipper.java | 49 +++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java index d09331a9..8d953de1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java @@ -10,6 +10,7 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.jsoup.Connection.Method; import org.jsoup.Connection.Response; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -49,15 +50,57 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { - Response resp = Http.url("http://www.hentai-foundry.com/").response(); - cookies = resp.cookies(); + Response resp; + Document doc; + resp = Http.url("http://www.hentai-foundry.com/?enterAgree=1&size=1500") .referrer("http://www.hentai-foundry.com/") .cookies(cookies) .response(); // The only cookie that seems to matter in getting around the age wall is the phpsession cookie cookies.putAll(resp.cookies()); - sleep(500); + + doc = resp.parse(); + String csrf_token = doc.select("input[name=YII_CSRF_TOKEN]") + .first().attr("value"); + if (csrf_token != null) { + Map data = new HashMap<>(); + data.put("YII_CSRF_TOKEN" , csrf_token); + data.put("rating_nudity" , "3"); + data.put("rating_violence" , "3"); + data.put("rating_profanity", "3"); + data.put("rating_racism" , "3"); + data.put("rating_sex" , "3"); + data.put("rating_spoilers" , "3"); + data.put("rating_yaoi" , "1"); + data.put("rating_yuri" , "1"); + data.put("rating_teen" , "1"); + data.put("rating_guro" , "1"); + data.put("rating_furry" , "1"); + data.put("rating_beast" , "1"); + data.put("rating_male" , "1"); + data.put("rating_female" , "1"); + data.put("rating_futa" , "1"); + data.put("rating_other" , "1"); + data.put("rating_scat" , "1"); + data.put("rating_incest" , "1"); + data.put("rating_rape" , "1"); + data.put("filter_media" , "A"); + data.put("filter_order" , "date_new"); + data.put("filter_type" , "0"); + + resp = Http.url("http://www.hentai-foundry.com/site/filters") + .referrer("http://www.hentai-foundry.com/") + .cookies(cookies) + .data(data) + .method(Method.POST) + .response(); + cookies.putAll(resp.cookies()); + } + else { + logger.info("unable to find csrf_token and set filter"); + } + resp = Http.url(url) .referrer("http://www.hentai-foundry.com/") .cookies(cookies) From eccce1b4a656af986a3171900e2d151d39cfc307 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 21 Apr 2018 14:22:48 -0400 Subject: [PATCH 26/57] 1.7.38: Added http and socks proxy support; Extended some unit tests to include getGid; Added HitomiRipper; hentaifoundry ripper now can rip all images from accounts --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index c8de11b2..d62354b6 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.37 + 1.7.38 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index a3783f9d..50199614 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.37", + "latestVersion": "1.7.38", "changeList": [ + "1.7.38: Added http and socks proxy support; Extended some unit tests to include getGid; Added HitomiRipper; hentaifoundry ripper now can rip all images from accounts", "1.7.37: MInor code clean up; Added socks proxy support; Added support for 8muses.download; Hentaifoundry no longer errors when there are no more pages; Fix bug that causes tumblr to replace https with httpss when downloading resized images", "1.7.36: Fixed Instagram ripper; Fixed hentai2read ripper test; Fixed tnbtu.com ripper", "1.7.35: Fixed instagram ripper; hentai2read ripper now properly names folders", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 3e0d5e5d..c3767e6a 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.37"; + private static final String DEFAULT_VERSION = "1.7.38"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 19ce96eae5236ce4766bf1bd2a90eead7853187a Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Mon, 23 Apr 2018 15:58:01 -0400 Subject: [PATCH 27/57] Fixed issue with downloading images without an extension --- .../ripme/ripper/AbstractRipper.java | 51 +++++++++++++------ .../ripme/tst/AbstractRipperTest.java | 4 ++ 2 files changed, 39 insertions(+), 16 deletions(-) create mode 100644 src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 6068ed18..4438ae2a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -212,7 +212,7 @@ public abstract class AbstractRipper * True if downloaded successfully * False if failed to download */ - protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName) { + protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName, String extension) { // Don't re-add the url if it was downloaded in a previous rip if (Utils.getConfigBoolean("remember.url_history", true) && !isThisATest()) { if (hasDownloadedURL(url.toExternalForm())) { @@ -228,21 +228,7 @@ public abstract class AbstractRipper return false; } logger.debug("url: " + url + ", prefix: " + prefix + ", subdirectory" + subdirectory + ", referrer: " + referrer + ", cookies: " + cookies + ", fileName: " + fileName); - String saveAs; - if (fileName != null) { - saveAs = fileName; - // Get the extension of the file - String extension = url.toExternalForm().substring(url.toExternalForm().lastIndexOf(".") + 1); - saveAs = saveAs + "." + extension; - } else { - saveAs = url.toExternalForm(); - saveAs = saveAs.substring(saveAs.lastIndexOf('/')+1); - } - - if (saveAs.indexOf('?') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('?')); } - if (saveAs.indexOf('#') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('#')); } - if (saveAs.indexOf('&') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('&')); } - if (saveAs.indexOf(':') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf(':')); } + String saveAs = getFileName(url, fileName, extension); File saveFileAs; try { if (!subdirectory.equals("")) { @@ -274,6 +260,10 @@ public abstract class AbstractRipper return addURLToDownload(url, saveFileAs, referrer, cookies); } + protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName) { + return addURLToDownload(url, prefix, subdirectory, referrer, cookies, fileName, null); + } + /** * Queues file to be downloaded and saved. With options. * @param url @@ -306,6 +296,35 @@ public abstract class AbstractRipper return addURLToDownload(url, prefix, ""); } + public static String getFileName(URL url, String fileName, String extension) { + String saveAs; + if (fileName != null) { + saveAs = fileName; + } else { + saveAs = url.toExternalForm(); + saveAs = saveAs.substring(saveAs.lastIndexOf('/')+1); + } + if (extension == null) { + // Get the extension of the file + String[] lastBitOfURL = url.toExternalForm().split("/"); + + String[] lastBit = lastBitOfURL[lastBitOfURL.length - 1].split("."); + if (lastBit.length != 0) { + extension = lastBit[lastBit.length - 1]; + saveAs = saveAs + "." + extension; + } + } + + if (saveAs.indexOf('?') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('?')); } + if (saveAs.indexOf('#') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('#')); } + if (saveAs.indexOf('&') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('&')); } + if (saveAs.indexOf(':') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf(':')); } + if (extension != null) { + saveAs = saveAs + "." + extension; + } + return saveAs; + } + /** * Waits for downloading threads to complete. diff --git a/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java b/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java new file mode 100644 index 00000000..3a71f472 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java @@ -0,0 +1,4 @@ +package com.rarchives.ripme.tst; + +public class AbstractRipperTest { +} From 56d20b15000e33f5ff9683938c5f786aa07afa95 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Mon, 23 Apr 2018 15:58:18 -0400 Subject: [PATCH 28/57] Added some unit tests for getting file name --- .../ripme/tst/AbstractRipperTest.java | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java b/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java index 3a71f472..f1d8eff5 100644 --- a/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java @@ -1,4 +1,30 @@ package com.rarchives.ripme.tst; -public class AbstractRipperTest { +import com.rarchives.ripme.ripper.AbstractRipper; +import junit.framework.TestCase; + +import java.io.IOException; +import java.net.URL; + + + +public class AbstractRipperTest extends TestCase { + + public void testGetFileName() throws IOException { + String fileName = AbstractRipper.getFileName(new URL("http://www.tsumino.com/Image/Object?name=U1EieteEGwm6N1dGszqCpA%3D%3D"), "test", "test"); + assertEquals("test.test", fileName); + + fileName = AbstractRipper.getFileName(new URL("http://www.tsumino.com/Image/Object?name=U1EieteEGwm6N1dGszqCpA%3D%3D"), "test", null); + assertEquals("test", fileName); + + fileName = AbstractRipper.getFileName(new URL("http://www.tsumino.com/Image/Object?name=U1EieteEGwm6N1dGszqCpA%3D%3D"), null, null); + assertEquals("Object", fileName); + + fileName = AbstractRipper.getFileName(new URL("http://www.test.com/file.png"), null, null); + assertEquals("file.png", fileName); + + fileName = AbstractRipper.getFileName(new URL("http://www.test.com/file."), null, null); + assertEquals("file.", fileName); + } + } From 0427073c9a47cba750dff8f3534e4d4c756199e0 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 26 Apr 2018 09:48:27 -0400 Subject: [PATCH 29/57] Added Gfycatporntube.com Ripper --- .../ripper/rippers/GfycatporntubeRipper.java | 61 +++++++++++++++++++ .../rippers/GfycatporntubeRipperTest.java | 13 ++++ 2 files changed, 74 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java new file mode 100644 index 00000000..504b89d6 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java @@ -0,0 +1,61 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class GfycatporntubeRipper extends AbstractHTMLRipper { + + public GfycatporntubeRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "gfycatporntube"; + } + + @Override + public String getDomain() { + return "gfycatporntube.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https?://gfycatporntube.com/([a-zA-Z1-9_-]*)/?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected gfycatporntube URL format: " + + "gfycatporntube.com/NAME - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + result.add(doc.select("source[id=mp4Source]").attr("src")); + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java new file mode 100644 index 00000000..6856eb06 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.GfycatporntubeRipper; + +public class GfycatporntubeRipperTest extends RippersTest { + public void testRip() throws IOException { + GfycatporntubeRipper ripper = new GfycatporntubeRipper(new URL("https://gfycatporntube.com/blowjob-bunny-puts-on-a-show/")); + testRipper(ripper); + } +} From 3342898e774ab6be7ee9815a0a351042889d3b2e Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 26 Apr 2018 10:48:07 -0400 Subject: [PATCH 30/57] Added ripper for rule34.xxx --- .../ripme/ripper/rippers/Rule34Ripper.java | 84 +++++++++++++++++++ .../ripme/ripper/rippers/rule34Ripper.java | 4 + 2 files changed, 88 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/Rule34Ripper.java create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Rule34Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Rule34Ripper.java new file mode 100644 index 00000000..8651bd69 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Rule34Ripper.java @@ -0,0 +1,84 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class Rule34Ripper extends AbstractHTMLRipper { + + public Rule34Ripper(URL url) throws IOException { + super(url); + } + + private String apiUrl; + private int pageNumber = 0; + + @Override + public String getHost() { + return "rule34"; + } + + @Override + public String getDomain() { + return "rule34.xxx"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https?://rule34.xxx/index.php\\?page=post&s=list&tags=([\\S]+)"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected rule34.xxx URL format: " + + "rule34.xxx/index.php\\?page=post&s=list&tags=TAG - got " + url + " instead"); + } + + public URL getAPIUrl() throws MalformedURLException { + URL urlToReturn = new URL("https://rule34.xxx/index.php?page=dapi&s=post&q=index&limit=100&tags=" + getGID(url)); + return urlToReturn; + } + + @Override + public Document getFirstPage() throws IOException { + apiUrl = getAPIUrl().toExternalForm(); + // "url" is an instance field of the superclass + return Http.url(getAPIUrl()).get(); + } + + @Override + public Document getNextPage(Document doc) throws IOException { + if (doc.html().contains("Search error: API limited due to abuse")) { + throw new IOException("No more pages"); + } + pageNumber += 1; + String nextPage = apiUrl + "&pid=" + pageNumber; + return Http.url(nextPage).get(); + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + for (Element el : doc.select("posts > post")) { + String imageSource = el.select("post").attr("file_url"); + result.add(imageSource); + + } + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java new file mode 100644 index 00000000..9402256b --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java @@ -0,0 +1,4 @@ +package com.rarchives.ripme.ripper.rippers; + +public class rule34Ripper { +} From 7cf8e59ee72ca906fb584ebafd021d8be5e6ef4f Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 26 Apr 2018 10:48:28 -0400 Subject: [PATCH 31/57] Removed misnamed file --- .../java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java deleted file mode 100644 index 9402256b..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -public class rule34Ripper { -} From c076fe2462dcdb1062f574405277ba8a80ab6b41 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 26 Apr 2018 10:58:35 -0400 Subject: [PATCH 32/57] Added unit test Rule34Ripper --- .../ripme/tst/ripper/rippers/Rule34RipperTest.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/Rule34RipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Rule34RipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Rule34RipperTest.java new file mode 100644 index 00000000..cffc807d --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Rule34RipperTest.java @@ -0,0 +1,14 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.Rule34Ripper; + +public class Rule34RipperTest extends RippersTest { + public void testShesFreakyRip() throws IOException { + Rule34Ripper ripper = new Rule34Ripper(new URL("https://rule34.xxx/index.php?page=post&s=list&tags=bimbo")); + testRipper(ripper); + } + +} From 388cd9936d16c2ec53a16749b179bf3d7d7fb13d Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 26 Apr 2018 12:42:24 -0400 Subject: [PATCH 33/57] 1.7.39: Added rule34.xxx ripper; Added Gfycatporntube.com ripper; Fixed AbstractRipper subdir bug; Added AbstractRipper unit tests" --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index d62354b6..b93ef11e 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.38 + 1.7.39 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 50199614..e1288c50 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.38", + "latestVersion": "1.7.39", "changeList": [ + "1.7.39: Added rule34.xxx ripper; Added Gfycatporntube.com ripper; Fixed AbstractRipper subdir bug; Added AbstractRipper unit tests"", "1.7.38: Added http and socks proxy support; Extended some unit tests to include getGid; Added HitomiRipper; hentaifoundry ripper now can rip all images from accounts", "1.7.37: MInor code clean up; Added socks proxy support; Added support for 8muses.download; Hentaifoundry no longer errors when there are no more pages; Fix bug that causes tumblr to replace https with httpss when downloading resized images", "1.7.36: Fixed Instagram ripper; Fixed hentai2read ripper test; Fixed tnbtu.com ripper", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index c3767e6a..b16d8399 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.38"; + private static final String DEFAULT_VERSION = "1.7.39"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 8f84d3f8ff3cbf391af7c738e2d6222b3f8b3657 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 26 Apr 2018 12:54:17 -0400 Subject: [PATCH 34/57] Removed extra qoute --- ripme.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ripme.json b/ripme.json index e1288c50..f4511057 100644 --- a/ripme.json +++ b/ripme.json @@ -1,7 +1,7 @@ { "latestVersion": "1.7.39", "changeList": [ - "1.7.39: Added rule34.xxx ripper; Added Gfycatporntube.com ripper; Fixed AbstractRipper subdir bug; Added AbstractRipper unit tests"", + "1.7.39: Added rule34.xxx ripper; Added Gfycatporntube.com ripper; Fixed AbstractRipper subdir bug; Added AbstractRipper unit tests", "1.7.38: Added http and socks proxy support; Extended some unit tests to include getGid; Added HitomiRipper; hentaifoundry ripper now can rip all images from accounts", "1.7.37: MInor code clean up; Added socks proxy support; Added support for 8muses.download; Hentaifoundry no longer errors when there are no more pages; Fix bug that causes tumblr to replace https with httpss when downloading resized images", "1.7.36: Fixed Instagram ripper; Fixed hentai2read ripper test; Fixed tnbtu.com ripper", From 5763c673ec88160679392886eb57ca0c2f076b4b Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 26 Apr 2018 15:46:50 -0400 Subject: [PATCH 35/57] TsuminoRipper now adds .png to filenames; Removed unneeded debug statment --- .../com/rarchives/ripme/ripper/rippers/TsuminoRipper.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java index 9ca91e45..6030c9f0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java @@ -17,7 +17,6 @@ import org.json.JSONObject; import org.jsoup.Connection; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; import com.rarchives.ripme.ripper.AbstractHTMLRipper; @@ -35,13 +34,10 @@ public class TsuminoRipper extends AbstractHTMLRipper { try { // This sessionId will expire and need to be replaced cookies.put("ASP.NET_SessionId","c4rbzccf0dvy3e0cloolmlkq"); - logger.info(cookies); Document doc = Jsoup.connect(postURL).data("q", getAlbumID()).userAgent(USER_AGENT).cookies(cookies).referrer("http://www.tsumino.com/Read/View/" + getAlbumID()).post(); String jsonInfo = doc.html().replaceAll("","").replaceAll("", "").replaceAll("", "").replaceAll("", "") .replaceAll("", "").replaceAll("\n", ""); - logger.info(jsonInfo); JSONObject json = new JSONObject(jsonInfo); - logger.info(json.getJSONArray("reader_page_urls")); return json.getJSONArray("reader_page_urls"); } catch (IOException e) { logger.info(e); @@ -85,7 +81,6 @@ public class TsuminoRipper extends AbstractHTMLRipper { public Document getFirstPage() throws IOException { Connection.Response resp = Http.url(url).response(); cookies.putAll(resp.cookies()); - logger.info(resp.parse()); return resp.parse(); } @@ -103,6 +98,6 @@ public class TsuminoRipper extends AbstractHTMLRipper { @Override public void downloadURL(URL url, int index) { sleep(1000); - addURLToDownload(url, getPrefix(index)); + addURLToDownload(url, getPrefix(index), "", null, null, null, "png"); } } From 1ccaa21ef89ea60f6e255674b57c17f6d8fa45eb Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 27 Apr 2018 13:38:16 -0400 Subject: [PATCH 36/57] Added canRip for rule34.xxx --- .../rarchives/ripme/ripper/rippers/Rule34Ripper.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Rule34Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Rule34Ripper.java index 8651bd69..b7e0f7b0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/Rule34Ripper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Rule34Ripper.java @@ -33,6 +33,16 @@ public class Rule34Ripper extends AbstractHTMLRipper { return "rule34.xxx"; } + @Override + public boolean canRip(URL url){ + Pattern p = Pattern.compile("https?://rule34.xxx/index.php\\?page=post&s=list&tags=([\\S]+)"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return true; + } + return false; + } + @Override public String getGID(URL url) throws MalformedURLException { Pattern p = Pattern.compile("https?://rule34.xxx/index.php\\?page=post&s=list&tags=([\\S]+)"); @@ -41,7 +51,7 @@ public class Rule34Ripper extends AbstractHTMLRipper { return m.group(1); } throw new MalformedURLException("Expected rule34.xxx URL format: " + - "rule34.xxx/index.php\\?page=post&s=list&tags=TAG - got " + url + " instead"); + "rule34.xxx/index.php?page=post&s=list&tags=TAG - got " + url + " instead"); } public URL getAPIUrl() throws MalformedURLException { From 7b1fbd3290010db410ef2c92afb75d1bc6e4797f Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 29 Apr 2018 12:22:27 -0400 Subject: [PATCH 37/57] Added ripper for hypnohub.net; removed misnamed file --- .../ripme/ripper/rippers/HypnohubRipper.java | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/HypnohubRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HypnohubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HypnohubRipper.java new file mode 100644 index 00000000..1eabefb9 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HypnohubRipper.java @@ -0,0 +1,91 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +import javax.print.Doc; + +public class HypnohubRipper extends AbstractHTMLRipper { + + public HypnohubRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "hypnohub"; + } + + @Override + public String getDomain() { + return "hypnohub.net"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https?://hypnohub.net/\\S+/show/([\\d]+)/?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + p = Pattern.compile("https?://hypnohub.net/\\S+/show/([\\d]+)/([\\S]+)/?$"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1) + "_" + m.group(2); + } + throw new MalformedURLException("Expected cfake URL format: " + + "hypnohub.net/pool/show/ID - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + private String ripPost(String url) throws IOException { + logger.info(url); + Document doc = Http.url(url).get(); + return "https:" + doc.select("img.image").attr("src"); + + } + + private String ripPost(Document doc) { + logger.info(url); + return "https:" + doc.select("img.image").attr("src"); + + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + if (url.toExternalForm().contains("/pool")) { + for (Element el : doc.select("ul[id=post-list-posts] > li > div > a.thumb")) { + try { + result.add(ripPost("https://hypnohub.net" + el.attr("href"))); + } catch (IOException e) { + return result; + } + } + } else if (url.toExternalForm().contains("/post")) { + result.add(ripPost(doc)); + } + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} From 60d5f95f938a2da169727b13a4b5af055ca16b9e Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 29 Apr 2018 12:40:50 -0400 Subject: [PATCH 38/57] Added unit test for HypnohubRipper --- .../ripper/rippers/HypnohubRipperTest.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/HypnohubRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HypnohubRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HypnohubRipperTest.java new file mode 100644 index 00000000..4466eadc --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HypnohubRipperTest.java @@ -0,0 +1,25 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.HypnohubRipper; + +public class HypnohubRipperTest extends RippersTest { + public void testRip() throws IOException { + URL poolURL = new URL("http://hypnohub.net/pool/show/2303"); + URL postURL = new URL("http://hypnohub.net/post/show/63464/black_hair-bracelet-collar-corruption-female_only-"); + HypnohubRipper ripper = new HypnohubRipper(poolURL); + testRipper(ripper); + ripper = new HypnohubRipper(postURL); + testRipper(ripper); + } + public void testGetGID() throws IOException { + URL poolURL = new URL("http://hypnohub.net/pool/show/2303"); + HypnohubRipper ripper = new HypnohubRipper(poolURL); + assertEquals("hypnohub_2303", ripper.getGID(poolURL)); + + URL postURL = new URL("http://hypnohub.net/post/show/63464/black_hair-bracelet-collar-corruption-female_only-"); + assertEquals("hypnohub_63464_black_hair-bracelet-collar-corruption-female_only-", ripper.getGID(postURL)); + } +} From 24f1b4fbdc6f26ab0a6a6dfe85d2edb1cf5ed9a7 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 29 Apr 2018 12:50:44 -0400 Subject: [PATCH 39/57] Fixed hypnohub unit test --- .../ripme/tst/ripper/rippers/HypnohubRipperTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HypnohubRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HypnohubRipperTest.java index 4466eadc..1d9ef4ad 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HypnohubRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HypnohubRipperTest.java @@ -17,9 +17,9 @@ public class HypnohubRipperTest extends RippersTest { public void testGetGID() throws IOException { URL poolURL = new URL("http://hypnohub.net/pool/show/2303"); HypnohubRipper ripper = new HypnohubRipper(poolURL); - assertEquals("hypnohub_2303", ripper.getGID(poolURL)); + assertEquals("2303", ripper.getGID(poolURL)); URL postURL = new URL("http://hypnohub.net/post/show/63464/black_hair-bracelet-collar-corruption-female_only-"); - assertEquals("hypnohub_63464_black_hair-bracelet-collar-corruption-female_only-", ripper.getGID(postURL)); + assertEquals("63464_black_hair-bracelet-collar-corruption-female_only-", ripper.getGID(postURL)); } } From e8f7f2cfae1f2ebbf631f6e861b4f36231901cc8 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 29 Apr 2018 16:28:17 -0400 Subject: [PATCH 40/57] 1.7.40: Added hypnohub.net ripper; Fixed rule34.xxx ripper; Tsumino Ripper now add .png to filenames --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index b93ef11e..d0be1909 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.39 + 1.7.40 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index f4511057..dcac88bc 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.39", + "latestVersion": "1.7.40", "changeList": [ + "1.7.40: Added hypnohub.net ripper; Fixed rule34.xxx ripper; Tsumino Ripper now add .png to filenames", "1.7.39: Added rule34.xxx ripper; Added Gfycatporntube.com ripper; Fixed AbstractRipper subdir bug; Added AbstractRipper unit tests", "1.7.38: Added http and socks proxy support; Extended some unit tests to include getGid; Added HitomiRipper; hentaifoundry ripper now can rip all images from accounts", "1.7.37: MInor code clean up; Added socks proxy support; Added support for 8muses.download; Hentaifoundry no longer errors when there are no more pages; Fix bug that causes tumblr to replace https with httpss when downloading resized images", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index b16d8399..5287194d 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.39"; + private static final String DEFAULT_VERSION = "1.7.40"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 5369a54caaff2b484f7c3c690d1a83893a9e10e1 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Mon, 30 Apr 2018 13:22:51 -0400 Subject: [PATCH 41/57] Added ripper and unit test for dynasty-scans.com --- .../ripper/rippers/DynastyscansRipper.java | 84 +++++++++++++++++++ .../rippers/DynastyscansRipperTest.java | 18 ++++ 2 files changed, 102 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/DynastyscansRipper.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/DynastyscansRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DynastyscansRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DynastyscansRipper.java new file mode 100644 index 00000000..37d3ad93 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DynastyscansRipper.java @@ -0,0 +1,84 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.json.JSONArray; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class DynastyscansRipper extends AbstractHTMLRipper { + + public DynastyscansRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "dynasty-scans"; + } + + @Override + public String getDomain() { + return "dynasty-scans.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https?://dynasty-scans.com/chapters/([\\S]+)/?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected dynasty-scans URL format: " + + "dynasty-scans.com/chapters/ID - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + @Override + public Document getNextPage(Document doc) throws IOException { + Element elem = doc.select("a[id=next_link]").first(); + if (elem == null || elem.attr("href").equals("#")) { + throw new IOException("No more pages"); + } + return Http.url("https://dynasty-scans.com" + elem.attr("href")).get(); + + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + String jsonText = null; + for (Element script : doc.select("script")) { + if (script.data().contains("var pages")) { + jsonText = script.data().replaceAll("var pages = ", ""); + jsonText = jsonText.replaceAll("//", ""); + } + } + JSONArray imageArray = new JSONArray(jsonText); + for (int i = 0; i < imageArray.length(); i++) { + result.add("https://dynasty-scans.com" + imageArray.getJSONObject(i).getString("image")); + } + + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DynastyscansRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DynastyscansRipperTest.java new file mode 100644 index 00000000..8eb8d88f --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DynastyscansRipperTest.java @@ -0,0 +1,18 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.DynastyscansRipper; + +public class DynastyscansRipperTest extends RippersTest { + public void testRip() throws IOException { + DynastyscansRipper ripper = new DynastyscansRipper(new URL("https://dynasty-scans.com/chapters/under_one_roof_ch01")); + testRipper(ripper); + } + + public void testGetGID() throws IOException { + DynastyscansRipper ripper = new DynastyscansRipper(new URL("https://dynasty-scans.com/chapters/under_one_roof_ch01")); + assertEquals("under_one_roof_ch01", ripper.getGID(new URL("https://dynasty-scans.com/chapters/under_one_roof_ch01"))); + } +} From 6a672f05c9762b2fabbc6c0619ab12bc565f2403 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Mon, 30 Apr 2018 17:47:19 -0400 Subject: [PATCH 42/57] Added ManganeloRipper --- .../ripme/ripper/rippers/ManganeloRipper.java | 116 ++++++++++++++++++ .../ripper/rippers/ManganeloRipperTest.java | 13 ++ 2 files changed, 129 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/ManganeloRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java new file mode 100644 index 00000000..8f8f8e68 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java @@ -0,0 +1,116 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +import javax.print.Doc; + +public class ManganeloRipper extends AbstractHTMLRipper { + + public ManganeloRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "manganelo"; + } + + @Override + public String getDomain() { + return "manganelo.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https?://manganelo.com/manga/([\\S]+)/?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + + p = Pattern.compile("http://manganelo.com/chapter/([\\S]+)/([\\S]+)/?$"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected manganelo URL format: " + + "/manganelo.com/manga/ID - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + @Override + public Document getNextPage(Document doc) throws IOException { + Element elem = doc.select("div.btn-navigation-chap > a.back").first(); + if (elem == null) { + throw new IOException("No more pages"); + } else { + return Http.url(elem.attr("href")).get(); + } + } + + private List getURLsFromChap(String url) { + logger.debug("Getting urls from " + url); + List result = new ArrayList<>(); + try { + Document doc = Http.url(url).get(); + for (Element el : doc.select("img.img_content")) { + result.add(el.attr("src")); + } + return result; + } catch (IOException e) { + return null; + } + + } + + private List getURLsFromChap(Document doc) { + logger.debug("Getting urls from " + url); + List result = new ArrayList<>(); + for (Element el : doc.select("img.img_content")) { + result.add(el.attr("src")); + } + return result; + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + List urlsToGrab = new ArrayList<>(); + if (url.toExternalForm().contains("/manga/")) { + for (Element el : doc.select("div.chapter-list > div.row > span > a")) { + urlsToGrab.add(el.attr("href")); + } + Collections.reverse(urlsToGrab); + + for (String url : urlsToGrab) { + result.addAll(getURLsFromChap(url)); + } + } else if (url.toExternalForm().contains("/chapter/")) { + result.addAll(getURLsFromChap(doc)); + } + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ManganeloRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ManganeloRipperTest.java new file mode 100644 index 00000000..ca355a2c --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ManganeloRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.ManganeloRipper; + +public class ManganeloRipperTest extends RippersTest { + public void testRip() throws IOException { + ManganeloRipper ripper = new ManganeloRipper(new URL("http://manganelo.com/manga/black_clover")); + testRipper(ripper); + } +} From d73744a73fb7a709e9c4a276646edfe545760fc2 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 1 May 2018 15:14:08 -0400 Subject: [PATCH 43/57] Added support for spyingwithlana.com; added some more unittests for the wordpress comic ripper --- .../ripper/rippers/WordpressComicRipper.java | 43 ++++++++++++------- .../rippers/WordpressComicRipperTest.java | 28 ++++++++++++ 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java index 13b4891e..8b7b7b80 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java @@ -45,7 +45,19 @@ public class WordpressComicRipper extends AbstractHTMLRipper { "thisis.delvecomic.com", "tnbtu.com", "shipinbottle.pepsaga.com", - "8muses.download" + "8muses.download", + "spyingwithlana.com" + ); + + private static List theme1 = Arrays.asList( + "www.totempole666.com", + "buttsmithy.com", + "themonsterunderthebed.net", + "prismblush.com", + "www.konradokonski.com", + "thisis.delvecomic.com", + "tnbtu.com", + "spyingwithlana.com" ); @Override @@ -142,6 +154,12 @@ public class WordpressComicRipper extends AbstractHTMLRipper { if (eight_musesMat.matches()) { return true; } + + Pattern spyingwithlanaPat = Pattern.compile("https?://spyingwithlana.com/comic/([a-zA-Z0-9_-]+)/?$"); + Matcher spyingwithlanaMat = spyingwithlanaPat.matcher(url.toExternalForm()); + if (spyingwithlanaMat.matches()) { + return true; + } } @@ -221,6 +239,13 @@ public class WordpressComicRipper extends AbstractHTMLRipper { if (eight_musesMat.matches()) { return getHost() + "_" + eight_musesMat.group(1); } + + Pattern spyingwithlanaPat = Pattern.compile("https?://spyingwithlana.com/comic/([a-zA-Z0-9_-]+)/?$"); + Matcher spyingwithlanaMat = spyingwithlanaPat.matcher(url.toExternalForm()); + if (spyingwithlanaMat.matches()) { + return "spyingwithlana_" + spyingwithlanaMat.group(1).replaceAll("-page-\\d", ""); + } + return super.getAlbumTitle(url); } @@ -239,13 +264,7 @@ public class WordpressComicRipper extends AbstractHTMLRipper { // Find next page String nextPage = ""; Element elem = null; - if (getHost().contains("www.totempole666.com") - || getHost().contains("buttsmithy.com") - || getHost().contains("themonsterunderthebed.net") - || getHost().contains("prismblush.com") - || getHost().contains("www.konradokonski.com") - || getHost().contains("thisis.delvecomic.com") - || getHost().contains("tnbtu.com")) { + if (theme1.contains(getHost())) { elem = doc.select("a.comic-nav-next").first(); if (elem == null) { throw new IOException("No more pages"); @@ -269,13 +288,7 @@ public class WordpressComicRipper extends AbstractHTMLRipper { @Override public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); - if (getHost().contains("www.totempole666.com") - || getHost().contains("buttsmithy.com") - || getHost().contains("themonsterunderthebed.net") - || getHost().contains("prismblush.com") - || getHost().contains("www.konradokonski.com") - || getHost().contains("thisis.delvecomic.com") - || getHost().contains("tnbtu.com")) { + if (theme1.contains(getHost())) { Element elem = doc.select("div.comic-table > div#comic > a > img").first(); // If doc is the last page in the comic then elem.attr("src") returns null // because there is no link to the next page diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java index ec8db0b0..8879c561 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java @@ -55,6 +55,7 @@ public class WordpressComicRipperTest extends RippersTest { WordpressComicRipper ripper = new WordpressComicRipper( new URL("http://www.konradokonski.com/sawdust/comic/get-up/")); testRipper(ripper); + } public void test_konradokonski_2() throws IOException { @@ -63,6 +64,13 @@ public class WordpressComicRipperTest extends RippersTest { testRipper(ripper); } + public void test_konradokonski_getAlbumTitle() throws IOException { + URL url = new URL("http://www.konradokonski.com/sawdust/comic/get-up/"); + WordpressComicRipper ripper = new WordpressComicRipper(url); + assertEquals("konradokonski.com_sawdust", ripper.getAlbumTitle(url)); + + } + /* // https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI public void test_freeadultcomix() throws IOException { @@ -89,6 +97,26 @@ public class WordpressComicRipperTest extends RippersTest { new URL("https://8muses.download/lustomic-playkittens-josh-samuel-porn-comics-8-muses/")); testRipper(ripper); } + + public void test_Eightmuses_getAlbumTitle() throws IOException { + URL url = new URL("https://8muses.download/lustomic-playkittens-josh-samuel-porn-comics-8-muses/"); + WordpressComicRipper ripper = new WordpressComicRipper(url); + assertEquals("8muses.download_lustomic-playkittens-josh-samuel-porn-comics-8-muses", + ripper.getAlbumTitle(url)); + } + + public void test_spyingwithlana_download() throws IOException { + WordpressComicRipper ripper = new WordpressComicRipper( + new URL("http://spyingwithlana.com/comic/the-big-hookup/")); + testRipper(ripper); + } + + public void test_spyingwithlana_getAlbumTitle() throws IOException { + URL url = new URL("http://spyingwithlana.com/comic/the-big-hookup/"); + WordpressComicRipper ripper = new WordpressComicRipper(url); + assertEquals("spyingwithlana_the-big-hookup", ripper.getAlbumTitle(url)); + } + // https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI // public void test_pepsaga() throws IOException { // WordpressComicRipper ripper = new WordpressComicRipper( From d604d4059659fca6653d70266feaa049b6d7d1b5 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 5 May 2018 05:36:00 -0400 Subject: [PATCH 44/57] Added option for addURLToDownload which sets the file extension to the files MIME type --- .../com/rarchives/ripme/ripper/AbstractRipper.java | 11 ++++++++--- .../com/rarchives/ripme/ripper/AlbumRipper.java | 6 +++--- .../rarchives/ripme/ripper/DownloadFileThread.java | 13 +++++++++++-- .../com/rarchives/ripme/ripper/VideoRipper.java | 3 ++- .../ripme/ripper/rippers/FivehundredpxRipper.java | 2 +- .../ripme/ripper/rippers/TsuminoRipper.java | 6 +++++- 6 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 4438ae2a..20889495 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -192,7 +192,8 @@ public abstract class AbstractRipper * True if downloaded successfully * False if failed to download */ - protected abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies); + protected abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies, + Boolean getFileExtFromMIME); /** * Queues image to be downloaded and saved. @@ -212,7 +213,7 @@ public abstract class AbstractRipper * True if downloaded successfully * False if failed to download */ - protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName, String extension) { + protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName, String extension, Boolean getFileExtFromMIME) { // Don't re-add the url if it was downloaded in a previous rip if (Utils.getConfigBoolean("remember.url_history", true) && !isThisATest()) { if (hasDownloadedURL(url.toExternalForm())) { @@ -257,7 +258,11 @@ public abstract class AbstractRipper logger.debug("Unable to write URL history file"); } } - return addURLToDownload(url, saveFileAs, referrer, cookies); + return addURLToDownload(url, saveFileAs, referrer, cookies, getFileExtFromMIME); + } + + protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName, String extension) { + return addURLToDownload(url, prefix, subdirectory, referrer, cookies, fileName, extension, false); } protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName) { diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 1726343a..f700f012 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -50,7 +50,7 @@ public abstract class AlbumRipper extends AbstractRipper { /** * Queues multiple URLs of single images to download from a single Album URL */ - public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies) { + public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { // Only download one file if this is a test. if (super.isThisATest() && (itemsPending.size() > 0 || itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { @@ -82,7 +82,7 @@ public abstract class AlbumRipper extends AbstractRipper { } else { itemsPending.put(url, saveAs); - DownloadFileThread dft = new DownloadFileThread(url, saveAs, this); + DownloadFileThread dft = new DownloadFileThread(url, saveAs, this, getFileExtFromMIME); if (referrer != null) { dft.setReferrer(referrer); } @@ -96,7 +96,7 @@ public abstract class AlbumRipper extends AbstractRipper { @Override public boolean addURLToDownload(URL url, File saveAs) { - return addURLToDownload(url, saveAs, null, null); + return addURLToDownload(url, saveAs, null, null, false); } /** diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index c62d58a6..42dedffe 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -8,6 +8,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URL; +import java.net.URLConnection; import java.util.HashMap; import java.util.Map; @@ -36,10 +37,11 @@ class DownloadFileThread extends Thread { private String prettySaveAs; private AbstractRipper observer; private int retries; + private Boolean getFileExtFromMIME; private final int TIMEOUT; - public DownloadFileThread(URL url, File saveAs, AbstractRipper observer) { + public DownloadFileThread(URL url, File saveAs, AbstractRipper observer, Boolean getFileExtFromMIME) { super(); this.url = url; this.saveAs = saveAs; @@ -47,6 +49,7 @@ class DownloadFileThread extends Thread { this.observer = observer; this.retries = Utils.getConfigInteger("download.retries", 1); this.TIMEOUT = Utils.getConfigInteger("download.timeout", 60000); + this.getFileExtFromMIME = getFileExtFromMIME; } public void setReferrer(String referrer) { @@ -143,9 +146,15 @@ class DownloadFileThread extends Thread { observer.downloadErrored(url, "Imgur image is 404: " + url.toExternalForm()); return; } - // Save file bis = new BufferedInputStream(huc.getInputStream()); + + // Check if we should get the file ext from the MIME type + if (getFileExtFromMIME) { + String fileExt = URLConnection.guessContentTypeFromStream(bis).replaceAll("image/", ""); + saveAs = new File(saveAs.toString() + "." + fileExt); + } + fos = new FileOutputStream(saveAs); IOUtils.copy(bis, fos); break; // Download successful: break out of infinite loop diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 13008cd9..29200d5a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -10,6 +10,7 @@ import java.util.Map; import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Utils; +import com.sun.org.apache.xpath.internal.operations.Bool; public abstract class VideoRipper extends AbstractRipper { @@ -70,7 +71,7 @@ public abstract class VideoRipper extends AbstractRipper { } @Override - public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies) { + public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { return addURLToDownload(url, saveAs); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java index 7e532943..93aedba2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java @@ -332,7 +332,7 @@ public class FivehundredpxRipper extends AbstractJSONRipper { String[] fields = u.split("/"); String prefix = getPrefix(index) + fields[fields.length - 3]; File saveAs = new File(getWorkingDir() + File.separator + prefix + ".jpg"); - addURLToDownload(url, saveAs, "", null); + addURLToDownload(url, saveAs, "", null, false); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java index 6030c9f0..7d35fc1d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java @@ -98,6 +98,10 @@ public class TsuminoRipper extends AbstractHTMLRipper { @Override public void downloadURL(URL url, int index) { sleep(1000); - addURLToDownload(url, getPrefix(index), "", null, null, null, "png"); + /* + There is no way to tell if an image returned from tsumino.com is a png to jpg. The content-type header is always + "image/jpeg" even when the image is a png. The file ext is not included in the url. + */ + addURLToDownload(url, getPrefix(index), "", null, null, null, null, true); } } From 2d1e648bd1b226c97a8303588162ab0d98695bf7 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 5 May 2018 05:41:48 -0400 Subject: [PATCH 45/57] 1.7.41: Added support for spyingwithlana.com; Added ManganeloRipper; Added support for dynasty-scans.com --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index d0be1909..9bd3de11 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.40 + 1.7.41 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index dcac88bc..ea3feafa 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.40", + "latestVersion": "1.7.41", "changeList": [ + "1.7.41: Added support for spyingwithlana.com; Added ManganeloRipper; Added support for dynasty-scans.com", "1.7.40: Added hypnohub.net ripper; Fixed rule34.xxx ripper; Tsumino Ripper now add .png to filenames", "1.7.39: Added rule34.xxx ripper; Added Gfycatporntube.com ripper; Fixed AbstractRipper subdir bug; Added AbstractRipper unit tests", "1.7.38: Added http and socks proxy support; Extended some unit tests to include getGid; Added HitomiRipper; hentaifoundry ripper now can rip all images from accounts", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 5287194d..92a75c78 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.40"; + private static final String DEFAULT_VERSION = "1.7.41"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 2001ada6f4291122a210e49db53f0b0d597ec308 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 6 May 2018 05:33:33 -0400 Subject: [PATCH 46/57] Fixed NudeGalsRipper --- .../ripme/ripper/rippers/NudeGalsRipper.java | 21 ++----------------- .../ripper/rippers/NudeGalsRipperTest.java | 4 ++++ 2 files changed, 6 insertions(+), 19 deletions(-) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/NudeGalsRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NudeGalsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NudeGalsRipper.java index d3cb0ab1..3300da50 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NudeGalsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NudeGalsRipper.java @@ -33,23 +33,6 @@ public class NudeGalsRipper extends AbstractHTMLRipper { return "nude-gals.com"; } - public String getAlbumTitle(URL url) throws MalformedURLException { - try { - Document doc = getFirstPage(); - Elements elems = doc.select("#left_col > #grid_title > .right"); - - String girl = elems.get(3).text(); - String magazine = elems.get(2).text(); - String title = elems.get(0).text(); - - return getHost() + "_" + girl + "-" + magazine + "-" + title; - } catch (Exception e) { - // Fall back to default album naming convention - logger.warn("Failed to get album title from " + url, e); - } - return super.getAlbumTitle(url); - } - @Override public String getGID(URL url) throws MalformedURLException { Pattern p; @@ -79,9 +62,9 @@ public class NudeGalsRipper extends AbstractHTMLRipper { public List getURLsFromPage(Document doc) { List imageURLs = new ArrayList<>(); - Elements thumbs = doc.select("#grid_container .grid > .grid_box"); + Elements thumbs = doc.select("img.thumbnail"); for (Element thumb : thumbs) { - String link = thumb.select("a").get(1).attr("href"); + String link = thumb.attr("src").replaceAll("thumbs/th_", ""); String imgSrc = "http://nude-gals.com/" + link; imageURLs.add(imgSrc); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NudeGalsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NudeGalsRipperTest.java new file mode 100644 index 00000000..e77fcea9 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NudeGalsRipperTest.java @@ -0,0 +1,4 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +public class NudeGalsRipperTest { +} From 7e766b2bc2703b61ca5261e7a24b6891466a18fa Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 6 May 2018 05:34:00 -0400 Subject: [PATCH 47/57] Added unit test for NudeGalsRipper --- .../tst/ripper/rippers/NudeGalsRipperTest.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NudeGalsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NudeGalsRipperTest.java index e77fcea9..3353eeb5 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NudeGalsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NudeGalsRipperTest.java @@ -1,4 +1,18 @@ package com.rarchives.ripme.tst.ripper.rippers; -public class NudeGalsRipperTest { +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.NudeGalsRipper; + +public class NudeGalsRipperTest extends RippersTest { + public void testRip() throws IOException { + NudeGalsRipper ripper = new NudeGalsRipper(new URL("https://nude-gals.com/photoshoot.php?photoshoot_id=5541")); + testRipper(ripper); + } + + public void testGetGID() throws IOException { + NudeGalsRipper ripper = new NudeGalsRipper(new URL("https://nude-gals.com/photoshoot.php?photoshoot_id=5541")); + assertEquals("5541", ripper.getGID( new URL("https://nude-gals.com/photoshoot.php?photoshoot_id=5541"))); + } } From f36a4dfd3b78ab5454b7b4978b32e58522156674 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 6 May 2018 05:43:57 -0400 Subject: [PATCH 48/57] Removed vine ripper --- .../ripme/ripper/rippers/VineRipper.java | 95 ------------------- 1 file changed, 95 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/VineRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VineRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VineRipper.java deleted file mode 100644 index 1ba53926..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VineRipper.java +++ /dev/null @@ -1,95 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.json.JSONArray; -import org.json.JSONObject; -import org.jsoup.HttpStatusException; - -import com.rarchives.ripme.ripper.AlbumRipper; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; -import com.rarchives.ripme.utils.Http; - -public class VineRipper extends AlbumRipper { - - private static final String DOMAIN = "vine.co", - HOST = "vine"; - - public VineRipper(URL url) throws IOException { - super(url); - } - - @Override - public boolean canRip(URL url) { - return url.getHost().endsWith(DOMAIN); - } - - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return new URL("http://vine.co/u/" + getGID(url)); - } - - @Override - public void rip() throws IOException { - int page = 0; - String baseURL = "https://vine.co/api/timelines/users/" + getGID(this.url); - JSONObject json = null; - while (true) { - page++; - String theURL = baseURL; - if (page > 1) { - theURL += "?page=" + page; - } - try { - logger.info(" Retrieving " + theURL); - sendUpdate(STATUS.LOADING_RESOURCE, theURL); - json = Http.url(theURL).getJSON(); - } catch (HttpStatusException e) { - logger.debug("Hit end of pages at page " + page, e); - break; - } - JSONArray records = json.getJSONObject("data").getJSONArray("records"); - for (int i = 0; i < records.length(); i++) { - String videoURL = records.getJSONObject(i).getString("videoUrl"); - addURLToDownload(new URL(videoURL)); - if (isThisATest()) { - break; - } - } - if (isThisATest()) { - break; - } - if (records.length() == 0) { - logger.info("Zero records returned"); - break; - } - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - logger.error("[!] Interrupted while waiting to load next page", e); - break; - } - } - waitForThreads(); - } - - @Override - public String getHost() { - return HOST; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://(www\\.)?vine\\.co/u/([0-9]+).*$"); - Matcher m = p.matcher(url.toExternalForm()); - if (!m.matches()) { - throw new MalformedURLException("Expected format: http://vine.co/u/######"); - } - return m.group(m.groupCount()); - } - -} From b8023fb4f1f501442a2ef88cccfe31fc6539b50a Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 6 May 2018 05:46:35 -0400 Subject: [PATCH 49/57] Removed test for removed ripper --- .../ripme/tst/ripper/rippers/VineRipperTest.java | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/VineRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VineRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VineRipperTest.java deleted file mode 100644 index 343a72b0..00000000 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VineRipperTest.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import java.io.IOException; -import java.net.URL; - -import com.rarchives.ripme.ripper.rippers.VineRipper; - -public class VineRipperTest extends RippersTest { - // https://github.com/RipMeApp/ripme/issues/181 - /* - public void testVineRip() throws IOException { - VineRipper ripper = new VineRipper(new URL("https://vine.co/u/954440445776334848")); - testRipper(ripper); - } - */ -} From 66bba724f9a0ef461d8e71219a60753f825cc6a0 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 6 May 2018 06:27:41 -0400 Subject: [PATCH 50/57] Added user support to SmuttyRipper --- .../ripme/ripper/rippers/SmuttyRipper.java | 125 +++++++++--------- 1 file changed, 62 insertions(+), 63 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SmuttyRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SmuttyRipper.java index 51992ec4..b61f2fef 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SmuttyRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SmuttyRipper.java @@ -3,24 +3,19 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.util.ArrayList; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import com.rarchives.ripme.ripper.AlbumRipper; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; +import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; -/** - * Appears to be broken as of 2015-02-11. - * Generating large image from thumbnail requires replacing "/m/" with something else: - * -> Sometimes "/b/" - * -> Sometimes "/p/" - * No way to know without loading the image page. - */ -public class SmuttyRipper extends AlbumRipper { + +public class SmuttyRipper extends AbstractHTMLRipper { private static final String DOMAIN = "smutty.com", HOST = "smutty"; @@ -29,6 +24,16 @@ public class SmuttyRipper extends AlbumRipper { super(url); } + @Override + public String getHost() { + return "smutty"; + } + + @Override + public String getDomain() { + return "smutty.com"; + } + @Override public boolean canRip(URL url) { return (url.getHost().endsWith(DOMAIN)); @@ -40,69 +45,57 @@ public class SmuttyRipper extends AlbumRipper { } @Override - public void rip() throws IOException { - int page = 0; - String url, tag = getGID(this.url); - boolean hasNextPage = true; - while (hasNextPage) { + public List getURLsFromPage(Document doc) { + List results = new ArrayList<>(); + for (Element image : doc.select("a.l > img")) { if (isStopped()) { break; } - page++; - url = "http://smutty.com/h/" + tag + "/?q=%23" + tag + "&page=" + page + "&sort=date&lazy=1"; - this.sendUpdate(STATUS.LOADING_RESOURCE, url); - logger.info(" Retrieving " + url); - Document doc; - try { - doc = Http.url(url) - .ignoreContentType() - .get(); - } catch (IOException e) { - if (e.toString().contains("Status=404")) { - logger.info("No more pages to load"); - } else { - logger.warn("Exception while loading " + url, e); - } - break; - } - for (Element image : doc.select("a.l > img")) { - if (isStopped()) { - break; - } - String imageUrl = image.attr("src"); + String imageUrl = image.attr("src"); - // Construct direct link to image based on thumbnail - StringBuilder sb = new StringBuilder(); - String[] fields = imageUrl.split("/"); - for (int i = 0; i < fields.length; i++) { - if (i == fields.length - 2 && fields[i].equals("m")) { - fields[i] = "b"; - } - sb.append(fields[i]); - if (i < fields.length - 1) { - sb.append("/"); - } + // Construct direct link to image based on thumbnail + StringBuilder sb = new StringBuilder(); + String[] fields = imageUrl.split("/"); + for (int i = 0; i < fields.length; i++) { + if (i == fields.length - 2 && fields[i].equals("m")) { + fields[i] = "b"; + } + sb.append(fields[i]); + if (i < fields.length - 1) { + sb.append("/"); } - imageUrl = sb.toString(); - addURLToDownload(new URL("http:" + imageUrl)); - } - if (doc.select("#next").size() == 0) { - break; // No more pages - } - // Wait before loading next page - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - logger.error("[!] Interrupted while waiting to load next album:", e); - break; } + imageUrl = sb.toString(); + results.add("http:" + imageUrl); } - waitForThreads(); + return results; } @Override - public String getHost() { - return HOST; + public Document getNextPage(Document doc) throws IOException { + Element elem = doc.select("a.next").first(); + if (elem == null) { + throw new IOException("No more pages"); + } + String nextPage = elem.attr("href"); + // Some times this returns a empty string + // This for stops that + if (nextPage.equals("")) { + throw new IOException("No more pages"); + } + else { + return Http.url("https://smutty.com" + nextPage).get(); + } + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); } @Override @@ -117,6 +110,12 @@ public class SmuttyRipper extends AlbumRipper { if (m.matches()) { return m.group(1).replace("%23", ""); } + + p = Pattern.compile("^https?://smutty.com/user/([a-zA-Z0-9\\-_]+)/?$"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } throw new MalformedURLException("Expected tag in URL (smutty.com/h/tag and not " + url); } From c500f20c02e5ab11931c19e1bf8bdd89b85ad196 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 6 May 2018 06:30:17 -0400 Subject: [PATCH 51/57] Added unit test for SmuttyRipper --- .../ripme/tst/ripper/rippers/SmuttyRipperTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java new file mode 100644 index 00000000..949e715f --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.SmuttyRipper; + +public class SmuttyRipperTest extends RippersTest { + public void testRip() throws IOException { + SmuttyRipper ripper = new SmuttyRipper(new URL("https://smutty.com/user/QUIGON/")); + testRipper(ripper); + } +} From 857b951956e44a76080de1255854e931c8e4fd9a Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 8 May 2018 07:28:18 -0400 Subject: [PATCH 52/57] Fixed regex --- .../com/rarchives/ripme/ripper/rippers/InstagramRipper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index 2b63a44f..717aaa30 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -430,7 +430,7 @@ public class InstagramRipper extends AbstractHTMLRipper { return null; } if (!rippingTag) { - Pattern jsP = Pattern.compile("o},queryId:.([a-zA-Z0-9]+)."); + Pattern jsP = Pattern.compile("\\?n.pagination:n},queryId:.([a-zA-Z0-9]+)."); Matcher m = jsP.matcher(sb.toString()); if (m.find()) { return m.group(1); @@ -442,7 +442,7 @@ public class InstagramRipper extends AbstractHTMLRipper { return m.group(1); } } - logger.info("Could not find query_hash on " + jsFileURL); + logger.error("Could not find query_hash on " + jsFileURL); return null; } From 925df36e3138b055dbff94833365ec7956184dc3 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 8 May 2018 08:41:39 -0400 Subject: [PATCH 53/57] 1.7.42: Added user support to SmuttyRipper; Removed vine ripper; Fixed NudeGalsRipper; addURLToDownload improvments; Fixed Instagram ripper --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 9bd3de11..41826a8e 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.41 + 1.7.42 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index ea3feafa..00677389 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.41", + "latestVersion": "1.7.42", "changeList": [ + "1.7.42: Added user support to SmuttyRipper; Removed vine ripper; Fixed NudeGalsRipper; addURLToDownload improvments; Fixed Instagram ripper", "1.7.41: Added support for spyingwithlana.com; Added ManganeloRipper; Added support for dynasty-scans.com", "1.7.40: Added hypnohub.net ripper; Fixed rule34.xxx ripper; Tsumino Ripper now add .png to filenames", "1.7.39: Added rule34.xxx ripper; Added Gfycatporntube.com ripper; Fixed AbstractRipper subdir bug; Added AbstractRipper unit tests", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 92a75c78..7b73f410 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.41"; + private static final String DEFAULT_VERSION = "1.7.42"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 52d14559e286939387b2ae26dedd4769ca5a0df8 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 9 May 2018 02:35:30 -0400 Subject: [PATCH 54/57] Fixed queryId regex --- .../com/rarchives/ripme/ripper/rippers/InstagramRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index 717aaa30..9c233b0d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -430,7 +430,7 @@ public class InstagramRipper extends AbstractHTMLRipper { return null; } if (!rippingTag) { - Pattern jsP = Pattern.compile("\\?n.pagination:n},queryId:.([a-zA-Z0-9]+)."); + Pattern jsP = Pattern.compile("o},queryId:.([a-zA-Z0-9]+)."); Matcher m = jsP.matcher(sb.toString()); if (m.find()) { return m.group(1); From c4eb31f1e9cbad13b80628fd0c1e20d96ef22ae3 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 9 May 2018 03:09:41 -0400 Subject: [PATCH 55/57] 1.7.43: Fixed queryId regex in instagram ripper --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 41826a8e..8566f2d6 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.42 + 1.7.43 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 00677389..6c4f3fc0 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.42", + "latestVersion": "1.7.43", "changeList": [ + "1.7.43: Fixed queryId regex in instagram ripper", "1.7.42: Added user support to SmuttyRipper; Removed vine ripper; Fixed NudeGalsRipper; addURLToDownload improvments; Fixed Instagram ripper", "1.7.41: Added support for spyingwithlana.com; Added ManganeloRipper; Added support for dynasty-scans.com", "1.7.40: Added hypnohub.net ripper; Fixed rule34.xxx ripper; Tsumino Ripper now add .png to filenames", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 7b73f410..4284eacc 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.42"; + private static final String DEFAULT_VERSION = "1.7.43"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From fbde7b9f97450a60a8a9ad489b76b9914877039a Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 10 May 2018 06:40:29 -0400 Subject: [PATCH 56/57] Fixed instagram ripper regex --- .../com/rarchives/ripme/ripper/rippers/InstagramRipper.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index 9c233b0d..efc4cb40 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -435,6 +435,12 @@ public class InstagramRipper extends AbstractHTMLRipper { if (m.find()) { return m.group(1); } + jsP = Pattern.compile("n.pagination:n},queryId:.([a-zA-Z0-9]+)."); + m = jsP.matcher(sb.toString()); + if (m.find()) { + return m.group(1); + } + } else { Pattern jsP = Pattern.compile("return e.tagMedia.byTagName.get\\(t\\).pagination},queryId:.([a-zA-Z0-9]+)."); Matcher m = jsP.matcher(sb.toString()); From 19e4a95c964a7dd0ac3cd42d50d72aa55c0083f8 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 10 May 2018 07:30:05 -0400 Subject: [PATCH 57/57] 1.7.44: Fixed instagram ripper regex --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 8566f2d6..0796e60f 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.43 + 1.7.44 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 6c4f3fc0..757058ae 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.43", + "latestVersion": "1.7.44", "changeList": [ + "1.7.44: Fixed instagram ripper regex", "1.7.43: Fixed queryId regex in instagram ripper", "1.7.42: Added user support to SmuttyRipper; Removed vine ripper; Fixed NudeGalsRipper; addURLToDownload improvments; Fixed Instagram ripper", "1.7.41: Added support for spyingwithlana.com; Added ManganeloRipper; Added support for dynasty-scans.com", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 4284eacc..11d18771 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.43"; + private static final String DEFAULT_VERSION = "1.7.44"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar";