From acb6356b946b433db563c2fda9ee6808a6b64f6f Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 17 Mar 2018 19:52:47 -0400 Subject: [PATCH 001/140] Hentaifoundry Ripper no longer errors out when there is no next page --- .../rarchives/ripme/ripper/rippers/HentaifoundryRipper.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java index 561c4249..c0031548 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java @@ -68,8 +68,7 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { @Override public Document getNextPage(Document doc) throws IOException { - if (doc.select("li.next.hidden").size() != 0) { - // Last page + if (doc.select("li.next > a").size() == 0) { throw new IOException("No more pages"); } Elements els = doc.select("li.next > a"); From 0f61dd00f0f88d12da327d946c8d675e3a65dfbf Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 21 Mar 2018 23:59:19 -0400 Subject: [PATCH 002/140] Reddit ripper now gets erome links --- .../ripme/ripper/rippers/RedditRipper.java | 6 +++ .../com/rarchives/ripme/utils/RipUtils.java | 40 +++++++++++++------ 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index 52e9a6d2..e8798476 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -4,10 +4,13 @@ import java.io.File; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.rarchives.ripme.ripper.AbstractRipper; +import com.rarchives.ripme.ripper.rippers.video.GfycatRipper; import org.json.JSONArray; import org.json.JSONObject; import org.json.JSONTokener; @@ -17,6 +20,9 @@ import com.rarchives.ripme.ui.UpdateUtils; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.RipUtils; import com.rarchives.ripme.utils.Utils; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; public class RedditRipper extends AlbumRipper { diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index b7b8c239..01d20e7c 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -9,19 +9,18 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.rarchives.ripme.ripper.AbstractRipper; +import com.rarchives.ripme.ripper.rippers.EroShareRipper; +import com.rarchives.ripme.ripper.rippers.EromeRipper; +import com.rarchives.ripme.ripper.rippers.ImgurRipper; +import com.rarchives.ripme.ripper.rippers.VidbleRipper; +import com.rarchives.ripme.ripper.rippers.video.GfycatRipper; import org.apache.commons.lang.math.NumberUtils; import org.apache.log4j.Logger; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import com.rarchives.ripme.ripper.AbstractRipper; -import com.rarchives.ripme.ripper.rippers.ImgurRipper; -import com.rarchives.ripme.ripper.rippers.ImgurRipper.ImgurAlbum; -import com.rarchives.ripme.ripper.rippers.ImgurRipper.ImgurImage; -import com.rarchives.ripme.ripper.rippers.VidbleRipper; -import com.rarchives.ripme.ripper.rippers.video.GfycatRipper; -import com.rarchives.ripme.ripper.rippers.EroShareRipper; public class RipUtils { private static final Logger logger = Logger.getLogger(RipUtils.class); @@ -35,8 +34,8 @@ public class RipUtils { && url.toExternalForm().contains("imgur.com/a/")) { try { logger.debug("Fetching imgur album at " + url); - ImgurAlbum imgurAlbum = ImgurRipper.getImgurAlbum(url); - for (ImgurImage imgurImage : imgurAlbum.images) { + ImgurRipper.ImgurAlbum imgurAlbum = ImgurRipper.getImgurAlbum(url); + for (ImgurRipper.ImgurImage imgurImage : imgurAlbum.images) { logger.debug("Got imgur image: " + imgurImage.url); result.add(imgurImage.url); } @@ -49,8 +48,8 @@ public class RipUtils { // Imgur image series. try { logger.debug("Fetching imgur series at " + url); - ImgurAlbum imgurAlbum = ImgurRipper.getImgurSeries(url); - for (ImgurImage imgurImage : imgurAlbum.images) { + ImgurRipper.ImgurAlbum imgurAlbum = ImgurRipper.getImgurSeries(url); + for (ImgurRipper.ImgurImage imgurImage : imgurAlbum.images) { logger.debug("Got imgur image: " + imgurImage.url); result.add(imgurImage.url); } @@ -91,6 +90,21 @@ public class RipUtils { return result; } + else if (url.toExternalForm().contains("erome.com")) { + try { + logger.info("Getting eroshare album " + url); + EromeRipper r = new EromeRipper(url); + Document tempDoc = r.getFirstPage(); + for (String u : r.getURLsFromPage(tempDoc)) { + result.add(new URL(u)); + } + } catch (IOException e) { + // Do nothing + logger.warn("Exception while retrieving eroshare page:", e); + } + return result; + } + Pattern p = Pattern.compile("https?://i.reddituploads.com/([a-zA-Z0-9]+)\\?.*"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { @@ -122,8 +136,8 @@ public class RipUtils { try { // Fetch the page Document doc = Jsoup.connect(url.toExternalForm()) - .userAgent(AbstractRipper.USER_AGENT) - .get(); + .userAgent(AbstractRipper.USER_AGENT) + .get(); for (Element el : doc.select("meta")) { if (el.attr("name").equals("twitter:image:src")) { result.add(new URL(el.attr("content"))); From 06e566ac63ff1205ed84379f3aa1628c9ceaa7da Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 22 Mar 2018 00:01:51 -0400 Subject: [PATCH 003/140] Revert "Hentaifoundry Ripper no longer errors out when there is no next page" This reverts commit acb6356b946b433db563c2fda9ee6808a6b64f6f. --- .../rarchives/ripme/ripper/rippers/HentaifoundryRipper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java index c0031548..561c4249 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java @@ -68,7 +68,8 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { @Override public Document getNextPage(Document doc) throws IOException { - if (doc.select("li.next > a").size() == 0) { + if (doc.select("li.next.hidden").size() != 0) { + // Last page throw new IOException("No more pages"); } Elements els = doc.select("li.next > a"); From ace84b868befbdb4e96f85cec99bd85f52342306 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 23 Mar 2018 23:06:43 -0400 Subject: [PATCH 004/140] IG ripper now uses display_url when downloading images --- .../com/rarchives/ripme/ripper/rippers/InstagramRipper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index bb2998f8..b38c2c08 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -284,9 +284,9 @@ public class InstagramRipper extends AbstractHTMLRipper { if (imageURLs.size() == 0) { // We add this one item to the array because either wise // the ripper will error out because we returned an empty array - imageURLs.add(getOriginalUrl(data.getString("thumbnail_src"))); + imageURLs.add(getOriginalUrl(data.getString("display_url"))); } - addURLToDownload(new URL(getOriginalUrl(data.getString("thumbnail_src"))), image_date); + addURLToDownload(new URL(data.getString("display_url")), image_date); } else { if (!Utils.getConfigBoolean("instagram.download_images_only", false)) { addURLToDownload(new URL(getVideoFromPage(data.getString("shortcode"))), image_date); From fc0e3588b63229cfb4f6d8b1a0c7fe568484fc03 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 23 Mar 2018 23:50:44 -0400 Subject: [PATCH 005/140] 1.7.28: IG ripper now uses display_url when downloading images; Reddit ripper now gets erome links; Hentaifoundry Ripper no longer errors out when there is no next page --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 77bedc9c..49b98ee0 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.27 + 1.7.28 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 5f47ad6b..5c3fc239 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.27", + "latestVersion": "1.7.28", "changeList": [ + "1.7.28: IG ripper now uses display_url when downloading images; Reddit ripper now gets erome links; Hentaifoundry Ripper no longer errors out when there is no next page", "1.7.27: IG ripper can now rip from tags; fixed json parsing issues", "1.7.26: fixed instagram ripper", "1.7.25: Fixed instagram ripper; Added an option to use short names for 8muses; Added tsuminoRipper; Added support for incase.buttsmithy.com", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 80282ccb..9a34cf98 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.27"; + private static final String DEFAULT_VERSION = "1.7.28"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 8237841df43782474a64ee142a3ff394627ca258 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 25 Mar 2018 01:56:07 -0400 Subject: [PATCH 006/140] Fixed eroshae ripper --- .../rarchives/ripme/ripper/rippers/EroShareRipper.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java index 30dcfd4f..e7019178 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java @@ -119,7 +119,7 @@ public class EroShareRipper extends AbstractHTMLRipper { for (Element img : imgs) { if (img.hasClass("album-image")) { String imageURL = img.attr("src"); - imageURL = "https:" + imageURL; + imageURL = imageURL; URLs.add(imageURL); } } @@ -129,7 +129,7 @@ public class EroShareRipper extends AbstractHTMLRipper { if (vid.hasClass("album-video")) { Elements source = vid.getElementsByTag("source"); String videoURL = source.first().attr("src"); - URLs.add("https:" + videoURL); + URLs.add(videoURL); } } // Profile videos @@ -148,7 +148,7 @@ public class EroShareRipper extends AbstractHTMLRipper { if (vid.hasClass("album-video")) { Elements source = vid.getElementsByTag("source"); String videoURL = source.first().attr("src"); - URLs.add("https:" + videoURL); + URLs.add(videoURL); } } } @@ -209,7 +209,6 @@ public class EroShareRipper extends AbstractHTMLRipper { for (Element img : imgs) { if (img.hasClass("album-image")) { String imageURL = img.attr("src"); - imageURL = "https:" + imageURL; URLs.add(new URL(imageURL)); } } @@ -219,7 +218,7 @@ public class EroShareRipper extends AbstractHTMLRipper { if (vid.hasClass("album-video")) { Elements source = vid.getElementsByTag("source"); String videoURL = source.first().attr("src"); - URLs.add(new URL("https:" + videoURL)); + URLs.add(new URL(videoURL)); } } From a88dcaad6acbbd9403bd76e275f15e03c1f9aeac Mon Sep 17 00:00:00 2001 From: dex-11 Date: Sun, 25 Mar 2018 09:42:38 +0200 Subject: [PATCH 007/140] Added modelx ripper --- .../ripme/ripper/rippers/ModelxRipper.java | 64 +++++++++++++++++++ .../tst/ripper/rippers/ModelxRipperTest.java | 13 ++++ 2 files changed, 77 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/ModelxRipper.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelxRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ModelxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ModelxRipper.java new file mode 100644 index 00000000..0b513b37 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ModelxRipper.java @@ -0,0 +1,64 @@ +package com.rarchives.ripme.ripper.rippers; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class ModelxRipper extends AbstractHTMLRipper { + + public ModelxRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "modelx"; + } + + @Override + public String getDomain() { + return "modelx.org"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^.*modelx.org/.*/(.+)$"); + Matcher m = p.matcher(url.toExternalForm()); + + if (m.matches()) { + return m.group(1); + } + + throw new MalformedURLException("Expected URL format: http://www.modelx.org/[category (one or more)]/xxxxx got: " + url); + } + + @Override + public Document getFirstPage() throws IOException { + return Http.url(url).get(); + } + + @Override + public List getURLsFromPage(Document page) { + List result = new ArrayList<>(); + + for (Element el : page.select(".gallery-icon > a")) { + result.add(el.attr("href")); + } + + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelxRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelxRipperTest.java new file mode 100644 index 00000000..2a0358d2 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelxRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.ModelxRipper; + +public class ModelxRipperTest extends RippersTest { + public void testModelxAlbum() throws IOException { + ModelxRipper ripper = new ModelxRipper(new URL("http://www.modelx.org/graphis-collection-2002-2016/ai-yuzuki-%e6%9f%9a%e6%9c%88%e3%81%82%e3%81%84-yuzuiro/")); + testRipper(ripper); + } +} \ No newline at end of file From b4832d32e8a6739cf7b6be5332c459adb5cf59db Mon Sep 17 00:00:00 2001 From: MrTimscampi Date: Fri, 30 Mar 2018 11:58:46 +0200 Subject: [PATCH 008/140] Improved handling of headless mode & OS-specific config This adds changes the check for command-lines arguments to avoir throwing an error when no argument is passed on an headless system. It will instead gracefully show the help page to the user. It also adds a check for the Mac OS-specific properties, to avoid setting them on other systems. The OS-specific properties and the app initialization have been moved to after the Headless check for cleanliness (No need to set GUI properties on an headless app and no need to initialize if we're only going to show an help page). --- src/main/java/com/rarchives/ripme/App.java | 48 ++++++++++++------- .../ripme/ripper/rippers/FacebookRipper.java | 4 ++ 2 files changed, 36 insertions(+), 16 deletions(-) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/FacebookRipper.java diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index 0bb5f3f6..9a6b1d09 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -1,5 +1,6 @@ package com.rarchives.ripme; +import java.awt.*; import java.io.File; import java.io.IOException; import java.io.BufferedReader; @@ -18,6 +19,7 @@ import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import org.apache.commons.lang.SystemUtils; import org.apache.log4j.Logger; import com.rarchives.ripme.ripper.AbstractRipper; @@ -34,35 +36,34 @@ import com.rarchives.ripme.utils.Utils; */ public class App { - public static final Logger logger; + public static final Logger logger = Logger.getLogger(App.class); private static final History HISTORY = new History(); - static { - //initialize logger - Utils.configureLogger(); - logger = Logger.getLogger(App.class); - } - public static void main(String[] args) throws MalformedURLException { CommandLine cl = getArgs(args); + if (args.length > 0 && cl.hasOption('v')){ - logger.error(UpdateUtils.getThisJarVersion()); + logger.info(UpdateUtils.getThisJarVersion()); System.exit(0); } - System.setProperty("apple.laf.useScreenMenuBar", "true"); - System.setProperty("com.apple.mrj.application.apple.menu.about.name", "RipMe"); - logger.info("Initialized ripme v" + UpdateUtils.getThisJarVersion()); - - if (args.length > 0) { - // CLI Mode + if (GraphicsEnvironment.isHeadless()) { handleArguments(args); } else { - // GUI Mode + if (SystemUtils.IS_OS_MAC_OSX) { + System.setProperty("apple.laf.useScreenMenuBar", "true"); + System.setProperty("com.apple.mrj.application.apple.menu.about.name", "RipMe"); + } + + Utils.configureLogger(); + + logger.info("Initialized ripme v" + UpdateUtils.getThisJarVersion()); + MainWindow mw = new MainWindow(); SwingUtilities.invokeLater(mw); } } + /** * Creates an abstract ripper and instructs it to rip. * @param url URL to be ripped @@ -80,20 +81,28 @@ public class App { */ private static void handleArguments(String[] args) { CommandLine cl = getArgs(args); - if (cl.hasOption('h')) { + + if (cl.hasOption('h') || args.length == 0) { HelpFormatter hf = new HelpFormatter(); hf.printHelp("java -jar ripme.jar [OPTIONS]", getOptions()); System.exit(0); } + + Utils.configureLogger(); + logger.info("Initialized ripme v" + UpdateUtils.getThisJarVersion()); + if (cl.hasOption('w')) { Utils.setConfigBoolean("file.overwrite", true); } + if (cl.hasOption('t')) { Utils.setConfigInteger("threads.size", Integer.parseInt(cl.getOptionValue('t'))); } + if (cl.hasOption('4')) { Utils.setConfigBoolean("errors.skip404", true); } + if (cl.hasOption('r')) { // Re-rip all via command-line List history = Utils.getConfigList("download.history"); @@ -115,6 +124,7 @@ public class App { // Exit System.exit(0); } + if (cl.hasOption('R')) { loadHistory(); if (HISTORY.toList().isEmpty()) { @@ -146,20 +156,25 @@ public class App { System.exit(-1); } } + if (cl.hasOption('d')) { Utils.setConfigBoolean("download.save_order", true); } + if (cl.hasOption('D')) { Utils.setConfigBoolean("download.save_order", false); } + if ((cl.hasOption('d'))&&(cl.hasOption('D'))) { logger.error("\nCannot specify '-d' and '-D' simultaneously"); System.exit(-1); } + if (cl.hasOption('l')) { // change the default rips directory Utils.setConfigString("rips.directory", cl.getOptionValue('l')); } + if (cl.hasOption('f')) { String filename = cl.getOptionValue('f'); try { @@ -175,6 +190,7 @@ public class App { logger.error("[!] Failed reading file containing list of URLs. Cannot continue."); } } + if (cl.hasOption('u')) { String url = cl.getOptionValue('u').trim(); ripURL(url, cl.hasOption("n")); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FacebookRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FacebookRipper.java new file mode 100644 index 00000000..acbf7a2e --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FacebookRipper.java @@ -0,0 +1,4 @@ +package com.rarchives.ripme.ripper.rippers; + +public class FacebookRipper { +} From b5abdb6ae057484b25c9724a1c6835e35d753eeb Mon Sep 17 00:00:00 2001 From: MrTimscampi Date: Fri, 30 Mar 2018 12:15:20 +0200 Subject: [PATCH 009/140] Removed empty file comitted by accident --- .../com/rarchives/ripme/ripper/rippers/FacebookRipper.java | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/FacebookRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FacebookRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FacebookRipper.java deleted file mode 100644 index acbf7a2e..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FacebookRipper.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -public class FacebookRipper { -} From 3716bbfb5414abfb30de77bfb19478b7d569d0e6 Mon Sep 17 00:00:00 2001 From: avesther Date: Mon, 2 Apr 2018 14:19:32 +0200 Subject: [PATCH 010/140] Fixes #410 Imgur - Cannot download albums with a single image in --- .../ripme/ripper/rippers/ImgurRipper.java | 138 +++++++++++------- .../tst/ripper/rippers/ImgurRipperTest.java | 16 +- 2 files changed, 100 insertions(+), 54 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java index b595d9e2..fe7937d3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java @@ -36,6 +36,7 @@ public class ImgurRipper extends AlbumRipper { USER, USER_ALBUM, USER_IMAGES, + SINGLE_IMAGE, SERIES_OF_IMAGES, SUBREDDIT } @@ -155,34 +156,48 @@ public class ImgurRipper extends AlbumRipper { @Override public void rip() throws IOException { switch (albumType) { - case ALBUM: - // Fall-through - case USER_ALBUM: - logger.info("Album type is USER_ALBUM"); - // Don't call getAlbumTitle(this.url) with this - // as it seems to cause the album to be downloaded to a subdir. - ripAlbum(this.url); - break; - case SERIES_OF_IMAGES: - logger.info("Album type is SERIES_OF_IMAGES"); - ripAlbum(this.url); - break; - case USER: - logger.info("Album type is USER"); - ripUserAccount(url); - break; - case SUBREDDIT: - logger.info("Album type is SUBREDDIT"); - ripSubreddit(url); - break; - case USER_IMAGES: - logger.info("Album type is USER_IMAGES"); - ripUserImages(url); - break; + case ALBUM: + // Fall-through + case USER_ALBUM: + logger.info("Album type is USER_ALBUM"); + // Don't call getAlbumTitle(this.url) with this + // as it seems to cause the album to be downloaded to a subdir. + ripAlbum(this.url); + break; + case SERIES_OF_IMAGES: + logger.info("Album type is SERIES_OF_IMAGES"); + ripAlbum(this.url); + break; + case SINGLE_IMAGE: + logger.info("Album type is SINGLE_IMAGE"); + ripSingleImage(this.url); + break; + case USER: + logger.info("Album type is USER"); + ripUserAccount(url); + break; + case SUBREDDIT: + logger.info("Album type is SUBREDDIT"); + ripSubreddit(url); + break; + case USER_IMAGES: + logger.info("Album type is USER_IMAGES"); + ripUserImages(url); + break; } waitForThreads(); } + private void ripSingleImage(URL url) throws IOException { + String strUrl = url.toExternalForm(); + Document document = getDocument(strUrl); + Matcher m = getEmbeddedJsonMatcher(document); + if (m.matches()) { + JSONObject json = new JSONObject(m.group(1)).getJSONObject("image"); + addURLToDownload(extractImageUrlFromJson(json), ""); + } + } + private void ripAlbum(URL url) throws IOException { ripAlbum(url, ""); } @@ -257,38 +272,16 @@ public class ImgurRipper extends AlbumRipper { strUrl += "/all"; } logger.info(" Retrieving " + strUrl); - Document doc = Jsoup.connect(strUrl) - .userAgent(USER_AGENT) - .timeout(10 * 1000) - .maxBodySize(0) - .get(); - + Document doc = getDocument(strUrl); // Try to use embedded JSON to retrieve images - Pattern p = Pattern.compile("^.*widgetFactory.mergeConfig\\('gallery', (.*?)\\);.*$", Pattern.DOTALL); - Matcher m = p.matcher(doc.body().html()); + Matcher m = getEmbeddedJsonMatcher(doc); if (m.matches()) { try { - ImgurAlbum imgurAlbum = new ImgurAlbum(url); JSONObject json = new JSONObject(m.group(1)); - JSONArray images = json.getJSONObject("image") + JSONArray jsonImages = json.getJSONObject("image") .getJSONObject("album_images") .getJSONArray("images"); - int imagesLength = images.length(); - for (int i = 0; i < imagesLength; i++) { - JSONObject image = images.getJSONObject(i); - String ext = image.getString("ext"); - if (ext.equals(".gif") && Utils.getConfigBoolean("prefer.mp4", false)) { - ext = ".mp4"; - } - URL imageURL = new URL( - "http://i.imgur.com/" - + image.getString("hash") - + ext); - ImgurImage imgurImage = new ImgurImage(imageURL); - imgurImage.extension = ext; - imgurAlbum.addImage(imgurImage); - } - return imgurAlbum; + return createImgurAlbumFromJsonArray(url, jsonImages); } catch (JSONException e) { logger.debug("Error while parsing JSON at " + url + ", continuing", e); } @@ -330,6 +323,44 @@ public class ImgurRipper extends AlbumRipper { return imgurAlbum; } + private static Matcher getEmbeddedJsonMatcher(Document doc) { + Pattern p = Pattern.compile("^.*widgetFactory.mergeConfig\\('gallery', (.*?)\\);.*$", Pattern.DOTALL); + return p.matcher(doc.body().html()); + } + + private static ImgurAlbum createImgurAlbumFromJsonArray(URL url, JSONArray jsonImages) throws MalformedURLException { + ImgurAlbum imgurAlbum = new ImgurAlbum(url); + int imagesLength = jsonImages.length(); + for (int i = 0; i < imagesLength; i++) { + JSONObject jsonImage = jsonImages.getJSONObject(i); + imgurAlbum.addImage(createImgurImageFromJson(jsonImage)); + } + return imgurAlbum; + } + + private static ImgurImage createImgurImageFromJson(JSONObject json) throws MalformedURLException { + return new ImgurImage(extractImageUrlFromJson(json)); + } + + private static URL extractImageUrlFromJson(JSONObject json) throws MalformedURLException { + String ext = json.getString("ext"); + if (ext.equals(".gif") && Utils.getConfigBoolean("prefer.mp4", false)) { + ext = ".mp4"; + } + return new URL( + "http://i.imgur.com/" + + json.getString("hash") + + ext); + } + + private static Document getDocument(String strUrl) throws IOException { + return Jsoup.connect(strUrl) + .userAgent(USER_AGENT) + .timeout(10 * 1000) + .maxBodySize(0) + .get(); + } + /** * Rips all albums in an imgur user's account. * @param url @@ -507,6 +538,13 @@ public class ImgurRipper extends AlbumRipper { this.url = new URL("http://imgur.com/r/" + subreddit + "/" + gid); return "r_" + subreddit + "_" + gid; } + p = Pattern.compile("^https?://(i\\.|www\\.|m\\.)?imgur\\.com/([a-zA-Z0-9]{5,})$"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + // Single imgur image + albumType = ALBUM_TYPE.SINGLE_IMAGE; + return m.group(m.groupCount()); + } p = Pattern.compile("^https?://(i\\.|www\\.|m\\.)?imgur\\.com/([a-zA-Z0-9,]{5,}).*$"); m = p.matcher(url.toExternalForm()); if (m.matches()) { diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java index 46f5679f..c321a99e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java @@ -1,13 +1,13 @@ package com.rarchives.ripme.tst.ripper.rippers; +import com.rarchives.ripme.ripper.rippers.ImgurRipper; +import com.rarchives.ripme.ripper.rippers.ImgurRipper.ImgurAlbum; + import java.io.IOException; import java.net.URL; import java.util.ArrayList; import java.util.List; -import com.rarchives.ripme.ripper.rippers.ImgurRipper; -import com.rarchives.ripme.ripper.rippers.ImgurRipper.ImgurAlbum; - public class ImgurRipperTest extends RippersTest { public void testImgurURLFailures() throws IOException { @@ -17,7 +17,6 @@ public class ImgurRipperTest extends RippersTest { failURLs.add(new URL("http://imgur.com/")); failURLs.add(new URL("http://i.imgur.com")); failURLs.add(new URL("http://i.imgur.com/")); - failURLs.add(new URL("http://imgur.com/image")); failURLs.add(new URL("http://imgur.com/image.jpg")); failURLs.add(new URL("http://i.imgur.com/image.jpg")); for (URL url : failURLs) { @@ -50,6 +49,15 @@ public class ImgurRipperTest extends RippersTest { } } + public void testImgurSingleImage() throws IOException { + List contentURLs = new ArrayList<>(); + contentURLs.add(new URL("http://imgur.com/qbfcLyG")); // Single image URL + contentURLs.add(new URL("https://imgur.com/KexUO")); // Single image URL + for (URL url : contentURLs) { + ImgurRipper ripper = new ImgurRipper(url); + testRipper(ripper); + } + } public void testImgurAlbumWithMoreThan20Pictures() throws IOException { ImgurAlbum album = ImgurRipper.getImgurAlbum(new URL("http://imgur.com/a/HUMsq")); From 338edd18f9e194cfb1fd2718802443da86009cfa Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Mon, 2 Apr 2018 16:48:32 -0400 Subject: [PATCH 011/140] 1.7.29: Cano now download single images from imgur; Improved handling of headless mode & OS-specific config; Added modelx ripper; Fixed eroshae ripper --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 49b98ee0..842dbe01 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.28 + 1.7.29 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 5c3fc239..50040faf 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.28", + "latestVersion": "1.7.29", "changeList": [ + "1.7.29: Cano now download single images from imgur; Improved handling of headless mode & OS-specific config; Added modelx ripper; Fixed eroshae ripper", "1.7.28: IG ripper now uses display_url when downloading images; Reddit ripper now gets erome links; Hentaifoundry Ripper no longer errors out when there is no next page", "1.7.27: IG ripper can now rip from tags; fixed json parsing issues", "1.7.26: fixed instagram ripper", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 9a34cf98..ced70f44 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.28"; + private static final String DEFAULT_VERSION = "1.7.29"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 9e138f2aad0bd3a5b327d25275575160c3679255 Mon Sep 17 00:00:00 2001 From: MrTimscampi Date: Tue, 3 Apr 2018 12:29:04 +0200 Subject: [PATCH 012/140] Fix usage of command-line on non-headless systems --- src/main/java/com/rarchives/ripme/App.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index 9a6b1d09..67c44ab1 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -47,7 +47,7 @@ public class App { System.exit(0); } - if (GraphicsEnvironment.isHeadless()) { + if (GraphicsEnvironment.isHeadless() || args.length > 0) { handleArguments(args); } else { if (SystemUtils.IS_OS_MAC_OSX) { From 24f07523086414256400f6b41af7f58350ed996b Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 3 Apr 2018 10:45:10 -0400 Subject: [PATCH 013/140] 1.7.30: Fixed usage of command-line on non-headless systems --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 842dbe01..20564bb4 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.29 + 1.7.30 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 50040faf..78ba8967 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.29", + "latestVersion": "1.7.30", "changeList": [ + "1.7.30: Fixed usage of command-line on non-headless systems", "1.7.29: Cano now download single images from imgur; Improved handling of headless mode & OS-specific config; Added modelx ripper; Fixed eroshae ripper", "1.7.28: IG ripper now uses display_url when downloading images; Reddit ripper now gets erome links; Hentaifoundry Ripper no longer errors out when there is no next page", "1.7.27: IG ripper can now rip from tags; fixed json parsing issues", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index ced70f44..ed8b8d6e 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.29"; + private static final String DEFAULT_VERSION = "1.7.30"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 7a1d6263b175b70fa53fa525a21de6077b2c38d8 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 7 Apr 2018 19:42:51 -0400 Subject: [PATCH 014/140] InstagramRipper now send ig_pr=1 cookie when getting next page --- .../rarchives/ripme/ripper/rippers/InstagramRipper.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index b38c2c08..4b87f992 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -24,6 +24,7 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.utils.Utils; +import java.util.HashMap; public class InstagramRipper extends AbstractHTMLRipper { @@ -316,12 +317,15 @@ public class InstagramRipper extends AbstractHTMLRipper { @Override public Document getNextPage(Document doc) throws IOException { Document toreturn; + java.util.Map cookies = new HashMap(); +// This shouldn't be hardcoded and will break one day + cookies.put("ig_pr", "1"); if (!nextPageID.equals("") && !isThisATest()) { if (rippingTag) { try { sleep(2500); toreturn = Http.url("https://www.instagram.com/graphql/query/?query_hash=" + qHash + - "&variables={\"tag_name\":\"" + tagName + "\",\"first\":4,\"after\":\"" + nextPageID + "\"}").ignoreContentType().get(); + "&variables={\"tag_name\":\"" + tagName + "\",\"first\":4,\"after\":\"" + nextPageID + "\"}").cookies(cookies).ignoreContentType().get(); // Sleep for a while to avoid a ban logger.info(toreturn.html()); return toreturn; @@ -335,7 +339,7 @@ public class InstagramRipper extends AbstractHTMLRipper { // Sleep for a while to avoid a ban sleep(2500); toreturn = Http.url("https://www.instagram.com/graphql/query/?query_hash=" + qHash + "&variables=" + - "{\"id\":\"" + userID + "\",\"first\":100,\"after\":\"" + nextPageID + "\"}").ignoreContentType().get(); + "{\"id\":\"" + userID + "\",\"first\":100,\"after\":\"" + nextPageID + "\"}").cookies(cookies).ignoreContentType().get(); if (!pageHasImages(toreturn)) { throw new IOException("No more pages"); } From 23fee2779200020b876b93395b73e804d6a69f3f Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 7 Apr 2018 20:00:14 -0400 Subject: [PATCH 015/140] 1.7.31: InstaGram ripper no longer errors out when getting next page --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 20564bb4..c6631f25 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.30 + 1.7.31 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 78ba8967..942daad4 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.30", + "latestVersion": "1.7.31", "changeList": [ + "1.7.31: InstaGram ripper no longer errors out when getting next page", "1.7.30: Fixed usage of command-line on non-headless systems", "1.7.29: Cano now download single images from imgur; Improved handling of headless mode & OS-specific config; Added modelx ripper; Fixed eroshae ripper", "1.7.28: IG ripper now uses display_url when downloading images; Reddit ripper now gets erome links; Hentaifoundry Ripper no longer errors out when there is no next page", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index ed8b8d6e..b02e7fed 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.30"; + private static final String DEFAULT_VERSION = "1.7.31"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 54f22eef3c4fd3056ce7fa5fed3ba9d266dbb89a Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 10 Apr 2018 18:23:31 -0400 Subject: [PATCH 016/140] Got getNextPage working again for non-tags --- .../ripme/ripper/rippers/InstagramRipper.java | 46 +++++++++++++++++-- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index 4b87f992..e4e4e39f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -3,6 +3,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; +import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLConnection; @@ -12,6 +13,7 @@ import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.security.*; import org.json.JSONArray; import org.json.JSONException; @@ -20,6 +22,7 @@ import org.json.JSONObject; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; +import org.jsoup.Connection; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import com.rarchives.ripme.ui.RipStatusMessage; @@ -34,6 +37,10 @@ public class InstagramRipper extends AbstractHTMLRipper { private String tagName; private String userID; + private String rhx_gis = null; + private String csrftoken; + + public InstagramRipper(URL url) throws IOException { super(url); @@ -178,7 +185,10 @@ public class InstagramRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { - Document p = Http.url(url).get(); + Connection.Response resp = Http.url(url).response(); + logger.info(resp.cookies()); + csrftoken = resp.cookie("csrftoken"); + Document p = resp.parse(); // Get the query hash so we can download the next page qHash = getQHash(p); return p; @@ -234,7 +244,10 @@ public class InstagramRipper extends AbstractHTMLRipper { logger.warn("Unable to exact json from page"); } - + // get the rhx_gis value so we can get the next page later on + if (rhx_gis == null) { + rhx_gis = json.getString("rhx_gis"); + } if (!url.toExternalForm().contains("/p/")) { JSONArray datas = new JSONArray(); if (!rippingTag) { @@ -314,12 +327,33 @@ public class InstagramRipper extends AbstractHTMLRipper { return imageURLs; } + private String getIGGis(String variables) { + String stringToMD5 = rhx_gis + ":" + csrftoken + ":" + USER_AGENT + ":" + variables; + logger.debug("String to md5 is \"" + stringToMD5 + "\""); + try { + byte[] bytesOfMessage = stringToMD5.getBytes("UTF-8"); + + MessageDigest md = MessageDigest.getInstance("MD5"); + byte[] hash = md.digest(bytesOfMessage); + StringBuffer sb = new StringBuffer(); + for (int i = 0; i < hash.length; ++i) { + sb.append(Integer.toHexString((hash[i] & 0xFF) | 0x100).substring(1,3)); + } + return sb.toString(); + } catch(UnsupportedEncodingException e) { + return null; + } catch(NoSuchAlgorithmException e) { + return null; + } + } + @Override public Document getNextPage(Document doc) throws IOException { Document toreturn; java.util.Map cookies = new HashMap(); // This shouldn't be hardcoded and will break one day cookies.put("ig_pr", "1"); + cookies.put("csrftoken", csrftoken); if (!nextPageID.equals("") && !isThisATest()) { if (rippingTag) { try { @@ -338,8 +372,11 @@ public class InstagramRipper extends AbstractHTMLRipper { try { // Sleep for a while to avoid a ban sleep(2500); - toreturn = Http.url("https://www.instagram.com/graphql/query/?query_hash=" + qHash + "&variables=" + - "{\"id\":\"" + userID + "\",\"first\":100,\"after\":\"" + nextPageID + "\"}").cookies(cookies).ignoreContentType().get(); + String vars = "{\"id\":\"" + userID + "\",\"first\":100,\"after\":\"" + nextPageID + "\"}"; + String ig_gis = getIGGis(vars); + logger.info(ig_gis); + toreturn = Http.url("https://www.instagram.com/graphql/query/?query_hash=" + qHash + "&variables=" + vars + ).header("x-instagram-gis", ig_gis).cookies(cookies).ignoreContentType().get(); if (!pageHasImages(toreturn)) { throw new IOException("No more pages"); } @@ -358,7 +395,6 @@ public class InstagramRipper extends AbstractHTMLRipper { } private boolean pageHasImages(Document doc) { - logger.info("BAD DATA: " + stripHTMLTags(doc.html())); JSONObject json = new JSONObject(stripHTMLTags(doc.html())); int numberOfImages = json.getJSONObject("data").getJSONObject("user") .getJSONObject("edge_owner_to_timeline_media").getJSONArray("edges").length(); From 73478344d8d5c6f43b55d134e727b6d8dc48452e Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 10 Apr 2018 18:28:13 -0400 Subject: [PATCH 017/140] Got getNextPage working again for tags --- .../com/rarchives/ripme/ripper/rippers/InstagramRipper.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index e4e4e39f..3f1433d0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -358,8 +358,10 @@ public class InstagramRipper extends AbstractHTMLRipper { if (rippingTag) { try { sleep(2500); + String vars = "{\"tag_name\":\"" + tagName + "\",\"first\":4,\"after\":\"" + nextPageID + "\"}"; + String ig_gis = getIGGis(vars); toreturn = Http.url("https://www.instagram.com/graphql/query/?query_hash=" + qHash + - "&variables={\"tag_name\":\"" + tagName + "\",\"first\":4,\"after\":\"" + nextPageID + "\"}").cookies(cookies).ignoreContentType().get(); + "&variables=" + vars).header("x-instagram-gis", ig_gis).cookies(cookies).ignoreContentType().get(); // Sleep for a while to avoid a ban logger.info(toreturn.html()); return toreturn; From 6b4d01af1d8f9c07413fb40c96047fda36a2488e Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 10 Apr 2018 18:52:52 -0400 Subject: [PATCH 018/140] 1.7.32: Instagram ripper update to use new enpoints --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index c6631f25..7e7b1c05 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.31 + 1.7.32 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 942daad4..6770542a 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.31", + "latestVersion": "1.7.32", "changeList": [ + "1.7.32: Instagram ripper update to use new enpoints", "1.7.31: InstaGram ripper no longer errors out when getting next page", "1.7.30: Fixed usage of command-line on non-headless systems", "1.7.29: Cano now download single images from imgur; Improved handling of headless mode & OS-specific config; Added modelx ripper; Fixed eroshae ripper", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index b02e7fed..8a05aaa4 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.31"; + private static final String DEFAULT_VERSION = "1.7.32"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 2871805b5da83ee73a33aa42941f48962637eda6 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 13 Apr 2018 05:45:22 -0400 Subject: [PATCH 019/140] Instagram ripper no longer errors out when downloading from more than 1 page --- .../com/rarchives/ripme/ripper/rippers/InstagramRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index 3f1433d0..12842aa8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -328,7 +328,7 @@ public class InstagramRipper extends AbstractHTMLRipper { } private String getIGGis(String variables) { - String stringToMD5 = rhx_gis + ":" + csrftoken + ":" + USER_AGENT + ":" + variables; + String stringToMD5 = rhx_gis + ":" + csrftoken + ":" + variables; logger.debug("String to md5 is \"" + stringToMD5 + "\""); try { byte[] bytesOfMessage = stringToMD5.getBytes("UTF-8"); From fd0db7e2424348345d1d3fb18d942826417229ed Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 13 Apr 2018 05:49:47 -0400 Subject: [PATCH 020/140] 1.7.33: Instagram ripper no longer errors out when downloading from more than 1 page --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 7e7b1c05..6b7084c8 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.32 + 1.7.33 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 6770542a..8dc6e2cf 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.32", + "latestVersion": "1.7.33", "changeList": [ + "1.7.33: Instagram ripper no longer errors out when downloading from more than 1 page", "1.7.32: Instagram ripper update to use new enpoints", "1.7.31: InstaGram ripper no longer errors out when getting next page", "1.7.30: Fixed usage of command-line on non-headless systems", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 8a05aaa4..e1d579a4 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.32"; + private static final String DEFAULT_VERSION = "1.7.33"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 1bd9885fbaaa594844b883ef0d4d5f10eb3a1281 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 13 Apr 2018 18:33:31 -0400 Subject: [PATCH 021/140] Added ripper for Blackbrickroadofoz.com --- .../ripper/rippers/Blackbrickroadofoz.java | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/Blackbrickroadofoz.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Blackbrickroadofoz.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Blackbrickroadofoz.java new file mode 100644 index 00000000..c290170f --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Blackbrickroadofoz.java @@ -0,0 +1,80 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class Blackbrickroadofoz extends AbstractHTMLRipper { + + public Blackbrickroadofoz(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "cfake"; + } + + @Override + public String getDomain() { + return "cfake.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https?://www.blackbrickroadofoz.com/comic/([a-zA-Z0-9_-]*)/?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected blackbrickroadofoz URL format: " + + "www.blackbrickroadofoz.com/comic/PAGE - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + @Override + public Document getNextPage(Document doc) throws IOException { + Element elem = doc.select("nav.cc-nav > a.cc-next").first(); + if (elem == null) { + throw new IOException("No more pages"); + } + String nextPage = elem.attr("href"); + // Some times this returns a empty string + // This for stops that + if (nextPage == "") { + throw new IOException("No more pages"); + } + else { + return Http.url(nextPage).get(); + } + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + Element elem = doc.select("img[id=cc-comic]").first(); + result.add(elem.attr("src")); + + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} From d8254b6e03c84551e6de206bcb3ef52bac601505 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 13 Apr 2018 18:52:38 -0400 Subject: [PATCH 022/140] Fixed url encoding error with Blackbrickroadofoz --- .../ripme/ripper/rippers/Blackbrickroadofoz.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Blackbrickroadofoz.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Blackbrickroadofoz.java index c290170f..2c9b14eb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/Blackbrickroadofoz.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Blackbrickroadofoz.java @@ -22,12 +22,12 @@ public class Blackbrickroadofoz extends AbstractHTMLRipper { @Override public String getHost() { - return "cfake"; + return "blackbrickroadofoz"; } @Override public String getDomain() { - return "cfake.com"; + return "blackbrickroadofoz.com"; } @Override @@ -49,7 +49,8 @@ public class Blackbrickroadofoz extends AbstractHTMLRipper { @Override public Document getNextPage(Document doc) throws IOException { - Element elem = doc.select("nav.cc-nav > a.cc-next").first(); + sleep(1000); + Element elem = doc.select("div[id=topnav] > nav.cc-nav > a.cc-next").first(); if (elem == null) { throw new IOException("No more pages"); } @@ -67,8 +68,9 @@ public class Blackbrickroadofoz extends AbstractHTMLRipper { @Override public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); - Element elem = doc.select("img[id=cc-comic]").first(); - result.add(elem.attr("src")); + Element elem = doc.select("div[id=cc-comicbody] > a > img[id=cc-comic]").first(); + // The site doesn't return properly encoded urls we replace all spaces ( ) with %20 + result.add(elem.attr("src").replaceAll(" ", "%20")); return result; } From 8cd3f5d77a22c880f23a3d4c9b734db8bb796b95 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 13 Apr 2018 18:54:27 -0400 Subject: [PATCH 023/140] Renamed misnamed file --- ...{Blackbrickroadofoz.java => BlackbrickroadofozRipper.java} | 0 .../tst/ripper/rippers/BlackbrickroadofozRipperTest.java | 4 ++++ 2 files changed, 4 insertions(+) rename src/main/java/com/rarchives/ripme/ripper/rippers/{Blackbrickroadofoz.java => BlackbrickroadofozRipper.java} (100%) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/BlackbrickroadofozRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Blackbrickroadofoz.java b/src/main/java/com/rarchives/ripme/ripper/rippers/BlackbrickroadofozRipper.java similarity index 100% rename from src/main/java/com/rarchives/ripme/ripper/rippers/Blackbrickroadofoz.java rename to src/main/java/com/rarchives/ripme/ripper/rippers/BlackbrickroadofozRipper.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BlackbrickroadofozRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BlackbrickroadofozRipperTest.java new file mode 100644 index 00000000..2c479c36 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BlackbrickroadofozRipperTest.java @@ -0,0 +1,4 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +public class BlackbrickroadofozRipperTest { +} From d16424d936b12e0b49fe7919ef40a94e3a7702de Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 13 Apr 2018 18:55:19 -0400 Subject: [PATCH 024/140] Renamed misnamed class --- .../ripme/ripper/rippers/BlackbrickroadofozRipper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/BlackbrickroadofozRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/BlackbrickroadofozRipper.java index 2c9b14eb..76340cf9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/BlackbrickroadofozRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/BlackbrickroadofozRipper.java @@ -14,9 +14,9 @@ import org.jsoup.nodes.Element; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; -public class Blackbrickroadofoz extends AbstractHTMLRipper { +public class BlackbrickroadofozRipper extends AbstractHTMLRipper { - public Blackbrickroadofoz(URL url) throws IOException { + public BlackbrickroadofozRipper(URL url) throws IOException { super(url); } From cbc286170e4f78a793a4351657ea888683574feb Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 13 Apr 2018 18:57:57 -0400 Subject: [PATCH 025/140] Added unit test for Blackbrickroadofoz Ripper --- .../ripper/rippers/BlackbrickroadofozRipperTest.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BlackbrickroadofozRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BlackbrickroadofozRipperTest.java index 2c479c36..a3c7d862 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BlackbrickroadofozRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BlackbrickroadofozRipperTest.java @@ -1,4 +1,13 @@ package com.rarchives.ripme.tst.ripper.rippers; -public class BlackbrickroadofozRipperTest { +import com.rarchives.ripme.ripper.rippers.BlackbrickroadofozRipper; + +import java.io.IOException; +import java.net.URL; + +public class BlackbrickroadofozRipperTest extends RippersTest { + public void testRip() throws IOException { + BlackbrickroadofozRipper ripper = new BlackbrickroadofozRipper(new URL("http://www.blackbrickroadofoz.com/comic/beginning")); + testRipper(ripper); + } } From 692dcaa9906a34b0a51dc57fdd77269231cfaec6 Mon Sep 17 00:00:00 2001 From: Kevin Jiang Date: Sat, 14 Apr 2018 17:45:56 -0400 Subject: [PATCH 026/140] Travis CL test because I'm too lazy to run it on my own computer TEST --- .../ripper/rippers/DeviantartRipper.java | 47 ++++++++++++++----- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index 4461188e..225c9c23 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -1,5 +1,9 @@ package com.rarchives.ripme.ripper.rippers; +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Base64; +import com.rarchives.ripme.utils.Http; +import com.rarchives.ripme.utils.Utils; import java.io.IOException; import java.net.HttpURLConnection; import java.net.MalformedURLException; @@ -13,7 +17,6 @@ import java.util.Map; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; - import org.jsoup.Connection.Method; import org.jsoup.Connection.Response; import org.jsoup.Jsoup; @@ -22,11 +25,6 @@ import org.jsoup.nodes.Element; import org.jsoup.safety.Whitelist; import org.jsoup.select.Elements; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Base64; -import com.rarchives.ripme.utils.Http; -import com.rarchives.ripme.utils.Utils; - public class DeviantartRipper extends AbstractHTMLRipper { private static final int PAGE_SLEEP_TIME = 3000, @@ -108,19 +106,46 @@ public class DeviantartRipper extends AbstractHTMLRipper { throw new MalformedURLException("Expected URL format: http://username.deviantart.com/[/gallery/#####], got: " + url); } + /** + * Gets first page. + * Will determine if login is supplied, + * if there is a login, then login and add that login cookies. + * Otherwise, just bypass the age gate with an anonymous flag. + * @return + * @throws IOException + */ @Override public Document getFirstPage() throws IOException { - // Login - try { - cookies = loginToDeviantart(); - } catch (Exception e) { - logger.warn("Failed to login: ", e); + + //Test to see if there is a login: + String username = Utils.getConfigString("deviantart.username", new String(Base64.decode("Z3JhYnB5"))); + String password = Utils.getConfigString("deviantart.password", new String(Base64.decode("ZmFrZXJz"))); + + if(username == null || password == null) { + logger.debug("No DeviantArt login provided."); cookies.put("agegate_state","1"); // Bypasses the age gate + } else { + // Attempt Login + try { + cookies = loginToDeviantart(); + } catch (Exception e) { + logger.warn("Failed to login: ", e); + cookies.put("agegate_state","1"); // Bypasses the age gate + } } + + return Http.url(this.url) .cookies(cookies) .get(); } + + /** + * + * @param page + * @param id + * @return + */ private String jsonToImage(Document page, String id) { Elements js = page.select("script[type=\"text/javascript\"]"); for (Element tag : js) { From 264980ad94a526ac7399cfe86af4831c93db11f2 Mon Sep 17 00:00:00 2001 From: Kevin Jiang Date: Sat, 14 Apr 2018 17:55:45 -0400 Subject: [PATCH 027/140] More specific Exception name IOException > Exception --- .../com/rarchives/ripme/ripper/rippers/DeviantartRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index 225c9c23..8ecb8086 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -128,7 +128,7 @@ public class DeviantartRipper extends AbstractHTMLRipper { // Attempt Login try { cookies = loginToDeviantart(); - } catch (Exception e) { + } catch (IOException e) { logger.warn("Failed to login: ", e); cookies.put("agegate_state","1"); // Bypasses the age gate } From dc9604b1155bf58f8c0b91854a37b59d6dc1aae6 Mon Sep 17 00:00:00 2001 From: Kevin Jiang Date: Sat, 14 Apr 2018 18:21:29 -0400 Subject: [PATCH 028/140] Update README.md Fix #454 Remove link to ded website --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index e56276c1..66ef3428 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,6 @@ For information about running the `.jar` file, see [the How To Run wiki](https:/ ## [Changelog](https://github.com/ripmeapp/ripme/blob/master/ripme.json) (ripme.json) -## [Website](http://rip.rarchives.com/) - # Features * Quickly downloads all images in an online album (see supported sites below) From d02080cd3dc7e01d459630e72fc4c859a2eda478 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 15 Apr 2018 05:22:56 -0400 Subject: [PATCH 029/140] Fix webtoons canRip regex --- .../com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java index df373689..de785a44 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java @@ -36,7 +36,7 @@ public class WebtoonsRipper extends AbstractHTMLRipper { @Override public boolean canRip(URL url) { - Pattern pat = Pattern.compile("https?://www.webtoons.com/[a-zA-Z]+/[a-zA-Z]+/([a-zA-Z0-9_-]*)/[a-zA-Z0-9_-]+/\\S*"); + Pattern pat = Pattern.compile("https?://www.webtoons.com/[a-zA-Z-_]+/[a-zA-Z_-]+/([a-zA-Z0-9_-]*)/[a-zA-Z0-9_-]+/\\S*"); Matcher mat = pat.matcher(url.toExternalForm()); if (mat.matches()) { return true; @@ -47,7 +47,7 @@ public class WebtoonsRipper extends AbstractHTMLRipper { @Override public String getAlbumTitle(URL url) throws MalformedURLException { - Pattern pat = Pattern.compile("https?://www.webtoons.com/[a-zA-Z]+/[a-zA-Z]+/([a-zA-Z0-9_-]*)/[a-zA-Z0-9_-]+/\\S*"); + Pattern pat = Pattern.compile("https?://www.webtoons.com/[a-zA-Z-_]+/[a-zA-Z_-]+/([a-zA-Z0-9_-]*)/[a-zA-Z0-9_-]+/\\S*"); Matcher mat = pat.matcher(url.toExternalForm()); if (mat.matches()) { return getHost() + "_" + mat.group(1); From 5206f1d30e36e42cf575559cf5e33e887e5617b5 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 15 Apr 2018 05:45:41 -0400 Subject: [PATCH 030/140] 1.7.34: Added Blackbrickroadofoz Ripper; Fixed webtoons regex --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 6b7084c8..8ce4b5ae 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.33 + 1.7.34 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 8dc6e2cf..3a07f1b7 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.33", + "latestVersion": "1.7.34", "changeList": [ + "1.7.34: Added Blackbrickroadofoz Ripper; Fixed webtoons regex", "1.7.33: Instagram ripper no longer errors out when downloading from more than 1 page", "1.7.32: Instagram ripper update to use new enpoints", "1.7.31: InstaGram ripper no longer errors out when getting next page", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index e1d579a4..01b6c7eb 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.33"; + private static final String DEFAULT_VERSION = "1.7.34"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 36f85282aada106a1c487eaaa49914023c782008 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 17 Apr 2018 03:56:34 -0400 Subject: [PATCH 031/140] Fixed instagram ripper --- .../com/rarchives/ripme/ripper/rippers/InstagramRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index 12842aa8..9e0fdac5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -328,7 +328,7 @@ public class InstagramRipper extends AbstractHTMLRipper { } private String getIGGis(String variables) { - String stringToMD5 = rhx_gis + ":" + csrftoken + ":" + variables; + String stringToMD5 = rhx_gis + ":" + variables; logger.debug("String to md5 is \"" + stringToMD5 + "\""); try { byte[] bytesOfMessage = stringToMD5.getBytes("UTF-8"); From f8ae2212c0af05470c05d95cb35378efee4a18df Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 17 Apr 2018 04:11:13 -0400 Subject: [PATCH 032/140] Fixed Hentai2Read folder naming --- .../rarchives/ripme/ripper/rippers/Hentai2readRipper.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java index 6e76bbc6..db4cbd97 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java @@ -33,7 +33,7 @@ public class Hentai2readRipper extends AbstractHTMLRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("https://hentai2read\\.com/([a-zA-Z0-9_-]*)/?"); + Pattern p = Pattern.compile("https?://hentai2read\\.com/([a-zA-Z0-9_-]*)/\\d/?"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); @@ -63,9 +63,7 @@ public class Hentai2readRipper extends AbstractHTMLRipper { @Override public String getAlbumTitle(URL url) throws MalformedURLException { try { - Document doc = getFirstPage(); - String title = doc.select("span[itemprop=title]").text(); - return getHost() + "_" + title; + return getHost() + "_" + getGID(url); } catch (Exception e) { // Fall back to default album naming convention logger.warn("Failed to get album title from " + url, e); From 1dadd26e0584c95fd74debff39294bb68de90a20 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 17 Apr 2018 04:23:05 -0400 Subject: [PATCH 033/140] 1.7.35: Fixed instagram ripper; hentai2read ripper now properly names folders --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 8ce4b5ae..d00e1746 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.34 + 1.7.35 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 3a07f1b7..af817532 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.34", + "latestVersion": "1.7.35", "changeList": [ + "1.7.35: Fixed instagram ripper; hentai2read ripper now properly names folders", "1.7.34: Added Blackbrickroadofoz Ripper; Fixed webtoons regex", "1.7.33: Instagram ripper no longer errors out when downloading from more than 1 page", "1.7.32: Instagram ripper update to use new enpoints", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 01b6c7eb..48f1f941 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.34"; + private static final String DEFAULT_VERSION = "1.7.35"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From bf056a75d0ed081123c188dd2dc9b04f24a568f0 Mon Sep 17 00:00:00 2001 From: randomcommitter <38479873+randomcommitter@users.noreply.github.com> Date: Wed, 18 Apr 2018 16:33:09 +0100 Subject: [PATCH 034/140] Added support for SOCKS v5 Server CLI only, the switch is "-s". Accepted formats: user:password@host user:password@host:port host host:port --- src/main/java/com/rarchives/ripme/App.java | 28 ++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index 67c44ab1..508401d2 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -7,7 +7,9 @@ import java.io.BufferedReader; import java.io.FileReader; import java.io.FileNotFoundException; +import java.net.Authenticator; import java.net.MalformedURLException; +import java.net.PasswordAuthentication; import java.net.URL; import java.util.Arrays; import java.util.List; @@ -94,6 +96,30 @@ public class App { if (cl.hasOption('w')) { Utils.setConfigBoolean("file.overwrite", true); } + + if (cl.hasOption('s')) { + String sservfull = cl.getOptionValue('s').trim(); + if (sservfull.lastIndexOf("@") != -1) { + int sservli = sservfull.lastIndexOf("@"); + String userpw = sservfull.substring(0, sservli); + String[] usersplit = userpw.split(":"); + String user = usersplit[0]; + String password = usersplit[1]; + Authenticator.setDefault(new Authenticator(){ + protected PasswordAuthentication getPasswordAuthentication(){ + PasswordAuthentication p = new PasswordAuthentication(user, password.toCharArray()); + return p; + } + }); + + sservfull = sservfull.substring(sservli + 1); + } + String[] servsplit = sservfull.split(":"); + if (servsplit.length == 2) { + System.setProperty("socksProxyPort", servsplit[1]); + } + System.setProperty("socksProxyHost", servsplit[0]); + } if (cl.hasOption('t')) { Utils.setConfigInteger("threads.size", Integer.parseInt(cl.getOptionValue('t'))); @@ -195,6 +221,7 @@ public class App { String url = cl.getOptionValue('u').trim(); ripURL(url, cl.hasOption("n")); } + } /** @@ -242,6 +269,7 @@ public class App { opts.addOption("n", "no-prop-file", false, "Do not create properties file."); opts.addOption("f", "urls-file", true, "Rip URLs from a file."); opts.addOption("v", "version", false, "Show current version"); + opts.addOption("s", "socks-server", true, "Use socks server ([user:password]@host[:port]"); return opts; } From 9251ee0a255908224207e8227f01a702253a3334 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 03:42:19 -0400 Subject: [PATCH 035/140] Instagram ripper no longer throws a 400 error when getting new pages --- .../com/rarchives/ripme/ripper/rippers/InstagramRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index 9e0fdac5..2b63a44f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -374,7 +374,7 @@ public class InstagramRipper extends AbstractHTMLRipper { try { // Sleep for a while to avoid a ban sleep(2500); - String vars = "{\"id\":\"" + userID + "\",\"first\":100,\"after\":\"" + nextPageID + "\"}"; + String vars = "{\"id\":\"" + userID + "\",\"first\":50,\"after\":\"" + nextPageID + "\"}"; String ig_gis = getIGGis(vars); logger.info(ig_gis); toreturn = Http.url("https://www.instagram.com/graphql/query/?query_hash=" + qHash + "&variables=" + vars From fdf3e7c68deecd08b4fc0432c026f219571ecbc9 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 04:53:34 -0400 Subject: [PATCH 036/140] tnbtu ripper now sets referrer header --- .../ripme/ripper/rippers/WordpressComicRipper.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java index dbc44585..19fd4352 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java @@ -327,8 +327,14 @@ public class WordpressComicRipper extends AbstractHTMLRipper { || getHost().contains("themonsterunderthebed.net")) { addURLToDownload(url, pageTitle + "_"); } - // If we're ripping a site where we can't get the page number/title we just rip normally - addURLToDownload(url, getPrefix(index)); + if (getHost().contains("tnbtu.com")) { + // We need to set the referrer header for tnbtu + addURLToDownload(url, getPrefix(index), "","http://www.tnbtu.com/comic", null); + } else { + // If we're ripping a site where we can't get the page number/title we just rip normally + addURLToDownload(url, getPrefix(index)); + } + } @Override From 1e3b309f4b321ff63839a3be722f870eea54da30 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 04:58:53 -0400 Subject: [PATCH 037/140] Fixed hentai2read regex --- .../com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java index db4cbd97..405d0563 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java @@ -33,7 +33,7 @@ public class Hentai2readRipper extends AbstractHTMLRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("https?://hentai2read\\.com/([a-zA-Z0-9_-]*)/\\d/?"); + Pattern p = Pattern.compile("https?://hentai2read\\.com/([a-zA-Z0-9_-]*)/(\\d)?/?"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); From 6cd6e2ee151ad4d4f4dd78caf174ac6489d572cb Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 04:59:07 -0400 Subject: [PATCH 038/140] Add another unit test --- .../ripme/tst/ripper/rippers/Hentai2readRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java index 144606fc..b5765047 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java @@ -9,5 +9,7 @@ public class Hentai2readRipperTest extends RippersTest { public void testHentai2readAlbum() throws IOException { Hentai2readRipper ripper = new Hentai2readRipper(new URL("https://hentai2read.com/sm_school_memorial/")); testRipper(ripper); + ripper = new Hentai2readRipper(new URL("https://hentai2read.com/sm_school_memorial/1/")); + testRipper(ripper); } } \ No newline at end of file From 1bc696504ad0e09b111c816db1c3d3688492988e Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 05:58:50 -0400 Subject: [PATCH 039/140] 1.7.36: Fixed Instagram ripper; Fixed hentai2read ripper test; Fixed tnbtu.com ripper --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index d00e1746..ed4efbc6 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.35 + 1.7.36 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index af817532..7c31ed69 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.35", + "latestVersion": "1.7.36", "changeList": [ + "1.7.36: Fixed Instagram ripper; Fixed hentai2read ripper test; Fixed tnbtu.com ripper", "1.7.35: Fixed instagram ripper; hentai2read ripper now properly names folders", "1.7.34: Added Blackbrickroadofoz Ripper; Fixed webtoons regex", "1.7.33: Instagram ripper no longer errors out when downloading from more than 1 page", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 48f1f941..608e0c58 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.35"; + private static final String DEFAULT_VERSION = "1.7.36"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 034225cf1ceaef838e3be367474bfb606d41d50a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Thu, 19 Apr 2018 15:14:24 +0200 Subject: [PATCH 040/140] [tumblr] avoid 'httpss' in image URLs fixes #523 replacing 'http' with 'https' in an URL already starting with 'https://...' changes its scheme/protocol to 'httpss' --- .../rarchives/ripme/ripper/rippers/TumblrRipper.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java index dc57c48f..89884854 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java @@ -44,7 +44,7 @@ public class TumblrRipper extends AlbumRipper { private static final String API_KEY = APIKEYS.get(genNum); // Select random API key from APIKEYS /** - * Gets the API key. + * Gets the API key. * Chooses between default/included keys & user specified ones (from the config file). * @return Tumblr API key */ @@ -57,7 +57,7 @@ public class TumblrRipper extends AlbumRipper { logger.info("Using user tumblr.auth api key: " + userDefinedAPIKey); return userDefinedAPIKey; } - + } public TumblrRipper(URL url) throws IOException { @@ -71,12 +71,12 @@ public class TumblrRipper extends AlbumRipper { public boolean canRip(URL url) { return url.getHost().endsWith(DOMAIN); } - + /** * Sanitizes URL. * @param url URL to be sanitized. * @return Sanitized URL - * @throws MalformedURLException + * @throws MalformedURLException */ @Override public URL sanitizeURL(URL url) throws MalformedURLException { @@ -230,7 +230,7 @@ public class TumblrRipper extends AlbumRipper { urlString = urlString.replaceAll("_\\d+\\.", "_raw."); fileURL = new URL(urlString); } else { - fileURL = new URL(photo.getJSONObject("original_size").getString("url").replaceAll("http", "https")); + fileURL = new URL(photo.getJSONObject("original_size").getString("url").replaceAll("http:", "https:")); } m = p.matcher(fileURL.toString()); if (m.matches()) { From be24df4d48a88d114da94205ec1e4359ab1064f1 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 11:35:36 -0400 Subject: [PATCH 041/140] Some minor code clean up --- .../ripme/ripper/rippers/BlackbrickroadofozRipper.java | 10 ++-------- .../rarchives/ripme/ripper/rippers/CfakeRipper.java | 2 +- .../ripme/ripper/rippers/EightmusesRipper.java | 1 - .../rarchives/ripme/ripper/rippers/EroShareRipper.java | 2 +- .../ripme/ripper/rippers/FuraffinityRipper.java | 6 ------ .../rarchives/ripme/ripper/rippers/LusciousRipper.java | 2 +- .../ripme/ripper/rippers/MyhentaicomicsRipper.java | 6 +++--- .../rarchives/ripme/ripper/rippers/SinfestRipper.java | 2 +- .../ripme/ripper/rippers/ViewcomicRipper.java | 2 -- .../ripme/ripper/rippers/WordpressComicRipper.java | 2 +- .../rarchives/ripme/ripper/rippers/XhamsterRipper.java | 2 +- .../ripme/ripper/rippers/video/XvideosRipper.java | 1 - .../ripme/tst/ripper/rippers/AerisdiesRipperTest.java | 2 +- 13 files changed, 12 insertions(+), 28 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/BlackbrickroadofozRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/BlackbrickroadofozRipper.java index 76340cf9..cb5d4b14 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/BlackbrickroadofozRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/BlackbrickroadofozRipper.java @@ -55,14 +55,8 @@ public class BlackbrickroadofozRipper extends AbstractHTMLRipper { throw new IOException("No more pages"); } String nextPage = elem.attr("href"); - // Some times this returns a empty string - // This for stops that - if (nextPage == "") { - throw new IOException("No more pages"); - } - else { - return Http.url(nextPage).get(); - } + return Http.url(nextPage).get(); + } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/CfakeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/CfakeRipper.java index fb29171b..7ccf558c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/CfakeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/CfakeRipper.java @@ -59,7 +59,7 @@ public class CfakeRipper extends AbstractHTMLRipper { String nextPage = elem.attr("href"); // Some times this returns a empty string // This for stops that - if (nextPage == "") { + if (nextPage.equals("")) { return null; } else { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java index 77ca9102..ca9c24e3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java @@ -4,7 +4,6 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java index e7019178..737b8092 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java @@ -86,7 +86,7 @@ public class EroShareRipper extends AbstractHTMLRipper { throw new IOException("No more pages"); } nextUrl = elem.attr("href"); - if (nextUrl == "") { + if (nextUrl.equals("")) { throw new IOException("No more pages"); } return Http.url("eroshae.com" + nextUrl).get(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java index 973796cf..68aa950a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java @@ -13,7 +13,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import com.rarchives.ripme.utils.Utils; -import org.jsoup.Connection.Method; import org.jsoup.Connection.Response; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -23,7 +22,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ripper.DownloadThreadPool; -import com.rarchives.ripme.utils.Base64; import com.rarchives.ripme.utils.Http; public class FuraffinityRipper extends AbstractHTMLRipper { @@ -162,10 +160,6 @@ public class FuraffinityRipper extends AbstractHTMLRipper { if (!subdirectory.equals("")) { subdirectory = File.separator + subdirectory; } - int o = url.toString().lastIndexOf('/')-1; - String test = url.toString().substring(url.toString().lastIndexOf('/',o)+1); - test = test.replace("/",""); // This is probably not the best way to do this. - test = test.replace("\\",""); // CLOSE ENOUGH! saveFileAs = new File( workingDir.getCanonicalPath() + subdirectory diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java index c98dac0c..376d1292 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java @@ -50,7 +50,7 @@ public class LusciousRipper extends AbstractHTMLRipper { // This is here for pages with mp4s instead of images String video_image = ""; video_image = page.select("div > video > source").attr("src"); - if (video_image != "") { + if (!video_image.equals("")) { urls.add(video_image); } return urls; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java index 21942a47..5b60c4f2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java @@ -73,7 +73,7 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper { if (m.matches()) { nextUrl = "http://myhentaicomics.com" + m.group(0); } - if (nextUrl == "") { + if (nextUrl.equals("")) { throw new IOException("No more pages"); } // Sleep for half a sec to avoid getting IP banned @@ -100,7 +100,7 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper { Element elem = nextAlbumPage.select("a.ui-icon-right").first(); String nextPage = elem.attr("href"); pageNumber = pageNumber + 1; - if (nextPage == "") { + if (nextPage.equals("")) { logger.info("Got " + pageNumber + " pages"); break; } @@ -220,7 +220,7 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper { // If true the page is a page of albums if (doc.toString().contains("class=\"g-item g-album\"")) { // This if checks that there is more than 1 page - if (doc.select("a.ui-icon-right").last().attr("href") != "") { + if (!doc.select("a.ui-icon-right").last().attr("href").equals("")) { // There is more than one page so we call getListOfPages List pagesToRip = getListOfPages(doc); logger.debug("Pages to rip = " + pagesToRip); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java index d30e9b63..9de3d2ae 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java @@ -57,7 +57,7 @@ public class SinfestRipper extends AbstractHTMLRipper { String nextPage = elem.parent().attr("href"); // Some times this returns a empty string // This for stops that - if (nextPage == "") { + if (nextPage.equals("")) { return null; } else { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java index ab34620c..abdb0320 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java @@ -4,14 +4,12 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java index dbc44585..e5014e92 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java @@ -247,7 +247,7 @@ public class WordpressComicRipper extends AbstractHTMLRipper { nextPage = elem.attr("href"); } - if (nextPage == "") { + if (nextPage.equals("")) { throw new IOException("No more pages"); } else { return Http.url(nextPage).get(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index 35fe56ff..b92aa9dd 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -72,7 +72,7 @@ public class XhamsterRipper extends AbstractHTMLRipper { @Override public Document getNextPage(Document doc) throws IOException { - if (doc.select("a.next").first().attr("href") != "") { + if (!doc.select("a.next").first().attr("href").equals("")) { return Http.url(doc.select("a.next").first().attr("href")).get(); } else { throw new IOException("No more pages"); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java index 4f2bac97..6dde798d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java @@ -3,7 +3,6 @@ package com.rarchives.ripme.ripper.rippers.video; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; -import java.net.URLDecoder; import java.util.regex.Matcher; import java.util.regex.Pattern; diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java index fb991ec7..503db2c3 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java @@ -3,7 +3,7 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; import java.net.URL; -import com.rarchives.ripme.ripper.rippers.AerisdiesRipper;; +import com.rarchives.ripme.ripper.rippers.AerisdiesRipper; public class AerisdiesRipperTest extends RippersTest { public void testAlbum() throws IOException { From 7d3c7f2b2007355e7f008b16885b5f2bc02998c6 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 15:07:26 -0400 Subject: [PATCH 042/140] Hentaifoundry ripper no long throws a null error when getting the next page; No longer makes unneeded requests --- .../ripper/rippers/HentaifoundryRipper.java | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java index 561c4249..f15d8b01 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java @@ -52,16 +52,16 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { Response resp = Http.url("http://www.hentai-foundry.com/").response(); cookies = resp.cookies(); resp = Http.url("http://www.hentai-foundry.com/?enterAgree=1&size=1500") - .referrer("http://www.hentai-foundry.com/") - .cookies(cookies) - .response(); + .referrer("http://www.hentai-foundry.com/") + .cookies(cookies) + .response(); // The only cookie that seems to matter in getting around the age wall is the phpsession cookie cookies.putAll(resp.cookies()); sleep(500); resp = Http.url(url) - .referrer("http://www.hentai-foundry.com/") - .cookies(cookies) - .response(); + .referrer("http://www.hentai-foundry.com/") + .cookies(cookies) + .response(); cookies.putAll(resp.cookies()); return resp.parse(); } @@ -74,12 +74,16 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { } Elements els = doc.select("li.next > a"); Element first = els.first(); - String nextURL = first.attr("href"); - nextURL = "http://www.hentai-foundry.com" + nextURL; - return Http.url(nextURL) - .referrer(url) - .cookies(cookies) - .get(); + try { + String nextURL = first.attr("href"); + nextURL = "http://www.hentai-foundry.com" + nextURL; + return Http.url(nextURL) + .referrer(url) + .cookies(cookies) + .get(); + } catch (NullPointerException e) { + throw new IOException("No more pages"); + } } @Override @@ -100,9 +104,9 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { Response resp = Http.url("http://www.hentai-foundry.com/").response(); cookies = resp.cookies(); resp = Http.url("http://www.hentai-foundry.com/?enterAgree=1&size=1500") - .referrer("http://www.hentai-foundry.com/") - .cookies(cookies) - .response(); + .referrer("http://www.hentai-foundry.com/") + .cookies(cookies) + .response(); cookies.putAll(resp.cookies()); logger.info("grabbing " + "http://www.hentai-foundry.com" + thumb.attr("href")); From 4d8f4095d257e996a3c5f94a37cab833f2adc060 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 15:10:37 -0400 Subject: [PATCH 043/140] Hentaifoundry ripper no long throws a null error when getting the next page; No longer makes unneeded requests --- .../ripme/ripper/rippers/HentaifoundryRipper.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java index f15d8b01..d09331a9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java @@ -101,13 +101,6 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { } Document imagePage; try { - Response resp = Http.url("http://www.hentai-foundry.com/").response(); - cookies = resp.cookies(); - resp = Http.url("http://www.hentai-foundry.com/?enterAgree=1&size=1500") - .referrer("http://www.hentai-foundry.com/") - .cookies(cookies) - .response(); - cookies.putAll(resp.cookies()); logger.info("grabbing " + "http://www.hentai-foundry.com" + thumb.attr("href")); imagePage = Http.url("http://www.hentai-foundry.com" + thumb.attr("href")).cookies(cookies).get(); From d6535cd2d3c5805f020c4e26e41a9d77c0aa5092 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 16:00:07 -0400 Subject: [PATCH 044/140] Added support for 8muses.download --- .../ripper/rippers/WordpressComicRipper.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java index dbc44585..6de586c8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java @@ -44,7 +44,8 @@ public class WordpressComicRipper extends AbstractHTMLRipper { "freeadultcomix.com", "thisis.delvecomic.com", "tnbtu.com", - "shipinbottle.pepsaga.com" + "shipinbottle.pepsaga.com", + "8muses.download" ); @Override @@ -135,6 +136,12 @@ public class WordpressComicRipper extends AbstractHTMLRipper { if (shipinbottleMat.matches()) { return true; } + + Pattern eight_musesPat = Pattern.compile("https?://8muses.download/([a-zA-Z0-9_-]+)/?$"); + Matcher eight_musesMat = eight_musesPat.matcher(url.toExternalForm()); + if (eight_musesMat.matches()) { + return true; + } } @@ -209,6 +216,11 @@ public class WordpressComicRipper extends AbstractHTMLRipper { return getHost() + "_" + "Ship_in_bottle"; } + Pattern eight_musesPat = Pattern.compile("https?://8muses.download/([a-zA-Z0-9_-]+)/?$"); + Matcher eight_musesMat = eight_musesPat.matcher(url.toExternalForm()); + if (eight_musesMat.matches()) { + return getHost() + "_" + eight_musesMat.group(1); + } return super.getAlbumTitle(url); } @@ -315,6 +327,12 @@ public class WordpressComicRipper extends AbstractHTMLRipper { } } + if (url.toExternalForm().contains("8muses.download")) { + for (Element elem : doc.select("div.popup-gallery > figure > a")) { + result.add(elem.attr("href")); + } + } + return result; } From 2a4bd6c67af4a9f339b2a18e212870fa4734f1ef Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 16:03:48 -0400 Subject: [PATCH 045/140] Added unit test for 8muses.download --- .../ripme/tst/ripper/rippers/WordpressComicRipperTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java index 2f7dbcf9..ec8db0b0 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java @@ -83,6 +83,12 @@ public class WordpressComicRipperTest extends RippersTest { new URL("http://tnbtu.com/comic/01-00/")); testRipper(ripper); } + + public void test_Eightmuses_download() throws IOException { + WordpressComicRipper ripper = new WordpressComicRipper( + new URL("https://8muses.download/lustomic-playkittens-josh-samuel-porn-comics-8-muses/")); + testRipper(ripper); + } // https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI // public void test_pepsaga() throws IOException { // WordpressComicRipper ripper = new WordpressComicRipper( From a52bbeb6d8ba46e6f47199783665c2335e4a6c83 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 16:23:34 -0400 Subject: [PATCH 046/140] 1.7.37: MInor code clean up; Added socks proxy support; Added support for 8muses.download; Hentaifoundry no longer errors when there are no more pages; Fix bug that causes tumblr to replace https with httpss when downloading resized images --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index ed4efbc6..c8de11b2 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.36 + 1.7.37 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 7c31ed69..a3783f9d 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.36", + "latestVersion": "1.7.37", "changeList": [ + "1.7.37: MInor code clean up; Added socks proxy support; Added support for 8muses.download; Hentaifoundry no longer errors when there are no more pages; Fix bug that causes tumblr to replace https with httpss when downloading resized images", "1.7.36: Fixed Instagram ripper; Fixed hentai2read ripper test; Fixed tnbtu.com ripper", "1.7.35: Fixed instagram ripper; hentai2read ripper now properly names folders", "1.7.34: Added Blackbrickroadofoz Ripper; Fixed webtoons regex", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 608e0c58..3e0d5e5d 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.36"; + private static final String DEFAULT_VERSION = "1.7.37"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From b73e943b9cf39e89b5bcf1fe08723bc9e77081d5 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 18:52:36 -0400 Subject: [PATCH 047/140] Added HitomiRipper --- .../ripme/ripper/rippers/HitomiRipper.java | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java new file mode 100644 index 00000000..3e92cc61 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java @@ -0,0 +1,73 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class HitomiRipper extends AbstractHTMLRipper { + + String galleryId = ""; + + public HitomiRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "hitomi"; + } + + @Override + public String getDomain() { + return "hitomi.la"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https://hitomi.la/galleries/([\\d]+).html"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + galleryId = m.group(1); + return m.group(1); + } + throw new MalformedURLException("Expected hitomi URL format: " + + "https://hitomi.la/galleries/ID.html - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { + // if we go to /GALLERYID.js we get a nice json array of all images in the gallery + return Http.url(new URL(url.toExternalForm().replaceAll(".html", ".js"))).ignoreContentType().get(); + } + + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + String json = doc.text().replaceAll("var galleryinfo =", ""); + logger.info(json); + JSONArray json_data = new JSONArray(json); + for (int i = 0; i < json_data.length(); i++) { + result.add("https://0a.hitomi.la/galleries/" + galleryId + "/" + json_data.getJSONObject(i).getString("name")); + } + + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} From a7445ec4ac3d4ccab090fd829706f8ad2ff68fc0 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 18:54:54 -0400 Subject: [PATCH 048/140] Added Hitomi Ripper unit test --- .../ripme/tst/ripper/rippers/HitomiRipperTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java new file mode 100644 index 00000000..78627a83 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.HitomiRipper; + +public class HitomiRipperTest extends RippersTest { + public void testRip() throws IOException { + HitomiRipper ripper = new HitomiRipper(new URL("https://hitomi.la/galleries/975973.html")); + testRipper(ripper); + } +} From f8a8fdb9dc74c54064bd021aa774166fff0a7f9e Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 18:58:47 -0400 Subject: [PATCH 049/140] Extended HitomiRipper unit test --- .../com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java index 78627a83..13c2798d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java @@ -9,5 +9,6 @@ public class HitomiRipperTest extends RippersTest { public void testRip() throws IOException { HitomiRipper ripper = new HitomiRipper(new URL("https://hitomi.la/galleries/975973.html")); testRipper(ripper); + assertTrue(ripper.getGID(new URL("https://hitomi.la/galleries/975973.html")).equals("975973")); } } From 7b71a7d3f5f80d35fd2c2bb0f674e257519c9a50 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 19 Apr 2018 19:16:15 -0400 Subject: [PATCH 050/140] Extended some unit tests to include getGid --- .../tst/ripper/rippers/EightmusesRipperTest.java | 5 +++++ .../tst/ripper/rippers/ModelmayhemRipperTest.java | 5 +++++ .../tst/ripper/rippers/MyhentaicomicsRipperTest.java | 11 +++++++++++ 3 files changed, 21 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java index 4a6c3539..e29c9ece 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java @@ -17,4 +17,9 @@ public class EightmusesRipperTest extends RippersTest { ripper = new EightmusesRipper(new URL("https://www.8muses.com/comix/album/Blacknwhitecomics_com-Comix/BlacknWhiteComics/The-Mayor")); testRipper(ripper); } + + public void testGID() throws IOException { + EightmusesRipper ripper = new EightmusesRipper(new URL("https://www.8muses.com/comix/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore")); + assertEquals("Affect3D-Comics", ripper.getGID(new URL("https://www.8muses.com/comics/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore"))); + } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelmayhemRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelmayhemRipperTest.java index 621d77c3..d2a9581b 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelmayhemRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelmayhemRipperTest.java @@ -10,4 +10,9 @@ public class ModelmayhemRipperTest extends RippersTest { ModelmayhemRipper ripper = new ModelmayhemRipper(new URL("https://www.modelmayhem.com/portfolio/520206/viewall")); testRipper(ripper); } + + public void testGetGID() throws IOException { + ModelmayhemRipper ripper = new ModelmayhemRipper(new URL("https://www.modelmayhem.com/portfolio/520206/viewall")); + assertEquals("520206", ripper.getGID(new URL("https://www.modelmayhem.com/portfolio/520206/viewall"))); + } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java index 6714195d..b4d01032 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java @@ -10,4 +10,15 @@ public class MyhentaicomicsRipperTest extends RippersTest { MyhentaicomicsRipper ripper = new MyhentaicomicsRipper(new URL("http://myhentaicomics.com/index.php/Nienna-Lost-Tales")); testRipper(ripper); } + + public void testGetGID() throws IOException { + URL url = new URL("http://myhentaicomics.com/index.php/Nienna-Lost-Tales"); + MyhentaicomicsRipper ripper = new MyhentaicomicsRipper(url); + // Test a comic + assertEquals("Nienna-Lost-Tales", ripper.getGID(url)); + // Test a search + assertEquals("test", ripper.getGID(new URL("http://myhentaicomics.com/index.php/search?q=test"))); + // Test a tag + assertEquals("2409", ripper.getGID(new URL("http://myhentaicomics.com/index.php/tag/2409/"))); + } } \ No newline at end of file From 34ee3a6f60d3d274cc19dedeba4ceaa3495b4bc8 Mon Sep 17 00:00:00 2001 From: Random Committer Date: Fri, 20 Apr 2018 07:51:40 -0700 Subject: [PATCH 051/140] Removed Socks Proxy setup from App.java, added new class Proxy under com.rarchives.ripme.utils which allows to set both HTTP Proxy and SOCKS Proxy --- src/main/java/com/rarchives/ripme/App.java | 40 +++----- .../java/com/rarchives/ripme/utils/Proxy.java | 99 +++++++++++++++++++ 2 files changed, 112 insertions(+), 27 deletions(-) create mode 100644 src/main/java/com/rarchives/ripme/utils/Proxy.java diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index 508401d2..f9357e62 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -7,9 +7,7 @@ import java.io.BufferedReader; import java.io.FileReader; import java.io.FileNotFoundException; -import java.net.Authenticator; import java.net.MalformedURLException; -import java.net.PasswordAuthentication; import java.net.URL; import java.util.Arrays; import java.util.List; @@ -29,6 +27,7 @@ import com.rarchives.ripme.ui.History; import com.rarchives.ripme.ui.HistoryEntry; import com.rarchives.ripme.ui.MainWindow; import com.rarchives.ripme.ui.UpdateUtils; +import com.rarchives.ripme.utils.Proxy; import com.rarchives.ripme.utils.RipUtils; import com.rarchives.ripme.utils.Utils; @@ -69,7 +68,7 @@ public class App { /** * Creates an abstract ripper and instructs it to rip. * @param url URL to be ripped - * @throws Exception + * @throws Exception */ private static void rip(URL url) throws Exception { AbstractRipper ripper = AbstractRipper.getRipper(url); @@ -96,29 +95,15 @@ public class App { if (cl.hasOption('w')) { Utils.setConfigBoolean("file.overwrite", true); } - + if (cl.hasOption('s')) { String sservfull = cl.getOptionValue('s').trim(); - if (sservfull.lastIndexOf("@") != -1) { - int sservli = sservfull.lastIndexOf("@"); - String userpw = sservfull.substring(0, sservli); - String[] usersplit = userpw.split(":"); - String user = usersplit[0]; - String password = usersplit[1]; - Authenticator.setDefault(new Authenticator(){ - protected PasswordAuthentication getPasswordAuthentication(){ - PasswordAuthentication p = new PasswordAuthentication(user, password.toCharArray()); - return p; - } - }); - - sservfull = sservfull.substring(sservli + 1); - } - String[] servsplit = sservfull.split(":"); - if (servsplit.length == 2) { - System.setProperty("socksProxyPort", servsplit[1]); - } - System.setProperty("socksProxyHost", servsplit[0]); + Proxy.setSocks(sservfull); + } + + if (cl.hasOption('p')) { + String proxyserverfull = cl.getOptionValue('p').trim(); + Proxy.setHTTPProxy(proxyserverfull); } if (cl.hasOption('t')) { @@ -221,7 +206,7 @@ public class App { String url = cl.getOptionValue('u').trim(); ripURL(url, cl.hasOption("n")); } - + } /** @@ -269,7 +254,8 @@ public class App { opts.addOption("n", "no-prop-file", false, "Do not create properties file."); opts.addOption("f", "urls-file", true, "Rip URLs from a file."); opts.addOption("v", "version", false, "Show current version"); - opts.addOption("s", "socks-server", true, "Use socks server ([user:password]@host[:port]"); + opts.addOption("s", "socks-server", true, "Use socks server ([user:password]@host[:port])"); + opts.addOption("p", "proxy-server", true, "Use HTTP Proxy server ([user:password]@host[:port])"); return opts; } @@ -288,7 +274,7 @@ public class App { return null; } } - + /** * Loads history from history file into memory. */ diff --git a/src/main/java/com/rarchives/ripme/utils/Proxy.java b/src/main/java/com/rarchives/ripme/utils/Proxy.java new file mode 100644 index 00000000..2f657b88 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/utils/Proxy.java @@ -0,0 +1,99 @@ +package com.rarchives.ripme.utils; + +import java.net.Authenticator; +import java.net.PasswordAuthentication; +import java.util.Map; +import java.util.HashMap; + +/** + * Proxy/Socks setter + */ +public class Proxy { + private Proxy() { + } + + /** + * Parse the proxy server settings from string, using the format + * [user:password]@host[:port]. + * + * @param fullproxy the string to parse + * @return HashMap containing proxy server, port, user and password + */ + private static Map parseServer(String fullproxy) { + Map proxy = new HashMap(); + + if (fullproxy.lastIndexOf("@") != -1) { + int sservli = fullproxy.lastIndexOf("@"); + String userpw = fullproxy.substring(0, sservli); + String[] usersplit = userpw.split(":"); + proxy.put("user", usersplit[0]); + proxy.put("password", usersplit[1]); + fullproxy = fullproxy.substring(sservli + 1); + } + String[] servsplit = fullproxy.split(":"); + if (servsplit.length == 2) { + proxy.put("port", servsplit[1]); + } + proxy.put("server", servsplit[0]); + return proxy; + } + + /** + * Set a HTTP Proxy. + * WARNING: Authenticated HTTP Proxy won't work from jdk1.8.111 unless + * passing the flag -Djdk.http.auth.tunneling.disabledSchemes="" to java + * see https://stackoverflow.com/q/41505219 + * + * @param fullproxy the proxy, using format [user:password]@host[:port] + */ + public static void setHTTPProxy(String fullproxy) { + Map proxyServer = parseServer(fullproxy); + + if (proxyServer.get("user") != null && proxyServer.get("password") != null) { + Authenticator.setDefault(new Authenticator(){ + protected PasswordAuthentication getPasswordAuthentication(){ + PasswordAuthentication p = new PasswordAuthentication(proxyServer.get("user"), proxyServer.get("password").toCharArray()); + return p; + } + }); + System.setProperty("http.proxyUser", proxyServer.get("user")); + System.setProperty("http.proxyPassword", proxyServer.get("password")); + System.setProperty("https.proxyUser", proxyServer.get("user")); + System.setProperty("https.proxyPassword", proxyServer.get("password")); + } + + if (proxyServer.get("port") != null) { + System.setProperty("http.proxyPort", proxyServer.get("port")); + System.setProperty("https.proxyPort", proxyServer.get("port")); + } + + System.setProperty("http.proxyHost", proxyServer.get("server")); + System.setProperty("https.proxyHost", proxyServer.get("server")); + } + + /** + * Set a Socks Proxy Server (globally). + * + * @param fullsocks the socks server, using format [user:password]@host[:port] + */ + public static void setSocks(String fullsocks) { + + Map socksServer = parseServer(fullsocks); + if (socksServer.get("user") != null && socksServer.get("password") != null) { + Authenticator.setDefault(new Authenticator(){ + protected PasswordAuthentication getPasswordAuthentication(){ + PasswordAuthentication p = new PasswordAuthentication(socksServer.get("user"), socksServer.get("password").toCharArray()); + return p; + } + }); + System.setProperty("java.net.socks.username", socksServer.get("user")); + System.setProperty("java.net.socks.password", socksServer.get("password")); + } + if (socksServer.get("port") != null) { + System.setProperty("socksProxyPort", socksServer.get("port")); + } + + System.setProperty("socksProxyHost", socksServer.get("server")); + } + +} \ No newline at end of file From f493b1ed80384349511ba31f7099e4576b0a92d3 Mon Sep 17 00:00:00 2001 From: Random Committer Date: Fri, 20 Apr 2018 08:03:45 -0700 Subject: [PATCH 052/140] Added test for class Proxy --- .../com/rarchives/ripme/tst/proxyTest.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/proxyTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/proxyTest.java b/src/test/java/com/rarchives/ripme/tst/proxyTest.java new file mode 100644 index 00000000..32b7958c --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/proxyTest.java @@ -0,0 +1,42 @@ +package com.rarchives.ripme.tst; + +import java.io.IOException; +import java.net.URL; +import com.rarchives.ripme.utils.Proxy; +import com.rarchives.ripme.utils.Utils; +import junit.framework.TestCase; +import com.rarchives.ripme.utils.Http; + + +public class proxyTest extends TestCase { + // This test will only run on machines where the user has added a entry for socks.server + public void testSocksProxy() throws IOException { + URL url = new URL("https://icanhazip.com"); + String proxyConfig = Utils.getConfigString("socks.server", ""); + if (!proxyConfig.equals("")) { + String ip1 = Http.url(url).ignoreContentType().get().text(); + Proxy.setSocks(Utils.getConfigString("socks.server", "")); + String ip2 = Http.url(url).ignoreContentType().get().text(); + assertFalse(ip1.equals(ip2)); + } else { + System.out.println("Skipping testSocksProxy"); + assert(true); + } + } + + // This test will only run on machines where the user has added a entry for proxy.server + public void testHTTPProxy() throws IOException { + URL url = new URL("https://icanhazip.com"); + String proxyConfig = Utils.getConfigString("proxy.server", ""); + if (!proxyConfig.equals("")) { + String ip1 = Http.url(url).ignoreContentType().get().text(); + Proxy.setHTTPProxy(Utils.getConfigString("proxy.server", "")); + String ip2 = Http.url(url).ignoreContentType().get().text(); + assertFalse(ip1.equals(ip2)); + } else { + System.out.println("Skipping testHTTPProxy"); + assert(true); + } + } + +} From 95d1e0a12046ff9ca1673a8df5ad5508630e2270 Mon Sep 17 00:00:00 2001 From: Random Committer Date: Fri, 20 Apr 2018 09:15:39 -0700 Subject: [PATCH 053/140] Changed socks and proxy config values to proxy.socks and proxy.http; Checking config on start for these values and set proxy/socks accordingly. --- src/main/java/com/rarchives/ripme/App.java | 6 ++++++ src/main/java/com/rarchives/ripme/utils/Proxy.java | 2 +- src/test/java/com/rarchives/ripme/tst/proxyTest.java | 12 ++++++------ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index f9357e62..6f650d56 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -48,6 +48,12 @@ public class App { System.exit(0); } + if (Utils.getConfigString("proxy.http", null) != null) { + Proxy.setHTTPProxy(Utils.getConfigString("proxy.http", null)); + } else if (Utils.getConfigString("proxy.socks", null) != null) { + Proxy.setSocks(Utils.getConfigString("proxy.socks", null)); + } + if (GraphicsEnvironment.isHeadless() || args.length > 0) { handleArguments(args); } else { diff --git a/src/main/java/com/rarchives/ripme/utils/Proxy.java b/src/main/java/com/rarchives/ripme/utils/Proxy.java index 2f657b88..be3c3b7e 100644 --- a/src/main/java/com/rarchives/ripme/utils/Proxy.java +++ b/src/main/java/com/rarchives/ripme/utils/Proxy.java @@ -96,4 +96,4 @@ public class Proxy { System.setProperty("socksProxyHost", socksServer.get("server")); } -} \ No newline at end of file +} diff --git a/src/test/java/com/rarchives/ripme/tst/proxyTest.java b/src/test/java/com/rarchives/ripme/tst/proxyTest.java index 32b7958c..1d702c6b 100644 --- a/src/test/java/com/rarchives/ripme/tst/proxyTest.java +++ b/src/test/java/com/rarchives/ripme/tst/proxyTest.java @@ -9,13 +9,13 @@ import com.rarchives.ripme.utils.Http; public class proxyTest extends TestCase { - // This test will only run on machines where the user has added a entry for socks.server + // This test will only run on machines where the user has added a entry for proxy.socks public void testSocksProxy() throws IOException { URL url = new URL("https://icanhazip.com"); - String proxyConfig = Utils.getConfigString("socks.server", ""); + String proxyConfig = Utils.getConfigString("proxy.socks", ""); if (!proxyConfig.equals("")) { String ip1 = Http.url(url).ignoreContentType().get().text(); - Proxy.setSocks(Utils.getConfigString("socks.server", "")); + Proxy.setSocks(Utils.getConfigString("proxy.socks", "")); String ip2 = Http.url(url).ignoreContentType().get().text(); assertFalse(ip1.equals(ip2)); } else { @@ -24,13 +24,13 @@ public class proxyTest extends TestCase { } } - // This test will only run on machines where the user has added a entry for proxy.server + // This test will only run on machines where the user has added a entry for proxy.http public void testHTTPProxy() throws IOException { URL url = new URL("https://icanhazip.com"); - String proxyConfig = Utils.getConfigString("proxy.server", ""); + String proxyConfig = Utils.getConfigString("proxy.http", ""); if (!proxyConfig.equals("")) { String ip1 = Http.url(url).ignoreContentType().get().text(); - Proxy.setHTTPProxy(Utils.getConfigString("proxy.server", "")); + Proxy.setHTTPProxy(Utils.getConfigString("proxy.http", "")); String ip2 = Http.url(url).ignoreContentType().get().text(); assertFalse(ip1.equals(ip2)); } else { From 34699bf8c66296f8eabb03e9658646ceac19c88b Mon Sep 17 00:00:00 2001 From: Random Committer Date: Fri, 20 Apr 2018 09:53:49 -0700 Subject: [PATCH 054/140] proxyTest.java: unset proxy/socks before testing it --- src/test/java/com/rarchives/ripme/tst/proxyTest.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/proxyTest.java b/src/test/java/com/rarchives/ripme/tst/proxyTest.java index 1d702c6b..36ea2f55 100644 --- a/src/test/java/com/rarchives/ripme/tst/proxyTest.java +++ b/src/test/java/com/rarchives/ripme/tst/proxyTest.java @@ -9,8 +9,14 @@ import com.rarchives.ripme.utils.Http; public class proxyTest extends TestCase { + + // This test will only run on machines where the user has added a entry for proxy.socks public void testSocksProxy() throws IOException { + // Unset proxy before testing + System.setProperty("http.proxyHost", ""); + System.setProperty("https.proxyHost", ""); + System.setProperty("socksProxyHost", ""); URL url = new URL("https://icanhazip.com"); String proxyConfig = Utils.getConfigString("proxy.socks", ""); if (!proxyConfig.equals("")) { @@ -26,6 +32,10 @@ public class proxyTest extends TestCase { // This test will only run on machines where the user has added a entry for proxy.http public void testHTTPProxy() throws IOException { + // Unset proxy before testing + System.setProperty("http.proxyHost", ""); + System.setProperty("https.proxyHost", ""); + System.setProperty("socksProxyHost", ""); URL url = new URL("https://icanhazip.com"); String proxyConfig = Utils.getConfigString("proxy.http", ""); if (!proxyConfig.equals("")) { From a587ec77faf0cb989dd8f15bc5813fdc644855b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sat, 21 Apr 2018 10:46:16 +0200 Subject: [PATCH 055/140] [hentaifoundry] set content filter --- .../ripper/rippers/HentaifoundryRipper.java | 49 +++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java index d09331a9..8d953de1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java @@ -10,6 +10,7 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.jsoup.Connection.Method; import org.jsoup.Connection.Response; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -49,15 +50,57 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { - Response resp = Http.url("http://www.hentai-foundry.com/").response(); - cookies = resp.cookies(); + Response resp; + Document doc; + resp = Http.url("http://www.hentai-foundry.com/?enterAgree=1&size=1500") .referrer("http://www.hentai-foundry.com/") .cookies(cookies) .response(); // The only cookie that seems to matter in getting around the age wall is the phpsession cookie cookies.putAll(resp.cookies()); - sleep(500); + + doc = resp.parse(); + String csrf_token = doc.select("input[name=YII_CSRF_TOKEN]") + .first().attr("value"); + if (csrf_token != null) { + Map data = new HashMap<>(); + data.put("YII_CSRF_TOKEN" , csrf_token); + data.put("rating_nudity" , "3"); + data.put("rating_violence" , "3"); + data.put("rating_profanity", "3"); + data.put("rating_racism" , "3"); + data.put("rating_sex" , "3"); + data.put("rating_spoilers" , "3"); + data.put("rating_yaoi" , "1"); + data.put("rating_yuri" , "1"); + data.put("rating_teen" , "1"); + data.put("rating_guro" , "1"); + data.put("rating_furry" , "1"); + data.put("rating_beast" , "1"); + data.put("rating_male" , "1"); + data.put("rating_female" , "1"); + data.put("rating_futa" , "1"); + data.put("rating_other" , "1"); + data.put("rating_scat" , "1"); + data.put("rating_incest" , "1"); + data.put("rating_rape" , "1"); + data.put("filter_media" , "A"); + data.put("filter_order" , "date_new"); + data.put("filter_type" , "0"); + + resp = Http.url("http://www.hentai-foundry.com/site/filters") + .referrer("http://www.hentai-foundry.com/") + .cookies(cookies) + .data(data) + .method(Method.POST) + .response(); + cookies.putAll(resp.cookies()); + } + else { + logger.info("unable to find csrf_token and set filter"); + } + resp = Http.url(url) .referrer("http://www.hentai-foundry.com/") .cookies(cookies) From eccce1b4a656af986a3171900e2d151d39cfc307 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 21 Apr 2018 14:22:48 -0400 Subject: [PATCH 056/140] 1.7.38: Added http and socks proxy support; Extended some unit tests to include getGid; Added HitomiRipper; hentaifoundry ripper now can rip all images from accounts --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index c8de11b2..d62354b6 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.37 + 1.7.38 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index a3783f9d..50199614 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.37", + "latestVersion": "1.7.38", "changeList": [ + "1.7.38: Added http and socks proxy support; Extended some unit tests to include getGid; Added HitomiRipper; hentaifoundry ripper now can rip all images from accounts", "1.7.37: MInor code clean up; Added socks proxy support; Added support for 8muses.download; Hentaifoundry no longer errors when there are no more pages; Fix bug that causes tumblr to replace https with httpss when downloading resized images", "1.7.36: Fixed Instagram ripper; Fixed hentai2read ripper test; Fixed tnbtu.com ripper", "1.7.35: Fixed instagram ripper; hentai2read ripper now properly names folders", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 3e0d5e5d..c3767e6a 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.37"; + private static final String DEFAULT_VERSION = "1.7.38"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 19ce96eae5236ce4766bf1bd2a90eead7853187a Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Mon, 23 Apr 2018 15:58:01 -0400 Subject: [PATCH 057/140] Fixed issue with downloading images without an extension --- .../ripme/ripper/AbstractRipper.java | 51 +++++++++++++------ .../ripme/tst/AbstractRipperTest.java | 4 ++ 2 files changed, 39 insertions(+), 16 deletions(-) create mode 100644 src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 6068ed18..4438ae2a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -212,7 +212,7 @@ public abstract class AbstractRipper * True if downloaded successfully * False if failed to download */ - protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName) { + protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName, String extension) { // Don't re-add the url if it was downloaded in a previous rip if (Utils.getConfigBoolean("remember.url_history", true) && !isThisATest()) { if (hasDownloadedURL(url.toExternalForm())) { @@ -228,21 +228,7 @@ public abstract class AbstractRipper return false; } logger.debug("url: " + url + ", prefix: " + prefix + ", subdirectory" + subdirectory + ", referrer: " + referrer + ", cookies: " + cookies + ", fileName: " + fileName); - String saveAs; - if (fileName != null) { - saveAs = fileName; - // Get the extension of the file - String extension = url.toExternalForm().substring(url.toExternalForm().lastIndexOf(".") + 1); - saveAs = saveAs + "." + extension; - } else { - saveAs = url.toExternalForm(); - saveAs = saveAs.substring(saveAs.lastIndexOf('/')+1); - } - - if (saveAs.indexOf('?') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('?')); } - if (saveAs.indexOf('#') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('#')); } - if (saveAs.indexOf('&') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('&')); } - if (saveAs.indexOf(':') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf(':')); } + String saveAs = getFileName(url, fileName, extension); File saveFileAs; try { if (!subdirectory.equals("")) { @@ -274,6 +260,10 @@ public abstract class AbstractRipper return addURLToDownload(url, saveFileAs, referrer, cookies); } + protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName) { + return addURLToDownload(url, prefix, subdirectory, referrer, cookies, fileName, null); + } + /** * Queues file to be downloaded and saved. With options. * @param url @@ -306,6 +296,35 @@ public abstract class AbstractRipper return addURLToDownload(url, prefix, ""); } + public static String getFileName(URL url, String fileName, String extension) { + String saveAs; + if (fileName != null) { + saveAs = fileName; + } else { + saveAs = url.toExternalForm(); + saveAs = saveAs.substring(saveAs.lastIndexOf('/')+1); + } + if (extension == null) { + // Get the extension of the file + String[] lastBitOfURL = url.toExternalForm().split("/"); + + String[] lastBit = lastBitOfURL[lastBitOfURL.length - 1].split("."); + if (lastBit.length != 0) { + extension = lastBit[lastBit.length - 1]; + saveAs = saveAs + "." + extension; + } + } + + if (saveAs.indexOf('?') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('?')); } + if (saveAs.indexOf('#') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('#')); } + if (saveAs.indexOf('&') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('&')); } + if (saveAs.indexOf(':') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf(':')); } + if (extension != null) { + saveAs = saveAs + "." + extension; + } + return saveAs; + } + /** * Waits for downloading threads to complete. diff --git a/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java b/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java new file mode 100644 index 00000000..3a71f472 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java @@ -0,0 +1,4 @@ +package com.rarchives.ripme.tst; + +public class AbstractRipperTest { +} From 56d20b15000e33f5ff9683938c5f786aa07afa95 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Mon, 23 Apr 2018 15:58:18 -0400 Subject: [PATCH 058/140] Added some unit tests for getting file name --- .../ripme/tst/AbstractRipperTest.java | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java b/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java index 3a71f472..f1d8eff5 100644 --- a/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java @@ -1,4 +1,30 @@ package com.rarchives.ripme.tst; -public class AbstractRipperTest { +import com.rarchives.ripme.ripper.AbstractRipper; +import junit.framework.TestCase; + +import java.io.IOException; +import java.net.URL; + + + +public class AbstractRipperTest extends TestCase { + + public void testGetFileName() throws IOException { + String fileName = AbstractRipper.getFileName(new URL("http://www.tsumino.com/Image/Object?name=U1EieteEGwm6N1dGszqCpA%3D%3D"), "test", "test"); + assertEquals("test.test", fileName); + + fileName = AbstractRipper.getFileName(new URL("http://www.tsumino.com/Image/Object?name=U1EieteEGwm6N1dGszqCpA%3D%3D"), "test", null); + assertEquals("test", fileName); + + fileName = AbstractRipper.getFileName(new URL("http://www.tsumino.com/Image/Object?name=U1EieteEGwm6N1dGszqCpA%3D%3D"), null, null); + assertEquals("Object", fileName); + + fileName = AbstractRipper.getFileName(new URL("http://www.test.com/file.png"), null, null); + assertEquals("file.png", fileName); + + fileName = AbstractRipper.getFileName(new URL("http://www.test.com/file."), null, null); + assertEquals("file.", fileName); + } + } From 0427073c9a47cba750dff8f3534e4d4c756199e0 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 26 Apr 2018 09:48:27 -0400 Subject: [PATCH 059/140] Added Gfycatporntube.com Ripper --- .../ripper/rippers/GfycatporntubeRipper.java | 61 +++++++++++++++++++ .../rippers/GfycatporntubeRipperTest.java | 13 ++++ 2 files changed, 74 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java new file mode 100644 index 00000000..504b89d6 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java @@ -0,0 +1,61 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class GfycatporntubeRipper extends AbstractHTMLRipper { + + public GfycatporntubeRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "gfycatporntube"; + } + + @Override + public String getDomain() { + return "gfycatporntube.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https?://gfycatporntube.com/([a-zA-Z1-9_-]*)/?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected gfycatporntube URL format: " + + "gfycatporntube.com/NAME - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + result.add(doc.select("source[id=mp4Source]").attr("src")); + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java new file mode 100644 index 00000000..6856eb06 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.GfycatporntubeRipper; + +public class GfycatporntubeRipperTest extends RippersTest { + public void testRip() throws IOException { + GfycatporntubeRipper ripper = new GfycatporntubeRipper(new URL("https://gfycatporntube.com/blowjob-bunny-puts-on-a-show/")); + testRipper(ripper); + } +} From 3342898e774ab6be7ee9815a0a351042889d3b2e Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 26 Apr 2018 10:48:07 -0400 Subject: [PATCH 060/140] Added ripper for rule34.xxx --- .../ripme/ripper/rippers/Rule34Ripper.java | 84 +++++++++++++++++++ .../ripme/ripper/rippers/rule34Ripper.java | 4 + 2 files changed, 88 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/Rule34Ripper.java create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Rule34Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Rule34Ripper.java new file mode 100644 index 00000000..8651bd69 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Rule34Ripper.java @@ -0,0 +1,84 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class Rule34Ripper extends AbstractHTMLRipper { + + public Rule34Ripper(URL url) throws IOException { + super(url); + } + + private String apiUrl; + private int pageNumber = 0; + + @Override + public String getHost() { + return "rule34"; + } + + @Override + public String getDomain() { + return "rule34.xxx"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https?://rule34.xxx/index.php\\?page=post&s=list&tags=([\\S]+)"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected rule34.xxx URL format: " + + "rule34.xxx/index.php\\?page=post&s=list&tags=TAG - got " + url + " instead"); + } + + public URL getAPIUrl() throws MalformedURLException { + URL urlToReturn = new URL("https://rule34.xxx/index.php?page=dapi&s=post&q=index&limit=100&tags=" + getGID(url)); + return urlToReturn; + } + + @Override + public Document getFirstPage() throws IOException { + apiUrl = getAPIUrl().toExternalForm(); + // "url" is an instance field of the superclass + return Http.url(getAPIUrl()).get(); + } + + @Override + public Document getNextPage(Document doc) throws IOException { + if (doc.html().contains("Search error: API limited due to abuse")) { + throw new IOException("No more pages"); + } + pageNumber += 1; + String nextPage = apiUrl + "&pid=" + pageNumber; + return Http.url(nextPage).get(); + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + for (Element el : doc.select("posts > post")) { + String imageSource = el.select("post").attr("file_url"); + result.add(imageSource); + + } + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java new file mode 100644 index 00000000..9402256b --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java @@ -0,0 +1,4 @@ +package com.rarchives.ripme.ripper.rippers; + +public class rule34Ripper { +} From 7cf8e59ee72ca906fb584ebafd021d8be5e6ef4f Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 26 Apr 2018 10:48:28 -0400 Subject: [PATCH 061/140] Removed misnamed file --- .../java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java deleted file mode 100644 index 9402256b..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -public class rule34Ripper { -} From c076fe2462dcdb1062f574405277ba8a80ab6b41 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 26 Apr 2018 10:58:35 -0400 Subject: [PATCH 062/140] Added unit test Rule34Ripper --- .../ripme/tst/ripper/rippers/Rule34RipperTest.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/Rule34RipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Rule34RipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Rule34RipperTest.java new file mode 100644 index 00000000..cffc807d --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Rule34RipperTest.java @@ -0,0 +1,14 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.Rule34Ripper; + +public class Rule34RipperTest extends RippersTest { + public void testShesFreakyRip() throws IOException { + Rule34Ripper ripper = new Rule34Ripper(new URL("https://rule34.xxx/index.php?page=post&s=list&tags=bimbo")); + testRipper(ripper); + } + +} From 388cd9936d16c2ec53a16749b179bf3d7d7fb13d Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 26 Apr 2018 12:42:24 -0400 Subject: [PATCH 063/140] 1.7.39: Added rule34.xxx ripper; Added Gfycatporntube.com ripper; Fixed AbstractRipper subdir bug; Added AbstractRipper unit tests" --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index d62354b6..b93ef11e 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.38 + 1.7.39 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 50199614..e1288c50 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.38", + "latestVersion": "1.7.39", "changeList": [ + "1.7.39: Added rule34.xxx ripper; Added Gfycatporntube.com ripper; Fixed AbstractRipper subdir bug; Added AbstractRipper unit tests"", "1.7.38: Added http and socks proxy support; Extended some unit tests to include getGid; Added HitomiRipper; hentaifoundry ripper now can rip all images from accounts", "1.7.37: MInor code clean up; Added socks proxy support; Added support for 8muses.download; Hentaifoundry no longer errors when there are no more pages; Fix bug that causes tumblr to replace https with httpss when downloading resized images", "1.7.36: Fixed Instagram ripper; Fixed hentai2read ripper test; Fixed tnbtu.com ripper", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index c3767e6a..b16d8399 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.38"; + private static final String DEFAULT_VERSION = "1.7.39"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 8f84d3f8ff3cbf391af7c738e2d6222b3f8b3657 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 26 Apr 2018 12:54:17 -0400 Subject: [PATCH 064/140] Removed extra qoute --- ripme.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ripme.json b/ripme.json index e1288c50..f4511057 100644 --- a/ripme.json +++ b/ripme.json @@ -1,7 +1,7 @@ { "latestVersion": "1.7.39", "changeList": [ - "1.7.39: Added rule34.xxx ripper; Added Gfycatporntube.com ripper; Fixed AbstractRipper subdir bug; Added AbstractRipper unit tests"", + "1.7.39: Added rule34.xxx ripper; Added Gfycatporntube.com ripper; Fixed AbstractRipper subdir bug; Added AbstractRipper unit tests", "1.7.38: Added http and socks proxy support; Extended some unit tests to include getGid; Added HitomiRipper; hentaifoundry ripper now can rip all images from accounts", "1.7.37: MInor code clean up; Added socks proxy support; Added support for 8muses.download; Hentaifoundry no longer errors when there are no more pages; Fix bug that causes tumblr to replace https with httpss when downloading resized images", "1.7.36: Fixed Instagram ripper; Fixed hentai2read ripper test; Fixed tnbtu.com ripper", From 5763c673ec88160679392886eb57ca0c2f076b4b Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 26 Apr 2018 15:46:50 -0400 Subject: [PATCH 065/140] TsuminoRipper now adds .png to filenames; Removed unneeded debug statment --- .../com/rarchives/ripme/ripper/rippers/TsuminoRipper.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java index 9ca91e45..6030c9f0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java @@ -17,7 +17,6 @@ import org.json.JSONObject; import org.jsoup.Connection; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; import com.rarchives.ripme.ripper.AbstractHTMLRipper; @@ -35,13 +34,10 @@ public class TsuminoRipper extends AbstractHTMLRipper { try { // This sessionId will expire and need to be replaced cookies.put("ASP.NET_SessionId","c4rbzccf0dvy3e0cloolmlkq"); - logger.info(cookies); Document doc = Jsoup.connect(postURL).data("q", getAlbumID()).userAgent(USER_AGENT).cookies(cookies).referrer("http://www.tsumino.com/Read/View/" + getAlbumID()).post(); String jsonInfo = doc.html().replaceAll("","").replaceAll("", "").replaceAll("", "").replaceAll("", "") .replaceAll("", "").replaceAll("\n", ""); - logger.info(jsonInfo); JSONObject json = new JSONObject(jsonInfo); - logger.info(json.getJSONArray("reader_page_urls")); return json.getJSONArray("reader_page_urls"); } catch (IOException e) { logger.info(e); @@ -85,7 +81,6 @@ public class TsuminoRipper extends AbstractHTMLRipper { public Document getFirstPage() throws IOException { Connection.Response resp = Http.url(url).response(); cookies.putAll(resp.cookies()); - logger.info(resp.parse()); return resp.parse(); } @@ -103,6 +98,6 @@ public class TsuminoRipper extends AbstractHTMLRipper { @Override public void downloadURL(URL url, int index) { sleep(1000); - addURLToDownload(url, getPrefix(index)); + addURLToDownload(url, getPrefix(index), "", null, null, null, "png"); } } From 1ccaa21ef89ea60f6e255674b57c17f6d8fa45eb Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 27 Apr 2018 13:38:16 -0400 Subject: [PATCH 066/140] Added canRip for rule34.xxx --- .../rarchives/ripme/ripper/rippers/Rule34Ripper.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Rule34Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Rule34Ripper.java index 8651bd69..b7e0f7b0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/Rule34Ripper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Rule34Ripper.java @@ -33,6 +33,16 @@ public class Rule34Ripper extends AbstractHTMLRipper { return "rule34.xxx"; } + @Override + public boolean canRip(URL url){ + Pattern p = Pattern.compile("https?://rule34.xxx/index.php\\?page=post&s=list&tags=([\\S]+)"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return true; + } + return false; + } + @Override public String getGID(URL url) throws MalformedURLException { Pattern p = Pattern.compile("https?://rule34.xxx/index.php\\?page=post&s=list&tags=([\\S]+)"); @@ -41,7 +51,7 @@ public class Rule34Ripper extends AbstractHTMLRipper { return m.group(1); } throw new MalformedURLException("Expected rule34.xxx URL format: " + - "rule34.xxx/index.php\\?page=post&s=list&tags=TAG - got " + url + " instead"); + "rule34.xxx/index.php?page=post&s=list&tags=TAG - got " + url + " instead"); } public URL getAPIUrl() throws MalformedURLException { From 7b1fbd3290010db410ef2c92afb75d1bc6e4797f Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 29 Apr 2018 12:22:27 -0400 Subject: [PATCH 067/140] Added ripper for hypnohub.net; removed misnamed file --- .../ripme/ripper/rippers/HypnohubRipper.java | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/HypnohubRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HypnohubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HypnohubRipper.java new file mode 100644 index 00000000..1eabefb9 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HypnohubRipper.java @@ -0,0 +1,91 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +import javax.print.Doc; + +public class HypnohubRipper extends AbstractHTMLRipper { + + public HypnohubRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "hypnohub"; + } + + @Override + public String getDomain() { + return "hypnohub.net"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https?://hypnohub.net/\\S+/show/([\\d]+)/?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + p = Pattern.compile("https?://hypnohub.net/\\S+/show/([\\d]+)/([\\S]+)/?$"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1) + "_" + m.group(2); + } + throw new MalformedURLException("Expected cfake URL format: " + + "hypnohub.net/pool/show/ID - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + private String ripPost(String url) throws IOException { + logger.info(url); + Document doc = Http.url(url).get(); + return "https:" + doc.select("img.image").attr("src"); + + } + + private String ripPost(Document doc) { + logger.info(url); + return "https:" + doc.select("img.image").attr("src"); + + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + if (url.toExternalForm().contains("/pool")) { + for (Element el : doc.select("ul[id=post-list-posts] > li > div > a.thumb")) { + try { + result.add(ripPost("https://hypnohub.net" + el.attr("href"))); + } catch (IOException e) { + return result; + } + } + } else if (url.toExternalForm().contains("/post")) { + result.add(ripPost(doc)); + } + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} From 60d5f95f938a2da169727b13a4b5af055ca16b9e Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 29 Apr 2018 12:40:50 -0400 Subject: [PATCH 068/140] Added unit test for HypnohubRipper --- .../ripper/rippers/HypnohubRipperTest.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/HypnohubRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HypnohubRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HypnohubRipperTest.java new file mode 100644 index 00000000..4466eadc --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HypnohubRipperTest.java @@ -0,0 +1,25 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.HypnohubRipper; + +public class HypnohubRipperTest extends RippersTest { + public void testRip() throws IOException { + URL poolURL = new URL("http://hypnohub.net/pool/show/2303"); + URL postURL = new URL("http://hypnohub.net/post/show/63464/black_hair-bracelet-collar-corruption-female_only-"); + HypnohubRipper ripper = new HypnohubRipper(poolURL); + testRipper(ripper); + ripper = new HypnohubRipper(postURL); + testRipper(ripper); + } + public void testGetGID() throws IOException { + URL poolURL = new URL("http://hypnohub.net/pool/show/2303"); + HypnohubRipper ripper = new HypnohubRipper(poolURL); + assertEquals("hypnohub_2303", ripper.getGID(poolURL)); + + URL postURL = new URL("http://hypnohub.net/post/show/63464/black_hair-bracelet-collar-corruption-female_only-"); + assertEquals("hypnohub_63464_black_hair-bracelet-collar-corruption-female_only-", ripper.getGID(postURL)); + } +} From 24f1b4fbdc6f26ab0a6a6dfe85d2edb1cf5ed9a7 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 29 Apr 2018 12:50:44 -0400 Subject: [PATCH 069/140] Fixed hypnohub unit test --- .../ripme/tst/ripper/rippers/HypnohubRipperTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HypnohubRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HypnohubRipperTest.java index 4466eadc..1d9ef4ad 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HypnohubRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HypnohubRipperTest.java @@ -17,9 +17,9 @@ public class HypnohubRipperTest extends RippersTest { public void testGetGID() throws IOException { URL poolURL = new URL("http://hypnohub.net/pool/show/2303"); HypnohubRipper ripper = new HypnohubRipper(poolURL); - assertEquals("hypnohub_2303", ripper.getGID(poolURL)); + assertEquals("2303", ripper.getGID(poolURL)); URL postURL = new URL("http://hypnohub.net/post/show/63464/black_hair-bracelet-collar-corruption-female_only-"); - assertEquals("hypnohub_63464_black_hair-bracelet-collar-corruption-female_only-", ripper.getGID(postURL)); + assertEquals("63464_black_hair-bracelet-collar-corruption-female_only-", ripper.getGID(postURL)); } } From e8f7f2cfae1f2ebbf631f6e861b4f36231901cc8 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 29 Apr 2018 16:28:17 -0400 Subject: [PATCH 070/140] 1.7.40: Added hypnohub.net ripper; Fixed rule34.xxx ripper; Tsumino Ripper now add .png to filenames --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index b93ef11e..d0be1909 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.39 + 1.7.40 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index f4511057..dcac88bc 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.39", + "latestVersion": "1.7.40", "changeList": [ + "1.7.40: Added hypnohub.net ripper; Fixed rule34.xxx ripper; Tsumino Ripper now add .png to filenames", "1.7.39: Added rule34.xxx ripper; Added Gfycatporntube.com ripper; Fixed AbstractRipper subdir bug; Added AbstractRipper unit tests", "1.7.38: Added http and socks proxy support; Extended some unit tests to include getGid; Added HitomiRipper; hentaifoundry ripper now can rip all images from accounts", "1.7.37: MInor code clean up; Added socks proxy support; Added support for 8muses.download; Hentaifoundry no longer errors when there are no more pages; Fix bug that causes tumblr to replace https with httpss when downloading resized images", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index b16d8399..5287194d 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.39"; + private static final String DEFAULT_VERSION = "1.7.40"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 5369a54caaff2b484f7c3c690d1a83893a9e10e1 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Mon, 30 Apr 2018 13:22:51 -0400 Subject: [PATCH 071/140] Added ripper and unit test for dynasty-scans.com --- .../ripper/rippers/DynastyscansRipper.java | 84 +++++++++++++++++++ .../rippers/DynastyscansRipperTest.java | 18 ++++ 2 files changed, 102 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/DynastyscansRipper.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/DynastyscansRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DynastyscansRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DynastyscansRipper.java new file mode 100644 index 00000000..37d3ad93 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DynastyscansRipper.java @@ -0,0 +1,84 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.json.JSONArray; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class DynastyscansRipper extends AbstractHTMLRipper { + + public DynastyscansRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "dynasty-scans"; + } + + @Override + public String getDomain() { + return "dynasty-scans.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https?://dynasty-scans.com/chapters/([\\S]+)/?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected dynasty-scans URL format: " + + "dynasty-scans.com/chapters/ID - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + @Override + public Document getNextPage(Document doc) throws IOException { + Element elem = doc.select("a[id=next_link]").first(); + if (elem == null || elem.attr("href").equals("#")) { + throw new IOException("No more pages"); + } + return Http.url("https://dynasty-scans.com" + elem.attr("href")).get(); + + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + String jsonText = null; + for (Element script : doc.select("script")) { + if (script.data().contains("var pages")) { + jsonText = script.data().replaceAll("var pages = ", ""); + jsonText = jsonText.replaceAll("//", ""); + } + } + JSONArray imageArray = new JSONArray(jsonText); + for (int i = 0; i < imageArray.length(); i++) { + result.add("https://dynasty-scans.com" + imageArray.getJSONObject(i).getString("image")); + } + + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DynastyscansRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DynastyscansRipperTest.java new file mode 100644 index 00000000..8eb8d88f --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DynastyscansRipperTest.java @@ -0,0 +1,18 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.DynastyscansRipper; + +public class DynastyscansRipperTest extends RippersTest { + public void testRip() throws IOException { + DynastyscansRipper ripper = new DynastyscansRipper(new URL("https://dynasty-scans.com/chapters/under_one_roof_ch01")); + testRipper(ripper); + } + + public void testGetGID() throws IOException { + DynastyscansRipper ripper = new DynastyscansRipper(new URL("https://dynasty-scans.com/chapters/under_one_roof_ch01")); + assertEquals("under_one_roof_ch01", ripper.getGID(new URL("https://dynasty-scans.com/chapters/under_one_roof_ch01"))); + } +} From 6a672f05c9762b2fabbc6c0619ab12bc565f2403 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Mon, 30 Apr 2018 17:47:19 -0400 Subject: [PATCH 072/140] Added ManganeloRipper --- .../ripme/ripper/rippers/ManganeloRipper.java | 116 ++++++++++++++++++ .../ripper/rippers/ManganeloRipperTest.java | 13 ++ 2 files changed, 129 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/ManganeloRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java new file mode 100644 index 00000000..8f8f8e68 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java @@ -0,0 +1,116 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +import javax.print.Doc; + +public class ManganeloRipper extends AbstractHTMLRipper { + + public ManganeloRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "manganelo"; + } + + @Override + public String getDomain() { + return "manganelo.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https?://manganelo.com/manga/([\\S]+)/?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + + p = Pattern.compile("http://manganelo.com/chapter/([\\S]+)/([\\S]+)/?$"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected manganelo URL format: " + + "/manganelo.com/manga/ID - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + @Override + public Document getNextPage(Document doc) throws IOException { + Element elem = doc.select("div.btn-navigation-chap > a.back").first(); + if (elem == null) { + throw new IOException("No more pages"); + } else { + return Http.url(elem.attr("href")).get(); + } + } + + private List getURLsFromChap(String url) { + logger.debug("Getting urls from " + url); + List result = new ArrayList<>(); + try { + Document doc = Http.url(url).get(); + for (Element el : doc.select("img.img_content")) { + result.add(el.attr("src")); + } + return result; + } catch (IOException e) { + return null; + } + + } + + private List getURLsFromChap(Document doc) { + logger.debug("Getting urls from " + url); + List result = new ArrayList<>(); + for (Element el : doc.select("img.img_content")) { + result.add(el.attr("src")); + } + return result; + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + List urlsToGrab = new ArrayList<>(); + if (url.toExternalForm().contains("/manga/")) { + for (Element el : doc.select("div.chapter-list > div.row > span > a")) { + urlsToGrab.add(el.attr("href")); + } + Collections.reverse(urlsToGrab); + + for (String url : urlsToGrab) { + result.addAll(getURLsFromChap(url)); + } + } else if (url.toExternalForm().contains("/chapter/")) { + result.addAll(getURLsFromChap(doc)); + } + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ManganeloRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ManganeloRipperTest.java new file mode 100644 index 00000000..ca355a2c --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ManganeloRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.ManganeloRipper; + +public class ManganeloRipperTest extends RippersTest { + public void testRip() throws IOException { + ManganeloRipper ripper = new ManganeloRipper(new URL("http://manganelo.com/manga/black_clover")); + testRipper(ripper); + } +} From d73744a73fb7a709e9c4a276646edfe545760fc2 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 1 May 2018 15:14:08 -0400 Subject: [PATCH 073/140] Added support for spyingwithlana.com; added some more unittests for the wordpress comic ripper --- .../ripper/rippers/WordpressComicRipper.java | 43 ++++++++++++------- .../rippers/WordpressComicRipperTest.java | 28 ++++++++++++ 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java index 13b4891e..8b7b7b80 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java @@ -45,7 +45,19 @@ public class WordpressComicRipper extends AbstractHTMLRipper { "thisis.delvecomic.com", "tnbtu.com", "shipinbottle.pepsaga.com", - "8muses.download" + "8muses.download", + "spyingwithlana.com" + ); + + private static List theme1 = Arrays.asList( + "www.totempole666.com", + "buttsmithy.com", + "themonsterunderthebed.net", + "prismblush.com", + "www.konradokonski.com", + "thisis.delvecomic.com", + "tnbtu.com", + "spyingwithlana.com" ); @Override @@ -142,6 +154,12 @@ public class WordpressComicRipper extends AbstractHTMLRipper { if (eight_musesMat.matches()) { return true; } + + Pattern spyingwithlanaPat = Pattern.compile("https?://spyingwithlana.com/comic/([a-zA-Z0-9_-]+)/?$"); + Matcher spyingwithlanaMat = spyingwithlanaPat.matcher(url.toExternalForm()); + if (spyingwithlanaMat.matches()) { + return true; + } } @@ -221,6 +239,13 @@ public class WordpressComicRipper extends AbstractHTMLRipper { if (eight_musesMat.matches()) { return getHost() + "_" + eight_musesMat.group(1); } + + Pattern spyingwithlanaPat = Pattern.compile("https?://spyingwithlana.com/comic/([a-zA-Z0-9_-]+)/?$"); + Matcher spyingwithlanaMat = spyingwithlanaPat.matcher(url.toExternalForm()); + if (spyingwithlanaMat.matches()) { + return "spyingwithlana_" + spyingwithlanaMat.group(1).replaceAll("-page-\\d", ""); + } + return super.getAlbumTitle(url); } @@ -239,13 +264,7 @@ public class WordpressComicRipper extends AbstractHTMLRipper { // Find next page String nextPage = ""; Element elem = null; - if (getHost().contains("www.totempole666.com") - || getHost().contains("buttsmithy.com") - || getHost().contains("themonsterunderthebed.net") - || getHost().contains("prismblush.com") - || getHost().contains("www.konradokonski.com") - || getHost().contains("thisis.delvecomic.com") - || getHost().contains("tnbtu.com")) { + if (theme1.contains(getHost())) { elem = doc.select("a.comic-nav-next").first(); if (elem == null) { throw new IOException("No more pages"); @@ -269,13 +288,7 @@ public class WordpressComicRipper extends AbstractHTMLRipper { @Override public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); - if (getHost().contains("www.totempole666.com") - || getHost().contains("buttsmithy.com") - || getHost().contains("themonsterunderthebed.net") - || getHost().contains("prismblush.com") - || getHost().contains("www.konradokonski.com") - || getHost().contains("thisis.delvecomic.com") - || getHost().contains("tnbtu.com")) { + if (theme1.contains(getHost())) { Element elem = doc.select("div.comic-table > div#comic > a > img").first(); // If doc is the last page in the comic then elem.attr("src") returns null // because there is no link to the next page diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java index ec8db0b0..8879c561 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java @@ -55,6 +55,7 @@ public class WordpressComicRipperTest extends RippersTest { WordpressComicRipper ripper = new WordpressComicRipper( new URL("http://www.konradokonski.com/sawdust/comic/get-up/")); testRipper(ripper); + } public void test_konradokonski_2() throws IOException { @@ -63,6 +64,13 @@ public class WordpressComicRipperTest extends RippersTest { testRipper(ripper); } + public void test_konradokonski_getAlbumTitle() throws IOException { + URL url = new URL("http://www.konradokonski.com/sawdust/comic/get-up/"); + WordpressComicRipper ripper = new WordpressComicRipper(url); + assertEquals("konradokonski.com_sawdust", ripper.getAlbumTitle(url)); + + } + /* // https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI public void test_freeadultcomix() throws IOException { @@ -89,6 +97,26 @@ public class WordpressComicRipperTest extends RippersTest { new URL("https://8muses.download/lustomic-playkittens-josh-samuel-porn-comics-8-muses/")); testRipper(ripper); } + + public void test_Eightmuses_getAlbumTitle() throws IOException { + URL url = new URL("https://8muses.download/lustomic-playkittens-josh-samuel-porn-comics-8-muses/"); + WordpressComicRipper ripper = new WordpressComicRipper(url); + assertEquals("8muses.download_lustomic-playkittens-josh-samuel-porn-comics-8-muses", + ripper.getAlbumTitle(url)); + } + + public void test_spyingwithlana_download() throws IOException { + WordpressComicRipper ripper = new WordpressComicRipper( + new URL("http://spyingwithlana.com/comic/the-big-hookup/")); + testRipper(ripper); + } + + public void test_spyingwithlana_getAlbumTitle() throws IOException { + URL url = new URL("http://spyingwithlana.com/comic/the-big-hookup/"); + WordpressComicRipper ripper = new WordpressComicRipper(url); + assertEquals("spyingwithlana_the-big-hookup", ripper.getAlbumTitle(url)); + } + // https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI // public void test_pepsaga() throws IOException { // WordpressComicRipper ripper = new WordpressComicRipper( From d604d4059659fca6653d70266feaa049b6d7d1b5 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 5 May 2018 05:36:00 -0400 Subject: [PATCH 074/140] Added option for addURLToDownload which sets the file extension to the files MIME type --- .../com/rarchives/ripme/ripper/AbstractRipper.java | 11 ++++++++--- .../com/rarchives/ripme/ripper/AlbumRipper.java | 6 +++--- .../rarchives/ripme/ripper/DownloadFileThread.java | 13 +++++++++++-- .../com/rarchives/ripme/ripper/VideoRipper.java | 3 ++- .../ripme/ripper/rippers/FivehundredpxRipper.java | 2 +- .../ripme/ripper/rippers/TsuminoRipper.java | 6 +++++- 6 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 4438ae2a..20889495 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -192,7 +192,8 @@ public abstract class AbstractRipper * True if downloaded successfully * False if failed to download */ - protected abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies); + protected abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies, + Boolean getFileExtFromMIME); /** * Queues image to be downloaded and saved. @@ -212,7 +213,7 @@ public abstract class AbstractRipper * True if downloaded successfully * False if failed to download */ - protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName, String extension) { + protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName, String extension, Boolean getFileExtFromMIME) { // Don't re-add the url if it was downloaded in a previous rip if (Utils.getConfigBoolean("remember.url_history", true) && !isThisATest()) { if (hasDownloadedURL(url.toExternalForm())) { @@ -257,7 +258,11 @@ public abstract class AbstractRipper logger.debug("Unable to write URL history file"); } } - return addURLToDownload(url, saveFileAs, referrer, cookies); + return addURLToDownload(url, saveFileAs, referrer, cookies, getFileExtFromMIME); + } + + protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName, String extension) { + return addURLToDownload(url, prefix, subdirectory, referrer, cookies, fileName, extension, false); } protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName) { diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 1726343a..f700f012 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -50,7 +50,7 @@ public abstract class AlbumRipper extends AbstractRipper { /** * Queues multiple URLs of single images to download from a single Album URL */ - public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies) { + public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { // Only download one file if this is a test. if (super.isThisATest() && (itemsPending.size() > 0 || itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { @@ -82,7 +82,7 @@ public abstract class AlbumRipper extends AbstractRipper { } else { itemsPending.put(url, saveAs); - DownloadFileThread dft = new DownloadFileThread(url, saveAs, this); + DownloadFileThread dft = new DownloadFileThread(url, saveAs, this, getFileExtFromMIME); if (referrer != null) { dft.setReferrer(referrer); } @@ -96,7 +96,7 @@ public abstract class AlbumRipper extends AbstractRipper { @Override public boolean addURLToDownload(URL url, File saveAs) { - return addURLToDownload(url, saveAs, null, null); + return addURLToDownload(url, saveAs, null, null, false); } /** diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index c62d58a6..42dedffe 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -8,6 +8,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URL; +import java.net.URLConnection; import java.util.HashMap; import java.util.Map; @@ -36,10 +37,11 @@ class DownloadFileThread extends Thread { private String prettySaveAs; private AbstractRipper observer; private int retries; + private Boolean getFileExtFromMIME; private final int TIMEOUT; - public DownloadFileThread(URL url, File saveAs, AbstractRipper observer) { + public DownloadFileThread(URL url, File saveAs, AbstractRipper observer, Boolean getFileExtFromMIME) { super(); this.url = url; this.saveAs = saveAs; @@ -47,6 +49,7 @@ class DownloadFileThread extends Thread { this.observer = observer; this.retries = Utils.getConfigInteger("download.retries", 1); this.TIMEOUT = Utils.getConfigInteger("download.timeout", 60000); + this.getFileExtFromMIME = getFileExtFromMIME; } public void setReferrer(String referrer) { @@ -143,9 +146,15 @@ class DownloadFileThread extends Thread { observer.downloadErrored(url, "Imgur image is 404: " + url.toExternalForm()); return; } - // Save file bis = new BufferedInputStream(huc.getInputStream()); + + // Check if we should get the file ext from the MIME type + if (getFileExtFromMIME) { + String fileExt = URLConnection.guessContentTypeFromStream(bis).replaceAll("image/", ""); + saveAs = new File(saveAs.toString() + "." + fileExt); + } + fos = new FileOutputStream(saveAs); IOUtils.copy(bis, fos); break; // Download successful: break out of infinite loop diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 13008cd9..29200d5a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -10,6 +10,7 @@ import java.util.Map; import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Utils; +import com.sun.org.apache.xpath.internal.operations.Bool; public abstract class VideoRipper extends AbstractRipper { @@ -70,7 +71,7 @@ public abstract class VideoRipper extends AbstractRipper { } @Override - public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies) { + public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { return addURLToDownload(url, saveAs); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java index 7e532943..93aedba2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java @@ -332,7 +332,7 @@ public class FivehundredpxRipper extends AbstractJSONRipper { String[] fields = u.split("/"); String prefix = getPrefix(index) + fields[fields.length - 3]; File saveAs = new File(getWorkingDir() + File.separator + prefix + ".jpg"); - addURLToDownload(url, saveAs, "", null); + addURLToDownload(url, saveAs, "", null, false); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java index 6030c9f0..7d35fc1d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java @@ -98,6 +98,10 @@ public class TsuminoRipper extends AbstractHTMLRipper { @Override public void downloadURL(URL url, int index) { sleep(1000); - addURLToDownload(url, getPrefix(index), "", null, null, null, "png"); + /* + There is no way to tell if an image returned from tsumino.com is a png to jpg. The content-type header is always + "image/jpeg" even when the image is a png. The file ext is not included in the url. + */ + addURLToDownload(url, getPrefix(index), "", null, null, null, null, true); } } From 2d1e648bd1b226c97a8303588162ab0d98695bf7 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 5 May 2018 05:41:48 -0400 Subject: [PATCH 075/140] 1.7.41: Added support for spyingwithlana.com; Added ManganeloRipper; Added support for dynasty-scans.com --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index d0be1909..9bd3de11 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.40 + 1.7.41 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index dcac88bc..ea3feafa 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.40", + "latestVersion": "1.7.41", "changeList": [ + "1.7.41: Added support for spyingwithlana.com; Added ManganeloRipper; Added support for dynasty-scans.com", "1.7.40: Added hypnohub.net ripper; Fixed rule34.xxx ripper; Tsumino Ripper now add .png to filenames", "1.7.39: Added rule34.xxx ripper; Added Gfycatporntube.com ripper; Fixed AbstractRipper subdir bug; Added AbstractRipper unit tests", "1.7.38: Added http and socks proxy support; Extended some unit tests to include getGid; Added HitomiRipper; hentaifoundry ripper now can rip all images from accounts", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 5287194d..92a75c78 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.40"; + private static final String DEFAULT_VERSION = "1.7.41"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 2001ada6f4291122a210e49db53f0b0d597ec308 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 6 May 2018 05:33:33 -0400 Subject: [PATCH 076/140] Fixed NudeGalsRipper --- .../ripme/ripper/rippers/NudeGalsRipper.java | 21 ++----------------- .../ripper/rippers/NudeGalsRipperTest.java | 4 ++++ 2 files changed, 6 insertions(+), 19 deletions(-) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/NudeGalsRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NudeGalsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NudeGalsRipper.java index d3cb0ab1..3300da50 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NudeGalsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NudeGalsRipper.java @@ -33,23 +33,6 @@ public class NudeGalsRipper extends AbstractHTMLRipper { return "nude-gals.com"; } - public String getAlbumTitle(URL url) throws MalformedURLException { - try { - Document doc = getFirstPage(); - Elements elems = doc.select("#left_col > #grid_title > .right"); - - String girl = elems.get(3).text(); - String magazine = elems.get(2).text(); - String title = elems.get(0).text(); - - return getHost() + "_" + girl + "-" + magazine + "-" + title; - } catch (Exception e) { - // Fall back to default album naming convention - logger.warn("Failed to get album title from " + url, e); - } - return super.getAlbumTitle(url); - } - @Override public String getGID(URL url) throws MalformedURLException { Pattern p; @@ -79,9 +62,9 @@ public class NudeGalsRipper extends AbstractHTMLRipper { public List getURLsFromPage(Document doc) { List imageURLs = new ArrayList<>(); - Elements thumbs = doc.select("#grid_container .grid > .grid_box"); + Elements thumbs = doc.select("img.thumbnail"); for (Element thumb : thumbs) { - String link = thumb.select("a").get(1).attr("href"); + String link = thumb.attr("src").replaceAll("thumbs/th_", ""); String imgSrc = "http://nude-gals.com/" + link; imageURLs.add(imgSrc); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NudeGalsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NudeGalsRipperTest.java new file mode 100644 index 00000000..e77fcea9 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NudeGalsRipperTest.java @@ -0,0 +1,4 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +public class NudeGalsRipperTest { +} From 7e766b2bc2703b61ca5261e7a24b6891466a18fa Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 6 May 2018 05:34:00 -0400 Subject: [PATCH 077/140] Added unit test for NudeGalsRipper --- .../tst/ripper/rippers/NudeGalsRipperTest.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NudeGalsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NudeGalsRipperTest.java index e77fcea9..3353eeb5 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NudeGalsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NudeGalsRipperTest.java @@ -1,4 +1,18 @@ package com.rarchives.ripme.tst.ripper.rippers; -public class NudeGalsRipperTest { +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.NudeGalsRipper; + +public class NudeGalsRipperTest extends RippersTest { + public void testRip() throws IOException { + NudeGalsRipper ripper = new NudeGalsRipper(new URL("https://nude-gals.com/photoshoot.php?photoshoot_id=5541")); + testRipper(ripper); + } + + public void testGetGID() throws IOException { + NudeGalsRipper ripper = new NudeGalsRipper(new URL("https://nude-gals.com/photoshoot.php?photoshoot_id=5541")); + assertEquals("5541", ripper.getGID( new URL("https://nude-gals.com/photoshoot.php?photoshoot_id=5541"))); + } } From f36a4dfd3b78ab5454b7b4978b32e58522156674 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 6 May 2018 05:43:57 -0400 Subject: [PATCH 078/140] Removed vine ripper --- .../ripme/ripper/rippers/VineRipper.java | 95 ------------------- 1 file changed, 95 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/VineRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VineRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VineRipper.java deleted file mode 100644 index 1ba53926..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VineRipper.java +++ /dev/null @@ -1,95 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.json.JSONArray; -import org.json.JSONObject; -import org.jsoup.HttpStatusException; - -import com.rarchives.ripme.ripper.AlbumRipper; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; -import com.rarchives.ripme.utils.Http; - -public class VineRipper extends AlbumRipper { - - private static final String DOMAIN = "vine.co", - HOST = "vine"; - - public VineRipper(URL url) throws IOException { - super(url); - } - - @Override - public boolean canRip(URL url) { - return url.getHost().endsWith(DOMAIN); - } - - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return new URL("http://vine.co/u/" + getGID(url)); - } - - @Override - public void rip() throws IOException { - int page = 0; - String baseURL = "https://vine.co/api/timelines/users/" + getGID(this.url); - JSONObject json = null; - while (true) { - page++; - String theURL = baseURL; - if (page > 1) { - theURL += "?page=" + page; - } - try { - logger.info(" Retrieving " + theURL); - sendUpdate(STATUS.LOADING_RESOURCE, theURL); - json = Http.url(theURL).getJSON(); - } catch (HttpStatusException e) { - logger.debug("Hit end of pages at page " + page, e); - break; - } - JSONArray records = json.getJSONObject("data").getJSONArray("records"); - for (int i = 0; i < records.length(); i++) { - String videoURL = records.getJSONObject(i).getString("videoUrl"); - addURLToDownload(new URL(videoURL)); - if (isThisATest()) { - break; - } - } - if (isThisATest()) { - break; - } - if (records.length() == 0) { - logger.info("Zero records returned"); - break; - } - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - logger.error("[!] Interrupted while waiting to load next page", e); - break; - } - } - waitForThreads(); - } - - @Override - public String getHost() { - return HOST; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://(www\\.)?vine\\.co/u/([0-9]+).*$"); - Matcher m = p.matcher(url.toExternalForm()); - if (!m.matches()) { - throw new MalformedURLException("Expected format: http://vine.co/u/######"); - } - return m.group(m.groupCount()); - } - -} From b8023fb4f1f501442a2ef88cccfe31fc6539b50a Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 6 May 2018 05:46:35 -0400 Subject: [PATCH 079/140] Removed test for removed ripper --- .../ripme/tst/ripper/rippers/VineRipperTest.java | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/VineRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VineRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VineRipperTest.java deleted file mode 100644 index 343a72b0..00000000 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VineRipperTest.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import java.io.IOException; -import java.net.URL; - -import com.rarchives.ripme.ripper.rippers.VineRipper; - -public class VineRipperTest extends RippersTest { - // https://github.com/RipMeApp/ripme/issues/181 - /* - public void testVineRip() throws IOException { - VineRipper ripper = new VineRipper(new URL("https://vine.co/u/954440445776334848")); - testRipper(ripper); - } - */ -} From 66bba724f9a0ef461d8e71219a60753f825cc6a0 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 6 May 2018 06:27:41 -0400 Subject: [PATCH 080/140] Added user support to SmuttyRipper --- .../ripme/ripper/rippers/SmuttyRipper.java | 125 +++++++++--------- 1 file changed, 62 insertions(+), 63 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SmuttyRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SmuttyRipper.java index 51992ec4..b61f2fef 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SmuttyRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SmuttyRipper.java @@ -3,24 +3,19 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.util.ArrayList; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import com.rarchives.ripme.ripper.AlbumRipper; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; +import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; -/** - * Appears to be broken as of 2015-02-11. - * Generating large image from thumbnail requires replacing "/m/" with something else: - * -> Sometimes "/b/" - * -> Sometimes "/p/" - * No way to know without loading the image page. - */ -public class SmuttyRipper extends AlbumRipper { + +public class SmuttyRipper extends AbstractHTMLRipper { private static final String DOMAIN = "smutty.com", HOST = "smutty"; @@ -29,6 +24,16 @@ public class SmuttyRipper extends AlbumRipper { super(url); } + @Override + public String getHost() { + return "smutty"; + } + + @Override + public String getDomain() { + return "smutty.com"; + } + @Override public boolean canRip(URL url) { return (url.getHost().endsWith(DOMAIN)); @@ -40,69 +45,57 @@ public class SmuttyRipper extends AlbumRipper { } @Override - public void rip() throws IOException { - int page = 0; - String url, tag = getGID(this.url); - boolean hasNextPage = true; - while (hasNextPage) { + public List getURLsFromPage(Document doc) { + List results = new ArrayList<>(); + for (Element image : doc.select("a.l > img")) { if (isStopped()) { break; } - page++; - url = "http://smutty.com/h/" + tag + "/?q=%23" + tag + "&page=" + page + "&sort=date&lazy=1"; - this.sendUpdate(STATUS.LOADING_RESOURCE, url); - logger.info(" Retrieving " + url); - Document doc; - try { - doc = Http.url(url) - .ignoreContentType() - .get(); - } catch (IOException e) { - if (e.toString().contains("Status=404")) { - logger.info("No more pages to load"); - } else { - logger.warn("Exception while loading " + url, e); - } - break; - } - for (Element image : doc.select("a.l > img")) { - if (isStopped()) { - break; - } - String imageUrl = image.attr("src"); + String imageUrl = image.attr("src"); - // Construct direct link to image based on thumbnail - StringBuilder sb = new StringBuilder(); - String[] fields = imageUrl.split("/"); - for (int i = 0; i < fields.length; i++) { - if (i == fields.length - 2 && fields[i].equals("m")) { - fields[i] = "b"; - } - sb.append(fields[i]); - if (i < fields.length - 1) { - sb.append("/"); - } + // Construct direct link to image based on thumbnail + StringBuilder sb = new StringBuilder(); + String[] fields = imageUrl.split("/"); + for (int i = 0; i < fields.length; i++) { + if (i == fields.length - 2 && fields[i].equals("m")) { + fields[i] = "b"; + } + sb.append(fields[i]); + if (i < fields.length - 1) { + sb.append("/"); } - imageUrl = sb.toString(); - addURLToDownload(new URL("http:" + imageUrl)); - } - if (doc.select("#next").size() == 0) { - break; // No more pages - } - // Wait before loading next page - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - logger.error("[!] Interrupted while waiting to load next album:", e); - break; } + imageUrl = sb.toString(); + results.add("http:" + imageUrl); } - waitForThreads(); + return results; } @Override - public String getHost() { - return HOST; + public Document getNextPage(Document doc) throws IOException { + Element elem = doc.select("a.next").first(); + if (elem == null) { + throw new IOException("No more pages"); + } + String nextPage = elem.attr("href"); + // Some times this returns a empty string + // This for stops that + if (nextPage.equals("")) { + throw new IOException("No more pages"); + } + else { + return Http.url("https://smutty.com" + nextPage).get(); + } + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); } @Override @@ -117,6 +110,12 @@ public class SmuttyRipper extends AlbumRipper { if (m.matches()) { return m.group(1).replace("%23", ""); } + + p = Pattern.compile("^https?://smutty.com/user/([a-zA-Z0-9\\-_]+)/?$"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } throw new MalformedURLException("Expected tag in URL (smutty.com/h/tag and not " + url); } From c500f20c02e5ab11931c19e1bf8bdd89b85ad196 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 6 May 2018 06:30:17 -0400 Subject: [PATCH 081/140] Added unit test for SmuttyRipper --- .../ripme/tst/ripper/rippers/SmuttyRipperTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java new file mode 100644 index 00000000..949e715f --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.SmuttyRipper; + +public class SmuttyRipperTest extends RippersTest { + public void testRip() throws IOException { + SmuttyRipper ripper = new SmuttyRipper(new URL("https://smutty.com/user/QUIGON/")); + testRipper(ripper); + } +} From 857b951956e44a76080de1255854e931c8e4fd9a Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 8 May 2018 07:28:18 -0400 Subject: [PATCH 082/140] Fixed regex --- .../com/rarchives/ripme/ripper/rippers/InstagramRipper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index 2b63a44f..717aaa30 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -430,7 +430,7 @@ public class InstagramRipper extends AbstractHTMLRipper { return null; } if (!rippingTag) { - Pattern jsP = Pattern.compile("o},queryId:.([a-zA-Z0-9]+)."); + Pattern jsP = Pattern.compile("\\?n.pagination:n},queryId:.([a-zA-Z0-9]+)."); Matcher m = jsP.matcher(sb.toString()); if (m.find()) { return m.group(1); @@ -442,7 +442,7 @@ public class InstagramRipper extends AbstractHTMLRipper { return m.group(1); } } - logger.info("Could not find query_hash on " + jsFileURL); + logger.error("Could not find query_hash on " + jsFileURL); return null; } From 925df36e3138b055dbff94833365ec7956184dc3 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 8 May 2018 08:41:39 -0400 Subject: [PATCH 083/140] 1.7.42: Added user support to SmuttyRipper; Removed vine ripper; Fixed NudeGalsRipper; addURLToDownload improvments; Fixed Instagram ripper --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 9bd3de11..41826a8e 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.41 + 1.7.42 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index ea3feafa..00677389 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.41", + "latestVersion": "1.7.42", "changeList": [ + "1.7.42: Added user support to SmuttyRipper; Removed vine ripper; Fixed NudeGalsRipper; addURLToDownload improvments; Fixed Instagram ripper", "1.7.41: Added support for spyingwithlana.com; Added ManganeloRipper; Added support for dynasty-scans.com", "1.7.40: Added hypnohub.net ripper; Fixed rule34.xxx ripper; Tsumino Ripper now add .png to filenames", "1.7.39: Added rule34.xxx ripper; Added Gfycatporntube.com ripper; Fixed AbstractRipper subdir bug; Added AbstractRipper unit tests", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 92a75c78..7b73f410 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.41"; + private static final String DEFAULT_VERSION = "1.7.42"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 52d14559e286939387b2ae26dedd4769ca5a0df8 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 9 May 2018 02:35:30 -0400 Subject: [PATCH 084/140] Fixed queryId regex --- .../com/rarchives/ripme/ripper/rippers/InstagramRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index 717aaa30..9c233b0d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -430,7 +430,7 @@ public class InstagramRipper extends AbstractHTMLRipper { return null; } if (!rippingTag) { - Pattern jsP = Pattern.compile("\\?n.pagination:n},queryId:.([a-zA-Z0-9]+)."); + Pattern jsP = Pattern.compile("o},queryId:.([a-zA-Z0-9]+)."); Matcher m = jsP.matcher(sb.toString()); if (m.find()) { return m.group(1); From c4eb31f1e9cbad13b80628fd0c1e20d96ef22ae3 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 9 May 2018 03:09:41 -0400 Subject: [PATCH 085/140] 1.7.43: Fixed queryId regex in instagram ripper --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 41826a8e..8566f2d6 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.42 + 1.7.43 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 00677389..6c4f3fc0 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.42", + "latestVersion": "1.7.43", "changeList": [ + "1.7.43: Fixed queryId regex in instagram ripper", "1.7.42: Added user support to SmuttyRipper; Removed vine ripper; Fixed NudeGalsRipper; addURLToDownload improvments; Fixed Instagram ripper", "1.7.41: Added support for spyingwithlana.com; Added ManganeloRipper; Added support for dynasty-scans.com", "1.7.40: Added hypnohub.net ripper; Fixed rule34.xxx ripper; Tsumino Ripper now add .png to filenames", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 7b73f410..4284eacc 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.42"; + private static final String DEFAULT_VERSION = "1.7.43"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From b760ed35c95a7039ce749f77babd59deb996c582 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 9 May 2018 10:34:39 -0400 Subject: [PATCH 086/140] Minor style changes of DeviantartRipper --- .../rarchives/ripme/ripper/rippers/DeviantartRipper.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index 4461188e..1b90be5c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -43,14 +43,17 @@ public class DeviantartRipper extends AbstractHTMLRipper { public String getHost() { return "deviantart"; } + @Override public String getDomain() { return "deviantart.com"; } + @Override public boolean hasDescriptionSupport() { return true; } + @Override public URL sanitizeURL(URL url) throws MalformedURLException { String u = url.toExternalForm(); @@ -121,6 +124,7 @@ public class DeviantartRipper extends AbstractHTMLRipper { .cookies(cookies) .get(); } + private String jsonToImage(Document page, String id) { Elements js = page.select("script[type=\"text/javascript\"]"); for (Element tag : js) { @@ -142,6 +146,7 @@ public class DeviantartRipper extends AbstractHTMLRipper { } return null; } + @Override public List getURLsFromPage(Document page) { List imageURLs = new ArrayList<>(); @@ -192,6 +197,7 @@ public class DeviantartRipper extends AbstractHTMLRipper { } return imageURLs; } + @Override public List getDescriptionsFromPage(Document page) { List textURLs = new ArrayList<>(); @@ -210,6 +216,7 @@ public class DeviantartRipper extends AbstractHTMLRipper { } return textURLs; } + @Override public Document getNextPage(Document page) throws IOException { if (isThisATest()) { From de6f696bb2c91075991de7790d4e46f046ef800c Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 9 May 2018 10:35:04 -0400 Subject: [PATCH 087/140] Added test for getGID --- .../ripme/tst/ripper/rippers/DeviantartRipperTest.java | 6 ++++++ .../ripme/tst/ripper/rippers/FuraffinityRipperTest.java | 6 ++++++ .../ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java | 6 ++++++ .../ripme/tst/ripper/rippers/ImagevenueRipperTest.java | 6 ++++++ .../ripme/tst/ripper/rippers/ImgboxRipperTest.java | 6 ++++++ .../ripme/tst/ripper/rippers/PornhubRipperTest.java | 6 ++++++ .../ripme/tst/ripper/rippers/Rule34RipperTest.java | 6 ++++++ .../ripme/tst/ripper/rippers/TheyiffgalleryRipperTest.java | 6 ++++++ .../ripme/tst/ripper/rippers/WebtoonsRipperTest.java | 6 ++++++ 9 files changed, 54 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java index a3fdfd81..a3e6a9c8 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java @@ -16,4 +16,10 @@ public class DeviantartRipperTest extends RippersTest { DeviantartRipper ripper = new DeviantartRipper(new URL("http://faterkcx.deviantart.com/gallery/")); testRipper(ripper); } + + public void testGetGID() throws IOException { + URL url = new URL("http://airgee.deviantart.com/gallery/"); + DeviantartRipper ripper = new DeviantartRipper(url); + assertEquals("airgee", ripper.getGID(url)); + } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java index 4285993b..90d66ecd 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java @@ -10,4 +10,10 @@ public class FuraffinityRipperTest extends RippersTest { FuraffinityRipper ripper = new FuraffinityRipper(new URL("https://www.furaffinity.net/gallery/mustardgas/")); testRipper(ripper); } + + public void testGetGID() throws IOException { + URL url = new URL("https://www.furaffinity.net/gallery/mustardgas/"); + FuraffinityRipper ripper = new FuraffinityRipper(url); + assertEquals("mustardgas", ripper.getGID(url)); + } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java index 6856eb06..e53c78e6 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java @@ -10,4 +10,10 @@ public class GfycatporntubeRipperTest extends RippersTest { GfycatporntubeRipper ripper = new GfycatporntubeRipper(new URL("https://gfycatporntube.com/blowjob-bunny-puts-on-a-show/")); testRipper(ripper); } + + public void testGetGID() throws IOException { + URL url = new URL("https://gfycatporntube.com/blowjob-bunny-puts-on-a-show/"); + GfycatporntubeRipper ripper = new GfycatporntubeRipper(url); + assertEquals("blowjob-bunny-puts-on-a-show", ripper.getGID(url)); + } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagevenueRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagevenueRipperTest.java index 135a7b0a..90d76442 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagevenueRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagevenueRipperTest.java @@ -10,4 +10,10 @@ public class ImagevenueRipperTest extends RippersTest { ImagevenueRipper ripper = new ImagevenueRipper(new URL("http://img120.imagevenue.com/galshow.php?gal=gallery_1373818527696_191lo")); testRipper(ripper); } + + public void testGetGID() throws IOException { + URL url = new URL("http://img120.imagevenue.com/galshow.php?gal=gallery_1373818527696_191lo"); + ImagevenueRipper ripper = new ImagevenueRipper(url); + assertEquals("gallery_1373818527696_191lo", ripper.getGID(url)); + } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgboxRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgboxRipperTest.java index db8e1680..27ebdca2 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgboxRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgboxRipperTest.java @@ -10,4 +10,10 @@ public class ImgboxRipperTest extends RippersTest { ImgboxRipper ripper = new ImgboxRipper(new URL("https://imgbox.com/g/FJPF7t26FD")); testRipper(ripper); } + + public void testGetGID() throws IOException { + URL url = new URL("https://imgbox.com/g/FJPF7t26FD"); + ImgboxRipper ripper = new ImgboxRipper(url); + assertEquals("FJPF7t26FD", ripper.getGID(url)); + } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java index 96685cbc..74bee8d9 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java @@ -10,4 +10,10 @@ public class PornhubRipperTest extends RippersTest { PornhubRipper ripper = new PornhubRipper(new URL("https://www.pornhub.com/album/15680522")); testRipper(ripper); } + + public void testGetGID() throws IOException { + URL url = new URL("https://www.pornhub.com/album/15680522"); + PornhubRipper ripper = new PornhubRipper(url); + assertEquals("15680522", ripper.getGID(url)); + } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Rule34RipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Rule34RipperTest.java index cffc807d..4c63d66e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Rule34RipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Rule34RipperTest.java @@ -11,4 +11,10 @@ public class Rule34RipperTest extends RippersTest { testRipper(ripper); } + public void testGetGID() throws IOException { + URL url = new URL("https://rule34.xxx/index.php?page=post&s=list&tags=bimbo"); + Rule34Ripper ripper = new Rule34Ripper(url); + assertEquals("bimbo", ripper.getGID(url)); + } + } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TheyiffgalleryRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TheyiffgalleryRipperTest.java index 3c7a8fbd..91fc0617 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TheyiffgalleryRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TheyiffgalleryRipperTest.java @@ -10,4 +10,10 @@ public class TheyiffgalleryRipperTest extends RippersTest { TheyiffgalleryRipper ripper = new TheyiffgalleryRipper(new URL("https://theyiffgallery.com/index?/category/4303")); testRipper(ripper); } + + public void testGetGID() throws IOException { + URL url = new URL("https://theyiffgallery.com/index?/category/4303"); + TheyiffgalleryRipper ripper = new TheyiffgalleryRipper(url); + assertEquals("4303", ripper.getGID(url)); + } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java index 4aa50d0a..16407ad7 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java @@ -15,4 +15,10 @@ public class WebtoonsRipperTest extends RippersTest { WebtoonsRipper ripper = new WebtoonsRipper(new URL("http://www.webtoons.com/en/drama/lookism/ep-145/viewer?title_no=1049&episode_no=145")); testRipper(ripper); } + + public void testGetGID() throws IOException { + URL url = new URL("http://www.webtoons.com/en/drama/my-boo/ep-33/viewer?title_no=1185&episode_no=33"); + WebtoonsRipper ripper = new WebtoonsRipper(url); + assertEquals("my-boo", ripper.getGID(url)); + } } From fbde7b9f97450a60a8a9ad489b76b9914877039a Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 10 May 2018 06:40:29 -0400 Subject: [PATCH 088/140] Fixed instagram ripper regex --- .../com/rarchives/ripme/ripper/rippers/InstagramRipper.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index 9c233b0d..efc4cb40 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -435,6 +435,12 @@ public class InstagramRipper extends AbstractHTMLRipper { if (m.find()) { return m.group(1); } + jsP = Pattern.compile("n.pagination:n},queryId:.([a-zA-Z0-9]+)."); + m = jsP.matcher(sb.toString()); + if (m.find()) { + return m.group(1); + } + } else { Pattern jsP = Pattern.compile("return e.tagMedia.byTagName.get\\(t\\).pagination},queryId:.([a-zA-Z0-9]+)."); Matcher m = jsP.matcher(sb.toString()); From 19e4a95c964a7dd0ac3cd42d50d72aa55c0083f8 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 10 May 2018 07:30:05 -0400 Subject: [PATCH 089/140] 1.7.44: Fixed instagram ripper regex --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 8566f2d6..0796e60f 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.43 + 1.7.44 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 6c4f3fc0..757058ae 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.43", + "latestVersion": "1.7.44", "changeList": [ + "1.7.44: Fixed instagram ripper regex", "1.7.43: Fixed queryId regex in instagram ripper", "1.7.42: Added user support to SmuttyRipper; Removed vine ripper; Fixed NudeGalsRipper; addURLToDownload improvments; Fixed Instagram ripper", "1.7.41: Added support for spyingwithlana.com; Added ManganeloRipper; Added support for dynasty-scans.com", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 4284eacc..11d18771 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.43"; + private static final String DEFAULT_VERSION = "1.7.44"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From bba39a5337de39f14cff1923a2886ab753f2a4a1 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 10 May 2018 12:42:15 -0400 Subject: [PATCH 090/140] Added getResourceBundle func --- .../java/com/rarchives/ripme/utils/Utils.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 9517c528..156c6ac0 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -1,9 +1,6 @@ package com.rarchives.ripme.utils; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.UnsupportedEncodingException; +import java.io.*; import java.lang.reflect.Constructor; import java.net.URISyntaxException; import java.net.URL; @@ -579,4 +576,14 @@ public class Utils { } return domainCookies; } + + public static ResourceBundle getResourceBundle() { + try { + ResourceBundle rb = ResourceBundle.getBundle("LabelsBundle", Locale.getDefault()); + return rb; + } catch (MissingResourceException e) { + ResourceBundle rb = ResourceBundle.getBundle("LabelsBundle", Locale.ROOT); + return rb; + } + } } From c51710b8335d4a14f6ec6d379b1d461ef60a0b33 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 10 May 2018 12:43:01 -0400 Subject: [PATCH 091/140] Started using resourceBUndle --- .../com/rarchives/ripme/ui/MainWindow.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 016d25f3..f76964bf 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -16,9 +16,7 @@ import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collections; -import java.util.Date; -import java.util.Enumeration; +import java.util.*; import java.util.List; import javax.imageio.ImageIO; @@ -138,6 +136,8 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static AbstractRipper ripper; + private ResourceBundle rb = Utils.getResourceBundle(); + private static void addCheckboxListener(JCheckBox checkBox, String configString) { checkBox.addActionListener(arg0 -> { Utils.setConfigBoolean(configString, checkBox.isSelected()); @@ -462,17 +462,17 @@ public final class MainWindow implements Runnable, RipStatusHandler { configUpdateButton = new JButton("Check for updates"); configUpdateLabel = new JLabel("Current version: " + UpdateUtils.getThisJarVersion(), JLabel.RIGHT); JLabel configThreadsLabel = new JLabel("Maximum download threads:", JLabel.RIGHT); - JLabel configTimeoutLabel = new JLabel("Timeout (in milliseconds):", JLabel.RIGHT); - JLabel configRetriesLabel = new JLabel("Retry download count:", JLabel.RIGHT); + JLabel configTimeoutLabel = new JLabel(rb.getString("timeout.mill"), JLabel.RIGHT); + JLabel configRetriesLabel = new JLabel(rb.getString("retry.download.count"), JLabel.RIGHT); configThreadsText = new JTextField(Integer.toString(Utils.getConfigInteger("threads.size", 3))); configTimeoutText = new JTextField(Integer.toString(Utils.getConfigInteger("download.timeout", 60000))); configRetriesText = new JTextField(Integer.toString(Utils.getConfigInteger("download.retries", 3))); - configOverwriteCheckbox = addNewCheckbox("Overwrite existing files?", "file.overwrite", false); - configAutoupdateCheckbox = addNewCheckbox("Auto-update?", "auto.update", true); - configPlaySound = addNewCheckbox("Sound when rip completes", "play.sound", false); + configOverwriteCheckbox = addNewCheckbox(rb.getString("overwrite.existing.files"), "file.overwrite", false); + configAutoupdateCheckbox = addNewCheckbox(rb.getString("auto.update"), "auto.update", true); + configPlaySound = addNewCheckbox(rb.getString("sound.when.rip.completes"), "play.sound", false); configShowPopup = addNewCheckbox("Notification when rip starts", "download.show_popup", false); - configSaveOrderCheckbox = addNewCheckbox("Preserve order", "download.save_order", true); - configSaveLogs = addNewCheckbox("Save logs", "log.save", false); + configSaveOrderCheckbox = addNewCheckbox(rb.getString("preserve.order"), "download.save_order", true); + configSaveLogs = addNewCheckbox(rb.getString("save.logs"), "log.save", false); configSaveURLsOnly = addNewCheckbox("Save URLs only", "urls_only.save", false); configSaveAlbumTitles = addNewCheckbox("Save album titles", "album_titles.save", true); configClipboardAutorip = addNewCheckbox("Autorip from Clipboard", "clipboard.autorip", false); From fe503e71b89b19929e21f53f90f69aad8311a039 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 10 May 2018 12:44:18 -0400 Subject: [PATCH 092/140] Started building ResourceBundle --- src/main/resources/LabelsBundle.properties | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 src/main/resources/LabelsBundle.properties diff --git a/src/main/resources/LabelsBundle.properties b/src/main/resources/LabelsBundle.properties new file mode 100644 index 00000000..ba884030 --- /dev/null +++ b/src/main/resources/LabelsBundle.properties @@ -0,0 +1,18 @@ +Log = Log +History = History +created = created +modified = modified +Queue = Queue +Configuration = Configuration + +# Keys for the Configuration menu + +current.version = Current version: +auto.update = Auto-update? +max.download.threads = Maximum download threads +timeout.mill = Timeout (in milliseconds): +retry.download.count = Retry download count +overwrite.existing.files = Overwrite existing files? +sound.when.rip.completes = Sound when rip completes +preserve.order = Preserve order +save.logs = Save logs \ No newline at end of file From c4ce162b481440825ffcf662207e499ce9cf59dd Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 10 May 2018 13:22:27 -0400 Subject: [PATCH 093/140] Extended RB --- src/main/resources/LabelsBundle.properties | 23 ++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/main/resources/LabelsBundle.properties b/src/main/resources/LabelsBundle.properties index ba884030..3a42ab0c 100644 --- a/src/main/resources/LabelsBundle.properties +++ b/src/main/resources/LabelsBundle.properties @@ -7,7 +7,8 @@ Configuration = Configuration # Keys for the Configuration menu -current.version = Current version: +current.version = Current version +check.for.updates = Check for updates auto.update = Auto-update? max.download.threads = Maximum download threads timeout.mill = Timeout (in milliseconds): @@ -15,4 +16,22 @@ retry.download.count = Retry download count overwrite.existing.files = Overwrite existing files? sound.when.rip.completes = Sound when rip completes preserve.order = Preserve order -save.logs = Save logs \ No newline at end of file +save.logs = Save logs +notification.when.rip.starts = Notification when rip starts +save.urls.only = Save URLs only +save.album.titles = Save album titles +autorip.from.clipboard = Autorip from Clipboard +save.descriptions = Save descriptions +prefer.mp4.over.gif = Prefer MP4 over GIF +restore.window.position = Restore window position +remember.url.history = Remember URL history +loading.history.from = Loading history from + +# Misc UI keys + +loading.history.from.configuration = Loading history from configuration +interrupted.while.waiting.to.rip.next.album = Interrupted while waiting to rip next album +inactive = Inactive +re-rip.checked = Re-rip Checked +remove = Remove +clear = Clear \ No newline at end of file From dc9f7ea33ee72c3fcf4cdef6114e39fedaea5fa5 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 10 May 2018 13:24:14 -0400 Subject: [PATCH 094/140] Now using RB for more UI elements --- .../com/rarchives/ripme/ui/MainWindow.java | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index f76964bf..f360351a 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -289,7 +289,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { gbc.gridx = 3; ripPanel.add(stopButton, gbc); gbc.weightx = 1; - statusLabel = new JLabel("Inactive"); + statusLabel = new JLabel(rb.getString("inactive")); statusLabel.setHorizontalAlignment(JLabel.CENTER); openButton = new JButton(); openButton.setVisible(false); @@ -402,9 +402,9 @@ public final class MainWindow implements Runnable, RipStatusHandler { historyTable.getColumnModel().getColumn(i).setPreferredWidth(width); } JScrollPane historyTableScrollPane = new JScrollPane(historyTable); - historyButtonRemove = new JButton("Remove"); - historyButtonClear = new JButton("Clear"); - historyButtonRerip = new JButton("Re-rip Checked"); + historyButtonRemove = new JButton(rb.getString("remove")); + historyButtonClear = new JButton(rb.getString("clear")); + historyButtonRerip = new JButton(rb.getString("re-rip.checked")); gbc.gridx = 0; // History List Panel JPanel historyTablePanel = new JPanel(new GridBagLayout()); @@ -459,9 +459,9 @@ public final class MainWindow implements Runnable, RipStatusHandler { configurationPanel.setBorder(emptyBorder); configurationPanel.setVisible(false); // TODO Configuration components - configUpdateButton = new JButton("Check for updates"); - configUpdateLabel = new JLabel("Current version: " + UpdateUtils.getThisJarVersion(), JLabel.RIGHT); - JLabel configThreadsLabel = new JLabel("Maximum download threads:", JLabel.RIGHT); + configUpdateButton = new JButton(rb.getString("check.for.updates")); + configUpdateLabel = new JLabel( rb.getString("current.version") + ": " + UpdateUtils.getThisJarVersion(), JLabel.RIGHT); + JLabel configThreadsLabel = new JLabel(rb.getString("max.download.threads") + ":", JLabel.RIGHT); JLabel configTimeoutLabel = new JLabel(rb.getString("timeout.mill"), JLabel.RIGHT); JLabel configRetriesLabel = new JLabel(rb.getString("retry.download.count"), JLabel.RIGHT); configThreadsText = new JTextField(Integer.toString(Utils.getConfigInteger("threads.size", 3))); @@ -470,16 +470,16 @@ public final class MainWindow implements Runnable, RipStatusHandler { configOverwriteCheckbox = addNewCheckbox(rb.getString("overwrite.existing.files"), "file.overwrite", false); configAutoupdateCheckbox = addNewCheckbox(rb.getString("auto.update"), "auto.update", true); configPlaySound = addNewCheckbox(rb.getString("sound.when.rip.completes"), "play.sound", false); - configShowPopup = addNewCheckbox("Notification when rip starts", "download.show_popup", false); + configShowPopup = addNewCheckbox(rb.getString("notification.when.rip.starts"), "download.show_popup", false); configSaveOrderCheckbox = addNewCheckbox(rb.getString("preserve.order"), "download.save_order", true); configSaveLogs = addNewCheckbox(rb.getString("save.logs"), "log.save", false); - configSaveURLsOnly = addNewCheckbox("Save URLs only", "urls_only.save", false); - configSaveAlbumTitles = addNewCheckbox("Save album titles", "album_titles.save", true); - configClipboardAutorip = addNewCheckbox("Autorip from Clipboard", "clipboard.autorip", false); - configSaveDescriptions = addNewCheckbox("Save descriptions", "descriptions.save", true); - configPreferMp4 = addNewCheckbox("Prefer MP4 over GIF","prefer.mp4", false); - configWindowPosition = addNewCheckbox("Restore window position", "window.position", true); - configURLHistoryCheckbox = addNewCheckbox("Remember URL history", "remember.url_history", true); + configSaveURLsOnly = addNewCheckbox(rb.getString("save.urls.only"), "urls_only.save", false); + configSaveAlbumTitles = addNewCheckbox(rb.getString("save.album.titles"), "album_titles.save", true); + configClipboardAutorip = addNewCheckbox(rb.getString("autorip.from.clipboard"), "clipboard.autorip", false); + configSaveDescriptions = addNewCheckbox(rb.getString("save.descriptions"), "descriptions.save", true); + configPreferMp4 = addNewCheckbox(rb.getString("prefer.mp4.over.gif"),"prefer.mp4", false); + configWindowPosition = addNewCheckbox(rb.getString("restore.window.position"), "window.position", true); + configURLHistoryCheckbox = addNewCheckbox(rb.getString("remember.url.history"), "remember.url_history", true); configLogLevelCombobox = new JComboBox(new String[] {"Log level: Error", "Log level: Warn", "Log level: Info", "Log level: Debug"}); configLogLevelCombobox.setSelectedItem(Utils.getConfigString("log.level", "Log level: Debug")); @@ -966,7 +966,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { HISTORY.clear(); if (historyFile.exists()) { try { - logger.info("Loading history from " + historyFile.getCanonicalPath()); + logger.info(rb.getString("loading.history.from") + " " + historyFile.getCanonicalPath()); HISTORY.fromFile(historyFile.getCanonicalPath()); } catch (IOException e) { logger.error("Failed to load history from file " + historyFile, e); @@ -979,7 +979,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { JOptionPane.ERROR_MESSAGE); } } else { - logger.info("Loading history from configuration"); + logger.info(rb.getString("loading.history.from.configuration")); HISTORY.fromList(Utils.getConfigList("download.history")); if (HISTORY.toList().size() == 0) { // Loaded from config, still no entries. @@ -1035,7 +1035,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { try { Thread.sleep(500); } catch (InterruptedException ie) { - logger.error("Interrupted while waiting to rip next album", ie); + logger.error(rb.getString("interrupted.while.waiting.to.rip.next.album"), ie); } ripNextAlbum(); } else { From 1c38a86d3d6fae4a4957c632e83e260120f386da Mon Sep 17 00:00:00 2001 From: Kevin Jiang Date: Fri, 11 May 2018 21:52:01 -0400 Subject: [PATCH 095/140] ImageBam album fixed, more Documentation Slightly fixed it, but don't know CSS so it's kinda a bootleg approach, but it works extremely well. Added bits of documentation here and there, and changed .size() == 0 to .isEmpty() for more readability. --- .../ripme/ripper/rippers/ImagebamRipper.java | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java index 323ad1de..b5fca14f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java @@ -1,5 +1,9 @@ package com.rarchives.ripme.ripper.rippers; +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.ripper.DownloadThreadPool; +import com.rarchives.ripme.utils.Http; +import com.rarchives.ripme.utils.Utils; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; @@ -7,16 +11,10 @@ import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; - import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; -import com.rarchives.ripme.utils.Http; -import com.rarchives.ripme.utils.Utils; - public class ImagebamRipper extends AbstractHTMLRipper { // Current HTML document @@ -71,7 +69,7 @@ public class ImagebamRipper extends AbstractHTMLRipper { public Document getNextPage(Document doc) throws IOException { // Find next page Elements hrefs = doc.select("a.pagination_current + a.pagination_link"); - if (hrefs.size() == 0) { + if (hrefs.isEmpty()) { throw new IOException("No more pages"); } String nextUrl = "http://www.imagebam.com" + hrefs.first().attr("href"); @@ -121,8 +119,8 @@ public class ImagebamRipper extends AbstractHTMLRipper { * Handles case when site has IP-banned the user. */ private class ImagebamImageThread extends Thread { - private URL url; - private int index; + private URL url; //link to "image page" + private int index; //index in album ImagebamImageThread(URL url, int index) { super(); @@ -134,24 +132,39 @@ public class ImagebamRipper extends AbstractHTMLRipper { public void run() { fetchImage(); } - + + /** + * Rips useful image from "image page" + */ private void fetchImage() { try { Document doc = Http.url(url).get(); // Find image - Elements images = doc.select(".image-container img"); - if (images.size() == 0) { + Elements metaTags = doc.getElementsByTag("meta"); + + String imgsrc = "";//initialize, so no NullPointerExceptions should ever happen. + + for (Element metaTag: metaTags) { + //the direct link to the image seems to always be linked in the part of the html. + if(metaTag.attr("property").equals("og:image")) { + imgsrc = metaTag.attr("content"); + logger.info("Found URL " + imgsrc); + break;//only one (useful) image possible for an "image page". + } + } + + //for debug, or something goes wrong. + if (imgsrc.isEmpty()) { logger.warn("Image not found at " + this.url); return; } - Element image = images.first(); - String imgsrc = image.attr("src"); - logger.info("Found URL " + imgsrc); + // Provide prefix and let the AbstractRipper "guess" the filename String prefix = ""; if (Utils.getConfigBoolean("download.save_order", true)) { prefix = String.format("%03d_", index); } + addURLToDownload(new URL(imgsrc), prefix); } catch (IOException e) { logger.error("[!] Exception while loading/parsing " + this.url, e); From 03963eac55e22b05f2af538232c169a05b50789b Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 11 May 2018 23:56:02 -0400 Subject: [PATCH 096/140] Fixed hentai2read ripper --- .../ripper/rippers/Hentai2readRipper.java | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java index 405d0563..a87575bd 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java @@ -44,20 +44,15 @@ public class Hentai2readRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { - Document tempDoc; - // get the first page of the comic - if (url.toExternalForm().substring(url.toExternalForm().length() - 1).equals("/")) { - tempDoc = Http.url(url + "1").get(); - } else { - tempDoc = Http.url(url + "/1").get(); + try { + Document tempDoc; + tempDoc = Http.url(url).get(); + // Get the thumbnail page so we can rip all images without loading every page in the comic + String thumbnailLink = tempDoc.select("a[data-original-title=Thumbnails").attr("href"); + return Http.url(thumbnailLink).get(); + } catch (IOException e) { + throw new IOException("Unable to get first page"); } - for (Element el : tempDoc.select("ul.nav > li > a")) { - if (el.attr("href").startsWith("https://hentai2read.com/thumbnails/")) { - // Get the page with the thumbnails - return Http.url(el.attr("href")).get(); - } - } - throw new IOException("Unable to get first page"); } @Override From 608c825ccab16bf7d5183157a5bf8813498e1f3c Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 12 May 2018 00:17:02 -0400 Subject: [PATCH 097/140] TsuminoRipper no longer requires album name to download --- .../com/rarchives/ripme/ripper/rippers/TsuminoRipper.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java index 7d35fc1d..70c023d3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java @@ -59,11 +59,16 @@ public class TsuminoRipper extends AbstractHTMLRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("https?://www.tsumino.com/Book/Info/([0-9]+)/([a-zA-Z0-9_-]*)"); + Pattern p = Pattern.compile("https?://www.tsumino.com/Book/Info/([0-9]+)/([a-zA-Z0-9_-]*)/?"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1) + "_" + m.group(2); } + p = Pattern.compile("https?://www.tsumino.com/Book/Info/([0-9]+)/?"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } throw new MalformedURLException("Expected tsumino URL format: " + "tsumino.com/Book/Info/ID/TITLE - got " + url + " instead"); } From 4a160deaa5ffaf4247b07845fb1ef7fc4ac69135 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 12 May 2018 03:00:25 -0400 Subject: [PATCH 098/140] Now uses translation for more strings --- .../com/rarchives/ripme/ui/MainWindow.java | 35 +++++++++---------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index f360351a..95961f44 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -138,6 +138,15 @@ public final class MainWindow implements Runnable, RipStatusHandler { private ResourceBundle rb = Utils.getResourceBundle(); + private void updateQueueLabel() { + if (queueListModel.size() > 0) { + optionQueue.setText( rb.getString("Queue") + " (" + queueListModel.size() + ")"); + } else { + optionQueue.setText(rb.getString("Queue")); + } + } + + private static void addCheckboxListener(JCheckBox checkBox, String configString) { checkBox.addActionListener(arg0 -> { Utils.setConfigBoolean(configString, checkBox.isSelected()); @@ -307,10 +316,10 @@ public final class MainWindow implements Runnable, RipStatusHandler { JPanel optionsPanel = new JPanel(new GridBagLayout()); optionsPanel.setBorder(emptyBorder); - optionLog = new JButton("Log"); - optionHistory = new JButton("History"); - optionQueue = new JButton("Queue"); - optionConfiguration = new JButton("Configuration"); + optionLog = new JButton(rb.getString("Log")); + optionHistory = new JButton(rb.getString("History")); + optionQueue = new JButton(rb.getString("Queue")); + optionConfiguration = new JButton(rb.getString("Configuration")); optionLog.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); optionHistory.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); optionQueue.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); @@ -440,11 +449,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { for (String item : Utils.getConfigList("queue")) { queueListModel.addElement(item); } - if (queueListModel.size() > 0) { - optionQueue.setText("Queue (" + queueListModel.size() + ")"); - } else { - optionQueue.setText("Queue"); - } + updateQueueLabel(); gbc.gridx = 0; JPanel queueListPanel = new JPanel(new GridBagLayout()); gbc.fill = GridBagConstraints.BOTH; @@ -785,11 +790,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { queueListModel.addListDataListener(new ListDataListener() { @Override public void intervalAdded(ListDataEvent arg0) { - if (queueListModel.size() > 0) { - optionQueue.setText("Queue (" + queueListModel.size() + ")"); - } else { - optionQueue.setText("Queue"); - } + updateQueueLabel(); if (!isRipping) { ripNextAlbum(); } @@ -1025,11 +1026,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { return; } String nextAlbum = (String) queueListModel.remove(0); - if (queueListModel.isEmpty()) { - optionQueue.setText("Queue"); - } else { - optionQueue.setText("Queue (" + queueListModel.size() + ")"); - } + updateQueueLabel(); Thread t = ripAlbum(nextAlbum); if (t == null) { try { From c1dfb1e5d8f9028e52b7c3d8d44fda4d7d455eef Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 12 May 2018 03:01:01 -0400 Subject: [PATCH 099/140] Added spainish UI translation --- .../resources/LabelsBundle_es_ES.properties | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 src/main/resources/LabelsBundle_es_ES.properties diff --git a/src/main/resources/LabelsBundle_es_ES.properties b/src/main/resources/LabelsBundle_es_ES.properties new file mode 100644 index 00000000..a1aa5a4a --- /dev/null +++ b/src/main/resources/LabelsBundle_es_ES.properties @@ -0,0 +1,37 @@ +Log = Log +History = Historia +created = creado +modified = modificado +Queue = Cola +Configuration = Configuracion + +# Keys for the Configuration menu + +current.version = Version Actual +check.for.updates = Buscar actualizaciones +auto.update = Auto-actualizar? +max.download.threads = Maximos procesos de descarga +timeout.mill = Timeout (in milliseconds): +retry.download.count = Numero de reintentos de descarga +overwrite.existing.files = Sobreescribir archivos existentes? +sound.when.rip.completes = Sonar cuando el Rip termina +preserve.order = Mantener orden +save.logs = Guardar logs +notification.when.rip.starts = Notificar cuando el Rip comienza +save.urls.only = Guardar solamente URLs +save.album.titles = Guardar titulos de albunes +autorip.from.clipboard = Autorip desde Portapapeles +save.descriptions = Guardar descripciones +prefer.mp4.over.gif = Preferir MP4 sobre GIF +restore.window.position = Restaurar posicion de ventana +remember.url.history = Recordar historia URL +loading.history.from = Cargando historia desde + +# Misc UI keys + +loading.history.from.configuration = Cargando historia desde la configuracion +interrupted.while.waiting.to.rip.next.album = Interrumpido esperando el Rip del proximo album +inactive = Inactivo +re-rip.checked = Re-rip marcado +remove = Quitar +clear = Limpiar \ No newline at end of file From 89119d48f916f41ed8e87b6a0f0570445c40ea40 Mon Sep 17 00:00:00 2001 From: TheRec Date: Sat, 12 May 2018 13:00:54 +0200 Subject: [PATCH 100/140] Added French (Switzerland) translation. --- .../resources/LabelsBundle_fr_CH.properties | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 src/main/resources/LabelsBundle_fr_CH.properties diff --git a/src/main/resources/LabelsBundle_fr_CH.properties b/src/main/resources/LabelsBundle_fr_CH.properties new file mode 100644 index 00000000..2b568891 --- /dev/null +++ b/src/main/resources/LabelsBundle_fr_CH.properties @@ -0,0 +1,37 @@ +Log = Journal +History = Historique +created = créé le +modified = modifié le +Queue = File d'attente +Configuration = Configuration + +# Keys for the Configuration menu + +current.version = Version actuelle +check.for.updates = Vérifier mises à jour +auto.update = Mises à jour automatiques? +max.download.threads = Maximum download threads +timeout.mill = Délai d'expiration (en millisecondes): +retry.download.count = Nombre d'essais téléchargement +overwrite.existing.files = Remplacer fichiers existants ? +sound.when.rip.completes = Son lorsque le rip est terminé +preserve.order = Conserver l'ordre +save.logs = Enregistrer journaux +notification.when.rip.starts = Notification lorsqu'un rip commence +save.urls.only = Enregistrer URL uniquement +save.album.titles = Enregistrer titres d'album +autorip.from.clipboard = Autorip depuis presse-papier +save.descriptions = Enregistrer descriptions +prefer.mp4.over.gif = Préférer MP4 à GIF +restore.window.position = Restaurer la position de la fenêtre +remember.url.history = Se souvenir de l'historique des URL +loading.history.from = Charger l'historique depuis + +# Misc UI keys + +loading.history.from.configuration = Charger l'historique depuis la configuration +interrupted.while.waiting.to.rip.next.album = Interrompu lors de l'attente pour ripper le prochain album +inactive = Inactif +re-rip.checked = Re-rip vérifié +remove = Enlever +clear = Effacer \ No newline at end of file From 3061125b5ba4fd2954d91da0abfa81a7add0ec27 Mon Sep 17 00:00:00 2001 From: TheRec Date: Sat, 12 May 2018 13:10:47 +0200 Subject: [PATCH 101/140] Added missing translation to French (Switzerland) --- src/main/resources/LabelsBundle_fr_CH.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/LabelsBundle_fr_CH.properties b/src/main/resources/LabelsBundle_fr_CH.properties index 2b568891..e2cedff6 100644 --- a/src/main/resources/LabelsBundle_fr_CH.properties +++ b/src/main/resources/LabelsBundle_fr_CH.properties @@ -10,7 +10,7 @@ Configuration = Configuration current.version = Version actuelle check.for.updates = Vérifier mises à jour auto.update = Mises à jour automatiques? -max.download.threads = Maximum download threads +max.download.threads = Nombre de téléchargements parallèles maximum timeout.mill = Délai d'expiration (en millisecondes): retry.download.count = Nombre d'essais téléchargement overwrite.existing.files = Remplacer fichiers existants ? From def01bfb45fc47e94cdc95fe8ceb882c4f5454ec Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 12 May 2018 10:49:31 -0400 Subject: [PATCH 102/140] Added German translation --- .../rarchives/ripme/utils/UTF8Control.java | 44 +++++++++++++++++++ .../java/com/rarchives/ripme/utils/Utils.java | 2 +- .../resources/LabelsBundle_de_DE.properties | 38 ++++++++++++++++ 3 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/rarchives/ripme/utils/UTF8Control.java create mode 100644 src/main/resources/LabelsBundle_de_DE.properties diff --git a/src/main/java/com/rarchives/ripme/utils/UTF8Control.java b/src/main/java/com/rarchives/ripme/utils/UTF8Control.java new file mode 100644 index 00000000..2441ed7c --- /dev/null +++ b/src/main/java/com/rarchives/ripme/utils/UTF8Control.java @@ -0,0 +1,44 @@ +package com.rarchives.ripme.utils; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URL; +import java.net.URLConnection; +import java.util.Locale; +import java.util.PropertyResourceBundle; +import java.util.ResourceBundle; + +public class UTF8Control extends ResourceBundle.Control { + public ResourceBundle newBundle + (String baseName, Locale locale, String format, ClassLoader loader, boolean reload) + throws IllegalAccessException, InstantiationException, IOException + { + // The below is a copy of the default implementation. + String bundleName = toBundleName(baseName, locale); + String resourceName = toResourceName(bundleName, "properties"); + ResourceBundle bundle = null; + InputStream stream = null; + if (reload) { + URL url = loader.getResource(resourceName); + if (url != null) { + URLConnection connection = url.openConnection(); + if (connection != null) { + connection.setUseCaches(false); + stream = connection.getInputStream(); + } + } + } else { + stream = loader.getResourceAsStream(resourceName); + } + if (stream != null) { + try { + // Only this line is changed to make it to read properties files as UTF-8. + bundle = new PropertyResourceBundle(new InputStreamReader(stream, "UTF-8")); + } finally { + stream.close(); + } + } + return bundle; + } +} \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 156c6ac0..e6617f46 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -579,7 +579,7 @@ public class Utils { public static ResourceBundle getResourceBundle() { try { - ResourceBundle rb = ResourceBundle.getBundle("LabelsBundle", Locale.getDefault()); + ResourceBundle rb = ResourceBundle.getBundle("LabelsBundle", Locale.getDefault(), new UTF8Control()); return rb; } catch (MissingResourceException e) { ResourceBundle rb = ResourceBundle.getBundle("LabelsBundle", Locale.ROOT); diff --git a/src/main/resources/LabelsBundle_de_DE.properties b/src/main/resources/LabelsBundle_de_DE.properties new file mode 100644 index 00000000..da1fc8e2 --- /dev/null +++ b/src/main/resources/LabelsBundle_de_DE.properties @@ -0,0 +1,38 @@ +Log = Log +History = Verlauf +created = erstellt +modified = geändert +Queue = Queue +Configuration = Konfiguration + +# Keys for the Configuration menu + +current.version = Aktuelle Version +check.for.updates = Suche nach Aktualisierungen +auto.update = Automatisch Aktualisieren? +max.download.threads = Maximum download threads +timeout.mill = Timeout (in milliseconds): +retry.download.count = Anzahl der Downloadversuche +overwrite.existing.files = Ãœberschreibe bereits existierende Dateien? +sound.when.rip.completes = Ton abspielen bei fertigem Download +preserve.order = Reihenfolge beibehalten +save.logs = Speichere Logs +notification.when.rip.starts = Benachrichtigung wenn Download startet +save.urls.only = Speicher nur URLs +save.album.titles = Speichere Albumtitels +autorip.from.clipboard = Automatisch Downloaden von der Zwischenablage +save.descriptions = Speichere Beschreibungen +prefer.mp4.over.gif = Bevorzuge MP4 über GIF +restore.window.position = Wieder herstellen der Fensterposition +remember.url.history = Erinnere URL Verlauf +loading.history.from = Lade Verlauf von + +# Misc UI keys + +loading.history.from.configuration = Lade Verlauf aus Konfiguration +interrupted.while.waiting.to.rip.next.album = Unterbrochen während Download des nächsten Albums +inactive = Inaktiv +re-rip.checked = Re-rip Ãœberprüft +remove = Entfernen +clear = Leeren + From 1fe3f10be72e85406ad7620591f43ce954d12b31 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 12 May 2018 10:54:15 -0400 Subject: [PATCH 103/140] Added comment showing where the code is from --- src/main/java/com/rarchives/ripme/utils/UTF8Control.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/utils/UTF8Control.java b/src/main/java/com/rarchives/ripme/utils/UTF8Control.java index 2441ed7c..6cd81bbb 100644 --- a/src/main/java/com/rarchives/ripme/utils/UTF8Control.java +++ b/src/main/java/com/rarchives/ripme/utils/UTF8Control.java @@ -9,6 +9,8 @@ import java.util.Locale; import java.util.PropertyResourceBundle; import java.util.ResourceBundle; +// Code taken from https://stackoverflow.com/questions/4659929/how-to-use-utf-8-in-resource-properties-with-resourcebundle/4660195#4660195 + public class UTF8Control extends ResourceBundle.Control { public ResourceBundle newBundle (String baseName, Locale locale, String format, ClassLoader loader, boolean reload) From ee27d69280c14844cefbafe27616a8e6d1aa3948 Mon Sep 17 00:00:00 2001 From: Jason <37483675+jasonpw@users.noreply.github.com> Date: Sat, 12 May 2018 17:25:01 +0200 Subject: [PATCH 104/140] Portuguese from Portugal translation I decided to help with the portuguese translation, I hope it's just as accurate as the english version! --- .../resources/LabelsBundle_pt_PT.properties | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 src/main/resources/LabelsBundle_pt_PT.properties diff --git a/src/main/resources/LabelsBundle_pt_PT.properties b/src/main/resources/LabelsBundle_pt_PT.properties new file mode 100644 index 00000000..61581728 --- /dev/null +++ b/src/main/resources/LabelsBundle_pt_PT.properties @@ -0,0 +1,37 @@ +Log = Registo +History = Histórico +created = criado +modified = modificado +Queue = Fila +Configuration = Configuração + +# Keys for the Configuration menu + +current.version = Versão atual +check.for.updates = Verificar atualizações +auto.update = Atualização automática? +max.download.threads = Número máximo de processos de transferência +timeout.mill = Timeout (em milissegundos): +retry.download.count = Número de novas tentativas de transferência +overwrite.existing.files = Sobrescrever ficheiros existentes? +sound.when.rip.completes = Notificar quando o rip é concluído +preserve.order = Manter a ordem +save.logs = Guardar registos +notification.when.rip.starts = Notificar quando o rip começar +save.urls.only = Apenas guardar URLs +save.album.titles = Guardar os títulos de álbuns +autorip.from.clipboard = Autorip da área de transferência +save.descriptions = Guardar descrições +prefer.mp4.over.gif = Preferir MP4 a GIF +restore.window.position = Restaurar posição da janela +remember.url.history = Lembrar histórico de URL +loading.history.from = Carregar histórico de + +# Misc UI keys + +loading.history.from.configuration = A carregar o histórico da configuração +interrupted.while.waiting.to.rip.next.album = Interrompido durante a espera do rip do próximo álbum +inactive = Inativo +re-rip.checked = Re-rip verificado +remove = Remover +clear = Limpar From 14acc08aa188fb6b22e34855b52ca8be2eb58e2c Mon Sep 17 00:00:00 2001 From: TheRec Date: Sat, 12 May 2018 23:35:02 +0200 Subject: [PATCH 105/140] Converted French (Switzerland) translation from ANSI to UTF-8 --- .../resources/LabelsBundle_fr_CH.properties | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/main/resources/LabelsBundle_fr_CH.properties b/src/main/resources/LabelsBundle_fr_CH.properties index e2cedff6..1e23645b 100644 --- a/src/main/resources/LabelsBundle_fr_CH.properties +++ b/src/main/resources/LabelsBundle_fr_CH.properties @@ -1,20 +1,20 @@ Log = Journal History = Historique -created = créé le -modified = modifié le +created = créé le +modified = modifié le Queue = File d'attente Configuration = Configuration # Keys for the Configuration menu current.version = Version actuelle -check.for.updates = Vérifier mises à jour -auto.update = Mises à jour automatiques? -max.download.threads = Nombre de téléchargements parallèles maximum -timeout.mill = Délai d'expiration (en millisecondes): -retry.download.count = Nombre d'essais téléchargement +check.for.updates = Vérifier mises à jour +auto.update = Mises à jour automatiques? +max.download.threads = Nombre de téléchargements parallèles maximum +timeout.mill = Délai d'expiration (en millisecondes): +retry.download.count = Nombre d'essais téléchargement overwrite.existing.files = Remplacer fichiers existants ? -sound.when.rip.completes = Son lorsque le rip est terminé +sound.when.rip.completes = Son lorsque le rip est terminé preserve.order = Conserver l'ordre save.logs = Enregistrer journaux notification.when.rip.starts = Notification lorsqu'un rip commence @@ -22,8 +22,8 @@ save.urls.only = Enregistrer URL uniquement save.album.titles = Enregistrer titres d'album autorip.from.clipboard = Autorip depuis presse-papier save.descriptions = Enregistrer descriptions -prefer.mp4.over.gif = Préférer MP4 à GIF -restore.window.position = Restaurer la position de la fenêtre +prefer.mp4.over.gif = Préférer MP4 à GIF +restore.window.position = Restaurer la position de la fenêtre remember.url.history = Se souvenir de l'historique des URL loading.history.from = Charger l'historique depuis @@ -32,6 +32,6 @@ loading.history.from = Charger l'historique depuis loading.history.from.configuration = Charger l'historique depuis la configuration interrupted.while.waiting.to.rip.next.album = Interrompu lors de l'attente pour ripper le prochain album inactive = Inactif -re-rip.checked = Re-rip vérifié +re-rip.checked = Re-rip vérifié remove = Enlever clear = Effacer \ No newline at end of file From 96bf2ad3a7d655c5d5311f36ae7491bd9620f6ec Mon Sep 17 00:00:00 2001 From: Kevin Jiang Date: Sat, 12 May 2018 17:58:27 -0400 Subject: [PATCH 106/140] Update ImagebamRipper.java --- .../com/rarchives/ripme/ripper/rippers/ImagebamRipper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java index b5fca14f..b33f5624 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java @@ -146,7 +146,7 @@ public class ImagebamRipper extends AbstractHTMLRipper { for (Element metaTag: metaTags) { //the direct link to the image seems to always be linked in the part of the html. - if(metaTag.attr("property").equals("og:image")) { + if (metaTag.attr("property").equals("og:image")) { imgsrc = metaTag.attr("content"); logger.info("Found URL " + imgsrc); break;//only one (useful) image possible for an "image page". @@ -171,4 +171,4 @@ public class ImagebamRipper extends AbstractHTMLRipper { } } } -} \ No newline at end of file +} From a670de675a0550d8fb93723407efbf0f8239d7d4 Mon Sep 17 00:00:00 2001 From: Kevin Jiang Date: Sat, 12 May 2018 17:58:55 -0400 Subject: [PATCH 107/140] if( if ( --- .../com/rarchives/ripme/ripper/rippers/DeviantartRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index 8ecb8086..9ab23ec8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -121,7 +121,7 @@ public class DeviantartRipper extends AbstractHTMLRipper { String username = Utils.getConfigString("deviantart.username", new String(Base64.decode("Z3JhYnB5"))); String password = Utils.getConfigString("deviantart.password", new String(Base64.decode("ZmFrZXJz"))); - if(username == null || password == null) { + if (username == null || password == null) { logger.debug("No DeviantArt login provided."); cookies.put("agegate_state","1"); // Bypasses the age gate } else { From 00696c3450e6c0321a13786dfd044d16fe44a669 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 13 May 2018 08:56:14 -0400 Subject: [PATCH 108/140] Can now set UI lang in config file --- src/main/java/com/rarchives/ripme/utils/Utils.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index e6617f46..cd1048df 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -578,6 +578,11 @@ public class Utils { } public static ResourceBundle getResourceBundle() { + if (!getConfigString("lang", "").equals("")) { + String[] langCode = getConfigString("lang", "").split("_"); + logger.info("Setting locale to " + getConfigString("lang", "")); + return ResourceBundle.getBundle("LabelsBundle", new Locale(langCode[0], langCode[1]), new UTF8Control()); + } try { ResourceBundle rb = ResourceBundle.getBundle("LabelsBundle", Locale.getDefault(), new UTF8Control()); return rb; From 34425361aeec2cc12f782b7a2afb8a4be820d266 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 13 May 2018 09:14:42 -0400 Subject: [PATCH 109/140] 1.7.45: Fixed hentai2read ripper; ImageBam album fixed; Added various translations; TsuminoRipper no longer requires album name to download --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 0796e60f..317b816e 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.44 + 1.7.45 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 757058ae..73b9b240 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.44", + "latestVersion": "1.7.45", "changeList": [ + "1.7.45: Fixed hentai2read ripper; ImageBam album fixed; Added various translations; TsuminoRipper no longer requires album name to download", "1.7.44: Fixed instagram ripper regex", "1.7.43: Fixed queryId regex in instagram ripper", "1.7.42: Added user support to SmuttyRipper; Removed vine ripper; Fixed NudeGalsRipper; addURLToDownload improvments; Fixed Instagram ripper", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 11d18771..38e88ae7 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.44"; + private static final String DEFAULT_VERSION = "1.7.45"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 9d8c6f6ce220080a17f1d4b3df5bd32e532ef0db Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 13 May 2018 09:38:59 -0400 Subject: [PATCH 110/140] EightmusesRipper now adds file extension to filename --- .../com/rarchives/ripme/ripper/rippers/EightmusesRipper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java index ca9c24e3..3e06695b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java @@ -126,9 +126,9 @@ public class EightmusesRipper extends AbstractHTMLRipper { image = getFullSizeImage(imageHref); URL imageUrl = new URL(image); if (Utils.getConfigBoolean("8muses.use_short_names", false)) { - addURLToDownload(imageUrl, getPrefixShort(x), getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies, ""); + addURLToDownload(imageUrl, getPrefixShort(x), getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies, "", null, true); } else { - addURLToDownload(imageUrl, getPrefixLong(x), getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies); + addURLToDownload(imageUrl, getPrefixLong(x), getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies, "", null, true); } // X is our page index x++; From 98ad786529e3a5d0e4c26538aa6eaf59403a7d19 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 13 May 2018 10:06:19 -0400 Subject: [PATCH 111/140] Disabled test for broken ripper --- .../tst/ripper/rippers/VideoRippersTest.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java index ae31192b..6a7df184 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java @@ -38,14 +38,17 @@ public class VideoRippersTest extends RippersTest { } } - public void testTwitchVideoRipper() throws IOException { - List contentURLs = new ArrayList<>(); - contentURLs.add(new URL("https://clips.twitch.tv/FaithfulIncredulousPotTBCheesePull")); - for (URL url : contentURLs) { - TwitchVideoRipper ripper = new TwitchVideoRipper(url); - videoTestHelper(ripper); - } - } + +// Test disbaled. See https://github.com/RipMeApp/ripme/issues/574 + +// public void testTwitchVideoRipper() throws IOException { +// List contentURLs = new ArrayList<>(); +// contentURLs.add(new URL("https://clips.twitch.tv/FaithfulIncredulousPotTBCheesePull")); +// for (URL url : contentURLs) { +// TwitchVideoRipper ripper = new TwitchVideoRipper(url); +// videoTestHelper(ripper); +// } +// } public void testXhamsterRipper() throws IOException { List contentURLs = new ArrayList<>(); From 046fa093a3adf55dc39ea39a7981774a31f694b3 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 13 May 2018 10:11:38 -0400 Subject: [PATCH 112/140] SinnercomicsRipper no longer adds -page-01 to folder names --- .../com/rarchives/ripme/ripper/rippers/SinnercomicsRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SinnercomicsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SinnercomicsRipper.java index 326de1a1..166bce88 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SinnercomicsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SinnercomicsRipper.java @@ -34,7 +34,7 @@ public class SinnercomicsRipper extends AbstractHTMLRipper { Pattern p = Pattern.compile("^https?://sinnercomics.com/comic/([a-zA-Z0-9-]*)/?$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { - return m.group(1); + return m.group(1).replaceAll("-page-\\d+", ""); } throw new MalformedURLException("Expected sinnercomics.com URL format: " + "sinnercomics.com/comic/albumName - got " + url + " instead"); From aca2d24a65cf78bd4748a09fdb3c939da30d6a13 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 13 May 2018 12:12:23 -0400 Subject: [PATCH 113/140] Can now blacklist nhentai tags --- .../ripme/ripper/rippers/NhentaiRipper.java | 41 +++++++++++++++++++ .../java/com/rarchives/ripme/utils/Utils.java | 10 +++++ 2 files changed, 51 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java index ac8f782d..50334f05 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ripper.DownloadThreadPool; +import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; import org.jsoup.nodes.Document; @@ -64,6 +65,39 @@ public class NhentaiRipper extends AbstractHTMLRipper { return "nhentai" + title; } + private List getTags(Document doc) { + List tags = new ArrayList<>(); + for (Element tag : doc.select("a.tag")) { + tags.add(tag.attr("href").replaceAll("/tag/", "").replaceAll("/", "")); + } + return tags; + } + + /** + * Checks for blacklisted tags on page. If it finds one it returns it, if not it return null + * + * @param doc + * @return String + */ + private String checkTags(Document doc) { + String[] blackListedTags = Utils.getConfigStringArray("nhentai.blacklist.tags"); + // If the user hasn't blacklisted any tags we return false; + if (blackListedTags == null) { + return null; + } + logger.info("Blacklisted tags " + blackListedTags[0]); + List tagsOnPage = getTags(doc); + for (String tag : blackListedTags) { + for (String pageTag : tagsOnPage) { + logger.info("tag: " + tag + " pageTag: " + pageTag); + if (tag.trim().toLowerCase().equals(pageTag.toLowerCase())) { + return tag; + } + } + } + return null; + } + @Override public String getGID(URL url) throws MalformedURLException { // Ex: https://nhentai.net/g/159174/ @@ -82,6 +116,13 @@ public class NhentaiRipper extends AbstractHTMLRipper { if (firstPage == null) { firstPage = Http.url(url).get(); } + + String blacklistedTag = checkTags(firstPage); + if (blacklistedTag != null) { + sendUpdate(RipStatusMessage.STATUS.DOWNLOAD_WARN, "Skipping " + url.toExternalForm() + " as it " + + "contains the blacklisted tag \"" + blacklistedTag + "\""); + return null; + } return firstPage; } diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index cd1048df..5560a425 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -92,6 +92,16 @@ public class Utils { public static String getConfigString(String key, String defaultValue) { return config.getString(key, defaultValue); } + + public static String[] getConfigStringArray(String key) { + String[] s = config.getStringArray(key); + if (s.length == 0) { + return null; + } else { + return s; + } + } + public static int getConfigInteger(String key, int defaultValue) { return config.getInt(key, defaultValue); } From 05cc8cec615dc5dc9d0d5888011c18a77ab569d1 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Mon, 14 May 2018 03:26:32 -0400 Subject: [PATCH 114/140] Added unit test for nhentai --- .../tst/ripper/rippers/NhentaiRipperTest.java | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/NhentaiRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NhentaiRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NhentaiRipperTest.java new file mode 100644 index 00000000..108feed2 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NhentaiRipperTest.java @@ -0,0 +1,33 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.NhentaiRipper; + +public class NhentaiRipperTest extends RippersTest { + public void testRip() throws IOException { + NhentaiRipper ripper = new NhentaiRipper(new URL("https://nhentai.net/g/233295/")); + testRipper(ripper); + } + + public void testGetGID() throws IOException { + NhentaiRipper ripper = new NhentaiRipper(new URL("https://nhentai.net/g/233295/")); + assertEquals("233295", ripper.getGID(new URL("https://nhentai.net/g/233295/"))); + } + + // Test the tag black listing + public void testTagBlackList() throws IOException { + URL url = new URL("https://nhentai.net/g/233295/"); + NhentaiRipper ripper = new NhentaiRipper(url); + // Test multiple blacklisted tags + String[] tags = {"test", "one", "blowjob"}; + String blacklistedTag = ripper.checkTags(ripper.getFirstPage(), tags); + assertEquals("blowjob", blacklistedTag); + + // test tags with spaces in them + String[] tags2 = {"test", "one", "sole female"}; + blacklistedTag = ripper.checkTags(ripper.getFirstPage(), tags2); + assertEquals("sole female", blacklistedTag); + } +} From 4dcfe3b712cd6b34f5279161c937e23ef3c5d911 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Mon, 14 May 2018 03:26:56 -0400 Subject: [PATCH 115/140] Can now black list tags with spaces in them --- .../rarchives/ripme/ripper/rippers/NhentaiRipper.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java index 50334f05..9c204a8d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java @@ -79,8 +79,7 @@ public class NhentaiRipper extends AbstractHTMLRipper { * @param doc * @return String */ - private String checkTags(Document doc) { - String[] blackListedTags = Utils.getConfigStringArray("nhentai.blacklist.tags"); + public String checkTags(Document doc, String[] blackListedTags) { // If the user hasn't blacklisted any tags we return false; if (blackListedTags == null) { return null; @@ -89,8 +88,9 @@ public class NhentaiRipper extends AbstractHTMLRipper { List tagsOnPage = getTags(doc); for (String tag : blackListedTags) { for (String pageTag : tagsOnPage) { - logger.info("tag: " + tag + " pageTag: " + pageTag); - if (tag.trim().toLowerCase().equals(pageTag.toLowerCase())) { + // We replace all dashes in the tag with spaces because the tags we get from the site are separated using + // dashes + if (tag.trim().toLowerCase().equals(pageTag.replaceAll("-", " ").toLowerCase())) { return tag; } } @@ -117,7 +117,7 @@ public class NhentaiRipper extends AbstractHTMLRipper { firstPage = Http.url(url).get(); } - String blacklistedTag = checkTags(firstPage); + String blacklistedTag = checkTags(firstPage, Utils.getConfigStringArray("nhentai.blacklist.tags")); if (blacklistedTag != null) { sendUpdate(RipStatusMessage.STATUS.DOWNLOAD_WARN, "Skipping " + url.toExternalForm() + " as it " + "contains the blacklisted tag \"" + blacklistedTag + "\""); From c5a8f0cd0885eb6f00b332d5746946a1b2832c92 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Mon, 14 May 2018 08:20:24 -0400 Subject: [PATCH 116/140] Added func to add urls to the download queue --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 95961f44..59f75e06 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -162,6 +162,11 @@ public final class MainWindow implements Runnable, RipStatusHandler { return checkbox; } + + public static void addUrlToQueue(String url) { + queueListModel.addElement(url); + } + public MainWindow() { mainFrame = new JFrame("RipMe v" + UpdateUtils.getThisJarVersion()); mainFrame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); From e33d24364e4b4116acf1a637cb4adb7d6cb7e927 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Mon, 14 May 2018 08:21:09 -0400 Subject: [PATCH 117/140] Added a way for rippers to add urls to the download queue --- .../ripme/ripper/AbstractHTMLRipper.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index e0fd3548..436eb122 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -11,6 +11,7 @@ import org.jsoup.nodes.Document; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Utils; +import com.rarchives.ripme.ui.MainWindow; /** * Simplified ripper, designed for ripping from sites by parsing HTML. @@ -53,12 +54,29 @@ public abstract class AbstractHTMLRipper extends AlbumRipper { protected boolean hasDescriptionSupport() { return false; } + protected String[] getDescription(String url, Document page) throws IOException { throw new IOException("getDescription not implemented"); // Do I do this or make an abstract function? } protected int descSleepTime() { return 100; } + + protected List getAlbumsToQueue(Document doc) { + return null; + } + + // If a page has Queue support than it has no images we want to download, just a list of urls we want to add to + // the queue + protected boolean hasQueueSupport() { + return false; + } + + // Takes a url and checks if it is for a page of albums + protected boolean pageContainsAlbums(URL url) { + return false; + } + @Override public void rip() throws IOException { int index = 0; @@ -67,6 +85,15 @@ public abstract class AbstractHTMLRipper extends AlbumRipper { sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm()); Document doc = getFirstPage(); + if (hasQueueSupport() && pageContainsAlbums(this.url)) { + List urls = getAlbumsToQueue(doc); + for (String url : urls) { + MainWindow.addUrlToQueue(url); + } + + doc = null; + } + while (doc != null) { if (alreadyDownloadedUrls >= Utils.getConfigInteger("history.end_rip_after_already_seen", 1000000000) && !isThisATest()) { sendUpdate(STATUS.DOWNLOAD_COMPLETE, "Already seen the last " + alreadyDownloadedUrls + " images ending rip"); From 21a81342d9f1eaa330d04d7b596d7d20966c1509 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Mon, 14 May 2018 08:21:59 -0400 Subject: [PATCH 118/140] Rewrote the myhentaicomics ripper to use the new getAlbumsToQueue func --- .../ripper/rippers/MyhentaicomicsRipper.java | 185 +++--------------- 1 file changed, 31 insertions(+), 154 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java index 5b60c4f2..60c90ae1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java @@ -34,21 +34,18 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper { Pattern p = Pattern.compile("^https?://myhentaicomics.com/index.php/([a-zA-Z0-9-]*)/?$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { - isTag = false; return m.group(1); } Pattern pa = Pattern.compile("^https?://myhentaicomics.com/index.php/search\\?q=([a-zA-Z0-9-]*)([a-zA-Z0-9=&]*)?$"); Matcher ma = pa.matcher(url.toExternalForm()); if (ma.matches()) { - isTag = true; return ma.group(1); } Pattern pat = Pattern.compile("^https?://myhentaicomics.com/index.php/tag/([0-9]*)/?([a-zA-Z%0-9+?=:]*)?$"); Matcher mat = pat.matcher(url.toExternalForm()); if (mat.matches()) { - isTag = true; return mat.group(1); } @@ -56,6 +53,37 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper { "myhentaicomics.com/index.php/albumName - got " + url + " instead"); } + @Override + public boolean hasQueueSupport() { + return true; + } + + @Override + public boolean pageContainsAlbums(URL url) { + Pattern pa = Pattern.compile("^https?://myhentaicomics.com/index.php/search\\?q=([a-zA-Z0-9-]*)([a-zA-Z0-9=&]*)?$"); + Matcher ma = pa.matcher(url.toExternalForm()); + if (ma.matches()) { + return true; + } + + Pattern pat = Pattern.compile("^https?://myhentaicomics.com/index.php/tag/([0-9]*)/?([a-zA-Z%0-9+?=:]*)?$"); + Matcher mat = pat.matcher(url.toExternalForm()); + if (mat.matches()) { + isTag = true; + return true; + } + return false; + } + + @Override + public List getAlbumsToQueue(Document doc) { + List urlsToAddToQueue = new ArrayList<>(); + for (Element elem : doc.select(".g-album > a")) { + urlsToAddToQueue.add(getDomain() + elem.attr("href")); + } + return urlsToAddToQueue; + } + @Override public Document getFirstPage() throws IOException { // "url" is an instance field of the superclass @@ -81,161 +109,11 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper { return Http.url(nextUrl).get(); } - // This replaces getNextPage when downloading from searchs and tags - private List getNextAlbumPage(String pageUrl) { - List albumPagesList = new ArrayList<>(); - int pageNumber = 1; - albumPagesList.add("http://myhentaicomics.com/index.php/" + pageUrl.split("\\?")[0] + "?page=" + Integer.toString(pageNumber)); - while (true) { - String urlToGet = "http://myhentaicomics.com/index.php/" + pageUrl.split("\\?")[0] + "?page=" + Integer.toString(pageNumber); - Document nextAlbumPage; - try { - logger.info("Grabbing " + urlToGet); - nextAlbumPage = Http.url(urlToGet).get(); - } catch (IOException e) { - logger.warn("Failed to log link in Jsoup"); - nextAlbumPage = null; - e.printStackTrace(); - } - Element elem = nextAlbumPage.select("a.ui-icon-right").first(); - String nextPage = elem.attr("href"); - pageNumber = pageNumber + 1; - if (nextPage.equals("")) { - logger.info("Got " + pageNumber + " pages"); - break; - } - else { - logger.info(nextPage); - albumPagesList.add(nextPage); - logger.info("Adding " + nextPage); - } - } - return albumPagesList; - } - private List getAlbumsFromPage(String url) { - List pagesToRip; - List result = new ArrayList<>(); - logger.info("Running getAlbumsFromPage"); - Document doc; - try { - doc = Http.url("http://myhentaicomics.com" + url).get(); - } catch (IOException e) { - logger.warn("Failed to log link in Jsoup"); - doc = null; - e.printStackTrace(); - } - // This for goes over every album on the page - for (Element elem : doc.select("li.g-album > a")) { - String link = elem.attr("href"); - logger.info("Grabbing album " + link); - pagesToRip = getNextAlbumPage(link); - logger.info(pagesToRip); - for (String element : pagesToRip) { - Document album_doc; - try { - logger.info("grabbing " + element + " with jsoup"); - boolean startsWithHttp = element.startsWith("http://"); - if (!startsWithHttp) { - album_doc = Http.url("http://myhentaicomics.com/" + element).get(); - } - else { - album_doc = Http.url(element).get(); - } - } catch (IOException e) { - logger.warn("Failed to log link in Jsoup"); - album_doc = null; - e.printStackTrace(); - } - for (Element el :album_doc.select("img")) { - String imageSource = el.attr("src"); - // This bool is here so we don't try and download the site logo - if (!imageSource.startsWith("http://")) { - // We replace thumbs with resizes so we can the full sized images - imageSource = imageSource.replace("thumbs", "resizes"); - String url_string = "http://myhentaicomics.com/" + imageSource; - url_string = url_string.replace("%20", "_"); - url_string = url_string.replace("%27", ""); - url_string = url_string.replace("%28", "_"); - url_string = url_string.replace("%29", "_"); - url_string = url_string.replace("%2C", "_"); - if (isTag) { - logger.info("Downloading from a tag or search"); - try { - sleep(500); - result.add("http://myhentaicomics.com/" + imageSource); - addURLToDownload(new URL("http://myhentaicomics.com/" + imageSource), "", url_string.split("/")[6]); - } - catch (MalformedURLException e) { - logger.warn("Malformed URL"); - e.printStackTrace(); - } - } - } - } - } - } - return result; - } - - private List getListOfPages(Document doc) { - List pages = new ArrayList<>(); - // Get the link from the last button - String nextPageUrl = doc.select("a.ui-icon-right").last().attr("href"); - Pattern pat = Pattern.compile("/index\\.php/tag/[0-9]*/[a-zA-Z0-9_\\-:+]*\\?page=(\\d+)"); - Matcher mat = pat.matcher(nextPageUrl); - if (mat.matches()) { - logger.debug("Getting pages from a tag"); - String base_link = mat.group(0).replaceAll("\\?page=\\d+", ""); - logger.debug("base_link is " + base_link); - int numOfPages = Integer.parseInt(mat.group(1)); - for (int x = 1; x != numOfPages +1; x++) { - logger.debug("running loop"); - String link = base_link + "?page=" + Integer.toString(x); - pages.add(link); - } - } else { - Pattern pa = Pattern.compile("/index\\.php/search\\?q=[a-zA-Z0-9_\\-:]*&page=(\\d+)"); - Matcher ma = pa.matcher(nextPageUrl); - if (ma.matches()) { - logger.debug("Getting pages from a search"); - String base_link = ma.group(0).replaceAll("page=\\d+", ""); - logger.debug("base_link is " + base_link); - int numOfPages = Integer.parseInt(ma.group(1)); - for (int x = 1; x != numOfPages +1; x++) { - logger.debug("running loop"); - String link = base_link + "page=" + Integer.toString(x); - logger.debug(link); - pages.add(link); - } - } - } - return pages; - } @Override public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); - // Checks if this is a comic page or a page of albums - // If true the page is a page of albums - if (doc.toString().contains("class=\"g-item g-album\"")) { - // This if checks that there is more than 1 page - if (!doc.select("a.ui-icon-right").last().attr("href").equals("")) { - // There is more than one page so we call getListOfPages - List pagesToRip = getListOfPages(doc); - logger.debug("Pages to rip = " + pagesToRip); - for (String url : pagesToRip) { - logger.debug("Getting albums from " + url); - result = getAlbumsFromPage(url); - } - } else { - logger.debug("There is only one page on this page of albums"); - // There is only 1 page so we call getAlbumsFromPage and pass it the page url - result = getAlbumsFromPage(doc.select("div.g-description > a").attr("href")); - } - return result; - } - else { for (Element el : doc.select("img")) { String imageSource = el.attr("src"); // This bool is here so we don't try and download the site logo @@ -245,7 +123,6 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper { result.add("http://myhentaicomics.com/" + imageSource); } } - } return result; } From 6a73e6c06bf7de45885175e75b211b3d9685b79d Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 16 May 2018 04:31:25 -0400 Subject: [PATCH 119/140] Fixed spelling mistake added another comment --- .../java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 436eb122..2eefe873 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -66,7 +66,7 @@ public abstract class AbstractHTMLRipper extends AlbumRipper { return null; } - // If a page has Queue support than it has no images we want to download, just a list of urls we want to add to + // If a page has Queue support then it has no images we want to download, just a list of urls we want to add to // the queue protected boolean hasQueueSupport() { return false; @@ -91,6 +91,7 @@ public abstract class AbstractHTMLRipper extends AlbumRipper { MainWindow.addUrlToQueue(url); } + // We set doc to null here so the while loop below this doesn't fire doc = null; } From 694dc3fe5b39d4093e6036a381fa512ff03d5a32 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 16 May 2018 04:55:36 -0400 Subject: [PATCH 120/140] hentai2read can now get the album thumbnail link again --- .../com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java index a87575bd..d994c11b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java @@ -48,7 +48,7 @@ public class Hentai2readRipper extends AbstractHTMLRipper { Document tempDoc; tempDoc = Http.url(url).get(); // Get the thumbnail page so we can rip all images without loading every page in the comic - String thumbnailLink = tempDoc.select("a[data-original-title=Thumbnails").attr("href"); + String thumbnailLink = tempDoc.select("div.col-xs-12 > div.reader-controls > div.controls-block > button > a").attr("href"); return Http.url(thumbnailLink).get(); } catch (IOException e) { throw new IOException("Unable to get first page"); From afdff0ebc48b5b794316141bf8cf26f1cf844a82 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 16 May 2018 11:38:49 -0400 Subject: [PATCH 121/140] hentai2read can now get the album thumbnail link again --- .../ripme/ripper/rippers/Hentai2readRipper.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java index d994c11b..7441d8d6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java @@ -44,12 +44,17 @@ public class Hentai2readRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { + String thumbnailLink; try { Document tempDoc; tempDoc = Http.url(url).get(); // Get the thumbnail page so we can rip all images without loading every page in the comic - String thumbnailLink = tempDoc.select("div.col-xs-12 > div.reader-controls > div.controls-block > button > a").attr("href"); - return Http.url(thumbnailLink).get(); + thumbnailLink = tempDoc.select("div.col-xs-12 > div.reader-controls > div.controls-block > button > a").attr("href"); + if (thumbnailLink != null) { + return Http.url(thumbnailLink).get(); + } else { + return Http.url(tempDoc.select("a[data-original-title=Thumbnails").attr("href")).get(); + } } catch (IOException e) { throw new IOException("Unable to get first page"); } From 59d31159704ee4763dd4f6e2bc349d173d636109 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 16 May 2018 11:46:36 -0400 Subject: [PATCH 122/140] Fixed hentai2read ripper for real this time --- .../com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java index 7441d8d6..78fc50fc 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java @@ -50,7 +50,7 @@ public class Hentai2readRipper extends AbstractHTMLRipper { tempDoc = Http.url(url).get(); // Get the thumbnail page so we can rip all images without loading every page in the comic thumbnailLink = tempDoc.select("div.col-xs-12 > div.reader-controls > div.controls-block > button > a").attr("href"); - if (thumbnailLink != null) { + if (!thumbnailLink.equals("")) { return Http.url(thumbnailLink).get(); } else { return Http.url(tempDoc.select("a[data-original-title=Thumbnails").attr("href")).get(); From 97856425126d440c3ad1697ad340890f4f157dd0 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 16 May 2018 12:04:19 -0400 Subject: [PATCH 123/140] 1.7.46: Fixed hentai2read ripper; Rewrote the myhentaicomics ripper to use the new getAlbumsToQueue func; Can now blacklist nhentai tags; SinnercomicsRipper no longer adds -page-01 to folder names; EightmusesRipper now adds file extension to filename; disbaled test for twitch ripper --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 317b816e..a46264f9 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.45 + 1.7.46 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 73b9b240..4858d2cc 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.45", + "latestVersion": "1.7.46", "changeList": [ + "1.7.46: Fixed hentai2read ripper; Rewrote the myhentaicomics ripper to use the new getAlbumsToQueue func; Can now blacklist nhentai tags; SinnercomicsRipper no longer adds -page-01 to folder names; EightmusesRipper now adds file extension to filename; disbaled test for twitch ripper", "1.7.45: Fixed hentai2read ripper; ImageBam album fixed; Added various translations; TsuminoRipper no longer requires album name to download", "1.7.44: Fixed instagram ripper regex", "1.7.43: Fixed queryId regex in instagram ripper", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 38e88ae7..f82b2137 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.45"; + private static final String DEFAULT_VERSION = "1.7.46"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 67821fb311ec82ec2528cc492fe69eaecb1dc2ba Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 16 May 2018 15:21:34 -0400 Subject: [PATCH 124/140] Added quick queue support for 8muses.download --- .../ripper/rippers/WordpressComicRipper.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java index 8b7b7b80..adcc7360 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java @@ -160,12 +160,54 @@ public class WordpressComicRipper extends AbstractHTMLRipper { if (spyingwithlanaMat.matches()) { return true; } + + Pattern pa = Pattern.compile("^https?://8muses.download/\\?s=([a-zA-Z0-9-]*)"); + Matcher ma = pa.matcher(url.toExternalForm()); + if (ma.matches()) { + return true; + } + + Pattern pat = Pattern.compile("https?://8muses.download/page/\\d+/\\?s=([a-zA-Z0-9-]*)"); + Matcher mat = pat.matcher(url.toExternalForm()); + if (mat.matches()) { + return true; + } } return false; } + @Override + public boolean hasQueueSupport() { + return true; + } + + @Override + public boolean pageContainsAlbums(URL url) { + Pattern pa = Pattern.compile("^https?://8muses.download/\\?s=([a-zA-Z0-9-]*)"); + Matcher ma = pa.matcher(url.toExternalForm()); + if (ma.matches()) { + return true; + } + + Pattern pat = Pattern.compile("https?://8muses.download/page/\\d+/\\?s=([a-zA-Z0-9-]*)"); + Matcher mat = pat.matcher(url.toExternalForm()); + if (mat.matches()) { + return true; + } + return false; + } + + @Override + public List getAlbumsToQueue(Document doc) { + List urlsToAddToQueue = new ArrayList<>(); + for (Element elem : doc.select("#post_masonry > article > div > figure > a")) { + urlsToAddToQueue.add(elem.attr("href")); + } + return urlsToAddToQueue; + } + @Override public String getAlbumTitle(URL url) throws MalformedURLException { Pattern totempole666Pat = Pattern.compile("(?:https?://)?(?:www\\.)?totempole666.com/comic/([a-zA-Z0-9_-]*)/?$"); From e541d1cba0e20ece318e45e8ee54e0c075c1db73 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 16 May 2018 15:32:18 -0400 Subject: [PATCH 125/140] Added quick queue support for 8muses.download --- .../ripme/ripper/rippers/WordpressComicRipper.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java index adcc7360..31509734 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java @@ -172,6 +172,12 @@ public class WordpressComicRipper extends AbstractHTMLRipper { if (mat.matches()) { return true; } + + pat = Pattern.compile("https://8muses.download/category/([a-zA-Z0-9-]*)/?"); + mat = pat.matcher(url.toExternalForm()); + if (mat.matches()) { + return true; + } } @@ -196,6 +202,13 @@ public class WordpressComicRipper extends AbstractHTMLRipper { if (mat.matches()) { return true; } + + pat = Pattern.compile("https://8muses.download/category/([a-zA-Z0-9-]*)/?"); + mat = pat.matcher(url.toExternalForm()); + if (mat.matches()) { + return true; + } + return false; } From 81a648681efea4d2b9426c418a62746cbd521521 Mon Sep 17 00:00:00 2001 From: Kevin Jiang Date: Thu, 17 May 2018 13:57:18 -0400 Subject: [PATCH 126/140] Add more documentation to App.java --- src/main/java/com/rarchives/ripme/App.java | 26 +++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index 6f650d56..d58f5d4d 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -33,13 +33,22 @@ import com.rarchives.ripme.utils.Utils; /** * Entry point to application. + * This is where all the fun happens, with the main method. * Decides to display UI or to run silently via command-line. + * + * As the "controller" to all other classes, it parses command line parameters and loads the history. */ public class App { public static final Logger logger = Logger.getLogger(App.class); private static final History HISTORY = new History(); + /** + * Where everything starts. Takes in, and tries to parse as many commandline arguments as possible. + * Otherwise, it launches a GUI. + * + * @param args Array of command line arguments. + */ public static void main(String[] args) throws MalformedURLException { CommandLine cl = getArgs(args); @@ -74,7 +83,8 @@ public class App { /** * Creates an abstract ripper and instructs it to rip. * @param url URL to be ripped - * @throws Exception + * @throws Exception Nothing too specific here, just a catch-all. + * */ private static void rip(URL url) throws Exception { AbstractRipper ripper = AbstractRipper.getRipper(url); @@ -89,6 +99,7 @@ public class App { private static void handleArguments(String[] args) { CommandLine cl = getArgs(args); + //Help (list commands) if (cl.hasOption('h') || args.length == 0) { HelpFormatter hf = new HelpFormatter(); hf.printHelp("java -jar ripme.jar [OPTIONS]", getOptions()); @@ -98,28 +109,34 @@ public class App { Utils.configureLogger(); logger.info("Initialized ripme v" + UpdateUtils.getThisJarVersion()); + //Allow file overwriting if (cl.hasOption('w')) { Utils.setConfigBoolean("file.overwrite", true); } + //SOCKS proxy server if (cl.hasOption('s')) { String sservfull = cl.getOptionValue('s').trim(); Proxy.setSocks(sservfull); } + //HTTP proxy server if (cl.hasOption('p')) { String proxyserverfull = cl.getOptionValue('p').trim(); Proxy.setHTTPProxy(proxyserverfull); } + //Number of threads if (cl.hasOption('t')) { Utils.setConfigInteger("threads.size", Integer.parseInt(cl.getOptionValue('t'))); } + //Ignore 404 if (cl.hasOption('4')) { Utils.setConfigBoolean("errors.skip404", true); } + //Re-rip all previous albums if (cl.hasOption('r')) { // Re-rip all via command-line List history = Utils.getConfigList("download.history"); @@ -142,6 +159,7 @@ public class App { System.exit(0); } + //Re-rip all selected albums if (cl.hasOption('R')) { loadHistory(); if (HISTORY.toList().isEmpty()) { @@ -174,24 +192,29 @@ public class App { } } + //Save the order of images in album if (cl.hasOption('d')) { Utils.setConfigBoolean("download.save_order", true); } + //Don't save the order of images in album if (cl.hasOption('D')) { Utils.setConfigBoolean("download.save_order", false); } + //In case specify both, break and exit since it isn't possible. if ((cl.hasOption('d'))&&(cl.hasOption('D'))) { logger.error("\nCannot specify '-d' and '-D' simultaneously"); System.exit(-1); } + //Destination directory if (cl.hasOption('l')) { // change the default rips directory Utils.setConfigString("rips.directory", cl.getOptionValue('l')); } + //Read URLs from File if (cl.hasOption('f')) { String filename = cl.getOptionValue('f'); try { @@ -208,6 +231,7 @@ public class App { } } + //The URL to rip. if (cl.hasOption('u')) { String url = cl.getOptionValue('u').trim(); ripURL(url, cl.hasOption("n")); From de105edffe0f9f61783ee8bea326de8af558afd3 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 17 May 2018 19:46:05 -0400 Subject: [PATCH 127/140] Added ripper for bato.to --- .../ripme/ripper/rippers/BatoRipper.java | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java new file mode 100644 index 00000000..a25aa16b --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java @@ -0,0 +1,109 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.json.JSONObject; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class BatoRipper extends AbstractHTMLRipper { + + public BatoRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "bato"; + } + + @Override + public String getDomain() { + return "bato.to"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https?://bato.to/chapter/([\\d]+)/?"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected bato.to URL format: " + + "bato.to/chapter/ID - got " + url + " instead"); + } + + @Override + public boolean canRip(URL url) { + Pattern p = Pattern.compile("https?://bato.to/series/([\\d]+)/?"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return true; + } + + p = Pattern.compile("https?://bato.to/chapter/([\\d]+)/?"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return true; + } + return false; + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + @Override + public Document getNextPage(Document doc) throws IOException { + // Find next page + String nextUrl = ""; + // We use comic-nav-next to the find the next page + Element elem = doc.select("div.nav-next > a").first(); + // If there are no more chapters to download the last chapter will link to bato.to/series/ID + if (elem == null || elem.attr("href").contains("/series/")) { + throw new IOException("No more pages"); + } + String nextPage = elem.attr("href"); + return Http.url("https://bato.to" + nextPage).get(); + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + for (Element script : doc.select("script")) { + if (script.data().contains("var images = ")) { + String s = script.data(); + s = s.replaceAll("var seriesId = \\d+;", ""); + s = s.replaceAll("var chapterId = \\d+;", ""); + s = s.replaceAll("var pages = \\d+;", ""); + s = s.replaceAll("var page = \\d+;", ""); + s = s.replaceAll("var prevCha = null;", ""); + s = s.replaceAll("var nextCha = \\.*;", ""); + String json = s.replaceAll("var images = ", "").replaceAll(";", ""); + logger.info(s); + JSONObject images = new JSONObject(json); + for (int i = 1; i < images.length() +1; i++) { + result.add(images.getString(Integer.toString(i))); + } + + } + } + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} From 8a27941ddee14d9c2858636c90d71670e37c8d2b Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 17 May 2018 19:47:54 -0400 Subject: [PATCH 128/140] Added test for BatoRipper --- .../ripme/tst/ripper/rippers/BatoRipperTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java new file mode 100644 index 00000000..073c304d --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.BatoRipper; + +public class BatoRipperTest extends RippersTest { + public void testRip() throws IOException { + BatoRipper ripper = new BatoRipper(new URL("https://bato.to/chapter/1207152")); + testRipper(ripper); + } +} From 05b50a4513e83e7838f03a0198c72dee4abb4e30 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 17 May 2018 19:56:41 -0400 Subject: [PATCH 129/140] Added getAlbumTitle to batoRipper --- .../rarchives/ripme/ripper/rippers/BatoRipper.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java index a25aa16b..6578b7b1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java @@ -42,6 +42,18 @@ public class BatoRipper extends AbstractHTMLRipper { "bato.to/chapter/ID - got " + url + " instead"); } + @Override + public String getAlbumTitle(URL url) throws MalformedURLException { + try { + // Attempt to use album title as GID + return getHost() + "_" + getGID(url) + "_" + getFirstPage().select("title").first().text().replaceAll(" ", "_"); + } catch (IOException e) { + // Fall back to default album naming convention + logger.info("Unable to find title at " + url); + } + return super.getAlbumTitle(url); + } + @Override public boolean canRip(URL url) { Pattern p = Pattern.compile("https?://bato.to/series/([\\d]+)/?"); From 9281766a71ea8a619f40b866358893e77b521392 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 17 May 2018 19:57:02 -0400 Subject: [PATCH 130/140] Extented BatoRipper tests --- .../ripme/tst/ripper/rippers/BatoRipperTest.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java index 073c304d..6bd8744a 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java @@ -10,4 +10,16 @@ public class BatoRipperTest extends RippersTest { BatoRipper ripper = new BatoRipper(new URL("https://bato.to/chapter/1207152")); testRipper(ripper); } + + public void testGetGID() throws IOException { + URL url = new URL("https://bato.to/chapter/1207152"); + BatoRipper ripper = new BatoRipper(url); + assertEquals("1207152", ripper.getGID(url)); + } + + public void testGetAlbumTitle() throws IOException { + URL url = new URL("https://bato.to/chapter/1207152"); + BatoRipper ripper = new BatoRipper(url); + assertEquals("bato_1207152_I_Messed_Up_by_Teaching_at_a_Black_Gyaru_School!_Ch.2", ripper.getAlbumTitle(url)); + } } From bb60779469a8c833645c848f6aff7338fcfefb50 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 17 May 2018 20:07:16 -0400 Subject: [PATCH 131/140] Removed getnextpage; add support for quickQueue --- .../ripme/ripper/rippers/BatoRipper.java | 44 +++++++++++++------ 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java index 6578b7b1..a3350e68 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java @@ -38,10 +38,40 @@ public class BatoRipper extends AbstractHTMLRipper { if (m.matches()) { return m.group(1); } + // As this is just for quick queue support it does matter what this if returns + p = Pattern.compile("https?://bato.to/series/([\\d]+)/?"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return ""; + } throw new MalformedURLException("Expected bato.to URL format: " + "bato.to/chapter/ID - got " + url + " instead"); } + @Override + public boolean hasQueueSupport() { + return true; + } + + @Override + public boolean pageContainsAlbums(URL url) { + Pattern p = Pattern.compile("https?://bato.to/series/([\\d]+)/?"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return true; + } + return false; + } + + @Override + public List getAlbumsToQueue(Document doc) { + List urlsToAddToQueue = new ArrayList<>(); + for (Element elem : doc.select("div.main > div > a")) { + urlsToAddToQueue.add("https://" + getDomain() + elem.attr("href")); + } + return urlsToAddToQueue; + } + @Override public String getAlbumTitle(URL url) throws MalformedURLException { try { @@ -76,20 +106,6 @@ public class BatoRipper extends AbstractHTMLRipper { return Http.url(url).get(); } - @Override - public Document getNextPage(Document doc) throws IOException { - // Find next page - String nextUrl = ""; - // We use comic-nav-next to the find the next page - Element elem = doc.select("div.nav-next > a").first(); - // If there are no more chapters to download the last chapter will link to bato.to/series/ID - if (elem == null || elem.attr("href").contains("/series/")) { - throw new IOException("No more pages"); - } - String nextPage = elem.attr("href"); - return Http.url("https://bato.to" + nextPage).get(); - } - @Override public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); From 318760e7ade4e5930f3ee1c7d09f85c3d7c16f94 Mon Sep 17 00:00:00 2001 From: Kevin Jiang Date: Fri, 18 May 2018 13:25:09 -0400 Subject: [PATCH 132/140] More documentation; working towards creating a fully functional JavaDoc --- .../java/com/rarchives/ripme/utils/Utils.java | 107 +++++++++++++++++- 1 file changed, 103 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 5560a425..828d552a 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -89,6 +89,12 @@ public class Utils { return workingDir; } + /** + * Gets the value of a specific config key. + * + * @param key The name of the config parameter you want to find. + * @param defaultValue What the default value would be. + */ public static String getConfigString(String key, String defaultValue) { return config.getString(key, defaultValue); } @@ -142,31 +148,53 @@ public class Utils { } } + /** + * Determines if your current system is a Windows system. + */ private static boolean isWindows() { return OS.contains("win"); } + /** + * Determines if your current system is a Mac system + */ private static boolean isMacOS() { return OS.contains("mac"); } + /** + * Determines if current system is based on UNIX + */ private static boolean isUnix() { return OS.contains("nix") || OS.contains("nux") || OS.contains("bsd"); } + /** + * Gets the directory of where the config file is stored on a Windows machine. + */ private static String getWindowsConfigDir() { return System.getenv("LOCALAPPDATA") + File.separator + "ripme"; } + + /** + * Gets the directory of where the config file is stored on a UNIX machine. + */ private static String getUnixConfigDir() { return System.getProperty("user.home") + File.separator + ".config" + File.separator + "ripme"; } - + + /** + * Gets the directory of where the config file is stored on a Mac machine. + */ private static String getMacOSConfigDir() { return System.getProperty("user.home") + File.separator + "Library" + File.separator + "Application Support" + File.separator + "ripme"; } + /** + * Determines if the app is running in a portable mode. i.e. on a USB stick + */ private static boolean portableMode() { try { File f = new File(new File(".").getCanonicalPath() + File.separator + configFile); @@ -179,7 +207,9 @@ public class Utils { return false; } - + /** + * Gets the directory of the config directory, for all systems. + */ public static String getConfigDir() { if (portableMode()) { try { @@ -199,17 +229,24 @@ public class Utils { return "."; } } - // Delete the url history file + /** + * Delete the url history file + */ public static void clearURLHistory() { File file = new File(getURLHistoryFile()); file.delete(); } - // Return the path of the url history file + /** + * Return the path of the url history file + */ public static String getURLHistoryFile() { return getConfigDir() + File.separator + "url_history.txt"; } + /** + * Gets the path to the configuration file. + */ private static String getConfigFilePath() { return getConfigDir() + File.separator + configFile; } @@ -235,6 +272,15 @@ public class Utils { return prettySaveAs; } + /** + * Strips away URL parameters, which usually appear at the end of URLs. + * E.g. the ?query on PHP + * + * @param url The URL to filter/strip + * @param parameter The parameter to strip + * + * @return The stripped URL + */ public static String stripURLParameter(String url, String parameter) { int paramIndex = url.indexOf("?" + parameter); boolean wasFirstParam = true; @@ -262,6 +308,7 @@ public class Utils { /** * Removes the current working directory from a given filename * @param file + * Path to the file * @return * 'file' without the leading current working directory */ @@ -345,9 +392,24 @@ public class Utils { } private static final int SHORTENED_PATH_LENGTH = 12; + /** + * Shortens the path to a file + * @param path + * String of the path to the file + * @return + * The simplified path to the file. + */ public static String shortenPath(String path) { return shortenPath(new File(path)); } + + /** + * Shortens the path to a file + * @param file + * File object that you want the shortened path of. + * @return + * The simplified path to the file. + */ public static String shortenPath(File file) { String path = removeCWD(file); if (path.length() < SHORTENED_PATH_LENGTH * 2) { @@ -358,6 +420,13 @@ public class Utils { + path.substring(path.length() - SHORTENED_PATH_LENGTH); } + /** + * Sanitizes a string so that a filesystem can handle it + * @param text + * The text to be sanitized. + * @return + * The sanitized text. + */ public static String filesystemSanitized(String text) { text = text.replaceAll("[^a-zA-Z0-9.-]", "_"); return text; @@ -407,6 +476,13 @@ public class Utils { return original; } + /** + * Converts an integer into a human readable string + * @param bytes + * Non-human readable integer. + * @return + * Human readable interpretation of a byte. + */ public static String bytesToHumanReadable(int bytes) { float fbytes = (float) bytes; String[] mags = new String[] {"", "K", "M", "G", "T"}; @@ -418,6 +494,10 @@ public class Utils { return String.format("%.2f%siB", fbytes, mags[magIndex]); } + /** + * Gets and returns a list of all the album rippers present in the "com.rarchives.ripme.ripper.rippers" package. + * @return List of all album rippers present. + */ public static List getListOfAlbumRippers() throws Exception { List list = new ArrayList<>(); for (Constructor ripper : AbstractRipper.getRipperConstructors("com.rarchives.ripme.ripper.rippers")) { @@ -425,6 +505,11 @@ public class Utils { } return list; } + + /** + * Gets and returns a list of all video rippers present in the "com.rarchives.rime.rippers.video" package + * @return List of all the video rippers. + */ public static List getListOfVideoRippers() throws Exception { List list = new ArrayList<>(); for (Constructor ripper : AbstractRipper.getRipperConstructors("com.rarchives.ripme.ripper.rippers.video")) { @@ -433,6 +518,11 @@ public class Utils { return list; } + /** + * Plays a sound from a file. + * @param filename + * Path to the sound file + */ public static void playSound(String filename) { URL resource = ClassLoader.getSystemClassLoader().getResource(filename); try { @@ -570,6 +660,9 @@ public class Utils { cookieCache = new HashMap>(); } + /** + * Gets all the cookies from a certain host + */ public static Map getCookies(String host) { HashMap domainCookies = cookieCache.get(host); if (domainCookies == null) { @@ -587,6 +680,12 @@ public class Utils { return domainCookies; } + /** + * Gets the ResourceBundle AKA language package. + * Used for choosing the language of the UI. + * + * @return Returns the default resource bundle using the language specified in the config file. + */ public static ResourceBundle getResourceBundle() { if (!getConfigString("lang", "").equals("")) { String[] langCode = getConfigString("lang", "").split("_"); From 0594e55d1fdd79af5e3a9f5776d2bbaee6e681f6 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 19 May 2018 10:42:50 -0400 Subject: [PATCH 133/140] SankakuComplexRipper can now download from different subdomains --- .../ripper/rippers/SankakuComplexRipper.java | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SankakuComplexRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SankakuComplexRipper.java index c6440bb8..d83d5930 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SankakuComplexRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SankakuComplexRipper.java @@ -43,7 +43,7 @@ public class SankakuComplexRipper extends AbstractHTMLRipper { Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { try { - return URLDecoder.decode(m.group(2), "UTF-8"); + return URLDecoder.decode(m.group(1) + "_" + m.group(2), "UTF-8"); } catch (UnsupportedEncodingException e) { throw new MalformedURLException("Cannot decode tag name '" + m.group(1) + "'"); } @@ -53,6 +53,20 @@ public class SankakuComplexRipper extends AbstractHTMLRipper { url + "instead"); } + public String getSubDomain(URL url){ + Pattern p = Pattern.compile("^https?://([a-zA-Z0-9]+\\.)?sankakucomplex\\.com/.*tags=([^&]+).*$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + try { + return URLDecoder.decode(m.group(1), "UTF-8"); + } catch (UnsupportedEncodingException e) { + return null; + } + } + return null; + + } + @Override public Document getFirstPage() throws IOException { if (albumDoc == null) { @@ -71,9 +85,11 @@ public class SankakuComplexRipper extends AbstractHTMLRipper { for (Element thumbSpan : doc.select("div.content > div > span.thumb > a")) { String postLink = thumbSpan.attr("href"); try { + String subDomain = getSubDomain(url); + String siteURL = "https://" + subDomain + "sankakucomplex.com"; // Get the page the full sized image is on - Document subPage = Http.url("https://chan.sankakucomplex.com" + postLink).get(); - logger.info("Checking page " + "https://chan.sankakucomplex.com" + postLink); + Document subPage = Http.url(siteURL + postLink).get(); + logger.info("Checking page " + siteURL + postLink); imageURLs.add("https:" + subPage.select("div[id=stats] > ul > li > a[id=highres]").attr("href")); } catch (IOException e) { logger.warn("Error while loading page " + postLink, e); From 846e2fe24520553c9da51061a04f41fa54ffe1d5 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 19 May 2018 11:31:05 -0400 Subject: [PATCH 134/140] Added some more unit tests --- .../tst/ripper/rippers/SankakuComplexRipperTest.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SankakuComplexRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SankakuComplexRipperTest.java index 6bbc8890..5b57e291 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SankakuComplexRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SankakuComplexRipperTest.java @@ -17,4 +17,15 @@ public class SankakuComplexRipperTest extends RippersTest { testRipper(ripper); } */ + public void testgetGID() throws IOException { + URL url = new URL("https://idol.sankakucomplex.com/?tags=meme_%28me%21me%21me%21%29_%28cosplay%29"); + SankakuComplexRipper ripper = new SankakuComplexRipper(url); + assertEquals("idol._meme_(me!me!me!)_(cosplay)", ripper.getGID(url)); + } + + public void testgetSubDomain() throws IOException { + URL url = new URL("https://idol.sankakucomplex.com/?tags=meme_%28me%21me%21me%21%29_%28cosplay%29"); + SankakuComplexRipper ripper = new SankakuComplexRipper(url); + assertEquals("idol.", ripper.getSubDomain(url)); + } } From fab905e4bc28b3ee1cff31dc9fe605b75f37a7a5 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 19 May 2018 12:35:38 -0400 Subject: [PATCH 135/140] Added more unit tests --- .../tst/ripper/rippers/ShesFreakyRipperTest.java | 6 ++++++ .../ripme/tst/ripper/rippers/SinfestRipperTest.java | 6 ++++++ .../tst/ripper/rippers/SinnercomicsRipperTest.java | 6 ++++++ .../ripme/tst/ripper/rippers/SmuttyRipperTest.java | 6 ++++++ .../ripme/tst/ripper/rippers/StaRipperTest.java | 6 ++++++ .../ripme/tst/ripper/rippers/TapasticRipperTest.java | 6 ++++++ .../tst/ripper/rippers/TeenplanetRipperTest.java | 6 ++++++ .../ripme/tst/ripper/rippers/VidbleRipperTest.java | 6 ++++++ .../ripme/tst/ripper/rippers/XbooruRipperTest.java | 6 ++++++ .../ripme/tst/ripper/rippers/XhamsterRipperTest.java | 6 ++++++ .../ripme/tst/ripper/rippers/YuvutuRipperTest.java | 6 ++++++ .../ripme/tst/ripper/rippers/ZizkiRipperTest.java | 12 ++++++++++++ 12 files changed, 78 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ShesFreakyRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ShesFreakyRipperTest.java index 3efa31b4..65f371d1 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ShesFreakyRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ShesFreakyRipperTest.java @@ -13,4 +13,10 @@ public class ShesFreakyRipperTest extends RippersTest { testRipper(ripper); } */ + + public void testGetGID() throws IOException { + URL url = new URL("http://www.shesfreaky.com/gallery/nicee-snow-bunny-579NbPjUcYa.html"); + ShesFreakyRipper ripper = new ShesFreakyRipper(url); + assertEquals("nicee-snow-bunny-579NbPjUcYa", ripper.getGID(url)); + } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinfestRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinfestRipperTest.java index c46e922c..c4f56432 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinfestRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinfestRipperTest.java @@ -10,4 +10,10 @@ public class SinfestRipperTest extends RippersTest { SinfestRipper ripper = new SinfestRipper(new URL("http://sinfest.net/view.php?date=2000-01-17")); testRipper(ripper); } + + public void testGetGID() throws IOException { + URL url = new URL("http://sinfest.net/view.php?date=2000-01-17"); + SinfestRipper ripper = new SinfestRipper(url); + assertEquals("2000-01-17", ripper.getGID(url)); + } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinnercomicsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinnercomicsRipperTest.java index d1ce6b33..3866b6ba 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinnercomicsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinnercomicsRipperTest.java @@ -10,4 +10,10 @@ public class SinnercomicsRipperTest extends RippersTest { SinnercomicsRipper ripper = new SinnercomicsRipper(new URL("https://sinnercomics.com/comic/beyond-the-hotel-page-01/")); testRipper(ripper); } + + public void testGetGID() throws IOException { + URL url = new URL("https://sinnercomics.com/comic/beyond-the-hotel-page-01/"); + SinnercomicsRipper ripper = new SinnercomicsRipper(url); + assertEquals("beyond-the-hotel", ripper.getGID(url)); + } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java index 949e715f..c7aa694e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java @@ -10,4 +10,10 @@ public class SmuttyRipperTest extends RippersTest { SmuttyRipper ripper = new SmuttyRipper(new URL("https://smutty.com/user/QUIGON/")); testRipper(ripper); } + + public void testGetGID() throws IOException { + URL url = new URL("https://smutty.com/user/QUIGON/"); + SmuttyRipper ripper = new SmuttyRipper(url); + assertEquals("QUIGON", ripper.getGID(url)); + } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StaRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StaRipperTest.java index 128f3f17..c9aded3e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StaRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StaRipperTest.java @@ -10,4 +10,10 @@ public class StaRipperTest extends RippersTest { StaRipper ripper = new StaRipper(new URL("https://sta.sh/2hn9rtavr1g")); testRipper(ripper); } + + public void testGetGID() throws IOException { + URL url = new URL("https://sta.sh/2hn9rtavr1g"); + StaRipper ripper = new StaRipper(url); + assertEquals("2hn9rtavr1g", ripper.getGID(url)); + } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TapasticRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TapasticRipperTest.java index 9617e4b8..f9e448e6 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TapasticRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TapasticRipperTest.java @@ -10,4 +10,10 @@ public class TapasticRipperTest extends RippersTest { TapasticRipper ripper = new TapasticRipper(new URL("https://tapas.io/series/tsiwbakd-comic")); testRipper(ripper); } + + public void testGetGID() throws IOException { + URL url = new URL("https://tapas.io/series/tsiwbakd-comic"); + TapasticRipper ripper = new TapasticRipper(url); + assertEquals("series_ tsiwbakd-comic", ripper.getGID(url)); + } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TeenplanetRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TeenplanetRipperTest.java index aa43103b..a402ebc3 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TeenplanetRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TeenplanetRipperTest.java @@ -10,4 +10,10 @@ public class TeenplanetRipperTest extends RippersTest { TeenplanetRipper ripper = new TeenplanetRipper(new URL("http://teenplanet.org/galleries/the-perfect-side-of-me-6588.html")); testRipper(ripper); } + + public void testGetGID() throws IOException { + URL url = new URL("http://teenplanet.org/galleries/the-perfect-side-of-me-6588.html"); + TeenplanetRipper ripper = new TeenplanetRipper(url); + assertEquals("the-perfect-side-of-me-6588", ripper.getGID(url)); + } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VidbleRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VidbleRipperTest.java index ccbeb8bb..9659d630 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VidbleRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VidbleRipperTest.java @@ -10,5 +10,11 @@ public class VidbleRipperTest extends RippersTest { VidbleRipper ripper = new VidbleRipper(new URL("http://www.vidble.com/album/y1oyh3zd")); testRipper(ripper); } + + public void testGetGID() throws IOException { + URL url = new URL("http://www.vidble.com/album/y1oyh3zd"); + VidbleRipper ripper = new VidbleRipper(url); + assertEquals("y1oyh3zd", ripper.getGID(url)); + } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XbooruRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XbooruRipperTest.java index 8eefd4e3..6dfc0bab 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XbooruRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XbooruRipperTest.java @@ -10,4 +10,10 @@ public class XbooruRipperTest extends RippersTest { XbooruRipper ripper = new XbooruRipper(new URL("http://xbooru.com/index.php?page=post&s=list&tags=furry")); testRipper(ripper); } + + public void testGetGID() throws IOException { + URL url = new URL("http://xbooru.com/index.php?page=post&s=list&tags=furry"); + XbooruRipper ripper = new XbooruRipper(url); + assertEquals("furry", ripper.getGID(url)); + } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java index f66b27d1..54b22eb0 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java @@ -16,4 +16,10 @@ public class XhamsterRipperTest extends RippersTest { XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.com/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664")); testRipper(ripper); } + + public void testGetGID() throws IOException { + URL url = new URL("https://xhamster.com/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664"); + XhamsterRipper ripper = new XhamsterRipper(url); + assertEquals("7254664", ripper.getGID(url)); + } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YuvutuRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YuvutuRipperTest.java index a74b76e9..1a304468 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YuvutuRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YuvutuRipperTest.java @@ -16,4 +16,10 @@ public class YuvutuRipperTest extends RippersTest { YuvutuRipper ripper = new YuvutuRipper(new URL("http://www.yuvutu.com/modules.php?name=YuGallery&action=view&set_id=420333")); testRipper(ripper); } + + public void testGetGID() throws IOException { + URL url = new URL("http://www.yuvutu.com/modules.php?name=YuGallery&action=view&set_id=420333"); + YuvutuRipper ripper = new YuvutuRipper(url); + assertEquals("420333", ripper.getGID(url)); + } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java index de953be8..9facf481 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java @@ -10,4 +10,16 @@ public class ZizkiRipperTest extends RippersTest { ZizkiRipper ripper = new ZizkiRipper(new URL("http://zizki.com/dee-chorde/we-got-spirit")); testRipper(ripper); } + + public void testGetGID() throws IOException { + URL url = new URL("http://zizki.com/dee-chorde/we-got-spirit"); + ZizkiRipper ripper = new ZizkiRipper(url); + assertEquals("dee-chorde", ripper.getGID(url)); + } + + public void testAlbumTitle() throws IOException { + URL url = new URL("http://zizki.com/dee-chorde/we-got-spirit"); + ZizkiRipper ripper = new ZizkiRipper(url); + assertEquals("zizki_Dee Chorde_We Got Spirit", ripper.getAlbumTitle(url)); + } } From 617d1b9a14296d5b71f3d4bffd1101516019f302 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 20 May 2018 10:21:45 -0400 Subject: [PATCH 136/140] Fixed instagram ripper using a total hack --- .../ripme/ripper/rippers/InstagramRipper.java | 42 +++++++++++++++++-- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index efc4cb40..a3f9c2d2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -23,6 +23,7 @@ import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; import org.jsoup.Connection; +import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import com.rarchives.ripme.ui.RipStatusMessage; @@ -39,6 +40,9 @@ public class InstagramRipper extends AbstractHTMLRipper { private String userID; private String rhx_gis = null; private String csrftoken; + // Run into a weird issue with Jsoup cutting some json pages in half, this is a work around + // see https://github.com/RipMeApp/ripme/issues/601 + private String workAroundJsonString; @@ -242,6 +246,10 @@ public class InstagramRipper extends AbstractHTMLRipper { json = getJSONFromPage(doc); } catch (IOException e) { logger.warn("Unable to exact json from page"); + } catch (JSONException e) { + // IF there's a json error it's almost certianly because our json string got cut off while being turned + // into a doc, so we try this work around + json = new JSONObject(workAroundJsonString); } // get the rhx_gis value so we can get the next page later on @@ -377,8 +385,8 @@ public class InstagramRipper extends AbstractHTMLRipper { String vars = "{\"id\":\"" + userID + "\",\"first\":50,\"after\":\"" + nextPageID + "\"}"; String ig_gis = getIGGis(vars); logger.info(ig_gis); - toreturn = Http.url("https://www.instagram.com/graphql/query/?query_hash=" + qHash + "&variables=" + vars - ).header("x-instagram-gis", ig_gis).cookies(cookies).ignoreContentType().get(); + + toreturn = getPage("https://www.instagram.com/graphql/query/?query_hash=" + qHash + "&variables=" + vars, ig_gis); if (!pageHasImages(toreturn)) { throw new IOException("No more pages"); } @@ -397,7 +405,7 @@ public class InstagramRipper extends AbstractHTMLRipper { } private boolean pageHasImages(Document doc) { - JSONObject json = new JSONObject(stripHTMLTags(doc.html())); + JSONObject json = new JSONObject(workAroundJsonString); int numberOfImages = json.getJSONObject("data").getJSONObject("user") .getJSONObject("edge_owner_to_timeline_media").getJSONArray("edges").length(); if (numberOfImages == 0) { @@ -406,6 +414,34 @@ public class InstagramRipper extends AbstractHTMLRipper { return true; } + private Document getPage(String url, String ig_gis) { + StringBuilder sb = new StringBuilder(); + try { + // We can't use Jsoup here because it won't download a non-html file larger than a MB + // even if you set maxBodySize to 0 + URLConnection connection = new URL(url).openConnection(); + connection.setRequestProperty("User-Agent", USER_AGENT); + connection.setRequestProperty("x-instagram-gis", ig_gis); + BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream())); + String line; + while ((line = in.readLine()) != null) { + sb.append(line); + + } + in.close(); + workAroundJsonString = sb.toString(); + return Jsoup.parse(sb.toString()); + + } catch (MalformedURLException e) { + logger.info("Unable to get query_hash, " + url + " is a malformed URL"); + return null; + } catch (IOException e) { + logger.info("Unable to get query_hash"); + logger.info(e.getMessage()); + return null; + } + } + private String getQHash(Document doc) { String jsFileURL = "https://www.instagram.com" + doc.select("link[rel=preload]").attr("href"); StringBuilder sb = new StringBuilder(); From d4c4a638e044712c1d12803ee2931a69b1007b4e Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 20 May 2018 10:40:47 -0400 Subject: [PATCH 137/140] Rewrote instagram ripper to use abstractJSONRipper --- .../ripme/ripper/rippers/InstagramRipper.java | 52 +++++++------------ 1 file changed, 19 insertions(+), 33 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index a3f9c2d2..12227418 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -19,7 +19,7 @@ import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.ripper.AbstractJSONRipper; import com.rarchives.ripme.utils.Http; import org.jsoup.Connection; @@ -31,7 +31,7 @@ import com.rarchives.ripme.utils.Utils; import java.util.HashMap; -public class InstagramRipper extends AbstractHTMLRipper { +public class InstagramRipper extends AbstractJSONRipper { String nextPageID = ""; private String qHash; private boolean rippingTag = false; @@ -77,11 +77,9 @@ public class InstagramRipper extends AbstractHTMLRipper { return url.replaceAll("/[A-Z0-9]{8}/", "/"); } - private List getPostsFromSinglePage(Document Doc) { + private List getPostsFromSinglePage(JSONObject json) { List imageURLs = new ArrayList<>(); JSONArray datas; - try { - JSONObject json = getJSONFromPage(Doc); if (json.getJSONObject("entry_data").getJSONArray("PostPage") .getJSONObject(0).getJSONObject("graphql").getJSONObject("shortcode_media") .has("edge_sidecar_to_children")) { @@ -107,10 +105,6 @@ public class InstagramRipper extends AbstractHTMLRipper { } } return imageURLs; - } catch (IOException e) { - logger.error("Unable to get JSON from page " + url.toExternalForm()); - return null; - } } @Override @@ -188,14 +182,14 @@ public class InstagramRipper extends AbstractHTMLRipper { } @Override - public Document getFirstPage() throws IOException { + public JSONObject getFirstPage() throws IOException { Connection.Response resp = Http.url(url).response(); logger.info(resp.cookies()); csrftoken = resp.cookie("csrftoken"); Document p = resp.parse(); // Get the query hash so we can download the next page qHash = getQHash(p); - return p; + return getJSONFromPage(p); } private String getVideoFromPage(String videoID) { @@ -239,18 +233,8 @@ public class InstagramRipper extends AbstractHTMLRipper { } @Override - public List getURLsFromPage(Document doc) { + public List getURLsFromJSON(JSONObject json) { List imageURLs = new ArrayList<>(); - JSONObject json = new JSONObject(); - try { - json = getJSONFromPage(doc); - } catch (IOException e) { - logger.warn("Unable to exact json from page"); - } catch (JSONException e) { - // IF there's a json error it's almost certianly because our json string got cut off while being turned - // into a doc, so we try this work around - json = new JSONObject(workAroundJsonString); - } // get the rhx_gis value so we can get the next page later on if (rhx_gis == null) { @@ -290,7 +274,7 @@ public class InstagramRipper extends AbstractHTMLRipper { if (data.getString("__typename").equals("GraphSidecar")) { try { Document slideShowDoc = Http.url(new URL("https://www.instagram.com/p/" + data.getString("shortcode"))).get(); - List toAdd = getPostsFromSinglePage(slideShowDoc); + List toAdd = getPostsFromSinglePage(getJSONFromPage(slideShowDoc)); for (int slideShowInt = 0; slideShowInt < toAdd.size(); slideShowInt++) { addURLToDownload(new URL(toAdd.get(slideShowInt)), image_date + data.getString("shortcode")); } @@ -329,7 +313,7 @@ public class InstagramRipper extends AbstractHTMLRipper { } else { // We're ripping from a single page logger.info("Ripping from single page"); - imageURLs = getPostsFromSinglePage(doc); + imageURLs = getPostsFromSinglePage(json); } return imageURLs; @@ -356,8 +340,8 @@ public class InstagramRipper extends AbstractHTMLRipper { } @Override - public Document getNextPage(Document doc) throws IOException { - Document toreturn; + public JSONObject getNextPage(JSONObject json) throws IOException { + JSONObject toreturn; java.util.Map cookies = new HashMap(); // This shouldn't be hardcoded and will break one day cookies.put("ig_pr", "1"); @@ -368,10 +352,13 @@ public class InstagramRipper extends AbstractHTMLRipper { sleep(2500); String vars = "{\"tag_name\":\"" + tagName + "\",\"first\":4,\"after\":\"" + nextPageID + "\"}"; String ig_gis = getIGGis(vars); - toreturn = Http.url("https://www.instagram.com/graphql/query/?query_hash=" + qHash + - "&variables=" + vars).header("x-instagram-gis", ig_gis).cookies(cookies).ignoreContentType().get(); + toreturn = getPage("https://www.instagram.com/graphql/query/?query_hash=" + qHash + + "&variables=" + vars, ig_gis); // Sleep for a while to avoid a ban - logger.info(toreturn.html()); + logger.info(toreturn); + if (!pageHasImages(toreturn)) { + throw new IOException("No more pages"); + } return toreturn; } catch (IOException e) { @@ -404,8 +391,7 @@ public class InstagramRipper extends AbstractHTMLRipper { addURLToDownload(url); } - private boolean pageHasImages(Document doc) { - JSONObject json = new JSONObject(workAroundJsonString); + private boolean pageHasImages(JSONObject json) { int numberOfImages = json.getJSONObject("data").getJSONObject("user") .getJSONObject("edge_owner_to_timeline_media").getJSONArray("edges").length(); if (numberOfImages == 0) { @@ -414,7 +400,7 @@ public class InstagramRipper extends AbstractHTMLRipper { return true; } - private Document getPage(String url, String ig_gis) { + private JSONObject getPage(String url, String ig_gis) { StringBuilder sb = new StringBuilder(); try { // We can't use Jsoup here because it won't download a non-html file larger than a MB @@ -430,7 +416,7 @@ public class InstagramRipper extends AbstractHTMLRipper { } in.close(); workAroundJsonString = sb.toString(); - return Jsoup.parse(sb.toString()); + return new JSONObject(sb.toString()); } catch (MalformedURLException e) { logger.info("Unable to get query_hash, " + url + " is a malformed URL"); From 7dbc3cfd9fc4075fc96d779a12e92165b3c28149 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 20 May 2018 13:06:15 -0400 Subject: [PATCH 138/140] Added quick queue support for hentai2read ripper --- .../ripper/rippers/Hentai2readRipper.java | 47 +++++++++++++++---- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java index 78fc50fc..e0dbff17 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java @@ -31,21 +31,50 @@ public class Hentai2readRipper extends AbstractHTMLRipper { return "hentai2read.com"; } - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("https?://hentai2read\\.com/([a-zA-Z0-9_-]*)/(\\d)?/?"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - throw new MalformedURLException("Expected hentai2read.com URL format: " + - "hbrowse.com/COMICID - got " + url + " instead"); + @Override + public boolean hasQueueSupport() { + return true; + } + + @Override + public boolean pageContainsAlbums(URL url) { + logger.info("Page contains albums"); + Pattern pat = Pattern.compile("https?://hentai2read\\.com/([a-zA-Z0-9_-]*)/?"); + Matcher mat = pat.matcher(url.toExternalForm()); + if (mat.matches()) { + return true; } + return false; + } + + @Override + public List getAlbumsToQueue(Document doc) { + List urlsToAddToQueue = new ArrayList<>(); + for (Element elem : doc.select(".nav-chapters > li > div.media > a")) { + urlsToAddToQueue.add(elem.attr("href")); + } + return urlsToAddToQueue; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https?://hentai2read\\.com/([a-zA-Z0-9_-]*)/(\\d+)?/?"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1) + "_" + m.group(2); + } + throw new MalformedURLException("Expected hentai2read.com URL format: " + + "hentai2read.com/COMICID - got " + url + " instead"); + } @Override public Document getFirstPage() throws IOException { String thumbnailLink; try { + // If the page contains albums we want to load the main page + if (pageContainsAlbums(url)) { + return Http.url(url).get(); + } Document tempDoc; tempDoc = Http.url(url).get(); // Get the thumbnail page so we can rip all images without loading every page in the comic From f240f2c49b65d7c90a82d75fd0ef7dd51d103ef6 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 20 May 2018 13:21:46 -0400 Subject: [PATCH 139/140] Fixed hentai2read ripper test --- .../ripme/tst/ripper/rippers/Hentai2readRipperTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java index b5765047..f142635d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java @@ -7,9 +7,7 @@ import com.rarchives.ripme.ripper.rippers.Hentai2readRipper; public class Hentai2readRipperTest extends RippersTest { public void testHentai2readAlbum() throws IOException { - Hentai2readRipper ripper = new Hentai2readRipper(new URL("https://hentai2read.com/sm_school_memorial/")); - testRipper(ripper); - ripper = new Hentai2readRipper(new URL("https://hentai2read.com/sm_school_memorial/1/")); + Hentai2readRipper ripper = new Hentai2readRipper(new URL("https://hentai2read.com/sm_school_memorial/1/")); testRipper(ripper); } } \ No newline at end of file From d11dfe814b7a681d004883da742954080b14ff6f Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 20 May 2018 13:42:33 -0400 Subject: [PATCH 140/140] 1.7.47: Added quick queue support for hentai2read ripper; Fixed instagram ripper; SankakuComplexRipper can now download from different subdomains; Added ripper for bato.to; Added quick queue support for 8muses.download; --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index a46264f9..1bd8dc0d 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.46 + 1.7.47 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 4858d2cc..2cc83cdc 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.46", + "latestVersion": "1.7.47", "changeList": [ + "1.7.47: Added quick queue support for hentai2read ripper; Fixed instagram ripper; SankakuComplexRipper can now download from different subdomains; Added ripper for bato.to; Added quick queue support for 8muses.download; ", "1.7.46: Fixed hentai2read ripper; Rewrote the myhentaicomics ripper to use the new getAlbumsToQueue func; Can now blacklist nhentai tags; SinnercomicsRipper no longer adds -page-01 to folder names; EightmusesRipper now adds file extension to filename; disbaled test for twitch ripper", "1.7.45: Fixed hentai2read ripper; ImageBam album fixed; Added various translations; TsuminoRipper no longer requires album name to download", "1.7.44: Fixed instagram ripper regex", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index f82b2137..59f21f09 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.46"; + private static final String DEFAULT_VERSION = "1.7.47"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar";