From de0dda0d9fb86125b4e761cd7f68ee38a421830b Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 1 Jul 2018 21:49:00 -0400 Subject: [PATCH 01/10] Added ripper for loveroms.com --- .../ripme/ripper/rippers/LoveromRipper.java | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java new file mode 100644 index 00000000..cfc264a9 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java @@ -0,0 +1,111 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.rarchives.ripme.utils.Utils; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class LoveromRipper extends AbstractHTMLRipper { + + public LoveromRipper(URL url) throws IOException { + super(url); + } + + private int bytesTotal = 1; + private int bytesCompleted = 1; + + @Override + public String getHost() { + return "loveroms"; + } + + @Override + public String getDomain() { + return "loveroms.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https://www.loveroms.com/download/([a-zA-Z0-9_-]+)/([a-zA-Z0-9_-]+)/\\d+"); + Matcher m = p.matcher(url.toExternalForm()); + if (!m.matches()) { + throw new MalformedURLException("Expected URL format: https://www.loveroms.com/download/CONSOLE/GAME, got: " + url); + } + return m.group(1) + "_" + m.group(2); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + String downloadLink = doc.select("a#download_link").attr("href"); + if (downloadLink != null && !downloadLink.isEmpty()) { + result.add(downloadLink); + } else { + for (Element el : doc.select("a.multi-file-btn")) { + result.add(el.attr("href")); + } + } + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } + + @Override + public String getStatusText() { + return String.valueOf(getCompletionPercentage()) + + "% - " + + Utils.bytesToHumanReadable(bytesCompleted) + + " / " + + Utils.bytesToHumanReadable(bytesTotal); + } + +// @Override +// public int getCompletionPercentage() { +// return (int) (100 * (bytesCompleted / (float) bytesTotal)); +// } +// +// @Override +// public void setBytesTotal(int bytes) { +// this.bytesTotal = bytes; +// } +// +// @Override +// public void setBytesCompleted(int bytes) { +// this.bytesCompleted = bytes; +// } + +// @Override +// public boolean useByteProgessBar() {return true;} +// +// @Override +// public boolean tryResumeDownload() {return true;} + + @Override + public String getPrefix(int index) { + String prefix = ""; + if (keepSortOrder() && Utils.getConfigBoolean("download.save_order", true)) { + prefix = String.format("%03d_", index); + } + return prefix; + } +} From 4b7bd12313467fafef8a225ba903a155afeefd5a Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 3 Jul 2018 03:52:53 -0400 Subject: [PATCH 02/10] Uncommented out some code --- .../ripme/ripper/rippers/LoveromRipper.java | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java index cfc264a9..88c9c0f5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java @@ -79,26 +79,26 @@ public class LoveromRipper extends AbstractHTMLRipper { Utils.bytesToHumanReadable(bytesTotal); } -// @Override -// public int getCompletionPercentage() { -// return (int) (100 * (bytesCompleted / (float) bytesTotal)); -// } -// -// @Override -// public void setBytesTotal(int bytes) { -// this.bytesTotal = bytes; -// } -// -// @Override -// public void setBytesCompleted(int bytes) { -// this.bytesCompleted = bytes; -// } + @Override + public int getCompletionPercentage() { + return (int) (100 * (bytesCompleted / (float) bytesTotal)); + } -// @Override -// public boolean useByteProgessBar() {return true;} -// -// @Override -// public boolean tryResumeDownload() {return true;} + @Override + public void setBytesTotal(int bytes) { + this.bytesTotal = bytes; + } + + @Override + public void setBytesCompleted(int bytes) { + this.bytesCompleted = bytes; + } + + @Override + public boolean useByteProgessBar() {return true;} + + @Override + public boolean tryResumeDownload() {return true;} @Override public String getPrefix(int index) { From 13dd2694b45ec21ead0f2ed2e7a1878645a44cf3 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Mon, 9 Jul 2018 11:19:15 -0400 Subject: [PATCH 03/10] Fixed multipart file downloading --- .../com/rarchives/ripme/ripper/rippers/LoveromRipper.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java index 88c9c0f5..bddecf64 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java @@ -23,6 +23,7 @@ public class LoveromRipper extends AbstractHTMLRipper { private int bytesTotal = 1; private int bytesCompleted = 1; + boolean multipart = false; @Override public String getHost() { @@ -58,6 +59,7 @@ public class LoveromRipper extends AbstractHTMLRipper { if (downloadLink != null && !downloadLink.isEmpty()) { result.add(downloadLink); } else { + multipart = true; for (Element el : doc.select("a.multi-file-btn")) { result.add(el.attr("href")); } @@ -72,6 +74,9 @@ public class LoveromRipper extends AbstractHTMLRipper { @Override public String getStatusText() { + if (multipart) { + return super.getStatusText(); + } return String.valueOf(getCompletionPercentage()) + "% - " + Utils.bytesToHumanReadable(bytesCompleted) + @@ -81,6 +86,9 @@ public class LoveromRipper extends AbstractHTMLRipper { @Override public int getCompletionPercentage() { + if (multipart) { + return super.getCompletionPercentage(); + } return (int) (100 * (bytesCompleted / (float) bytesTotal)); } From 67e2451744af30b3ec2c221a7239dfba12e5c606 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 10 Jul 2018 13:24:56 -0400 Subject: [PATCH 04/10] Now properly names multi-part files; Updated to work with recent changes to site --- .../rarchives/ripme/ripper/rippers/LoveromRipper.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java index bddecf64..83af405f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java @@ -55,7 +55,7 @@ public class LoveromRipper extends AbstractHTMLRipper { @Override public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); - String downloadLink = doc.select("a#download_link").attr("href"); + String downloadLink = doc.select("a#start_download_link").attr("href"); if (downloadLink != null && !downloadLink.isEmpty()) { result.add(downloadLink); } else { @@ -69,7 +69,11 @@ public class LoveromRipper extends AbstractHTMLRipper { @Override public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); + if (multipart) { + addURLToDownload(url, "", "", "", null, null, getPrefix(index)); + } else { + addURLToDownload(url); + } } @Override @@ -112,7 +116,7 @@ public class LoveromRipper extends AbstractHTMLRipper { public String getPrefix(int index) { String prefix = ""; if (keepSortOrder() && Utils.getConfigBoolean("download.save_order", true)) { - prefix = String.format("%03d_", index); + prefix = String.format("7z.%03d", index); } return prefix; } From f7a1255bab7d2bbfc1225a461b598adf930b459d Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 11 Jul 2018 13:52:29 -0400 Subject: [PATCH 05/10] Added loverom unit test --- .../ripme/tst/ripper/rippers/LoveromRipperTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/LoveromRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LoveromRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LoveromRipperTest.java new file mode 100644 index 00000000..377b2212 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LoveromRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import com.rarchives.ripme.ripper.rippers.LoveromRipper; + +import java.io.IOException; +import java.net.URL; + +public class LoveromRipperTest extends RippersTest{ + public void testRip() throws IOException { + LoveromRipper ripper = new LoveromRipper(new URL("https://www.loveroms.com/download/nintendo/adventures-of-tom-sawyer-u/107165")); + testRipper(ripper); + } +} From 3553118b54e84968c8bf5530dc548001227de2f7 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 12 Jul 2018 13:44:45 -0400 Subject: [PATCH 06/10] Added support for Desuarchive.org --- .../java/com/rarchives/ripme/ripper/rippers/ChanRipper.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java index 0ab86244..85528e8d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java @@ -97,6 +97,9 @@ public class ChanRipper extends AbstractHTMLRipper { if (url.toExternalForm().contains("xchan.pw") && url.toExternalForm().contains("/board/")) { return true; } + if (url.toExternalForm().contains("desuarchive.org")) { + return true; + } return false; } From a65ad6f75616b8f022a5dee0fd773353248e1783 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 13 Jul 2018 18:14:36 -0400 Subject: [PATCH 07/10] Fixed ManganeloRipper --- .../com/rarchives/ripme/ripper/rippers/ManganeloRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java index 79e670d1..cabb4188 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java @@ -99,7 +99,7 @@ public class ManganeloRipper extends AbstractHTMLRipper { Collections.reverse(urlsToGrab); for (String url : urlsToGrab) { - result.addAll(getURLsFromChap("https:" + url)); + result.addAll(getURLsFromChap(url)); } } else if (url.toExternalForm().contains("/chapter/")) { result.addAll(getURLsFromChap(doc)); From 3287b8c59afdb02f55def83aae3c3ff4a8f20d51 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 13 Jul 2018 19:01:55 -0400 Subject: [PATCH 08/10] Fixed erome video downloading --- .../java/com/rarchives/ripme/ripper/rippers/EromeRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java index 2711cd1d..27e0e0be 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java @@ -105,7 +105,7 @@ public class EromeRipper extends AbstractHTMLRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://www.erome.com/a/([a-zA-Z0-9]*)/?$"); + Pattern p = Pattern.compile("^https?://www.erome.com/[ai]/([a-zA-Z0-9]*)/?$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); From 3f18d8a9909a0e8578c6d6b2b3f16e26dc367297 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 13 Jul 2018 20:06:54 -0400 Subject: [PATCH 09/10] Readded Imagearn ripper and test --- .../ripme/ripper/rippers/ImagearnRipper.java | 112 ++++++++++++++++++ .../ripper/rippers/ImagearnRipperTest.java | 13 ++ 2 files changed, 125 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagearnRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java new file mode 100644 index 00000000..062217b2 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java @@ -0,0 +1,112 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class ImagearnRipper extends AbstractHTMLRipper { + + public ImagearnRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "imagearn"; + } + @Override + public String getDomain() { + return "imagearn.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^.*imagearn.com/+gallery.php\\?id=([0-9]+).*$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException( + "Expected imagearn.com gallery formats: " + + "imagearn.com/gallery.php?id=####..." + + " Got: " + url); + } + + public URL sanitizeURL(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^.*imagearn.com/+image.php\\?id=[0-9]+.*$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + // URL points to imagearn *image*, not gallery + try { + url = getGalleryFromImage(url); + } catch (Exception e) { + LOGGER.error("[!] " + e.getMessage(), e); + } + } + return url; + } + + private URL getGalleryFromImage(URL url) throws IOException { + Document doc = Http.url(url).get(); + for (Element link : doc.select("a[href~=^gallery\\.php.*$]")) { + LOGGER.info("LINK: " + link.toString()); + if (link.hasAttr("href") + && link.attr("href").contains("gallery.php")) { + url = new URL("http://imagearn.com/" + link.attr("href")); + LOGGER.info("[!] Found gallery from given link: " + url); + return url; + } + } + throw new IOException("Failed to find gallery at URL " + url); + } + + @Override + public Document getFirstPage() throws IOException { + return Http.url(url).get(); + } + + @Override + public String getAlbumTitle(URL url) throws MalformedURLException { + try { + Document doc = getFirstPage(); + String title = doc.select("h3 > strong").first().text(); // profile name + return getHost() + "_" + title + "_" + getGID(url); + } catch (Exception e) { + // Fall back to default album naming convention + LOGGER.warn("Failed to get album title from " + url, e); + } + return super.getAlbumTitle(url); + } + + @Override + public List getURLsFromPage(Document doc) { + List imageURLs = new ArrayList<>(); + for (Element thumb : doc.select("div#gallery > div > a")) { + String imageURL = thumb.attr("href"); + try { + Document imagedoc = new Http("http://imagearn.com/" + imageURL).get(); + String image = imagedoc.select("a.thickbox").first().attr("href"); + imageURLs.add(image); + } catch (IOException e) { + LOGGER.warn("Was unable to download page: " + imageURL); + } + } + return imageURLs; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + sleep(1000); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagearnRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagearnRipperTest.java new file mode 100644 index 00000000..b9710635 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagearnRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.ImagearnRipper; + +public class ImagearnRipperTest extends RippersTest { + public void testImagearnRip() throws IOException { + ImagearnRipper ripper = new ImagearnRipper(new URL("http://imagearn.com//gallery.php?id=578682")); + testRipper(ripper); + } +} From cbfd3638425282eeb4cadc15747ff04315486400 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 14 Jul 2018 14:56:45 -0400 Subject: [PATCH 10/10] 1.7.59: Added Loverom ripper; added Imagearn ripper; Added support for Desuarchive.org; Fixed erome ripper --- pom.xml | 2 +- ripme.json | 7 ++++--- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 9de7c444..1010bdac 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.58 + 1.7.59 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index d835ecfe..f4a73d8b 100644 --- a/ripme.json +++ b/ripme.json @@ -1,7 +1,7 @@ { - "latestVersion": "1.7.58", - "currentHash": "d94ae3f80ae38b5bcd7abe996cf70ed3de610bba794665be01518892a11c70c5", + "currentHash": "8861d48484c0b029249408f81ccc8bd70c7d4de9343d867e9615969cc67c9fd7", "changeList": [ + "1.7.59: Added Loverom ripper; added Imagearn ripper; Added support for Desuarchive.org; Fixed erome ripper", "1.7.58: Fixed Deviantart ripper; fixed HitomiRipper; Fixed ManganeloRipper; Fixed update box formating", "1.7.57: Got DeviantartRipper working again; Imagefap ripper now downloads full sized images; Twitter ripper can now rip extended tweets; Added nl_NL translation", "1.7.56: Fixed DeviantartRipper ripper; Added support for resuming file downloads; Fixed erome ripper; Fixed ModelmayhemRipper NSFW image downloading", @@ -230,5 +230,6 @@ "1.0.4: Fixed spaces-in-directory bug", "1.0.3: Added VK.com ripper", "1.0.1: Added auto-update functionality" - ] + ], + "latestVersion": "1.7.59" } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 77ccc1b1..188a5dc0 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -20,7 +20,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.58"; + private static final String DEFAULT_VERSION = "1.7.59"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar";