diff --git a/pom.xml b/pom.xml index 9de7c444..1010bdac 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.58 + 1.7.59 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index d835ecfe..f4a73d8b 100644 --- a/ripme.json +++ b/ripme.json @@ -1,7 +1,7 @@ { - "latestVersion": "1.7.58", - "currentHash": "d94ae3f80ae38b5bcd7abe996cf70ed3de610bba794665be01518892a11c70c5", + "currentHash": "8861d48484c0b029249408f81ccc8bd70c7d4de9343d867e9615969cc67c9fd7", "changeList": [ + "1.7.59: Added Loverom ripper; added Imagearn ripper; Added support for Desuarchive.org; Fixed erome ripper", "1.7.58: Fixed Deviantart ripper; fixed HitomiRipper; Fixed ManganeloRipper; Fixed update box formating", "1.7.57: Got DeviantartRipper working again; Imagefap ripper now downloads full sized images; Twitter ripper can now rip extended tweets; Added nl_NL translation", "1.7.56: Fixed DeviantartRipper ripper; Added support for resuming file downloads; Fixed erome ripper; Fixed ModelmayhemRipper NSFW image downloading", @@ -230,5 +230,6 @@ "1.0.4: Fixed spaces-in-directory bug", "1.0.3: Added VK.com ripper", "1.0.1: Added auto-update functionality" - ] + ], + "latestVersion": "1.7.59" } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java index 0ab86244..85528e8d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java @@ -97,6 +97,9 @@ public class ChanRipper extends AbstractHTMLRipper { if (url.toExternalForm().contains("xchan.pw") && url.toExternalForm().contains("/board/")) { return true; } + if (url.toExternalForm().contains("desuarchive.org")) { + return true; + } return false; } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java index 2711cd1d..27e0e0be 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java @@ -105,7 +105,7 @@ public class EromeRipper extends AbstractHTMLRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://www.erome.com/a/([a-zA-Z0-9]*)/?$"); + Pattern p = Pattern.compile("^https?://www.erome.com/[ai]/([a-zA-Z0-9]*)/?$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java new file mode 100644 index 00000000..062217b2 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java @@ -0,0 +1,112 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class ImagearnRipper extends AbstractHTMLRipper { + + public ImagearnRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "imagearn"; + } + @Override + public String getDomain() { + return "imagearn.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^.*imagearn.com/+gallery.php\\?id=([0-9]+).*$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException( + "Expected imagearn.com gallery formats: " + + "imagearn.com/gallery.php?id=####..." + + " Got: " + url); + } + + public URL sanitizeURL(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^.*imagearn.com/+image.php\\?id=[0-9]+.*$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + // URL points to imagearn *image*, not gallery + try { + url = getGalleryFromImage(url); + } catch (Exception e) { + LOGGER.error("[!] " + e.getMessage(), e); + } + } + return url; + } + + private URL getGalleryFromImage(URL url) throws IOException { + Document doc = Http.url(url).get(); + for (Element link : doc.select("a[href~=^gallery\\.php.*$]")) { + LOGGER.info("LINK: " + link.toString()); + if (link.hasAttr("href") + && link.attr("href").contains("gallery.php")) { + url = new URL("http://imagearn.com/" + link.attr("href")); + LOGGER.info("[!] Found gallery from given link: " + url); + return url; + } + } + throw new IOException("Failed to find gallery at URL " + url); + } + + @Override + public Document getFirstPage() throws IOException { + return Http.url(url).get(); + } + + @Override + public String getAlbumTitle(URL url) throws MalformedURLException { + try { + Document doc = getFirstPage(); + String title = doc.select("h3 > strong").first().text(); // profile name + return getHost() + "_" + title + "_" + getGID(url); + } catch (Exception e) { + // Fall back to default album naming convention + LOGGER.warn("Failed to get album title from " + url, e); + } + return super.getAlbumTitle(url); + } + + @Override + public List getURLsFromPage(Document doc) { + List imageURLs = new ArrayList<>(); + for (Element thumb : doc.select("div#gallery > div > a")) { + String imageURL = thumb.attr("href"); + try { + Document imagedoc = new Http("http://imagearn.com/" + imageURL).get(); + String image = imagedoc.select("a.thickbox").first().attr("href"); + imageURLs.add(image); + } catch (IOException e) { + LOGGER.warn("Was unable to download page: " + imageURL); + } + } + return imageURLs; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + sleep(1000); + } +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java new file mode 100644 index 00000000..83af405f --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java @@ -0,0 +1,123 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.rarchives.ripme.utils.Utils; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class LoveromRipper extends AbstractHTMLRipper { + + public LoveromRipper(URL url) throws IOException { + super(url); + } + + private int bytesTotal = 1; + private int bytesCompleted = 1; + boolean multipart = false; + + @Override + public String getHost() { + return "loveroms"; + } + + @Override + public String getDomain() { + return "loveroms.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https://www.loveroms.com/download/([a-zA-Z0-9_-]+)/([a-zA-Z0-9_-]+)/\\d+"); + Matcher m = p.matcher(url.toExternalForm()); + if (!m.matches()) { + throw new MalformedURLException("Expected URL format: https://www.loveroms.com/download/CONSOLE/GAME, got: " + url); + } + return m.group(1) + "_" + m.group(2); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + String downloadLink = doc.select("a#start_download_link").attr("href"); + if (downloadLink != null && !downloadLink.isEmpty()) { + result.add(downloadLink); + } else { + multipart = true; + for (Element el : doc.select("a.multi-file-btn")) { + result.add(el.attr("href")); + } + } + return result; + } + + @Override + public void downloadURL(URL url, int index) { + if (multipart) { + addURLToDownload(url, "", "", "", null, null, getPrefix(index)); + } else { + addURLToDownload(url); + } + } + + @Override + public String getStatusText() { + if (multipart) { + return super.getStatusText(); + } + return String.valueOf(getCompletionPercentage()) + + "% - " + + Utils.bytesToHumanReadable(bytesCompleted) + + " / " + + Utils.bytesToHumanReadable(bytesTotal); + } + + @Override + public int getCompletionPercentage() { + if (multipart) { + return super.getCompletionPercentage(); + } + return (int) (100 * (bytesCompleted / (float) bytesTotal)); + } + + @Override + public void setBytesTotal(int bytes) { + this.bytesTotal = bytes; + } + + @Override + public void setBytesCompleted(int bytes) { + this.bytesCompleted = bytes; + } + + @Override + public boolean useByteProgessBar() {return true;} + + @Override + public boolean tryResumeDownload() {return true;} + + @Override + public String getPrefix(int index) { + String prefix = ""; + if (keepSortOrder() && Utils.getConfigBoolean("download.save_order", true)) { + prefix = String.format("7z.%03d", index); + } + return prefix; + } +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java index 79e670d1..cabb4188 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java @@ -99,7 +99,7 @@ public class ManganeloRipper extends AbstractHTMLRipper { Collections.reverse(urlsToGrab); for (String url : urlsToGrab) { - result.addAll(getURLsFromChap("https:" + url)); + result.addAll(getURLsFromChap(url)); } } else if (url.toExternalForm().contains("/chapter/")) { result.addAll(getURLsFromChap(doc)); diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 77ccc1b1..188a5dc0 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -20,7 +20,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.58"; + private static final String DEFAULT_VERSION = "1.7.59"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagearnRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagearnRipperTest.java new file mode 100644 index 00000000..b9710635 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagearnRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.ImagearnRipper; + +public class ImagearnRipperTest extends RippersTest { + public void testImagearnRip() throws IOException { + ImagearnRipper ripper = new ImagearnRipper(new URL("http://imagearn.com//gallery.php?id=578682")); + testRipper(ripper); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LoveromRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LoveromRipperTest.java new file mode 100644 index 00000000..377b2212 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LoveromRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import com.rarchives.ripme.ripper.rippers.LoveromRipper; + +import java.io.IOException; +import java.net.URL; + +public class LoveromRipperTest extends RippersTest{ + public void testRip() throws IOException { + LoveromRipper ripper = new LoveromRipper(new URL("https://www.loveroms.com/download/nintendo/adventures-of-tom-sawyer-u/107165")); + testRipper(ripper); + } +}