From c4f2d52370b9a92d306f7ec4715431e31d3cdbaa Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Mon, 25 Jun 2018 05:32:37 -0400 Subject: [PATCH 1/2] Fixed erome ripper and added quickQueue support --- .../ripme/ripper/rippers/EromeRipper.java | 78 +++++++++---------- 1 file changed, 35 insertions(+), 43 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java index a01c134a..2711cd1d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java @@ -22,6 +22,8 @@ import com.rarchives.ripme.utils.Http; */ public class EromeRipper extends AbstractHTMLRipper { + boolean rippingProfile; + public EromeRipper (URL url) throws IOException { super(url); @@ -42,6 +44,27 @@ public class EromeRipper extends AbstractHTMLRipper { addURLToDownload(url, getPrefix(index)); } + @Override + public boolean hasQueueSupport() { + return true; + } + + @Override + public boolean pageContainsAlbums(URL url) { + Pattern pa = Pattern.compile("https?://www.erome.com/([a-zA-Z0-9_-]*)/?"); + Matcher ma = pa.matcher(url.toExternalForm()); + return ma.matches(); + } + + @Override + public List getAlbumsToQueue(Document doc) { + List urlsToAddToQueue = new ArrayList<>(); + for (Element elem : doc.select("div#albums > div.album > a")) { + urlsToAddToQueue.add(elem.attr("href")); + } + return urlsToAddToQueue; + } + @Override public String getAlbumTitle(URL url) throws MalformedURLException { try { @@ -53,6 +76,8 @@ public class EromeRipper extends AbstractHTMLRipper { } catch (IOException e) { // Fall back to default album naming convention LOGGER.info("Unable to find title at " + url); + } catch (NullPointerException e) { + return getHost() + "_" + getGID(url); } return super.getAlbumTitle(url); } @@ -66,21 +91,7 @@ public class EromeRipper extends AbstractHTMLRipper { @Override public List getURLsFromPage(Document doc) { List URLs = new ArrayList<>(); - //Pictures - Elements imgs = doc.select("div.img > img.img-front"); - for (Element img : imgs) { - String imageURL = img.attr("src"); - imageURL = "https:" + imageURL; - URLs.add(imageURL); - } - //Videos - Elements vids = doc.select("div.video > video > source"); - for (Element vid : vids) { - String videoURL = vid.attr("src"); - URLs.add("https:" + videoURL); - } - - return URLs; + return getMediaFromPage(doc); } @Override @@ -100,7 +111,7 @@ public class EromeRipper extends AbstractHTMLRipper { return m.group(1); } - p = Pattern.compile("^https?://erome.com/a/([a-zA-Z0-9]*)/?$"); + p = Pattern.compile("^https?://www.erome.com/([a-zA-Z0-9_-]+)/?$"); m = p.matcher(url.toExternalForm()); if (m.matches()) { @@ -110,34 +121,15 @@ public class EromeRipper extends AbstractHTMLRipper { throw new MalformedURLException("erome album not found in " + url + ", expected https://www.erome.com/album"); } - public static List getURLs(URL url) throws IOException{ - - Response resp = Http.url(url) - .ignoreContentType() - .response(); - - Document doc = resp.parse(); - - List URLs = new ArrayList<>(); - //Pictures - Elements imgs = doc.getElementsByTag("img"); - for (Element img : imgs) { - if (img.hasClass("album-image")) { - String imageURL = img.attr("src"); - imageURL = "https:" + imageURL; - URLs.add(new URL(imageURL)); - } + private List getMediaFromPage(Document doc) { + List results = new ArrayList<>(); + for (Element el : doc.select("img.img-front")) { + results.add("https:" + el.attr("src")); } - //Videos - Elements vids = doc.getElementsByTag("video"); - for (Element vid : vids) { - if (vid.hasClass("album-video")) { - Elements source = vid.getElementsByTag("source"); - String videoURL = source.first().attr("src"); - URLs.add(new URL(videoURL)); - } + for (Element el : doc.select("source[label=HD]")) { + results.add("https:" + el.attr("src")); } - - return URLs; + return results; } + } From b63f8f16b9474e2023e23c9d51a296373daeebed Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Mon, 25 Jun 2018 06:03:10 -0400 Subject: [PATCH 2/2] Added unit test for erome --- .../tst/ripper/rippers/EromeRipperTest.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/EromeRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EromeRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EromeRipperTest.java new file mode 100644 index 00000000..c07c41ac --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EromeRipperTest.java @@ -0,0 +1,40 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.EromeRipper; + +public class EromeRipperTest extends RippersTest { + + public void testGetGIDProfilePage() throws IOException { + URL url = new URL("https://www.erome.com/Jay-Jenna"); + EromeRipper ripper = new EromeRipper(url); + assertEquals("Jay-Jenna", ripper.getGID(url)); + } + + public void testGetGIDAlbum() throws IOException { + URL url = new URL("https://www.erome.com/a/KbDAM1XT"); + EromeRipper ripper = new EromeRipper(url); + assertEquals("KbDAM1XT", ripper.getGID(url)); + } + + public void testGetAlbumsToQueue() throws IOException { + URL url = new URL("https://www.erome.com/Jay-Jenna"); + EromeRipper ripper = new EromeRipper(url); + assert(2 >= ripper.getAlbumsToQueue(ripper.getFirstPage()).size()); + } + + public void testPageContainsAlbums() throws IOException { + URL url = new URL("https://www.erome.com/Jay-Jenna"); + EromeRipper ripper = new EromeRipper(url); + assert(ripper.pageContainsAlbums(url)); + assert(!ripper.pageContainsAlbums(new URL("https://www.erome.com/a/KbDAM1XT"))); + } + + public void testRip() throws IOException { + URL url = new URL("https://www.erome.com/a/4FqeUxil"); + EromeRipper ripper = new EromeRipper(url); + testRipper(ripper); + } +}