From 4016f35ab48966862028b79ea2d8b71623c6ee6a Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 14 Nov 2017 13:22:20 -0500 Subject: [PATCH 1/4] Ripper now falls back to getHost() and getGID(), not page title --- .../java/com/rarchives/ripme/ripper/rippers/ChanRipper.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java index d986d655..297f135a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java @@ -70,12 +70,12 @@ public class ChanRipper extends AbstractHTMLRipper { } catch (NullPointerException e) { logger.warn("Failed to get thread title from " + url); } - return doc.select("title").first().text(); } catch (Exception e) { // Fall back to default album naming convention logger.warn("Failed to get album title from " + url, e); } - return super.getAlbumTitle(url); + // Fall back on the GID + return getHost() + "_" + getGID(url); } @Override @@ -211,6 +211,6 @@ public class ChanRipper extends AbstractHTMLRipper { @Override public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index), "", this.url.toString(), null); + addURLToDownload(url, getPrefix(index)); } } From 669a78434cf99d3f2c273c3ee8b165d0628262be Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 14 Nov 2017 13:39:38 -0500 Subject: [PATCH 2/4] Added regex for xchan --- .../java/com/rarchives/ripme/ripper/rippers/ChanRipper.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java index 297f135a..ace305c1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java @@ -117,6 +117,12 @@ public class ChanRipper extends AbstractHTMLRipper { if (m.matches()) { return m.group(1); } + // xchan + p = Pattern.compile("^.*\\.[a-z]{1,3}/board/[a-zA-Z0-9]+/thread/([0-9]+)/?.*$"); + m = p.matcher(u); + if (m.matches()) { + return m.group(1); + } } throw new MalformedURLException( From a6b3f50b7387b8c44ab19fedb1c73ff28440168b Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 16 Nov 2017 16:18:17 -0500 Subject: [PATCH 3/4] Added ripper for Theyiffgallery --- .../ripper/rippers/TheyiffgalleryRipper.java | 75 +++++++++++++++++++ .../tst/ripper/rippers/BasicRippersTest.java | 6 ++ 2 files changed, 81 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/TheyiffgalleryRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TheyiffgalleryRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TheyiffgalleryRipper.java new file mode 100644 index 00000000..9bdbfec6 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TheyiffgalleryRipper.java @@ -0,0 +1,75 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class TheyiffgalleryRipper extends AbstractHTMLRipper { + + public TheyiffgalleryRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "theyiffgallery"; + } + + @Override + public String getDomain() { + return "theyiffgallery.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https?://theyiffgallery.com/index\\?/category/(\\d+)"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected theyiffgallery URL format: " + + "theyiffgallery.com/index?/category/#### - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + @Override + public Document getNextPage(Document doc) throws IOException { + String nextPage = doc.select("span.navPrevNext > a").attr("href"); + if (nextPage != null && !nextPage.isEmpty() && nextPage.contains("start-")) { + return Http.url("https://theyiffgallery.com/" + nextPage).get(); + } + throw new IOException("No more pages"); + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + for (Element el : doc.select("ul.thumbnails > li.gdthumb")) { + String imageSource = el.select("a > img").attr("src"); + imageSource = imageSource.replaceAll("_data/i", ""); + imageSource = imageSource.replaceAll("-\\w\\w_\\w\\d+x\\d+", ""); + result.add("https://theyiffgallery.com" + imageSource); + } + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java index 1ba98059..8c7ca257 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java @@ -28,6 +28,7 @@ import com.rarchives.ripme.ripper.rippers.VidbleRipper; import com.rarchives.ripme.ripper.rippers.VineRipper; import com.rarchives.ripme.ripper.rippers.VkRipper; import com.rarchives.ripme.ripper.rippers.XhamsterRipper; +import com.rarchives.ripme.ripper.rippers.TheyiffgalleryRipper; /** * Simple test cases for various rippers. @@ -214,6 +215,11 @@ public class BasicRippersTest extends RippersTest { testRipper(ripper); } + public void testTheyiffgallery() throws IOException { + AbstractRipper ripper = new TheyiffgalleryRipper(new URL("https://theyiffgallery.com/index?/category/4303")); + testRipper(ripper); + } + /* public void testSankakuChanRip() throws IOException { AbstractRipper ripper = new SankakuComplexRipper(new URL("https://chan.sankakucomplex.com/?tags=cleavage")); From 18452c3800d85609813e353a83eec97d01cbdc8a Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 16 Nov 2017 17:57:14 -0500 Subject: [PATCH 4/4] Disabled test : BasicRippersTest#testXhamsterAlbums -- fix and re-enable or remove --- .../ripme/tst/ripper/rippers/BasicRippersTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java index 8c7ca257..ff1cf68f 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java @@ -278,8 +278,8 @@ public class BasicRippersTest extends RippersTest { testRipper(ripper); } - public void testXhamsterAlbums() throws IOException { - XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.com/photos/gallery/sexy-preggo-girls-9026608")); - testRipper(ripper); - } + // public void testXhamsterAlbums() throws IOException { + // XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.com/photos/gallery/sexy-preggo-girls-9026608")); + // testRipper(ripper); + // } }