From 3342898e774ab6be7ee9815a0a351042889d3b2e Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 26 Apr 2018 10:48:07 -0400 Subject: [PATCH 1/3] Added ripper for rule34.xxx --- .../ripme/ripper/rippers/Rule34Ripper.java | 84 +++++++++++++++++++ .../ripme/ripper/rippers/rule34Ripper.java | 4 + 2 files changed, 88 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/Rule34Ripper.java create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Rule34Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Rule34Ripper.java new file mode 100644 index 00000000..8651bd69 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Rule34Ripper.java @@ -0,0 +1,84 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class Rule34Ripper extends AbstractHTMLRipper { + + public Rule34Ripper(URL url) throws IOException { + super(url); + } + + private String apiUrl; + private int pageNumber = 0; + + @Override + public String getHost() { + return "rule34"; + } + + @Override + public String getDomain() { + return "rule34.xxx"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https?://rule34.xxx/index.php\\?page=post&s=list&tags=([\\S]+)"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected rule34.xxx URL format: " + + "rule34.xxx/index.php\\?page=post&s=list&tags=TAG - got " + url + " instead"); + } + + public URL getAPIUrl() throws MalformedURLException { + URL urlToReturn = new URL("https://rule34.xxx/index.php?page=dapi&s=post&q=index&limit=100&tags=" + getGID(url)); + return urlToReturn; + } + + @Override + public Document getFirstPage() throws IOException { + apiUrl = getAPIUrl().toExternalForm(); + // "url" is an instance field of the superclass + return Http.url(getAPIUrl()).get(); + } + + @Override + public Document getNextPage(Document doc) throws IOException { + if (doc.html().contains("Search error: API limited due to abuse")) { + throw new IOException("No more pages"); + } + pageNumber += 1; + String nextPage = apiUrl + "&pid=" + pageNumber; + return Http.url(nextPage).get(); + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + for (Element el : doc.select("posts > post")) { + String imageSource = el.select("post").attr("file_url"); + result.add(imageSource); + + } + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java new file mode 100644 index 00000000..9402256b --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java @@ -0,0 +1,4 @@ +package com.rarchives.ripme.ripper.rippers; + +public class rule34Ripper { +} From 7cf8e59ee72ca906fb584ebafd021d8be5e6ef4f Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 26 Apr 2018 10:48:28 -0400 Subject: [PATCH 2/3] Removed misnamed file --- .../java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java deleted file mode 100644 index 9402256b..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/rule34Ripper.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -public class rule34Ripper { -} From c076fe2462dcdb1062f574405277ba8a80ab6b41 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 26 Apr 2018 10:58:35 -0400 Subject: [PATCH 3/3] Added unit test Rule34Ripper --- .../ripme/tst/ripper/rippers/Rule34RipperTest.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/Rule34RipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Rule34RipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Rule34RipperTest.java new file mode 100644 index 00000000..cffc807d --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Rule34RipperTest.java @@ -0,0 +1,14 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.Rule34Ripper; + +public class Rule34RipperTest extends RippersTest { + public void testShesFreakyRip() throws IOException { + Rule34Ripper ripper = new Rule34Ripper(new URL("https://rule34.xxx/index.php?page=post&s=list&tags=bimbo")); + testRipper(ripper); + } + +}