From 47c67b173369d610106728405908516672eb69eb Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 19 Aug 2017 00:13:22 -0400 Subject: [PATCH 1/3] Added basic webtoons ripper --- .../ripme/ripper/rippers/WebtoonsRipper.java | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java new file mode 100644 index 00000000..dba049cc --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java @@ -0,0 +1,91 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import java.util.Map; +import java.util.HashMap; +import org.jsoup.Connection.Response; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class WebtoonsRipper extends AbstractHTMLRipper { + private Map cookies = new HashMap(); + + public WebtoonsRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "webtoons"; + } + + @Override + public String getDomain() { + return "www.webtoons.com"; + } + + @Override + public boolean canRip(URL url) { + Pattern pat = Pattern.compile("https?://www\\.webtoons\\.com/[a-zA-Z]+/[a-zA-Z]+/([a-zA-Z0-9_-]*)/ep-\\d+/\\S*"); + Matcher mat = pat.matcher(url.toExternalForm()); + if (mat.matches()) { + return true; + } + return false; + } + + + @Override + public String getAlbumTitle(URL url) throws MalformedURLException { + Pattern pat = Pattern.compile("https?://www\\.webtoons\\.com/[a-zA-Z]+/[a-zA-Z]+/([a-zA-Z0-9_-]*)/ep-\\d+/\\S*"); + Matcher mat = pat.matcher(url.toExternalForm()); + if (mat.matches()) { + return mat.group(1); + } + + return super.getAlbumTitle(url); + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern pat = Pattern.compile("https?://www\\.webtoons\\.com/[a-zA-Z]+/[a-zA-Z]+/([a-zA-Z0-9_-]*)/ep-\\d+/\\S*"); + Matcher mat = pat.matcher(url.toExternalForm()); + if (mat.matches()) { + return mat.group(1); + } + throw new MalformedURLException("You should never see this error message"); + } + + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList(); + for (Element elem : doc.select("div.viewer_img > img")) { + result.add(elem.attr("data-url")); + } + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index), "", this.url.toExternalForm(), cookies); + } + + @Override + public Document getFirstPage() throws IOException { + Response resp = Http.url(url).response(); + cookies = resp.cookies(); + return Http.url(url).get(); + } +} From e323495b657b6d215fb8814f5d989d685eb03e92 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 19 Aug 2017 00:27:00 -0400 Subject: [PATCH 2/3] Webtoon ripper can now rip from all pages --- .../ripme/ripper/rippers/WebtoonsRipper.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java index dba049cc..96471c72 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java @@ -88,4 +88,16 @@ public class WebtoonsRipper extends AbstractHTMLRipper { cookies = resp.cookies(); return Http.url(url).get(); } + + @Override + public Document getNextPage(Document doc) throws IOException { + // Find next page + String nextUrl = ""; + Element elem = doc.select("a.pg_next").first(); + nextUrl = elem.attr("href"); + if (nextUrl == "") { + throw new IOException("No more pages"); + } + return Http.url(nextUrl).get(); + } } From edd6dfa0a3f45bf05567aa9a9bd034bff67e0439 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 3 Nov 2017 00:01:13 -0400 Subject: [PATCH 3/3] Can now rip all webtoons comics --- .../rarchives/ripme/ripper/rippers/WebtoonsRipper.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java index 96471c72..5541fb98 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java @@ -37,7 +37,7 @@ public class WebtoonsRipper extends AbstractHTMLRipper { @Override public boolean canRip(URL url) { - Pattern pat = Pattern.compile("https?://www\\.webtoons\\.com/[a-zA-Z]+/[a-zA-Z]+/([a-zA-Z0-9_-]*)/ep-\\d+/\\S*"); + Pattern pat = Pattern.compile("https?://www.webtoons.com/[a-zA-Z]+/[a-zA-Z]+/([a-zA-Z0-9_-]*)/[a-zA-Z0-9_-]+/\\S*"); Matcher mat = pat.matcher(url.toExternalForm()); if (mat.matches()) { return true; @@ -48,7 +48,7 @@ public class WebtoonsRipper extends AbstractHTMLRipper { @Override public String getAlbumTitle(URL url) throws MalformedURLException { - Pattern pat = Pattern.compile("https?://www\\.webtoons\\.com/[a-zA-Z]+/[a-zA-Z]+/([a-zA-Z0-9_-]*)/ep-\\d+/\\S*"); + Pattern pat = Pattern.compile("https?://www.webtoons.com/[a-zA-Z]+/[a-zA-Z]+/([a-zA-Z0-9_-]*)/[a-zA-Z0-9_-]+/\\S*"); Matcher mat = pat.matcher(url.toExternalForm()); if (mat.matches()) { return mat.group(1); @@ -59,7 +59,7 @@ public class WebtoonsRipper extends AbstractHTMLRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern pat = Pattern.compile("https?://www\\.webtoons\\.com/[a-zA-Z]+/[a-zA-Z]+/([a-zA-Z0-9_-]*)/ep-\\d+/\\S*"); + Pattern pat = Pattern.compile("https?://www.webtoons.com/[a-zA-Z]+/[a-zA-Z]+/([a-zA-Z0-9_-]*)/[a-zA-Z0-9_-]+/\\S*"); Matcher mat = pat.matcher(url.toExternalForm()); if (mat.matches()) { return mat.group(1); @@ -95,7 +95,7 @@ public class WebtoonsRipper extends AbstractHTMLRipper { String nextUrl = ""; Element elem = doc.select("a.pg_next").first(); nextUrl = elem.attr("href"); - if (nextUrl == "") { + if (nextUrl.equals("") || nextUrl.equals("#")) { throw new IOException("No more pages"); } return Http.url(nextUrl).get();