From 66bba724f9a0ef461d8e71219a60753f825cc6a0 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 6 May 2018 06:27:41 -0400 Subject: [PATCH 1/2] Added user support to SmuttyRipper --- .../ripme/ripper/rippers/SmuttyRipper.java | 125 +++++++++--------- 1 file changed, 62 insertions(+), 63 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SmuttyRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SmuttyRipper.java index 51992ec4..b61f2fef 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SmuttyRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SmuttyRipper.java @@ -3,24 +3,19 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.util.ArrayList; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import com.rarchives.ripme.ripper.AlbumRipper; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; +import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; -/** - * Appears to be broken as of 2015-02-11. - * Generating large image from thumbnail requires replacing "/m/" with something else: - * -> Sometimes "/b/" - * -> Sometimes "/p/" - * No way to know without loading the image page. - */ -public class SmuttyRipper extends AlbumRipper { + +public class SmuttyRipper extends AbstractHTMLRipper { private static final String DOMAIN = "smutty.com", HOST = "smutty"; @@ -29,6 +24,16 @@ public class SmuttyRipper extends AlbumRipper { super(url); } + @Override + public String getHost() { + return "smutty"; + } + + @Override + public String getDomain() { + return "smutty.com"; + } + @Override public boolean canRip(URL url) { return (url.getHost().endsWith(DOMAIN)); @@ -40,69 +45,57 @@ public class SmuttyRipper extends AlbumRipper { } @Override - public void rip() throws IOException { - int page = 0; - String url, tag = getGID(this.url); - boolean hasNextPage = true; - while (hasNextPage) { + public List getURLsFromPage(Document doc) { + List results = new ArrayList<>(); + for (Element image : doc.select("a.l > img")) { if (isStopped()) { break; } - page++; - url = "http://smutty.com/h/" + tag + "/?q=%23" + tag + "&page=" + page + "&sort=date&lazy=1"; - this.sendUpdate(STATUS.LOADING_RESOURCE, url); - logger.info(" Retrieving " + url); - Document doc; - try { - doc = Http.url(url) - .ignoreContentType() - .get(); - } catch (IOException e) { - if (e.toString().contains("Status=404")) { - logger.info("No more pages to load"); - } else { - logger.warn("Exception while loading " + url, e); - } - break; - } - for (Element image : doc.select("a.l > img")) { - if (isStopped()) { - break; - } - String imageUrl = image.attr("src"); + String imageUrl = image.attr("src"); - // Construct direct link to image based on thumbnail - StringBuilder sb = new StringBuilder(); - String[] fields = imageUrl.split("/"); - for (int i = 0; i < fields.length; i++) { - if (i == fields.length - 2 && fields[i].equals("m")) { - fields[i] = "b"; - } - sb.append(fields[i]); - if (i < fields.length - 1) { - sb.append("/"); - } + // Construct direct link to image based on thumbnail + StringBuilder sb = new StringBuilder(); + String[] fields = imageUrl.split("/"); + for (int i = 0; i < fields.length; i++) { + if (i == fields.length - 2 && fields[i].equals("m")) { + fields[i] = "b"; + } + sb.append(fields[i]); + if (i < fields.length - 1) { + sb.append("/"); } - imageUrl = sb.toString(); - addURLToDownload(new URL("http:" + imageUrl)); - } - if (doc.select("#next").size() == 0) { - break; // No more pages - } - // Wait before loading next page - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - logger.error("[!] Interrupted while waiting to load next album:", e); - break; } + imageUrl = sb.toString(); + results.add("http:" + imageUrl); } - waitForThreads(); + return results; } @Override - public String getHost() { - return HOST; + public Document getNextPage(Document doc) throws IOException { + Element elem = doc.select("a.next").first(); + if (elem == null) { + throw new IOException("No more pages"); + } + String nextPage = elem.attr("href"); + // Some times this returns a empty string + // This for stops that + if (nextPage.equals("")) { + throw new IOException("No more pages"); + } + else { + return Http.url("https://smutty.com" + nextPage).get(); + } + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); } @Override @@ -117,6 +110,12 @@ public class SmuttyRipper extends AlbumRipper { if (m.matches()) { return m.group(1).replace("%23", ""); } + + p = Pattern.compile("^https?://smutty.com/user/([a-zA-Z0-9\\-_]+)/?$"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } throw new MalformedURLException("Expected tag in URL (smutty.com/h/tag and not " + url); } From c500f20c02e5ab11931c19e1bf8bdd89b85ad196 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 6 May 2018 06:30:17 -0400 Subject: [PATCH 2/2] Added unit test for SmuttyRipper --- .../ripme/tst/ripper/rippers/SmuttyRipperTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java new file mode 100644 index 00000000..949e715f --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.SmuttyRipper; + +public class SmuttyRipperTest extends RippersTest { + public void testRip() throws IOException { + SmuttyRipper ripper = new SmuttyRipper(new URL("https://smutty.com/user/QUIGON/")); + testRipper(ripper); + } +}