From e88392b8b09467f3f8e92f909222df04fc383e11 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 10 Mar 2018 22:53:27 -0500 Subject: [PATCH] Added sta ripper --- .../ripme/ripper/rippers/StaRipper.java | 112 ++++++++++++++++++ .../tst/ripper/rippers/StaRipperTest.java | 13 ++ 2 files changed, 125 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/StaRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java new file mode 100644 index 00000000..4cfaf485 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java @@ -0,0 +1,112 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.Connection; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class StaRipper extends AbstractHTMLRipper { + + public StaRipper(URL url) throws IOException { + super(url); + } + + private Map cookies = new HashMap<>(); + + @Override + public String getHost() { + return "sta"; + } + + @Override + public String getDomain() { + return "sta.sh"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https://sta.sh/([A-Za-z0-9]+)"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected sta.sh URL format: " + + "sta.sh/ALBUMID - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + for (Element el : doc.select("span > span > a.thumb")) { + String thumbPageURL = el.attr("href"); + Document thumbPage = null; + if (checkURL(thumbPageURL)) { + try { + Connection.Response resp = Http.url(new URL(thumbPageURL)).response(); + cookies.putAll(resp.cookies()); + thumbPage = resp.parse(); + } catch (MalformedURLException e) { + logger.info(thumbPageURL + " is a malformed URL"); + } catch (IOException e) { + logger.info(e.getMessage()); + } + String imageDownloadUrl = thumbPage.select("a.dev-page-download").attr("href"); + if (imageDownloadUrl != null && !imageDownloadUrl.equals("")) { + result.add(getImageLinkFromDLLink(imageDownloadUrl)); + } + } + + } + return result; + } + + private boolean checkURL(String url) { + try { + new URL(url); + return true; + } catch (MalformedURLException e) { + return false; + } + } + + private String getImageLinkFromDLLink(String url) { + try { + Connection.Response response = Jsoup.connect(url) + .userAgent(USER_AGENT) + .timeout(10000) + .cookies(cookies) + .followRedirects(false) + .execute(); + String imageURL = response.header("Location"); + logger.info(imageURL); + return imageURL; + } catch (IOException e) { + logger.info("Got error message " + e.getMessage() + " trying to download " + url); + return null; + } + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StaRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StaRipperTest.java new file mode 100644 index 00000000..128f3f17 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StaRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.StaRipper; + +public class StaRipperTest extends RippersTest { + public void testRip() throws IOException { + StaRipper ripper = new StaRipper(new URL("https://sta.sh/2hn9rtavr1g")); + testRipper(ripper); + } +} \ No newline at end of file