From e88392b8b09467f3f8e92f909222df04fc383e11 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 10 Mar 2018 22:53:27 -0500 Subject: [PATCH 1/8] Added sta ripper --- .../ripme/ripper/rippers/StaRipper.java | 112 ++++++++++++++++++ .../tst/ripper/rippers/StaRipperTest.java | 13 ++ 2 files changed, 125 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/StaRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java new file mode 100644 index 00000000..4cfaf485 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java @@ -0,0 +1,112 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.Connection; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class StaRipper extends AbstractHTMLRipper { + + public StaRipper(URL url) throws IOException { + super(url); + } + + private Map cookies = new HashMap<>(); + + @Override + public String getHost() { + return "sta"; + } + + @Override + public String getDomain() { + return "sta.sh"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https://sta.sh/([A-Za-z0-9]+)"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected sta.sh URL format: " + + "sta.sh/ALBUMID - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + for (Element el : doc.select("span > span > a.thumb")) { + String thumbPageURL = el.attr("href"); + Document thumbPage = null; + if (checkURL(thumbPageURL)) { + try { + Connection.Response resp = Http.url(new URL(thumbPageURL)).response(); + cookies.putAll(resp.cookies()); + thumbPage = resp.parse(); + } catch (MalformedURLException e) { + logger.info(thumbPageURL + " is a malformed URL"); + } catch (IOException e) { + logger.info(e.getMessage()); + } + String imageDownloadUrl = thumbPage.select("a.dev-page-download").attr("href"); + if (imageDownloadUrl != null && !imageDownloadUrl.equals("")) { + result.add(getImageLinkFromDLLink(imageDownloadUrl)); + } + } + + } + return result; + } + + private boolean checkURL(String url) { + try { + new URL(url); + return true; + } catch (MalformedURLException e) { + return false; + } + } + + private String getImageLinkFromDLLink(String url) { + try { + Connection.Response response = Jsoup.connect(url) + .userAgent(USER_AGENT) + .timeout(10000) + .cookies(cookies) + .followRedirects(false) + .execute(); + String imageURL = response.header("Location"); + logger.info(imageURL); + return imageURL; + } catch (IOException e) { + logger.info("Got error message " + e.getMessage() + " trying to download " + url); + return null; + } + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StaRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StaRipperTest.java new file mode 100644 index 00000000..128f3f17 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StaRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.StaRipper; + +public class StaRipperTest extends RippersTest { + public void testRip() throws IOException { + StaRipper ripper = new StaRipper(new URL("https://sta.sh/2hn9rtavr1g")); + testRipper(ripper); + } +} \ No newline at end of file From 6245680fefdfbd13f63b838b6023a93e64bdeb1d Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 11 Mar 2018 22:35:27 -0400 Subject: [PATCH 2/8] Added ripper for femjoyhunter.com --- .../ripper/rippers/FemjoyhunterRipper.java | 64 +++++++++++++++++++ .../tst/ripper/rippers/FemJoyRipperTest.java | 4 ++ .../rippers/FemJoyhunterRipperTest.java | 13 ++++ 3 files changed, 81 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/FemjoyhunterRipper.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemJoyRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemJoyhunterRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FemjoyhunterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FemjoyhunterRipper.java new file mode 100644 index 00000000..522a9720 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FemjoyhunterRipper.java @@ -0,0 +1,64 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class FemjoyhunterRipper extends AbstractHTMLRipper { + + public FemjoyhunterRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "femjoyhunter"; + } + + @Override + public String getDomain() { + return "femjoyhunter.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https?://www.femjoyhunter.com/([a-zA-Z0-9_-]+)/?"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected femjoyhunter URL format: " + + "femjoyhunter.com/ID - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + for (Element el : doc.select("ul.gallery-b > li > a")) { + result.add(el.attr("href")); + } + return result; + } + + @Override + public void downloadURL(URL url, int index) { + + addURLToDownload(url, getPrefix(index), "", "https://a2h6m3w6.ssl.hwcdn.net/", null); + } +} \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemJoyRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemJoyRipperTest.java new file mode 100644 index 00000000..85a26d07 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemJoyRipperTest.java @@ -0,0 +1,4 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +public class FemJoyRipperTest { +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemJoyhunterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemJoyhunterRipperTest.java new file mode 100644 index 00000000..2d7dbdc8 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemJoyhunterRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.FemjoyhunterRipper; + +public class FemjoyRipperTest extends RippersTest { + public void testRip() throws IOException { + FemjoyhunterRipper ripper = new FemjoyhunterRipper(new URL("https://www.femjoyhunter.com/alisa-i-got-nice-big-breasts-and-fine-ass-so-she-seems-to-be-a-hottest-brunette-5936/")); + testRipper(ripper); + } +} \ No newline at end of file From cdc70f50cc8b7c549b18cbba9b445ab49aedc321 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 11 Mar 2018 22:35:52 -0400 Subject: [PATCH 3/8] Removed wrongly named file --- .../rarchives/ripme/tst/ripper/rippers/FemJoyRipperTest.java | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemJoyRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemJoyRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemJoyRipperTest.java deleted file mode 100644 index 85a26d07..00000000 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemJoyRipperTest.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -public class FemJoyRipperTest { -} From 1555a6c276eeed0ca0c2c58f382c84656f76f7a5 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 11 Mar 2018 22:42:07 -0400 Subject: [PATCH 4/8] Fixed test --- ...{FemJoyhunterRipperTest.java => FemjoyhunterRipperTest.java} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/test/java/com/rarchives/ripme/tst/ripper/rippers/{FemJoyhunterRipperTest.java => FemjoyhunterRipperTest.java} (88%) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemJoyhunterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemjoyhunterRipperTest.java similarity index 88% rename from src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemJoyhunterRipperTest.java rename to src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemjoyhunterRipperTest.java index 2d7dbdc8..fee634ef 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemJoyhunterRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemjoyhunterRipperTest.java @@ -5,7 +5,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.FemjoyhunterRipper; -public class FemjoyRipperTest extends RippersTest { +public class FemjoyhunterRipperTest extends RippersTest { public void testRip() throws IOException { FemjoyhunterRipper ripper = new FemjoyhunterRipper(new URL("https://www.femjoyhunter.com/alisa-i-got-nice-big-breasts-and-fine-ass-so-she-seems-to-be-a-hottest-brunette-5936/")); testRipper(ripper); From 28ce42a54f2ad36de99fe825f0e8b4f0203d9076 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 11 Mar 2018 23:03:24 -0400 Subject: [PATCH 5/8] Added sinfest ripper --- .../ripme/ripper/rippers/SinfestRipper.java | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java new file mode 100644 index 00000000..d30e9b63 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java @@ -0,0 +1,80 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class SinfestRipper extends AbstractHTMLRipper { + + public SinfestRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "sinfest"; + } + + @Override + public String getDomain() { + return "sinfest.net"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https?://sinfest.net/view.php\\?date=([0-9-]*)/?"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected sinfest URL format: " + + "sinfest.net/view.php?date=XXXX-XX-XX/ - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + @Override + public Document getNextPage(Document doc) throws IOException { + Element elem = doc.select("td.style5 > a > img").last(); + logger.info(elem.parent().attr("href")); + if (elem == null || elem.parent().attr("href").equals("view.php?date=")) { + throw new IOException("No more pages"); + } + String nextPage = elem.parent().attr("href"); + // Some times this returns a empty string + // This for stops that + if (nextPage == "") { + return null; + } + else { + return Http.url("http://sinfest.net/" + nextPage).get(); + } + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + Element elem = doc.select("tbody > tr > td > img").last(); + result.add("http://sinfest.net/" + elem.attr("src")); + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} From 8a145d1ff458b13c007db5c14ca46b5c12d69111 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 11 Mar 2018 23:05:23 -0400 Subject: [PATCH 6/8] Added sinfest ripper test --- .../ripme/tst/ripper/rippers/SinfestRipperTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinfestRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinfestRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinfestRipperTest.java new file mode 100644 index 00000000..c46e922c --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinfestRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.SinfestRipper; + +public class SinfestRipperTest extends RippersTest { + public void testRip() throws IOException { + SinfestRipper ripper = new SinfestRipper(new URL("http://sinfest.net/view.php?date=2000-01-17")); + testRipper(ripper); + } +} \ No newline at end of file From f51d9c03e933674e93094f709290b6e3c2edfc70 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Mon, 12 Mar 2018 11:35:26 -0400 Subject: [PATCH 7/8] Disabled flaky unit test --- .../tst/ripper/rippers/WordpressComicRipperTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java index e3d5c4a0..2f7dbcf9 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java @@ -83,10 +83,10 @@ public class WordpressComicRipperTest extends RippersTest { new URL("http://tnbtu.com/comic/01-00/")); testRipper(ripper); } - - public void test_pepsaga() throws IOException { - WordpressComicRipper ripper = new WordpressComicRipper( - new URL("http://shipinbottle.pepsaga.com/?p=281")); - testRipper(ripper); - } + // https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI +// public void test_pepsaga() throws IOException { +// WordpressComicRipper ripper = new WordpressComicRipper( +// new URL("http://shipinbottle.pepsaga.com/?p=281")); +// testRipper(ripper); +// } } From c1f46d82516cda02cb086841d7f0dea14a402c4c Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Mon, 12 Mar 2018 11:50:03 -0400 Subject: [PATCH 8/8] 1.7.24: Added sta.sh ripper; Added sinfest.com ripper; Added femjoyhunter.com ripper; Disabled flaky unit tests --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 547d26e9..7c7b831f 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.23 + 1.7.24 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index bc2acca8..3fc7fde5 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.23", + "latestVersion": "1.7.24", "changeList": [ + "1.7.24: Added sta.sh ripper; Added sinfest.com ripper; Added femjoyhunter.com ripper; Disabled flaky unit tests", "1.7.23: Fixed xvideos ripper; InstagramRipper now works with lastseenfeature", "1.7.22: Added func to normalize urls before reading from/writing to url history file; last seen feature now works with instagram", "1.7.21: Fixed last seen feature", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 318eec4c..98191c33 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.23"; + private static final String DEFAULT_VERSION = "1.7.24"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar";