diff --git a/pom.xml b/pom.xml index 547d26e9..7c7b831f 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.23 + 1.7.24 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index bc2acca8..3fc7fde5 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.23", + "latestVersion": "1.7.24", "changeList": [ + "1.7.24: Added sta.sh ripper; Added sinfest.com ripper; Added femjoyhunter.com ripper; Disabled flaky unit tests", "1.7.23: Fixed xvideos ripper; InstagramRipper now works with lastseenfeature", "1.7.22: Added func to normalize urls before reading from/writing to url history file; last seen feature now works with instagram", "1.7.21: Fixed last seen feature", diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FemjoyhunterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FemjoyhunterRipper.java new file mode 100644 index 00000000..522a9720 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FemjoyhunterRipper.java @@ -0,0 +1,64 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class FemjoyhunterRipper extends AbstractHTMLRipper { + + public FemjoyhunterRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "femjoyhunter"; + } + + @Override + public String getDomain() { + return "femjoyhunter.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https?://www.femjoyhunter.com/([a-zA-Z0-9_-]+)/?"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected femjoyhunter URL format: " + + "femjoyhunter.com/ID - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + for (Element el : doc.select("ul.gallery-b > li > a")) { + result.add(el.attr("href")); + } + return result; + } + + @Override + public void downloadURL(URL url, int index) { + + addURLToDownload(url, getPrefix(index), "", "https://a2h6m3w6.ssl.hwcdn.net/", null); + } +} \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java new file mode 100644 index 00000000..d30e9b63 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java @@ -0,0 +1,80 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class SinfestRipper extends AbstractHTMLRipper { + + public SinfestRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "sinfest"; + } + + @Override + public String getDomain() { + return "sinfest.net"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https?://sinfest.net/view.php\\?date=([0-9-]*)/?"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected sinfest URL format: " + + "sinfest.net/view.php?date=XXXX-XX-XX/ - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + @Override + public Document getNextPage(Document doc) throws IOException { + Element elem = doc.select("td.style5 > a > img").last(); + logger.info(elem.parent().attr("href")); + if (elem == null || elem.parent().attr("href").equals("view.php?date=")) { + throw new IOException("No more pages"); + } + String nextPage = elem.parent().attr("href"); + // Some times this returns a empty string + // This for stops that + if (nextPage == "") { + return null; + } + else { + return Http.url("http://sinfest.net/" + nextPage).get(); + } + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + Element elem = doc.select("tbody > tr > td > img").last(); + result.add("http://sinfest.net/" + elem.attr("src")); + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java new file mode 100644 index 00000000..4cfaf485 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java @@ -0,0 +1,112 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.Connection; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class StaRipper extends AbstractHTMLRipper { + + public StaRipper(URL url) throws IOException { + super(url); + } + + private Map cookies = new HashMap<>(); + + @Override + public String getHost() { + return "sta"; + } + + @Override + public String getDomain() { + return "sta.sh"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https://sta.sh/([A-Za-z0-9]+)"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected sta.sh URL format: " + + "sta.sh/ALBUMID - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + for (Element el : doc.select("span > span > a.thumb")) { + String thumbPageURL = el.attr("href"); + Document thumbPage = null; + if (checkURL(thumbPageURL)) { + try { + Connection.Response resp = Http.url(new URL(thumbPageURL)).response(); + cookies.putAll(resp.cookies()); + thumbPage = resp.parse(); + } catch (MalformedURLException e) { + logger.info(thumbPageURL + " is a malformed URL"); + } catch (IOException e) { + logger.info(e.getMessage()); + } + String imageDownloadUrl = thumbPage.select("a.dev-page-download").attr("href"); + if (imageDownloadUrl != null && !imageDownloadUrl.equals("")) { + result.add(getImageLinkFromDLLink(imageDownloadUrl)); + } + } + + } + return result; + } + + private boolean checkURL(String url) { + try { + new URL(url); + return true; + } catch (MalformedURLException e) { + return false; + } + } + + private String getImageLinkFromDLLink(String url) { + try { + Connection.Response response = Jsoup.connect(url) + .userAgent(USER_AGENT) + .timeout(10000) + .cookies(cookies) + .followRedirects(false) + .execute(); + String imageURL = response.header("Location"); + logger.info(imageURL); + return imageURL; + } catch (IOException e) { + logger.info("Got error message " + e.getMessage() + " trying to download " + url); + return null; + } + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 318eec4c..98191c33 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.23"; + private static final String DEFAULT_VERSION = "1.7.24"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemjoyhunterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemjoyhunterRipperTest.java new file mode 100644 index 00000000..fee634ef --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemjoyhunterRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.FemjoyhunterRipper; + +public class FemjoyhunterRipperTest extends RippersTest { + public void testRip() throws IOException { + FemjoyhunterRipper ripper = new FemjoyhunterRipper(new URL("https://www.femjoyhunter.com/alisa-i-got-nice-big-breasts-and-fine-ass-so-she-seems-to-be-a-hottest-brunette-5936/")); + testRipper(ripper); + } +} \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinfestRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinfestRipperTest.java new file mode 100644 index 00000000..c46e922c --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinfestRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.SinfestRipper; + +public class SinfestRipperTest extends RippersTest { + public void testRip() throws IOException { + SinfestRipper ripper = new SinfestRipper(new URL("http://sinfest.net/view.php?date=2000-01-17")); + testRipper(ripper); + } +} \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StaRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StaRipperTest.java new file mode 100644 index 00000000..128f3f17 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StaRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.StaRipper; + +public class StaRipperTest extends RippersTest { + public void testRip() throws IOException { + StaRipper ripper = new StaRipper(new URL("https://sta.sh/2hn9rtavr1g")); + testRipper(ripper); + } +} \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java index e3d5c4a0..2f7dbcf9 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java @@ -83,10 +83,10 @@ public class WordpressComicRipperTest extends RippersTest { new URL("http://tnbtu.com/comic/01-00/")); testRipper(ripper); } - - public void test_pepsaga() throws IOException { - WordpressComicRipper ripper = new WordpressComicRipper( - new URL("http://shipinbottle.pepsaga.com/?p=281")); - testRipper(ripper); - } + // https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI +// public void test_pepsaga() throws IOException { +// WordpressComicRipper ripper = new WordpressComicRipper( +// new URL("http://shipinbottle.pepsaga.com/?p=281")); +// testRipper(ripper); +// } }