From 2c0010f9dfd83910601fcfdefe0be9d7057ae3e2 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 17 Mar 2017 18:09:52 -0400 Subject: [PATCH 1/6] Changed hentaifoundry file name format --- .../ripme/ripper/rippers/HentaifoundryRipper.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java index d476969d..ac0489fc 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java @@ -64,7 +64,7 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { cookies.putAll(resp.cookies()); return resp.parse(); } - + @Override public Document getNextPage(Document doc) throws IOException { if (doc.select("li.next.hidden").size() > 0) { @@ -94,16 +94,18 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { logger.info("Couldn't find user & image ID in " + thumb.attr("href")); continue; } + String[] titleSplit = thumb.attr("herf").split("/"); + String title = titleSplit[titleSplit.length -1]; String user = imgMatcher.group(1), imageId = imgMatcher.group(2); String image = "http://pictures.hentai-foundry.com//"; image += user.toLowerCase().charAt(0); - image += "/" + user + "/" + imageId + ".jpg"; + image += "/" + user + "/" + imageId + "/" + user + "-" + imageId + title + ".png"; imageURLs.add(image); } return imageURLs; } - + @Override public void downloadURL(URL url, int index) { addURLToDownload(url, getPrefix(index)); From 6d352fcdc5fc0786c1abb00da1842a3fbbd01fea Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 23 Mar 2017 17:22:36 -0400 Subject: [PATCH 2/6] Fixed typo --- .../com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java index ac0489fc..fd0dfe3a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java @@ -94,7 +94,7 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { logger.info("Couldn't find user & image ID in " + thumb.attr("href")); continue; } - String[] titleSplit = thumb.attr("herf").split("/"); + String[] titleSplit = thumb.attr("href").split("/"); String title = titleSplit[titleSplit.length -1]; String user = imgMatcher.group(1), imageId = imgMatcher.group(2); From b8b706de90348dc6e965cf436b6141d70a319cbb Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 28 Mar 2017 15:17:51 -0400 Subject: [PATCH 3/6] Fix null error, Now rips all file types --- .../ripper/rippers/HentaifoundryRipper.java | 37 ++++++++++++++----- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java index fd0dfe3a..2601a7cf 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java @@ -67,7 +67,7 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { @Override public Document getNextPage(Document doc) throws IOException { - if (doc.select("li.next.hidden").size() > 0) { + if (doc.select("li.next.hidden").size() != 0) { // Last page throw new IOException("No more pages"); } @@ -94,14 +94,33 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { logger.info("Couldn't find user & image ID in " + thumb.attr("href")); continue; } - String[] titleSplit = thumb.attr("href").split("/"); - String title = titleSplit[titleSplit.length -1]; - String user = imgMatcher.group(1), - imageId = imgMatcher.group(2); - String image = "http://pictures.hentai-foundry.com//"; - image += user.toLowerCase().charAt(0); - image += "/" + user + "/" + imageId + "/" + user + "-" + imageId + title + ".png"; - imageURLs.add(image); + Document imagePage; + try { + Response resp = Http.url("http://www.hentai-foundry.com/").response(); + cookies = resp.cookies(); + resp = Http.url("http://www.hentai-foundry.com/?enterAgree=1&size=1500") + .referrer("http://www.hentai-foundry.com/") + .cookies(cookies) + .response(); + cookies.putAll(resp.cookies()); + + logger.info("grabbing " + "http://www.hentai-foundry.com" + thumb.attr("href")); + imagePage = Http.url("http://www.hentai-foundry.com" + thumb.attr("href")).cookies(cookies).get(); + } + + catch (IOException e) { + logger.debug(e.getMessage()); + logger.debug("Warning: imagePage is null!"); + imagePage = null; + } + // String[] titleSplit = thumb.attr("href").split("/"); + // String title = titleSplit[titleSplit.length -1]; + // String user = imgMatcher.group(1), + // imageId = imgMatcher.group(2); + // String image = "http://pictures.hentai-foundry.com//"; + // image += user.toLowerCase().charAt(0); + // image += "/" + user + "/" + imageId + "/" + user + "-" + imageId + title + ".png"; + imageURLs.add("http:" + imagePage.select("div.boxbody > center > img").attr("src")); } return imageURLs; } From ac7cad038b50d618800003e76a97a9519ac28e7e Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 25 Apr 2017 14:02:18 -0400 Subject: [PATCH 4/6] Updated hentaifoundry ripper for new site css --- .../ripme/ripper/rippers/HentaifoundryRipper.java | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java index 2601a7cf..b84d551e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java @@ -55,6 +55,7 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { .referrer("http://www.hentai-foundry.com/") .cookies(cookies) .response(); + // The only cookie that seems to matter in getting around the age wall is the phpsession cookie cookies.putAll(resp.cookies()); sleep(500); resp = Http.url(url) @@ -67,7 +68,7 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { @Override public Document getNextPage(Document doc) throws IOException { - if (doc.select("li.next.hidden").size() != 0) { + if (doc.select("li.next").size() != 0) { // Last page throw new IOException("No more pages"); } @@ -85,7 +86,7 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { public List getURLsFromPage(Document doc) { List imageURLs = new ArrayList(); Pattern imgRegex = Pattern.compile(".*/user/([a-zA-Z0-9\\-_]+)/(\\d+)/.*"); - for (Element thumb : doc.select("td > a:first-child")) { + for (Element thumb : doc.select("div.thumb_square > a.thumbLink")) { if (isStopped()) { break; } @@ -113,14 +114,7 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { logger.debug("Warning: imagePage is null!"); imagePage = null; } - // String[] titleSplit = thumb.attr("href").split("/"); - // String title = titleSplit[titleSplit.length -1]; - // String user = imgMatcher.group(1), - // imageId = imgMatcher.group(2); - // String image = "http://pictures.hentai-foundry.com//"; - // image += user.toLowerCase().charAt(0); - // image += "/" + user + "/" + imageId + "/" + user + "-" + imageId + title + ".png"; - imageURLs.add("http:" + imagePage.select("div.boxbody > center > img").attr("src")); + imageURLs.add("http:" + imagePage.select("div.boxbody img.center").attr("src")); } return imageURLs; } From d733e9f6dd1e33081a4ee996f35882a42c33f71f Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 25 Apr 2017 14:39:05 -0400 Subject: [PATCH 5/6] now rips images of all sizes --- .../ripper/rippers/HentaifoundryRipper.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java index b84d551e..6127ce29 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java @@ -76,6 +76,14 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { Element first = els.first(); String nextURL = first.attr("href"); nextURL = "http://www.hentai-foundry.com" + nextURL; + Response resp = Http.url("http://www.hentai-foundry.com/").response(); + cookies = resp.cookies(); + resp = Http.url("http://www.hentai-foundry.com/?enterAgree=1&size=1500") + .referrer("http://www.hentai-foundry.com/") + .cookies(cookies) + .response(); + // The only cookie that seems to matter in getting around the age wall is the phpsession cookie + cookies.putAll(resp.cookies()); return Http.url(nextURL) .referrer(url) .cookies(cookies) @@ -114,7 +122,13 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { logger.debug("Warning: imagePage is null!"); imagePage = null; } - imageURLs.add("http:" + imagePage.select("div.boxbody img.center").attr("src")); + // This is here for when the image is resized to a thumbnail because ripme doesn't report a screensize + if (imagePage.select("div.boxbody > img.center").attr("src").contains("thumbs.") == true) { + imageURLs.add("http:" + imagePage.select("div.boxbody > img.center").attr("onclick").replace("this.src=", "").replace("'", "").replace("; $(#resize_message).hide();", "")); + } + else { + imageURLs.add("http:" + imagePage.select("div.boxbody > img.center").attr("src")); + } } return imageURLs; } From a5e30634ad3095bcf974e1309ca05e359d29ea72 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 25 Apr 2017 14:42:57 -0400 Subject: [PATCH 6/6] Now rips all pages --- .../ripme/ripper/rippers/HentaifoundryRipper.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java index 6127ce29..ca4e906e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java @@ -68,7 +68,7 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { @Override public Document getNextPage(Document doc) throws IOException { - if (doc.select("li.next").size() != 0) { + if (doc.select("li.next.hidden").size() != 0) { // Last page throw new IOException("No more pages"); } @@ -76,14 +76,6 @@ public class HentaifoundryRipper extends AbstractHTMLRipper { Element first = els.first(); String nextURL = first.attr("href"); nextURL = "http://www.hentai-foundry.com" + nextURL; - Response resp = Http.url("http://www.hentai-foundry.com/").response(); - cookies = resp.cookies(); - resp = Http.url("http://www.hentai-foundry.com/?enterAgree=1&size=1500") - .referrer("http://www.hentai-foundry.com/") - .cookies(cookies) - .response(); - // The only cookie that seems to matter in getting around the age wall is the phpsession cookie - cookies.putAll(resp.cookies()); return Http.url(nextURL) .referrer(url) .cookies(cookies)