Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

8Muses ripper now supports ripping from pages containing both images and subalbums

Browse Source
This commit is contained in:
cyian-1756 2018-02-10 02:36:28 -05:00
parent c496678874
commit 9afa1a9316
1 changed files with 27 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java
View File

@ -88,47 +88,26 @@ public class EightmusesRipper extends AbstractHTMLRipper {
@Override @Override
public List<String> getURLsFromPage(Document page) { public List<String> getURLsFromPage(Document page) {
List<String> imageURLs = new ArrayList<>(); List<String> imageURLs = new ArrayList<>();
// get the first image link on the page and check if the last char in it is a number int x = 1;
// if it is a number then we're ripping a comic if not it's a subalbum // This contains the thumbnails of all images on the page
String firstImageLink = page.select("div.gallery > a.t-hover").first().attr("href"); Elements pageImages = page.getElementsByClass("c-tile");
Pattern p = Pattern.compile("/comix/picture/([a-zA-Z0-9\\-_/]*/)?\\d+"); for (Element thumb : pageImages) {
Matcher m = p.matcher(firstImageLink); // If true this link is a sub album
if (!m.matches()) { if (thumb.attr("href").contains("/comix/album/")) {
logger.info("Ripping subalbums"); String subUrl = "https://www.8muses.com" + thumb.attr("href");
// Page contains subalbums (not images) try {
Elements albumElements = page.select("div.gallery > a.t-hover"); logger.info("Retrieving " + subUrl);
List<Element> albumsList = albumElements.subList(0, albumElements.size()); sendUpdate(STATUS.LOADING_RESOURCE, subUrl);
Collections.reverse(albumsList); Document subPage = Http.url(subUrl).get();
// Iterate over elements in reverse order // If the page below this one has images this line will download them
for (Element subalbum : albumsList) { List<String> subalbumImages = getURLsFromPage(subPage);
String subUrl = subalbum.attr("href"); logger.info("Found " + subalbumImages.size() + " images in subalbum");
// This if is to skip ads which don't have a href } catch (IOException e) {
if (subUrl != "") { logger.warn("Error while loading subalbum " + subUrl, e);
subUrl = subUrl.replaceAll("\\.\\./", "");
if (subUrl.startsWith("//")) {
subUrl = "https:";
}
else if (!subUrl.startsWith("http://")) {
subUrl = "https://www.8muses.com" + subUrl;
}
try {
logger.info("Retrieving " + subUrl);
sendUpdate(STATUS.LOADING_RESOURCE, subUrl);
Document subPage = Http.url(subUrl).get();
// If the page below this one has images this line will download them
List<String> subalbumImages = getURLsFromPage(subPage);
logger.info("Found " + subalbumImages.size() + " images in subalbum");
} catch (IOException e) {
logger.warn("Error while loading subalbum " + subUrl, e);
}
} }
}
} } else if (thumb.attr("href").contains("/comix/picture/")) {
else { logger.info("Ripping image");
// Page contains images
// X is our page index
int x = 1;
for (Element thumb : page.select(".image")) {
if (super.isStopped()) break; if (super.isStopped()) break;
// Find thumbnail image source // Find thumbnail image source
String image = null; String image = null;
@ -136,21 +115,21 @@ public class EightmusesRipper extends AbstractHTMLRipper {
image = thumb.attr("data-cfsrc"); image = thumb.attr("data-cfsrc");
} }
else { else {
String parentHref = thumb.parent().attr("href"); String imageHref = thumb.attr("href");
if (parentHref.equals("")) continue; if (imageHref.equals("")) continue;
if (parentHref.startsWith("/")) { if (imageHref.startsWith("/")) {
parentHref = "https://www.8muses.com" + parentHref; imageHref = "https://www.8muses.com" + imageHref;
} }
try { try {
logger.info("Retrieving full-size image location from " + parentHref); logger.info("Retrieving full-size image location from " + imageHref);
image = getFullSizeImage(parentHref); image = getFullSizeImage(imageHref);
URL imageUrl = new URL(image); URL imageUrl = new URL(image);
addURLToDownload(imageUrl, getPrefix(x), getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies); addURLToDownload(imageUrl, getPrefix(x), getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies);
// X is our page index // X is our page index
x++; x++;
} catch (IOException e) { } catch (IOException e) {
logger.error("Failed to get full-size image from " + parentHref); logger.error("Failed to get full-size image from " + imageHref);
continue; continue;
} }
} }
@ -161,6 +140,7 @@ public class EightmusesRipper extends AbstractHTMLRipper {
imageURLs.add(image); imageURLs.add(image);
if (isThisATest()) break; if (isThisATest()) break;
} }
} }
return imageURLs; return imageURLs;
} }