Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

DeviantartRipper can now rip nsfw images

Browse Source
This commit is contained in:
cyian-1756 2018-07-01 16:42:11 -04:00
parent ae6d09c6e8
commit 0ed2a7ddac
1 changed files with 34 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
View File

@ -54,26 +54,23 @@ public class DeviantartRipper extends AbstractJSONRipper {
public String getHost() { public String getHost() {
return "deviantart"; return "deviantart";
} }
@Override @Override
public String getDomain() { public String getDomain() {
return "deviantart.com"; return "deviantart.com";
} }
// @Override
// public boolean hasDescriptionSupport() {
// return true;
// }
@Override @Override
public URL sanitizeURL(URL url) throws MalformedURLException { public URL sanitizeURL(URL url) throws MalformedURLException {
String u = url.toExternalForm(); String u = url.toExternalForm();
if (u.replace("/", "").endsWith(".deviantart.com")) { if (!u.endsWith("/gallery/")) {
// Root user page, get all albums if (!u.endsWith("/gallery")) {
if (!u.endsWith("/")) { u += "gallery/";
u += "/";
} }
u += "gallery/?";
} }
Pattern p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/favou?rites/([0-9]+)/*?$"); Pattern p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/favou?rites/([0-9]+)/*?$");
Matcher m = p.matcher(url.toExternalForm()); Matcher m = p.matcher(url.toExternalForm());
if (!m.matches()) { if (!m.matches()) {
@ -119,6 +116,25 @@ public class DeviantartRipper extends AbstractJSONRipper {
throw new MalformedURLException("Expected URL format: http://www.deviantart.com/username[/gallery/#####], got: " + url); throw new MalformedURLException("Expected URL format: http://www.deviantart.com/username[/gallery/#####], got: " + url);
} }
private String getFullsizedNSFWImage(String pageURL) {
try {
Document doc = Http.url(pageURL).cookies(cookies).get();
String imageToReturn = "";
String[] d = doc.select("img").attr("srcset").split(",");
String s = d[d.length -1].split(" ")[0];
LOGGER.info("2:" + s);
if (s == null || s.equals("")) {
LOGGER.error("Could not find full sized image at " + pageURL);
}
return s;
} catch (IOException e) {
LOGGER.error("Could not find full sized image at " + pageURL);
return null;
}
}
/** /**
* Gets first page. * Gets first page.
* Will determine if login is supplied, * Will determine if login is supplied,
@ -178,7 +194,6 @@ public class DeviantartRipper extends AbstractJSONRipper {
private JSONObject getFirstPageJSON(Document doc) { private JSONObject getFirstPageJSON(Document doc) {
for (Element js : doc.select("script")) { for (Element js : doc.select("script")) {
LOGGER.info(js.html());
if (js.html().contains("requestid")) { if (js.html().contains("requestid")) {
String json = js.html().replaceAll("window.__initial_body_data=", "").replaceAll("\\);", "") String json = js.html().replaceAll("window.__initial_body_data=", "").replaceAll("\\);", "")
.replaceAll(";__wake\\(.+", ""); .replaceAll(";__wake\\(.+", "");
@ -204,7 +219,8 @@ public class DeviantartRipper extends AbstractJSONRipper {
} }
public String getUsername(Document doc) { public String getUsername(Document doc) {
return doc.select("meta[property=og:title]").attr("content").replaceAll("'s DeviantArt gallery", ""); return doc.select("meta[property=og:title]").attr("content")
.replaceAll("'s DeviantArt gallery", "").replaceAll("'s DeviantArt Gallery", "");
} }
@ -215,6 +231,13 @@ public class DeviantartRipper extends AbstractJSONRipper {
JSONArray results = json.getJSONObject("content").getJSONArray("results"); JSONArray results = json.getJSONObject("content").getJSONArray("results");
for (int i = 0; i < results.length(); i++) { for (int i = 0; i < results.length(); i++) {
Document doc = Jsoup.parseBodyFragment(results.getJSONObject(i).getString("html")); Document doc = Jsoup.parseBodyFragment(results.getJSONObject(i).getString("html"));
if (doc.html().contains("ismature")) {
LOGGER.info("Downloading nsfw image");
String nsfwImage = getFullsizedNSFWImage(doc.select("span").attr("href"));
if (nsfwImage != null) {
imageURLs.add(nsfwImage);
}
}
try { try {
String imageURL = doc.select("span").first().attr("data-super-full-img"); String imageURL = doc.select("span").first().attr("data-super-full-img");
if (!imageURL.isEmpty()) { if (!imageURL.isEmpty()) {