Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #754 from cyian-1756/daNewProfileUrl

Browse Source 
Fixed  DeviantartRipper ripper
This commit is contained in:
cyian-1756 2018-06-30 18:21:40 -04:00 committed by GitHub
commit 3136f7bcdf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 155 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

324
src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
View File

@ -1,6 +1,6 @@
package com.rarchives.ripme.ripper.rippers; package com.rarchives.ripme.ripper.rippers;
import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ripper.AbstractJSONRipper;
import com.rarchives.ripme.utils.Base64; import com.rarchives.ripme.utils.Base64;
import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.RipUtils; import com.rarchives.ripme.utils.RipUtils;
@ -18,15 +18,23 @@ import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.jsoup.Connection.Method;
import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.Connection.Response; import org.jsoup.Connection.Response;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.jsoup.safety.Whitelist;
import org.jsoup.select.Elements; import org.jsoup.select.Elements;
public class DeviantartRipper extends AbstractHTMLRipper {
public class DeviantartRipper extends AbstractJSONRipper {
String requestID;
String galleryID;
String username;
String baseApiUrl = "https://www.deviantart.com/dapi/v1/gallery/";
String csrf;
Map<String, String> pageCookies = new HashMap<>();
private static final int PAGE_SLEEP_TIME = 3000, private static final int PAGE_SLEEP_TIME = 3000,
IMAGE_SLEEP_TIME = 2000; IMAGE_SLEEP_TIME = 2000;
@ -50,10 +58,10 @@ public class DeviantartRipper extends AbstractHTMLRipper {
public String getDomain() { public String getDomain() {
return "deviantart.com"; return "deviantart.com";
} }
@Override // @Override
public boolean hasDescriptionSupport() { // public boolean hasDescriptionSupport() {
return true; // return true;
} // }
@Override @Override
public URL sanitizeURL(URL url) throws MalformedURLException { public URL sanitizeURL(URL url) throws MalformedURLException {
String u = url.toExternalForm(); String u = url.toExternalForm();
@ -66,7 +74,7 @@ public class DeviantartRipper extends AbstractHTMLRipper {
u += "gallery/?"; u += "gallery/?";
} }
Pattern p = Pattern.compile("^https?://([a-zA-Z0-9\\-]+)\\.deviantart\\.com/favou?rites/([0-9]+)/*?$"); Pattern p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/favou?rites/([0-9]+)/*?$");
Matcher m = p.matcher(url.toExternalForm()); Matcher m = p.matcher(url.toExternalForm());
if (!m.matches()) { if (!m.matches()) {
String subdir = "/"; String subdir = "/";
@ -80,7 +88,7 @@ public class DeviantartRipper extends AbstractHTMLRipper {
@Override @Override
public String getGID(URL url) throws MalformedURLException { public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^https?://([a-zA-Z0-9\\-]+)\\.deviantart\\.com(/gallery)?/?(\\?.*)?$"); Pattern p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)(/gallery)?/?(\\?.*)?$");
Matcher m = p.matcher(url.toExternalForm()); Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) { if (m.matches()) {
// Root gallery // Root gallery
@ -91,24 +99,24 @@ public class DeviantartRipper extends AbstractHTMLRipper {
return m.group(1); return m.group(1);
} }
} }
p = Pattern.compile("^https?://([a-zA-Z0-9\\-]+)\\.deviantart\\.com/gallery/([0-9]+).*$"); p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/gallery/([0-9]+).*$");
m = p.matcher(url.toExternalForm()); m = p.matcher(url.toExternalForm());
if (m.matches()) { if (m.matches()) {
// Subgallery // Subgallery
return m.group(1) + "_" + m.group(2); return m.group(1) + "_" + m.group(2);
} }
p = Pattern.compile("^https?://([a-zA-Z0-9\\-]+)\\.deviantart\\.com/favou?rites/([0-9]+)/.*?$"); p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/favou?rites/([0-9]+)/.*?$");
m = p.matcher(url.toExternalForm()); m = p.matcher(url.toExternalForm());
if (m.matches()) { if (m.matches()) {
return m.group(1) + "_faves_" + m.group(2); return m.group(1) + "_faves_" + m.group(2);
} }
p = Pattern.compile("^https?://([a-zA-Z0-9\\-]+)\\.deviantart\\.com/favou?rites/?$"); p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/favou?rites/?$");
m = p.matcher(url.toExternalForm()); m = p.matcher(url.toExternalForm());
if (m.matches()) { if (m.matches()) {
// Subgallery // Subgallery
return m.group(1) + "_faves"; return m.group(1) + "_faves";
} }
throw new MalformedURLException("Expected URL format: http://username.deviantart.com/[/gallery/#####], got: " + url); throw new MalformedURLException("Expected URL format: http://www.deviantart.com/username[/gallery/#####], got: " + url);
} }
/** /**
@ -120,7 +128,7 @@ public class DeviantartRipper extends AbstractHTMLRipper {
* @throws IOException * @throws IOException
*/ */
@Override @Override
public Document getFirstPage() throws IOException { public JSONObject getFirstPage() throws IOException {
// Base64 da login // Base64 da login
// username: Z3JhYnB5 // username: Z3JhYnB5
@ -133,131 +141,103 @@ public class DeviantartRipper extends AbstractHTMLRipper {
cookies.put("agegate_state","1"); // Bypasses the age gate cookies.put("agegate_state","1"); // Bypasses the age gate
} }
return Http.url(this.url) Response res = Http.url(this.url)
.cookies(cookies) .cookies(cookies)
.get(); .response();
Document page = res.parse();
JSONObject firstPageJSON = getFirstPageJSON(page);
requestID = firstPageJSON.getJSONObject("dapx").getString("requestid");
galleryID = page.select("input[name=set]").attr("value");
username = page.select("div.tt-tv150").attr("username");
csrf = firstPageJSON.getString("csrf");
pageCookies = res.cookies();
return requestPage(0, galleryID, username, requestID, csrf, pageCookies);
} }
/** private JSONObject requestPage(int offset, String galleryID, String username, String requestID, String csfr, Map<String, String> c) {
* LOGGER.debug("offset: " + Integer.toString(offset));
* @param page LOGGER.debug("galleryID: " + galleryID);
* @param id LOGGER.debug("username: " + username);
* @return LOGGER.debug("requestID: " + requestID);
*/ String url = baseApiUrl + galleryID + "?iid=" + requestID;
private String jsonToImage(Document page, String id) { try {
Elements js = page.select("script[type=\"text/javascript\"]"); Document doc = Http.url(url).cookies(c).data("username", username).data("offset", Integer.toString(offset))
for (Element tag : js) { .data("limit", "24").data("_csrf", csfr).data("id", requestID)
if (tag.html().contains("window.__pageload")) { .ignoreContentType().post();
try { return new JSONObject(doc.body().text());
String script = tag.html(); } catch (IOException e) {
script = script.substring(script.indexOf("window.__pageload")); LOGGER.error("Got error trying to get page: " + e.getMessage());
if (!script.contains(id)) { e.printStackTrace();
continue; return null;
} }
script = script.substring(script.indexOf(id));
// first },"src":"url" after id
script = script.substring(script.indexOf("},\"src\":\"") + 9, script.indexOf("\",\"type\"")); }
return script.replace("\\/", "/");
} catch (StringIndexOutOfBoundsException e) { private JSONObject getFirstPageJSON(Document doc) {
LOGGER.debug("Unable to get json link from " + page.location()); for (Element js : doc.select("script")) {
} LOGGER.info(js.html());
if (js.html().contains("requestid")) {
String json = js.html().replaceAll("window.__initial_body_data=", "").replaceAll("\\);", "")
.replaceAll(";__wake\\(.+", "");
LOGGER.info("json: " + json);
JSONObject j = new JSONObject(json);
return j;
} }
} }
return null; return null;
} }
@Override @Override
public List<String> getURLsFromPage(Document page) { public List<String> getURLsFromJSON(JSONObject json) {
List<String> imageURLs = new ArrayList<>(); List<String> imageURLs = new ArrayList<>();
LOGGER.info(json);
JSONArray results = json.getJSONObject("content").getJSONArray("results");
for (int i = 0; i < results.length(); i++) {
LOGGER.info(results.getJSONObject(i).toString());
Document doc = Jsoup.parseBodyFragment(results.getJSONObject(i).getString("html"));
try {
String imageURL = doc.select("span").first().attr("data-super-full-img");
if (!imageURL.isEmpty()) {
imageURLs.add(imageURL);
}
} catch (NullPointerException e) {
LOGGER.info(i + " does not contain any images");
}
// Iterate over all thumbnails
for (Element thumb : page.select("div.zones-container span.thumb")) {
if (isStopped()) {
break;
}
Element img = thumb.select("img").get(0);
if (img.attr("transparent").equals("false")) {
continue; // a.thumbs to other albums are invisible
}
// Get full-sized image via helper methods
String fullSize = null;
if (thumb.attr("data-super-full-img").contains("//orig")) {
fullSize = thumb.attr("data-super-full-img");
} else {
String spanUrl = thumb.attr("href");
String fullSize1 = jsonToImage(page,spanUrl.substring(spanUrl.lastIndexOf('-') + 1));
if (fullSize1 == null || !fullSize1.contains("//orig")) {
fullSize = smallToFull(img.attr("src"), spanUrl);
}
if (fullSize == null && fullSize1 != null) {
fullSize = fullSize1;
}
}
if (fullSize == null) {
if (thumb.attr("data-super-full-img") != null) {
fullSize = thumb.attr("data-super-full-img");
} else if (thumb.attr("data-super-img") != null) {
fullSize = thumb.attr("data-super-img");
} else {
continue;
}
}
if (triedURLs.contains(fullSize)) {
LOGGER.warn("Already tried to download " + fullSize);
continue;
}
triedURLs.add(fullSize);
imageURLs.add(fullSize);
if (isThisATest()) {
// Only need one image for a test
break;
}
} }
return imageURLs; return imageURLs;
} }
@Override // @Override
public List<String> getDescriptionsFromPage(Document page) { // public List<String> getDescriptionsFromPage(Document page) {
List<String> textURLs = new ArrayList<>(); // List<String> textURLs = new ArrayList<>();
// Iterate over all thumbnails // // Iterate over all thumbnails
for (Element thumb : page.select("div.zones-container span.thumb")) { // for (Element thumb : page.select("div.zones-container span.thumb")) {
LOGGER.info(thumb.attr("href")); // LOGGER.info(thumb.attr("href"));
if (isStopped()) { // if (isStopped()) {
break; // break;
} // }
Element img = thumb.select("img").get(0); // Element img = thumb.select("img").get(0);
if (img.attr("transparent").equals("false")) { // if (img.attr("transparent").equals("false")) {
continue; // a.thumbs to other albums are invisible // continue; // a.thumbs to other albums are invisible
} // }
textURLs.add(thumb.attr("href")); // textURLs.add(thumb.attr("href"));
//
// }
// return textURLs;
// }
}
return textURLs;
}
@Override @Override
public Document getNextPage(Document page) throws IOException { public JSONObject getNextPage(JSONObject page) throws IOException {
if (isThisATest()) { boolean hasMore = page.getJSONObject("content").getBoolean("has_more");
return null; if (hasMore) {
return requestPage(page.getJSONObject("content").getInt("next_offset"), galleryID, username, requestID, csrf, pageCookies);
} }
Elements nextButtons = page.select("link[rel=\"next\"]");
if (nextButtons.isEmpty()) { throw new IOException("No more pages");
if (page.select("link[rel=\"prev\"]").isEmpty()) {
throw new IOException("No next page found");
} else {
throw new IOException("Hit end of pages");
}
}
Element a = nextButtons.first();
String nextPage = a.attr("href");
if (nextPage.startsWith("/")) {
nextPage = "http://" + this.url.getHost() + nextPage;
}
if (!sleep(PAGE_SLEEP_TIME)) {
throw new IOException("Interrupted while waiting to load next page: " + nextPage);
}
LOGGER.info("Found next page: " + nextPage);
return Http.url(nextPage)
.cookies(cookies)
.get();
} }
@Override @Override
@ -306,53 +286,53 @@ public class DeviantartRipper extends AbstractHTMLRipper {
* @param page The gallery page the URL was found on * @param page The gallery page the URL was found on
* @return A String[] with first object being the description, and the second object being image file name if found. * @return A String[] with first object being the description, and the second object being image file name if found.
*/ */
@Override // @Override
public String[] getDescription(String url,Document page) { // public String[] getDescription(String url,Document page) {
if (isThisATest()) { // if (isThisATest()) {
return null; // return null;
} // }
try { // try {
// Fetch the image page // // Fetch the image page
Response resp = Http.url(url) // Response resp = Http.url(url)
.referrer(this.url) // .referrer(this.url)
.cookies(cookies) // .cookies(cookies)
.response(); // .response();
cookies.putAll(resp.cookies()); // cookies.putAll(resp.cookies());
//
// Try to find the description // // Try to find the description
Document documentz = resp.parse(); // Document documentz = resp.parse();
Element ele = documentz.select("div.dev-description").first(); // Element ele = documentz.select("div.dev-description").first();
if (ele == null) { // if (ele == null) {
throw new IOException("No description found"); // throw new IOException("No description found");
} // }
documentz.outputSettings(new Document.OutputSettings().prettyPrint(false)); // documentz.outputSettings(new Document.OutputSettings().prettyPrint(false));
ele.select("br").append("\\n"); // ele.select("br").append("\\n");
ele.select("p").prepend("\\n\\n"); // ele.select("p").prepend("\\n\\n");
String fullSize = null; // String fullSize = null;
Element thumb = page.select("div.zones-container span.thumb[href=\"" + url + "\"]").get(0); // Element thumb = page.select("div.zones-container span.thumb[href=\"" + url + "\"]").get(0);
if (!thumb.attr("data-super-full-img").isEmpty()) { // if (!thumb.attr("data-super-full-img").isEmpty()) {
fullSize = thumb.attr("data-super-full-img"); // fullSize = thumb.attr("data-super-full-img");
String[] split = fullSize.split("/"); // String[] split = fullSize.split("/");
fullSize = split[split.length - 1]; // fullSize = split[split.length - 1];
} else { // } else {
String spanUrl = thumb.attr("href"); // String spanUrl = thumb.attr("href");
fullSize = jsonToImage(page,spanUrl.substring(spanUrl.lastIndexOf('-') + 1)); // fullSize = jsonToImage(page,spanUrl.substring(spanUrl.lastIndexOf('-') + 1));
if (fullSize != null) { // if (fullSize != null) {
String[] split = fullSize.split("/"); // String[] split = fullSize.split("/");
fullSize = split[split.length - 1]; // fullSize = split[split.length - 1];
} // }
} // }
if (fullSize == null) { // if (fullSize == null) {
return new String[] {Jsoup.clean(ele.html().replaceAll("\\\\n", System.getProperty("line.separator")), "", Whitelist.none(), new Document.OutputSettings().prettyPrint(false))}; // return new String[] {Jsoup.clean(ele.html().replaceAll("\\\\n", System.getProperty("line.separator")), "", Whitelist.none(), new Document.OutputSettings().prettyPrint(false))};
} // }
fullSize = fullSize.substring(0, fullSize.lastIndexOf(".")); // fullSize = fullSize.substring(0, fullSize.lastIndexOf("."));
return new String[] {Jsoup.clean(ele.html().replaceAll("\\\\n", System.getProperty("line.separator")), "", Whitelist.none(), new Document.OutputSettings().prettyPrint(false)),fullSize}; // return new String[] {Jsoup.clean(ele.html().replaceAll("\\\\n", System.getProperty("line.separator")), "", Whitelist.none(), new Document.OutputSettings().prettyPrint(false)),fullSize};
// TODO Make this not make a newline if someone just types \n into the description. // // TODO Make this not make a newline if someone just types \n into the description.
} catch (IOException ioe) { // } catch (IOException ioe) {
LOGGER.info("Failed to get description at " + url + ": '" + ioe.getMessage() + "'"); // LOGGER.info("Failed to get description at " + url + ": '" + ioe.getMessage() + "'");
return null; // return null;
} // }
} // }
/** /**
* If largest resolution for image at 'thumb' is found, starts downloading * If largest resolution for image at 'thumb' is found, starts downloading

6
src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java
View File

@ -7,18 +7,18 @@ import com.rarchives.ripme.ripper.rippers.DeviantartRipper;
public class DeviantartRipperTest extends RippersTest { public class DeviantartRipperTest extends RippersTest {
public void testDeviantartAlbum() throws IOException { public void testDeviantartAlbum() throws IOException {
DeviantartRipper ripper = new DeviantartRipper(new URL("http://airgee.deviantart.com/gallery/")); DeviantartRipper ripper = new DeviantartRipper(new URL("https://www.deviantart.com/airgee/gallery/"));
testRipper(ripper); testRipper(ripper);
} }
public void testDeviantartNSFWAlbum() throws IOException { public void testDeviantartNSFWAlbum() throws IOException {
// NSFW gallery // NSFW gallery
DeviantartRipper ripper = new DeviantartRipper(new URL("http://faterkcx.deviantart.com/gallery/")); DeviantartRipper ripper = new DeviantartRipper(new URL("https://www.deviantart.com/faterkcx/gallery/"));
testRipper(ripper); testRipper(ripper);
} }
public void testGetGID() throws IOException { public void testGetGID() throws IOException {
URL url = new URL("http://airgee.deviantart.com/gallery/"); URL url = new URL("https://www.deviantart.com/airgee/gallery/");
DeviantartRipper ripper = new DeviantartRipper(url); DeviantartRipper ripper = new DeviantartRipper(url);
assertEquals("airgee", ripper.getGID(url)); assertEquals("airgee", ripper.getGID(url));
} }