Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #759 from cyian-1756/deviantartGalleryFixes

Browse Source 
Looks good
This commit is contained in:
Kevin Jiang 2018-07-02 12:24:11 -04:00 committed by GitHub
commit 21199d6bda
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 83 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
View File

@ -54,26 +54,25 @@ public class DeviantartRipper extends AbstractJSONRipper {
public String getHost() {
return "deviantart";
}
@Override
public String getDomain() {
return "deviantart.com";
}
// @Override
// public boolean hasDescriptionSupport() {
// return true;
// }
@Override
public URL sanitizeURL(URL url) throws MalformedURLException {
String u = url.toExternalForm();
if (u.replace("/", "").endsWith(".deviantart.com")) {
// Root user page, get all albums
if (!u.endsWith("/gallery/") && !u.endsWith("/gallery")) {
if (!u.endsWith("/")) {
u += "/";
u += "/gallery/";
} else {
u += "gallery/";
}
u += "gallery/?";
}
Pattern p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/favou?rites/([0-9]+)/*?$");
Matcher m = p.matcher(url.toExternalForm());
if (!m.matches()) {
@ -119,6 +118,25 @@ public class DeviantartRipper extends AbstractJSONRipper {
throw new MalformedURLException("Expected URL format: http://www.deviantart.com/username[/gallery/#####], got: " + url);
}
private String getFullsizedNSFWImage(String pageURL) {
try {
Document doc = Http.url(pageURL).cookies(cookies).get();
String imageToReturn = "";
String[] d = doc.select("img").attr("srcset").split(",");
String s = d[d.length -1].split(" ")[0];
LOGGER.info("2:" + s);
if (s == null || s.equals("")) {
LOGGER.error("Could not find full sized image at " + pageURL);
}
return s;
} catch (IOException e) {
LOGGER.error("Could not find full sized image at " + pageURL);
return null;
}
}
/**
* Gets first page.
* Will determine if login is supplied,
@ -148,8 +166,8 @@ public class DeviantartRipper extends AbstractJSONRipper {
JSONObject firstPageJSON = getFirstPageJSON(page);
requestID = firstPageJSON.getJSONObject("dapx").getString("requestid");
galleryID = page.select("input[name=set]").attr("value");
username = page.select("div.tt-tv150").attr("username");
galleryID = getGalleryID(page);
username = getUsername(page);
csrf = firstPageJSON.getString("csrf");
pageCookies = res.cookies();
@ -178,7 +196,6 @@ public class DeviantartRipper extends AbstractJSONRipper {
private JSONObject getFirstPageJSON(Document doc) {
for (Element js : doc.select("script")) {
LOGGER.info(js.html());
if (js.html().contains("requestid")) {
String json = js.html().replaceAll("window.__initial_body_data=", "").replaceAll("\\);", "")
.replaceAll(";__wake\\(.+", "");
@ -189,6 +206,24 @@ public class DeviantartRipper extends AbstractJSONRipper {
}
return null;
}
public String getGalleryID(Document doc) {
for (Element el : doc.select("input[name=set]")) {
try {
String galleryID = el.attr("value");
return galleryID;
} catch (NullPointerException e) {
continue;
}
}
LOGGER.error("Could not find gallery ID");
return null;
}
public String getUsername(Document doc) {
return doc.select("meta[property=og:title]").attr("content")
.replaceAll("'s DeviantArt gallery", "").replaceAll("'s DeviantArt Gallery", "");
}
@Override
@ -197,8 +232,14 @@ public class DeviantartRipper extends AbstractJSONRipper {
LOGGER.info(json);
JSONArray results = json.getJSONObject("content").getJSONArray("results");
for (int i = 0; i < results.length(); i++) {
LOGGER.info(results.getJSONObject(i).toString());
Document doc = Jsoup.parseBodyFragment(results.getJSONObject(i).getString("html"));
if (doc.html().contains("ismature")) {
LOGGER.info("Downloading nsfw image");
String nsfwImage = getFullsizedNSFWImage(doc.select("span").attr("href"));
if (nsfwImage != null) {
imageURLs.add(nsfwImage);
}
}
try {
String imageURL = doc.select("span").first().attr("data-super-full-img");
if (!imageURL.isEmpty()) {
@ -240,12 +281,12 @@ public class DeviantartRipper extends AbstractJSONRipper {
throw new IOException("No more pages");
}
@Override
public boolean keepSortOrder() {
// Don't keep sort order (do not add prefixes).
// Causes file duplication, as outlined in https://github.com/4pr0n/ripme/issues/113
return false;
}
// @Override
// public boolean keepSortOrder() {
// // Don't keep sort order (do not add prefixes).
// // Causes file duplication, as outlined in https://github.com/4pr0n/ripme/issues/113
// return false;
// }
@Override
public void downloadURL(URL url, int index) {

24
src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java
View File

@ -2,8 +2,12 @@ package com.rarchives.ripme.tst.ripper.rippers;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import com.rarchives.ripme.ripper.rippers.DeviantartRipper;
import com.rarchives.ripme.utils.Http;
import org.jsoup.nodes.Document;
public class DeviantartRipperTest extends RippersTest {
public void testDeviantartAlbum() throws IOException {
@ -22,4 +26,24 @@ public class DeviantartRipperTest extends RippersTest {
DeviantartRipper ripper = new DeviantartRipper(url);
assertEquals("airgee", ripper.getGID(url));
}
public void testGetGalleryIDAndUsername() throws IOException {
URL url = new URL("https://www.deviantart.com/airgee/gallery/");
DeviantartRipper ripper = new DeviantartRipper(url);
Document doc = Http.url(url).get();
assertEquals("airgee", ripper.getUsername(doc));
assertEquals("714589", ripper.getGalleryID(doc));
}
public void testSanitizeURL() throws IOException {
List<URL> urls = new ArrayList<URL>();
urls.add(new URL("https://www.deviantart.com/airgee/"));
urls.add(new URL("https://www.deviantart.com/airgee"));
urls.add(new URL("https://www.deviantart.com/airgee/gallery/"));
for (URL url : urls) {
DeviantartRipper ripper = new DeviantartRipper(url);
assertEquals("https://www.deviantart.com/airgee/gallery/", ripper.sanitizeURL(url).toExternalForm());
}
}
}