diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index 1d7fb17c..cb3e7595 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -54,26 +54,25 @@ public class DeviantartRipper extends AbstractJSONRipper { public String getHost() { return "deviantart"; } + @Override public String getDomain() { return "deviantart.com"; } -// @Override -// public boolean hasDescriptionSupport() { -// return true; -// } + @Override public URL sanitizeURL(URL url) throws MalformedURLException { String u = url.toExternalForm(); - if (u.replace("/", "").endsWith(".deviantart.com")) { - // Root user page, get all albums + if (!u.endsWith("/gallery/") && !u.endsWith("/gallery")) { if (!u.endsWith("/")) { - u += "/"; + u += "/gallery/"; + } else { + u += "gallery/"; } - u += "gallery/?"; } + Pattern p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/favou?rites/([0-9]+)/*?$"); Matcher m = p.matcher(url.toExternalForm()); if (!m.matches()) { @@ -119,6 +118,25 @@ public class DeviantartRipper extends AbstractJSONRipper { throw new MalformedURLException("Expected URL format: http://www.deviantart.com/username[/gallery/#####], got: " + url); } + private String getFullsizedNSFWImage(String pageURL) { + try { + Document doc = Http.url(pageURL).cookies(cookies).get(); + String imageToReturn = ""; + String[] d = doc.select("img").attr("srcset").split(","); + + String s = d[d.length -1].split(" ")[0]; + LOGGER.info("2:" + s); + + if (s == null || s.equals("")) { + LOGGER.error("Could not find full sized image at " + pageURL); + } + return s; + } catch (IOException e) { + LOGGER.error("Could not find full sized image at " + pageURL); + return null; + } + } + /** * Gets first page. * Will determine if login is supplied, @@ -148,8 +166,8 @@ public class DeviantartRipper extends AbstractJSONRipper { JSONObject firstPageJSON = getFirstPageJSON(page); requestID = firstPageJSON.getJSONObject("dapx").getString("requestid"); - galleryID = page.select("input[name=set]").attr("value"); - username = page.select("div.tt-tv150").attr("username"); + galleryID = getGalleryID(page); + username = getUsername(page); csrf = firstPageJSON.getString("csrf"); pageCookies = res.cookies(); @@ -178,7 +196,6 @@ public class DeviantartRipper extends AbstractJSONRipper { private JSONObject getFirstPageJSON(Document doc) { for (Element js : doc.select("script")) { - LOGGER.info(js.html()); if (js.html().contains("requestid")) { String json = js.html().replaceAll("window.__initial_body_data=", "").replaceAll("\\);", "") .replaceAll(";__wake\\(.+", ""); @@ -189,6 +206,24 @@ public class DeviantartRipper extends AbstractJSONRipper { } return null; } + + public String getGalleryID(Document doc) { + for (Element el : doc.select("input[name=set]")) { + try { + String galleryID = el.attr("value"); + return galleryID; + } catch (NullPointerException e) { + continue; + } + } + LOGGER.error("Could not find gallery ID"); + return null; + } + + public String getUsername(Document doc) { + return doc.select("meta[property=og:title]").attr("content") + .replaceAll("'s DeviantArt gallery", "").replaceAll("'s DeviantArt Gallery", ""); + } @Override @@ -197,8 +232,14 @@ public class DeviantartRipper extends AbstractJSONRipper { LOGGER.info(json); JSONArray results = json.getJSONObject("content").getJSONArray("results"); for (int i = 0; i < results.length(); i++) { - LOGGER.info(results.getJSONObject(i).toString()); Document doc = Jsoup.parseBodyFragment(results.getJSONObject(i).getString("html")); + if (doc.html().contains("ismature")) { + LOGGER.info("Downloading nsfw image"); + String nsfwImage = getFullsizedNSFWImage(doc.select("span").attr("href")); + if (nsfwImage != null) { + imageURLs.add(nsfwImage); + } + } try { String imageURL = doc.select("span").first().attr("data-super-full-img"); if (!imageURL.isEmpty()) { @@ -240,12 +281,12 @@ public class DeviantartRipper extends AbstractJSONRipper { throw new IOException("No more pages"); } - @Override - public boolean keepSortOrder() { - // Don't keep sort order (do not add prefixes). - // Causes file duplication, as outlined in https://github.com/4pr0n/ripme/issues/113 - return false; - } +// @Override +// public boolean keepSortOrder() { +// // Don't keep sort order (do not add prefixes). +// // Causes file duplication, as outlined in https://github.com/4pr0n/ripme/issues/113 +// return false; +// } @Override public void downloadURL(URL url, int index) { diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java index f68d1db5..6a1a054b 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java @@ -2,8 +2,12 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; import java.net.URL; +import java.util.ArrayList; +import java.util.List; import com.rarchives.ripme.ripper.rippers.DeviantartRipper; +import com.rarchives.ripme.utils.Http; +import org.jsoup.nodes.Document; public class DeviantartRipperTest extends RippersTest { public void testDeviantartAlbum() throws IOException { @@ -22,4 +26,24 @@ public class DeviantartRipperTest extends RippersTest { DeviantartRipper ripper = new DeviantartRipper(url); assertEquals("airgee", ripper.getGID(url)); } + + public void testGetGalleryIDAndUsername() throws IOException { + URL url = new URL("https://www.deviantart.com/airgee/gallery/"); + DeviantartRipper ripper = new DeviantartRipper(url); + Document doc = Http.url(url).get(); + assertEquals("airgee", ripper.getUsername(doc)); + assertEquals("714589", ripper.getGalleryID(doc)); + } + + public void testSanitizeURL() throws IOException { + List urls = new ArrayList(); + urls.add(new URL("https://www.deviantart.com/airgee/")); + urls.add(new URL("https://www.deviantart.com/airgee")); + urls.add(new URL("https://www.deviantart.com/airgee/gallery/")); + + for (URL url : urls) { + DeviantartRipper ripper = new DeviantartRipper(url); + assertEquals("https://www.deviantart.com/airgee/gallery/", ripper.sanitizeURL(url).toExternalForm()); + } + } }