From dab771f2da740516fccceb2cc3cfd069201ebccd Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 1 Jul 2018 14:55:48 -0400 Subject: [PATCH 1/4] Got DeviantartRipper working again --- .../ripper/rippers/DeviantartRipper.java | 36 ++++++++++++++----- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index 1d7fb17c..a3428ae6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -148,8 +148,8 @@ public class DeviantartRipper extends AbstractJSONRipper { JSONObject firstPageJSON = getFirstPageJSON(page); requestID = firstPageJSON.getJSONObject("dapx").getString("requestid"); - galleryID = page.select("input[name=set]").attr("value"); - username = page.select("div.tt-tv150").attr("username"); + galleryID = getGalleryID(page); + username = getUsername(page); csrf = firstPageJSON.getString("csrf"); pageCookies = res.cookies(); @@ -189,6 +189,25 @@ public class DeviantartRipper extends AbstractJSONRipper { } return null; } + + private String getGalleryID(Document doc) { + for (Element el : doc.select("input[name=set]")) { + try { + String galleryID = el.attr("value"); + if (galleryID.length() == 8) { + return galleryID; + } + } catch (NullPointerException e) { + continue; + } + } + LOGGER.error("Could not find gallery ID"); + return null; + } + + private String getUsername(Document doc) { + return doc.select("meta[property=og:title]").attr("content").replaceAll("'s DeviantArt gallery", ""); + } @Override @@ -197,7 +216,6 @@ public class DeviantartRipper extends AbstractJSONRipper { LOGGER.info(json); JSONArray results = json.getJSONObject("content").getJSONArray("results"); for (int i = 0; i < results.length(); i++) { - LOGGER.info(results.getJSONObject(i).toString()); Document doc = Jsoup.parseBodyFragment(results.getJSONObject(i).getString("html")); try { String imageURL = doc.select("span").first().attr("data-super-full-img"); @@ -240,12 +258,12 @@ public class DeviantartRipper extends AbstractJSONRipper { throw new IOException("No more pages"); } - @Override - public boolean keepSortOrder() { - // Don't keep sort order (do not add prefixes). - // Causes file duplication, as outlined in https://github.com/4pr0n/ripme/issues/113 - return false; - } +// @Override +// public boolean keepSortOrder() { +// // Don't keep sort order (do not add prefixes). +// // Causes file duplication, as outlined in https://github.com/4pr0n/ripme/issues/113 +// return false; +// } @Override public void downloadURL(URL url, int index) { From ae6d09c6e85578c301f789b7c7e113a5204cf0fc Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 1 Jul 2018 15:14:09 -0400 Subject: [PATCH 2/4] Added some unit tests for DeviantartRipper --- .../ripme/ripper/rippers/DeviantartRipper.java | 8 +++----- .../ripme/tst/ripper/rippers/DeviantartRipperTest.java | 10 ++++++++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index a3428ae6..8d9f3da2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -190,13 +190,11 @@ public class DeviantartRipper extends AbstractJSONRipper { return null; } - private String getGalleryID(Document doc) { + public String getGalleryID(Document doc) { for (Element el : doc.select("input[name=set]")) { try { String galleryID = el.attr("value"); - if (galleryID.length() == 8) { - return galleryID; - } + return galleryID; } catch (NullPointerException e) { continue; } @@ -205,7 +203,7 @@ public class DeviantartRipper extends AbstractJSONRipper { return null; } - private String getUsername(Document doc) { + public String getUsername(Document doc) { return doc.select("meta[property=og:title]").attr("content").replaceAll("'s DeviantArt gallery", ""); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java index f68d1db5..52b6bc59 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java @@ -4,6 +4,8 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.DeviantartRipper; +import com.rarchives.ripme.utils.Http; +import org.jsoup.nodes.Document; public class DeviantartRipperTest extends RippersTest { public void testDeviantartAlbum() throws IOException { @@ -22,4 +24,12 @@ public class DeviantartRipperTest extends RippersTest { DeviantartRipper ripper = new DeviantartRipper(url); assertEquals("airgee", ripper.getGID(url)); } + + public void testGetGalleryIDAndUsername() throws IOException { + URL url = new URL("https://www.deviantart.com/airgee/gallery/"); + DeviantartRipper ripper = new DeviantartRipper(url); + Document doc = Http.url(url).get(); + assertEquals("airgee", ripper.getUsername(doc)); + assertEquals("714589", ripper.getGalleryID(doc)); + } } From 0ed2a7ddac29db06536cd999481b28749f1fe6b7 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 1 Jul 2018 16:42:11 -0400 Subject: [PATCH 3/4] DeviantartRipper can now rip nsfw images --- .../ripper/rippers/DeviantartRipper.java | 45 ++++++++++++++----- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index 8d9f3da2..279d82d1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -54,26 +54,23 @@ public class DeviantartRipper extends AbstractJSONRipper { public String getHost() { return "deviantart"; } + @Override public String getDomain() { return "deviantart.com"; } -// @Override -// public boolean hasDescriptionSupport() { -// return true; -// } + @Override public URL sanitizeURL(URL url) throws MalformedURLException { String u = url.toExternalForm(); - if (u.replace("/", "").endsWith(".deviantart.com")) { - // Root user page, get all albums - if (!u.endsWith("/")) { - u += "/"; + if (!u.endsWith("/gallery/")) { + if (!u.endsWith("/gallery")) { + u += "gallery/"; } - u += "gallery/?"; } + Pattern p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/favou?rites/([0-9]+)/*?$"); Matcher m = p.matcher(url.toExternalForm()); if (!m.matches()) { @@ -119,6 +116,25 @@ public class DeviantartRipper extends AbstractJSONRipper { throw new MalformedURLException("Expected URL format: http://www.deviantart.com/username[/gallery/#####], got: " + url); } + private String getFullsizedNSFWImage(String pageURL) { + try { + Document doc = Http.url(pageURL).cookies(cookies).get(); + String imageToReturn = ""; + String[] d = doc.select("img").attr("srcset").split(","); + + String s = d[d.length -1].split(" ")[0]; + LOGGER.info("2:" + s); + + if (s == null || s.equals("")) { + LOGGER.error("Could not find full sized image at " + pageURL); + } + return s; + } catch (IOException e) { + LOGGER.error("Could not find full sized image at " + pageURL); + return null; + } + } + /** * Gets first page. * Will determine if login is supplied, @@ -178,7 +194,6 @@ public class DeviantartRipper extends AbstractJSONRipper { private JSONObject getFirstPageJSON(Document doc) { for (Element js : doc.select("script")) { - LOGGER.info(js.html()); if (js.html().contains("requestid")) { String json = js.html().replaceAll("window.__initial_body_data=", "").replaceAll("\\);", "") .replaceAll(";__wake\\(.+", ""); @@ -204,7 +219,8 @@ public class DeviantartRipper extends AbstractJSONRipper { } public String getUsername(Document doc) { - return doc.select("meta[property=og:title]").attr("content").replaceAll("'s DeviantArt gallery", ""); + return doc.select("meta[property=og:title]").attr("content") + .replaceAll("'s DeviantArt gallery", "").replaceAll("'s DeviantArt Gallery", ""); } @@ -215,6 +231,13 @@ public class DeviantartRipper extends AbstractJSONRipper { JSONArray results = json.getJSONObject("content").getJSONArray("results"); for (int i = 0; i < results.length(); i++) { Document doc = Jsoup.parseBodyFragment(results.getJSONObject(i).getString("html")); + if (doc.html().contains("ismature")) { + LOGGER.info("Downloading nsfw image"); + String nsfwImage = getFullsizedNSFWImage(doc.select("span").attr("href")); + if (nsfwImage != null) { + imageURLs.add(nsfwImage); + } + } try { String imageURL = doc.select("span").first().attr("data-super-full-img"); if (!imageURL.isEmpty()) { From d0960fad75d5f3f79055281e3d963f4a63582a10 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 1 Jul 2018 19:25:32 -0400 Subject: [PATCH 4/4] Added better url sanitization for DeviantartRipper and some unit tests --- .../ripme/ripper/rippers/DeviantartRipper.java | 6 ++++-- .../tst/ripper/rippers/DeviantartRipperTest.java | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index 279d82d1..cb3e7595 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -64,8 +64,10 @@ public class DeviantartRipper extends AbstractJSONRipper { public URL sanitizeURL(URL url) throws MalformedURLException { String u = url.toExternalForm(); - if (!u.endsWith("/gallery/")) { - if (!u.endsWith("/gallery")) { + if (!u.endsWith("/gallery/") && !u.endsWith("/gallery")) { + if (!u.endsWith("/")) { + u += "/gallery/"; + } else { u += "gallery/"; } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java index 52b6bc59..6a1a054b 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; import java.net.URL; +import java.util.ArrayList; +import java.util.List; import com.rarchives.ripme.ripper.rippers.DeviantartRipper; import com.rarchives.ripme.utils.Http; @@ -32,4 +34,16 @@ public class DeviantartRipperTest extends RippersTest { assertEquals("airgee", ripper.getUsername(doc)); assertEquals("714589", ripper.getGalleryID(doc)); } + + public void testSanitizeURL() throws IOException { + List urls = new ArrayList(); + urls.add(new URL("https://www.deviantart.com/airgee/")); + urls.add(new URL("https://www.deviantart.com/airgee")); + urls.add(new URL("https://www.deviantart.com/airgee/gallery/")); + + for (URL url : urls) { + DeviantartRipper ripper = new DeviantartRipper(url); + assertEquals("https://www.deviantart.com/airgee/gallery/", ripper.sanitizeURL(url).toExternalForm()); + } + } }