Merge pull request #759 from cyian-1756/deviantartGalleryFixes

Looks good

src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java

@@ -54,26 +54,25 @@ public class DeviantartRipper extends AbstractJSONRipper {
     public String getHost() {
         return "deviantart";
     }
+
     @Override
     public String getDomain() {
         return "deviantart.com";
     }
-//    @Override
+
-//    public boolean hasDescriptionSupport() {
-//        return true;
-//    }
     @Override
     public URL sanitizeURL(URL url) throws MalformedURLException {
         String u = url.toExternalForm();
 
-        if (u.replace("/", "").endsWith(".deviantart.com")) {
+        if (!u.endsWith("/gallery/") && !u.endsWith("/gallery")) {
-            // Root user page, get all albums
             if (!u.endsWith("/")) {
-                u += "/";
+                u += "/gallery/";
+            } else {
+                u += "gallery/";
             }
-            u += "gallery/?";
         }
 
+
         Pattern p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/favou?rites/([0-9]+)/*?$");
         Matcher m = p.matcher(url.toExternalForm());
         if (!m.matches()) {
@@ -119,6 +118,25 @@ public class DeviantartRipper extends AbstractJSONRipper {
         throw new MalformedURLException("Expected URL format: http://www.deviantart.com/username[/gallery/#####], got: " + url);
     }
 
+    private String getFullsizedNSFWImage(String pageURL) {
+        try {
+            Document doc = Http.url(pageURL).cookies(cookies).get();
+            String imageToReturn = "";
+            String[] d = doc.select("img").attr("srcset").split(",");
+
+            String s = d[d.length -1].split(" ")[0];
+            LOGGER.info("2:" + s);
+
+            if (s == null || s.equals("")) {
+                LOGGER.error("Could not find full sized image at " + pageURL);
+            }
+            return s;
+        } catch (IOException e) {
+            LOGGER.error("Could not find full sized image at " + pageURL);
+            return null;
+        }
+    }
+
     /**
      * Gets first page.
      * Will determine if login is supplied,
@@ -148,8 +166,8 @@ public class DeviantartRipper extends AbstractJSONRipper {
 
         JSONObject firstPageJSON = getFirstPageJSON(page);
         requestID = firstPageJSON.getJSONObject("dapx").getString("requestid");
-        galleryID = page.select("input[name=set]").attr("value");
+        galleryID = getGalleryID(page);
-        username = page.select("div.tt-tv150").attr("username");
+        username = getUsername(page);
         csrf = firstPageJSON.getString("csrf");
         pageCookies = res.cookies();
 
@@ -178,7 +196,6 @@ public class DeviantartRipper extends AbstractJSONRipper {
 
     private JSONObject getFirstPageJSON(Document doc) {
         for (Element js : doc.select("script")) {
-            LOGGER.info(js.html());
             if (js.html().contains("requestid")) {
                 String json = js.html().replaceAll("window.__initial_body_data=", "").replaceAll("\\);", "")
                         .replaceAll(";__wake\\(.+", "");
@@ -190,6 +207,24 @@ public class DeviantartRipper extends AbstractJSONRipper {
         return null;
     }
 
+    public String getGalleryID(Document doc) {
+        for (Element el : doc.select("input[name=set]")) {
+            try {
+                String galleryID = el.attr("value");
+                return galleryID;
+            } catch (NullPointerException e) {
+                continue;
+            }
+        }
+        LOGGER.error("Could not find gallery ID");
+        return null;
+    }
+
+    public String getUsername(Document doc) {
+        return doc.select("meta[property=og:title]").attr("content")
+                .replaceAll("'s DeviantArt gallery", "").replaceAll("'s DeviantArt Gallery", "");
+    }
+    
 
     @Override
     public List<String> getURLsFromJSON(JSONObject json) {
@@ -197,8 +232,14 @@ public class DeviantartRipper extends AbstractJSONRipper {
         LOGGER.info(json);
         JSONArray results = json.getJSONObject("content").getJSONArray("results");
         for (int i = 0; i < results.length(); i++) {
-            LOGGER.info(results.getJSONObject(i).toString());
             Document doc = Jsoup.parseBodyFragment(results.getJSONObject(i).getString("html"));
+            if (doc.html().contains("ismature")) {
+                LOGGER.info("Downloading nsfw image");
+                String nsfwImage = getFullsizedNSFWImage(doc.select("span").attr("href"));
+                if (nsfwImage != null) {
+                    imageURLs.add(nsfwImage);
+                }
+            }
             try {
                 String imageURL = doc.select("span").first().attr("data-super-full-img");
                 if (!imageURL.isEmpty()) {
@@ -240,12 +281,12 @@ public class DeviantartRipper extends AbstractJSONRipper {
         throw new IOException("No more pages");
     }
 
-    @Override
+//    @Override
-    public boolean keepSortOrder() {
+//    public boolean keepSortOrder() {
-         // Don't keep sort order (do not add prefixes).
+//         // Don't keep sort order (do not add prefixes).
-         // Causes file duplication, as outlined in https://github.com/4pr0n/ripme/issues/113
+//         // Causes file duplication, as outlined in https://github.com/4pr0n/ripme/issues/113
-        return false;
+//        return false;
-    }
+//    }
 
     @Override
     public void downloadURL(URL url, int index) {

src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java

@@ -2,8 +2,12 @@ package com.rarchives.ripme.tst.ripper.rippers;
 
 import java.io.IOException;
 import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
 
 import com.rarchives.ripme.ripper.rippers.DeviantartRipper;
+import com.rarchives.ripme.utils.Http;
+import org.jsoup.nodes.Document;
 
 public class DeviantartRipperTest extends RippersTest {
     public void testDeviantartAlbum() throws IOException {
@@ -22,4 +26,24 @@ public class DeviantartRipperTest extends RippersTest {
         DeviantartRipper ripper = new DeviantartRipper(url);
         assertEquals("airgee", ripper.getGID(url));
     }
+
+    public void testGetGalleryIDAndUsername() throws IOException {
+        URL url = new URL("https://www.deviantart.com/airgee/gallery/");
+        DeviantartRipper ripper = new DeviantartRipper(url);
+        Document doc = Http.url(url).get();
+        assertEquals("airgee", ripper.getUsername(doc));
+        assertEquals("714589", ripper.getGalleryID(doc));
+    }
+
+    public void testSanitizeURL() throws IOException {
+        List<URL> urls = new ArrayList<URL>();
+        urls.add(new URL("https://www.deviantart.com/airgee/"));
+        urls.add(new URL("https://www.deviantart.com/airgee"));
+        urls.add(new URL("https://www.deviantart.com/airgee/gallery/"));
+
+        for (URL url : urls) {
+            DeviantartRipper ripper = new DeviantartRipper(url);
+            assertEquals("https://www.deviantart.com/airgee/gallery/", ripper.sanitizeURL(url).toExternalForm());
+        }
+    }
 }