From 692dcaa9906a34b0a51dc57fdd77269231cfaec6 Mon Sep 17 00:00:00 2001 From: Kevin Jiang Date: Sat, 14 Apr 2018 17:45:56 -0400 Subject: [PATCH 1/5] Travis CL test because I'm too lazy to run it on my own computer TEST --- .../ripper/rippers/DeviantartRipper.java | 47 ++++++++++++++----- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index 4461188e..225c9c23 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -1,5 +1,9 @@ package com.rarchives.ripme.ripper.rippers; +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Base64; +import com.rarchives.ripme.utils.Http; +import com.rarchives.ripme.utils.Utils; import java.io.IOException; import java.net.HttpURLConnection; import java.net.MalformedURLException; @@ -13,7 +17,6 @@ import java.util.Map; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; - import org.jsoup.Connection.Method; import org.jsoup.Connection.Response; import org.jsoup.Jsoup; @@ -22,11 +25,6 @@ import org.jsoup.nodes.Element; import org.jsoup.safety.Whitelist; import org.jsoup.select.Elements; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Base64; -import com.rarchives.ripme.utils.Http; -import com.rarchives.ripme.utils.Utils; - public class DeviantartRipper extends AbstractHTMLRipper { private static final int PAGE_SLEEP_TIME = 3000, @@ -108,19 +106,46 @@ public class DeviantartRipper extends AbstractHTMLRipper { throw new MalformedURLException("Expected URL format: http://username.deviantart.com/[/gallery/#####], got: " + url); } + /** + * Gets first page. + * Will determine if login is supplied, + * if there is a login, then login and add that login cookies. + * Otherwise, just bypass the age gate with an anonymous flag. + * @return + * @throws IOException + */ @Override public Document getFirstPage() throws IOException { - // Login - try { - cookies = loginToDeviantart(); - } catch (Exception e) { - logger.warn("Failed to login: ", e); + + //Test to see if there is a login: + String username = Utils.getConfigString("deviantart.username", new String(Base64.decode("Z3JhYnB5"))); + String password = Utils.getConfigString("deviantart.password", new String(Base64.decode("ZmFrZXJz"))); + + if(username == null || password == null) { + logger.debug("No DeviantArt login provided."); cookies.put("agegate_state","1"); // Bypasses the age gate + } else { + // Attempt Login + try { + cookies = loginToDeviantart(); + } catch (Exception e) { + logger.warn("Failed to login: ", e); + cookies.put("agegate_state","1"); // Bypasses the age gate + } } + + return Http.url(this.url) .cookies(cookies) .get(); } + + /** + * + * @param page + * @param id + * @return + */ private String jsonToImage(Document page, String id) { Elements js = page.select("script[type=\"text/javascript\"]"); for (Element tag : js) { From 264980ad94a526ac7399cfe86af4831c93db11f2 Mon Sep 17 00:00:00 2001 From: Kevin Jiang Date: Sat, 14 Apr 2018 17:55:45 -0400 Subject: [PATCH 2/5] More specific Exception name IOException > Exception --- .../com/rarchives/ripme/ripper/rippers/DeviantartRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index 225c9c23..8ecb8086 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -128,7 +128,7 @@ public class DeviantartRipper extends AbstractHTMLRipper { // Attempt Login try { cookies = loginToDeviantart(); - } catch (Exception e) { + } catch (IOException e) { logger.warn("Failed to login: ", e); cookies.put("agegate_state","1"); // Bypasses the age gate } From 1c38a86d3d6fae4a4957c632e83e260120f386da Mon Sep 17 00:00:00 2001 From: Kevin Jiang Date: Fri, 11 May 2018 21:52:01 -0400 Subject: [PATCH 3/5] ImageBam album fixed, more Documentation Slightly fixed it, but don't know CSS so it's kinda a bootleg approach, but it works extremely well. Added bits of documentation here and there, and changed .size() == 0 to .isEmpty() for more readability. --- .../ripme/ripper/rippers/ImagebamRipper.java | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java index 323ad1de..b5fca14f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java @@ -1,5 +1,9 @@ package com.rarchives.ripme.ripper.rippers; +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.ripper.DownloadThreadPool; +import com.rarchives.ripme.utils.Http; +import com.rarchives.ripme.utils.Utils; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; @@ -7,16 +11,10 @@ import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; - import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; -import com.rarchives.ripme.utils.Http; -import com.rarchives.ripme.utils.Utils; - public class ImagebamRipper extends AbstractHTMLRipper { // Current HTML document @@ -71,7 +69,7 @@ public class ImagebamRipper extends AbstractHTMLRipper { public Document getNextPage(Document doc) throws IOException { // Find next page Elements hrefs = doc.select("a.pagination_current + a.pagination_link"); - if (hrefs.size() == 0) { + if (hrefs.isEmpty()) { throw new IOException("No more pages"); } String nextUrl = "http://www.imagebam.com" + hrefs.first().attr("href"); @@ -121,8 +119,8 @@ public class ImagebamRipper extends AbstractHTMLRipper { * Handles case when site has IP-banned the user. */ private class ImagebamImageThread extends Thread { - private URL url; - private int index; + private URL url; //link to "image page" + private int index; //index in album ImagebamImageThread(URL url, int index) { super(); @@ -134,24 +132,39 @@ public class ImagebamRipper extends AbstractHTMLRipper { public void run() { fetchImage(); } - + + /** + * Rips useful image from "image page" + */ private void fetchImage() { try { Document doc = Http.url(url).get(); // Find image - Elements images = doc.select(".image-container img"); - if (images.size() == 0) { + Elements metaTags = doc.getElementsByTag("meta"); + + String imgsrc = "";//initialize, so no NullPointerExceptions should ever happen. + + for (Element metaTag: metaTags) { + //the direct link to the image seems to always be linked in the part of the html. + if(metaTag.attr("property").equals("og:image")) { + imgsrc = metaTag.attr("content"); + logger.info("Found URL " + imgsrc); + break;//only one (useful) image possible for an "image page". + } + } + + //for debug, or something goes wrong. + if (imgsrc.isEmpty()) { logger.warn("Image not found at " + this.url); return; } - Element image = images.first(); - String imgsrc = image.attr("src"); - logger.info("Found URL " + imgsrc); + // Provide prefix and let the AbstractRipper "guess" the filename String prefix = ""; if (Utils.getConfigBoolean("download.save_order", true)) { prefix = String.format("%03d_", index); } + addURLToDownload(new URL(imgsrc), prefix); } catch (IOException e) { logger.error("[!] Exception while loading/parsing " + this.url, e); From 96bf2ad3a7d655c5d5311f36ae7491bd9620f6ec Mon Sep 17 00:00:00 2001 From: Kevin Jiang Date: Sat, 12 May 2018 17:58:27 -0400 Subject: [PATCH 4/5] Update ImagebamRipper.java --- .../com/rarchives/ripme/ripper/rippers/ImagebamRipper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java index b5fca14f..b33f5624 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java @@ -146,7 +146,7 @@ public class ImagebamRipper extends AbstractHTMLRipper { for (Element metaTag: metaTags) { //the direct link to the image seems to always be linked in the part of the html. - if(metaTag.attr("property").equals("og:image")) { + if (metaTag.attr("property").equals("og:image")) { imgsrc = metaTag.attr("content"); logger.info("Found URL " + imgsrc); break;//only one (useful) image possible for an "image page". @@ -171,4 +171,4 @@ public class ImagebamRipper extends AbstractHTMLRipper { } } } -} \ No newline at end of file +} From a670de675a0550d8fb93723407efbf0f8239d7d4 Mon Sep 17 00:00:00 2001 From: Kevin Jiang Date: Sat, 12 May 2018 17:58:55 -0400 Subject: [PATCH 5/5] if( if ( --- .../com/rarchives/ripme/ripper/rippers/DeviantartRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index 8ecb8086..9ab23ec8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -121,7 +121,7 @@ public class DeviantartRipper extends AbstractHTMLRipper { String username = Utils.getConfigString("deviantart.username", new String(Base64.decode("Z3JhYnB5"))); String password = Utils.getConfigString("deviantart.password", new String(Base64.decode("ZmFrZXJz"))); - if(username == null || password == null) { + if (username == null || password == null) { logger.debug("No DeviantArt login provided."); cookies.put("agegate_state","1"); // Bypasses the age gate } else {