From 567f25349147e8fff11254b4eefb10e5d7184bb3 Mon Sep 17 00:00:00 2001 From: 4pr0n Date: Sat, 17 May 2014 09:50:07 -0700 Subject: [PATCH] 1.0.44 - Deviantart gets full-sized images when possible #44 Ripping slows down a lot, but it gets full-size image when there's a "download" button on the deviantart page. --- pom.xml | 2 +- .../ripper/rippers/DeviantartRipper.java | 64 ++++++++++++++++--- .../com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 57 insertions(+), 11 deletions(-) diff --git a/pom.xml b/pom.xml index e0344199..fb39cbf7 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.0.43 + 1.0.44 ripme http://rip.rarchives.com diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index d59ad26c..f8f3046b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -17,6 +17,7 @@ import org.jsoup.Connection.Response; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AlbumRipper; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; @@ -59,21 +60,34 @@ public class DeviantartRipper extends AlbumRipper { .userAgent(USER_AGENT) .get(); - for (Element thumb : doc.select("div.zones-container a.thumb img")) { - if (thumb.attr("transparent").equals("false")) { + for (Element thumb : doc.select("div.zones-container a.thumb")) { + if (isStopped()) { + break; + } + Element img = thumb.select("img").get(0); + if (img.attr("transparent").equals("false")) { continue; // a.thumbs to other albums are invisible } + index++; - String fullSize = thumbToFull(thumb.attr("src")); + String fullSize = null; + try { + fullSize = thumbToFull(img.attr("src"), true); + } catch (Exception e) { + logger.info("Attempting to get full size image from " + thumb.attr("href")); + fullSize = smallToFull(img.attr("src"), thumb.attr("href")); + if (fullSize == null) { + continue; + } + } try { URL fullsizeURL = new URL(fullSize); String imageId = fullSize.substring(fullSize.lastIndexOf('-') + 1); imageId = imageId.substring(0, imageId.indexOf('.')); long imageIdLong = alphaToLong(imageId); - index++; addURLToDownload(fullsizeURL, String.format("%010d_", imageIdLong)); } catch (MalformedURLException e) { - logger.error("[!] Invalid thumbnail image: " + thumbToFull(fullSize)); + logger.error("[!] Invalid thumbnail image: " + fullSize); continue; } } @@ -101,7 +115,6 @@ public class DeviantartRipper extends AlbumRipper { long result = 0; for (int i = 0; i < alpha.length(); i++) { result += charToInt(alpha, i); - System.err.println("\t result: " + result); } return result; } @@ -109,17 +122,20 @@ public class DeviantartRipper extends AlbumRipper { private static int charToInt(String text, int index) { char c = text.charAt(text.length() - index - 1); c = Character.toLowerCase(c); - System.err.print(" " + c + ": "); int number = "0123456789abcdefghijklmnopqrstuvwxyz".indexOf(c); number *= Math.pow(36, index); - System.err.print(number); return number; } - public static String thumbToFull(String thumb) { + public static String thumbToFull(String thumb, boolean throwException) throws Exception { thumb = thumb.replace("http://th", "http://fc"); List fields = new ArrayList(Arrays.asList(thumb.split("/"))); fields.remove(4); + if (!fields.get(4).equals("f") && throwException) { + // Not a full-size image + logger.warn("Can't get full size image from " + thumb); + throw new Exception("Can't get full size image from " + thumb); + } StringBuilder result = new StringBuilder(); for (int i = 0; i < fields.size(); i++) { if (i > 0) { @@ -130,6 +146,36 @@ public class DeviantartRipper extends AlbumRipper { return result.toString(); } + public String smallToFull(String thumb, String page) { + try { + Response resp = Jsoup.connect(page) + .userAgent(USER_AGENT) + .referrer(this.url.toExternalForm()) + .method(Method.GET) + .execute(); + Map cookies = resp.cookies(); + Elements els = resp.parse().select("a.dev-page-download"); + if (els.size() == 0) { + throw new IOException("no download page found"); + } + String fsimage = els.get(0).attr("href"); + String imageId = fsimage.substring(fsimage.lastIndexOf('-') + 1); + imageId = imageId.substring(0, imageId.indexOf('.')); + long imageIdLong = alphaToLong(imageId); + addURLToDownload(new URL(fsimage), String.format("%010d_", imageIdLong), "", page, cookies); + return null; + } catch (IOException ioe) { + try { + logger.info("Failed to get full size download image at " + page + " : '" + ioe.getMessage() + "'"); + String lessThanFull = thumbToFull(thumb, false); + logger.info("Falling back to less-than-full-size image " + lessThanFull); + return lessThanFull; + } catch (Exception e) { + return null; + } + } + } + @Override public String getHost() { return HOST; diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 03a5d1c6..0340daab 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.0.43"; + private static final String DEFAULT_VERSION = "1.0.44"; private static final String updateJsonURL = "http://rarchives.com/ripme.json"; private static final String updateJarURL = "http://rarchives.com/ripme.jar"; private static final String mainFileName = "ripme.jar";