Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

1.0.44 - Deviantart gets full-sized images when possible #44

Browse Source 
Ripping slows down a lot, but it gets full-size image when there's a
"download" button on the deviantart page.
This commit is contained in:
4pr0n 2014-05-17 09:50:07 -07:00
parent 863f722f9c
commit 567f253491
3 changed files with 57 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@ -4,7 +4,7 @@
<groupId>com.rarchives.ripme</groupId>
<artifactId>ripme</artifactId>
<packaging>jar</packaging>
<version>1.0.43</version>
<version>1.0.44</version>
<name>ripme</name>
<url>http://rip.rarchives.com</url>
<properties>

64
src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
View File

@ -17,6 +17,7 @@ import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
@ -59,21 +60,34 @@ public class DeviantartRipper extends AlbumRipper {
.userAgent(USER_AGENT)
.get();
for (Element thumb : doc.select("div.zones-container a.thumb img")) {
if (thumb.attr("transparent").equals("false")) {
for (Element thumb : doc.select("div.zones-container a.thumb")) {
if (isStopped()) {
break;
}
Element img = thumb.select("img").get(0);
if (img.attr("transparent").equals("false")) {
continue; // a.thumbs to other albums are invisible
}
index++;
String fullSize = thumbToFull(thumb.attr("src"));
String fullSize = null;
try {
fullSize = thumbToFull(img.attr("src"), true);
} catch (Exception e) {
logger.info("Attempting to get full size image from " + thumb.attr("href"));
fullSize = smallToFull(img.attr("src"), thumb.attr("href"));
if (fullSize == null) {
continue;
}
}
try {
URL fullsizeURL = new URL(fullSize);
String imageId = fullSize.substring(fullSize.lastIndexOf('-') + 1);
imageId = imageId.substring(0, imageId.indexOf('.'));
long imageIdLong = alphaToLong(imageId);
index++;
addURLToDownload(fullsizeURL, String.format("%010d_", imageIdLong));
} catch (MalformedURLException e) {
logger.error("[!] Invalid thumbnail image: " + thumbToFull(fullSize));
logger.error("[!] Invalid thumbnail image: " + fullSize);
continue;
}
}
@ -101,7 +115,6 @@ public class DeviantartRipper extends AlbumRipper {
long result = 0;
for (int i = 0; i < alpha.length(); i++) {
result += charToInt(alpha, i);
System.err.println("\t result: " + result);
}
return result;
}
@ -109,17 +122,20 @@ public class DeviantartRipper extends AlbumRipper {
private static int charToInt(String text, int index) {
char c = text.charAt(text.length() - index - 1);
c = Character.toLowerCase(c);
System.err.print(" " + c + ": ");
int number = "0123456789abcdefghijklmnopqrstuvwxyz".indexOf(c);
number *= Math.pow(36, index);
System.err.print(number);
return number;
}
public static String thumbToFull(String thumb) {
public static String thumbToFull(String thumb, boolean throwException) throws Exception {
thumb = thumb.replace("http://th", "http://fc");
List<String> fields = new ArrayList<String>(Arrays.asList(thumb.split("/")));
fields.remove(4);
if (!fields.get(4).equals("f") && throwException) {
// Not a full-size image
logger.warn("Can't get full size image from " + thumb);
throw new Exception("Can't get full size image from " + thumb);
}
StringBuilder result = new StringBuilder();
for (int i = 0; i < fields.size(); i++) {
if (i > 0) {
@ -130,6 +146,36 @@ public class DeviantartRipper extends AlbumRipper {
return result.toString();
}
public String smallToFull(String thumb, String page) {
try {
Response resp = Jsoup.connect(page)
.userAgent(USER_AGENT)
.referrer(this.url.toExternalForm())
.method(Method.GET)
.execute();
Map<String,String> cookies = resp.cookies();
Elements els = resp.parse().select("a.dev-page-download");
if (els.size() == 0) {
throw new IOException("no download page found");
}
String fsimage = els.get(0).attr("href");
String imageId = fsimage.substring(fsimage.lastIndexOf('-') + 1);
imageId = imageId.substring(0, imageId.indexOf('.'));
long imageIdLong = alphaToLong(imageId);
addURLToDownload(new URL(fsimage), String.format("%010d_", imageIdLong), "", page, cookies);
return null;
} catch (IOException ioe) {
try {
logger.info("Failed to get full size download image at " + page + " : '" + ioe.getMessage() + "'");
String lessThanFull = thumbToFull(thumb, false);
logger.info("Falling back to less-than-full-size image " + lessThanFull);
return lessThanFull;
} catch (Exception e) {
return null;
}
}
}
@Override
public String getHost() {
return HOST;

2
src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
View File

@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils;
public class UpdateUtils {
private static final Logger logger = Logger.getLogger(UpdateUtils.class);
private static final String DEFAULT_VERSION = "1.0.43";
private static final String DEFAULT_VERSION = "1.0.44";
private static final String updateJsonURL = "http://rarchives.com/ripme.json";
private static final String updateJarURL = "http://rarchives.com/ripme.jar";
private static final String mainFileName = "ripme.jar";