2014-03-13 10:26:55 +01:00
|
|
|
package com.rarchives.ripme.ripper.rippers;
|
|
|
|
|
|
|
|
import java.io.IOException;
|
|
|
|
import java.net.MalformedURLException;
|
|
|
|
import java.net.URL;
|
|
|
|
import java.util.ArrayList;
|
|
|
|
import java.util.Arrays;
|
|
|
|
import java.util.HashMap;
|
|
|
|
import java.util.List;
|
|
|
|
import java.util.Map;
|
|
|
|
import java.util.regex.Matcher;
|
|
|
|
import java.util.regex.Pattern;
|
|
|
|
|
|
|
|
import org.apache.log4j.Logger;
|
|
|
|
import org.jsoup.Connection.Method;
|
|
|
|
import org.jsoup.Connection.Response;
|
|
|
|
import org.jsoup.Jsoup;
|
|
|
|
import org.jsoup.nodes.Document;
|
|
|
|
import org.jsoup.nodes.Element;
|
|
|
|
|
2014-04-20 07:41:11 +02:00
|
|
|
import com.rarchives.ripme.ripper.AlbumRipper;
|
2014-03-13 10:26:55 +01:00
|
|
|
import com.rarchives.ripme.utils.Utils;
|
|
|
|
|
2014-04-20 07:41:11 +02:00
|
|
|
public class DeviantartRipper extends AlbumRipper {
|
2014-03-13 10:26:55 +01:00
|
|
|
|
|
|
|
private static final String DOMAIN = "deviantart.com",
|
|
|
|
HOST = "deviantart";
|
|
|
|
|
|
|
|
private static final int SLEEP_TIME = 2000;
|
|
|
|
private static final Logger logger = Logger.getLogger(DeviantartRipper.class);
|
|
|
|
|
|
|
|
public DeviantartRipper(URL url) throws IOException {
|
|
|
|
super(url);
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public boolean canRip(URL url) {
|
|
|
|
return url.getHost().endsWith(DOMAIN);
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public URL sanitizeURL(URL url) throws MalformedURLException {
|
|
|
|
String u = url.toExternalForm();
|
|
|
|
u = u.replaceAll("\\?.*", "");
|
|
|
|
return new URL(u);
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public void rip() throws IOException {
|
|
|
|
int index = 0;
|
|
|
|
String nextURL = this.url.toExternalForm();
|
|
|
|
while (nextURL != null) {
|
|
|
|
logger.info(" Retrieving " + nextURL);
|
|
|
|
Document doc = Jsoup.connect(nextURL)
|
|
|
|
.userAgent(USER_AGENT)
|
|
|
|
.get();
|
|
|
|
try {
|
|
|
|
Thread.sleep(SLEEP_TIME);
|
|
|
|
} catch (InterruptedException e) {
|
|
|
|
logger.error("[!] Interrupted while waiting for page to load", e);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
for (Element thumb : doc.select("a.thumb img")) {
|
|
|
|
String fullSize = thumbToFull(thumb.attr("src"));
|
|
|
|
URL pageURL;
|
|
|
|
try {
|
|
|
|
pageURL = new URL(fullSize);
|
|
|
|
} catch (MalformedURLException e) {
|
|
|
|
logger.error("[!] Invalid thumbnail image: " + thumbToFull(fullSize));
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
index++;
|
|
|
|
addURLToDownload(pageURL, String.format("%03d_", index));
|
|
|
|
}
|
|
|
|
nextURL = null;
|
|
|
|
for (Element nextButton : doc.select("a.away")) {
|
|
|
|
if (nextButton.attr("href").contains("offset=" + index)) {
|
|
|
|
nextURL = this.url.toExternalForm() + "?offset=" + index;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
waitForThreads();
|
|
|
|
}
|
|
|
|
|
|
|
|
public static String thumbToFull(String thumb) {
|
|
|
|
thumb = thumb.replace("http://th", "http://fc");
|
|
|
|
List<String> fields = new ArrayList<String>(Arrays.asList(thumb.split("/")));
|
|
|
|
fields.remove(4);
|
|
|
|
StringBuilder result = new StringBuilder();
|
|
|
|
for (int i = 0; i < fields.size(); i++) {
|
|
|
|
if (i > 0) {
|
|
|
|
result.append("/");
|
|
|
|
}
|
|
|
|
result.append(fields.get(i));
|
|
|
|
}
|
|
|
|
return result.toString();
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public String getHost() {
|
|
|
|
return HOST;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public String getGID(URL url) throws MalformedURLException {
|
|
|
|
Pattern p = Pattern.compile("^https?://([a-zA-Z0-9\\-]{1,})\\.deviantart\\.com(/gallery)?/?$");
|
|
|
|
Matcher m = p.matcher(url.toExternalForm());
|
|
|
|
if (m.matches()) {
|
|
|
|
// Root gallery
|
|
|
|
return m.group(1);
|
|
|
|
}
|
|
|
|
p = Pattern.compile("^https?://([a-zA-Z0-9\\-]{1,})\\.deviantart\\.com/gallery/([0-9]{1,}).*$");
|
|
|
|
m = p.matcher(url.toExternalForm());
|
|
|
|
if (m.matches()) {
|
|
|
|
// Subgallery
|
|
|
|
return m.group(1) + "_" + m.group(2);
|
|
|
|
}
|
|
|
|
throw new MalformedURLException("Expected URL format: http://username.deviantart.com/[/gallery/#####], got: " + url);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Logs into deviant art. Not required to rip NSFW images.
|
|
|
|
* @return Map of cookies containing session data.
|
|
|
|
*/
|
|
|
|
@SuppressWarnings("unused")
|
|
|
|
private Map<String, String> loginToDeviantart() throws IOException {
|
|
|
|
// Populate postData fields
|
|
|
|
Map<String,String> postData = new HashMap<String,String>();
|
|
|
|
String username = Utils.getConfigString("deviantart.username", null);
|
|
|
|
String password = Utils.getConfigString("deviantart.password", null);
|
|
|
|
if (username == null || password == null) {
|
|
|
|
throw new IOException("could not find username or password in config");
|
|
|
|
}
|
|
|
|
Response resp = Jsoup.connect("http://www.deviantart.com/")
|
|
|
|
.userAgent(USER_AGENT)
|
|
|
|
.method(Method.GET)
|
|
|
|
.execute();
|
|
|
|
for (Element input : resp.parse().select("form#form-login input[type=hidden]")) {
|
|
|
|
postData.put(input.attr("name"), input.attr("value"));
|
|
|
|
}
|
|
|
|
postData.put("username", username);
|
|
|
|
postData.put("password", password);
|
|
|
|
postData.put("remember_me", "1");
|
|
|
|
|
|
|
|
// Send login request
|
|
|
|
resp = Jsoup.connect("https://www.deviantart.com/users/login")
|
|
|
|
.userAgent(USER_AGENT)
|
|
|
|
.data(postData)
|
|
|
|
.cookies(resp.cookies())
|
|
|
|
.method(Method.POST)
|
|
|
|
.execute();
|
|
|
|
|
|
|
|
// Assert we are logged in
|
|
|
|
if (resp.hasHeader("Location") && resp.header("Location").contains("password")) {
|
|
|
|
// Wrong password
|
|
|
|
throw new IOException("Wrong pasword");
|
|
|
|
}
|
|
|
|
if (resp.url().toExternalForm().contains("bad_form")) {
|
|
|
|
throw new IOException("Login form was incorrectly submitted");
|
|
|
|
}
|
|
|
|
if (resp.cookie("auth_secure") == null ||
|
|
|
|
resp.cookie("auth") == null) {
|
|
|
|
throw new IOException("No auth_secure or auth cookies received");
|
|
|
|
}
|
|
|
|
// We are logged in, save the cookies
|
|
|
|
return resp.cookies();
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|