Deviantart support
This commit is contained in:
4pr0n
parent
d8294568a6
commit
b662b46a44
@ -99,7 +99,7 @@ public abstract class AbstractRipper
|
||||
|| itemsCompleted.containsKey(url)
|
||||
|| itemsErrored.containsKey(url)) {
|
||||
// Item is already downloaded/downloading, skip it.
|
||||
logger.info(" Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs));
|
||||
logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs));
|
||||
return;
|
||||
}
|
||||
itemsPending.put(url, saveAs);
|
||||
|
||||
@ -0,0 +1,169 @@
|
||||
package com.rarchives.ripme.ripper.rippers;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
import org.jsoup.Connection.Method;
|
||||
import org.jsoup.Connection.Response;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
|
||||
import com.rarchives.ripme.ripper.AbstractRipper;
|
||||
import com.rarchives.ripme.utils.Utils;
|
||||
|
||||
public class DeviantartRipper extends AbstractRipper {
|
||||
|
||||
private static final String DOMAIN = "deviantart.com",
|
||||
HOST = "deviantart";
|
||||
|
||||
private static final int SLEEP_TIME = 2000;
|
||||
private static final Logger logger = Logger.getLogger(DeviantartRipper.class);
|
||||
|
||||
public DeviantartRipper(URL url) throws IOException {
|
||||
super(url);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean canRip(URL url) {
|
||||
return url.getHost().endsWith(DOMAIN);
|
||||
}
|
||||
|
||||
@Override
|
||||
public URL sanitizeURL(URL url) throws MalformedURLException {
|
||||
String u = url.toExternalForm();
|
||||
u = u.replaceAll("\\?.*", "");
|
||||
return new URL(u);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void rip() throws IOException {
|
||||
int index = 0;
|
||||
String nextURL = this.url.toExternalForm();
|
||||
while (nextURL != null) {
|
||||
logger.info(" Retrieving " + nextURL);
|
||||
Document doc = Jsoup.connect(nextURL)
|
||||
.userAgent(USER_AGENT)
|
||||
.get();
|
||||
try {
|
||||
Thread.sleep(SLEEP_TIME);
|
||||
} catch (InterruptedException e) {
|
||||
logger.error("[!] Interrupted while waiting for page to load", e);
|
||||
break;
|
||||
}
|
||||
for (Element thumb : doc.select("a.thumb img")) {
|
||||
String fullSize = thumbToFull(thumb.attr("src"));
|
||||
URL pageURL;
|
||||
try {
|
||||
pageURL = new URL(fullSize);
|
||||
} catch (MalformedURLException e) {
|
||||
logger.error("[!] Invalid thumbnail image: " + thumbToFull(fullSize));
|
||||
continue;
|
||||
}
|
||||
index++;
|
||||
addURLToDownload(pageURL, String.format("%03d_", index));
|
||||
}
|
||||
nextURL = null;
|
||||
for (Element nextButton : doc.select("a.away")) {
|
||||
if (nextButton.attr("href").contains("offset=" + index)) {
|
||||
nextURL = this.url.toExternalForm() + "?offset=" + index;
|
||||
}
|
||||
}
|
||||
}
|
||||
waitForThreads();
|
||||
}
|
||||
|
||||
public static String thumbToFull(String thumb) {
|
||||
thumb = thumb.replace("http://th", "http://fc");
|
||||
List<String> fields = new ArrayList<String>(Arrays.asList(thumb.split("/")));
|
||||
fields.remove(4);
|
||||
StringBuilder result = new StringBuilder();
|
||||
for (int i = 0; i < fields.size(); i++) {
|
||||
if (i > 0) {
|
||||
result.append("/");
|
||||
}
|
||||
result.append(fields.get(i));
|
||||
}
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
return HOST;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getGID(URL url) throws MalformedURLException {
|
||||
Pattern p = Pattern.compile("^https?://([a-zA-Z0-9\\-]{1,})\\.deviantart\\.com(/gallery)?/?$");
|
||||
Matcher m = p.matcher(url.toExternalForm());
|
||||
if (m.matches()) {
|
||||
// Root gallery
|
||||
return m.group(1);
|
||||
}
|
||||
p = Pattern.compile("^https?://([a-zA-Z0-9\\-]{1,})\\.deviantart\\.com/gallery/([0-9]{1,}).*$");
|
||||
m = p.matcher(url.toExternalForm());
|
||||
if (m.matches()) {
|
||||
// Subgallery
|
||||
return m.group(1) + "_" + m.group(2);
|
||||
}
|
||||
throw new MalformedURLException("Expected URL format: http://username.deviantart.com/[/gallery/#####], got: " + url);
|
||||
}
|
||||
|
||||
/**
|
||||
* Logs into deviant art. Not required to rip NSFW images.
|
||||
* @return Map of cookies containing session data.
|
||||
*/
|
||||
@SuppressWarnings("unused")
|
||||
private Map<String, String> loginToDeviantart() throws IOException {
|
||||
// Populate postData fields
|
||||
Map<String,String> postData = new HashMap<String,String>();
|
||||
String username = Utils.getConfigString("deviantart.username", null);
|
||||
String password = Utils.getConfigString("deviantart.password", null);
|
||||
if (username == null || password == null) {
|
||||
throw new IOException("could not find username or password in config");
|
||||
}
|
||||
Response resp = Jsoup.connect("http://www.deviantart.com/")
|
||||
.userAgent(USER_AGENT)
|
||||
.method(Method.GET)
|
||||
.execute();
|
||||
for (Element input : resp.parse().select("form#form-login input[type=hidden]")) {
|
||||
postData.put(input.attr("name"), input.attr("value"));
|
||||
}
|
||||
postData.put("username", username);
|
||||
postData.put("password", password);
|
||||
postData.put("remember_me", "1");
|
||||
|
||||
// Send login request
|
||||
resp = Jsoup.connect("https://www.deviantart.com/users/login")
|
||||
.userAgent(USER_AGENT)
|
||||
.data(postData)
|
||||
.cookies(resp.cookies())
|
||||
.method(Method.POST)
|
||||
.execute();
|
||||
|
||||
// Assert we are logged in
|
||||
if (resp.hasHeader("Location") && resp.header("Location").contains("password")) {
|
||||
// Wrong password
|
||||
throw new IOException("Wrong pasword");
|
||||
}
|
||||
if (resp.url().toExternalForm().contains("bad_form")) {
|
||||
throw new IOException("Login form was incorrectly submitted");
|
||||
}
|
||||
if (resp.cookie("auth_secure") == null ||
|
||||
resp.cookie("auth") == null) {
|
||||
throw new IOException("No auth_secure or auth cookies received");
|
||||
}
|
||||
// We are logged in, save the cookies
|
||||
return resp.cookies();
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
package com.rarchives.ripme.tst.ripper.rippers;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.rarchives.ripme.ripper.rippers.DeviantartRipper;
|
||||
|
||||
public class DeviantartRipperTest extends RippersTest {
|
||||
|
||||
public void testDeviantartAlbums() throws IOException {
|
||||
if (!DOWNLOAD_CONTENT) {
|
||||
return;
|
||||
}
|
||||
List<URL> contentURLs = new ArrayList<URL>();
|
||||
|
||||
// Small gallery
|
||||
contentURLs.add(new URL("http://airgee.deviantart.com/gallery/"));
|
||||
// NSFW gallery
|
||||
contentURLs.add(new URL("http://faterkcx.deviantart.com/gallery/"));
|
||||
// Multi-page NSFW
|
||||
contentURLs.add(new URL("http://geekysica.deviantart.com/gallery/35209412"));
|
||||
|
||||
for (URL url : contentURLs) {
|
||||
try {
|
||||
DeviantartRipper ripper = new DeviantartRipper(url);
|
||||
ripper.rip();
|
||||
assert(ripper.getWorkingDir().listFiles().length > 1);
|
||||
deleteDir(ripper.getWorkingDir());
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
fail("Error while ripping URL " + url + ": " + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user