Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Motherless ripper can rip search terms / tags

Browse Source
This commit is contained in:
4pr0n 2014-05-25 20:10:21 -07:00
parent b800200304
commit 7beb6a402d
1 changed files with 24 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java
View File

@ -39,22 +39,23 @@ public class MotherlessRipper extends AlbumRipper {
@Override @Override
public URL sanitizeURL(URL url) throws MalformedURLException { public URL sanitizeURL(URL url) throws MalformedURLException {
String gid = getGID(url); return url;
URL newURL = new URL("http://motherless.com/G" + gid);
logger.debug("Sanitized URL from " + url + " to " + newURL);
return newURL;
} }
@Override @Override
public String getGID(URL url) throws MalformedURLException { public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^https?://(www\\.)?motherless\\.com/G([MVI]?[A-F0-9]{6,8}).*$"); Pattern p = Pattern.compile("^https?://(www\\.)?motherless\\.com/G([MVI]?[A-F0-9]{6,8}).*$");
Matcher m = p.matcher(url.toExternalForm()); Matcher m = p.matcher(url.toExternalForm());
System.err.println(url.toExternalForm()); if (m.matches()) {
if (!m.matches()) {
throw new MalformedURLException("Expected URL format: http://motherless.com/GIXXXXXXX, got: " + url);
}
return m.group(m.groupCount()); return m.group(m.groupCount());
} }
p = Pattern.compile("^https?://(www\\.)?motherless\\.com/term/(images/|videos/)([a-zA-Z0-9%]+)$");
m = p.matcher(url.toExternalForm());
if (m.matches()) {
return m.group(m.groupCount());
}
throw new MalformedURLException("Expected URL format: http://motherless.com/GIXXXXXXX, got: " + url);
}
@Override @Override
public void rip() throws IOException { public void rip() throws IOException {
@ -64,9 +65,21 @@ public class MotherlessRipper extends AlbumRipper {
logger.info(" Retrieving " + nextURL); logger.info(" Retrieving " + nextURL);
Document doc = Jsoup.connect(nextURL) Document doc = Jsoup.connect(nextURL)
.userAgent(USER_AGENT) .userAgent(USER_AGENT)
.timeout(5000)
.referrer("http://motherless.com")
.get(); .get();
for (Element thumb : doc.select("div.thumb a.img-container")) { for (Element thumb : doc.select("div.thumb a.img-container")) {
URL url = new URL("http://" + DOMAIN + thumb.attr("href")); String thumbURL = thumb.attr("href");
if (thumbURL.contains("pornmd.com")) {
continue;
}
URL url;
if (!thumbURL.startsWith("http")) {
url = new URL("http://" + DOMAIN + thumbURL);
}
else {
url = new URL(thumbURL);
}
index += 1; index += 1;
// Create thread for finding image at "url" page // Create thread for finding image at "url" page
MotherlessImageThread mit = new MotherlessImageThread(url, index); MotherlessImageThread mit = new MotherlessImageThread(url, index);
@ -101,6 +114,8 @@ public class MotherlessRipper extends AlbumRipper {
try { try {
Document doc = Jsoup.connect(this.url.toExternalForm()) Document doc = Jsoup.connect(this.url.toExternalForm())
.userAgent(USER_AGENT) .userAgent(USER_AGENT)
.timeout(5000)
.referrer(this.url.toExternalForm())
.get(); .get();
Pattern p = Pattern.compile("^.*__fileurl = '([^']{1,})';.*$", Pattern.DOTALL); Pattern p = Pattern.compile("^.*__fileurl = '([^']{1,})';.*$", Pattern.DOTALL);
Matcher m = p.matcher(doc.outerHtml()); Matcher m = p.matcher(doc.outerHtml());