From 91ef1c1b31232c1822fe3359c3822c5350f24555 Mon Sep 17 00:00:00 2001 From: 4pr0n Date: Sun, 3 Aug 2014 12:45:11 -0700 Subject: [PATCH] 1.0.82 - Fix 2dgalleries ripper Closes #96 --- pom.xml | 2 +- .../ripper/rippers/TwodgalleriesRipper.java | 56 ++++++++++++++++--- .../com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 49 insertions(+), 11 deletions(-) diff --git a/pom.xml b/pom.xml index 937eca2f..d07feb2f 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.0.82 + 1.0.83 ripme http://rip.rarchives.com diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TwodgalleriesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TwodgalleriesRipper.java index 8be2d191..c48a11c3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TwodgalleriesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TwodgalleriesRipper.java @@ -4,19 +4,25 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.jsoup.Connection.Method; +import org.jsoup.Connection.Response; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Base64; import com.rarchives.ripme.utils.Http; public class TwodgalleriesRipper extends AbstractHTMLRipper { private int offset = 0; + private Map cookies = new HashMap(); public TwodgalleriesRipper(URL url) throws IOException { super(url); @@ -35,36 +41,48 @@ public class TwodgalleriesRipper extends AbstractHTMLRipper { public String getGID(URL url) throws MalformedURLException { Pattern p; Matcher m; - p = Pattern.compile("^.*2dgalleries.com/browse/profile\\?id=([0-9]+).*$"); + p = Pattern.compile("^.*2dgalleries.com/artist/([a-zA-Z0-9\\-]+).*$"); m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); } throw new MalformedURLException( "Expected 2dgalleries.com album format: " - + "2dgalleries.com/browse/profile?id=####" + + "2dgalleries.com/artist/..." + " Got: " + url); } private String getURL(String userid, int offset) { - return "http://en.2dgalleries.com/browse/user-artworks?uid=" + userid + return "http://en.2dgalleries.com/artist/" + userid + + "?timespan=4" + + "&order=1" + + "&catid=2" + "&offset=" + offset - + "&ajx=1&pager=1&hr=1&pid=" + userid; + + "&ajx=1&pager=1"; } @Override public Document getFirstPage() throws IOException { + try { + login(); + } catch (IOException e) { + logger.error("Failed to login", e); + } String url = getURL(getGID(this.url), offset); - return Http.url(url).get(); + return Http.url(url) + .cookies(cookies) + .get(); } @Override public Document getNextPage(Document doc) throws IOException { - offset += 3; + offset += 24; String url = getURL(getGID(this.url), offset); sleep(500); - Document nextDoc = Http.url(url).get(); - if (nextDoc.select(".noartwork").size() > 0) { + Document nextDoc = Http.url(url) + .cookies(cookies) + .get(); + if (nextDoc.select("div.hcaption > img").size() == 0) { throw new IOException("No more images to retrieve"); } return nextDoc; @@ -73,7 +91,7 @@ public class TwodgalleriesRipper extends AbstractHTMLRipper { @Override public List getURLsFromPage(Document doc) { List imageURLs = new ArrayList(); - for (Element thumb : doc.select("img")) { + for (Element thumb : doc.select("div.hcaption > img")) { String image = thumb.attr("src"); image = image.replace("/200H/", "/"); if (image.startsWith("//")) { @@ -90,4 +108,24 @@ public class TwodgalleriesRipper extends AbstractHTMLRipper { public void downloadURL(URL url, int index) { addURLToDownload(url, getPrefix(index)); } + + private void login() throws IOException { + Response resp = Http.url(this.url).response(); + cookies = resp.cookies(); + String ctoken = resp.parse().select("form > input[name=ctoken]").first().attr("value"); + + Map postdata = new HashMap(); + postdata.put("user[login]", new String(Base64.decode("cmlwbWU="))); + postdata.put("user[password]", new String(Base64.decode("cmlwcGVy"))); + postdata.put("rememberme", "1"); + postdata.put("ctoken", ctoken); + + resp = Http.url("http://en.2dgalleries.com/account/login") + .referrer("http://en.2dgalleries.com/") + .cookies(cookies) + .data(postdata) + .method(Method.POST) + .response(); + cookies = resp.cookies(); + } } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 381695ad..888e86f3 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.0.82"; + private static final String DEFAULT_VERSION = "1.0.83"; private static final String updateJsonURL = "http://rarchives.com/ripme.json"; private static final String updateJarURL = "http://rarchives.com/ripme.jar"; private static final String mainFileName = "ripme.jar";