From ac04bb9053b73bfc4419075d847ea4a32e4ab070 Mon Sep 17 00:00:00 2001 From: Daniel Schulte Date: Fri, 27 Jun 2014 17:18:09 +0200 Subject: [PATCH 1/3] Furaffinity ripper --- .../ripper/rippers/FuraffinityRipper.java | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java new file mode 100644 index 00000000..7ddebe1d --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java @@ -0,0 +1,143 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.swing.JOptionPane; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.ripper.DownloadThreadPool; +import com.rarchives.ripme.utils.Http; + +public class FuraffinityRipper extends AbstractHTMLRipper { + + static Map cookies; + static final String urlBase = "http://www.furaffinity.net"; + + // Thread pool for finding direct image links from "image" pages (html) + private DownloadThreadPool furaffinityThreadPool = new DownloadThreadPool( + "furaffinity"); + + @Override + public DownloadThreadPool getThreadPool() { + return furaffinityThreadPool; + } + + public FuraffinityRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getDomain() { + return "furaffinity.net"; + } + + @Override + public String getHost() { + return "furaffinity"; + } + + @Override + public Document getFirstPage() throws IOException { + if (cookies == null || cookies.size() == 0) { + String a = JOptionPane.showInputDialog("Value of a-cookie"); + String b = JOptionPane.showInputDialog("Value of b-cookie"); + + cookies = new HashMap(); + cookies.put("a", a); + cookies.put("b", b); + } + + return Http.url(url).cookies(cookies).get(); + } + + @Override + public Document getNextPage(Document doc) throws IOException { + // Find next page + Elements nextPageUrl = doc.select("td[align=right] form"); + String nextUrl = urlBase+nextPageUrl.first().attr("action"); + if (nextPageUrl.size() == 0) { + throw new IOException("No more pages"); + } + sleep(500); + Document nextPage = Http.url(nextUrl).cookies(cookies).get(); + + Elements hrefs = nextPage.select("div#no-images"); + if (hrefs.size() != 0) { + throw new IOException("No more pages"); + } + return nextPage; + } + + @Override + public List getURLsFromPage(Document page) { + List urls = new ArrayList(); + Elements urlElements = page.select("b[id^=sid_]"); + for (Element e : urlElements) { + urls.add(urlBase + e.select("a").first().attr("href")); + } + return urls; + } + + @Override + public void downloadURL(URL url, int index) { + furaffinityThreadPool.addThread(new FuraffinityDocumentThread(url, + index)); + sleep(250); + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern + .compile("^https?://www\\.furaffinity\\.net/gallery/([-_.0-9a-zA-Z]+).*$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected furaffinity.net URL format: " + + "www.furaffinity.net/gallery/username - got " + url + + " instead"); + } + + private class FuraffinityDocumentThread extends Thread { + private URL url; + private int index; + + public FuraffinityDocumentThread(URL url, int index) { + super(); + this.url = url; + this.index = index; + } + + @Override + public void run() { + try { + Document doc = Http.url(url).cookies(cookies).get(); + // Find image + Elements donwloadLink = doc.select("div.alt1 b a[href^=//d.facdn.net/]"); + if (donwloadLink.size() == 0) { + logger.warn("Could not download " + this.url); + return; + } + String link = "http:" + donwloadLink.first().attr("href"); + logger.info("Found URL " + link); + addURLToDownload(new URL(link),"","",url.toExternalForm(),cookies); + } catch (IOException e) { + logger.error("[!] Exception while loading/parsing " + this.url, + e); + } + } + } + +} From 5428cb899165ce5995bf4ce3c63a24090cb0ae85 Mon Sep 17 00:00:00 2001 From: Daniel Schulte Date: Fri, 27 Jun 2014 22:40:03 +0200 Subject: [PATCH 2/3] Trying to changed code to login the user instead of manually supplying cookie values. Currently the "a" cookie does not get captured for some reason. --- .../ripper/rippers/FuraffinityRipper.java | 36 +++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java index 7ddebe1d..c12c41dc 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -11,7 +12,10 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.swing.JOptionPane; +import javax.swing.JPasswordField; +import org.jsoup.Connection.Method; +import org.jsoup.Connection.Response; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; @@ -22,7 +26,7 @@ import com.rarchives.ripme.utils.Http; public class FuraffinityRipper extends AbstractHTMLRipper { - static Map cookies; + static Map cookies=null; static final String urlBase = "http://www.furaffinity.net"; // Thread pool for finding direct image links from "image" pages (html) @@ -51,12 +55,32 @@ public class FuraffinityRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { if (cookies == null || cookies.size() == 0) { - String a = JOptionPane.showInputDialog("Value of a-cookie"); - String b = JOptionPane.showInputDialog("Value of b-cookie"); + JPasswordField passwordField=new JPasswordField(); + String user = JOptionPane.showInputDialog("Username"); + JOptionPane.showMessageDialog(null,passwordField,"Password",JOptionPane.QUESTION_MESSAGE|JOptionPane.OK_OPTION); + String pass = Arrays.toString(passwordField.getPassword()); - cookies = new HashMap(); - cookies.put("a", a); - cookies.put("b", b); + Response loginPage=Http.url(urlBase+"/login/") + .referrer(urlBase) + .response(); + cookies=loginPage.cookies(); + System.out.println("Cookies: "+cookies); + + Map formData=new HashMap(); + formData.put("action", "login"); + formData.put("retard_protection", "1"); + formData.put("name", user); + formData.put("pass", pass); + formData.put("login", "Login to FurAffinity"); + + Response doLogin=Http.url(urlBase+"/login/") + .referrer(urlBase+"/login/") + .cookies(cookies) + .data(formData) + .method(Method.POST) + .response(); + cookies.putAll(doLogin.cookies()); + System.out.println("Cookies: "+cookies); } return Http.url(url).cookies(cookies).get(); From cce85e99cd3cb310e8b1852d7f509119a98bea99 Mon Sep 17 00:00:00 2001 From: Daniel Schulte Date: Fri, 27 Jun 2014 22:44:27 +0200 Subject: [PATCH 3/3] Trying to hunt down the cookie bug --- .../com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java index c12c41dc..aa0e07a7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java @@ -73,7 +73,7 @@ public class FuraffinityRipper extends AbstractHTMLRipper { formData.put("pass", pass); formData.put("login", "Login to FurAffinity"); - Response doLogin=Http.url(urlBase+"/login/") + Response doLogin=Http.url(urlBase+"/login/?ref=http://www.furaffinity.net/") .referrer(urlBase+"/login/") .cookies(cookies) .data(formData)