Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fixed furaffinity ripper. #84

Browse Source 
Logs in correctly via https.
Spaces instead of tabs.
No more warnings.
This commit is contained in:
4pr0n 2014-06-27 23:20:52 -07:00
parent 9e40e55ee9
commit b0e5fa8cb0
1 changed files with 118 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

239
src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java
View File

@ -4,16 +4,12 @@ import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import javax.swing.JOptionPane;
import javax.swing.JPasswordField;
import org.jsoup.Connection.Method; import org.jsoup.Connection.Method;
import org.jsoup.Connection.Response; import org.jsoup.Connection.Response;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
@ -22,146 +18,143 @@ import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.ripper.DownloadThreadPool;
import com.rarchives.ripme.utils.Base64;
import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Http;
public class FuraffinityRipper extends AbstractHTMLRipper { public class FuraffinityRipper extends AbstractHTMLRipper {
static Map<String, String> cookies=null; static Map<String, String> cookies=null;
static final String urlBase = "http://www.furaffinity.net"; static final String urlBase = "https://www.furaffinity.net";
// Thread pool for finding direct image links from "image" pages (html) // Thread pool for finding direct image links from "image" pages (html)
private DownloadThreadPool furaffinityThreadPool = new DownloadThreadPool( private DownloadThreadPool furaffinityThreadPool
"furaffinity"); = new DownloadThreadPool( "furaffinity");
@Override @Override
public DownloadThreadPool getThreadPool() { public DownloadThreadPool getThreadPool() {
return furaffinityThreadPool; return furaffinityThreadPool;
} }
public FuraffinityRipper(URL url) throws IOException { public FuraffinityRipper(URL url) throws IOException {
super(url); super(url);
} }
@Override @Override
public String getDomain() { public String getDomain() {
return "furaffinity.net"; return "furaffinity.net";
} }
@Override @Override
public String getHost() { public String getHost() {
return "furaffinity"; return "furaffinity";
} }
@Override @Override
public Document getFirstPage() throws IOException { public Document getFirstPage() throws IOException {
if (cookies == null || cookies.size() == 0) { if (cookies == null || cookies.size() == 0) {
JPasswordField passwordField=new JPasswordField(); login();
String user = JOptionPane.showInputDialog("Username"); }
JOptionPane.showMessageDialog(null,passwordField,"Password",JOptionPane.QUESTION_MESSAGE|JOptionPane.OK_OPTION);
String pass = Arrays.toString(passwordField.getPassword());
Response loginPage=Http.url(urlBase+"/login/") return Http.url(url).cookies(cookies).get();
.referrer(urlBase) }
.response();
cookies=loginPage.cookies();
System.out.println("Cookies: "+cookies);
Map<String,String> formData=new HashMap<String,String>(); private void login() throws IOException {
formData.put("action", "login"); String user = new String(Base64.decode("cmlwbWU="));
formData.put("retard_protection", "1"); String pass = new String(Base64.decode("cmlwbWVwYXNzd29yZA=="));
formData.put("name", user);
formData.put("pass", pass);
formData.put("login", "Login to FurAffinity");
Response doLogin=Http.url(urlBase+"/login/?ref=http://www.furaffinity.net/") Response loginPage = Http.url(urlBase + "/login/")
.referrer(urlBase+"/login/") .referrer(urlBase)
.cookies(cookies) .response();
.data(formData) cookies = loginPage.cookies();
.method(Method.POST)
.response();
cookies.putAll(doLogin.cookies());
System.out.println("Cookies: "+cookies);
}
return Http.url(url).cookies(cookies).get(); Map<String,String> formData = new HashMap<String,String>();
} formData.put("action", "login");
formData.put("retard_protection", "1");
formData.put("name", user);
formData.put("pass", pass);
formData.put("login", "Login to FurAffinity");
@Override Response doLogin = Http.url(urlBase + "/login/?ref=" + url)
public Document getNextPage(Document doc) throws IOException { .referrer(urlBase + "/login/")
// Find next page .cookies(cookies)
Elements nextPageUrl = doc.select("td[align=right] form"); .data(formData)
String nextUrl = urlBase+nextPageUrl.first().attr("action"); .method(Method.POST)
if (nextPageUrl.size() == 0) { .response();
throw new IOException("No more pages"); cookies.putAll(doLogin.cookies());
} }
sleep(500);
Document nextPage = Http.url(nextUrl).cookies(cookies).get();
Elements hrefs = nextPage.select("div#no-images"); @Override
if (hrefs.size() != 0) { public Document getNextPage(Document doc) throws IOException {
throw new IOException("No more pages"); // Find next page
} Elements nextPageUrl = doc.select("td[align=right] form");
return nextPage; String nextUrl = urlBase+nextPageUrl.first().attr("action");
} if (nextPageUrl.size() == 0) {
throw new IOException("No more pages");
}
sleep(500);
Document nextPage = Http.url(nextUrl).cookies(cookies).get();
@Override Elements hrefs = nextPage.select("div#no-images");
public List<String> getURLsFromPage(Document page) { if (hrefs.size() != 0) {
List<String> urls = new ArrayList<String>(); throw new IOException("No more pages");
Elements urlElements = page.select("b[id^=sid_]"); }
for (Element e : urlElements) { return nextPage;
urls.add(urlBase + e.select("a").first().attr("href")); }
}
return urls;
}
@Override @Override
public void downloadURL(URL url, int index) { public List<String> getURLsFromPage(Document page) {
furaffinityThreadPool.addThread(new FuraffinityDocumentThread(url, List<String> urls = new ArrayList<String>();
index)); Elements urlElements = page.select("b[id^=sid_]");
sleep(250); for (Element e : urlElements) {
} urls.add(urlBase + e.select("a").first().attr("href"));
}
return urls;
}
@Override @Override
public String getGID(URL url) throws MalformedURLException { public void downloadURL(URL url, int index) {
Pattern p = Pattern furaffinityThreadPool.addThread(new FuraffinityDocumentThread(url));
.compile("^https?://www\\.furaffinity\\.net/gallery/([-_.0-9a-zA-Z]+).*$"); sleep(250);
Matcher m = p.matcher(url.toExternalForm()); }
if (m.matches()) {
return m.group(1);
}
throw new MalformedURLException("Expected furaffinity.net URL format: "
+ "www.furaffinity.net/gallery/username - got " + url
+ " instead");
}
private class FuraffinityDocumentThread extends Thread { @Override
private URL url; public String getGID(URL url) throws MalformedURLException {
private int index; Pattern p = Pattern
.compile("^https?://www\\.furaffinity\\.net/gallery/([-_.0-9a-zA-Z]+).*$");
Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) {
return m.group(1);
}
throw new MalformedURLException("Expected furaffinity.net URL format: "
+ "www.furaffinity.net/gallery/username - got " + url
+ " instead");
}
public FuraffinityDocumentThread(URL url, int index) { private class FuraffinityDocumentThread extends Thread {
super(); private URL url;
this.url = url;
this.index = index;
}
@Override public FuraffinityDocumentThread(URL url) {
public void run() { super();
try { this.url = url;
Document doc = Http.url(url).cookies(cookies).get(); }
// Find image
Elements donwloadLink = doc.select("div.alt1 b a[href^=//d.facdn.net/]"); @Override
if (donwloadLink.size() == 0) { public void run() {
logger.warn("Could not download " + this.url); try {
return; Document doc = Http.url(url).cookies(cookies).get();
} // Find image
String link = "http:" + donwloadLink.first().attr("href"); Elements donwloadLink = doc.select("div.alt1 b a[href^=//d.facdn.net/]");
logger.info("Found URL " + link); if (donwloadLink.size() == 0) {
addURLToDownload(new URL(link),"","",url.toExternalForm(),cookies); logger.warn("Could not download " + this.url);
} catch (IOException e) { return;
logger.error("[!] Exception while loading/parsing " + this.url, }
e); String link = "http:" + donwloadLink.first().attr("href");
} logger.info("Found URL " + link);
} addURLToDownload(new URL(link),"","",url.toExternalForm(),cookies);
} } catch (IOException e) {
logger.error("[!] Exception while loading/parsing " + this.url, e);
}
}
}
} }