diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FapprovedRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FapprovedRipper.java index dd423691..f33d40ce 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FapprovedRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FapprovedRipper.java @@ -1,101 +1,101 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; - -public class FapprovedRipper extends AbstractHTMLRipper { - - private int pageIndex = 1; - private String username = null; - - public FapprovedRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return "fapproved"; - } - @Override - public String getDomain() { - return "fapproved.com"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://[w.]*fapproved.com/users/([a-zA-Z0-9\\-_]{3,}).*$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - username = m.group(1); - return username; - } - throw new MalformedURLException("Fapproved user not found in " + url + ", expected http://fapproved.com/users/username/images"); - } - - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return new URL("http://fapproved.com/users/" + getGID(url)); - } - - @Override - public Document getFirstPage() throws IOException { - pageIndex = 1; - String pageURL = getPageURL(pageIndex); - return Http.url(pageURL) - .ignoreContentType() - .get(); - } - - @Override - public Document getNextPage(Document doc) throws IOException { - if ( (doc.select("div.pagination li.next.disabled").size() != 0) - || (doc.select("div.pagination").size() == 0) ) { - throw new IOException("No more pages found"); - } - sleep(1000); - pageIndex++; - String pageURL = getPageURL(pageIndex); - return Http.url(pageURL) - .ignoreContentType() - .get(); - } - - private String getPageURL(int index) throws IOException { - if (username == null) { - username = getGID(this.url); - } - return "http://fapproved.com/users/" + username + "/images?page=" + pageIndex; - } - - @Override - public List getURLsFromPage(Document page) { - List imageURLs = new ArrayList(); - for (Element image : page.select("div.actual-image img")) { - String imageURL = image.attr("src"); - if (imageURL.startsWith("//")) { - imageURL = "http:" + imageURL; - } - else if (imageURL.startsWith("/")) { - imageURL = "http://fapproved.com" + imageURL; - } - imageURLs.add(imageURL); - } - return imageURLs; - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - } -} +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class FapprovedRipper extends AbstractHTMLRipper { + + private int pageIndex = 1; + private String username = null; + + public FapprovedRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "fapproved"; + } + @Override + public String getDomain() { + return "fapproved.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^https?://[w.]*fapproved.com/users/([a-zA-Z0-9\\-_]{3,}).*$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + username = m.group(1); + return username; + } + throw new MalformedURLException("Fapproved user not found in " + url + ", expected http://fapproved.com/users/username/images"); + } + + @Override + public URL sanitizeURL(URL url) throws MalformedURLException { + return new URL("http://fapproved.com/users/" + getGID(url)); + } + + @Override + public Document getFirstPage() throws IOException { + pageIndex = 1; + String pageURL = getPageURL(pageIndex); + return Http.url(pageURL) + .ignoreContentType() + .get(); + } + + @Override + public Document getNextPage(Document doc) throws IOException { + if ((doc.select("div.pagination li.next.disabled").size() != 0) + || (doc.select("div.pagination").size() == 0)) { + throw new IOException("No more pages found"); + } + sleep(1000); + pageIndex++; + String pageURL = getPageURL(pageIndex); + return Http.url(pageURL) + .ignoreContentType() + .get(); + } + + private String getPageURL(int index) throws IOException { + if (username == null) { + username = getGID(this.url); + } + return "http://fapproved.com/users/" + username + "/images?page=" + pageIndex; + } + + @Override + public List getURLsFromPage(Document page) { + List imageURLs = new ArrayList(); + for (Element image : page.select("div.actual-image img")) { + String imageURL = image.attr("src"); + if (imageURL.startsWith("//")) { + imageURL = "http:" + imageURL; + } + else if (imageURL.startsWith("/")) { + imageURL = "http://fapproved.com" + imageURL; + } + imageURLs.add(imageURL); + } + return imageURLs; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java index b200c413..32cda5c3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java @@ -4,7 +4,6 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -14,10 +13,8 @@ import java.util.regex.Pattern; import org.jsoup.Connection.Response; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; public class ZizkiRipper extends AbstractHTMLRipper { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ripperhelpers/ChanSite.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ripperhelpers/ChanSite.java index c8ec3d4f..4b4dd87f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ripperhelpers/ChanSite.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ripperhelpers/ChanSite.java @@ -3,23 +3,27 @@ package com.rarchives.ripme.ripper.rippers.ripperhelpers; import java.util.List; public class ChanSite { - //The domains where the threads are hosted. + // The domains where the threads are hosted. public List domains; - //The domains where the images are hosted. + // The domains where the images are hosted. public List cdnDomains; - public ChanSite(List Domains, List CdnDomains){ - if(Domains.isEmpty()) + public ChanSite(List Domains, List CdnDomains) { + if (Domains.isEmpty()) { throw new IllegalArgumentException("Domains"); - if(CdnDomains.isEmpty()) + } + if (CdnDomains.isEmpty()) { throw new IllegalArgumentException("CdnDomains"); + } domains = Domains; - cdnDomains = CdnDomains; + cdnDomains = CdnDomains; } - public ChanSite(List Domains){ - if(Domains.isEmpty()) - throw new IllegalArgumentException("Domains"); + + public ChanSite(List Domains) { + if (Domains.isEmpty()) { + throw new IllegalArgumentException("Domains"); + } domains = Domains; - cdnDomains = Domains; + cdnDomains = Domains; } } diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index c36942d9..710c681e 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -164,18 +164,6 @@ public class RipUtils { if (url == null) url = urlFromSiteDirectoryName(dir, "vinebox", "http://finebox.co/u/", ""); if (url == null) url = urlFromSiteDirectoryName(dir, "imgbox", "http://imgbox.com/g/", ""); if (url == null) url = urlFromSiteDirectoryName(dir, "modelmayhem", "http://www.modelmayhem.com/", ""); - /* - if (url == null) url = urlFromSiteDirectoryName(dir, "", "", ""); - if (url == null) url = urlFromSiteDirectoryName(dir, "", "", ""); - if (url == null) url = urlFromSiteDirectoryName(dir, "", "", ""); - if (url == null) url = urlFromSiteDirectoryName(dir, "", "", ""); - if (url == null) url = urlFromSiteDirectoryName(dir, "", "", ""); - if (url == null) url = urlFromSiteDirectoryName(dir, "", "", ""); - if (url == null) url = urlFromSiteDirectoryName(dir, "", "", ""); - if (url == null) url = urlFromSiteDirectoryName(dir, "", "", ""); - if (url == null) url = urlFromSiteDirectoryName(dir, "", "", ""); - if (url == null) url = urlFromSiteDirectoryName(dir, "", "", ""); - */ //if (url == null) url = urlFromSiteDirectoryName(dir, "8muses", "http://www.8muses.com/index/category/", ""); return url; } @@ -247,9 +235,8 @@ public class RipUtils { List fields = Arrays.asList(dir.split("_")); String album = fields.get(1); String url = "http://"; - if ( (fields.contains("top") || fields.contains("new")) - && (fields.contains("year") || fields.contains("month") || fields.contains("week") || fields.contains("all")) - ) { + if ((fields.contains("top") || fields.contains("new")) + && (fields.contains("year") || fields.contains("month") || fields.contains("week") || fields.contains("all"))) { // Subreddit fields.remove(0); // "imgur" String sub = ""; diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index e5343cf2..362b6cc8 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -34,7 +34,6 @@ import com.rarchives.ripme.ripper.AbstractRipper; * Common utility functions used in various places throughout the project. */ public class Utils { - public static final String RIP_DIRECTORY = "rips"; private static final String configFile = "rip.properties"; private static final Logger logger = Logger.getLogger(Utils.class); @@ -50,7 +49,7 @@ public class Utils { } config = new PropertiesConfiguration(configPath); logger.info("Loaded " + config.getPath()); - if (f.exists()){ + if (f.exists()) { // Config was loaded from file if ( !config.containsKey("twitter.auth") || !config.containsKey("twitter.max_requests") @@ -171,16 +170,18 @@ public class Utils { public static String stripURLParameter(String url, String parameter) { int paramIndex = url.indexOf("?" + parameter); boolean wasFirstParam = true; - if(paramIndex < 0) { + if (paramIndex < 0) { wasFirstParam = false; paramIndex = url.indexOf("&" + parameter); } - if(paramIndex > 0) { + if (paramIndex > 0) { int nextParam = url.indexOf("&", paramIndex+1); - if(nextParam != -1) { + if (nextParam != -1) { String c = "&"; - if(wasFirstParam) c = "?"; + if (wasFirstParam) { + c = "?"; + } url = url.substring(0, paramIndex) + c + url.substring(nextParam+1, url.length()); } else { url = url.substring(0, paramIndex); @@ -253,7 +254,7 @@ public class Utils { while(entries.hasMoreElements()) { JarEntry nextElement = entries.nextElement(); String entryName = nextElement.getName(); - if(entryName.startsWith(relPath) + if (entryName.startsWith(relPath) && entryName.length() > (relPath.length() + "/".length()) && !nextElement.isDirectory()) { String className = entryName.replace('/', '.').replace('\\', '.').replace(".class", ""); @@ -401,7 +402,7 @@ public class Utils { public static Map parseUrlQuery(String query) { Map res = new HashMap(); - if (query.equals("")){ + if (query.equals("")) { return res; } @@ -410,9 +411,9 @@ public class Utils { try { for (String part : parts) { - if ((pos = part.indexOf('=')) >= 0){ + if ((pos = part.indexOf('=')) >= 0) { res.put(URLDecoder.decode(part.substring(0, pos), "UTF-8"), URLDecoder.decode(part.substring(pos + 1), "UTF-8")); - }else{ + } else { res.put(URLDecoder.decode(part, "UTF-8"), ""); } } @@ -434,7 +435,7 @@ public class Utils { * @return The associated value or null if key wasn't found */ public static String parseUrlQuery(String query, String key) { - if (query.equals("")){ + if (query.equals("")) { return null; } @@ -444,7 +445,7 @@ public class Utils { try { for (String part : parts) { if ((pos = part.indexOf('=')) >= 0) { - if (URLDecoder.decode(part.substring(0, pos), "UTF-8").equals(key)){ + if (URLDecoder.decode(part.substring(0, pos), "UTF-8").equals(key)) { return URLDecoder.decode(part.substring(pos + 1), "UTF-8"); }