diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index 95bf5ed7..a1728fe6 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -146,7 +146,7 @@ public class App { try { String url; BufferedReader br = new BufferedReader(new FileReader(filename)); - while((url = br.readLine()) != null) { + while ((url = br.readLine()) != null) { // loop through each url in the file and proces each url individually. ripURL(url.trim(), cl.hasOption("n")); } @@ -171,7 +171,7 @@ public class App { if (!history.contains(url.toExternalForm())) { history.add(url.toExternalForm()); Utils.setConfigList("download.history", Arrays.asList(history.toArray())); - if(saveConfig) { + if (saveConfig) { Utils.saveConfig(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 35b2aa43..6c89cb8e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -16,7 +16,7 @@ import com.rarchives.ripme.utils.Utils; * Simplified ripper, designed for ripping from sites by parsing HTML. */ public abstract class AbstractHTMLRipper extends AlbumRipper { - + public AbstractHTMLRipper(URL url) throws IOException { super(url); } @@ -30,7 +30,7 @@ public abstract class AbstractHTMLRipper extends AlbumRipper { } public abstract List getURLsFromPage(Document page); public List getDescriptionsFromPage(Document doc) throws IOException { - throw new IOException("getDescriptionsFromPage not implemented"); // Do I do this or make an abstract function? + throw new IOException("getDescriptionsFromPage not implemented"); // Do I do this or make an abstract function? } public abstract void downloadURL(URL url, int index); public DownloadThreadPool getThreadPool() { @@ -45,16 +45,16 @@ public abstract class AbstractHTMLRipper extends AlbumRipper { public boolean canRip(URL url) { return url.getHost().endsWith(getDomain()); } - + @Override public URL sanitizeURL(URL url) throws MalformedURLException { return url; } public boolean hasDescriptionSupport() { - return false; + return false; } public String getDescription(String page) throws IOException { - throw new IOException("getDescription not implemented"); // Do I do this or make an abstract function? + throw new IOException("getDescription not implemented"); // Do I do this or make an abstract function? } public int descSleepTime() { return 0; @@ -66,7 +66,7 @@ public abstract class AbstractHTMLRipper extends AlbumRipper { logger.info("Retrieving " + this.url); sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm()); Document doc = getFirstPage(); - + while (doc != null) { List imageURLs = getURLsFromPage(doc); // Remove all but 1 image @@ -79,7 +79,7 @@ public abstract class AbstractHTMLRipper extends AlbumRipper { if (imageURLs.size() == 0) { throw new IOException("No images found at " + doc.location()); } - + for (String imageURL : imageURLs) { index += 1; logger.debug("Found image url #" + index + ": " + imageURL); @@ -90,23 +90,23 @@ public abstract class AbstractHTMLRipper extends AlbumRipper { } if (hasDescriptionSupport() && Utils.getConfigBoolean("descriptions.save", false)) { logger.debug("Fetching description(s) from " + doc.location()); - List textURLs = getDescriptionsFromPage(doc); - if (textURLs.size() > 0) { + List textURLs = getDescriptionsFromPage(doc); + if (textURLs.size() > 0) { logger.debug("Found description link(s) from " + doc.location()); - for (String textURL : textURLs) { - if (isStopped()) { - break; - } - textindex += 1; - logger.debug("Getting description from " + textURL); + for (String textURL : textURLs) { + if (isStopped()) { + break; + } + textindex += 1; + logger.debug("Getting description from " + textURL); sleep(descSleepTime()); - String tempDesc = getDescription(textURL); - if (tempDesc != null) { - logger.debug("Got description: " + tempDesc); - saveText(new URL(textURL), "", tempDesc, textindex); - } - } - } + String tempDesc = getDescription(textURL); + if (tempDesc != null) { + logger.debug("Got description: " + tempDesc); + saveText(new URL(textURL), "", tempDesc, textindex); + } + } + } } if (isStopped() || isThisATest()) { diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 85d4c706..15551204 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -195,7 +195,7 @@ public abstract class AbstractRipper public void retrievingSource(String url) { RipStatusMessage msg = new RipStatusMessage(STATUS.LOADING_RESOURCE, url); if (observer != null) { - observer.update(this, msg); + observer.update(this, msg); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java index 9b385500..30e3c2b7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java @@ -19,7 +19,7 @@ public class DownloadThreadPool { public DownloadThreadPool() { initialize("Main"); } - + public DownloadThreadPool(String threadPoolName) { initialize(threadPoolName); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/BcfakesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/BcfakesRipper.java index edc63e02..682519cc 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/BcfakesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/BcfakesRipper.java @@ -46,12 +46,12 @@ public class BcfakesRipper extends AbstractHTMLRipper { + "http://www.bcfakes.com/celebritylist/name" + " Got: " + url); } - + @Override public Document getFirstPage() throws IOException { return Http.url(url).get(); } - + @Override public Document getNextPage(Document doc) throws IOException { // Find next page @@ -63,7 +63,7 @@ public class BcfakesRipper extends AbstractHTMLRipper { sleep(500); return Http.url(nextUrl).get(); } - + @Override public List getURLsFromPage(Document doc) { List imageURLs = new ArrayList(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java index 9de7845e..e9993321 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java @@ -158,13 +158,13 @@ public class ChanRipper extends AbstractHTMLRipper { Boolean self_hosted = false; if (!generalChanSite) { for (String cdnDomain : chanSite.cdnDomains) { - if (href.contains(cdnDomain)){ + if (href.contains(cdnDomain)) { self_hosted = true; } } } - if (self_hosted || generalChanSite){ + if (self_hosted || generalChanSite) { p = Pattern.compile("^.*\\.(jpg|jpeg|png|gif|apng|webp|tif|tiff|webm)$", Pattern.CASE_INSENSITIVE); m = p.matcher(href); if (m.matches()) { @@ -194,7 +194,7 @@ public class ChanRipper extends AbstractHTMLRipper { } List urls = RipUtils.getFilesFromURL(originalURL); - for(URL imageurl : urls){ + for (URL imageurl : urls) { imageURLs.add(imageurl.toString()); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DatwinRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DatwinRipper.java index 7d067e35..2f276bd8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DatwinRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DatwinRipper.java @@ -46,7 +46,7 @@ public class DatwinRipper extends AbstractHTMLRipper { public Document getFirstPage() throws IOException { return Http.url(url).get(); } - + @Override public List getURLsFromPage(Document doc) { List imageURLs = new ArrayList(); @@ -57,7 +57,7 @@ public class DatwinRipper extends AbstractHTMLRipper { } return imageURLs; } - + @Override public void downloadURL(URL url, int index) { addURLToDownload(url, getPrefix(index)); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index b170c2e7..1ea1479a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -48,7 +48,7 @@ public class DeviantartRipper extends AbstractHTMLRipper { } @Override public boolean hasDescriptionSupport() { - return true; + return true; } @Override public URL sanitizeURL(URL url) throws MalformedURLException { @@ -239,7 +239,7 @@ public class DeviantartRipper extends AbstractHTMLRipper { } return result.toString(); } - + /** * Attempts to download description for image. * Comes in handy when people put entire stories in their description. @@ -277,7 +277,7 @@ public class DeviantartRipper extends AbstractHTMLRipper { return null; } } - + /** * If largest resolution for image at 'thumb' is found, starts downloading * and returns null. diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java index e45d3980..3cb606c7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java @@ -1,4 +1,3 @@ - package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.ripper.AbstractHTMLRipper; @@ -8,174 +7,165 @@ import com.rarchives.ripme.utils.Utils; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; import java.net.URL; import java.net.URLDecoder; import java.util.ArrayList; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; -/** - * - * @author - */ public class E621Ripper extends AbstractHTMLRipper { - public static final int POOL_IMAGES_PER_PAGE = 24; + public static final int POOL_IMAGES_PER_PAGE = 24; - private DownloadThreadPool e621ThreadPool = new DownloadThreadPool("e621"); + private DownloadThreadPool e621ThreadPool = new DownloadThreadPool("e621"); - public E621Ripper(URL url) throws IOException { - super(url); - } + public E621Ripper(URL url) throws IOException { + super(url); + } - @Override - public DownloadThreadPool getThreadPool() { - return e621ThreadPool; - } + @Override + public DownloadThreadPool getThreadPool() { + return e621ThreadPool; + } - @Override - public String getDomain() { - return "e621.net"; - } + @Override + public String getDomain() { + return "e621.net"; + } - @Override - public String getHost() { - return "e621"; - } + @Override + public String getHost() { + return "e621"; + } - @Override - public Document getFirstPage() throws IOException { - if (url.getPath().startsWith("/pool/show/")) { - return Http.url("https://e621.net/pool/show/" + getTerm(url)).get(); - } else { - return Http.url("https://e621.net/post/index/1/" + getTerm(url)).get(); - } - } + @Override + public Document getFirstPage() throws IOException { + if (url.getPath().startsWith("/pool/show/")) { + return Http.url("https://e621.net/pool/show/" + getTerm(url)).get(); + } else { + return Http.url("https://e621.net/post/index/1/" + getTerm(url)).get(); + } + } - @Override - public List getURLsFromPage(Document page) { - Elements elements = page.select("#post-list .thumb a,#pool-show .thumb a"); - List res = new ArrayList(elements.size()); + @Override + public List getURLsFromPage(Document page) { + Elements elements = page.select("#post-list .thumb a,#pool-show .thumb a"); + List res = new ArrayList(elements.size()); - if (page.getElementById("pool-show") != null) { - int index = 0; + if (page.getElementById("pool-show") != null) { + int index = 0; - Element e = page.getElementById("paginator"); - if (e != null) { - e = e.getElementsByClass("current").first(); - if (e != null) { - index = (Integer.parseInt(e.text()) - 1) * POOL_IMAGES_PER_PAGE; - } - } + Element e = page.getElementById("paginator"); + if (e != null) { + e = e.getElementsByClass("current").first(); + if (e != null) { + index = (Integer.parseInt(e.text()) - 1) * POOL_IMAGES_PER_PAGE; + } + } - for (Element e_ : elements) { - res.add(e_.absUrl("href") + "#" + ++index); - } + for (Element e_ : elements) { + res.add(e_.absUrl("href") + "#" + ++index); + } - } else { - for (Element e : elements) { - res.add(e.absUrl("href") + "#" + e.child(0).attr("id").substring(1)); - } - } + } else { + for (Element e : elements) { + res.add(e.absUrl("href") + "#" + e.child(0).attr("id").substring(1)); + } + } - return res; - } + return res; + } - @Override - public Document getNextPage(Document page) throws IOException { - for (Element e : page.select("#paginator a")) { - if (e.attr("rel").equals("next")) { - return Http.url(e.absUrl("href")).get(); - } - } + @Override + public Document getNextPage(Document page) throws IOException { + for (Element e : page.select("#paginator a")) { + if (e.attr("rel").equals("next")) { + return Http.url(e.absUrl("href")).get(); + } + } - return null; - } + return null; + } - @Override - public void downloadURL(final URL url, int index) { - e621ThreadPool.addThread(new Thread(new Runnable() { - public void run() { - try { - Document page = Http.url(url).get(); - Element e = page.getElementById("image"); + @Override + public void downloadURL(final URL url, int index) { + e621ThreadPool.addThread(new Thread(new Runnable() { + public void run() { + try { + Document page = Http.url(url).get(); + Element e = page.getElementById("image"); - if (e != null) { - addURLToDownload(new URL(e.absUrl("src")), Utils.getConfigBoolean("download.save_order", true) ? url.getRef() + "-" : ""); - } else if ((e = page.select(".content object>param[name=\"movie\"]").first()) != null) { - addURLToDownload(new URL(e.absUrl("value")), Utils.getConfigBoolean("download.save_order", true) ? url.getRef() + "-" : ""); - } else { - Logger.getLogger(E621Ripper.class.getName()).log(Level.WARNING, "Unsupported media type - please report to program author: " + url.toString()); - } + if (e != null) { + addURLToDownload(new URL(e.absUrl("src")), Utils.getConfigBoolean("download.save_order", true) ? url.getRef() + "-" : ""); + } else if ((e = page.select(".content object>param[name=\"movie\"]").first()) != null) { + addURLToDownload(new URL(e.absUrl("value")), Utils.getConfigBoolean("download.save_order", true) ? url.getRef() + "-" : ""); + } else { + Logger.getLogger(E621Ripper.class.getName()).log(Level.WARNING, "Unsupported media type - please report to program author: " + url.toString()); + } - } catch (IOException ex) { - Logger.getLogger(E621Ripper.class.getName()).log(Level.SEVERE, null, ex); - } - } - })); - } + } catch (IOException ex) { + Logger.getLogger(E621Ripper.class.getName()).log(Level.SEVERE, null, ex); + } + } + })); + } - private String getTerm(URL url) throws MalformedURLException { - String query = url.getQuery(); + private String getTerm(URL url) throws MalformedURLException { + String query = url.getQuery(); - if (query != null) { - return Utils.parseUrlQuery(query, "tags"); - } + if (query != null) { + return Utils.parseUrlQuery(query, "tags"); + } - if (query == null) { - if ((query = url.getPath()).startsWith("/post/index/")) { - query = query.substring(12); + if (query == null) { + if ((query = url.getPath()).startsWith("/post/index/")) { + query = query.substring(12); - int pos = query.indexOf('/'); - if (pos == -1) { - return null; - } + int pos = query.indexOf('/'); + if (pos == -1) { + return null; + } - // skip page number - query = query.substring(pos + 1); + // skip page number + query = query.substring(pos + 1); - if (query.endsWith("/")) { - query = query.substring(0, query.length() - 1); - } + if (query.endsWith("/")) { + query = query.substring(0, query.length() - 1); + } - try { - return URLDecoder.decode(query, "UTF-8"); - } catch (UnsupportedEncodingException e) { - // Shouldn't happen since UTF-8 is required to be supported - throw new RuntimeException(e); - } + try { + return URLDecoder.decode(query, "UTF-8"); + } catch (UnsupportedEncodingException e) { + // Shouldn't happen since UTF-8 is required to be supported + throw new RuntimeException(e); + } - } else if (query.startsWith("/pool/show/")) { - query = query.substring(11); + } else if (query.startsWith("/pool/show/")) { + query = query.substring(11); - if (query.endsWith("/")) { - query = query.substring(0, query.length() - 1); - } + if (query.endsWith("/")) { + query = query.substring(0, query.length() - 1); + } - return query; - } - } + return query; + } + } - return null; - } + return null; + } - @Override - public String getGID(URL url) throws MalformedURLException { - String prefix = ""; - if (url.getPath().startsWith("/pool/show/")) { - prefix = "pool_"; - } else { - prefix = "term_"; - } + @Override + public String getGID(URL url) throws MalformedURLException { + String prefix = ""; + if (url.getPath().startsWith("/pool/show/")) { + prefix = "pool_"; + } else { + prefix = "term_"; + } - return Utils.filesystemSafe(prefix + getTerm(url)); - } - -} \ No newline at end of file + return Utils.filesystemSafe(prefix + getTerm(url)); + } +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java index 97ab2815..10d848fb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java @@ -9,7 +9,6 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -18,10 +17,8 @@ import org.jsoup.Connection.Response; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; -import org.jsoup.Connection.Method; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; /** @@ -45,7 +42,7 @@ public class EroShareRipper extends AbstractHTMLRipper { } @Override - public void downloadURL(URL url, int index){ + public void downloadURL(URL url, int index) { addURLToDownload(url); } @@ -66,12 +63,12 @@ public class EroShareRipper extends AbstractHTMLRipper { @Override - public List getURLsFromPage(Document doc){ + public List getURLsFromPage(Document doc) { List URLs = new ArrayList(); //Pictures Elements imgs = doc.getElementsByTag("img"); - for (Element img : imgs){ - if (img.hasClass("album-image")){ + for (Element img : imgs) { + if (img.hasClass("album-image")) { String imageURL = img.attr("src"); imageURL = "https:" + imageURL; URLs.add(imageURL); @@ -79,8 +76,8 @@ public class EroShareRipper extends AbstractHTMLRipper { } //Videos Elements vids = doc.getElementsByTag("video"); - for (Element vid : vids){ - if (vid.hasClass("album-video")){ + for (Element vid : vids) { + if (vid.hasClass("album-video")) { Elements source = vid.getElementsByTag("source"); String videoURL = source.first().attr("src"); URLs.add(videoURL); @@ -122,8 +119,8 @@ public class EroShareRipper extends AbstractHTMLRipper { List URLs = new ArrayList(); //Pictures Elements imgs = doc.getElementsByTag("img"); - for (Element img : imgs){ - if (img.hasClass("album-image")){ + for (Element img : imgs) { + if (img.hasClass("album-image")) { String imageURL = img.attr("src"); imageURL = "https:" + imageURL; URLs.add(new URL(imageURL)); @@ -131,8 +128,8 @@ public class EroShareRipper extends AbstractHTMLRipper { } //Videos Elements vids = doc.getElementsByTag("video"); - for (Element vid : vids){ - if (vid.hasClass("album-video")){ + for (Element vid : vids) { + if (vid.hasClass("album-video")) { Elements source = vid.getElementsByTag("source"); String videoURL = source.first().attr("src"); URLs.add(new URL(videoURL)); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FapprovedRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FapprovedRipper.java index dd423691..f33d40ce 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FapprovedRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FapprovedRipper.java @@ -1,101 +1,101 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; - -public class FapprovedRipper extends AbstractHTMLRipper { - - private int pageIndex = 1; - private String username = null; - - public FapprovedRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return "fapproved"; - } - @Override - public String getDomain() { - return "fapproved.com"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://[w.]*fapproved.com/users/([a-zA-Z0-9\\-_]{3,}).*$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - username = m.group(1); - return username; - } - throw new MalformedURLException("Fapproved user not found in " + url + ", expected http://fapproved.com/users/username/images"); - } - - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return new URL("http://fapproved.com/users/" + getGID(url)); - } - - @Override - public Document getFirstPage() throws IOException { - pageIndex = 1; - String pageURL = getPageURL(pageIndex); - return Http.url(pageURL) - .ignoreContentType() - .get(); - } - - @Override - public Document getNextPage(Document doc) throws IOException { - if ( (doc.select("div.pagination li.next.disabled").size() != 0) - || (doc.select("div.pagination").size() == 0) ) { - throw new IOException("No more pages found"); - } - sleep(1000); - pageIndex++; - String pageURL = getPageURL(pageIndex); - return Http.url(pageURL) - .ignoreContentType() - .get(); - } - - private String getPageURL(int index) throws IOException { - if (username == null) { - username = getGID(this.url); - } - return "http://fapproved.com/users/" + username + "/images?page=" + pageIndex; - } - - @Override - public List getURLsFromPage(Document page) { - List imageURLs = new ArrayList(); - for (Element image : page.select("div.actual-image img")) { - String imageURL = image.attr("src"); - if (imageURL.startsWith("//")) { - imageURL = "http:" + imageURL; - } - else if (imageURL.startsWith("/")) { - imageURL = "http://fapproved.com" + imageURL; - } - imageURLs.add(imageURL); - } - return imageURLs; - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - } -} +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class FapprovedRipper extends AbstractHTMLRipper { + + private int pageIndex = 1; + private String username = null; + + public FapprovedRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "fapproved"; + } + @Override + public String getDomain() { + return "fapproved.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^https?://[w.]*fapproved.com/users/([a-zA-Z0-9\\-_]{3,}).*$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + username = m.group(1); + return username; + } + throw new MalformedURLException("Fapproved user not found in " + url + ", expected http://fapproved.com/users/username/images"); + } + + @Override + public URL sanitizeURL(URL url) throws MalformedURLException { + return new URL("http://fapproved.com/users/" + getGID(url)); + } + + @Override + public Document getFirstPage() throws IOException { + pageIndex = 1; + String pageURL = getPageURL(pageIndex); + return Http.url(pageURL) + .ignoreContentType() + .get(); + } + + @Override + public Document getNextPage(Document doc) throws IOException { + if ((doc.select("div.pagination li.next.disabled").size() != 0) + || (doc.select("div.pagination").size() == 0)) { + throw new IOException("No more pages found"); + } + sleep(1000); + pageIndex++; + String pageURL = getPageURL(pageIndex); + return Http.url(pageURL) + .ignoreContentType() + .get(); + } + + private String getPageURL(int index) throws IOException { + if (username == null) { + username = getGID(this.url); + } + return "http://fapproved.com/users/" + username + "/images?page=" + pageIndex; + } + + @Override + public List getURLsFromPage(Document page) { + List imageURLs = new ArrayList(); + for (Element image : page.select("div.actual-image img")) { + String imageURL = image.attr("src"); + if (imageURL.startsWith("//")) { + imageURL = "http:" + imageURL; + } + else if (imageURL.startsWith("/")) { + imageURL = "http://fapproved.com" + imageURL; + } + imageURLs.add(imageURL); + } + return imageURLs; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java index 1c57cf82..42649af9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java @@ -155,7 +155,7 @@ public class FivehundredpxRipper extends AbstractJSONRipper { private String getUserID(String username) throws IOException { logger.info("Fetching user ID for " + username); JSONObject json = new Http("https://api.500px.com/v1/" + - "users/show" + + "users/show" + "?username=" + username + "&consumer_key=" + CONSUMER_KEY) .getJSON(); @@ -259,55 +259,55 @@ public class FivehundredpxRipper extends AbstractJSONRipper { List imageURLs = new ArrayList(); JSONArray photos = json.getJSONArray("photos"); for (int i = 0; i < photos.length(); i++) { - if (super.isStopped()) { - break; - } + if (super.isStopped()) { + break; + } JSONObject photo = photos.getJSONObject(i); String imageURL = null; String rawUrl = "https://500px.com" + photo.getString("url"); Document doc; Elements images = new Elements(); try { - logger.debug("Loading " + rawUrl); - super.retrievingSource(rawUrl); - doc = Http.url(rawUrl).get(); - images = doc.select("div#preload img"); + logger.debug("Loading " + rawUrl); + super.retrievingSource(rawUrl); + doc = Http.url(rawUrl).get(); + images = doc.select("div#preload img"); } catch (IOException e) { - logger.error("Error fetching full-size image from " + rawUrl, e); + logger.error("Error fetching full-size image from " + rawUrl, e); } if (images.size() > 0) { - imageURL = images.first().attr("src"); - logger.debug("Found full-size non-watermarked image: " + imageURL); + imageURL = images.first().attr("src"); + logger.debug("Found full-size non-watermarked image: " + imageURL); } else { - logger.debug("Falling back to image_url from API response"); - imageURL = photo.getString("image_url"); - imageURL = imageURL.replaceAll("/4\\.", "/5."); - // See if there's larger images - for (String imageSize : new String[] { "2048" } ) { - String fsURL = imageURL.replaceAll("/5\\.", "/" + imageSize + "."); - sleep(10); - if (urlExists(fsURL)) { - logger.info("Found larger image at " + fsURL); - imageURL = fsURL; - break; - } - } + logger.debug("Falling back to image_url from API response"); + imageURL = photo.getString("image_url"); + imageURL = imageURL.replaceAll("/4\\.", "/5."); + // See if there's larger images + for (String imageSize : new String[] { "2048" } ) { + String fsURL = imageURL.replaceAll("/5\\.", "/" + imageSize + "."); + sleep(10); + if (urlExists(fsURL)) { + logger.info("Found larger image at " + fsURL); + imageURL = fsURL; + break; + } + } } if (imageURL == null) { - logger.error("Failed to find image for photo " + photo.toString()); + logger.error("Failed to find image for photo " + photo.toString()); } else { - imageURLs.add(imageURL); - if (isThisATest()) { - break; - } + imageURLs.add(imageURL); + if (isThisATest()) { + break; + } } } return imageURLs; } - + private boolean urlExists(String url) { try { HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java index 03205e1a..08d53eec 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java @@ -246,7 +246,7 @@ public class FuraffinityRipper extends AbstractHTMLRipper { if (donwloadLink.size() == 0) { logger.warn("Could not download " + this.url); return; - } + } String link = "http:" + donwloadLink.first().attr("href"); logger.info("Found URL " + link); String[] fileNameSplit = link.split("/"); @@ -266,7 +266,7 @@ public class FuraffinityRipper extends AbstractHTMLRipper { addURLToDownload(new URL(link),saveAS,"",cookies); } catch (IOException e) { logger.error("[!] Exception while loading/parsing " + this.url, e); - } + } } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java index a7130ed3..1950cf8e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java @@ -69,7 +69,7 @@ public class GirlsOfDesireRipper extends AbstractHTMLRipper { } return albumDoc; } - + @Override public List getURLsFromPage(Document doc) { List imageURLs = new ArrayList(); @@ -83,7 +83,7 @@ public class GirlsOfDesireRipper extends AbstractHTMLRipper { } return imageURLs; } - + @Override public void downloadURL(URL url, int index) { // Send referrer when downloading images diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java index 9fede305..6f82a9c9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java @@ -69,12 +69,12 @@ public class ImagearnRipper extends AbstractHTMLRipper { } throw new IOException("Failed to find gallery at URL " + url); } - + @Override public Document getFirstPage() throws IOException { return Http.url(url).get(); } - + @Override public List getURLsFromPage(Document doc) { List imageURLs = new ArrayList(); @@ -85,7 +85,7 @@ public class ImagearnRipper extends AbstractHTMLRipper { } return imageURLs; } - + @Override public void downloadURL(URL url, int index) { addURLToDownload(url, getPrefix(index)); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java index 0a358783..f4608d73 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java @@ -58,7 +58,7 @@ public class ImagebamRipper extends AbstractHTMLRipper { + "http://www.imagebam.com/gallery/galleryid" + " Got: " + url); } - + @Override public Document getFirstPage() throws IOException { if (albumDoc == null) { @@ -66,7 +66,7 @@ public class ImagebamRipper extends AbstractHTMLRipper { } return albumDoc; } - + @Override public Document getNextPage(Document doc) throws IOException { // Find next page @@ -78,7 +78,7 @@ public class ImagebamRipper extends AbstractHTMLRipper { sleep(500); return Http.url(nextUrl).get(); } - + @Override public List getURLsFromPage(Document doc) { List imageURLs = new ArrayList(); @@ -117,7 +117,7 @@ public class ImagebamRipper extends AbstractHTMLRipper { /** * Helper class to find and download images found on "image" pages - * + * * Handles case when site has IP-banned the user. */ private class ImagebamImageThread extends Thread { @@ -134,7 +134,7 @@ public class ImagebamRipper extends AbstractHTMLRipper { public void run() { fetchImage(); } - + private void fetchImage() { try { Document doc = Http.url(url).get(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java index ad6a82ae..b405caff 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java @@ -93,7 +93,7 @@ public class ImagefapRipper extends AbstractHTMLRipper { + "imagefap.com/pictures/####..." + " Got: " + url); } - + @Override public Document getFirstPage() throws IOException { if (albumDoc == null) { @@ -101,7 +101,7 @@ public class ImagefapRipper extends AbstractHTMLRipper { } return albumDoc; } - + @Override public Document getNextPage(Document doc) throws IOException { String nextURL = null; @@ -117,7 +117,7 @@ public class ImagefapRipper extends AbstractHTMLRipper { sleep(1000); return Http.url(nextURL).get(); } - + @Override public List getURLsFromPage(Document doc) { List imageURLs = new ArrayList(); @@ -137,7 +137,7 @@ public class ImagefapRipper extends AbstractHTMLRipper { } return imageURLs; } - + @Override public void downloadURL(URL url, int index) { // Send referrer for image downloads diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java index 0bb2b6a4..c818f3ec 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java @@ -55,12 +55,12 @@ public class ImagevenueRipper extends AbstractHTMLRipper { + "http://...imagevenue.com/galshow.php?gal=gallery_...." + " Got: " + url); } - + @Override public Document getFirstPage() throws IOException { return Http.url(url).get(); } - + public List getURLsFromPage(Document doc) { List imageURLs = new ArrayList(); for (Element thumb : doc.select("a[target=_blank]")) { @@ -68,7 +68,7 @@ public class ImagevenueRipper extends AbstractHTMLRipper { } return imageURLs; } - + public void downloadURL(URL url, int index) { ImagevenueImageThread t = new ImagevenueImageThread(url, index); imagevenueThreadPool.addThread(t); @@ -76,7 +76,7 @@ public class ImagevenueRipper extends AbstractHTMLRipper { /** * Helper class to find and download images found on "image" pages - * + * * Handles case when site has IP-banned the user. */ private class ImagevenueImageThread extends Thread { @@ -93,7 +93,7 @@ public class ImagevenueRipper extends AbstractHTMLRipper { public void run() { fetchImage(); } - + private void fetchImage() { try { Document doc = Http.url(url) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgboxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgboxRipper.java index d8804d61..69392c95 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgboxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgboxRipper.java @@ -39,7 +39,7 @@ public class ImgboxRipper extends AbstractHTMLRipper { throw new MalformedURLException("Expected imgbox.com URL format: " + "imgbox.com/g/albumid - got " + url + "instead"); } - + @Override public Document getFirstPage() throws IOException { return Http.url(url).get(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index 43b8eda3..e2828709 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -11,11 +11,8 @@ import java.util.regex.Pattern; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; import com.rarchives.ripme.ripper.AbstractJSONRipper; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; public class InstagramRipper extends AbstractJSONRipper { @@ -60,18 +57,18 @@ public class InstagramRipper extends AbstractJSONRipper { throw new MalformedURLException("Expected username in URL (instagram.com/username and not " + url); } - + private String getUserID(URL url) throws IOException { Pattern p = Pattern.compile("^https?://instagram\\.com/([^/]+)"); Matcher m = p.matcher(url.toExternalForm()); - if(m.matches()) { + if (m.matches()) { return m.group(1); } throw new IOException("Unable to find userID at " + this.url); } - + @Override public JSONObject getFirstPage() throws IOException { userID = getUserID(url); @@ -95,7 +92,7 @@ public class InstagramRipper extends AbstractJSONRipper { throw new IOException("No additional pages found"); } - if(nextPageAvailable) { + if (nextPageAvailable) { JSONArray items = json.getJSONArray("items"); JSONObject last_item = items.getJSONObject(items.length() - 1); String nextMaxID = last_item.getString("id"); @@ -111,7 +108,7 @@ public class InstagramRipper extends AbstractJSONRipper { throw new IOException("No more images found"); } } - + @Override public List getURLsFromJSON(JSONObject json) { List imageURLs = new ArrayList(); @@ -142,7 +139,7 @@ public class InstagramRipper extends AbstractJSONRipper { } return imageURLs; } - + @Override public void downloadURL(URL url, int index) { addURLToDownload(url); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MediacrushRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MediacrushRipper.java index 8970c22b..b3fdbdaf 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MediacrushRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MediacrushRipper.java @@ -138,9 +138,9 @@ public class MediacrushRipper extends AbstractJSONRipper { } /** - * Iterates over list if "file" objects and returns the preferred + * Iterates over list if "file" objects and returns the preferred * image format. - * @param subfiles Array of "files" (JSONObjects) which contain + * @param subfiles Array of "files" (JSONObjects) which contain * @return Preferred media format. */ private String getPreferredUrl(JSONArray subfiles) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java index 32612012..b5a1ecca 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java @@ -2,24 +2,15 @@ package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; -import com.rarchives.ripme.utils.Utils; -import java.io.File; import java.io.IOException; import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; public class MyhentaicomicsRipper extends AbstractHTMLRipper { public static boolean isTag; @@ -95,13 +86,13 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper { List albumPagesList = new ArrayList(); int pageNumber = 1; albumPagesList.add("http://myhentaicomics.com/index.php/" + pageUrl.split("\\?")[0] + "?page=" + Integer.toString(pageNumber)); - while(true) { + while (true) { String urlToGet = "http://myhentaicomics.com/index.php/" + pageUrl.split("\\?")[0] + "?page=" + Integer.toString(pageNumber); Document nextAlbumPage; try { logger.info("Grabbing " + urlToGet); nextAlbumPage = Http.url(urlToGet).get(); - } catch(IOException e){ + } catch(IOException e) { logger.warn("Failed to log link in Jsoup"); nextAlbumPage = null; e.printStackTrace(); @@ -109,7 +100,7 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper { Element elem = nextAlbumPage.select("a.ui-icon-right").first(); String nextPage = elem.attr("href"); pageNumber = pageNumber + 1; - if(nextPage == ""){ + if (nextPage == "") { logger.info("Got " + pageNumber + " pages"); break; } @@ -145,7 +136,7 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper { else { album_doc = Http.url(element).get(); } - } catch(IOException e){ + } catch(IOException e) { logger.warn("Failed to log link in Jsoup"); album_doc = null; e.printStackTrace(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NatalieMuRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NatalieMuRipper.java index 51f2bdda..9526c6b1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NatalieMuRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NatalieMuRipper.java @@ -4,7 +4,6 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -13,15 +12,14 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.rippers.ripperhelpers.ChanSite; import com.rarchives.ripme.utils.Http; public class NatalieMuRipper extends AbstractHTMLRipper { - + public int news_id = 0; - + public NatalieMuRipper(URL url) throws IOException { - super(url); + super(url); } @Override @@ -37,12 +35,12 @@ public class NatalieMuRipper extends AbstractHTMLRipper { } @Override - public boolean canRip(URL url) { + public boolean canRip(URL url) { //urls like: // http://cdn2.natalie.mu/music/gallery/show/news_id/xxxxxx/image_id/xxxxxx // http://cdn2.natalie.mu/music/news/140411 return url.toExternalForm().contains("natalie.mu") // Most chans - && (url.toExternalForm().contains("/news_id/") + && (url.toExternalForm().contains("/news_id/") || url.toExternalForm().contains("/news/")); // 4chan, archive.moe } @@ -61,13 +59,13 @@ public class NatalieMuRipper extends AbstractHTMLRipper { m = p.matcher(u); if (m.find()) { return m.group(1); - } + } } else if (u.contains("/news/")) { p = Pattern.compile("/news/([0-9]+)/?"); m = p.matcher(u); if (m.find()) { return m.group(1); - } + } } throw new MalformedURLException( @@ -85,18 +83,18 @@ public class NatalieMuRipper extends AbstractHTMLRipper { public Document getFirstPage() throws IOException { return Http.url(this.url).get(); } - + @Override public List getURLsFromPage(Document page) { List imageURLs = new ArrayList(); Pattern p; Matcher m; //select all album thumbnails for (Element span : page.select(".NA_articleGallery span")) { - if (!span.hasAttr("style")) { + if (!span.hasAttr("style")) { continue; } String style = span.attr("style").trim(); - + p = Pattern.compile("background-image: url\\((.*list_thumb_inbox.*)\\);", Pattern.CASE_INSENSITIVE); m = p.matcher(style); if (m.find()) { @@ -118,7 +116,7 @@ public class NatalieMuRipper extends AbstractHTMLRipper { if (isThisATest()) { break; } - } + } if (isStopped()) { break; @@ -130,5 +128,5 @@ public class NatalieMuRipper extends AbstractHTMLRipper { @Override public void downloadURL(URL url, int index) { addURLToDownload(url, getPrefix(index), "", this.url.toString(), null); - } + } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PahealRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PahealRipper.java index 4db47885..1dc9eaee 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PahealRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PahealRipper.java @@ -21,102 +21,105 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; -/** - * - * @author - */ -public class PahealRipper extends AbstractHTMLRipper{ - private static Map cookies=null; - private static Pattern gidPattern=null; +public class PahealRipper extends AbstractHTMLRipper { + private static Map cookies = null; + private static Pattern gidPattern = null; - private static Map getCookies() { - if(cookies==null){ - cookies=new HashMap(1); - cookies.put("ui-tnc-agreed","true"); - } - return cookies; - } + private static Map getCookies() { + if (cookies == null) { + cookies = new HashMap(1); + cookies.put("ui-tnc-agreed", "true"); + } + return cookies; + } - public PahealRipper(URL url) throws IOException { - super(url); - } + public PahealRipper(URL url) throws IOException { + super(url); + } - @Override - public String getDomain() { - return "rule34.paheal.net"; - } + @Override + public String getDomain() { + return "rule34.paheal.net"; + } - @Override - public String getHost() { - return "paheal"; - } + @Override + public String getHost() { + return "paheal"; + } - @Override - public Document getFirstPage() throws IOException { - return Http.url("http://rule34.paheal.net/post/list/"+getTerm(url)+"/1").cookies(getCookies()).get(); - } + @Override + public Document getFirstPage() throws IOException { + return Http.url("http://rule34.paheal.net/post/list/" + getTerm(url) + "/1").cookies(getCookies()).get(); + } - @Override - public Document getNextPage(Document page) throws IOException { - for(Element e:page.select("#paginator a")){ - if(e.text().toLowerCase().equals("next")) - return Http.url(e.absUrl("href")).cookies(getCookies()).get(); - } - - return null; - } + @Override + public Document getNextPage(Document page) throws IOException { + for (Element e : page.select("#paginator a")) { + if (e.text().toLowerCase().equals("next")) { + return Http.url(e.absUrl("href")).cookies(getCookies()).get(); + } + } - @Override - public List getURLsFromPage(Document page) { - Elements elements=page.select(".shm-thumb.thumb>a").not(".shm-thumb-link"); - List res=new ArrayList(elements.size()); - - for(Element e:elements) - res.add(e.absUrl("href")); - - return res; - } + return null; + } - @Override - public void downloadURL(URL url, int index) { - try { - String name=url.getPath(); - String ext=".png"; - - name=name.substring(name.lastIndexOf('/')+1); - if(name.indexOf('.')>=0){ - ext=name.substring(name.lastIndexOf('.')); - name=name.substring(0,name.length()-ext.length()); - } - - addURLToDownload(url,new File(workingDir.getCanonicalPath()+File.separator+Utils.filesystemSafe(new URI(name).getPath())+ext)); - } catch (IOException ex) { - Logger.getLogger(PahealRipper.class.getName()).log(Level.SEVERE, null, ex); - } catch (URISyntaxException ex) { - Logger.getLogger(PahealRipper.class.getName()).log(Level.SEVERE, null, ex); - } - } - - private String getTerm(URL url) throws MalformedURLException{ - if(gidPattern==null) - gidPattern=Pattern.compile("^https?://(www\\.)?rule34\\.paheal\\.net/post/list/([a-zA-Z0-9$_.+!*'(),%-]+)(/.*)?(#.*)?$"); + @Override + public List getURLsFromPage(Document page) { + Elements elements = page.select(".shm-thumb.thumb>a").not(".shm-thumb-link"); + List res = new ArrayList(elements.size()); - Matcher m = gidPattern.matcher(url.toExternalForm()); - if(m.matches()) - return m.group(2); - - throw new MalformedURLException("Expected paheal.net URL format: rule34.paheal.net/post/list/searchterm - got "+url+" instead"); - } + for (Element e : elements) { + res.add(e.absUrl("href")); + } - @Override - public String getGID(URL url) throws MalformedURLException { - try { - return Utils.filesystemSafe(new URI(getTerm(url)).getPath()); - } catch (URISyntaxException ex) { - Logger.getLogger(PahealRipper.class.getName()).log(Level.SEVERE, null, ex); - } - - throw new MalformedURLException("Expected paheal.net URL format: rule34.paheal.net/post/list/searchterm - got "+url+" instead"); - } - + return res; + } + + @Override + public void downloadURL(URL url, int index) { + try { + String name = url.getPath(); + String ext = ".png"; + + name = name.substring(name.lastIndexOf('/') + 1); + if (name.indexOf('.') >= 0) { + ext = name.substring(name.lastIndexOf('.')); + name = name.substring(0, name.length() - ext.length()); + } + + File outFile = new File(workingDir.getCanonicalPath() + + File.separator + + Utils.filesystemSafe(new URI(name).getPath()) + + ext); + addURLToDownload(url, outFile); + } catch (IOException ex) { + Logger.getLogger(PahealRipper.class.getName()).log(Level.SEVERE, null, ex); + } catch (URISyntaxException ex) { + Logger.getLogger(PahealRipper.class.getName()).log(Level.SEVERE, null, ex); + } + } + + private String getTerm(URL url) throws MalformedURLException { + if (gidPattern == null) { + gidPattern = Pattern.compile("^https?://(www\\.)?rule34\\.paheal\\.net/post/list/([a-zA-Z0-9$_.+!*'(),%-]+)(/.*)?(#.*)?$"); + } + + Matcher m = gidPattern.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(2); + } + + throw new MalformedURLException("Expected paheal.net URL format: rule34.paheal.net/post/list/searchterm - got " + url + " instead"); + } + + @Override + public String getGID(URL url) throws MalformedURLException { + try { + return Utils.filesystemSafe(new URI(getTerm(url)).getPath()); + } catch (URISyntaxException ex) { + Logger.getLogger(PahealRipper.class.getName()).log(Level.SEVERE, null, ex); + } + + throw new MalformedURLException("Expected paheal.net URL format: rule34.paheal.net/post/list/searchterm - got " + url + " instead"); + } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java index c217f86b..a18ee118 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java @@ -80,7 +80,7 @@ public class PornhubRipper extends AlbumRipper { public void rip() throws IOException { int index = 0; String nextUrl = this.url.toExternalForm(); - + if (albumDoc == null) { logger.info(" Retrieving album page " + nextUrl); sendUpdate(STATUS.LOADING_RESOURCE, nextUrl); @@ -127,7 +127,7 @@ public class PornhubRipper extends AlbumRipper { /** * Helper class to find and download images found on "image" pages - * + * * Handles case when site has IP-banned the user. */ private class PornhubImageThread extends Thread { @@ -144,28 +144,28 @@ public class PornhubRipper extends AlbumRipper { public void run() { fetchImage(); } - + private void fetchImage() { try { Document doc = Http.url(this.url) .referrer(this.url) .get(); - + // Find image Elements images = doc.select("#photoImageSection img"); Element image = images.first(); String imgsrc = image.attr("src"); logger.info("Found URL " + imgsrc + " via " + images.get(0)); - + // Provide prefix and let the AbstractRipper "guess" the filename String prefix = ""; if (Utils.getConfigBoolean("download.save_order", true)) { prefix = String.format("%03d_", index); } - + URL imgurl = new URL(url, imgsrc); addURLToDownload(imgurl, prefix); - + } catch (IOException e) { logger.error("[!] Exception while loading/parsing " + this.url, e); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index 067e7866..2dd2e38b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -26,13 +26,13 @@ public class RedditRipper extends AlbumRipper { private static final String HOST = "reddit"; private static final String DOMAIN = "reddit.com"; - + private static final String REDDIT_USER_AGENT = "RipMe:github/4pr0n/ripme:" + UpdateUtils.getThisJarVersion() + " (by /u/4_pr0n)"; private static final int SLEEP_TIME = 2000; //private static final String USER_AGENT = "ripme by /u/4_pr0n github.com/4pr0n/ripme"; - + private long lastRequestTime = 0; @Override @@ -68,9 +68,9 @@ public class RedditRipper extends AlbumRipper { } waitForThreads(); } - - - + + + private URL getAndParseAndReturnNext(URL url) throws IOException { JSONArray jsonArray = getJsonArrayFromURL(url), children; JSONObject json, data; @@ -108,7 +108,7 @@ public class RedditRipper extends AlbumRipper { } return nextURL; } - + private JSONArray getJsonArrayFromURL(URL url) throws IOException { // Wait 2 seconds before the next request long timeDiff = System.currentTimeMillis() - lastRequestTime; @@ -132,7 +132,7 @@ public class RedditRipper extends AlbumRipper { JSONArray jsonArray = new JSONArray(); if (jsonObj instanceof JSONObject) { jsonArray.put( (JSONObject) jsonObj); - } else if (jsonObj instanceof JSONArray){ + } else if (jsonObj instanceof JSONArray) { jsonArray = (JSONArray) jsonObj; } else { logger.warn("[!] Unable to parse JSON: " + jsonString); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java index 2d2e72aa..10caab9b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java @@ -194,9 +194,9 @@ public class TumblrRipper extends AlbumRipper { try { fileURL = new URL(photo.getJSONObject("original_size").getString("url")); m = p.matcher(fileURL.toString()); - if(m.matches()) { + if (m.matches()) { addURLToDownload(fileURL); - } else{ + } else { URL redirectedURL = Http.url(fileURL).ignoreContentType().response().url(); addURLToDownload(redirectedURL); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TwodgalleriesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TwodgalleriesRipper.java index c48a11c3..fa5281eb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TwodgalleriesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TwodgalleriesRipper.java @@ -36,7 +36,7 @@ public class TwodgalleriesRipper extends AbstractHTMLRipper { public String getDomain() { return "2dgalleries.com"; } - + @Override public String getGID(URL url) throws MalformedURLException { Pattern p; Matcher m; @@ -108,7 +108,7 @@ public class TwodgalleriesRipper extends AbstractHTMLRipper { public void downloadURL(URL url, int index) { addURLToDownload(url, getPrefix(index)); } - + private void login() throws IOException { Response resp = Http.url(this.url).response(); cookies = resp.cookies(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java index 7b0c8bbb..93bfeab6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java @@ -29,7 +29,7 @@ public class VidbleRipper extends AbstractHTMLRipper { public String getDomain() { return "vidble.com"; } - + @Override public String getGID(URL url) throws MalformedURLException { Pattern p; Matcher m; @@ -62,7 +62,7 @@ public class VidbleRipper extends AbstractHTMLRipper { for (Element img : imgs) { String src = img.absUrl("src"); src = src.replaceAll("_[a-zA-Z]{3,5}", ""); - + if (!src.equals("")) { imageURLs.add(src); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java index 15ed3671..515d227f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java @@ -11,155 +11,151 @@ import java.util.regex.Pattern; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; public class WordpressComicRipper extends AbstractHTMLRipper { - public WordpressComicRipper(URL url) throws IOException { - super(url); + super(url); } public static List explicit_domains = Arrays.asList("www.totempole666.com", - "buttsmithy.com", "themonsterunderthebed.net", "prismblush.com"); - @Override - public String getHost() { - String host = url.toExternalForm().split("/")[2]; - return host; - } + "buttsmithy.com", "themonsterunderthebed.net", "prismblush.com"); - @Override - public String getDomain() { - String host = url.toExternalForm().split("/")[2]; - return host; - } + @Override + public String getHost() { + String host = url.toExternalForm().split("/")[2]; + return host; + } - @Override - public boolean canRip(URL url) { - String url_name = url.toExternalForm(); - if (explicit_domains.contains(url_name.split("/")[2]) == true) { - Pattern totempole666Pat = Pattern.compile("https?://www\\.totempole666.com/comic/([a-zA-Z0-9_-]*)/?$"); - Matcher totempole666Mat = totempole666Pat.matcher(url.toExternalForm()); - if (totempole666Mat.matches()) { - return true; - } + @Override + public String getDomain() { + String host = url.toExternalForm().split("/")[2]; + return host; + } - Pattern buttsmithyPat = Pattern.compile("https?://buttsmithy.com/archives/comic/([a-zA-Z0-9_-]*)/?$"); - Matcher buttsmithyMat = buttsmithyPat.matcher(url.toExternalForm()); - if (buttsmithyMat.matches()) { - return true; - } - - Pattern theMonsterUnderTheBedPat = Pattern.compile("https?://themonsterunderthebed.net/\\?comic=([a-zA-Z0-9_-]*)/?$"); - Matcher theMonsterUnderTheBedMat = theMonsterUnderTheBedPat.matcher(url.toExternalForm()); - if (theMonsterUnderTheBedMat.matches()) { - return true; - } - - Pattern prismblushPat = Pattern.compile("https?://prismblush.com/comic/([a-zA-Z0-9_-]*)/?$"); - Matcher prismblushMat = prismblushPat.matcher(url.toExternalForm()); - if (prismblushMat.matches()) { - return true; - } - - } - return false; - } - - @Override - public String getAlbumTitle(URL url) throws MalformedURLException { - Pattern totempole666Pat = Pattern.compile("(?:https?://)?(?:www\\.)?totempole666.com\\/comic/([a-zA-Z0-9_-]*)/?$"); + @Override + public boolean canRip(URL url) { + String url_name = url.toExternalForm(); + if (explicit_domains.contains(url_name.split("/")[2]) == true) { + Pattern totempole666Pat = Pattern.compile("https?://www\\.totempole666.com/comic/([a-zA-Z0-9_-]*)/?$"); Matcher totempole666Mat = totempole666Pat.matcher(url.toExternalForm()); if (totempole666Mat.matches()) { - return "totempole666.com" + "_" + "The_cummoner"; + return true; } Pattern buttsmithyPat = Pattern.compile("https?://buttsmithy.com/archives/comic/([a-zA-Z0-9_-]*)/?$"); Matcher buttsmithyMat = buttsmithyPat.matcher(url.toExternalForm()); if (buttsmithyMat.matches()) { - return "buttsmithy.com" + "_" + "Alfie"; + return true; } - Pattern theMonsterUnderTheBedPat = Pattern.compile("https?://themonsterunderthebed.net/?comic=([a-zA-Z0-9_-]*)/?$"); + Pattern theMonsterUnderTheBedPat = Pattern.compile("https?://themonsterunderthebed.net/\\?comic=([a-zA-Z0-9_-]*)/?$"); Matcher theMonsterUnderTheBedMat = theMonsterUnderTheBedPat.matcher(url.toExternalForm()); if (theMonsterUnderTheBedMat.matches()) { - return "themonsterunderthebed.net_TheMonsterUnderTheBed"; + return true; } Pattern prismblushPat = Pattern.compile("https?://prismblush.com/comic/([a-zA-Z0-9_-]*)/?$"); Matcher prismblushMat = prismblushPat.matcher(url.toExternalForm()); if (prismblushMat.matches()) { - return "prismblush.com_" + prismblushMat.group(1).replaceAll("-pg-\\d+", ""); + return true; } - - return super.getAlbumTitle(url); } - - @Override - public String getGID(URL url) throws MalformedURLException { - String url_name = url.toExternalForm(); - // We shouldn't need to return any GID - if (explicit_domains.contains(url_name.split("/")[2]) == true) { - return ""; - } - throw new MalformedURLException("You should never see this error message"); - } - - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - - @Override - public Document getNextPage(Document doc) throws IOException { - // Find next page - String nextPage = ""; - Element elem = null; - if (explicit_domains.contains("www.totempole666.com") == true - || explicit_domains.contains("buttsmithy.com") == true - || explicit_domains.contains("themonsterunderthebed.net") - || explicit_domains.contains("prismblush.com")) { - elem = doc.select("a.comic-nav-next").first(); - if (elem == null) { - throw new IOException("No more pages"); - } - nextPage = elem.attr("href"); - } - if (nextPage == "") { - throw new IOException("No more pages"); - } - else { - return Http.url(nextPage).get(); - } - } - - @Override - public List getURLsFromPage(Document doc) { - List result = new ArrayList(); - if (explicit_domains.contains("www.totempole666.com") == true - || explicit_domains.contains("buttsmithy.com") == true - || explicit_domains.contains("themonsterunderthebed.net") - || explicit_domains.contains("prismblush.com")) { - Element elem = doc.select("div.comic-table > div#comic > a > img").first(); - // If doc is the last page in the comic then elem.attr("src") returns null - // because there is no link to the next page - if (elem == null) { - logger.debug("Got last page in totempole666 comic"); - elem = doc.select("div.comic-table > div#comic > img").first(); - } - result.add(elem.attr("src")); - } - return result; - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - } - - + return false; } + + @Override + public String getAlbumTitle(URL url) throws MalformedURLException { + Pattern totempole666Pat = Pattern.compile("(?:https?://)?(?:www\\.)?totempole666.com\\/comic/([a-zA-Z0-9_-]*)/?$"); + Matcher totempole666Mat = totempole666Pat.matcher(url.toExternalForm()); + if (totempole666Mat.matches()) { + return "totempole666.com" + "_" + "The_cummoner"; + } + + Pattern buttsmithyPat = Pattern.compile("https?://buttsmithy.com/archives/comic/([a-zA-Z0-9_-]*)/?$"); + Matcher buttsmithyMat = buttsmithyPat.matcher(url.toExternalForm()); + if (buttsmithyMat.matches()) { + return "buttsmithy.com" + "_" + "Alfie"; + } + + Pattern theMonsterUnderTheBedPat = Pattern.compile("https?://themonsterunderthebed.net/?comic=([a-zA-Z0-9_-]*)/?$"); + Matcher theMonsterUnderTheBedMat = theMonsterUnderTheBedPat.matcher(url.toExternalForm()); + if (theMonsterUnderTheBedMat.matches()) { + return "themonsterunderthebed.net_TheMonsterUnderTheBed"; + } + + Pattern prismblushPat = Pattern.compile("https?://prismblush.com/comic/([a-zA-Z0-9_-]*)/?$"); + Matcher prismblushMat = prismblushPat.matcher(url.toExternalForm()); + if (prismblushMat.matches()) { + return "prismblush.com_" + prismblushMat.group(1).replaceAll("-pg-\\d+", ""); + } + + return super.getAlbumTitle(url); + } + + @Override + public String getGID(URL url) throws MalformedURLException { + String url_name = url.toExternalForm(); + // We shouldn't need to return any GID + if (explicit_domains.contains(url_name.split("/")[2]) == true) { + return ""; + } + throw new MalformedURLException("You should never see this error message"); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + @Override + public Document getNextPage(Document doc) throws IOException { + // Find next page + String nextPage = ""; + Element elem = null; + if (explicit_domains.contains("www.totempole666.com") == true + || explicit_domains.contains("buttsmithy.com") == true + || explicit_domains.contains("themonsterunderthebed.net") + || explicit_domains.contains("prismblush.com")) { + elem = doc.select("a.comic-nav-next").first(); + if (elem == null) { + throw new IOException("No more pages"); + } + nextPage = elem.attr("href"); + } + + if (nextPage == "") { + throw new IOException("No more pages"); + } else { + return Http.url(nextPage).get(); + } + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList(); + if (explicit_domains.contains("www.totempole666.com") == true + || explicit_domains.contains("buttsmithy.com") == true + || explicit_domains.contains("themonsterunderthebed.net") + || explicit_domains.contains("prismblush.com")) { + Element elem = doc.select("div.comic-table > div#comic > a > img").first(); + // If doc is the last page in the comic then elem.attr("src") returns null + // because there is no link to the next page + if (elem == null) { + logger.debug("Got last page in totempole666 comic"); + elem = doc.select("div.comic-table > div#comic > img").first(); + } + result.add(elem.attr("src")); + } + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XbooruRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XbooruRipper.java index 6d7d70fc..12f4bdf2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XbooruRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XbooruRipper.java @@ -1,4 +1,3 @@ - package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.ripper.AbstractHTMLRipper; @@ -18,80 +17,79 @@ import java.util.regex.Pattern; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -/** - * - * @author - */ -public class XbooruRipper extends AbstractHTMLRipper{ - private static Pattern gidPattern=null; +public class XbooruRipper extends AbstractHTMLRipper { + private static Pattern gidPattern = null; - public XbooruRipper(URL url) throws IOException { - super(url); - } + public XbooruRipper(URL url) throws IOException { + super(url); + } - @Override - public String getDomain() { - return "xbooru.com"; - } + @Override + public String getDomain() { + return "xbooru.com"; + } - @Override - public String getHost() { - return "xbooru"; - } - - private String getPage(int num) throws MalformedURLException{ - return "http://xbooru.com/index.php?page=dapi&s=post&q=index&pid="+num+"&tags="+getTerm(url); - } + @Override + public String getHost() { + return "xbooru"; + } - @Override - public Document getFirstPage() throws IOException { - return Http.url(getPage(0)).get(); - } + private String getPage(int num) throws MalformedURLException { + return "http://xbooru.com/index.php?page=dapi&s=post&q=index&pid=" + num + "&tags=" + getTerm(url); + } - @Override - public Document getNextPage(Document doc) throws IOException { - int offset=Integer.parseInt(doc.getElementsByTag("posts").first().attr("offset")); - int num=Integer.parseInt(doc.getElementsByTag("posts").first().attr("count")); - - if(offset+100>num) - return null; - - return Http.url(getPage(offset/100+1)).get(); - } + @Override + public Document getFirstPage() throws IOException { + return Http.url(getPage(0)).get(); + } - @Override - public List getURLsFromPage(Document page) { - List res=new ArrayList(100); - for(Element e:page.getElementsByTag("post")) - res.add(e.absUrl("file_url")+"#"+e.attr("id")); - return res; - } + @Override + public Document getNextPage(Document doc) throws IOException { + int offset = Integer.parseInt(doc.getElementsByTag("posts").first().attr("offset")); + int num = Integer.parseInt(doc.getElementsByTag("posts").first().attr("count")); - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url,Utils.getConfigBoolean("download.save_order",true)?url.getRef()+"-":""); - } - - private String getTerm(URL url) throws MalformedURLException{ - if(gidPattern==null) - gidPattern=Pattern.compile("^https?://(www\\.)?xbooru\\.com/(index.php)?.*([?&]tags=([a-zA-Z0-9$_.+!*'(),%-]+))(\\&|(#.*)?$)"); + if (offset + 100 > num) { + return null; + } - Matcher m = gidPattern.matcher(url.toExternalForm()); - if(m.matches()) - return m.group(4); - - throw new MalformedURLException("Expected xbooru.com URL format: xbooru.com/index.php?tags=searchterm - got "+url+" instead"); - } + return Http.url(getPage(offset / 100 + 1)).get(); + } - @Override - public String getGID(URL url) throws MalformedURLException { - try { - return Utils.filesystemSafe(new URI(getTerm(url)).getPath()); - } catch (URISyntaxException ex) { - Logger.getLogger(PahealRipper.class.getName()).log(Level.SEVERE, null, ex); - } - - throw new MalformedURLException("Expected xbooru.com URL format: xbooru.com/index.php?tags=searchterm - got "+url+" instead"); - } - + @Override + public List getURLsFromPage(Document page) { + List res = new ArrayList(100); + for (Element e : page.getElementsByTag("post")) { + res.add(e.absUrl("file_url") + "#" + e.attr("id")); + } + return res; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, Utils.getConfigBoolean("download.save_order", true) ? url.getRef() + "-" : ""); + } + + private String getTerm(URL url) throws MalformedURLException { + if (gidPattern == null) { + gidPattern = Pattern.compile("^https?://(www\\.)?xbooru\\.com/(index.php)?.*([?&]tags=([a-zA-Z0-9$_.+!*'(),%-]+))(\\&|(#.*)?$)"); + } + + Matcher m = gidPattern.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(4); + } + + throw new MalformedURLException("Expected xbooru.com URL format: xbooru.com/index.php?tags=searchterm - got " + url + " instead"); + } + + @Override + public String getGID(URL url) throws MalformedURLException { + try { + return Utils.filesystemSafe(new URI(getTerm(url)).getPath()); + } catch (URISyntaxException ex) { + Logger.getLogger(PahealRipper.class.getName()).log(Level.SEVERE, null, ex); + } + + throw new MalformedURLException("Expected xbooru.com URL format: xbooru.com/index.php?tags=searchterm - got " + url + " instead"); + } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java index b200c413..32cda5c3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java @@ -4,7 +4,6 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -14,10 +13,8 @@ import java.util.regex.Pattern; import org.jsoup.Connection.Response; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; public class ZizkiRipper extends AbstractHTMLRipper { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ripperhelpers/ChanSite.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ripperhelpers/ChanSite.java index c8ec3d4f..4b4dd87f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ripperhelpers/ChanSite.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ripperhelpers/ChanSite.java @@ -3,23 +3,27 @@ package com.rarchives.ripme.ripper.rippers.ripperhelpers; import java.util.List; public class ChanSite { - //The domains where the threads are hosted. + // The domains where the threads are hosted. public List domains; - //The domains where the images are hosted. + // The domains where the images are hosted. public List cdnDomains; - public ChanSite(List Domains, List CdnDomains){ - if(Domains.isEmpty()) + public ChanSite(List Domains, List CdnDomains) { + if (Domains.isEmpty()) { throw new IllegalArgumentException("Domains"); - if(CdnDomains.isEmpty()) + } + if (CdnDomains.isEmpty()) { throw new IllegalArgumentException("CdnDomains"); + } domains = Domains; - cdnDomains = CdnDomains; + cdnDomains = CdnDomains; } - public ChanSite(List Domains){ - if(Domains.isEmpty()) - throw new IllegalArgumentException("Domains"); + + public ChanSite(List Domains) { + if (Domains.isEmpty()) { + throw new IllegalArgumentException("Domains"); + } domains = Domains; - cdnDomains = Domains; + cdnDomains = Domains; } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/tamindirmp3.java b/src/main/java/com/rarchives/ripme/ripper/rippers/tamindirmp3.java index 208932d0..7657a5af 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/tamindirmp3.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/tamindirmp3.java @@ -39,13 +39,13 @@ public class tamindirmp3 extends AbstractHTMLRipper { throw new MalformedURLException("Expected tamindir.com URL format: " + "tamindir.com/files/albumid - got " + url + "instead"); } - + @Override public Document getFirstPage() throws IOException { return Http.url(url).get(); - + } - + @Override public List getURLsFromPage(Document doc) { List music = new ArrayList(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java index 19048f8a..fb38b216 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java @@ -33,7 +33,7 @@ public class CliphunterRipper extends VideoRipper { Matcher m = p.matcher(url.toExternalForm()); return m.matches(); } - + @Override public URL sanitizeURL(URL url) throws MalformedURLException { return url; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java index 54a66769..dd0d20eb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java @@ -29,7 +29,7 @@ public class GfycatRipper extends VideoRipper { public boolean canRip(URL url) { return url.getHost().endsWith(HOST); } - + @Override public URL sanitizeURL(URL url) throws MalformedURLException { return url; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java index 3721362f..7f10323d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java @@ -69,9 +69,9 @@ public class PornhubRipper extends VideoRipper { vidUrl = null; for (String quality : new String[] {"quality_1080p", "quality_720p", "quality_480p", "quality_240p"}) { - Pattern pv = Pattern.compile("^.*var player_" + quality + " = '([^']*)'.*$", Pattern.DOTALL); - Matcher mv = pv.matcher(html); - if (mv.matches()) { + Pattern pv = Pattern.compile("^.*var player_" + quality + " = '([^']*)'.*$", Pattern.DOTALL); + Matcher mv = pv.matcher(html); + if (mv.matches()) { vidUrl = mv.group(1); break; } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java index 5d971b87..7f215770 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java @@ -32,7 +32,7 @@ public class VidearnRipper extends VideoRipper { Matcher m = p.matcher(url.toExternalForm()); return m.matches(); } - + @Override public URL sanitizeURL(URL url) throws MalformedURLException { return url; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VineRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VineRipper.java index 02aea14e..09637b84 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VineRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VineRipper.java @@ -32,7 +32,7 @@ public class VineRipper extends VideoRipper { Matcher m = p.matcher(url.toExternalForm()); return m.matches(); } - + @Override public URL sanitizeURL(URL url) throws MalformedURLException { return url; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java index a5ce360d..c610a470 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java @@ -30,7 +30,7 @@ public class VkRipper extends VideoRipper { Matcher m = p.matcher(url.toExternalForm()); return m.matches(); } - + @Override public URL sanitizeURL(URL url) throws MalformedURLException { return url; @@ -57,7 +57,7 @@ public class VkRipper extends VideoRipper { addURLToDownload(new URL(videoURL), HOST + "_" + getGID(this.url)); waitForThreads(); } - + public static String getVideoURLAtPage(String url) throws IOException { Document doc = Http.url(url) .userAgent(USER_AGENT) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/XhamsterRipper.java index cb6d5725..061ec2fb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/XhamsterRipper.java @@ -31,7 +31,7 @@ public class XhamsterRipper extends VideoRipper { Matcher m = p.matcher(url.toExternalForm()); return m.matches(); } - + @Override public URL sanitizeURL(URL url) throws MalformedURLException { return url; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java index 53edc9f6..06841ce9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java @@ -33,7 +33,7 @@ public class XvideosRipper extends VideoRipper { Matcher m = p.matcher(url.toExternalForm()); return m.matches(); } - + @Override public URL sanitizeURL(URL url) throws MalformedURLException { return url; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/YoupornRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/YoupornRipper.java index 493bb89a..a5e00b4b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/YoupornRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/YoupornRipper.java @@ -32,7 +32,7 @@ public class YoupornRipper extends VideoRipper { Matcher m = p.matcher(url.toExternalForm()); return m.matches(); } - + @Override public URL sanitizeURL(URL url) throws MalformedURLException { return url; diff --git a/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java b/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java index 8b68a4b2..1a07da97 100644 --- a/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java @@ -14,7 +14,7 @@ import static com.rarchives.ripme.App.logger; public class ClipboardUtils { private static AutoripThread autoripThread = new AutoripThread(); - + public static void setClipboardAutoRip(boolean enabled) { if (enabled) { autoripThread.kill(); @@ -44,7 +44,7 @@ public class ClipboardUtils { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); - } + } return null; } } @@ -61,15 +61,15 @@ class AutoripThread extends Thread { String clipboard = ClipboardUtils.getClipboardString(); if (clipboard != null) { Pattern p = Pattern.compile( - "\\b(((ht|f)tp(s?)\\:\\/\\/|~\\/|\\/)|www.)" + - "(\\w+:\\w+@)?(([-\\w]+\\.)+(com|org|net|gov" + - "|mil|biz|info|mobi|name|aero|jobs|museum" + - "|travel|[a-z]{2}))(:[\\d]{1,5})?" + - "(((\\/([-\\w~!$+|.,=]|%[a-f\\d]{2})+)+|\\/)+|\\?|#)?" + - "((\\?([-\\w~!$+|.,*:]|%[a-f\\d{2}])+=?" + - "([-\\w~!$+|.,*:=]|%[a-f\\d]{2})*)" + - "(&(?:[-\\w~!$+|.,*:]|%[a-f\\d{2}])+=?" + - "([-\\w~!$+|.,*:=]|%[a-f\\d]{2})*)*)*" + + "\\b(((ht|f)tp(s?)\\:\\/\\/|~\\/|\\/)|www.)" + + "(\\w+:\\w+@)?(([-\\w]+\\.)+(com|org|net|gov" + + "|mil|biz|info|mobi|name|aero|jobs|museum" + + "|travel|[a-z]{2}))(:[\\d]{1,5})?" + + "(((\\/([-\\w~!$+|.,=]|%[a-f\\d]{2})+)+|\\/)+|\\?|#)?" + + "((\\?([-\\w~!$+|.,*:]|%[a-f\\d{2}])+=?" + + "([-\\w~!$+|.,*:=]|%[a-f\\d]{2})*)" + + "(&(?:[-\\w~!$+|.,*:]|%[a-f\\d{2}])+=?" + + "([-\\w~!$+|.,*:=]|%[a-f\\d]{2})*)*)*" + "(#([-\\w~!$+|.,*:=]|%[a-f\\d]{2})*)?\\b"); Matcher m = p.matcher(clipboard); while (m.find()) { @@ -87,7 +87,7 @@ class AutoripThread extends Thread { e.printStackTrace(); } } - + public void kill() { isRunning = false; } diff --git a/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java b/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java index c487cc64..cd085d20 100644 --- a/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java +++ b/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java @@ -27,12 +27,12 @@ public class QueueMenuMouseListener extends MouseAdapter { removeSelected = new AbstractAction("Remove Selected") { @Override public void actionPerformed(ActionEvent ae) { - Object o = queueList.getSelectedValue(); - while (o != null) { - queueListModel.removeElement(o); - o = queueList.getSelectedValue(); - } - updateUI(); + Object o = queueList.getSelectedValue(); + while (o != null) { + queueListModel.removeElement(o); + o = queueList.getSelectedValue(); + } + updateUI(); } }; popup.add(removeSelected); @@ -40,8 +40,8 @@ public class QueueMenuMouseListener extends MouseAdapter { clearQueue = new AbstractAction("Remove All") { @Override public void actionPerformed(ActionEvent ae) { - queueListModel.removeAllElements(); - updateUI(); + queueListModel.removeAllElements(); + updateUI(); } }; popup.add(clearQueue); diff --git a/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java b/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java index 9d1c73d3..bdd68e3b 100644 --- a/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java +++ b/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java @@ -10,7 +10,7 @@ public class RipStatusMessage { DOWNLOAD_STARTED("Download Started"), DOWNLOAD_COMPLETE("Download Complete"), DOWNLOAD_ERRORED("Download Errored"), - RIP_COMPLETE("Rip Complete"), + RIP_COMPLETE("Rip Complete"), DOWNLOAD_WARN("Download problem"), TOTAL_BYTES("Total bytes"), COMPLETED_BYTES("Completed bytes"), @@ -29,11 +29,11 @@ public class RipStatusMessage { this.status = status; this.object = object; } - + public STATUS getStatus() { return status; } - + public Object getObject() { return object; } diff --git a/src/main/java/com/rarchives/ripme/utils/Base64.java b/src/main/java/com/rarchives/ripme/utils/Base64.java index d38f545e..961ce9df 100644 --- a/src/main/java/com/rarchives/ripme/utils/Base64.java +++ b/src/main/java/com/rarchives/ripme/utils/Base64.java @@ -3,16 +3,15 @@ package com.rarchives.ripme.utils; /** * Base64 encoder/decoder * From http://stackoverflow.com/a/4265472 - * */ public class Base64 { private final static char[] ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".toCharArray(); - private static int[] toInt = new int[128]; + private static int[] toInt = new int[128]; static { - for(int i=0; i< ALPHABET.length; i++){ - toInt[ALPHABET[i]]= i; + for (int i = 0; i < ALPHABET.length; i++) { + toInt[ALPHABET[i]] = i; } } @@ -22,12 +21,12 @@ public class Base64 { * @param buf the byte array (not null) * @return the translated Base64 string (not null) */ - public static String encode(byte[] buf){ + public static String encode(byte[] buf) { int size = buf.length; char[] ar = new char[((size + 2) / 3) * 4]; int a = 0; - int i=0; - while(i < size){ + int i = 0; + while (i < size) { byte b0 = buf[i++]; byte b1 = (i < size) ? buf[i++] : 0; byte b2 = (i < size) ? buf[i++] : 0; @@ -38,9 +37,9 @@ public class Base64 { ar[a++] = ALPHABET[((b1 << 2) | ((b2 & 0xFF) >> 6)) & mask]; ar[a++] = ALPHABET[b2 & mask]; } - switch(size % 3){ - case 1: ar[--a] = '='; - case 2: ar[--a] = '='; + switch (size % 3) { + case 1: ar[--a] = '='; + case 2: ar[--a] = '='; } return new String(ar); } @@ -51,26 +50,26 @@ public class Base64 { * @param s the Base64 string (not null) * @return the byte array (not null) */ - public static byte[] decode(String s){ - int delta = s.endsWith( "==" ) ? 2 : s.endsWith( "=" ) ? 1 : 0; - byte[] buffer = new byte[s.length()*3/4 - delta]; + public static byte[] decode(String s) { + int delta = s.endsWith("==") ? 2 : s.endsWith("=") ? 1 : 0; + byte[] buffer = new byte[s.length() * 3 / 4 - delta]; int mask = 0xFF; int index = 0; - for(int i=0; i< s.length(); i+=4){ - int c0 = toInt[s.charAt( i )]; - int c1 = toInt[s.charAt( i + 1)]; - buffer[index++]= (byte)(((c0 << 2) | (c1 >> 4)) & mask); - if(index >= buffer.length){ + for (int i = 0; i < s.length(); i += 4) { + int c0 = toInt[s.charAt(i)]; + int c1 = toInt[s.charAt(i + 1)]; + buffer[index++] = (byte) (((c0 << 2) | (c1 >> 4)) & mask); + if (index >= buffer.length) { return buffer; } - int c2 = toInt[s.charAt( i + 2)]; - buffer[index++]= (byte)(((c1 << 4) | (c2 >> 2)) & mask); - if(index >= buffer.length){ + int c2 = toInt[s.charAt(i + 2)]; + buffer[index++] = (byte) (((c1 << 4) | (c2 >> 2)) & mask); + if (index >= buffer.length) { return buffer; } - int c3 = toInt[s.charAt( i + 3 )]; - buffer[index++]= (byte)(((c2 << 6) | c3) & mask); + int c3 = toInt[s.charAt(i + 3)]; + buffer[index++] = (byte) (((c2 << 6) | c3) & mask); } return buffer; - } + } } diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index 073d4717..25bf67d7 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -23,7 +23,6 @@ import com.rarchives.ripme.ripper.rippers.VidbleRipper; import com.rarchives.ripme.ripper.rippers.video.GfycatRipper; import com.rarchives.ripme.ripper.rippers.EroShareRipper; - public class RipUtils { private static final Logger logger = Logger.getLogger(RipUtils.class); @@ -32,7 +31,7 @@ public class RipUtils { logger.debug("Checking " + url); // Imgur album - if ((url.getHost().endsWith("imgur.com")) + if ((url.getHost().endsWith("imgur.com")) && url.toExternalForm().contains("imgur.com/a/")) { try { logger.debug("Fetching imgur album at " + url); @@ -47,7 +46,7 @@ public class RipUtils { return result; } else if (url.getHost().endsWith("imgur.com") && url.toExternalForm().contains(",")) { - // Imgur image series. + // Imgur image series. try { logger.debug("Fetching imgur series at " + url); ImgurAlbum imgurAlbum = ImgurRipper.getImgurSeries(url); @@ -91,7 +90,7 @@ public class RipUtils { } return result; } - + Pattern p = Pattern.compile("https?://i.reddituploads.com/([a-zA-Z0-9]+)\\?.*"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { @@ -117,9 +116,9 @@ public class RipUtils { logger.error("[!] Not a valid URL: '" + url + "'", e); } } - - if (url.getHost().equals("imgur.com") || - url.getHost().equals("m.imgur.com")){ + + if (url.getHost().equals("imgur.com") || + url.getHost().equals("m.imgur.com")) { try { // Fetch the page Document doc = Jsoup.connect(url.toExternalForm()) @@ -138,13 +137,13 @@ public class RipUtils { } catch (IOException ex) { logger.error("[!] Error", ex); } - + } - + logger.error("[!] Unable to rip URL: " + url); return result; } - + public static Pattern getURLRegex() { return Pattern.compile("(https?://[a-zA-Z0-9\\-\\.]+\\.[a-zA-Z]{2,3}(/\\S*))"); } @@ -165,18 +164,6 @@ public class RipUtils { if (url == null) url = urlFromSiteDirectoryName(dir, "vinebox", "http://finebox.co/u/", ""); if (url == null) url = urlFromSiteDirectoryName(dir, "imgbox", "http://imgbox.com/g/", ""); if (url == null) url = urlFromSiteDirectoryName(dir, "modelmayhem", "http://www.modelmayhem.com/", ""); - /* - if (url == null) url = urlFromSiteDirectoryName(dir, "", "", ""); - if (url == null) url = urlFromSiteDirectoryName(dir, "", "", ""); - if (url == null) url = urlFromSiteDirectoryName(dir, "", "", ""); - if (url == null) url = urlFromSiteDirectoryName(dir, "", "", ""); - if (url == null) url = urlFromSiteDirectoryName(dir, "", "", ""); - if (url == null) url = urlFromSiteDirectoryName(dir, "", "", ""); - if (url == null) url = urlFromSiteDirectoryName(dir, "", "", ""); - if (url == null) url = urlFromSiteDirectoryName(dir, "", "", ""); - if (url == null) url = urlFromSiteDirectoryName(dir, "", "", ""); - if (url == null) url = urlFromSiteDirectoryName(dir, "", "", ""); - */ //if (url == null) url = urlFromSiteDirectoryName(dir, "8muses", "http://www.8muses.com/index/category/", ""); return url; } @@ -248,9 +235,8 @@ public class RipUtils { List fields = Arrays.asList(dir.split("_")); String album = fields.get(1); String url = "http://"; - if ( (fields.contains("top") || fields.contains("new")) - && (fields.contains("year") || fields.contains("month") || fields.contains("week") || fields.contains("all")) - ) { + if ((fields.contains("top") || fields.contains("new")) + && (fields.contains("year") || fields.contains("month") || fields.contains("week") || fields.contains("all"))) { // Subreddit fields.remove(0); // "imgur" String sub = ""; diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index f3957360..451fbf1a 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -34,7 +34,6 @@ import com.rarchives.ripme.ripper.AbstractRipper; * Common utility functions used in various places throughout the project. */ public class Utils { - public static final String RIP_DIRECTORY = "rips"; private static final String configFile = "rip.properties"; private static final Logger logger = Logger.getLogger(Utils.class); @@ -50,7 +49,7 @@ public class Utils { } config = new PropertiesConfiguration(configPath); logger.info("Loaded " + config.getPath()); - if (f.exists()){ + if (f.exists()) { // Config was loaded from file if ( !config.containsKey("twitter.auth") || !config.containsKey("twitter.max_requests") @@ -171,16 +170,18 @@ public class Utils { public static String stripURLParameter(String url, String parameter) { int paramIndex = url.indexOf("?" + parameter); boolean wasFirstParam = true; - if(paramIndex < 0) { + if (paramIndex < 0) { wasFirstParam = false; paramIndex = url.indexOf("&" + parameter); } - if(paramIndex > 0) { + if (paramIndex > 0) { int nextParam = url.indexOf("&", paramIndex+1); - if(nextParam != -1) { + if (nextParam != -1) { String c = "&"; - if(wasFirstParam) c = "?"; + if (wasFirstParam) { + c = "?"; + } url = url.substring(0, paramIndex) + c + url.substring(nextParam+1, url.length()); } else { url = url.substring(0, paramIndex); @@ -250,10 +251,10 @@ public class Utils { jarPath = URLDecoder.decode(jarPath, "UTF-8"); JarFile jarFile = new JarFile(jarPath); Enumeration entries = jarFile.entries(); - while(entries.hasMoreElements()) { + while (entries.hasMoreElements()) { JarEntry nextElement = entries.nextElement(); String entryName = nextElement.getName(); - if(entryName.startsWith(relPath) + if (entryName.startsWith(relPath) && entryName.length() > (relPath.length() + "/".length()) && !nextElement.isDirectory()) { String className = entryName.replace('/', '.').replace('\\', '.').replace(".class", ""); @@ -393,7 +394,7 @@ public class Utils { /** * Parses an URL query - * + * * @param query * The query part of an URL * @return The map of all query parameters @@ -401,7 +402,7 @@ public class Utils { public static Map parseUrlQuery(String query) { Map res = new HashMap(); - if (query.equals("")){ + if (query.equals("")) { return res; } @@ -410,9 +411,9 @@ public class Utils { try { for (String part : parts) { - if ((pos = part.indexOf('=')) >= 0){ + if ((pos = part.indexOf('=')) >= 0) { res.put(URLDecoder.decode(part.substring(0, pos), "UTF-8"), URLDecoder.decode(part.substring(pos + 1), "UTF-8")); - }else{ + } else { res.put(URLDecoder.decode(part, "UTF-8"), ""); } } @@ -426,7 +427,7 @@ public class Utils { /** * Parses an URL query and returns the requested parameter's value - * + * * @param query * The query part of an URL * @param key @@ -434,7 +435,7 @@ public class Utils { * @return The associated value or null if key wasn't found */ public static String parseUrlQuery(String query, String key) { - if (query.equals("")){ + if (query.equals("")) { return null; } @@ -444,7 +445,7 @@ public class Utils { try { for (String part : parts) { if ((pos = part.indexOf('=')) >= 0) { - if (URLDecoder.decode(part.substring(0, pos), "UTF-8").equals(key)){ + if (URLDecoder.decode(part.substring(0, pos), "UTF-8").equals(key)) { return URLDecoder.decode(part.substring(pos + 1), "UTF-8"); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java index 39f57576..153288d8 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java @@ -106,7 +106,7 @@ public class BasicRippersTest extends RippersTest { testRipper(ripper); } */ - + public void test500pxAlbum() throws IOException { FivehundredpxRipper ripper = new FivehundredpxRipper(new URL("https://marketplace.500px.com/alexander_hurman")); testRipper(ripper); @@ -261,7 +261,7 @@ public class BasicRippersTest extends RippersTest { AbstractRipper ripper = new VineRipper(new URL("https://vine.co/u/954440445776334848")); testRipper(ripper); } - + public void testVkSubalbumRip() throws IOException { VkRipper ripper = new VkRipper(new URL("http://vk.com/album45506334_0")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/InstagramRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/InstagramRipperTest.java index 984bc29f..bcf8a711 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/InstagramRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/InstagramRipperTest.java @@ -10,7 +10,7 @@ import java.util.Map; import com.rarchives.ripme.ripper.rippers.InstagramRipper; public class InstagramRipperTest extends RippersTest { - + public void testInstagramGID() throws IOException { Map testURLs = new HashMap(); testURLs.put(new URL("http://instagram.com/Test_User"), "Test_User"); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java index 72249fd8..409dfee9 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java @@ -12,7 +12,7 @@ import com.rarchives.ripme.ripper.rippers.video.XvideosRipper; import com.rarchives.ripme.ripper.rippers.video.YoupornRipper; public class VideoRippersTest extends RippersTest { - + /** * Helper method for testing a video ripper * @param ripper The video ripper @@ -44,7 +44,7 @@ public class VideoRippersTest extends RippersTest { videoTestHelper(ripper); } } - + public void testPornhubRipper() throws IOException { List contentURLs = new ArrayList(); contentURLs.add(new URL("http://www.pornhub.com/view_video.php?viewkey=993166542"));