Trim trailing whitespace, remove some unused imports.
This commit is contained in:
parent
3e219089f9
commit
65b11d1e00
@ -16,7 +16,7 @@ import com.rarchives.ripme.utils.Utils;
|
|||||||
* Simplified ripper, designed for ripping from sites by parsing HTML.
|
* Simplified ripper, designed for ripping from sites by parsing HTML.
|
||||||
*/
|
*/
|
||||||
public abstract class AbstractHTMLRipper extends AlbumRipper {
|
public abstract class AbstractHTMLRipper extends AlbumRipper {
|
||||||
|
|
||||||
public AbstractHTMLRipper(URL url) throws IOException {
|
public AbstractHTMLRipper(URL url) throws IOException {
|
||||||
super(url);
|
super(url);
|
||||||
}
|
}
|
||||||
@ -45,7 +45,7 @@ public abstract class AbstractHTMLRipper extends AlbumRipper {
|
|||||||
public boolean canRip(URL url) {
|
public boolean canRip(URL url) {
|
||||||
return url.getHost().endsWith(getDomain());
|
return url.getHost().endsWith(getDomain());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public URL sanitizeURL(URL url) throws MalformedURLException {
|
public URL sanitizeURL(URL url) throws MalformedURLException {
|
||||||
return url;
|
return url;
|
||||||
@ -66,7 +66,7 @@ public abstract class AbstractHTMLRipper extends AlbumRipper {
|
|||||||
logger.info("Retrieving " + this.url);
|
logger.info("Retrieving " + this.url);
|
||||||
sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm());
|
sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm());
|
||||||
Document doc = getFirstPage();
|
Document doc = getFirstPage();
|
||||||
|
|
||||||
while (doc != null) {
|
while (doc != null) {
|
||||||
List<String> imageURLs = getURLsFromPage(doc);
|
List<String> imageURLs = getURLsFromPage(doc);
|
||||||
// Remove all but 1 image
|
// Remove all but 1 image
|
||||||
@ -79,7 +79,7 @@ public abstract class AbstractHTMLRipper extends AlbumRipper {
|
|||||||
if (imageURLs.size() == 0) {
|
if (imageURLs.size() == 0) {
|
||||||
throw new IOException("No images found at " + doc.location());
|
throw new IOException("No images found at " + doc.location());
|
||||||
}
|
}
|
||||||
|
|
||||||
for (String imageURL : imageURLs) {
|
for (String imageURL : imageURLs) {
|
||||||
index += 1;
|
index += 1;
|
||||||
logger.debug("Found image url #" + index + ": " + imageURL);
|
logger.debug("Found image url #" + index + ": " + imageURL);
|
||||||
|
@ -19,7 +19,7 @@ public class DownloadThreadPool {
|
|||||||
public DownloadThreadPool() {
|
public DownloadThreadPool() {
|
||||||
initialize("Main");
|
initialize("Main");
|
||||||
}
|
}
|
||||||
|
|
||||||
public DownloadThreadPool(String threadPoolName) {
|
public DownloadThreadPool(String threadPoolName) {
|
||||||
initialize(threadPoolName);
|
initialize(threadPoolName);
|
||||||
}
|
}
|
||||||
|
@ -46,12 +46,12 @@ public class BcfakesRipper extends AbstractHTMLRipper {
|
|||||||
+ "http://www.bcfakes.com/celebritylist/name"
|
+ "http://www.bcfakes.com/celebritylist/name"
|
||||||
+ " Got: " + url);
|
+ " Got: " + url);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Document getFirstPage() throws IOException {
|
public Document getFirstPage() throws IOException {
|
||||||
return Http.url(url).get();
|
return Http.url(url).get();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Document getNextPage(Document doc) throws IOException {
|
public Document getNextPage(Document doc) throws IOException {
|
||||||
// Find next page
|
// Find next page
|
||||||
@ -63,7 +63,7 @@ public class BcfakesRipper extends AbstractHTMLRipper {
|
|||||||
sleep(500);
|
sleep(500);
|
||||||
return Http.url(nextUrl).get();
|
return Http.url(nextUrl).get();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<String> getURLsFromPage(Document doc) {
|
public List<String> getURLsFromPage(Document doc) {
|
||||||
List<String> imageURLs = new ArrayList<String>();
|
List<String> imageURLs = new ArrayList<String>();
|
||||||
|
@ -46,7 +46,7 @@ public class DatwinRipper extends AbstractHTMLRipper {
|
|||||||
public Document getFirstPage() throws IOException {
|
public Document getFirstPage() throws IOException {
|
||||||
return Http.url(url).get();
|
return Http.url(url).get();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<String> getURLsFromPage(Document doc) {
|
public List<String> getURLsFromPage(Document doc) {
|
||||||
List<String> imageURLs = new ArrayList<String>();
|
List<String> imageURLs = new ArrayList<String>();
|
||||||
@ -57,7 +57,7 @@ public class DatwinRipper extends AbstractHTMLRipper {
|
|||||||
}
|
}
|
||||||
return imageURLs;
|
return imageURLs;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void downloadURL(URL url, int index) {
|
public void downloadURL(URL url, int index) {
|
||||||
addURLToDownload(url, getPrefix(index));
|
addURLToDownload(url, getPrefix(index));
|
||||||
|
@ -239,7 +239,7 @@ public class DeviantartRipper extends AbstractHTMLRipper {
|
|||||||
}
|
}
|
||||||
return result.toString();
|
return result.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Attempts to download description for image.
|
* Attempts to download description for image.
|
||||||
* Comes in handy when people put entire stories in their description.
|
* Comes in handy when people put entire stories in their description.
|
||||||
@ -277,7 +277,7 @@ public class DeviantartRipper extends AbstractHTMLRipper {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* If largest resolution for image at 'thumb' is found, starts downloading
|
* If largest resolution for image at 'thumb' is found, starts downloading
|
||||||
* and returns null.
|
* and returns null.
|
||||||
|
@ -155,7 +155,7 @@ public class FivehundredpxRipper extends AbstractJSONRipper {
|
|||||||
private String getUserID(String username) throws IOException {
|
private String getUserID(String username) throws IOException {
|
||||||
logger.info("Fetching user ID for " + username);
|
logger.info("Fetching user ID for " + username);
|
||||||
JSONObject json = new Http("https://api.500px.com/v1/" +
|
JSONObject json = new Http("https://api.500px.com/v1/" +
|
||||||
"users/show" +
|
"users/show" +
|
||||||
"?username=" + username +
|
"?username=" + username +
|
||||||
"&consumer_key=" + CONSUMER_KEY)
|
"&consumer_key=" + CONSUMER_KEY)
|
||||||
.getJSON();
|
.getJSON();
|
||||||
@ -307,7 +307,7 @@ public class FivehundredpxRipper extends AbstractJSONRipper {
|
|||||||
}
|
}
|
||||||
return imageURLs;
|
return imageURLs;
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean urlExists(String url) {
|
private boolean urlExists(String url) {
|
||||||
try {
|
try {
|
||||||
HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
|
HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
|
||||||
|
@ -69,7 +69,7 @@ public class GirlsOfDesireRipper extends AbstractHTMLRipper {
|
|||||||
}
|
}
|
||||||
return albumDoc;
|
return albumDoc;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<String> getURLsFromPage(Document doc) {
|
public List<String> getURLsFromPage(Document doc) {
|
||||||
List<String> imageURLs = new ArrayList<String>();
|
List<String> imageURLs = new ArrayList<String>();
|
||||||
@ -83,7 +83,7 @@ public class GirlsOfDesireRipper extends AbstractHTMLRipper {
|
|||||||
}
|
}
|
||||||
return imageURLs;
|
return imageURLs;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void downloadURL(URL url, int index) {
|
public void downloadURL(URL url, int index) {
|
||||||
// Send referrer when downloading images
|
// Send referrer when downloading images
|
||||||
|
@ -69,12 +69,12 @@ public class ImagearnRipper extends AbstractHTMLRipper {
|
|||||||
}
|
}
|
||||||
throw new IOException("Failed to find gallery at URL " + url);
|
throw new IOException("Failed to find gallery at URL " + url);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Document getFirstPage() throws IOException {
|
public Document getFirstPage() throws IOException {
|
||||||
return Http.url(url).get();
|
return Http.url(url).get();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<String> getURLsFromPage(Document doc) {
|
public List<String> getURLsFromPage(Document doc) {
|
||||||
List<String> imageURLs = new ArrayList<String>();
|
List<String> imageURLs = new ArrayList<String>();
|
||||||
@ -85,7 +85,7 @@ public class ImagearnRipper extends AbstractHTMLRipper {
|
|||||||
}
|
}
|
||||||
return imageURLs;
|
return imageURLs;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void downloadURL(URL url, int index) {
|
public void downloadURL(URL url, int index) {
|
||||||
addURLToDownload(url, getPrefix(index));
|
addURLToDownload(url, getPrefix(index));
|
||||||
|
@ -58,7 +58,7 @@ public class ImagebamRipper extends AbstractHTMLRipper {
|
|||||||
+ "http://www.imagebam.com/gallery/galleryid"
|
+ "http://www.imagebam.com/gallery/galleryid"
|
||||||
+ " Got: " + url);
|
+ " Got: " + url);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Document getFirstPage() throws IOException {
|
public Document getFirstPage() throws IOException {
|
||||||
if (albumDoc == null) {
|
if (albumDoc == null) {
|
||||||
@ -66,7 +66,7 @@ public class ImagebamRipper extends AbstractHTMLRipper {
|
|||||||
}
|
}
|
||||||
return albumDoc;
|
return albumDoc;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Document getNextPage(Document doc) throws IOException {
|
public Document getNextPage(Document doc) throws IOException {
|
||||||
// Find next page
|
// Find next page
|
||||||
@ -78,7 +78,7 @@ public class ImagebamRipper extends AbstractHTMLRipper {
|
|||||||
sleep(500);
|
sleep(500);
|
||||||
return Http.url(nextUrl).get();
|
return Http.url(nextUrl).get();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<String> getURLsFromPage(Document doc) {
|
public List<String> getURLsFromPage(Document doc) {
|
||||||
List<String> imageURLs = new ArrayList<String>();
|
List<String> imageURLs = new ArrayList<String>();
|
||||||
@ -117,7 +117,7 @@ public class ImagebamRipper extends AbstractHTMLRipper {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Helper class to find and download images found on "image" pages
|
* Helper class to find and download images found on "image" pages
|
||||||
*
|
*
|
||||||
* Handles case when site has IP-banned the user.
|
* Handles case when site has IP-banned the user.
|
||||||
*/
|
*/
|
||||||
private class ImagebamImageThread extends Thread {
|
private class ImagebamImageThread extends Thread {
|
||||||
@ -134,7 +134,7 @@ public class ImagebamRipper extends AbstractHTMLRipper {
|
|||||||
public void run() {
|
public void run() {
|
||||||
fetchImage();
|
fetchImage();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void fetchImage() {
|
private void fetchImage() {
|
||||||
try {
|
try {
|
||||||
Document doc = Http.url(url).get();
|
Document doc = Http.url(url).get();
|
||||||
|
@ -93,7 +93,7 @@ public class ImagefapRipper extends AbstractHTMLRipper {
|
|||||||
+ "imagefap.com/pictures/####..."
|
+ "imagefap.com/pictures/####..."
|
||||||
+ " Got: " + url);
|
+ " Got: " + url);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Document getFirstPage() throws IOException {
|
public Document getFirstPage() throws IOException {
|
||||||
if (albumDoc == null) {
|
if (albumDoc == null) {
|
||||||
@ -101,7 +101,7 @@ public class ImagefapRipper extends AbstractHTMLRipper {
|
|||||||
}
|
}
|
||||||
return albumDoc;
|
return albumDoc;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Document getNextPage(Document doc) throws IOException {
|
public Document getNextPage(Document doc) throws IOException {
|
||||||
String nextURL = null;
|
String nextURL = null;
|
||||||
@ -117,7 +117,7 @@ public class ImagefapRipper extends AbstractHTMLRipper {
|
|||||||
sleep(1000);
|
sleep(1000);
|
||||||
return Http.url(nextURL).get();
|
return Http.url(nextURL).get();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<String> getURLsFromPage(Document doc) {
|
public List<String> getURLsFromPage(Document doc) {
|
||||||
List<String> imageURLs = new ArrayList<String>();
|
List<String> imageURLs = new ArrayList<String>();
|
||||||
@ -137,7 +137,7 @@ public class ImagefapRipper extends AbstractHTMLRipper {
|
|||||||
}
|
}
|
||||||
return imageURLs;
|
return imageURLs;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void downloadURL(URL url, int index) {
|
public void downloadURL(URL url, int index) {
|
||||||
// Send referrer for image downloads
|
// Send referrer for image downloads
|
||||||
|
@ -55,12 +55,12 @@ public class ImagevenueRipper extends AbstractHTMLRipper {
|
|||||||
+ "http://...imagevenue.com/galshow.php?gal=gallery_...."
|
+ "http://...imagevenue.com/galshow.php?gal=gallery_...."
|
||||||
+ " Got: " + url);
|
+ " Got: " + url);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Document getFirstPage() throws IOException {
|
public Document getFirstPage() throws IOException {
|
||||||
return Http.url(url).get();
|
return Http.url(url).get();
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<String> getURLsFromPage(Document doc) {
|
public List<String> getURLsFromPage(Document doc) {
|
||||||
List<String> imageURLs = new ArrayList<String>();
|
List<String> imageURLs = new ArrayList<String>();
|
||||||
for (Element thumb : doc.select("a[target=_blank]")) {
|
for (Element thumb : doc.select("a[target=_blank]")) {
|
||||||
@ -68,7 +68,7 @@ public class ImagevenueRipper extends AbstractHTMLRipper {
|
|||||||
}
|
}
|
||||||
return imageURLs;
|
return imageURLs;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void downloadURL(URL url, int index) {
|
public void downloadURL(URL url, int index) {
|
||||||
ImagevenueImageThread t = new ImagevenueImageThread(url, index);
|
ImagevenueImageThread t = new ImagevenueImageThread(url, index);
|
||||||
imagevenueThreadPool.addThread(t);
|
imagevenueThreadPool.addThread(t);
|
||||||
@ -76,7 +76,7 @@ public class ImagevenueRipper extends AbstractHTMLRipper {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Helper class to find and download images found on "image" pages
|
* Helper class to find and download images found on "image" pages
|
||||||
*
|
*
|
||||||
* Handles case when site has IP-banned the user.
|
* Handles case when site has IP-banned the user.
|
||||||
*/
|
*/
|
||||||
private class ImagevenueImageThread extends Thread {
|
private class ImagevenueImageThread extends Thread {
|
||||||
@ -93,7 +93,7 @@ public class ImagevenueRipper extends AbstractHTMLRipper {
|
|||||||
public void run() {
|
public void run() {
|
||||||
fetchImage();
|
fetchImage();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void fetchImage() {
|
private void fetchImage() {
|
||||||
try {
|
try {
|
||||||
Document doc = Http.url(url)
|
Document doc = Http.url(url)
|
||||||
|
@ -39,7 +39,7 @@ public class ImgboxRipper extends AbstractHTMLRipper {
|
|||||||
throw new MalformedURLException("Expected imgbox.com URL format: " +
|
throw new MalformedURLException("Expected imgbox.com URL format: " +
|
||||||
"imgbox.com/g/albumid - got " + url + "instead");
|
"imgbox.com/g/albumid - got " + url + "instead");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Document getFirstPage() throws IOException {
|
public Document getFirstPage() throws IOException {
|
||||||
return Http.url(url).get();
|
return Http.url(url).get();
|
||||||
|
@ -11,11 +11,8 @@ import java.util.regex.Pattern;
|
|||||||
import org.json.JSONArray;
|
import org.json.JSONArray;
|
||||||
import org.json.JSONException;
|
import org.json.JSONException;
|
||||||
import org.json.JSONObject;
|
import org.json.JSONObject;
|
||||||
import org.jsoup.nodes.Document;
|
|
||||||
import org.jsoup.nodes.Element;
|
|
||||||
|
|
||||||
import com.rarchives.ripme.ripper.AbstractJSONRipper;
|
import com.rarchives.ripme.ripper.AbstractJSONRipper;
|
||||||
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
|
|
||||||
import com.rarchives.ripme.utils.Http;
|
import com.rarchives.ripme.utils.Http;
|
||||||
|
|
||||||
public class InstagramRipper extends AbstractJSONRipper {
|
public class InstagramRipper extends AbstractJSONRipper {
|
||||||
@ -60,7 +57,7 @@ public class InstagramRipper extends AbstractJSONRipper {
|
|||||||
|
|
||||||
throw new MalformedURLException("Expected username in URL (instagram.com/username and not " + url);
|
throw new MalformedURLException("Expected username in URL (instagram.com/username and not " + url);
|
||||||
}
|
}
|
||||||
|
|
||||||
private String getUserID(URL url) throws IOException {
|
private String getUserID(URL url) throws IOException {
|
||||||
|
|
||||||
Pattern p = Pattern.compile("^https?://instagram\\.com/([^/]+)");
|
Pattern p = Pattern.compile("^https?://instagram\\.com/([^/]+)");
|
||||||
@ -71,7 +68,7 @@ public class InstagramRipper extends AbstractJSONRipper {
|
|||||||
|
|
||||||
throw new IOException("Unable to find userID at " + this.url);
|
throw new IOException("Unable to find userID at " + this.url);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public JSONObject getFirstPage() throws IOException {
|
public JSONObject getFirstPage() throws IOException {
|
||||||
userID = getUserID(url);
|
userID = getUserID(url);
|
||||||
@ -111,7 +108,7 @@ public class InstagramRipper extends AbstractJSONRipper {
|
|||||||
throw new IOException("No more images found");
|
throw new IOException("No more images found");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<String> getURLsFromJSON(JSONObject json) {
|
public List<String> getURLsFromJSON(JSONObject json) {
|
||||||
List<String> imageURLs = new ArrayList<String>();
|
List<String> imageURLs = new ArrayList<String>();
|
||||||
@ -142,7 +139,7 @@ public class InstagramRipper extends AbstractJSONRipper {
|
|||||||
}
|
}
|
||||||
return imageURLs;
|
return imageURLs;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void downloadURL(URL url, int index) {
|
public void downloadURL(URL url, int index) {
|
||||||
addURLToDownload(url);
|
addURLToDownload(url);
|
||||||
|
@ -138,9 +138,9 @@ public class MediacrushRipper extends AbstractJSONRipper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Iterates over list if "file" objects and returns the preferred
|
* Iterates over list if "file" objects and returns the preferred
|
||||||
* image format.
|
* image format.
|
||||||
* @param subfiles Array of "files" (JSONObjects) which contain
|
* @param subfiles Array of "files" (JSONObjects) which contain
|
||||||
* @return Preferred media format.
|
* @return Preferred media format.
|
||||||
*/
|
*/
|
||||||
private String getPreferredUrl(JSONArray subfiles) {
|
private String getPreferredUrl(JSONArray subfiles) {
|
||||||
|
@ -4,7 +4,6 @@ import java.io.IOException;
|
|||||||
import java.net.MalformedURLException;
|
import java.net.MalformedURLException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.regex.Matcher;
|
import java.util.regex.Matcher;
|
||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
@ -13,15 +12,14 @@ import org.jsoup.nodes.Document;
|
|||||||
import org.jsoup.nodes.Element;
|
import org.jsoup.nodes.Element;
|
||||||
|
|
||||||
import com.rarchives.ripme.ripper.AbstractHTMLRipper;
|
import com.rarchives.ripme.ripper.AbstractHTMLRipper;
|
||||||
import com.rarchives.ripme.ripper.rippers.ripperhelpers.ChanSite;
|
|
||||||
import com.rarchives.ripme.utils.Http;
|
import com.rarchives.ripme.utils.Http;
|
||||||
|
|
||||||
public class NatalieMuRipper extends AbstractHTMLRipper {
|
public class NatalieMuRipper extends AbstractHTMLRipper {
|
||||||
|
|
||||||
public int news_id = 0;
|
public int news_id = 0;
|
||||||
|
|
||||||
public NatalieMuRipper(URL url) throws IOException {
|
public NatalieMuRipper(URL url) throws IOException {
|
||||||
super(url);
|
super(url);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -37,12 +35,12 @@ public class NatalieMuRipper extends AbstractHTMLRipper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean canRip(URL url) {
|
public boolean canRip(URL url) {
|
||||||
//urls like:
|
//urls like:
|
||||||
// http://cdn2.natalie.mu/music/gallery/show/news_id/xxxxxx/image_id/xxxxxx
|
// http://cdn2.natalie.mu/music/gallery/show/news_id/xxxxxx/image_id/xxxxxx
|
||||||
// http://cdn2.natalie.mu/music/news/140411
|
// http://cdn2.natalie.mu/music/news/140411
|
||||||
return url.toExternalForm().contains("natalie.mu") // Most chans
|
return url.toExternalForm().contains("natalie.mu") // Most chans
|
||||||
&& (url.toExternalForm().contains("/news_id/")
|
&& (url.toExternalForm().contains("/news_id/")
|
||||||
|| url.toExternalForm().contains("/news/")); // 4chan, archive.moe
|
|| url.toExternalForm().contains("/news/")); // 4chan, archive.moe
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -61,13 +59,13 @@ public class NatalieMuRipper extends AbstractHTMLRipper {
|
|||||||
m = p.matcher(u);
|
m = p.matcher(u);
|
||||||
if (m.find()) {
|
if (m.find()) {
|
||||||
return m.group(1);
|
return m.group(1);
|
||||||
}
|
}
|
||||||
} else if (u.contains("/news/")) {
|
} else if (u.contains("/news/")) {
|
||||||
p = Pattern.compile("/news/([0-9]+)/?");
|
p = Pattern.compile("/news/([0-9]+)/?");
|
||||||
m = p.matcher(u);
|
m = p.matcher(u);
|
||||||
if (m.find()) {
|
if (m.find()) {
|
||||||
return m.group(1);
|
return m.group(1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
throw new MalformedURLException(
|
throw new MalformedURLException(
|
||||||
@ -85,18 +83,18 @@ public class NatalieMuRipper extends AbstractHTMLRipper {
|
|||||||
public Document getFirstPage() throws IOException {
|
public Document getFirstPage() throws IOException {
|
||||||
return Http.url(this.url).get();
|
return Http.url(this.url).get();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<String> getURLsFromPage(Document page) {
|
public List<String> getURLsFromPage(Document page) {
|
||||||
List<String> imageURLs = new ArrayList<String>();
|
List<String> imageURLs = new ArrayList<String>();
|
||||||
Pattern p; Matcher m;
|
Pattern p; Matcher m;
|
||||||
//select all album thumbnails
|
//select all album thumbnails
|
||||||
for (Element span : page.select(".NA_articleGallery span")) {
|
for (Element span : page.select(".NA_articleGallery span")) {
|
||||||
if (!span.hasAttr("style")) {
|
if (!span.hasAttr("style")) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
String style = span.attr("style").trim();
|
String style = span.attr("style").trim();
|
||||||
|
|
||||||
p = Pattern.compile("background-image: url\\((.*list_thumb_inbox.*)\\);", Pattern.CASE_INSENSITIVE);
|
p = Pattern.compile("background-image: url\\((.*list_thumb_inbox.*)\\);", Pattern.CASE_INSENSITIVE);
|
||||||
m = p.matcher(style);
|
m = p.matcher(style);
|
||||||
if (m.find()) {
|
if (m.find()) {
|
||||||
@ -118,7 +116,7 @@ public class NatalieMuRipper extends AbstractHTMLRipper {
|
|||||||
if (isThisATest()) {
|
if (isThisATest()) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isStopped()) {
|
if (isStopped()) {
|
||||||
break;
|
break;
|
||||||
@ -130,5 +128,5 @@ public class NatalieMuRipper extends AbstractHTMLRipper {
|
|||||||
@Override
|
@Override
|
||||||
public void downloadURL(URL url, int index) {
|
public void downloadURL(URL url, int index) {
|
||||||
addURLToDownload(url, getPrefix(index), "", this.url.toString(), null);
|
addURLToDownload(url, getPrefix(index), "", this.url.toString(), null);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -80,7 +80,7 @@ public class PornhubRipper extends AlbumRipper {
|
|||||||
public void rip() throws IOException {
|
public void rip() throws IOException {
|
||||||
int index = 0;
|
int index = 0;
|
||||||
String nextUrl = this.url.toExternalForm();
|
String nextUrl = this.url.toExternalForm();
|
||||||
|
|
||||||
if (albumDoc == null) {
|
if (albumDoc == null) {
|
||||||
logger.info(" Retrieving album page " + nextUrl);
|
logger.info(" Retrieving album page " + nextUrl);
|
||||||
sendUpdate(STATUS.LOADING_RESOURCE, nextUrl);
|
sendUpdate(STATUS.LOADING_RESOURCE, nextUrl);
|
||||||
@ -127,7 +127,7 @@ public class PornhubRipper extends AlbumRipper {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Helper class to find and download images found on "image" pages
|
* Helper class to find and download images found on "image" pages
|
||||||
*
|
*
|
||||||
* Handles case when site has IP-banned the user.
|
* Handles case when site has IP-banned the user.
|
||||||
*/
|
*/
|
||||||
private class PornhubImageThread extends Thread {
|
private class PornhubImageThread extends Thread {
|
||||||
@ -144,28 +144,28 @@ public class PornhubRipper extends AlbumRipper {
|
|||||||
public void run() {
|
public void run() {
|
||||||
fetchImage();
|
fetchImage();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void fetchImage() {
|
private void fetchImage() {
|
||||||
try {
|
try {
|
||||||
Document doc = Http.url(this.url)
|
Document doc = Http.url(this.url)
|
||||||
.referrer(this.url)
|
.referrer(this.url)
|
||||||
.get();
|
.get();
|
||||||
|
|
||||||
// Find image
|
// Find image
|
||||||
Elements images = doc.select("#photoImageSection img");
|
Elements images = doc.select("#photoImageSection img");
|
||||||
Element image = images.first();
|
Element image = images.first();
|
||||||
String imgsrc = image.attr("src");
|
String imgsrc = image.attr("src");
|
||||||
logger.info("Found URL " + imgsrc + " via " + images.get(0));
|
logger.info("Found URL " + imgsrc + " via " + images.get(0));
|
||||||
|
|
||||||
// Provide prefix and let the AbstractRipper "guess" the filename
|
// Provide prefix and let the AbstractRipper "guess" the filename
|
||||||
String prefix = "";
|
String prefix = "";
|
||||||
if (Utils.getConfigBoolean("download.save_order", true)) {
|
if (Utils.getConfigBoolean("download.save_order", true)) {
|
||||||
prefix = String.format("%03d_", index);
|
prefix = String.format("%03d_", index);
|
||||||
}
|
}
|
||||||
|
|
||||||
URL imgurl = new URL(url, imgsrc);
|
URL imgurl = new URL(url, imgsrc);
|
||||||
addURLToDownload(imgurl, prefix);
|
addURLToDownload(imgurl, prefix);
|
||||||
|
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
logger.error("[!] Exception while loading/parsing " + this.url, e);
|
logger.error("[!] Exception while loading/parsing " + this.url, e);
|
||||||
}
|
}
|
||||||
|
@ -26,13 +26,13 @@ public class RedditRipper extends AlbumRipper {
|
|||||||
|
|
||||||
private static final String HOST = "reddit";
|
private static final String HOST = "reddit";
|
||||||
private static final String DOMAIN = "reddit.com";
|
private static final String DOMAIN = "reddit.com";
|
||||||
|
|
||||||
private static final String REDDIT_USER_AGENT = "RipMe:github/4pr0n/ripme:" + UpdateUtils.getThisJarVersion() + " (by /u/4_pr0n)";
|
private static final String REDDIT_USER_AGENT = "RipMe:github/4pr0n/ripme:" + UpdateUtils.getThisJarVersion() + " (by /u/4_pr0n)";
|
||||||
|
|
||||||
private static final int SLEEP_TIME = 2000;
|
private static final int SLEEP_TIME = 2000;
|
||||||
|
|
||||||
//private static final String USER_AGENT = "ripme by /u/4_pr0n github.com/4pr0n/ripme";
|
//private static final String USER_AGENT = "ripme by /u/4_pr0n github.com/4pr0n/ripme";
|
||||||
|
|
||||||
private long lastRequestTime = 0;
|
private long lastRequestTime = 0;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -68,9 +68,9 @@ public class RedditRipper extends AlbumRipper {
|
|||||||
}
|
}
|
||||||
waitForThreads();
|
waitForThreads();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
private URL getAndParseAndReturnNext(URL url) throws IOException {
|
private URL getAndParseAndReturnNext(URL url) throws IOException {
|
||||||
JSONArray jsonArray = getJsonArrayFromURL(url), children;
|
JSONArray jsonArray = getJsonArrayFromURL(url), children;
|
||||||
JSONObject json, data;
|
JSONObject json, data;
|
||||||
@ -108,7 +108,7 @@ public class RedditRipper extends AlbumRipper {
|
|||||||
}
|
}
|
||||||
return nextURL;
|
return nextURL;
|
||||||
}
|
}
|
||||||
|
|
||||||
private JSONArray getJsonArrayFromURL(URL url) throws IOException {
|
private JSONArray getJsonArrayFromURL(URL url) throws IOException {
|
||||||
// Wait 2 seconds before the next request
|
// Wait 2 seconds before the next request
|
||||||
long timeDiff = System.currentTimeMillis() - lastRequestTime;
|
long timeDiff = System.currentTimeMillis() - lastRequestTime;
|
||||||
|
@ -36,7 +36,7 @@ public class TwodgalleriesRipper extends AbstractHTMLRipper {
|
|||||||
public String getDomain() {
|
public String getDomain() {
|
||||||
return "2dgalleries.com";
|
return "2dgalleries.com";
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getGID(URL url) throws MalformedURLException {
|
public String getGID(URL url) throws MalformedURLException {
|
||||||
Pattern p; Matcher m;
|
Pattern p; Matcher m;
|
||||||
@ -108,7 +108,7 @@ public class TwodgalleriesRipper extends AbstractHTMLRipper {
|
|||||||
public void downloadURL(URL url, int index) {
|
public void downloadURL(URL url, int index) {
|
||||||
addURLToDownload(url, getPrefix(index));
|
addURLToDownload(url, getPrefix(index));
|
||||||
}
|
}
|
||||||
|
|
||||||
private void login() throws IOException {
|
private void login() throws IOException {
|
||||||
Response resp = Http.url(this.url).response();
|
Response resp = Http.url(this.url).response();
|
||||||
cookies = resp.cookies();
|
cookies = resp.cookies();
|
||||||
|
@ -29,7 +29,7 @@ public class VidbleRipper extends AbstractHTMLRipper {
|
|||||||
public String getDomain() {
|
public String getDomain() {
|
||||||
return "vidble.com";
|
return "vidble.com";
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getGID(URL url) throws MalformedURLException {
|
public String getGID(URL url) throws MalformedURLException {
|
||||||
Pattern p; Matcher m;
|
Pattern p; Matcher m;
|
||||||
@ -62,7 +62,7 @@ public class VidbleRipper extends AbstractHTMLRipper {
|
|||||||
for (Element img : imgs) {
|
for (Element img : imgs) {
|
||||||
String src = img.absUrl("src");
|
String src = img.absUrl("src");
|
||||||
src = src.replaceAll("_[a-zA-Z]{3,5}", "");
|
src = src.replaceAll("_[a-zA-Z]{3,5}", "");
|
||||||
|
|
||||||
if (!src.equals("")) {
|
if (!src.equals("")) {
|
||||||
imageURLs.add(src);
|
imageURLs.add(src);
|
||||||
}
|
}
|
||||||
|
@ -39,13 +39,13 @@ public class tamindirmp3 extends AbstractHTMLRipper {
|
|||||||
throw new MalformedURLException("Expected tamindir.com URL format: " +
|
throw new MalformedURLException("Expected tamindir.com URL format: " +
|
||||||
"tamindir.com/files/albumid - got " + url + "instead");
|
"tamindir.com/files/albumid - got " + url + "instead");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Document getFirstPage() throws IOException {
|
public Document getFirstPage() throws IOException {
|
||||||
return Http.url(url).get();
|
return Http.url(url).get();
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<String> getURLsFromPage(Document doc) {
|
public List<String> getURLsFromPage(Document doc) {
|
||||||
List<String> music = new ArrayList<String>();
|
List<String> music = new ArrayList<String>();
|
||||||
|
@ -33,7 +33,7 @@ public class CliphunterRipper extends VideoRipper {
|
|||||||
Matcher m = p.matcher(url.toExternalForm());
|
Matcher m = p.matcher(url.toExternalForm());
|
||||||
return m.matches();
|
return m.matches();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public URL sanitizeURL(URL url) throws MalformedURLException {
|
public URL sanitizeURL(URL url) throws MalformedURLException {
|
||||||
return url;
|
return url;
|
||||||
|
@ -29,7 +29,7 @@ public class GfycatRipper extends VideoRipper {
|
|||||||
public boolean canRip(URL url) {
|
public boolean canRip(URL url) {
|
||||||
return url.getHost().endsWith(HOST);
|
return url.getHost().endsWith(HOST);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public URL sanitizeURL(URL url) throws MalformedURLException {
|
public URL sanitizeURL(URL url) throws MalformedURLException {
|
||||||
return url;
|
return url;
|
||||||
|
@ -32,7 +32,7 @@ public class VidearnRipper extends VideoRipper {
|
|||||||
Matcher m = p.matcher(url.toExternalForm());
|
Matcher m = p.matcher(url.toExternalForm());
|
||||||
return m.matches();
|
return m.matches();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public URL sanitizeURL(URL url) throws MalformedURLException {
|
public URL sanitizeURL(URL url) throws MalformedURLException {
|
||||||
return url;
|
return url;
|
||||||
|
@ -32,7 +32,7 @@ public class VineRipper extends VideoRipper {
|
|||||||
Matcher m = p.matcher(url.toExternalForm());
|
Matcher m = p.matcher(url.toExternalForm());
|
||||||
return m.matches();
|
return m.matches();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public URL sanitizeURL(URL url) throws MalformedURLException {
|
public URL sanitizeURL(URL url) throws MalformedURLException {
|
||||||
return url;
|
return url;
|
||||||
|
@ -30,7 +30,7 @@ public class VkRipper extends VideoRipper {
|
|||||||
Matcher m = p.matcher(url.toExternalForm());
|
Matcher m = p.matcher(url.toExternalForm());
|
||||||
return m.matches();
|
return m.matches();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public URL sanitizeURL(URL url) throws MalformedURLException {
|
public URL sanitizeURL(URL url) throws MalformedURLException {
|
||||||
return url;
|
return url;
|
||||||
@ -57,7 +57,7 @@ public class VkRipper extends VideoRipper {
|
|||||||
addURLToDownload(new URL(videoURL), HOST + "_" + getGID(this.url));
|
addURLToDownload(new URL(videoURL), HOST + "_" + getGID(this.url));
|
||||||
waitForThreads();
|
waitForThreads();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static String getVideoURLAtPage(String url) throws IOException {
|
public static String getVideoURLAtPage(String url) throws IOException {
|
||||||
Document doc = Http.url(url)
|
Document doc = Http.url(url)
|
||||||
.userAgent(USER_AGENT)
|
.userAgent(USER_AGENT)
|
||||||
|
@ -31,7 +31,7 @@ public class XhamsterRipper extends VideoRipper {
|
|||||||
Matcher m = p.matcher(url.toExternalForm());
|
Matcher m = p.matcher(url.toExternalForm());
|
||||||
return m.matches();
|
return m.matches();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public URL sanitizeURL(URL url) throws MalformedURLException {
|
public URL sanitizeURL(URL url) throws MalformedURLException {
|
||||||
return url;
|
return url;
|
||||||
|
@ -33,7 +33,7 @@ public class XvideosRipper extends VideoRipper {
|
|||||||
Matcher m = p.matcher(url.toExternalForm());
|
Matcher m = p.matcher(url.toExternalForm());
|
||||||
return m.matches();
|
return m.matches();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public URL sanitizeURL(URL url) throws MalformedURLException {
|
public URL sanitizeURL(URL url) throws MalformedURLException {
|
||||||
return url;
|
return url;
|
||||||
|
@ -32,7 +32,7 @@ public class YoupornRipper extends VideoRipper {
|
|||||||
Matcher m = p.matcher(url.toExternalForm());
|
Matcher m = p.matcher(url.toExternalForm());
|
||||||
return m.matches();
|
return m.matches();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public URL sanitizeURL(URL url) throws MalformedURLException {
|
public URL sanitizeURL(URL url) throws MalformedURLException {
|
||||||
return url;
|
return url;
|
||||||
|
@ -14,7 +14,7 @@ import static com.rarchives.ripme.App.logger;
|
|||||||
|
|
||||||
public class ClipboardUtils {
|
public class ClipboardUtils {
|
||||||
private static AutoripThread autoripThread = new AutoripThread();
|
private static AutoripThread autoripThread = new AutoripThread();
|
||||||
|
|
||||||
public static void setClipboardAutoRip(boolean enabled) {
|
public static void setClipboardAutoRip(boolean enabled) {
|
||||||
if (enabled) {
|
if (enabled) {
|
||||||
autoripThread.kill();
|
autoripThread.kill();
|
||||||
@ -44,7 +44,7 @@ public class ClipboardUtils {
|
|||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -61,15 +61,15 @@ class AutoripThread extends Thread {
|
|||||||
String clipboard = ClipboardUtils.getClipboardString();
|
String clipboard = ClipboardUtils.getClipboardString();
|
||||||
if (clipboard != null) {
|
if (clipboard != null) {
|
||||||
Pattern p = Pattern.compile(
|
Pattern p = Pattern.compile(
|
||||||
"\\b(((ht|f)tp(s?)\\:\\/\\/|~\\/|\\/)|www.)" +
|
"\\b(((ht|f)tp(s?)\\:\\/\\/|~\\/|\\/)|www.)" +
|
||||||
"(\\w+:\\w+@)?(([-\\w]+\\.)+(com|org|net|gov" +
|
"(\\w+:\\w+@)?(([-\\w]+\\.)+(com|org|net|gov" +
|
||||||
"|mil|biz|info|mobi|name|aero|jobs|museum" +
|
"|mil|biz|info|mobi|name|aero|jobs|museum" +
|
||||||
"|travel|[a-z]{2}))(:[\\d]{1,5})?" +
|
"|travel|[a-z]{2}))(:[\\d]{1,5})?" +
|
||||||
"(((\\/([-\\w~!$+|.,=]|%[a-f\\d]{2})+)+|\\/)+|\\?|#)?" +
|
"(((\\/([-\\w~!$+|.,=]|%[a-f\\d]{2})+)+|\\/)+|\\?|#)?" +
|
||||||
"((\\?([-\\w~!$+|.,*:]|%[a-f\\d{2}])+=?" +
|
"((\\?([-\\w~!$+|.,*:]|%[a-f\\d{2}])+=?" +
|
||||||
"([-\\w~!$+|.,*:=]|%[a-f\\d]{2})*)" +
|
"([-\\w~!$+|.,*:=]|%[a-f\\d]{2})*)" +
|
||||||
"(&(?:[-\\w~!$+|.,*:]|%[a-f\\d{2}])+=?" +
|
"(&(?:[-\\w~!$+|.,*:]|%[a-f\\d{2}])+=?" +
|
||||||
"([-\\w~!$+|.,*:=]|%[a-f\\d]{2})*)*)*" +
|
"([-\\w~!$+|.,*:=]|%[a-f\\d]{2})*)*)*" +
|
||||||
"(#([-\\w~!$+|.,*:=]|%[a-f\\d]{2})*)?\\b");
|
"(#([-\\w~!$+|.,*:=]|%[a-f\\d]{2})*)?\\b");
|
||||||
Matcher m = p.matcher(clipboard);
|
Matcher m = p.matcher(clipboard);
|
||||||
while (m.find()) {
|
while (m.find()) {
|
||||||
@ -87,7 +87,7 @@ class AutoripThread extends Thread {
|
|||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void kill() {
|
public void kill() {
|
||||||
isRunning = false;
|
isRunning = false;
|
||||||
}
|
}
|
||||||
|
@ -10,7 +10,7 @@ public class RipStatusMessage {
|
|||||||
DOWNLOAD_STARTED("Download Started"),
|
DOWNLOAD_STARTED("Download Started"),
|
||||||
DOWNLOAD_COMPLETE("Download Complete"),
|
DOWNLOAD_COMPLETE("Download Complete"),
|
||||||
DOWNLOAD_ERRORED("Download Errored"),
|
DOWNLOAD_ERRORED("Download Errored"),
|
||||||
RIP_COMPLETE("Rip Complete"),
|
RIP_COMPLETE("Rip Complete"),
|
||||||
DOWNLOAD_WARN("Download problem"),
|
DOWNLOAD_WARN("Download problem"),
|
||||||
TOTAL_BYTES("Total bytes"),
|
TOTAL_BYTES("Total bytes"),
|
||||||
COMPLETED_BYTES("Completed bytes"),
|
COMPLETED_BYTES("Completed bytes"),
|
||||||
@ -29,11 +29,11 @@ public class RipStatusMessage {
|
|||||||
this.status = status;
|
this.status = status;
|
||||||
this.object = object;
|
this.object = object;
|
||||||
}
|
}
|
||||||
|
|
||||||
public STATUS getStatus() {
|
public STATUS getStatus() {
|
||||||
return status;
|
return status;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Object getObject() {
|
public Object getObject() {
|
||||||
return object;
|
return object;
|
||||||
}
|
}
|
||||||
|
@ -72,5 +72,5 @@ public class Base64 {
|
|||||||
buffer[index++]= (byte)(((c2 << 6) | c3) & mask);
|
buffer[index++]= (byte)(((c2 << 6) | c3) & mask);
|
||||||
}
|
}
|
||||||
return buffer;
|
return buffer;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -23,7 +23,6 @@ import com.rarchives.ripme.ripper.rippers.VidbleRipper;
|
|||||||
import com.rarchives.ripme.ripper.rippers.video.GfycatRipper;
|
import com.rarchives.ripme.ripper.rippers.video.GfycatRipper;
|
||||||
import com.rarchives.ripme.ripper.rippers.EroShareRipper;
|
import com.rarchives.ripme.ripper.rippers.EroShareRipper;
|
||||||
|
|
||||||
|
|
||||||
public class RipUtils {
|
public class RipUtils {
|
||||||
private static final Logger logger = Logger.getLogger(RipUtils.class);
|
private static final Logger logger = Logger.getLogger(RipUtils.class);
|
||||||
|
|
||||||
@ -32,7 +31,7 @@ public class RipUtils {
|
|||||||
|
|
||||||
logger.debug("Checking " + url);
|
logger.debug("Checking " + url);
|
||||||
// Imgur album
|
// Imgur album
|
||||||
if ((url.getHost().endsWith("imgur.com"))
|
if ((url.getHost().endsWith("imgur.com"))
|
||||||
&& url.toExternalForm().contains("imgur.com/a/")) {
|
&& url.toExternalForm().contains("imgur.com/a/")) {
|
||||||
try {
|
try {
|
||||||
logger.debug("Fetching imgur album at " + url);
|
logger.debug("Fetching imgur album at " + url);
|
||||||
@ -91,7 +90,7 @@ public class RipUtils {
|
|||||||
}
|
}
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
Pattern p = Pattern.compile("https?://i.reddituploads.com/([a-zA-Z0-9]+)\\?.*");
|
Pattern p = Pattern.compile("https?://i.reddituploads.com/([a-zA-Z0-9]+)\\?.*");
|
||||||
Matcher m = p.matcher(url.toExternalForm());
|
Matcher m = p.matcher(url.toExternalForm());
|
||||||
if (m.matches()) {
|
if (m.matches()) {
|
||||||
@ -117,8 +116,8 @@ public class RipUtils {
|
|||||||
logger.error("[!] Not a valid URL: '" + url + "'", e);
|
logger.error("[!] Not a valid URL: '" + url + "'", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (url.getHost().equals("imgur.com") ||
|
if (url.getHost().equals("imgur.com") ||
|
||||||
url.getHost().equals("m.imgur.com")){
|
url.getHost().equals("m.imgur.com")){
|
||||||
try {
|
try {
|
||||||
// Fetch the page
|
// Fetch the page
|
||||||
@ -138,13 +137,13 @@ public class RipUtils {
|
|||||||
} catch (IOException ex) {
|
} catch (IOException ex) {
|
||||||
logger.error("[!] Error", ex);
|
logger.error("[!] Error", ex);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.error("[!] Unable to rip URL: " + url);
|
logger.error("[!] Unable to rip URL: " + url);
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static Pattern getURLRegex() {
|
public static Pattern getURLRegex() {
|
||||||
return Pattern.compile("(https?://[a-zA-Z0-9\\-\\.]+\\.[a-zA-Z]{2,3}(/\\S*))");
|
return Pattern.compile("(https?://[a-zA-Z0-9\\-\\.]+\\.[a-zA-Z]{2,3}(/\\S*))");
|
||||||
}
|
}
|
||||||
|
@ -393,7 +393,7 @@ public class Utils {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Parses an URL query
|
* Parses an URL query
|
||||||
*
|
*
|
||||||
* @param query
|
* @param query
|
||||||
* The query part of an URL
|
* The query part of an URL
|
||||||
* @return The map of all query parameters
|
* @return The map of all query parameters
|
||||||
@ -426,7 +426,7 @@ public class Utils {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Parses an URL query and returns the requested parameter's value
|
* Parses an URL query and returns the requested parameter's value
|
||||||
*
|
*
|
||||||
* @param query
|
* @param query
|
||||||
* The query part of an URL
|
* The query part of an URL
|
||||||
* @param key
|
* @param key
|
||||||
|
@ -106,7 +106,7 @@ public class BasicRippersTest extends RippersTest {
|
|||||||
testRipper(ripper);
|
testRipper(ripper);
|
||||||
}
|
}
|
||||||
*/
|
*/
|
||||||
|
|
||||||
public void test500pxAlbum() throws IOException {
|
public void test500pxAlbum() throws IOException {
|
||||||
FivehundredpxRipper ripper = new FivehundredpxRipper(new URL("https://marketplace.500px.com/alexander_hurman"));
|
FivehundredpxRipper ripper = new FivehundredpxRipper(new URL("https://marketplace.500px.com/alexander_hurman"));
|
||||||
testRipper(ripper);
|
testRipper(ripper);
|
||||||
@ -261,7 +261,7 @@ public class BasicRippersTest extends RippersTest {
|
|||||||
AbstractRipper ripper = new VineRipper(new URL("https://vine.co/u/954440445776334848"));
|
AbstractRipper ripper = new VineRipper(new URL("https://vine.co/u/954440445776334848"));
|
||||||
testRipper(ripper);
|
testRipper(ripper);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testVkSubalbumRip() throws IOException {
|
public void testVkSubalbumRip() throws IOException {
|
||||||
VkRipper ripper = new VkRipper(new URL("http://vk.com/album45506334_0"));
|
VkRipper ripper = new VkRipper(new URL("http://vk.com/album45506334_0"));
|
||||||
testRipper(ripper);
|
testRipper(ripper);
|
||||||
|
@ -10,7 +10,7 @@ import java.util.Map;
|
|||||||
import com.rarchives.ripme.ripper.rippers.InstagramRipper;
|
import com.rarchives.ripme.ripper.rippers.InstagramRipper;
|
||||||
|
|
||||||
public class InstagramRipperTest extends RippersTest {
|
public class InstagramRipperTest extends RippersTest {
|
||||||
|
|
||||||
public void testInstagramGID() throws IOException {
|
public void testInstagramGID() throws IOException {
|
||||||
Map<URL, String> testURLs = new HashMap<URL, String>();
|
Map<URL, String> testURLs = new HashMap<URL, String>();
|
||||||
testURLs.put(new URL("http://instagram.com/Test_User"), "Test_User");
|
testURLs.put(new URL("http://instagram.com/Test_User"), "Test_User");
|
||||||
|
@ -12,7 +12,7 @@ import com.rarchives.ripme.ripper.rippers.video.XvideosRipper;
|
|||||||
import com.rarchives.ripme.ripper.rippers.video.YoupornRipper;
|
import com.rarchives.ripme.ripper.rippers.video.YoupornRipper;
|
||||||
|
|
||||||
public class VideoRippersTest extends RippersTest {
|
public class VideoRippersTest extends RippersTest {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Helper method for testing a video ripper
|
* Helper method for testing a video ripper
|
||||||
* @param ripper The video ripper
|
* @param ripper The video ripper
|
||||||
@ -44,7 +44,7 @@ public class VideoRippersTest extends RippersTest {
|
|||||||
videoTestHelper(ripper);
|
videoTestHelper(ripper);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testPornhubRipper() throws IOException {
|
public void testPornhubRipper() throws IOException {
|
||||||
List<URL> contentURLs = new ArrayList<URL>();
|
List<URL> contentURLs = new ArrayList<URL>();
|
||||||
contentURLs.add(new URL("http://www.pornhub.com/view_video.php?viewkey=993166542"));
|
contentURLs.add(new URL("http://www.pornhub.com/view_video.php?viewkey=993166542"));
|
||||||
|
Loading…
Reference in New Issue
Block a user