Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #6 from RipMeApp/master

Browse Source 
test
This commit is contained in:
Kevin Jiang 2018-02-19 01:44:50 -05:00 committed by GitHub
commit a1dd7a64dd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 448 additions and 296 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
docs/options.md
View File

@ -1,40 +0,0 @@
file.overwrite | bool | If true ripme will overwrite existing files rather than skip them
clipboard.autorip | bool | If true ripme will try to download any links in the clip board
error.skip404 | bool | Don't retry on 404 errors
download.save_order| bool | If true ripme will prefix each downloaded file with a number in the order the file was download
auto.update | bool | If true ripme will auto-update every time it's started
play.sound | bool | If true ripme will play a sound every time a rip finishes
download.show_popup| bool | TODO figure out what this is for
log.save | bool | If true ripme will save it's logs
urls_only.save | bool | If true ripme will save all urls to a text file and download no files
album_titles.save | bool | Currently does nothing
prefer.mp4 | bool | Prefer mp4 when downloading a video that has more than 1 format
download.timeout | int | File download timeout (in milliseconds)
page.timeout | int | Page download timeout (in milliseconds)
download.max_size | int | Maximum size of downloaded files in bytes
threads.size | int | The number of threads to use
twitter.auth | String | Twitter API key (Base64'd)
tumblr.auth | String | Tumblr API key
log.level | String | The debug log level (Example: Log level: Debug)
gw.api | String | TODO figure out what this is for
twitter.max_requests | int | TODO figure out what this is for

2
pom.xml
View File

@ -4,7 +4,7 @@
<groupId>com.rarchives.ripme</groupId>
<artifactId>ripme</artifactId>
<packaging>jar</packaging>
<version>1.7.12</version>
<version>1.7.19</version>
<name>ripme</name>
<url>http://rip.rarchives.com</url>
<properties>

9
ripme.json
View File

@ -1,6 +1,13 @@
{
"latestVersion": "1.7.12",
"latestVersion": "1.7.19",
"changeList": [
"1.7.19: imgurRipper no longer tries to add ?1 to file names",
"1.7.18: AlbumRipper now removes bad chars from file names",
"1.7.17: Fixed hentai.cafe autorip from clipboard",
"1.7.16: Eightmuses now supports pages containing both images and subpages",
"1.7.15: Eigthmuses ripper now uses ASAP ripping; Remove ripper and tests for gwarchives.com and hushpix.com; Remove ripper and tests for imagearn.com; Fixed pornhub video downloader",
"1.7.14: Tumblr API Key Choosing Fix; Make webtoons ripper download maximum quality images; Added twitch ripper; Added VSCO ripper; Fixed pornhub video ripper",
"1.7.13: disabled FuskatorRipperTest; Fixes xhamster.com video ripper; Add yuvutu.com ripper",
"1.7.12: Instagram ripper no longer 403s on certain images",
"1.7.11: Added gwarchives support to the cheveretoRipper; Gfycat Tests & Fix for bad reddit submissions; instagram ripper can now be made to skip videos",
"1.7.10: Added basic pornpics.com ripper; Fixed hentai.cafe regex",

30
src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java
View File

@ -69,23 +69,25 @@ public abstract class AbstractHTMLRipper extends AlbumRipper {
while (doc != null) {
List<String> imageURLs = getURLsFromPage(doc);
// Remove all but 1 image
if (isThisATest()) {
while (imageURLs.size() > 1) {
imageURLs.remove(1);
if (!hasASAPRipping()) {
// Remove all but 1 image
if (isThisATest()) {
while (imageURLs.size() > 1) {
imageURLs.remove(1);
}
}
}
if (imageURLs.size() == 0) {
throw new IOException("No images found at " + doc.location());
}
if (imageURLs.size() == 0) {
throw new IOException("No images found at " + doc.location());
}
for (String imageURL : imageURLs) {
index += 1;
logger.debug("Found image url #" + index + ": " + imageURL);
downloadURL(new URL(imageURL), index);
if (isStopped()) {
break;
for (String imageURL : imageURLs) {
index += 1;
logger.debug("Found image url #" + index + ": " + imageURL);
downloadURL(new URL(imageURL), index);
if (isStopped()) {
break;
}
}
}
if (hasDescriptionSupport() && Utils.getConfigBoolean("descriptions.save", false)) {

1
src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
View File

@ -43,6 +43,7 @@ public abstract class AbstractRipper
public abstract void rip() throws IOException;
public abstract String getHost();
public abstract String getGID(URL url) throws MalformedURLException;
public boolean hasASAPRipping() { return false; }
private boolean shouldStop = false;
private boolean thisIsATest = false;

6
src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java
View File

@ -13,6 +13,7 @@ import com.rarchives.ripme.ui.RipStatusMessage;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Utils;
// Should this file even exist? It does the same thing as abstractHTML ripper
/**'
* For ripping delicious albums off the interwebz.
@ -197,8 +198,11 @@ public abstract class AlbumRipper extends AbstractRipper {
title = super.getAlbumTitle(this.url);
}
logger.debug("Using album title '" + title + "'");
title = Utils.filesystemSafe(title);
path += title + File.separator;
path += title;
path = Utils.getOriginalDirectory(path) + File.separator; // check for case sensitive (unix only)
this.workingDir = new File(path);
if (!this.workingDir.exists()) {
logger.info("[+] Creating directory: " + Utils.removeCWD(this.workingDir));

2
src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java
View File

@ -28,7 +28,7 @@ public class CheveretoRipper extends AbstractHTMLRipper {
super(url);
}
private static List<String> explicit_domains_1 = Arrays.asList("hushpix.com", "tag-fox.com", "gwarchives.com");
private static List<String> explicit_domains_1 = Arrays.asList("tag-fox.com");
@Override
public String getHost() {

124
src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java
View File

@ -34,10 +34,16 @@ public class EightmusesRipper extends AbstractHTMLRipper {
super(url);
}
@Override
public boolean hasASAPRipping() {
return true;
}
@Override
public String getHost() {
return "8muses";
}
@Override
public String getDomain() {
return "8muses.com";
@ -82,67 +88,26 @@ public class EightmusesRipper extends AbstractHTMLRipper {
@Override
public List<String> getURLsFromPage(Document page) {
List<String> imageURLs = new ArrayList<>();
// get the first image link on the page and check if the last char in it is a number
// if it is a number then we're ripping a comic if not it's a subalbum
String firstImageLink = page.select("div.gallery > a.t-hover").first().attr("href");
Pattern p = Pattern.compile("/comix/picture/([a-zA-Z0-9\\-_/]*/)?\\d+");
Matcher m = p.matcher(firstImageLink);
if (!m.matches()) {
logger.info("Ripping subalbums");
// Page contains subalbums (not images)
Elements albumElements = page.select("div.gallery > a.t-hover");
List<Element> albumsList = albumElements.subList(0, albumElements.size());
Collections.reverse(albumsList);
// Iterate over elements in reverse order
for (Element subalbum : albumsList) {
String subUrl = subalbum.attr("href");
// This if is to skip ads which don't have a href
if (subUrl != "") {
subUrl = subUrl.replaceAll("\\.\\./", "");
if (subUrl.startsWith("//")) {
subUrl = "https:";
}
else if (!subUrl.startsWith("http://")) {
subUrl = "https://www.8muses.com" + subUrl;
}
try {
logger.info("Retrieving " + subUrl);
sendUpdate(STATUS.LOADING_RESOURCE, subUrl);
Document subPage = Http.url(subUrl).get();
// Get all images in subalbum, add to list.
List<String> subalbumImages = getURLsFromPage(subPage);
String albumTitle = subPage.select("meta[name=description]").attr("content");
albumTitle = albumTitle.replace("A huge collection of free porn comics for adults. Read ", "");
albumTitle = albumTitle.replace(" online for free at 8muses.com", "");
albumTitle = albumTitle.replace(" ", "_");
// albumTitle = albumTitle.replace("Sex and Porn Comics", "");
// albumTitle = albumTitle.replace("|", "");
// albumTitle = albumTitle.replace("8muses", "");
// albumTitle = albumTitle.replaceAll("-", "_");
// albumTitle = albumTitle.replaceAll(" ", "_");
// albumTitle = albumTitle.replaceAll("___", "_");
// albumTitle = albumTitle.replaceAll("__", "_");
// // This is here to remove the trailing __ from folder names
// albumTitle = albumTitle.replaceAll("__", "");
logger.info("Found " + subalbumImages.size() + " images in subalbum");
int prefix = 1;
for (String image : subalbumImages) {
URL imageUrl = new URL(image);
// urlTitles.put(imageUrl, albumTitle);
addURLToDownload(imageUrl, getPrefix(prefix), albumTitle, this.url.toExternalForm(), cookies);
prefix = prefix + 1;
}
rippingSubalbums = true;
imageURLs.addAll(subalbumImages);
} catch (IOException e) {
logger.warn("Error while loading subalbum " + subUrl, e);
}
int x = 1;
// This contains the thumbnails of all images on the page
Elements pageImages = page.getElementsByClass("c-tile");
for (Element thumb : pageImages) {
// If true this link is a sub album
if (thumb.attr("href").contains("/comix/album/")) {
String subUrl = "https://www.8muses.com" + thumb.attr("href");
try {
logger.info("Retrieving " + subUrl);
sendUpdate(STATUS.LOADING_RESOURCE, subUrl);
Document subPage = Http.url(subUrl).get();
// If the page below this one has images this line will download them
List<String> subalbumImages = getURLsFromPage(subPage);
logger.info("Found " + subalbumImages.size() + " images in subalbum");
} catch (IOException e) {
logger.warn("Error while loading subalbum " + subUrl, e);
}
}
}
else {
// Page contains images
for (Element thumb : page.select(".image")) {
} else if (thumb.attr("href").contains("/comix/picture/")) {
logger.info("Ripping image");
if (super.isStopped()) break;
// Find thumbnail image source
String image = null;
@ -150,16 +115,21 @@ public class EightmusesRipper extends AbstractHTMLRipper {
image = thumb.attr("data-cfsrc");
}
else {
String parentHref = thumb.parent().attr("href");
if (parentHref.equals("")) continue;
if (parentHref.startsWith("/")) {
parentHref = "https://www.8muses.com" + parentHref;
String imageHref = thumb.attr("href");
if (imageHref.equals("")) continue;
if (imageHref.startsWith("/")) {
imageHref = "https://www.8muses.com" + imageHref;
}
try {
logger.info("Retrieving full-size image location from " + parentHref);
image = getFullSizeImage(parentHref);
logger.info("Retrieving full-size image location from " + imageHref);
image = getFullSizeImage(imageHref);
URL imageUrl = new URL(image);
addURLToDownload(imageUrl, getPrefix(x), getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies);
// X is our page index
x++;
} catch (IOException e) {
logger.error("Failed to get full-size image from " + parentHref);
logger.error("Failed to get full-size image from " + imageHref);
continue;
}
}
@ -170,6 +140,7 @@ public class EightmusesRipper extends AbstractHTMLRipper {
imageURLs.add(image);
if (isThisATest()) break;
}
}
return imageURLs;
}
@ -182,6 +153,25 @@ public class EightmusesRipper extends AbstractHTMLRipper {
return "https://www.8muses.com/image/fm/" + imageName;
}
private String getTitle(String albumTitle) {
albumTitle = albumTitle.replace("A huge collection of free porn comics for adults. Read ", "");
albumTitle = albumTitle.replace(" online for free at 8muses.com", "");
albumTitle = albumTitle.replace(" ", "_");
return albumTitle;
}
private String getSubdir(String rawHref) {
logger.info("Raw title: " + rawHref);
String title = rawHref;
title = title.replaceAll("8muses - Sex and Porn Comics", "");
title = title.replaceAll("\t\t", "");
title = title.replaceAll("\n", "");
title = title.replaceAll("\\| ", "");
title = title.replace(" ", "-");
logger.info(title);
return title;
}
@Override
public void downloadURL(URL url, int index) {
addURLToDownload(url, getPrefix(index), "", this.url.toExternalForm(), cookies);

112
src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java
View File

@ -1,112 +0,0 @@
package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.utils.Http;
public class ImagearnRipper extends AbstractHTMLRipper {
public ImagearnRipper(URL url) throws IOException {
super(url);
}
@Override
public String getHost() {
return "imagearn";
}
@Override
public String getDomain() {
return "imagearn.com";
}
@Override
public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^.*imagearn.com/+gallery.php\\?id=([0-9]+).*$");
Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) {
return m.group(1);
}
throw new MalformedURLException(
"Expected imagearn.com gallery formats: "
+ "imagearn.com/gallery.php?id=####..."
+ " Got: " + url);
}
public URL sanitizeURL(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^.*imagearn.com/+image.php\\?id=[0-9]+.*$");
Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) {
// URL points to imagearn *image*, not gallery
try {
url = getGalleryFromImage(url);
} catch (Exception e) {
logger.error("[!] " + e.getMessage(), e);
}
}
return url;
}
private URL getGalleryFromImage(URL url) throws IOException {
Document doc = Http.url(url).get();
for (Element link : doc.select("a[href~=^gallery\\.php.*$]")) {
logger.info("LINK: " + link.toString());
if (link.hasAttr("href")
&& link.attr("href").contains("gallery.php")) {
url = new URL("http://imagearn.com/" + link.attr("href"));
logger.info("[!] Found gallery from given link: " + url);
return url;
}
}
throw new IOException("Failed to find gallery at URL " + url);
}
@Override
public Document getFirstPage() throws IOException {
return Http.url(url).get();
}
@Override
public String getAlbumTitle(URL url) throws MalformedURLException {
try {
Document doc = getFirstPage();
String title = doc.select("h3 > strong").first().text(); // profile name
return getHost() + "_" + title + "_" + getGID(url);
} catch (Exception e) {
// Fall back to default album naming convention
logger.warn("Failed to get album title from " + url, e);
}
return super.getAlbumTitle(url);
}
@Override
public List<String> getURLsFromPage(Document doc) {
List<String> imageURLs = new ArrayList<>();
for (Element thumb : doc.select("div#gallery > div > a")) {
String imageURL = thumb.attr("href");
try {
Document imagedoc = new Http("http://imagearn.com/" + imageURL).get();
String image = imagedoc.select("a.thickbox").first().attr("href");
imageURLs.add(image);
} catch (IOException e) {
logger.warn("Was unable to download page: " + imageURL);
}
}
return imageURLs;
}
@Override
public void downloadURL(URL url, int index) {
addURLToDownload(url, getPrefix(index));
sleep(1000);
}
}

1
src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java
View File

@ -213,6 +213,7 @@ public class ImgurRipper extends AlbumRipper {
saveAs += String.format("%03d_", index);
}
saveAs += imgurImage.getSaveAs();
saveAs = saveAs.replaceAll("\\?\\d", "");
addURLToDownload(imgurImage.url, new File(saveAs));
}
}

4
src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java
View File

@ -71,7 +71,9 @@ public class WebtoonsRipper extends AbstractHTMLRipper {
public List<String> getURLsFromPage(Document doc) {
List<String> result = new ArrayList<String>();
for (Element elem : doc.select("div.viewer_img > img")) {
result.add(elem.attr("data-url"));
String origUrl = elem.attr("data-url");
String[] finalUrl = origUrl.split("\\?type");
result.add(finalUrl[0]);
}
return result;
}

70
src/main/java/com/rarchives/ripme/ripper/rippers/YuvutuRipper.java Normal file
View File

@ -0,0 +1,70 @@
package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.utils.Http;
public class YuvutuRipper extends AbstractHTMLRipper {
private static final String DOMAIN = "yuvutu.com",
HOST = "yuvutu";
public YuvutuRipper(URL url) throws IOException {
super(url);
}
@Override
public String getHost() {
return HOST;
}
@Override
public String getDomain() {
return DOMAIN;
}
@Override
public boolean canRip(URL url) {
Pattern p = Pattern.compile("^http://www\\.yuvutu\\.com/modules\\.php\\?name=YuGallery&action=view&set_id=([0-9]+)$");
Matcher m = p.matcher(url.toExternalForm());
return m.matches();
}
@Override
public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^http://www\\.yuvutu\\.com/modules\\.php\\?name=YuGallery&action=view&set_id=([0-9]+)$");
Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) {
return m.group(1);
}
throw new MalformedURLException("Expected yuvutu.com URL format: " +
"yuvutu.com/modules.php?name=YuGallery&action=view&set_id=albumid - got " + url + "instead");
}
@Override
public Document getFirstPage() throws IOException {
return Http.url(url).get();
}
@Override
public List<String> getURLsFromPage(Document doc) {
List<String> imageURLs = new ArrayList<>();
for (Element thumb : doc.select("div#galleria > a > img")) {
String image = thumb.attr("src");
imageURLs.add(image);
}
return imageURLs;
}
@Override
public void downloadURL(URL url, int index) {
addURLToDownload(url, getPrefix(index));
}
}

12
src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java
View File

@ -28,7 +28,7 @@ public class PornhubRipper extends VideoRipper {
@Override
public boolean canRip(URL url) {
Pattern p = Pattern.compile("^https?://[wm.]*pornhub\\.com/view_video.php\\?viewkey=[0-9]+.*$");
Pattern p = Pattern.compile("^https?://[wm.]*pornhub\\.com/view_video.php\\?viewkey=[a-z0-9]+$");
Matcher m = p.matcher(url.toExternalForm());
return m.matches();
}
@ -40,7 +40,7 @@ public class PornhubRipper extends VideoRipper {
@Override
public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^https?://[wm.]*pornhub\\.com/view_video.php\\?viewkey=([0-9]+).*$");
Pattern p = Pattern.compile("^https?://[wm.]*pornhub\\.com/view_video.php\\?viewkey=([a-z0-9]+)$");
Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) {
return m.group(1);
@ -68,11 +68,11 @@ public class PornhubRipper extends VideoRipper {
title = title.replaceAll("\\+", " ");
vidUrl = null;
for (String quality : new String[] {"quality_1080p", "quality_720p", "quality_480p", "quality_240p"}) {
Pattern pv = Pattern.compile("^.*var player_" + quality + " = '([^']*)'.*$", Pattern.DOTALL);
for (String quality : new String[] {"1080", "720", "480", "240"}) {
Pattern pv = Pattern.compile("\"format\":\"\",\"quality\":\"" + quality + "\",\"videoUrl\":\"(.*?)\"");
Matcher mv = pv.matcher(html);
if (mv.matches()) {
vidUrl = mv.group(1);
if (mv.find()) {
vidUrl = mv.group(1).replace("\\/", "/");
break;
}
}

80
src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java Normal file
View File

@ -0,0 +1,80 @@
package com.rarchives.ripme.ripper.rippers.video;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.VideoRipper;
import com.rarchives.ripme.utils.Http;
public class TwitchVideoRipper extends VideoRipper {
private static final String HOST = "twitch";
public TwitchVideoRipper(URL url) throws IOException {
super(url);
}
@Override
public String getHost() {
return HOST;
}
@Override
public boolean canRip(URL url) {
Pattern p = Pattern.compile("^https://clips\\.twitch\\.tv/.*$");
Matcher m = p.matcher(url.toExternalForm());
return m.matches();
}
@Override
public URL sanitizeURL(URL url) throws MalformedURLException {
return url;
}
@Override
public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^https://clips\\.twitch\\.tv/(.*)$");
Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) {
return m.group(m.groupCount());
}
throw new MalformedURLException(
"Expected Twitch.tv format:"
+ "https://clips.twitch.tv/####"
+ " Got: " + url);
}
@Override
public void rip() throws IOException {
logger.info("Retrieving " + this.url);
Document doc = Http.url(url).get();
//Get user friendly filename from page title
String title = doc.title();
Elements script = doc.select("script");
if (script.size() == 0) {
throw new IOException("Could not find script code at " + url);
}
//Regex assumes highest quality source is listed first
Pattern p = Pattern.compile("\"source\":\"(.*?)\"");
for (Element element : script) {
Matcher m = p.matcher(element.data());
if (m.find()){
String vidUrl = m.group(1);
addURLToDownload(new URL(vidUrl), HOST + "_" + title);
}
}
waitForThreads();
}
}

6
src/main/java/com/rarchives/ripme/ripper/rippers/video/XhamsterRipper.java
View File

@ -39,10 +39,10 @@ public class XhamsterRipper extends VideoRipper {
@Override
public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^https?://.*xhamster\\.com/(movies|videos)/.*$");
Pattern p = Pattern.compile("^https?://.*xhamster\\.com/(movies|videos)/(.*)$");
Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) {
return m.group(1);
return m.group(2);
}
throw new MalformedURLException(
@ -56,7 +56,7 @@ public class XhamsterRipper extends VideoRipper {
public void rip() throws IOException {
logger.info("Retrieving " + this.url);
Document doc = Http.url(url).get();
Elements videos = doc.select("a.mp4Thumb");
Elements videos = doc.select("div.player-container > a");
if (videos.size() == 0) {
throw new IOException("Could not find Embed code at " + url);
}

82
src/main/java/com/rarchives/ripme/ripper/rippers/video/YuvutuRipper.java Normal file
View File

@ -0,0 +1,82 @@
package com.rarchives.ripme.ripper.rippers.video;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.VideoRipper;
import com.rarchives.ripme.utils.Http;
public class YuvutuRipper extends VideoRipper {
private static final String HOST = "yuvutu";
public YuvutuRipper(URL url) throws IOException {
super(url);
}
@Override
public String getHost() {
return HOST;
}
@Override
public boolean canRip(URL url) {
Pattern p = Pattern.compile("^http://www\\.yuvutu\\.com/video/[0-9]+/(.*)$");
Matcher m = p.matcher(url.toExternalForm());
return m.matches();
}
@Override
public URL sanitizeURL(URL url) throws MalformedURLException {
return url;
}
@Override
public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^http://www\\.yuvutu\\.com/video/[0-9]+/(.*)$");
Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) {
return m.group(1);
}
throw new MalformedURLException(
"Expected yuvutu format:"
+ "yuvutu.com/video/####"
+ " Got: " + url);
}
@Override
public void rip() throws IOException {
logger.info("Retrieving " + this.url);
Document doc = Http.url(url).get();
Element iframe = doc.select("iframe").first();
String iframeSrc = iframe.attr("src");
if (iframeSrc != null) {
doc = Http.url("http://www.yuvutu.com" + iframeSrc).get();
} else {
throw new IOException("Could not find iframe code at " + url);
}
Elements script = doc.select("script");
if (script.size() == 0) {
throw new IOException("Could not find script code at " + url);
}
Pattern p = Pattern.compile("file: \"(.*?)\"");
for (Element element : script) {
Matcher m = p.matcher(element.data());
if (m.find()){
String vidUrl = m.group(1);
addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url));
}
}
waitForThreads();
}
}

3
src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java
View File

@ -57,10 +57,11 @@ class AutoripThread extends Thread {
String clipboard = ClipboardUtils.getClipboardString();
if (clipboard != null) {
Pattern p = Pattern.compile(
// TODO: This regex is a monster and doesn't match all links; It needs to be rewritten
"\\b(((ht|f)tp(s?)://|~/|/)|www.)" +
"(\\w+:\\w+@)?(([-\\w]+\\.)+(com|org|net|gov" +
"|mil|biz|info|mobi|name|aero|jobs|museum" +
"|travel|[a-z]{2}))(:[\\d]{1,5})?" +
"|travel|cafe|[a-z]{2}))(:[\\d]{1,5})?" +
"(((/([-\\w~!$+|.,=]|%[a-f\\d]{2})+)+|/)+|\\?|#)?" +
"((\\?([-\\w~!$+|.,*:]|%[a-f\\d{2}])+=?" +
"([-\\w~!$+|.,*:=]|%[a-f\\d]{2})*)" +

2
src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
View File

@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils;
public class UpdateUtils {
private static final Logger logger = Logger.getLogger(UpdateUtils.class);
private static final String DEFAULT_VERSION = "1.7.12";
private static final String DEFAULT_VERSION = "1.7.19";
private static final String REPO_NAME = "ripmeapp/ripme";
private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json";
private static final String mainFileName = "ripme.jar";

40
src/main/java/com/rarchives/ripme/utils/Utils.java
View File

@ -8,11 +8,7 @@ import java.lang.reflect.Constructor;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
@ -370,6 +366,40 @@ public class Utils {
return text;
}
/**
* Checks if given path already exists as lowercase
*
* @param path - original path entered to be ripped
* @return path of existing folder or the original path if not present
*/
public static String getOriginalDirectory(String path) {
int index;
if(isUnix() || isMacOS()) {
index = path.lastIndexOf('/');
} else {
// current OS is windows - nothing to do here
return path;
}
String original = path; // needs to be checked if lowercase exists
String lastPart = original.substring(index+1).toLowerCase(); // setting lowercase to check if it exists
// Get a List of all Directories and check its lowercase
// if file exists return it
File f = new File(path.substring(0, index));
ArrayList<String> names = new ArrayList<String>(Arrays.asList(f.list()));
for (String s : names) {
if(s.toLowerCase().equals(lastPart)) {
// Building Path of existing file
return path.substring(0, index) + File.separator + s;
}
}
return original;
}
public static String bytesToHumanReadable(int bytes) {
float fbytes = (float) bytes;
String[] mags = new String[] {"", "K", "M", "G", "T"};

10
src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java
View File

@ -6,18 +6,8 @@ import java.net.URL;
import com.rarchives.ripme.ripper.rippers.CheveretoRipper;
public class CheveretoRipperTest extends RippersTest {
public void testHushpix() throws IOException {
CheveretoRipper ripper = new CheveretoRipper(new URL("https://hushpix.com/album/gKcu"));
testRipper(ripper);
}
public void testTagFox() throws IOException {
CheveretoRipper ripper = new CheveretoRipper(new URL("http://tag-fox.com/album/Thjb"));
testRipper(ripper);
}
public void testgwarchives() throws IOException {
CheveretoRipper ripper = new CheveretoRipper(new URL("https://gwarchives.com/album/ns4q"));
testRipper(ripper);
}
}

8
src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java
View File

@ -7,7 +7,11 @@ import com.rarchives.ripme.ripper.rippers.EightmusesRipper;
public class EightmusesRipperTest extends RippersTest {
public void testEightmusesAlbum() throws IOException {
EightmusesRipper ripper = new EightmusesRipper(new URL("https://www.8muses.com/album/jab-comics/a-model-life"));
// A simple image album
EightmusesRipper ripper = new EightmusesRipper(new URL("https://www.8muses.com/comix/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore"));
testRipper(ripper);
// Test pages with subalbums
ripper = new EightmusesRipper(new URL("https://www.8muses.com/comix/album/Blacknwhitecomics_com-Comix/BlacknWhiteComics/The-Mayor"));
testRipper(ripper);
}
}
}

26
src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuskatorRipperTest.java
View File

@ -1,13 +1,15 @@
package com.rarchives.ripme.tst.ripper.rippers;
//package com.rarchives.ripme.tst.ripper.rippers;
//
//import java.io.IOException;
//import java.net.URL;
//
//import com.rarchives.ripme.ripper.rippers.FuskatorRipper;
//
//public class FuskatorRipperTest extends RippersTest {
// public void testFuskatorAlbum() throws IOException {
// FuskatorRipper ripper = new FuskatorRipper(new URL("https://fuskator.com/thumbs/hqt6pPXAf9z/Shaved-Blonde-Babe-Katerina-Ambre.html"));
// testRipper(ripper);
// }
//}
import java.io.IOException;
import java.net.URL;
import com.rarchives.ripme.ripper.rippers.FuskatorRipper;
public class FuskatorRipperTest extends RippersTest {
public void testFuskatorAlbum() throws IOException {
FuskatorRipper ripper = new FuskatorRipper(new URL("https://fuskator.com/thumbs/hqt6pPXAf9z/Shaved-Blonde-Babe-Katerina-Ambre.html"));
testRipper(ripper);
}
}
// Disabled because of https://github.com/RipMeApp/ripme/issues/393

13
src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagearnRipperTest.java
View File

@ -1,13 +0,0 @@
package com.rarchives.ripme.tst.ripper.rippers;
import java.io.IOException;
import java.net.URL;
import com.rarchives.ripme.ripper.rippers.ImagearnRipper;
public class ImagearnRipperTest extends RippersTest {
public void testImagearnRip() throws IOException {
ImagearnRipper ripper = new ImagearnRipper(new URL("http://imagearn.com//gallery.php?id=578682"));
testRipper(ripper);
}
}

2
src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java
View File

@ -8,7 +8,7 @@ import com.rarchives.ripme.ripper.rippers.MotherlessRipper;
public class MotherlessRipperTest extends RippersTest {
// https://github.com/RipMeApp/ripme/issues/238 - MotherlessRipperTest is flaky on Travis CI
public void testMotherlessAlbumRip() throws IOException {
MotherlessRipper ripper = new MotherlessRipper(new URL("http://motherless.com/G4DAA18D"));
MotherlessRipper ripper = new MotherlessRipper(new URL("http://motherless.com/G1E5C971"));
testRipper(ripper);
}
}

35
src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java
View File

@ -7,9 +7,12 @@ import java.util.List;
import com.rarchives.ripme.ripper.VideoRipper;
import com.rarchives.ripme.ripper.rippers.video.PornhubRipper;
import com.rarchives.ripme.ripper.rippers.video.TwitchVideoRipper;
import com.rarchives.ripme.ripper.rippers.video.VineRipper;
import com.rarchives.ripme.ripper.rippers.video.XhamsterRipper;
import com.rarchives.ripme.ripper.rippers.video.XvideosRipper;
import com.rarchives.ripme.ripper.rippers.video.YoupornRipper;
import com.rarchives.ripme.ripper.rippers.video.YuvutuRipper;
public class VideoRippersTest extends RippersTest {
@ -35,6 +38,24 @@ public class VideoRippersTest extends RippersTest {
}
}
public void testTwitchVideoRipper() throws IOException {
List<URL> contentURLs = new ArrayList<>();
contentURLs.add(new URL("https://clips.twitch.tv/FaithfulIncredulousPotTBCheesePull"));
for (URL url : contentURLs) {
TwitchVideoRipper ripper = new TwitchVideoRipper(url);
videoTestHelper(ripper);
}
}
public void testXhamsterRipper() throws IOException {
List<URL> contentURLs = new ArrayList<>();
contentURLs.add(new URL("https://xhamster.com/videos/brazzers-busty-big-booty-milf-lisa-ann-fucks-her-masseur-1492828"));
for (URL url : contentURLs) {
XhamsterRipper ripper = new XhamsterRipper(url);
videoTestHelper(ripper);
}
}
public void testXvideosRipper() throws IOException {
List<URL> contentURLs = new ArrayList<>();
contentURLs.add(new URL("https://www.xvideos.com/video19719109/ziggy_star_ultra_hard_anal_pounding"));
@ -45,17 +66,14 @@ public class VideoRippersTest extends RippersTest {
}
}
// https://github.com/RipMeApp/ripme/issues/187
/*
public void testPornhubRipper() throws IOException {
List<URL> contentURLs = new ArrayList<>();
contentURLs.add(new URL("http://www.pornhub.com/view_video.php?viewkey=993166542"));
contentURLs.add(new URL("https://www.pornhub.com/view_video.php?viewkey=ph5a329fa707269"));
for (URL url : contentURLs) {
PornhubRipper ripper = new PornhubRipper(url);
videoTestHelper(ripper);
}
}
*/
// https://github.com/RipMeApp/ripme/issues/186
/*
@ -77,5 +95,14 @@ public class VideoRippersTest extends RippersTest {
videoTestHelper(ripper);
}
}
public void testYuvutuRipper() throws IOException {
List<URL> contentURLs = new ArrayList<>();
contentURLs.add(new URL("http://www.yuvutu.com/video/828499/female-reader-armpit-job/"));
for (URL url : contentURLs) {
YuvutuRipper ripper = new YuvutuRipper(url);
videoTestHelper(ripper);
}
}
}

5
src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java
View File

@ -10,4 +10,9 @@ public class WebtoonsRipperTest extends RippersTest {
WebtoonsRipper ripper = new WebtoonsRipper(new URL("http://www.webtoons.com/en/drama/my-boo/ep-33/viewer?title_no=1185&episode_no=33"));
testRipper(ripper);
}
public void testWebtoonsType() throws IOException {
WebtoonsRipper ripper = new WebtoonsRipper(new URL("http://www.webtoons.com/en/drama/lookism/ep-145/viewer?title_no=1049&episode_no=145"));
testRipper(ripper);
}
}

19
src/test/java/com/rarchives/ripme/tst/ripper/rippers/YuvutuRipperTest.java Normal file
View File

@ -0,0 +1,19 @@
package com.rarchives.ripme.tst.ripper.rippers;
import java.io.IOException;
import java.net.URL;
import com.rarchives.ripme.ripper.rippers.YuvutuRipper;
public class YuvutuRipperTest extends RippersTest {
public void testYuvutuAlbum1() throws IOException {
YuvutuRipper ripper = new YuvutuRipper(new URL("http://www.yuvutu.com/modules.php?name=YuGallery&action=view&set_id=127013"));
testRipper(ripper);
}
public void testYuvutuAlbum2() throws IOException {
YuvutuRipper ripper = new YuvutuRipper(new URL("http://www.yuvutu.com/modules.php?name=YuGallery&action=view&set_id=420333"));
testRipper(ripper);
}
}