diff --git a/docs/options.md b/docs/options.md deleted file mode 100644 index d972561b..00000000 --- a/docs/options.md +++ /dev/null @@ -1,40 +0,0 @@ -file.overwrite | bool | If true ripme will overwrite existing files rather than skip them - -clipboard.autorip | bool | If true ripme will try to download any links in the clip board - -error.skip404 | bool | Don't retry on 404 errors - -download.save_order| bool | If true ripme will prefix each downloaded file with a number in the order the file was download - -auto.update | bool | If true ripme will auto-update every time it's started - -play.sound | bool | If true ripme will play a sound every time a rip finishes - -download.show_popup| bool | TODO figure out what this is for - -log.save | bool | If true ripme will save it's logs - -urls_only.save | bool | If true ripme will save all urls to a text file and download no files - -album_titles.save | bool | Currently does nothing - -prefer.mp4 | bool | Prefer mp4 when downloading a video that has more than 1 format - -download.timeout | int | File download timeout (in milliseconds) - -page.timeout | int | Page download timeout (in milliseconds) - -download.max_size | int | Maximum size of downloaded files in bytes - -threads.size | int | The number of threads to use - -twitter.auth | String | Twitter API key (Base64'd) - -tumblr.auth | String | Tumblr API key - -log.level | String | The debug log level (Example: Log level: Debug) - -gw.api | String | TODO figure out what this is for - -twitter.max_requests | int | TODO figure out what this is for - diff --git a/pom.xml b/pom.xml index fe587fb9..476068b2 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.12 + 1.7.19 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 55b96a97..de0dc0a6 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,13 @@ { - "latestVersion": "1.7.12", + "latestVersion": "1.7.19", "changeList": [ + "1.7.19: imgurRipper no longer tries to add ?1 to file names", + "1.7.18: AlbumRipper now removes bad chars from file names", + "1.7.17: Fixed hentai.cafe autorip from clipboard", + "1.7.16: Eightmuses now supports pages containing both images and subpages", + "1.7.15: Eigthmuses ripper now uses ASAP ripping; Remove ripper and tests for gwarchives.com and hushpix.com; Remove ripper and tests for imagearn.com; Fixed pornhub video downloader", + "1.7.14: Tumblr API Key Choosing Fix; Make webtoons ripper download maximum quality images; Added twitch ripper; Added VSCO ripper; Fixed pornhub video ripper", + "1.7.13: disabled FuskatorRipperTest; Fixes xhamster.com video ripper; Add yuvutu.com ripper", "1.7.12: Instagram ripper no longer 403s on certain images", "1.7.11: Added gwarchives support to the cheveretoRipper; Gfycat Tests & Fix for bad reddit submissions; instagram ripper can now be made to skip videos", "1.7.10: Added basic pornpics.com ripper; Fixed hentai.cafe regex", diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 25d51007..e71eb389 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -69,23 +69,25 @@ public abstract class AbstractHTMLRipper extends AlbumRipper { while (doc != null) { List imageURLs = getURLsFromPage(doc); - // Remove all but 1 image - if (isThisATest()) { - while (imageURLs.size() > 1) { - imageURLs.remove(1); + if (!hasASAPRipping()) { + // Remove all but 1 image + if (isThisATest()) { + while (imageURLs.size() > 1) { + imageURLs.remove(1); + } } - } - if (imageURLs.size() == 0) { - throw new IOException("No images found at " + doc.location()); - } + if (imageURLs.size() == 0) { + throw new IOException("No images found at " + doc.location()); + } - for (String imageURL : imageURLs) { - index += 1; - logger.debug("Found image url #" + index + ": " + imageURL); - downloadURL(new URL(imageURL), index); - if (isStopped()) { - break; + for (String imageURL : imageURLs) { + index += 1; + logger.debug("Found image url #" + index + ": " + imageURL); + downloadURL(new URL(imageURL), index); + if (isStopped()) { + break; + } } } if (hasDescriptionSupport() && Utils.getConfigBoolean("descriptions.save", false)) { diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 8209df68..edddea78 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -43,6 +43,7 @@ public abstract class AbstractRipper public abstract void rip() throws IOException; public abstract String getHost(); public abstract String getGID(URL url) throws MalformedURLException; + public boolean hasASAPRipping() { return false; } private boolean shouldStop = false; private boolean thisIsATest = false; diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index a92f3870..1726343a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -13,6 +13,7 @@ import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Utils; +// Should this file even exist? It does the same thing as abstractHTML ripper /**' * For ripping delicious albums off the interwebz. @@ -197,8 +198,11 @@ public abstract class AlbumRipper extends AbstractRipper { title = super.getAlbumTitle(this.url); } logger.debug("Using album title '" + title + "'"); + title = Utils.filesystemSafe(title); - path += title + File.separator; + path += title; + path = Utils.getOriginalDirectory(path) + File.separator; // check for case sensitive (unix only) + this.workingDir = new File(path); if (!this.workingDir.exists()) { logger.info("[+] Creating directory: " + Utils.removeCWD(this.workingDir)); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java index 4b5fb9e6..160febfc 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java @@ -28,7 +28,7 @@ public class CheveretoRipper extends AbstractHTMLRipper { super(url); } - private static List explicit_domains_1 = Arrays.asList("hushpix.com", "tag-fox.com", "gwarchives.com"); + private static List explicit_domains_1 = Arrays.asList("tag-fox.com"); @Override public String getHost() { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java index 59fd63ba..b7b5658f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java @@ -34,10 +34,16 @@ public class EightmusesRipper extends AbstractHTMLRipper { super(url); } + @Override + public boolean hasASAPRipping() { + return true; + } + @Override public String getHost() { return "8muses"; } + @Override public String getDomain() { return "8muses.com"; @@ -82,67 +88,26 @@ public class EightmusesRipper extends AbstractHTMLRipper { @Override public List getURLsFromPage(Document page) { List imageURLs = new ArrayList<>(); - // get the first image link on the page and check if the last char in it is a number - // if it is a number then we're ripping a comic if not it's a subalbum - String firstImageLink = page.select("div.gallery > a.t-hover").first().attr("href"); - Pattern p = Pattern.compile("/comix/picture/([a-zA-Z0-9\\-_/]*/)?\\d+"); - Matcher m = p.matcher(firstImageLink); - if (!m.matches()) { - logger.info("Ripping subalbums"); - // Page contains subalbums (not images) - Elements albumElements = page.select("div.gallery > a.t-hover"); - List albumsList = albumElements.subList(0, albumElements.size()); - Collections.reverse(albumsList); - // Iterate over elements in reverse order - for (Element subalbum : albumsList) { - String subUrl = subalbum.attr("href"); - // This if is to skip ads which don't have a href - if (subUrl != "") { - subUrl = subUrl.replaceAll("\\.\\./", ""); - if (subUrl.startsWith("//")) { - subUrl = "https:"; - } - else if (!subUrl.startsWith("http://")) { - subUrl = "https://www.8muses.com" + subUrl; - } - try { - logger.info("Retrieving " + subUrl); - sendUpdate(STATUS.LOADING_RESOURCE, subUrl); - Document subPage = Http.url(subUrl).get(); - // Get all images in subalbum, add to list. - List subalbumImages = getURLsFromPage(subPage); - String albumTitle = subPage.select("meta[name=description]").attr("content"); - albumTitle = albumTitle.replace("A huge collection of free porn comics for adults. Read ", ""); - albumTitle = albumTitle.replace(" online for free at 8muses.com", ""); - albumTitle = albumTitle.replace(" ", "_"); - // albumTitle = albumTitle.replace("Sex and Porn Comics", ""); - // albumTitle = albumTitle.replace("|", ""); - // albumTitle = albumTitle.replace("8muses", ""); - // albumTitle = albumTitle.replaceAll("-", "_"); - // albumTitle = albumTitle.replaceAll(" ", "_"); - // albumTitle = albumTitle.replaceAll("___", "_"); - // albumTitle = albumTitle.replaceAll("__", "_"); - // // This is here to remove the trailing __ from folder names - // albumTitle = albumTitle.replaceAll("__", ""); - logger.info("Found " + subalbumImages.size() + " images in subalbum"); - int prefix = 1; - for (String image : subalbumImages) { - URL imageUrl = new URL(image); - // urlTitles.put(imageUrl, albumTitle); - addURLToDownload(imageUrl, getPrefix(prefix), albumTitle, this.url.toExternalForm(), cookies); - prefix = prefix + 1; - } - rippingSubalbums = true; - imageURLs.addAll(subalbumImages); - } catch (IOException e) { - logger.warn("Error while loading subalbum " + subUrl, e); - } + int x = 1; + // This contains the thumbnails of all images on the page + Elements pageImages = page.getElementsByClass("c-tile"); + for (Element thumb : pageImages) { + // If true this link is a sub album + if (thumb.attr("href").contains("/comix/album/")) { + String subUrl = "https://www.8muses.com" + thumb.attr("href"); + try { + logger.info("Retrieving " + subUrl); + sendUpdate(STATUS.LOADING_RESOURCE, subUrl); + Document subPage = Http.url(subUrl).get(); + // If the page below this one has images this line will download them + List subalbumImages = getURLsFromPage(subPage); + logger.info("Found " + subalbumImages.size() + " images in subalbum"); + } catch (IOException e) { + logger.warn("Error while loading subalbum " + subUrl, e); } - } - } - else { - // Page contains images - for (Element thumb : page.select(".image")) { + + } else if (thumb.attr("href").contains("/comix/picture/")) { + logger.info("Ripping image"); if (super.isStopped()) break; // Find thumbnail image source String image = null; @@ -150,16 +115,21 @@ public class EightmusesRipper extends AbstractHTMLRipper { image = thumb.attr("data-cfsrc"); } else { - String parentHref = thumb.parent().attr("href"); - if (parentHref.equals("")) continue; - if (parentHref.startsWith("/")) { - parentHref = "https://www.8muses.com" + parentHref; + String imageHref = thumb.attr("href"); + if (imageHref.equals("")) continue; + if (imageHref.startsWith("/")) { + imageHref = "https://www.8muses.com" + imageHref; } try { - logger.info("Retrieving full-size image location from " + parentHref); - image = getFullSizeImage(parentHref); + logger.info("Retrieving full-size image location from " + imageHref); + image = getFullSizeImage(imageHref); + URL imageUrl = new URL(image); + addURLToDownload(imageUrl, getPrefix(x), getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies); + // X is our page index + x++; + } catch (IOException e) { - logger.error("Failed to get full-size image from " + parentHref); + logger.error("Failed to get full-size image from " + imageHref); continue; } } @@ -170,6 +140,7 @@ public class EightmusesRipper extends AbstractHTMLRipper { imageURLs.add(image); if (isThisATest()) break; } + } return imageURLs; } @@ -182,6 +153,25 @@ public class EightmusesRipper extends AbstractHTMLRipper { return "https://www.8muses.com/image/fm/" + imageName; } + private String getTitle(String albumTitle) { + albumTitle = albumTitle.replace("A huge collection of free porn comics for adults. Read ", ""); + albumTitle = albumTitle.replace(" online for free at 8muses.com", ""); + albumTitle = albumTitle.replace(" ", "_"); + return albumTitle; + } + + private String getSubdir(String rawHref) { + logger.info("Raw title: " + rawHref); + String title = rawHref; + title = title.replaceAll("8muses - Sex and Porn Comics", ""); + title = title.replaceAll("\t\t", ""); + title = title.replaceAll("\n", ""); + title = title.replaceAll("\\| ", ""); + title = title.replace(" ", "-"); + logger.info(title); + return title; + } + @Override public void downloadURL(URL url, int index) { addURLToDownload(url, getPrefix(index), "", this.url.toExternalForm(), cookies); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java deleted file mode 100644 index 39fdaeba..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java +++ /dev/null @@ -1,112 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; - -public class ImagearnRipper extends AbstractHTMLRipper { - - public ImagearnRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return "imagearn"; - } - @Override - public String getDomain() { - return "imagearn.com"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^.*imagearn.com/+gallery.php\\?id=([0-9]+).*$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - throw new MalformedURLException( - "Expected imagearn.com gallery formats: " - + "imagearn.com/gallery.php?id=####..." - + " Got: " + url); - } - - public URL sanitizeURL(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^.*imagearn.com/+image.php\\?id=[0-9]+.*$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - // URL points to imagearn *image*, not gallery - try { - url = getGalleryFromImage(url); - } catch (Exception e) { - logger.error("[!] " + e.getMessage(), e); - } - } - return url; - } - - private URL getGalleryFromImage(URL url) throws IOException { - Document doc = Http.url(url).get(); - for (Element link : doc.select("a[href~=^gallery\\.php.*$]")) { - logger.info("LINK: " + link.toString()); - if (link.hasAttr("href") - && link.attr("href").contains("gallery.php")) { - url = new URL("http://imagearn.com/" + link.attr("href")); - logger.info("[!] Found gallery from given link: " + url); - return url; - } - } - throw new IOException("Failed to find gallery at URL " + url); - } - - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } - - @Override - public String getAlbumTitle(URL url) throws MalformedURLException { - try { - Document doc = getFirstPage(); - String title = doc.select("h3 > strong").first().text(); // profile name - return getHost() + "_" + title + "_" + getGID(url); - } catch (Exception e) { - // Fall back to default album naming convention - logger.warn("Failed to get album title from " + url, e); - } - return super.getAlbumTitle(url); - } - - @Override - public List getURLsFromPage(Document doc) { - List imageURLs = new ArrayList<>(); - for (Element thumb : doc.select("div#gallery > div > a")) { - String imageURL = thumb.attr("href"); - try { - Document imagedoc = new Http("http://imagearn.com/" + imageURL).get(); - String image = imagedoc.select("a.thickbox").first().attr("href"); - imageURLs.add(image); - } catch (IOException e) { - logger.warn("Was unable to download page: " + imageURL); - } - } - return imageURLs; - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - sleep(1000); - } -} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java index 7c271442..b595d9e2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java @@ -213,6 +213,7 @@ public class ImgurRipper extends AlbumRipper { saveAs += String.format("%03d_", index); } saveAs += imgurImage.getSaveAs(); + saveAs = saveAs.replaceAll("\\?\\d", ""); addURLToDownload(imgurImage.url, new File(saveAs)); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java index f7b74133..df373689 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java @@ -71,7 +71,9 @@ public class WebtoonsRipper extends AbstractHTMLRipper { public List getURLsFromPage(Document doc) { List result = new ArrayList(); for (Element elem : doc.select("div.viewer_img > img")) { - result.add(elem.attr("data-url")); + String origUrl = elem.attr("data-url"); + String[] finalUrl = origUrl.split("\\?type"); + result.add(finalUrl[0]); } return result; } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/YuvutuRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/YuvutuRipper.java new file mode 100644 index 00000000..97365aa8 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/YuvutuRipper.java @@ -0,0 +1,70 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class YuvutuRipper extends AbstractHTMLRipper { + + private static final String DOMAIN = "yuvutu.com", + HOST = "yuvutu"; + + public YuvutuRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return HOST; + } + @Override + public String getDomain() { + return DOMAIN; + } + + @Override + public boolean canRip(URL url) { + Pattern p = Pattern.compile("^http://www\\.yuvutu\\.com/modules\\.php\\?name=YuGallery&action=view&set_id=([0-9]+)$"); + Matcher m = p.matcher(url.toExternalForm()); + return m.matches(); + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^http://www\\.yuvutu\\.com/modules\\.php\\?name=YuGallery&action=view&set_id=([0-9]+)$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected yuvutu.com URL format: " + + "yuvutu.com/modules.php?name=YuGallery&action=view&set_id=albumid - got " + url + "instead"); + } + + @Override + public Document getFirstPage() throws IOException { + return Http.url(url).get(); + } + @Override + public List getURLsFromPage(Document doc) { + List imageURLs = new ArrayList<>(); + for (Element thumb : doc.select("div#galleria > a > img")) { + String image = thumb.attr("src"); + imageURLs.add(image); + } + return imageURLs; + } + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java index 7f10323d..ea98bcfd 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java @@ -28,7 +28,7 @@ public class PornhubRipper extends VideoRipper { @Override public boolean canRip(URL url) { - Pattern p = Pattern.compile("^https?://[wm.]*pornhub\\.com/view_video.php\\?viewkey=[0-9]+.*$"); + Pattern p = Pattern.compile("^https?://[wm.]*pornhub\\.com/view_video.php\\?viewkey=[a-z0-9]+$"); Matcher m = p.matcher(url.toExternalForm()); return m.matches(); } @@ -40,7 +40,7 @@ public class PornhubRipper extends VideoRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://[wm.]*pornhub\\.com/view_video.php\\?viewkey=([0-9]+).*$"); + Pattern p = Pattern.compile("^https?://[wm.]*pornhub\\.com/view_video.php\\?viewkey=([a-z0-9]+)$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); @@ -68,11 +68,11 @@ public class PornhubRipper extends VideoRipper { title = title.replaceAll("\\+", " "); vidUrl = null; - for (String quality : new String[] {"quality_1080p", "quality_720p", "quality_480p", "quality_240p"}) { - Pattern pv = Pattern.compile("^.*var player_" + quality + " = '([^']*)'.*$", Pattern.DOTALL); + for (String quality : new String[] {"1080", "720", "480", "240"}) { + Pattern pv = Pattern.compile("\"format\":\"\",\"quality\":\"" + quality + "\",\"videoUrl\":\"(.*?)\""); Matcher mv = pv.matcher(html); - if (mv.matches()) { - vidUrl = mv.group(1); + if (mv.find()) { + vidUrl = mv.group(1).replace("\\/", "/"); break; } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java new file mode 100644 index 00000000..9cca50aa --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java @@ -0,0 +1,80 @@ +package com.rarchives.ripme.ripper.rippers.video; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + + +import com.rarchives.ripme.ripper.VideoRipper; +import com.rarchives.ripme.utils.Http; + +public class TwitchVideoRipper extends VideoRipper { + + private static final String HOST = "twitch"; + + public TwitchVideoRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return HOST; + } + + @Override + public boolean canRip(URL url) { + Pattern p = Pattern.compile("^https://clips\\.twitch\\.tv/.*$"); + Matcher m = p.matcher(url.toExternalForm()); + return m.matches(); + } + + @Override + public URL sanitizeURL(URL url) throws MalformedURLException { + return url; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^https://clips\\.twitch\\.tv/(.*)$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(m.groupCount()); + } + + throw new MalformedURLException( + "Expected Twitch.tv format:" + + "https://clips.twitch.tv/####" + + " Got: " + url); + } + + @Override + public void rip() throws IOException { + logger.info("Retrieving " + this.url); + Document doc = Http.url(url).get(); + + //Get user friendly filename from page title + String title = doc.title(); + + Elements script = doc.select("script"); + if (script.size() == 0) { + throw new IOException("Could not find script code at " + url); + } + //Regex assumes highest quality source is listed first + Pattern p = Pattern.compile("\"source\":\"(.*?)\""); + + for (Element element : script) { + Matcher m = p.matcher(element.data()); + if (m.find()){ + String vidUrl = m.group(1); + addURLToDownload(new URL(vidUrl), HOST + "_" + title); + } + } + waitForThreads(); + } +} \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/XhamsterRipper.java index e559ebec..b30f5a1a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/XhamsterRipper.java @@ -39,10 +39,10 @@ public class XhamsterRipper extends VideoRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://.*xhamster\\.com/(movies|videos)/.*$"); + Pattern p = Pattern.compile("^https?://.*xhamster\\.com/(movies|videos)/(.*)$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { - return m.group(1); + return m.group(2); } throw new MalformedURLException( @@ -56,7 +56,7 @@ public class XhamsterRipper extends VideoRipper { public void rip() throws IOException { logger.info("Retrieving " + this.url); Document doc = Http.url(url).get(); - Elements videos = doc.select("a.mp4Thumb"); + Elements videos = doc.select("div.player-container > a"); if (videos.size() == 0) { throw new IOException("Could not find Embed code at " + url); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/YuvutuRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/YuvutuRipper.java new file mode 100644 index 00000000..f455abe2 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/YuvutuRipper.java @@ -0,0 +1,82 @@ +package com.rarchives.ripme.ripper.rippers.video; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + + +import com.rarchives.ripme.ripper.VideoRipper; +import com.rarchives.ripme.utils.Http; + +public class YuvutuRipper extends VideoRipper { + + private static final String HOST = "yuvutu"; + + public YuvutuRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return HOST; + } + + @Override + public boolean canRip(URL url) { + Pattern p = Pattern.compile("^http://www\\.yuvutu\\.com/video/[0-9]+/(.*)$"); + Matcher m = p.matcher(url.toExternalForm()); + return m.matches(); + } + + @Override + public URL sanitizeURL(URL url) throws MalformedURLException { + return url; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^http://www\\.yuvutu\\.com/video/[0-9]+/(.*)$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + + throw new MalformedURLException( + "Expected yuvutu format:" + + "yuvutu.com/video/####" + + " Got: " + url); + } + + @Override + public void rip() throws IOException { + logger.info("Retrieving " + this.url); + Document doc = Http.url(url).get(); + Element iframe = doc.select("iframe").first(); + String iframeSrc = iframe.attr("src"); + if (iframeSrc != null) { + doc = Http.url("http://www.yuvutu.com" + iframeSrc).get(); + } else { + throw new IOException("Could not find iframe code at " + url); + } + Elements script = doc.select("script"); + if (script.size() == 0) { + throw new IOException("Could not find script code at " + url); + } + Pattern p = Pattern.compile("file: \"(.*?)\""); + + for (Element element : script) { + Matcher m = p.matcher(element.data()); + if (m.find()){ + String vidUrl = m.group(1); + addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url)); + } + } + waitForThreads(); + } +} diff --git a/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java b/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java index 34915146..24c46cd4 100644 --- a/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java @@ -57,10 +57,11 @@ class AutoripThread extends Thread { String clipboard = ClipboardUtils.getClipboardString(); if (clipboard != null) { Pattern p = Pattern.compile( + // TODO: This regex is a monster and doesn't match all links; It needs to be rewritten "\\b(((ht|f)tp(s?)://|~/|/)|www.)" + "(\\w+:\\w+@)?(([-\\w]+\\.)+(com|org|net|gov" + "|mil|biz|info|mobi|name|aero|jobs|museum" + - "|travel|[a-z]{2}))(:[\\d]{1,5})?" + + "|travel|cafe|[a-z]{2}))(:[\\d]{1,5})?" + "(((/([-\\w~!$+|.,=]|%[a-f\\d]{2})+)+|/)+|\\?|#)?" + "((\\?([-\\w~!$+|.,*:]|%[a-f\\d{2}])+=?" + "([-\\w~!$+|.,*:=]|%[a-f\\d]{2})*)" + diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 15654f5a..37ce6cfe 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.12"; + private static final String DEFAULT_VERSION = "1.7.19"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index d9f8bf9e..9517c528 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -8,11 +8,7 @@ import java.lang.reflect.Constructor; import java.net.URISyntaxException; import java.net.URL; import java.net.URLDecoder; -import java.util.ArrayList; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.jar.JarEntry; import java.util.jar.JarFile; @@ -370,6 +366,40 @@ public class Utils { return text; } + /** + * Checks if given path already exists as lowercase + * + * @param path - original path entered to be ripped + * @return path of existing folder or the original path if not present + */ + public static String getOriginalDirectory(String path) { + + int index; + if(isUnix() || isMacOS()) { + index = path.lastIndexOf('/'); + } else { + // current OS is windows - nothing to do here + return path; + } + + String original = path; // needs to be checked if lowercase exists + String lastPart = original.substring(index+1).toLowerCase(); // setting lowercase to check if it exists + + // Get a List of all Directories and check its lowercase + // if file exists return it + File f = new File(path.substring(0, index)); + ArrayList names = new ArrayList(Arrays.asList(f.list())); + + for (String s : names) { + if(s.toLowerCase().equals(lastPart)) { + // Building Path of existing file + return path.substring(0, index) + File.separator + s; + } + } + + return original; + } + public static String bytesToHumanReadable(int bytes) { float fbytes = (float) bytes; String[] mags = new String[] {"", "K", "M", "G", "T"}; diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java index 0c6344ea..4f456b8d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java @@ -6,18 +6,8 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.CheveretoRipper; public class CheveretoRipperTest extends RippersTest { - public void testHushpix() throws IOException { - CheveretoRipper ripper = new CheveretoRipper(new URL("https://hushpix.com/album/gKcu")); - testRipper(ripper); - } - public void testTagFox() throws IOException { CheveretoRipper ripper = new CheveretoRipper(new URL("http://tag-fox.com/album/Thjb")); testRipper(ripper); } - - public void testgwarchives() throws IOException { - CheveretoRipper ripper = new CheveretoRipper(new URL("https://gwarchives.com/album/ns4q")); - testRipper(ripper); - } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java index 068f9a81..469c330a 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java @@ -7,7 +7,11 @@ import com.rarchives.ripme.ripper.rippers.EightmusesRipper; public class EightmusesRipperTest extends RippersTest { public void testEightmusesAlbum() throws IOException { - EightmusesRipper ripper = new EightmusesRipper(new URL("https://www.8muses.com/album/jab-comics/a-model-life")); + // A simple image album + EightmusesRipper ripper = new EightmusesRipper(new URL("https://www.8muses.com/comix/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore")); + testRipper(ripper); + // Test pages with subalbums + ripper = new EightmusesRipper(new URL("https://www.8muses.com/comix/album/Blacknwhitecomics_com-Comix/BlacknWhiteComics/The-Mayor")); testRipper(ripper); } -} +} \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuskatorRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuskatorRipperTest.java index f5bc6f0d..20c8473d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuskatorRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuskatorRipperTest.java @@ -1,13 +1,15 @@ -package com.rarchives.ripme.tst.ripper.rippers; +//package com.rarchives.ripme.tst.ripper.rippers; +// +//import java.io.IOException; +//import java.net.URL; +// +//import com.rarchives.ripme.ripper.rippers.FuskatorRipper; +// +//public class FuskatorRipperTest extends RippersTest { +// public void testFuskatorAlbum() throws IOException { +// FuskatorRipper ripper = new FuskatorRipper(new URL("https://fuskator.com/thumbs/hqt6pPXAf9z/Shaved-Blonde-Babe-Katerina-Ambre.html")); +// testRipper(ripper); +// } +//} -import java.io.IOException; -import java.net.URL; - -import com.rarchives.ripme.ripper.rippers.FuskatorRipper; - -public class FuskatorRipperTest extends RippersTest { - public void testFuskatorAlbum() throws IOException { - FuskatorRipper ripper = new FuskatorRipper(new URL("https://fuskator.com/thumbs/hqt6pPXAf9z/Shaved-Blonde-Babe-Katerina-Ambre.html")); - testRipper(ripper); - } -} +// Disabled because of https://github.com/RipMeApp/ripme/issues/393 \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagearnRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagearnRipperTest.java deleted file mode 100644 index b9710635..00000000 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagearnRipperTest.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import java.io.IOException; -import java.net.URL; - -import com.rarchives.ripme.ripper.rippers.ImagearnRipper; - -public class ImagearnRipperTest extends RippersTest { - public void testImagearnRip() throws IOException { - ImagearnRipper ripper = new ImagearnRipper(new URL("http://imagearn.com//gallery.php?id=578682")); - testRipper(ripper); - } -} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java index 6b00f170..44caa03b 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java @@ -8,7 +8,7 @@ import com.rarchives.ripme.ripper.rippers.MotherlessRipper; public class MotherlessRipperTest extends RippersTest { // https://github.com/RipMeApp/ripme/issues/238 - MotherlessRipperTest is flaky on Travis CI public void testMotherlessAlbumRip() throws IOException { - MotherlessRipper ripper = new MotherlessRipper(new URL("http://motherless.com/G4DAA18D")); + MotherlessRipper ripper = new MotherlessRipper(new URL("http://motherless.com/G1E5C971")); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java index 73c818d0..ae31192b 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java @@ -7,9 +7,12 @@ import java.util.List; import com.rarchives.ripme.ripper.VideoRipper; import com.rarchives.ripme.ripper.rippers.video.PornhubRipper; +import com.rarchives.ripme.ripper.rippers.video.TwitchVideoRipper; import com.rarchives.ripme.ripper.rippers.video.VineRipper; +import com.rarchives.ripme.ripper.rippers.video.XhamsterRipper; import com.rarchives.ripme.ripper.rippers.video.XvideosRipper; import com.rarchives.ripme.ripper.rippers.video.YoupornRipper; +import com.rarchives.ripme.ripper.rippers.video.YuvutuRipper; public class VideoRippersTest extends RippersTest { @@ -35,6 +38,24 @@ public class VideoRippersTest extends RippersTest { } } + public void testTwitchVideoRipper() throws IOException { + List contentURLs = new ArrayList<>(); + contentURLs.add(new URL("https://clips.twitch.tv/FaithfulIncredulousPotTBCheesePull")); + for (URL url : contentURLs) { + TwitchVideoRipper ripper = new TwitchVideoRipper(url); + videoTestHelper(ripper); + } + } + + public void testXhamsterRipper() throws IOException { + List contentURLs = new ArrayList<>(); + contentURLs.add(new URL("https://xhamster.com/videos/brazzers-busty-big-booty-milf-lisa-ann-fucks-her-masseur-1492828")); + for (URL url : contentURLs) { + XhamsterRipper ripper = new XhamsterRipper(url); + videoTestHelper(ripper); + } + } + public void testXvideosRipper() throws IOException { List contentURLs = new ArrayList<>(); contentURLs.add(new URL("https://www.xvideos.com/video19719109/ziggy_star_ultra_hard_anal_pounding")); @@ -45,17 +66,14 @@ public class VideoRippersTest extends RippersTest { } } - // https://github.com/RipMeApp/ripme/issues/187 - /* public void testPornhubRipper() throws IOException { List contentURLs = new ArrayList<>(); - contentURLs.add(new URL("http://www.pornhub.com/view_video.php?viewkey=993166542")); + contentURLs.add(new URL("https://www.pornhub.com/view_video.php?viewkey=ph5a329fa707269")); for (URL url : contentURLs) { PornhubRipper ripper = new PornhubRipper(url); videoTestHelper(ripper); } } - */ // https://github.com/RipMeApp/ripme/issues/186 /* @@ -77,5 +95,14 @@ public class VideoRippersTest extends RippersTest { videoTestHelper(ripper); } } + + public void testYuvutuRipper() throws IOException { + List contentURLs = new ArrayList<>(); + contentURLs.add(new URL("http://www.yuvutu.com/video/828499/female-reader-armpit-job/")); + for (URL url : contentURLs) { + YuvutuRipper ripper = new YuvutuRipper(url); + videoTestHelper(ripper); + } + } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java index 20cf17b8..4aa50d0a 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java @@ -10,4 +10,9 @@ public class WebtoonsRipperTest extends RippersTest { WebtoonsRipper ripper = new WebtoonsRipper(new URL("http://www.webtoons.com/en/drama/my-boo/ep-33/viewer?title_no=1185&episode_no=33")); testRipper(ripper); } + + public void testWebtoonsType() throws IOException { + WebtoonsRipper ripper = new WebtoonsRipper(new URL("http://www.webtoons.com/en/drama/lookism/ep-145/viewer?title_no=1049&episode_no=145")); + testRipper(ripper); + } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YuvutuRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YuvutuRipperTest.java new file mode 100644 index 00000000..a74b76e9 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YuvutuRipperTest.java @@ -0,0 +1,19 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.YuvutuRipper; + +public class YuvutuRipperTest extends RippersTest { + + public void testYuvutuAlbum1() throws IOException { + YuvutuRipper ripper = new YuvutuRipper(new URL("http://www.yuvutu.com/modules.php?name=YuGallery&action=view&set_id=127013")); + testRipper(ripper); + } + + public void testYuvutuAlbum2() throws IOException { + YuvutuRipper ripper = new YuvutuRipper(new URL("http://www.yuvutu.com/modules.php?name=YuGallery&action=view&set_id=420333")); + testRipper(ripper); + } +}