Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #14 from RipMeApp/master

Browse Source 
asdf
This commit is contained in:
Kevin Jiang 2018-08-01 04:36:37 -04:00 committed by GitHub
commit a56c2f8ba6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
61 changed files with 2020 additions and 743 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
patch.py
View File

@ -8,8 +8,8 @@ from hashlib import sha256
# - update version in a few places # - update version in a few places
# - insert new line in ripme.json with message # - insert new line in ripme.json with message
# - build ripme # - build ripme
# - add the hash of the lastest binary to ripme.json # - add the hash of the latest binary to ripme.json
# - commit all changes
message = input('message: ') message = input('message: ')
@ -30,8 +30,7 @@ def update_hash(current_hash):
def update_change_list(message): def update_change_list(message):
ripmeJson = get_ripme_json() ripmeJson = get_ripme_json()
with open('ripme.json', 'w') as dataFile: with open('ripme.json', 'w') as dataFile:
ripmeJson["changeList"] = ripmeJson["changeList"].insert(0, message) ripmeJson["changeList"].insert(0, message)
print(ripmeJson["currentHash"])
json.dump(ripmeJson, dataFile, indent=4) json.dump(ripmeJson, dataFile, indent=4)
@ -63,17 +62,9 @@ subprocess.call(['sed', '-i', '-e', substrExpr, 'pom.xml'])
subprocess.call(['git', 'grep', '<version>' + nextVersion + '</version>', 'pom.xml']) subprocess.call(['git', 'grep', '<version>' + nextVersion + '</version>', 'pom.xml'])
commitMessage = nextVersion + ': ' + message commitMessage = nextVersion + ': ' + message
changeLogLine = ' \"' + commitMessage + '\",\n'
dataFile = open("ripme.json", "r") update_change_list(commitMessage)
ripmeJsonLines = dataFile.readlines()
ripmeJsonLines.insert(3, changeLogLine)
outputContent = ''.join(ripmeJsonLines)
dataFile.close()
dataFile = open("ripme.json", "w")
dataFile.write(outputContent)
dataFile.close()
print("Building ripme") print("Building ripme")
subprocess.call(["mvn", "clean", "compile", "assembly:single"]) subprocess.call(["mvn", "clean", "compile", "assembly:single"])
@ -89,3 +80,4 @@ update_hash(file_hash)
subprocess.call(['git', 'add', '-u']) subprocess.call(['git', 'add', '-u'])
subprocess.call(['git', 'commit', '-m', commitMessage]) subprocess.call(['git', 'commit', '-m', commitMessage])
subprocess.call(['git', 'tag', nextVersion]) subprocess.call(['git', 'tag', nextVersion])
print("Remember to run `git push origin master` before release.py")

2
pom.xml
View File

@ -4,7 +4,7 @@
<groupId>com.rarchives.ripme</groupId> <groupId>com.rarchives.ripme</groupId>
<artifactId>ripme</artifactId> <artifactId>ripme</artifactId>
<packaging>jar</packaging> <packaging>jar</packaging>
<version>1.7.51</version> <version>1.7.60</version>
<name>ripme</name> <name>ripme</name>
<url>http://rip.rarchives.com</url> <url>http://rip.rarchives.com</url>
<properties> <properties>

61
release.py Normal file → Executable file
View File

@ -1,4 +1,4 @@
#!/usr/bin/python3 #!/usr/bin/env python3
import re import re
@ -15,9 +15,12 @@ parser.add_argument("-f", "--file", help="Path to the version of ripme to releas
parser.add_argument("-t", "--token", help="Your github personal access token") parser.add_argument("-t", "--token", help="Your github personal access token")
parser.add_argument("-d", "--debug", help="Run in debug mode", action="store_true") parser.add_argument("-d", "--debug", help="Run in debug mode", action="store_true")
parser.add_argument("-n", "--non-interactive", help="Do not ask for any input from the user", action="store_true") parser.add_argument("-n", "--non-interactive", help="Do not ask for any input from the user", action="store_true")
parser.add_argument("--test", help="Perform a dry run (Do everything but upload new release)", action="store_true")
parser.add_argument("--skip-hash-check", help="Skip hash check (This should only be used for testing)", action="store_true")
args = parser.parse_args() args = parser.parse_args()
try: try:
# This binds input to raw_input on python2, we do this because input acts like eval on python2
input = raw_input input = raw_input
except NameError: except NameError:
pass pass
@ -34,13 +37,24 @@ def isJar(filename):
# false if not # false if not
def isValidCommitMessage(message): def isValidCommitMessage(message):
if debug: if debug:
print("Checking if {} matchs pattern ^\d+\.\d+\.\d+:".format(message)) print(r"Checking if {} matches pattern ^\d+\.\d+\.\d+:".format(message))
pattern = re.compile("^\d+\.\d+\.\d+:") pattern = re.compile(r"^\d+\.\d+\.\d+:")
return re.match(pattern, message) return re.match(pattern, message)
# Checks if the update has the name ripme.jar, if not it renames the file
def checkAndRenameFile(path):
"""Check if path (a string) points to a ripme.jar. Returns the possibly renamed file path"""
if not path.endswith("ripme.jar"):
print("Specified file is not named ripme.jar, renaming")
new_path = os.path.join(os.path.dirname(path), "ripme.jar")
os.rename(path, new_path)
return new_path
return path
ripmeJson = json.loads(open("ripme.json").read()) ripmeJson = json.loads(open("ripme.json").read())
fileToUploadPath = args.file fileToUploadPath = checkAndRenameFile(args.file)
InNoninteractiveMode = args.non_interactive InNoninteractiveMode = args.non_interactive
commitMessage = ripmeJson.get("changeList")[0] commitMessage = ripmeJson.get("changeList")[0]
releaseVersion = ripmeJson.get("latestVersion") releaseVersion = ripmeJson.get("latestVersion")
@ -61,22 +75,27 @@ if not isValidCommitMessage(commitMessage):
print("[!] Error: {} is not a valid commit message as it does not start with a version".format(fileToUploadPath)) print("[!] Error: {} is not a valid commit message as it does not start with a version".format(fileToUploadPath))
sys.exit(1) sys.exit(1)
ripmeUpdate = open(fileToUploadPath, mode='rb').read()
# The hash that we expect the update to have if not args.skip_hash_check:
expectedHash = ripmeJson.get("currentHash") if debug:
print("Reading file {}".format(fileToUploadPath))
ripmeUpdate = open(fileToUploadPath, mode='rb').read()
# The actual hash of the file on disk # The actual hash of the file on disk
actualHash = sha256(ripmeUpdate).hexdigest() actualHash = sha256(ripmeUpdate).hexdigest()
# Make sure that the hash of the file we're uploading matches the hash in ripme.json. These hashes not matching will # The hash that we expect the update to have
# cause ripme to refuse to install the update for all users who haven't disabled update hash checking expectedHash = ripmeJson.get("currentHash")
if expectedHash != actualHash:
# Make sure that the hash of the file we're uploading matches the hash in ripme.json. These hashes not matching will
# cause ripme to refuse to install the update for all users who haven't disabled update hash checking
if expectedHash != actualHash:
print("[!] Error: expected hash of file and actual hash differ") print("[!] Error: expected hash of file and actual hash differ")
print("[!] Expected hash is {}".format(expectedHash)) print("[!] Expected hash is {}".format(expectedHash))
print("[!] Actual hash is {}".format(actualHash)) print("[!] Actual hash is {}".format(actualHash))
sys.exit(1) sys.exit(1)
else:
print("[*] WARNING: SKIPPING HASH CHECK")
# Ask the user to review the information before we precede # Ask the user to review the information before we precede
# This only runs in we're in interactive mode # This only runs in we're in interactive mode
if not InNoninteractiveMode: if not InNoninteractiveMode:
@ -85,12 +104,14 @@ if not InNoninteractiveMode:
print("Repo: {}/{}".format(repoOwner, repoName)) print("Repo: {}/{}".format(repoOwner, repoName))
input("\nPlease review the information above and ensure it is correct and then press enter") input("\nPlease review the information above and ensure it is correct and then press enter")
print("Accessing github using token") if not args.test:
g = Github(accessToken) print("Accessing github using token")
g = Github(accessToken)
print("Creating release")
release = g.get_user(repoOwner).get_repo(repoName).create_git_release(releaseVersion, commitMessage, "")
print("Creating release") print("Uploading file")
release = g.get_user(repoOwner).get_repo(repoName).create_git_release(releaseVersion, commitMessage, "") release.upload_asset(fileToUploadPath, "ripme.jar")
else:
print("Uploading file") print("Not uploading release being script was run with --test flag")
release.upload_asset(fileToUploadPath, "ripme.jar")

13
ripme.json
View File

@ -1,6 +1,14 @@
{ {
"currentHash": "aadb71bf5cdf46fe92e270b50a55c8d8d7200a6dd304a4c2ac9f68cddc687d7e",
"changeList": [ "changeList": [
"1.7.60: Fixed EightmusesRipper; added Jab Archives ripper; loveroms ripper now properly names files; Fixed ArtStationRipper",
"1.7.59: Added Loverom ripper; added Imagearn ripper; Added support for Desuarchive.org; Fixed erome ripper",
"1.7.58: Fixed Deviantart ripper; fixed HitomiRipper; Fixed ManganeloRipper; Fixed update box formating",
"1.7.57: Got DeviantartRipper working again; Imagefap ripper now downloads full sized images; Twitter ripper can now rip extended tweets; Added nl_NL translation",
"1.7.56: Fixed DeviantartRipper ripper; Added support for resuming file downloads; Fixed erome ripper; Fixed ModelmayhemRipper NSFW image downloading",
"1.7.55: Fixed instagram ripper; Reddit ripper now respects history.end_rip_after_already_seen; Improvements to patch.py and release.py",
"1.7.54: Fixed twitter ripper video downloading; fixed instagram ripper",
"1.7.53: Added Picstatio ripper; Fixed instagram ripper; Reddit ripper now gets videos from v.redd.it; Fixed ZikiRipper getAlbumTitle; fixed twitter ripper",
"1.7.52: Added warning about using furaffinty shared account; Refactoring in Utils class; XhamsterRipper now accepts all countries subdomains; E621 ripper now accepts urls with order:Score at the end; release.py imrpovements; DeviantartRipper now logs in using cookies; patch.py imrpovements",
"1.7.51: Fixed instagram ripper; Added the ability to rip from vsco profiles; Fixed TheyiffgalleryRipper; Can now update ripme using the -j flag; added script to automate releases; Code style fixes", "1.7.51: Fixed instagram ripper; Added the ability to rip from vsco profiles; Fixed TheyiffgalleryRipper; Can now update ripme using the -j flag; added script to automate releases; Code style fixes",
"1.7.50: Ripme now checks file hash before running update; fixed update bug which cased ripme to report every update as new", "1.7.50: Ripme now checks file hash before running update; fixed update bug which cased ripme to report every update as new",
"1.7.49: Fixed -n flag; Added ability to change locale at runtime and from gui; Update kr_KR translation; Removed support for tnbtu.com; No longer writes url to url_history file is save urls only is checked", "1.7.49: Fixed -n flag; Added ability to change locale at runtime and from gui; Update kr_KR translation; Removed support for tnbtu.com; No longer writes url to url_history file is save urls only is checked",
@ -223,5 +231,6 @@
"1.0.3: Added VK.com ripper", "1.0.3: Added VK.com ripper",
"1.0.1: Added auto-update functionality" "1.0.1: Added auto-update functionality"
], ],
"latestVersion": "1.7.51" "latestVersion": "1.7.60",
"currentHash": "f206e478822134328763fc41676f438ee5cc795f31984613619952dab8402301"
} }

3
src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java
View File

@ -97,7 +97,7 @@ public abstract class AbstractHTMLRipper extends AlbumRipper {
while (doc != null) { while (doc != null) {
if (alreadyDownloadedUrls >= Utils.getConfigInteger("history.end_rip_after_already_seen", 1000000000) && !isThisATest()) { if (alreadyDownloadedUrls >= Utils.getConfigInteger("history.end_rip_after_already_seen", 1000000000) && !isThisATest()) {
sendUpdate(STATUS.DOWNLOAD_COMPLETE, "Already seen the last " + alreadyDownloadedUrls + " images ending rip"); sendUpdate(STATUS.DOWNLOAD_COMPLETE_HISTORY, "Already seen the last " + alreadyDownloadedUrls + " images ending rip");
break; break;
} }
List<String> imageURLs = getURLsFromPage(doc); List<String> imageURLs = getURLsFromPage(doc);
@ -224,7 +224,6 @@ public abstract class AbstractHTMLRipper extends AlbumRipper {
if (!subdirectory.equals("")) { // Not sure about this part if (!subdirectory.equals("")) { // Not sure about this part
subdirectory = File.separator + subdirectory; subdirectory = File.separator + subdirectory;
} }
// TODO Get prefix working again, probably requires reworking a lot of stuff! (Might be fixed now)
saveFileAs = new File( saveFileAs = new File(
workingDir.getCanonicalPath() workingDir.getCanonicalPath()
+ subdirectory + subdirectory

2
src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java
View File

@ -69,7 +69,7 @@ public abstract class AbstractJSONRipper extends AlbumRipper {
} }
} }
if (imageURLs.isEmpty()) { if (imageURLs.isEmpty() && !hasASAPRipping()) {
throw new IOException("No images found at " + this.url); throw new IOException("No images found at " + this.url);
} }

5
src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
View File

@ -613,4 +613,9 @@ public abstract class AbstractRipper
protected boolean isThisATest() { protected boolean isThisATest() {
return thisIsATest; return thisIsATest;
} }
// If true ripme uses a byte progress bar
protected boolean useByteProgessBar() { return false;}
// If true ripme will try to resume a broken download for this ripper
protected boolean tryResumeDownload() { return false;}
} }

43
src/main/java/com/rarchives/ripme/ripper/AbstractSingleFileRipper.java Normal file
View File

@ -0,0 +1,43 @@
package com.rarchives.ripme.ripper;
import com.rarchives.ripme.utils.Utils;
import java.io.IOException;
import java.net.URL;
/**
* This is just an extension of AbstractHTMLRipper that auto overrides a few things
* to help cut down on copy pasted code
*/
public abstract class AbstractSingleFileRipper extends AbstractHTMLRipper {
private int bytesTotal = 1;
private int bytesCompleted = 1;
protected AbstractSingleFileRipper(URL url) throws IOException {
super(url);
}
@Override
public String getStatusText() {
return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal);
}
@Override
public int getCompletionPercentage() {
return (int) (100 * (bytesCompleted / (float) bytesTotal));
}
@Override
public void setBytesTotal(int bytes) {
this.bytesTotal = bytes;
}
@Override
public void setBytesCompleted(int bytes) {
this.bytesCompleted = bytes;
}
@Override
public boolean useByteProgessBar() {return true;}
}

3
src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java
View File

@ -71,10 +71,7 @@ public abstract class AlbumRipper extends AbstractRipper {
try (FileWriter fw = new FileWriter(urlFile, true)) { try (FileWriter fw = new FileWriter(urlFile, true)) {
fw.write(url.toExternalForm()); fw.write(url.toExternalForm());
fw.write("\n"); fw.write("\n");
RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, urlFile);
itemsCompleted.put(url, new File(urlFile)); itemsCompleted.put(url, new File(urlFile));
observer.update(this, msg);
} catch (IOException e) { } catch (IOException e) {
LOGGER.error("Error while writing to " + urlFile, e); LOGGER.error("Error while writing to " + urlFile, e);
} }

74
src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
View File

@ -6,9 +6,11 @@ import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.OutputStream; import java.io.OutputStream;
import java.lang.reflect.Array;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.net.URLConnection; import java.net.URLConnection;
import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
@ -59,18 +61,26 @@ class DownloadFileThread extends Thread {
this.cookies = cookies; this.cookies = cookies;
} }
/** /**
* Attempts to download the file. Retries as needed. * Attempts to download the file. Retries as needed.
* Notifies observers upon completion/error/warn. * Notifies observers upon completion/error/warn.
*/ */
public void run() { public void run() {
long fileSize = 0;
int bytesTotal = 0;
int bytesDownloaded = 0;
if (saveAs.exists() && observer.tryResumeDownload()) {
fileSize = saveAs.length();
}
try { try {
observer.stopCheck(); observer.stopCheck();
} catch (IOException e) { } catch (IOException e) {
observer.downloadErrored(url, "Download interrupted"); observer.downloadErrored(url, "Download interrupted");
return; return;
} }
if (saveAs.exists()) { if (saveAs.exists() && !observer.tryResumeDownload() && !getFileExtFromMIME ||
Utils.fuzzyExists(new File(saveAs.getParent()), saveAs.getName()) && getFileExtFromMIME && !observer.tryResumeDownload()) {
if (Utils.getConfigBoolean("file.overwrite", false)) { if (Utils.getConfigBoolean("file.overwrite", false)) {
logger.info("[!] Deleting existing file" + prettySaveAs); logger.info("[!] Deleting existing file" + prettySaveAs);
saveAs.delete(); saveAs.delete();
@ -80,7 +90,6 @@ class DownloadFileThread extends Thread {
return; return;
} }
} }
URL urlToDownload = this.url; URL urlToDownload = this.url;
boolean redirected = false; boolean redirected = false;
int tries = 0; // Number of attempts to download int tries = 0; // Number of attempts to download
@ -114,11 +123,20 @@ class DownloadFileThread extends Thread {
cookie += key + "=" + cookies.get(key); cookie += key + "=" + cookies.get(key);
} }
huc.setRequestProperty("Cookie", cookie); huc.setRequestProperty("Cookie", cookie);
if (observer.tryResumeDownload()) {
if (fileSize != 0) {
huc.setRequestProperty("Range", "bytes=" + fileSize + "-");
}
}
logger.debug("Request properties: " + huc.getRequestProperties()); logger.debug("Request properties: " + huc.getRequestProperties());
huc.connect(); huc.connect();
int statusCode = huc.getResponseCode(); int statusCode = huc.getResponseCode();
logger.debug("Status code: " + statusCode); logger.debug("Status code: " + statusCode);
if (statusCode != 206 && observer.tryResumeDownload() && saveAs.exists()) {
// TODO find a better way to handle servers that don't support resuming downloads then just erroring out
throw new IOException("Server doesn't support resuming downloads");
}
if (statusCode / 100 == 3) { // 3xx Redirect if (statusCode / 100 == 3) { // 3xx Redirect
if (!redirected) { if (!redirected) {
// Don't increment retries on the first redirect // Don't increment retries on the first redirect
@ -146,17 +164,63 @@ class DownloadFileThread extends Thread {
observer.downloadErrored(url, "Imgur image is 404: " + url.toExternalForm()); observer.downloadErrored(url, "Imgur image is 404: " + url.toExternalForm());
return; return;
} }
// If the ripper is using the bytes progress bar set bytesTotal to huc.getContentLength()
if (observer.useByteProgessBar()) {
bytesTotal = huc.getContentLength();
observer.setBytesTotal(bytesTotal);
observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal);
logger.debug("Size of file at " + this.url + " = " + bytesTotal + "b");
}
// Save file // Save file
bis = new BufferedInputStream(huc.getInputStream()); bis = new BufferedInputStream(huc.getInputStream());
// Check if we should get the file ext from the MIME type // Check if we should get the file ext from the MIME type
if (getFileExtFromMIME) { if (getFileExtFromMIME) {
String fileExt = URLConnection.guessContentTypeFromStream(bis).replaceAll("image/", ""); String fileExt = URLConnection.guessContentTypeFromStream(bis);
if (fileExt != null) {
fileExt = fileExt.replaceAll("image/", "");
saveAs = new File(saveAs.toString() + "." + fileExt); saveAs = new File(saveAs.toString() + "." + fileExt);
} else {
logger.error("Was unable to get content type from stream");
// Try to get the file type from the magic number
byte[] magicBytes = new byte[8];
bis.read(magicBytes,0, 5);
bis.reset();
fileExt = Utils.getEXTFromMagic(magicBytes);
if (fileExt != null) {
saveAs = new File(saveAs.toString() + "." + fileExt);
} else {
logger.error("Was unable to get content type using magic number");
logger.error("Magic number was: " + Arrays.toString(magicBytes));
} }
}
}
// If we're resuming a download we append data to the existing file
if (statusCode == 206) {
fos = new FileOutputStream(saveAs, true);
} else {
fos = new FileOutputStream(saveAs); fos = new FileOutputStream(saveAs);
IOUtils.copy(bis, fos); }
byte[] data = new byte[1024 * 256];
int bytesRead;
while ( (bytesRead = bis.read(data)) != -1) {
try {
observer.stopCheck();
} catch (IOException e) {
observer.downloadErrored(url, "Download interrupted");
return;
}
fos.write(data, 0, bytesRead);
if (observer.useByteProgessBar()) {
bytesDownloaded += bytesRead;
observer.setBytesCompleted(bytesDownloaded);
observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded);
}
}
bis.close();
fos.close();
break; // Download successful: break out of infinite loop break; // Download successful: break out of infinite loop
} catch (HttpStatusException hse) { } catch (HttpStatusException hse) {
logger.debug("HTTP status exception", hse); logger.debug("HTTP status exception", hse);

3
src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java
View File

@ -76,7 +76,8 @@ class DownloadVideoThread extends Thread {
int tries = 0; // Number of attempts to download int tries = 0; // Number of attempts to download
do { do {
InputStream bis = null; OutputStream fos = null; InputStream bis = null; OutputStream fos = null;
byte[] data = new byte[1024 * 256]; int bytesRead; byte[] data = new byte[1024 * 256];
int bytesRead;
try { try {
logger.info(" Downloading file: " + url + (tries > 0 ? " Retry #" + tries : "")); logger.info(" Downloading file: " + url + (tries > 0 ? " Retry #" + tries : ""));
observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm()); observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm());

270
src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java Normal file
View File

@ -0,0 +1,270 @@
package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.rarchives.ripme.ripper.AbstractJSONRipper;
import com.rarchives.ripme.utils.Http;
import org.json.JSONObject;
public class ArtStationRipper extends AbstractJSONRipper {
enum URL_TYPE {
SINGLE_PROJECT, USER_PORTFOLIO, UNKNOWN
}
private ParsedURL albumURL;
private String projectName;
private Integer projectIndex;
private Integer projectPageNumber;
public ArtStationRipper(URL url) throws IOException {
super(url);
}
@Override
protected String getDomain() {
return "artstation.com";
}
@Override
public String getHost() {
return "ArtStation";
}
@Override
public String getGID(URL url) throws MalformedURLException {
JSONObject groupData;
// Parse URL and store for later use
albumURL = parseURL(url);
if (albumURL.getType() == URL_TYPE.SINGLE_PROJECT) {
// URL points to single project, use project title as GID
try {
groupData = Http.url(albumURL.getLocation()).getJSON();
} catch (IOException e) {
throw new MalformedURLException("Couldn't load JSON from " + albumURL.getLocation());
}
return groupData.getString("title");
}
if (albumURL.getType() == URL_TYPE.USER_PORTFOLIO) {
// URL points to user portfolio, use user's full name as GID
String userInfoURL = "https://www.artstation.com/users/" + albumURL.getID() + "/quick.json";
try {
groupData = Http.url(userInfoURL).getJSON();
} catch (IOException e) {
throw new MalformedURLException("Couldn't load JSON from " + userInfoURL);
}
return groupData.getString("full_name");
}
// No JSON found in the URL entered, can't rip
throw new MalformedURLException(
"Expected URL to an ArtStation project or user profile - got " + url + " instead");
}
@Override
protected JSONObject getFirstPage() throws IOException {
if (albumURL.getType() == URL_TYPE.SINGLE_PROJECT) {
// URL points to JSON of a single project, just return it
return Http.url(albumURL.getLocation()).getJSON();
}
if (albumURL.getType() == URL_TYPE.USER_PORTFOLIO) {
// URL points to JSON of a list of projects, load it to parse individual
// projects
JSONObject albumContent = Http.url(albumURL.getLocation()).getJSON();
if (albumContent.getInt("total_count") > 0) {
// Get JSON of the first project and return it
JSONObject projectInfo = albumContent.getJSONArray("data").getJSONObject(0);
ParsedURL projectURL = parseURL(new URL(projectInfo.getString("permalink")));
return Http.url(projectURL.getLocation()).getJSON();
}
}
throw new IOException("URL specified points to an user with empty portfolio");
}
@Override
protected JSONObject getNextPage(JSONObject doc) throws IOException {
if (albumURL.getType() == URL_TYPE.USER_PORTFOLIO) {
// Initialize the page number if it hasn't been initialized already
if (projectPageNumber == null) {
projectPageNumber = 1;
}
// Each page holds a maximum of 50 projects. Initialize the index if it hasn't
// been initialized already or increment page number and reset the index if all
// projects of the current page were already processed
if (projectIndex == null) {
projectIndex = 0;
} else if (projectIndex > 49) {
projectPageNumber++;
projectIndex = 0;
}
Integer currentProject = ((projectPageNumber - 1) * 50) + (projectIndex + 1);
JSONObject albumContent = Http.url(albumURL.getLocation() + "?page=" + projectPageNumber).getJSON();
if (albumContent.getInt("total_count") > currentProject) {
// Get JSON of the next project and return it
JSONObject projectInfo = albumContent.getJSONArray("data").getJSONObject(projectIndex);
ParsedURL projectURL = parseURL(new URL(projectInfo.getString("permalink")));
projectIndex++;
return Http.url(projectURL.getLocation()).getJSON();
}
throw new IOException("No more projects");
}
throw new IOException("Downloading a single project");
}
@Override
protected List<String> getURLsFromJSON(JSONObject json) {
List<String> assetURLs = new ArrayList<>();
JSONObject currentObject;
// Update project name variable from JSON data. Used by downloadURL() to create
// subfolders when input URL is URL_TYPE.USER_PORTFOLIO
projectName = json.getString("title");
for (int i = 0; i < json.getJSONArray("assets").length(); i++) {
currentObject = json.getJSONArray("assets").getJSONObject(i);
if (!currentObject.getString("image_url").isEmpty()) {
// TODO: Find a way to rip external content.
// ArtStation hosts only image content, everything else (videos, 3D Models, etc)
// is hosted in other websites and displayed through embedded HTML5 players
assetURLs.add(currentObject.getString("image_url"));
}
}
return assetURLs;
}
@Override
protected void downloadURL(URL url, int index) {
if (albumURL.getType() == URL_TYPE.USER_PORTFOLIO) {
// Replace not allowed characters with underlines
String folderName = projectName.replaceAll("[\\\\/:*?\"<>|]", "_");
// Folder name also can't end with dots or spaces, strip them
folderName = folderName.replaceAll("\\s+$", "");
folderName = folderName.replaceAll("\\.+$", "");
// Downloading multiple projects, separate each one in subfolders
addURLToDownload(url, "", folderName);
} else {
addURLToDownload(url);
}
}
@Override
public String normalizeUrl(String url) {
// Strip URL parameters
return url.replaceAll("\\?\\w+$", "");
}
private static class ParsedURL {
URL_TYPE urlType;
String jsonURL, urlID;
/**
* Construct a new ParsedURL object.
*
* @param urlType URL_TYPE enum containing the URL type
* @param jsonURL String containing the JSON URL location
* @param urlID String containing the ID of this URL
*
*/
ParsedURL(URL_TYPE urlType, String jsonURL, String urlID) {
this.urlType = urlType;
this.jsonURL = jsonURL;
this.urlID = urlID;
}
/**
* Get URL Type of this ParsedURL object.
*
* @return URL_TYPE enum containing this object type
*
*/
URL_TYPE getType() {
return this.urlType;
}
/**
* Get JSON location of this ParsedURL object.
*
* @return String containing the JSON URL
*
*/
String getLocation() {
return this.jsonURL;
}
/**
* Get ID of this ParsedURL object.
*
* @return For URL_TYPE.SINGLE_PROJECT, returns the project hash. For
* URL_TYPE.USER_PORTFOLIO, returns the account name
*/
String getID() {
return this.urlID;
}
}
/**
* Parses an ArtStation URL.
*
* @param url URL to an ArtStation user profile
* (https://www.artstation.com/username) or single project
* (https://www.artstation.com/artwork/projectid)
* @return ParsedURL object containing URL type, JSON location and ID (stores
* account name or project hash, depending of the URL type identified)
*
*/
private ParsedURL parseURL(URL url) {
String htmlSource;
ParsedURL parsedURL;
// Load HTML Source of the specified URL
try {
htmlSource = Http.url(url).get().html();
} catch (IOException e) {
htmlSource = "";
}
// Check if HTML Source of the specified URL references a project
Pattern p = Pattern.compile("'/projects/(\\w+)\\.json'");
Matcher m = p.matcher(htmlSource);
if (m.find()) {
parsedURL = new ParsedURL(URL_TYPE.SINGLE_PROJECT,
"https://www.artstation.com/projects/" + m.group(1) + ".json", m.group(1));
return parsedURL;
}
// Check if HTML Source of the specified URL references a user profile
p = Pattern.compile("'/users/([\\w-]+)/quick\\.json'");
m = p.matcher(htmlSource);
if (m.find()) {
parsedURL = new ParsedURL(URL_TYPE.USER_PORTFOLIO,
"https://www.artstation.com/users/" + m.group(1) + "/projects.json", m.group(1));
return parsedURL;
}
// HTML Source of the specified URL doesn't reference a user profile or project
parsedURL = new ParsedURL(URL_TYPE.UNKNOWN, null, null);
return parsedURL;
}
}

20
src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
View File

@ -19,7 +19,7 @@ import com.rarchives.ripme.utils.RipUtils;
public class ChanRipper extends AbstractHTMLRipper { public class ChanRipper extends AbstractHTMLRipper {
private static List<ChanSite> explicit_domains = Arrays.asList( private static List<ChanSite> explicit_domains = Arrays.asList(
new ChanSite(Arrays.asList("boards.4chan.org"), Arrays.asList("4cdn.org", "is.4chan.org", "is2.4chan.org")), new ChanSite(Arrays.asList("boards.4chan.org"), Arrays.asList("4cdn.org", "is.4chan.org", "is2.4chan.org", "is3.4chan.org")),
new ChanSite(Arrays.asList("4archive.org"), Arrays.asList("imgur.com")), new ChanSite(Arrays.asList("4archive.org"), Arrays.asList("imgur.com")),
new ChanSite(Arrays.asList("archive.4plebs.org"), Arrays.asList("img.4plebs.org")) new ChanSite(Arrays.asList("archive.4plebs.org"), Arrays.asList("img.4plebs.org"))
); );
@ -85,8 +85,22 @@ public class ChanRipper extends AbstractHTMLRipper {
return true; return true;
} }
} }
return url.toExternalForm().contains("/res/") // Most chans if (url.toExternalForm().contains("desuchan.net") && url.toExternalForm().contains("/res/")) {
|| url.toExternalForm().contains("/thread/"); // 4chan, archive.moe return true;
}
if (url.toExternalForm().contains("boards.420chan.org") && url.toExternalForm().contains("/res/")) {
return true;
}
if (url.toExternalForm().contains("7chan.org") && url.toExternalForm().contains("/res/")) {
return true;
}
if (url.toExternalForm().contains("xchan.pw") && url.toExternalForm().contains("/board/")) {
return true;
}
if (url.toExternalForm().contains("desuarchive.org")) {
return true;
}
return false;
} }
/** /**

10
src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java
View File

@ -28,7 +28,7 @@ public class CheveretoRipper extends AbstractHTMLRipper {
super(url); super(url);
} }
private static List<String> explicit_domains_1 = Arrays.asList("tag-fox.com"); private static List<String> explicit_domains = Arrays.asList("tag-fox.com", "kenzato.uk");
@Override @Override
public String getHost() { public String getHost() {
@ -43,13 +43,9 @@ public class CheveretoRipper extends AbstractHTMLRipper {
@Override @Override
public boolean canRip(URL url) { public boolean canRip(URL url) {
String url_name = url.toExternalForm(); String url_name = url.toExternalForm();
if (explicit_domains_1.contains(url_name.split("/")[2])) { if (explicit_domains.contains(url_name.split("/")[2])) {
Pattern pa = Pattern.compile("(?:https?://)?(?:www\\.)?[a-z1-9-]*\\.[a-z1-9]*/album/([a-zA-Z1-9]*)/?$");
Matcher ma = pa.matcher(url.toExternalForm());
if (ma.matches()) {
return true; return true;
} }
}
return false; return false;
} }
@ -70,7 +66,7 @@ public class CheveretoRipper extends AbstractHTMLRipper {
@Override @Override
public String getGID(URL url) throws MalformedURLException { public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("(?:https?://)?(?:www\\.)?[a-z1-9-]*\\.[a-z1-9]*/album/([a-zA-Z1-9]*)/?$"); Pattern p = Pattern.compile("(?:https?://)?(?:www\\.)?[a-z1-9-]*\\.[a-z1-9]*(?:[a-zA-Z1-9]*)/album/([a-zA-Z1-9]*)/?$");
Matcher m = p.matcher(url.toExternalForm()); Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) { if (m.matches()) {
return m.group(1); return m.group(1);

401
src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
View File

@ -1,8 +1,9 @@
package com.rarchives.ripme.ripper.rippers; package com.rarchives.ripme.ripper.rippers;
import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ripper.AbstractJSONRipper;
import com.rarchives.ripme.utils.Base64; import com.rarchives.ripme.utils.Base64;
import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.RipUtils;
import com.rarchives.ripme.utils.Utils; import com.rarchives.ripme.utils.Utils;
import java.io.IOException; import java.io.IOException;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
@ -17,15 +18,23 @@ import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.jsoup.Connection.Method;
import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.Connection.Response; import org.jsoup.Connection.Response;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.jsoup.safety.Whitelist;
import org.jsoup.select.Elements; import org.jsoup.select.Elements;
public class DeviantartRipper extends AbstractHTMLRipper {
public class DeviantartRipper extends AbstractJSONRipper {
String requestID;
String galleryID;
String username;
String baseApiUrl = "https://www.deviantart.com/dapi/v1/gallery/";
String csrf;
Map<String, String> pageCookies = new HashMap<>();
private static final int PAGE_SLEEP_TIME = 3000, private static final int PAGE_SLEEP_TIME = 3000,
IMAGE_SLEEP_TIME = 2000; IMAGE_SLEEP_TIME = 2000;
@ -37,31 +46,37 @@ public class DeviantartRipper extends AbstractHTMLRipper {
super(url); super(url);
} }
String loginCookies = "auth=__0f9158aaec09f417b235%3B%221ff79836392a515d154216d919eae573%22;" +
"auth_secure=__41d14dd0da101f411bb0%3B%2281cf2cf9477776162a1172543aae85ce%22;" +
"userinfo=__bf84ac233bfa8ae642e8%3B%7B%22username%22%3A%22grabpy%22%2C%22uniqueid%22%3A%22a0a876aa37dbd4b30e1c80406ee9c280%22%2C%22vd%22%3A%22BbHUXZ%2CBbHUXZ%2CA%2CU%2CA%2C%2CB%2CA%2CB%2CBbHUXZ%2CBbHUdj%2CL%2CL%2CA%2CBbHUdj%2C13%2CA%2CB%2CA%2C%2CA%2CA%2CB%2CA%2CA%2C%2CA%22%2C%22attr%22%3A56%7D";
@Override @Override
public String getHost() { public String getHost() {
return "deviantart"; return "deviantart";
} }
@Override @Override
public String getDomain() { public String getDomain() {
return "deviantart.com"; return "deviantart.com";
} }
@Override
public boolean hasDescriptionSupport() {
return true;
}
@Override @Override
public URL sanitizeURL(URL url) throws MalformedURLException { public URL sanitizeURL(URL url) throws MalformedURLException {
String u = url.toExternalForm(); String u = url.toExternalForm();
if (u.contains("/gallery/")) {
return url;
}
if (u.replace("/", "").endsWith(".deviantart.com")) { if (!u.endsWith("/gallery/") && !u.endsWith("/gallery")) {
// Root user page, get all albums
if (!u.endsWith("/")) { if (!u.endsWith("/")) {
u += "/"; u += "/gallery/";
} else {
u += "gallery/";
} }
u += "gallery/?";
} }
Pattern p = Pattern.compile("^https?://([a-zA-Z0-9\\-]+)\\.deviantart\\.com/favou?rites/([0-9]+)/*?$");
Pattern p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/favou?rites/([0-9]+)/*?$");
Matcher m = p.matcher(url.toExternalForm()); Matcher m = p.matcher(url.toExternalForm());
if (!m.matches()) { if (!m.matches()) {
String subdir = "/"; String subdir = "/";
@ -75,7 +90,7 @@ public class DeviantartRipper extends AbstractHTMLRipper {
@Override @Override
public String getGID(URL url) throws MalformedURLException { public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^https?://([a-zA-Z0-9\\-]+)\\.deviantart\\.com(/gallery)?/?(\\?.*)?$"); Pattern p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)(/gallery)?/?(\\?.*)?$");
Matcher m = p.matcher(url.toExternalForm()); Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) { if (m.matches()) {
// Root gallery // Root gallery
@ -86,24 +101,53 @@ public class DeviantartRipper extends AbstractHTMLRipper {
return m.group(1); return m.group(1);
} }
} }
p = Pattern.compile("^https?://([a-zA-Z0-9\\-]+)\\.deviantart\\.com/gallery/([0-9]+).*$"); p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/gallery/([0-9]+).*$");
m = p.matcher(url.toExternalForm()); m = p.matcher(url.toExternalForm());
if (m.matches()) { if (m.matches()) {
// Subgallery // Subgallery
return m.group(1) + "_" + m.group(2); return m.group(1) + "_" + m.group(2);
} }
p = Pattern.compile("^https?://([a-zA-Z0-9\\-]+)\\.deviantart\\.com/favou?rites/([0-9]+)/.*?$"); p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/favou?rites/([0-9]+)/.*?$");
m = p.matcher(url.toExternalForm()); m = p.matcher(url.toExternalForm());
if (m.matches()) { if (m.matches()) {
return m.group(1) + "_faves_" + m.group(2); return m.group(1) + "_faves_" + m.group(2);
} }
p = Pattern.compile("^https?://([a-zA-Z0-9\\-]+)\\.deviantart\\.com/favou?rites/?$"); p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/favou?rites/?$");
m = p.matcher(url.toExternalForm()); m = p.matcher(url.toExternalForm());
if (m.matches()) { if (m.matches()) {
// Subgallery // Subgallery
return m.group(1) + "_faves"; return m.group(1) + "_faves";
} }
throw new MalformedURLException("Expected URL format: http://username.deviantart.com/[/gallery/#####], got: " + url); throw new MalformedURLException("Expected URL format: http://www.deviantart.com/username[/gallery/#####], got: " + url);
}
private String getUsernameFromURL(String u) {
Pattern p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/gallery/?(\\S+)?");
Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) {
return m.group(1);
}
return null;
}
private String getFullsizedNSFWImage(String pageURL) {
try {
Document doc = Http.url(pageURL).cookies(cookies).get();
String imageToReturn = "";
String[] d = doc.select("img").attr("srcset").split(",");
String s = d[d.length -1].split(" ")[0];
LOGGER.info("2:" + s);
if (s == null || s.equals("")) {
LOGGER.error("Could not find full sized image at " + pageURL);
}
return s;
} catch (IOException e) {
LOGGER.error("Could not find full sized image at " + pageURL);
return null;
}
} }
/** /**
@ -115,151 +159,133 @@ public class DeviantartRipper extends AbstractHTMLRipper {
* @throws IOException * @throws IOException
*/ */
@Override @Override
public Document getFirstPage() throws IOException { public JSONObject getFirstPage() throws IOException {
//Test to see if there is a login: // Base64 da login
String username = Utils.getConfigString("deviantart.username", new String(Base64.decode("Z3JhYnB5"))); // username: Z3JhYnB5
String password = Utils.getConfigString("deviantart.password", new String(Base64.decode("ZmFrZXJz"))); // password: ZmFrZXJz
if (username == null || password == null) {
LOGGER.debug("No DeviantArt login provided."); cookies = getDACookies();
cookies.put("agegate_state","1"); // Bypasses the age gate if (cookies.isEmpty()) {
} else { LOGGER.warn("Failed to get login cookies");
// Attempt Login
try {
cookies = loginToDeviantart();
} catch (IOException e) {
LOGGER.warn("Failed to login: ", e);
cookies.put("agegate_state","1"); // Bypasses the age gate cookies.put("agegate_state","1"); // Bypasses the age gate
} }
} cookies.put("agegate_state", "1");
Response res = Http.url(this.url)
return Http.url(this.url)
.cookies(cookies) .cookies(cookies)
.get(); .response();
Document page = res.parse();
JSONObject firstPageJSON = getFirstPageJSON(page);
requestID = firstPageJSON.getJSONObject("dapx").getString("requestid");
galleryID = getGalleryID(page);
username = getUsernameFromURL(url.toExternalForm());
csrf = firstPageJSON.getString("csrf");
pageCookies = res.cookies();
return requestPage(0, galleryID, username, requestID, csrf, pageCookies);
} }
/** private JSONObject requestPage(int offset, String galleryID, String username, String requestID, String csfr, Map<String, String> c) {
* LOGGER.debug("offset: " + Integer.toString(offset));
* @param page LOGGER.debug("galleryID: " + galleryID);
* @param id LOGGER.debug("username: " + username);
* @return LOGGER.debug("requestID: " + requestID);
*/ String url = baseApiUrl + galleryID + "?iid=" + requestID;
private String jsonToImage(Document page, String id) {
Elements js = page.select("script[type=\"text/javascript\"]");
for (Element tag : js) {
if (tag.html().contains("window.__pageload")) {
try { try {
String script = tag.html(); Document doc = Http.url(url).cookies(c).data("username", username).data("offset", Integer.toString(offset))
script = script.substring(script.indexOf("window.__pageload")); .data("limit", "24").data("_csrf", csfr).data("id", requestID)
if (!script.contains(id)) { .ignoreContentType().post();
continue; return new JSONObject(doc.body().text());
} catch (IOException e) {
LOGGER.error("Got error trying to get page: " + e.getMessage());
e.printStackTrace();
return null;
} }
script = script.substring(script.indexOf(id));
// first },"src":"url" after id
script = script.substring(script.indexOf("},\"src\":\"") + 9, script.indexOf("\",\"type\""));
return script.replace("\\/", "/");
} catch (StringIndexOutOfBoundsException e) {
LOGGER.debug("Unable to get json link from " + page.location());
} }
private JSONObject getFirstPageJSON(Document doc) {
for (Element js : doc.select("script")) {
if (js.html().contains("requestid")) {
String json = js.html().replaceAll("window.__initial_body_data=", "").replaceAll("\\);", "")
.replaceAll(";__wake\\(.+", "");
LOGGER.info("json: " + json);
JSONObject j = new JSONObject(json);
return j;
} }
} }
return null; return null;
} }
public String getGalleryID(Document doc) {
// If the url contains catpath we return 0 as the DA api will provide all galery images if you sent the
// gallery id to 0
if (url.toExternalForm().contains("catpath=")) {
return "0";
}
Pattern p = Pattern.compile("^https?://www\\.deviantart\\.com/[a-zA-Z0-9\\-]+/gallery/([0-9]+)/?\\S+");
Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) {
return m.group(1);
}
for (Element el : doc.select("input[name=set]")) {
try {
String galleryID = el.attr("value");
return galleryID;
} catch (NullPointerException e) {
continue;
}
}
LOGGER.error("Could not find gallery ID");
return null;
}
public String getUsername(Document doc) {
return doc.select("meta[property=og:title]").attr("content")
.replaceAll("'s DeviantArt gallery", "").replaceAll("'s DeviantArt Gallery", "");
}
@Override @Override
public List<String> getURLsFromPage(Document page) { public List<String> getURLsFromJSON(JSONObject json) {
List<String> imageURLs = new ArrayList<>(); List<String> imageURLs = new ArrayList<>();
LOGGER.info(json);
JSONArray results = json.getJSONObject("content").getJSONArray("results");
for (int i = 0; i < results.length(); i++) {
Document doc = Jsoup.parseBodyFragment(results.getJSONObject(i).getString("html"));
if (doc.html().contains("ismature")) {
LOGGER.info("Downloading nsfw image");
String nsfwImage = getFullsizedNSFWImage(doc.select("span").attr("href"));
if (nsfwImage != null && nsfwImage.startsWith("http")) {
imageURLs.add(nsfwImage);
}
}
try {
String imageURL = doc.select("span").first().attr("data-super-full-img");
if (!imageURL.isEmpty() && imageURL.startsWith("http")) {
imageURLs.add(imageURL);
}
} catch (NullPointerException e) {
LOGGER.info(i + " does not contain any images");
}
// Iterate over all thumbnails
for (Element thumb : page.select("div.zones-container span.thumb")) {
if (isStopped()) {
break;
}
Element img = thumb.select("img").get(0);
if (img.attr("transparent").equals("false")) {
continue; // a.thumbs to other albums are invisible
}
// Get full-sized image via helper methods
String fullSize = null;
if (thumb.attr("data-super-full-img").contains("//orig")) {
fullSize = thumb.attr("data-super-full-img");
} else {
String spanUrl = thumb.attr("href");
String fullSize1 = jsonToImage(page,spanUrl.substring(spanUrl.lastIndexOf('-') + 1));
if (fullSize1 == null || !fullSize1.contains("//orig")) {
fullSize = smallToFull(img.attr("src"), spanUrl);
}
if (fullSize == null && fullSize1 != null) {
fullSize = fullSize1;
}
}
if (fullSize == null) {
if (thumb.attr("data-super-full-img") != null) {
fullSize = thumb.attr("data-super-full-img");
} else if (thumb.attr("data-super-img") != null) {
fullSize = thumb.attr("data-super-img");
} else {
continue;
}
}
if (triedURLs.contains(fullSize)) {
LOGGER.warn("Already tried to download " + fullSize);
continue;
}
triedURLs.add(fullSize);
imageURLs.add(fullSize);
if (isThisATest()) {
// Only need one image for a test
break;
}
} }
return imageURLs; return imageURLs;
} }
@Override
public List<String> getDescriptionsFromPage(Document page) {
List<String> textURLs = new ArrayList<>();
// Iterate over all thumbnails
for (Element thumb : page.select("div.zones-container span.thumb")) {
LOGGER.info(thumb.attr("href"));
if (isStopped()) {
break;
}
Element img = thumb.select("img").get(0);
if (img.attr("transparent").equals("false")) {
continue; // a.thumbs to other albums are invisible
}
textURLs.add(thumb.attr("href"));
}
return textURLs;
}
@Override @Override
public Document getNextPage(Document page) throws IOException { public JSONObject getNextPage(JSONObject page) throws IOException {
if (isThisATest()) { boolean hasMore = page.getJSONObject("content").getBoolean("has_more");
return null; if (hasMore) {
return requestPage(page.getJSONObject("content").getInt("next_offset"), galleryID, username, requestID, csrf, pageCookies);
} }
Elements nextButtons = page.select("link[rel=\"next\"]");
if (nextButtons.isEmpty()) { throw new IOException("No more pages");
if (page.select("link[rel=\"prev\"]").isEmpty()) {
throw new IOException("No next page found");
} else {
throw new IOException("Hit end of pages");
}
}
Element a = nextButtons.first();
String nextPage = a.attr("href");
if (nextPage.startsWith("/")) {
nextPage = "http://" + this.url.getHost() + nextPage;
}
if (!sleep(PAGE_SLEEP_TIME)) {
throw new IOException("Interrupted while waiting to load next page: " + nextPage);
}
LOGGER.info("Found next page: " + nextPage);
return Http.url(nextPage)
.cookies(cookies)
.get();
} }
@Override @Override
@ -300,61 +326,7 @@ public class DeviantartRipper extends AbstractHTMLRipper {
return result.toString(); return result.toString();
} }
/**
* Attempts to download description for image.
* Comes in handy when people put entire stories in their description.
* If no description was found, returns null.
* @param url The URL the description will be retrieved from
* @param page The gallery page the URL was found on
* @return A String[] with first object being the description, and the second object being image file name if found.
*/
@Override
public String[] getDescription(String url,Document page) {
if (isThisATest()) {
return null;
}
try {
// Fetch the image page
Response resp = Http.url(url)
.referrer(this.url)
.cookies(cookies)
.response();
cookies.putAll(resp.cookies());
// Try to find the description
Document documentz = resp.parse();
Element ele = documentz.select("div.dev-description").first();
if (ele == null) {
throw new IOException("No description found");
}
documentz.outputSettings(new Document.OutputSettings().prettyPrint(false));
ele.select("br").append("\\n");
ele.select("p").prepend("\\n\\n");
String fullSize = null;
Element thumb = page.select("div.zones-container span.thumb[href=\"" + url + "\"]").get(0);
if (!thumb.attr("data-super-full-img").isEmpty()) {
fullSize = thumb.attr("data-super-full-img");
String[] split = fullSize.split("/");
fullSize = split[split.length - 1];
} else {
String spanUrl = thumb.attr("href");
fullSize = jsonToImage(page,spanUrl.substring(spanUrl.lastIndexOf('-') + 1));
if (fullSize != null) {
String[] split = fullSize.split("/");
fullSize = split[split.length - 1];
}
}
if (fullSize == null) {
return new String[] {Jsoup.clean(ele.html().replaceAll("\\\\n", System.getProperty("line.separator")), "", Whitelist.none(), new Document.OutputSettings().prettyPrint(false))};
}
fullSize = fullSize.substring(0, fullSize.lastIndexOf("."));
return new String[] {Jsoup.clean(ele.html().replaceAll("\\\\n", System.getProperty("line.separator")), "", Whitelist.none(), new Document.OutputSettings().prettyPrint(false)),fullSize};
// TODO Make this not make a newline if someone just types \n into the description.
} catch (IOException ioe) {
LOGGER.info("Failed to get description at " + url + ": '" + ioe.getMessage() + "'");
return null;
}
}
/** /**
* If largest resolution for image at 'thumb' is found, starts downloading * If largest resolution for image at 'thumb' is found, starts downloading
@ -426,47 +398,10 @@ public class DeviantartRipper extends AbstractHTMLRipper {
} }
/** /**
* Logs into deviant art. Required to rip full-size NSFW content. * Returns DA cookies.
* @return Map of cookies containing session data. * @return Map of cookies containing session data.
*/ */
private Map<String, String> loginToDeviantart() throws IOException { private Map<String, String> getDACookies() {
// Populate postData fields return RipUtils.getCookiesFromString(Utils.getConfigString("deviantart.cookies", loginCookies));
Map<String,String> postData = new HashMap<>();
String username = Utils.getConfigString("deviantart.username", new String(Base64.decode("Z3JhYnB5")));
String password = Utils.getConfigString("deviantart.password", new String(Base64.decode("ZmFrZXJz")));
if (username == null || password == null) {
throw new IOException("could not find username or password in config");
}
Response resp = Http.url("http://www.deviantart.com/")
.response();
for (Element input : resp.parse().select("form#form-login input[type=hidden]")) {
postData.put(input.attr("name"), input.attr("value"));
}
postData.put("username", username);
postData.put("password", password);
postData.put("remember_me", "1");
// Send login request
resp = Http.url("https://www.deviantart.com/users/login")
.userAgent(USER_AGENT)
.data(postData)
.cookies(resp.cookies())
.method(Method.POST)
.response();
// Assert we are logged in
if (resp.hasHeader("Location") && resp.header("Location").contains("password")) {
// Wrong password
throw new IOException("Wrong password");
}
if (resp.url().toExternalForm().contains("bad_form")) {
throw new IOException("Login form was incorrectly submitted");
}
if (resp.cookie("auth_secure") == null ||
resp.cookie("auth") == null) {
throw new IOException("No auth_secure or auth cookies received");
}
// We are logged in, save the cookies
return resp.cookies();
} }
} }

26
src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java
View File

@ -96,40 +96,40 @@ public class E621Ripper extends AbstractHTMLRipper{
private String getTerm(URL url) throws MalformedURLException{ private String getTerm(URL url) throws MalformedURLException{
if(gidPattern==null) if(gidPattern==null)
gidPattern=Pattern.compile("^https?://(www\\.)?e621\\.net/post/index/[^/]+/([a-zA-Z0-9$_.+!*'(),%-]+)(/.*)?(#.*)?$"); gidPattern=Pattern.compile("^https?://(www\\.)?e621\\.net/post/index/[^/]+/([a-zA-Z0-9$_.+!*'():,%\\-]+)(/.*)?(#.*)?$");
if(gidPatternPool==null) if(gidPatternPool==null)
gidPatternPool=Pattern.compile("^https?://(www\\.)?e621\\.net/pool/show/([a-zA-Z0-9$_.+!*'(),%-]+)(\\?.*)?(/.*)?(#.*)?$"); gidPatternPool=Pattern.compile("^https?://(www\\.)?e621\\.net/pool/show/([a-zA-Z0-9$_.+!*'(),%:\\-]+)(\\?.*)?(/.*)?(#.*)?$");
Matcher m = gidPattern.matcher(url.toExternalForm()); Matcher m = gidPattern.matcher(url.toExternalForm());
if(m.matches()) if(m.matches()) {
LOGGER.info(m.group(2));
return m.group(2); return m.group(2);
}
m = gidPatternPool.matcher(url.toExternalForm()); m = gidPatternPool.matcher(url.toExternalForm());
if(m.matches()) if(m.matches()) {
return m.group(2); return m.group(2);
}
throw new MalformedURLException("Expected e621.net URL format: e621.net/post/index/1/searchterm - got "+url+" instead"); throw new MalformedURLException("Expected e621.net URL format: e621.net/post/index/1/searchterm - got "+url+" instead");
} }
@Override @Override
public String getGID(URL url) throws MalformedURLException { public String getGID(URL url) throws MalformedURLException {
try {
String prefix="";
if(url.getPath().startsWith("/pool/show/"))
prefix="pool_";
return Utils.filesystemSafe(prefix+new URI(getTerm(url)).getPath()); String prefix="";
} catch (URISyntaxException ex) { if (url.getPath().startsWith("/pool/show/")) {
logger.error(ex); prefix = "pool_";
} }
throw new MalformedURLException("Expected e621.net URL format: e621.net/post/index/1/searchterm - got "+url+" instead"); return Utils.filesystemSafe(prefix+getTerm(url));
} }
@Override @Override
public URL sanitizeURL(URL url) throws MalformedURLException { public URL sanitizeURL(URL url) throws MalformedURLException {
if(gidPattern2==null) if(gidPattern2==null)
gidPattern2=Pattern.compile("^https?://(www\\.)?e621\\.net/post/search\\?tags=([a-zA-Z0-9$_.+!*'(),%-]+)(/.*)?(#.*)?$"); gidPattern2=Pattern.compile("^https?://(www\\.)?e621\\.net/post/search\\?tags=([a-zA-Z0-9$_.+!*'():,%-]+)(/.*)?(#.*)?$");
Matcher m = gidPattern2.matcher(url.toExternalForm()); Matcher m = gidPattern2.matcher(url.toExternalForm());
if(m.matches()) if(m.matches())

39
src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java
View File

@ -11,6 +11,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import com.rarchives.ripme.utils.Utils; import com.rarchives.ripme.utils.Utils;
import org.json.JSONObject;
import org.jsoup.Connection.Response; import org.jsoup.Connection.Response;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
@ -116,14 +117,13 @@ public class EightmusesRipper extends AbstractHTMLRipper {
image = thumb.attr("data-cfsrc"); image = thumb.attr("data-cfsrc");
} }
else { else {
String imageHref = thumb.attr("href"); // Deobfustace the json data
if (imageHref.equals("")) continue; String rawJson = deobfuscateJSON(page.select("script#ractive-public").html()
if (imageHref.startsWith("/")) { .replaceAll("&gt;", ">").replaceAll("&lt;", "<").replace("&amp;", "&"));
imageHref = "https://www.8muses.com" + imageHref; JSONObject json = new JSONObject(rawJson);
}
try { try {
LOGGER.info("Retrieving full-size image location from " + imageHref); for (int i = 0; i != json.getJSONArray("pictures").length(); i++) {
image = getFullSizeImage(imageHref); image = "https://www.8muses.com/image/fm/" + json.getJSONArray("pictures").getJSONObject(i).getString("publicUri");
URL imageUrl = new URL(image); URL imageUrl = new URL(image);
if (Utils.getConfigBoolean("8muses.use_short_names", false)) { if (Utils.getConfigBoolean("8muses.use_short_names", false)) {
addURLToDownload(imageUrl, getPrefixShort(x), getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies, "", null, true); addURLToDownload(imageUrl, getPrefixShort(x), getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies, "", null, true);
@ -132,9 +132,9 @@ public class EightmusesRipper extends AbstractHTMLRipper {
} }
// X is our page index // X is our page index
x++; x++;
}
} catch (IOException e) { } catch (IOException e) {
LOGGER.error("Failed to get full-size image from " + imageHref);
continue; continue;
} }
} }
@ -154,7 +154,7 @@ public class EightmusesRipper extends AbstractHTMLRipper {
sendUpdate(STATUS.LOADING_RESOURCE, imageUrl); sendUpdate(STATUS.LOADING_RESOURCE, imageUrl);
LOGGER.info("Getting full sized image from " + imageUrl); LOGGER.info("Getting full sized image from " + imageUrl);
Document doc = new Http(imageUrl).get(); // Retrieve the webpage of the image URL Document doc = new Http(imageUrl).get(); // Retrieve the webpage of the image URL
String imageName = doc.select("input[id=imageName]").attr("value"); // Select the "input" element from the page String imageName = doc.select("div.photo > a > img").attr("src");
return "https://www.8muses.com/image/fm/" + imageName; return "https://www.8muses.com/image/fm/" + imageName;
} }
@ -189,4 +189,25 @@ public class EightmusesRipper extends AbstractHTMLRipper {
public String getPrefixShort(int index) { public String getPrefixShort(int index) {
return String.format("%03d", index); return String.format("%03d", index);
} }
private String deobfuscateJSON(String obfuscatedString) {
StringBuilder deobfuscatedString = new StringBuilder();
// The first char in one of 8muses obfuscated strings is always ! so we replace it
for (char ch : obfuscatedString.replaceFirst("!", "").toCharArray()){
deobfuscatedString.append(deobfuscateChar(ch));
}
return deobfuscatedString.toString();
}
private String deobfuscateChar(char c) {
if ((int) c == 32) {
return fromCharCode(32);
}
return fromCharCode(33 + (c + 14) % 94);
}
private static String fromCharCode(int... codePoints) {
return new String(codePoints, 0, codePoints.length);
}
} }

78
src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java
View File

@ -22,6 +22,8 @@ import com.rarchives.ripme.utils.Http;
*/ */
public class EromeRipper extends AbstractHTMLRipper { public class EromeRipper extends AbstractHTMLRipper {
boolean rippingProfile;
public EromeRipper (URL url) throws IOException { public EromeRipper (URL url) throws IOException {
super(url); super(url);
@ -42,6 +44,27 @@ public class EromeRipper extends AbstractHTMLRipper {
addURLToDownload(url, getPrefix(index)); addURLToDownload(url, getPrefix(index));
} }
@Override
public boolean hasQueueSupport() {
return true;
}
@Override
public boolean pageContainsAlbums(URL url) {
Pattern pa = Pattern.compile("https?://www.erome.com/([a-zA-Z0-9_-]*)/?");
Matcher ma = pa.matcher(url.toExternalForm());
return ma.matches();
}
@Override
public List<String> getAlbumsToQueue(Document doc) {
List<String> urlsToAddToQueue = new ArrayList<>();
for (Element elem : doc.select("div#albums > div.album > a")) {
urlsToAddToQueue.add(elem.attr("href"));
}
return urlsToAddToQueue;
}
@Override @Override
public String getAlbumTitle(URL url) throws MalformedURLException { public String getAlbumTitle(URL url) throws MalformedURLException {
try { try {
@ -53,6 +76,8 @@ public class EromeRipper extends AbstractHTMLRipper {
} catch (IOException e) { } catch (IOException e) {
// Fall back to default album naming convention // Fall back to default album naming convention
LOGGER.info("Unable to find title at " + url); LOGGER.info("Unable to find title at " + url);
} catch (NullPointerException e) {
return getHost() + "_" + getGID(url);
} }
return super.getAlbumTitle(url); return super.getAlbumTitle(url);
} }
@ -66,21 +91,7 @@ public class EromeRipper extends AbstractHTMLRipper {
@Override @Override
public List<String> getURLsFromPage(Document doc) { public List<String> getURLsFromPage(Document doc) {
List<String> URLs = new ArrayList<>(); List<String> URLs = new ArrayList<>();
//Pictures return getMediaFromPage(doc);
Elements imgs = doc.select("div.img > img.img-front");
for (Element img : imgs) {
String imageURL = img.attr("src");
imageURL = "https:" + imageURL;
URLs.add(imageURL);
}
//Videos
Elements vids = doc.select("div.video > video > source");
for (Element vid : vids) {
String videoURL = vid.attr("src");
URLs.add("https:" + videoURL);
}
return URLs;
} }
@Override @Override
@ -94,13 +105,13 @@ public class EromeRipper extends AbstractHTMLRipper {
@Override @Override
public String getGID(URL url) throws MalformedURLException { public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^https?://www.erome.com/a/([a-zA-Z0-9]*)/?$"); Pattern p = Pattern.compile("^https?://www.erome.com/[ai]/([a-zA-Z0-9]*)/?$");
Matcher m = p.matcher(url.toExternalForm()); Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) { if (m.matches()) {
return m.group(1); return m.group(1);
} }
p = Pattern.compile("^https?://erome.com/a/([a-zA-Z0-9]*)/?$"); p = Pattern.compile("^https?://www.erome.com/([a-zA-Z0-9_-]+)/?$");
m = p.matcher(url.toExternalForm()); m = p.matcher(url.toExternalForm());
if (m.matches()) { if (m.matches()) {
@ -110,34 +121,15 @@ public class EromeRipper extends AbstractHTMLRipper {
throw new MalformedURLException("erome album not found in " + url + ", expected https://www.erome.com/album"); throw new MalformedURLException("erome album not found in " + url + ", expected https://www.erome.com/album");
} }
public static List<URL> getURLs(URL url) throws IOException{ private List<String> getMediaFromPage(Document doc) {
List<String> results = new ArrayList<>();
Response resp = Http.url(url) for (Element el : doc.select("img.img-front")) {
.ignoreContentType() results.add("https:" + el.attr("src"));
.response();
Document doc = resp.parse();
List<URL> URLs = new ArrayList<>();
//Pictures
Elements imgs = doc.getElementsByTag("img");
for (Element img : imgs) {
if (img.hasClass("album-image")) {
String imageURL = img.attr("src");
imageURL = "https:" + imageURL;
URLs.add(new URL(imageURL));
} }
for (Element el : doc.select("source[label=HD]")) {
results.add("https:" + el.attr("src"));
} }
//Videos return results;
Elements vids = doc.getElementsByTag("video");
for (Element vid : vids) {
if (vid.hasClass("album-video")) {
Elements source = vid.getElementsByTag("source");
String videoURL = source.first().attr("src");
URLs.add(new URL(videoURL));
}
} }
return URLs;
}
} }

61
src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java
View File

@ -12,8 +12,10 @@ import java.util.Map;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import com.rarchives.ripme.ui.RipStatusMessage;
import com.rarchives.ripme.utils.Utils; import com.rarchives.ripme.utils.Utils;
import org.jsoup.Connection.Response; import org.jsoup.Connection.Response;
import org.jsoup.HttpStatusException;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
@ -24,13 +26,27 @@ import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.ripper.DownloadThreadPool;
import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Http;
import static com.rarchives.ripme.utils.RipUtils.getCookiesFromString;
public class FuraffinityRipper extends AbstractHTMLRipper { public class FuraffinityRipper extends AbstractHTMLRipper {
private static final String urlBase = "https://www.furaffinity.net"; private static final String urlBase = "https://www.furaffinity.net";
private static Map<String,String> cookies = new HashMap<>(); private Map<String,String> cookies = new HashMap<>();
static {
cookies.put("b", "bd5ccac8-51dc-4265-8ae1-7eac685ad667"); private void setCookies() {
cookies.put("a", "7c41b782-d01d-4b0e-b45b-62a4f0b2a369"); if (Utils.getConfigBoolean("furaffinity.login", true)) {
LOGGER.info("Logging in using cookies");
String faCookies = Utils.getConfigString("furaffinity.cookies", "a=897bc45b-1f87-49f1-8a85-9412bc103e7a;b=c8807f36-7a85-4caf-80ca-01c2a2368267");
warnAboutSharedAccount(faCookies);
cookies = getCookiesFromString(faCookies);
}
}
private void warnAboutSharedAccount(String loginCookies) {
if (loginCookies.equals("a=897bc45b-1f87-49f1-8a85-9412bc103e7a;b=c8807f36-7a85-4caf-80ca-01c2a2368267")) {
sendUpdate(RipStatusMessage.STATUS.DOWNLOAD_ERRORED,
"WARNING: Using the shared furaffinity account exposes both your IP and how many items you downloaded to the other users of the share account");
}
} }
// Thread pool for finding direct image links from "image" pages (html) // Thread pool for finding direct image links from "image" pages (html)
@ -61,6 +77,8 @@ public class FuraffinityRipper extends AbstractHTMLRipper {
} }
@Override @Override
public Document getFirstPage() throws IOException { public Document getFirstPage() throws IOException {
setCookies();
LOGGER.info(Http.url(url).cookies(cookies).get().html());
return Http.url(url).cookies(cookies).get(); return Http.url(url).cookies(cookies).get();
} }
@ -80,20 +98,34 @@ public class FuraffinityRipper extends AbstractHTMLRipper {
} }
private String getImageFromPost(String url) { private String getImageFromPost(String url) {
sleep(1000);
Document d = null;
try { try {
LOGGER.info("found url " + Http.url(url).cookies(cookies).get().select("meta[property=og:image]").attr("content")); d = Http.url(url).cookies(cookies).get();
return Http.url(url).cookies(cookies).get().select("meta[property=og:image]").attr("content"); Elements links = d.getElementsByTag("a");
} catch (IOException e) { for (Element link : links) {
return ""; if (link.text().equals("Download")) {
LOGGER.info("Found image " + link.attr("href"));
return "https:" + link.attr("href");
} }
} }
} catch (IOException e) {
return null;
}
return null;
}
@Override @Override
public List<String> getURLsFromPage(Document page) { public List<String> getURLsFromPage(Document page) {
List<String> urls = new ArrayList<>(); List<String> urls = new ArrayList<>();
Elements urlElements = page.select("figure.t-image > b > u > a"); Elements urlElements = page.select("figure.t-image > b > u > a");
for (Element e : urlElements) { for (Element e : urlElements) {
urls.add(getImageFromPost(urlBase + e.select("a").first().attr("href"))); String urlToAdd = getImageFromPost(urlBase + e.select("a").first().attr("href"));
if (url != null) {
if (urlToAdd.startsWith("http")) {
urls.add(urlToAdd);
}
}
} }
return urls; return urls;
} }
@ -200,16 +232,5 @@ public class FuraffinityRipper extends AbstractHTMLRipper {
+ " instead"); + " instead");
} }
private class FuraffinityDocumentThread extends Thread {
private URL url;
FuraffinityDocumentThread(URL url) {
super();
this.url = url;
}
}
} }

40
src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java → src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java
View File

@ -1,18 +1,22 @@
package com.rarchives.ripme.ripper.rippers.video; package com.rarchives.ripme.ripper.rippers;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import com.rarchives.ripme.ripper.AbstractSingleFileRipper;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.select.Elements; import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.VideoRipper;
import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Http;
public class GfycatRipper extends VideoRipper {
public class GfycatRipper extends AbstractSingleFileRipper {
private static final String HOST = "gfycat.com"; private static final String HOST = "gfycat.com";
@ -20,9 +24,14 @@ public class GfycatRipper extends VideoRipper {
super(url); super(url);
} }
@Override
public String getDomain() {
return "gfycat.com";
}
@Override @Override
public String getHost() { public String getHost() {
return HOST; return "gfycat";
} }
@Override @Override
@ -37,6 +46,16 @@ public class GfycatRipper extends VideoRipper {
return url; return url;
} }
@Override
public Document getFirstPage() throws IOException {
return Http.url(url).get();
}
@Override
public void downloadURL(URL url, int index) {
addURLToDownload(url, getPrefix(index));
}
@Override @Override
public String getGID(URL url) throws MalformedURLException { public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^https?://[wm.]*gfycat\\.com/([a-zA-Z0-9]+).*$"); Pattern p = Pattern.compile("^https?://[wm.]*gfycat\\.com/([a-zA-Z0-9]+).*$");
@ -52,10 +71,15 @@ public class GfycatRipper extends VideoRipper {
} }
@Override @Override
public void rip() throws IOException { public List<String> getURLsFromPage(Document doc) {
String vidUrl = getVideoURL(this.url); List<String> result = new ArrayList<>();
addURLToDownload(new URL(vidUrl), "gfycat_" + getGID(this.url)); Elements videos = doc.select("source#mp4Source");
waitForThreads(); String vidUrl = videos.first().attr("src");
if (vidUrl.startsWith("//")) {
vidUrl = "http:" + vidUrl;
}
result.add(vidUrl);
return result;
} }
/** /**

5
src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java
View File

@ -8,13 +8,12 @@ import java.util.List;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import com.rarchives.ripme.ripper.AbstractSingleFileRipper;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Http;
public class GfycatporntubeRipper extends AbstractHTMLRipper { public class GfycatporntubeRipper extends AbstractSingleFileRipper {
public GfycatporntubeRipper(URL url) throws IOException { public GfycatporntubeRipper(URL url) throws IOException {
super(url); super(url);

18
src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java
View File

@ -13,10 +13,11 @@ import org.jsoup.nodes.Document;
import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Http;
import org.jsoup.nodes.Element;
public class HitomiRipper extends AbstractHTMLRipper { public class HitomiRipper extends AbstractHTMLRipper {
String galleryId = ""; private String galleryId = "";
public HitomiRipper(URL url) throws IOException { public HitomiRipper(URL url) throws IOException {
super(url); super(url);
@ -47,7 +48,7 @@ public class HitomiRipper extends AbstractHTMLRipper {
@Override @Override
public Document getFirstPage() throws IOException { public Document getFirstPage() throws IOException {
// if we go to /GALLERYID.js we get a nice json array of all images in the gallery // if we go to /GALLERYID.js we get a nice json array of all images in the gallery
return Http.url(new URL(url.toExternalForm().replaceAll(".html", ".js"))).ignoreContentType().get(); return Http.url(new URL(url.toExternalForm().replaceAll("hitomi", "ltn.hitomi").replaceAll(".html", ".js"))).ignoreContentType().get();
} }
@ -64,6 +65,19 @@ public class HitomiRipper extends AbstractHTMLRipper {
return result; return result;
} }
@Override
public String getAlbumTitle(URL url) throws MalformedURLException {
try {
// Attempt to use album title and username as GID
Document doc = Http.url(url).get();
return getHost() + "_" + getGID(url) + "_" +
doc.select("title").text().replaceAll(" - Read Online - hentai artistcg \\| Hitomi.la", "");
} catch (IOException e) {
LOGGER.info("Falling back");
}
return super.getAlbumTitle(url);
}
@Override @Override
public void downloadURL(URL url, int index) { public void downloadURL(URL url, int index) {
addURLToDownload(url, getPrefix(index)); addURLToDownload(url, getPrefix(index));

112
src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java Normal file
View File

@ -0,0 +1,112 @@
package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.utils.Http;
public class ImagearnRipper extends AbstractHTMLRipper {
public ImagearnRipper(URL url) throws IOException {
super(url);
}
@Override
public String getHost() {
return "imagearn";
}
@Override
public String getDomain() {
return "imagearn.com";
}
@Override
public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^.*imagearn.com/+gallery.php\\?id=([0-9]+).*$");
Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) {
return m.group(1);
}
throw new MalformedURLException(
"Expected imagearn.com gallery formats: "
+ "imagearn.com/gallery.php?id=####..."
+ " Got: " + url);
}
public URL sanitizeURL(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^.*imagearn.com/+image.php\\?id=[0-9]+.*$");
Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) {
// URL points to imagearn *image*, not gallery
try {
url = getGalleryFromImage(url);
} catch (Exception e) {
LOGGER.error("[!] " + e.getMessage(), e);
}
}
return url;
}
private URL getGalleryFromImage(URL url) throws IOException {
Document doc = Http.url(url).get();
for (Element link : doc.select("a[href~=^gallery\\.php.*$]")) {
LOGGER.info("LINK: " + link.toString());
if (link.hasAttr("href")
&& link.attr("href").contains("gallery.php")) {
url = new URL("http://imagearn.com/" + link.attr("href"));
LOGGER.info("[!] Found gallery from given link: " + url);
return url;
}
}
throw new IOException("Failed to find gallery at URL " + url);
}
@Override
public Document getFirstPage() throws IOException {
return Http.url(url).get();
}
@Override
public String getAlbumTitle(URL url) throws MalformedURLException {
try {
Document doc = getFirstPage();
String title = doc.select("h3 > strong").first().text(); // profile name
return getHost() + "_" + title + "_" + getGID(url);
} catch (Exception e) {
// Fall back to default album naming convention
LOGGER.warn("Failed to get album title from " + url, e);
}
return super.getAlbumTitle(url);
}
@Override
public List<String> getURLsFromPage(Document doc) {
List<String> imageURLs = new ArrayList<>();
for (Element thumb : doc.select("div#gallery > div > a")) {
String imageURL = thumb.attr("href");
try {
Document imagedoc = new Http("http://imagearn.com/" + imageURL).get();
String image = imagedoc.select("a.thickbox").first().attr("href");
imageURLs.add(image);
} catch (IOException e) {
LOGGER.warn("Was unable to download page: " + imageURL);
}
}
return imageURLs;
}
@Override
public void downloadURL(URL url, int index) {
addURLToDownload(url, getPrefix(index));
sleep(1000);
}
}

15
src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java
View File

@ -125,11 +125,7 @@ public class ImagefapRipper extends AbstractHTMLRipper {
if (!thumb.hasAttr("src") || !thumb.hasAttr("width")) { if (!thumb.hasAttr("src") || !thumb.hasAttr("width")) {
continue; continue;
} }
String image = thumb.attr("src"); String image = getFullSizedImage("https://www.imagefap.com" + thumb.parent().attr("href"));
image = image.replaceAll(
"http://x.*.fap.to/images/thumb/",
"http://fap.to/images/full/");
image = image.replaceAll("w[0-9]+-h[0-9]+/", "");
imageURLs.add(image); imageURLs.add(image);
if (isThisATest()) { if (isThisATest()) {
break; break;
@ -160,4 +156,13 @@ public class ImagefapRipper extends AbstractHTMLRipper {
return super.getAlbumTitle(url); return super.getAlbumTitle(url);
} }
private String getFullSizedImage(String pageURL) {
try {
Document doc = Http.url(pageURL).get();
return doc.select("img#mainPhoto").attr("src");
} catch (IOException e) {
return null;
}
}
} }

45
src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java
View File

@ -76,6 +76,10 @@ public class InstagramRipper extends AbstractJSONRipper {
return url.replaceAll("/[A-Z0-9]{8}/", "/"); return url.replaceAll("/[A-Z0-9]{8}/", "/");
} }
@Override public boolean hasASAPRipping() {
return true;
}
private List<String> getPostsFromSinglePage(JSONObject json) { private List<String> getPostsFromSinglePage(JSONObject json) {
List<String> imageURLs = new ArrayList<>(); List<String> imageURLs = new ArrayList<>();
JSONArray datas; JSONArray datas;
@ -231,9 +235,23 @@ public class InstagramRipper extends AbstractJSONRipper {
return imageURL; return imageURL;
} }
public String getAfter(JSONObject json) {
try {
return json.getJSONObject("entry_data").getJSONArray("ProfilePage").getJSONObject(0)
.getJSONObject("graphql").getJSONObject("user")
.getJSONObject("edge_owner_to_timeline_media").getJSONObject("page_info").getString("end_cursor");
} catch (JSONException e) {
return json.getJSONObject("data").getJSONObject("user")
.getJSONObject("edge_owner_to_timeline_media").getJSONObject("page_info").getString("end_cursor");
}
}
@Override @Override
public List<String> getURLsFromJSON(JSONObject json) { public List<String> getURLsFromJSON(JSONObject json) {
List<String> imageURLs = new ArrayList<>(); List<String> imageURLs = new ArrayList<>();
if (!url.toExternalForm().contains("/p/")) {
nextPageID = getAfter(json);
}
// get the rhx_gis value so we can get the next page later on // get the rhx_gis value so we can get the next page later on
if (rhx_gis == null) { if (rhx_gis == null) {
@ -246,7 +264,8 @@ public class InstagramRipper extends AbstractJSONRipper {
try { try {
JSONArray profilePage = json.getJSONObject("entry_data").getJSONArray("ProfilePage"); JSONArray profilePage = json.getJSONObject("entry_data").getJSONArray("ProfilePage");
userID = profilePage.getJSONObject(0).getString("logging_page_id").replaceAll("profilePage_", ""); userID = profilePage.getJSONObject(0).getString("logging_page_id").replaceAll("profilePage_", "");
datas = profilePage.getJSONObject(0).getJSONObject("graphql").getJSONObject("user") datas = json.getJSONObject("entry_data").getJSONArray("ProfilePage").getJSONObject(0)
.getJSONObject("graphql").getJSONObject("user")
.getJSONObject("edge_owner_to_timeline_media").getJSONArray("edges"); .getJSONObject("edge_owner_to_timeline_media").getJSONArray("edges");
} catch (JSONException e) { } catch (JSONException e) {
datas = json.getJSONObject("data").getJSONObject("user") datas = json.getJSONObject("data").getJSONObject("user")
@ -300,11 +319,10 @@ public class InstagramRipper extends AbstractJSONRipper {
} }
} }
} catch (MalformedURLException e) { } catch (MalformedURLException e) {
LOGGER.info("Got MalformedURLException");
return imageURLs; return imageURLs;
} }
nextPageID = data.getString("id");
if (isThisATest()) { if (isThisATest()) {
break; break;
} }
@ -368,10 +386,11 @@ public class InstagramRipper extends AbstractJSONRipper {
try { try {
// Sleep for a while to avoid a ban // Sleep for a while to avoid a ban
sleep(2500); sleep(2500);
String vars = "{\"id\":\"" + userID + "\",\"first\":50,\"after\":\"" + nextPageID + "\"}"; String vars = "{\"id\":\"" + userID + "\",\"first\":12,\"after\":\"" + nextPageID + "\"}";
String ig_gis = getIGGis(vars); String ig_gis = getIGGis(vars);
LOGGER.info(ig_gis); LOGGER.info(ig_gis);
LOGGER.info("https://www.instagram.com/graphql/query/?query_hash=" + qHash + "&variables=" + vars);
toreturn = getPage("https://www.instagram.com/graphql/query/?query_hash=" + qHash + "&variables=" + vars, ig_gis); toreturn = getPage("https://www.instagram.com/graphql/query/?query_hash=" + qHash + "&variables=" + vars, ig_gis);
if (!pageHasImages(toreturn)) { if (!pageHasImages(toreturn)) {
throw new IOException("No more pages"); throw new IOException("No more pages");
@ -391,6 +410,7 @@ public class InstagramRipper extends AbstractJSONRipper {
} }
private boolean pageHasImages(JSONObject json) { private boolean pageHasImages(JSONObject json) {
LOGGER.info(json);
int numberOfImages = json.getJSONObject("data").getJSONObject("user") int numberOfImages = json.getJSONObject("data").getJSONObject("user")
.getJSONObject("edge_owner_to_timeline_media").getJSONArray("edges").length(); .getJSONObject("edge_owner_to_timeline_media").getJSONArray("edges").length();
if (numberOfImages == 0) { if (numberOfImages == 0) {
@ -451,26 +471,11 @@ public class InstagramRipper extends AbstractJSONRipper {
return null; return null;
} }
if (!rippingTag) { if (!rippingTag) {
Pattern jsP = Pattern.compile("o},queryId:.([a-zA-Z0-9]+)."); Pattern jsP = Pattern.compile("byUserId\\.get\\(t\\)\\)\\|\\|void 0===r\\?void 0:r\\.pagination},queryId:.([a-zA-Z0-9]+)");
Matcher m = jsP.matcher(sb.toString()); Matcher m = jsP.matcher(sb.toString());
if (m.find()) { if (m.find()) {
return m.group(1); return m.group(1);
} }
jsP = Pattern.compile("n.pagination:n},queryId:.([a-zA-Z0-9]+).");
m = jsP.matcher(sb.toString());
if (m.find()) {
return m.group(1);
}
jsP = Pattern.compile("0:n.pagination},queryId:.([a-zA-Z0-9]+).");
m = jsP.matcher(sb.toString());
if (m.find()) {
return m.group(1);
}
jsP = Pattern.compile("o.pagination},queryId:.([a-zA-Z0-9]+).");
m = jsP.matcher(sb.toString());
if (m.find()) {
return m.group(1);
}
} else { } else {
Pattern jsP = Pattern.compile("return e.tagMedia.byTagName.get\\(t\\).pagination},queryId:.([a-zA-Z0-9]+)."); Pattern jsP = Pattern.compile("return e.tagMedia.byTagName.get\\(t\\).pagination},queryId:.([a-zA-Z0-9]+).");

78
src/main/java/com/rarchives/ripme/ripper/rippers/JabArchivesRipper.java Normal file
View File

@ -0,0 +1,78 @@
package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.utils.Http;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class JabArchivesRipper extends AbstractHTMLRipper {
public JabArchivesRipper(URL url) throws IOException {
super(url);
}
@Override
public String getHost() {
return "jabarchives";
}
@Override
public String getDomain() {
return "jabarchives.com";
}
@Override
public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^https?://jabarchives.com/main/view/([a-zA-Z0-9_]+).*$");
Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) {
// Return the text contained between () in the regex
return m.group(1);
}
throw new MalformedURLException(
"Expected javarchives.com URL format: " +
"jabarchives.com/main/view/albumname - got " + url + " instead");
}
@Override
public Document getFirstPage() throws IOException {
// "url" is an instance field of the superclass
return Http.url(url).get();
}
@Override
public Document getNextPage(Document doc) throws IOException {
// Find next page
Elements hrefs = doc.select("a[title=\"Next page\"]");
if (hrefs.isEmpty()) {
throw new IOException("No more pages");
}
String nextUrl = "https://jabarchives.com" + hrefs.first().attr("href");
sleep(500);
return Http.url(nextUrl).get();
}
@Override
public List<String> getURLsFromPage(Document doc) {
List<String> result = new ArrayList<String>();
for (Element el : doc.select("#contentMain img")) {
result.add("https://jabarchives.com" + el.attr("src").replace("thumb", "large"));
}
return result;
}
@Override
public void downloadURL(URL url, int index) {
addURLToDownload(url, getPrefix(index));
}
}

123
src/main/java/com/rarchives/ripme/ripper/rippers/LoveromRipper.java Normal file
View File

@ -0,0 +1,123 @@
package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.rarchives.ripme.utils.Utils;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.utils.Http;
public class LoveromRipper extends AbstractHTMLRipper {
public LoveromRipper(URL url) throws IOException {
super(url);
}
private int bytesTotal = 1;
private int bytesCompleted = 1;
boolean multipart = false;
@Override
public String getHost() {
return "loveroms";
}
@Override
public String getDomain() {
return "loveroms.com";
}
@Override
public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("https://www.loveroms.com/download/([a-zA-Z0-9_-]+)/([a-zA-Z0-9_-]+)/\\d+");
Matcher m = p.matcher(url.toExternalForm());
if (!m.matches()) {
throw new MalformedURLException("Expected URL format: https://www.loveroms.com/download/CONSOLE/GAME, got: " + url);
}
return m.group(1) + "_" + m.group(2);
}
@Override
public Document getFirstPage() throws IOException {
// "url" is an instance field of the superclass
return Http.url(url).get();
}
@Override
public List<String> getURLsFromPage(Document doc) {
List<String> result = new ArrayList<>();
String downloadLink = doc.select("a#start_download_link").attr("href");
if (downloadLink != null && !downloadLink.isEmpty()) {
result.add(downloadLink);
} else {
multipart = true;
for (Element el : doc.select("a.multi-file-btn")) {
result.add(el.attr("href"));
}
}
return result;
}
@Override
public void downloadURL(URL url, int index) {
if (multipart) {
addURLToDownload(url, "", "", "", null, null, "7z." + getPrefix(index));
} else {
addURLToDownload(url, "", "", "", null, null, "7z");
}
}
@Override
public String getStatusText() {
if (multipart) {
return super.getStatusText();
}
return String.valueOf(getCompletionPercentage()) +
"% - " +
Utils.bytesToHumanReadable(bytesCompleted) +
" / " +
Utils.bytesToHumanReadable(bytesTotal);
}
@Override
public int getCompletionPercentage() {
if (multipart) {
return super.getCompletionPercentage();
}
return (int) (100 * (bytesCompleted / (float) bytesTotal));
}
@Override
public void setBytesTotal(int bytes) {
this.bytesTotal = bytes;
}
@Override
public void setBytesCompleted(int bytes) {
this.bytesCompleted = bytes;
}
@Override
public boolean useByteProgessBar() {return true;}
@Override
public boolean tryResumeDownload() {return true;}
@Override
public String getPrefix(int index) {
String prefix = "";
if (keepSortOrder() && Utils.getConfigBoolean("download.save_order", true)) {
prefix = String.format("7z.%03d", index);
}
return prefix;
}
}

8
src/main/java/com/rarchives/ripme/ripper/rippers/ModelmayhemRipper.java
View File

@ -4,7 +4,9 @@ import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@ -16,6 +18,8 @@ import com.rarchives.ripme.utils.Http;
public class ModelmayhemRipper extends AbstractHTMLRipper { public class ModelmayhemRipper extends AbstractHTMLRipper {
private Map<String,String> cookies = new HashMap<>();
public ModelmayhemRipper(URL url) throws IOException { public ModelmayhemRipper(URL url) throws IOException {
super(url); super(url);
} }
@ -43,8 +47,10 @@ public class ModelmayhemRipper extends AbstractHTMLRipper {
@Override @Override
public Document getFirstPage() throws IOException { public Document getFirstPage() throws IOException {
// Bypass NSFW filter
cookies.put("worksafe", "0");
// "url" is an instance field of the superclass // "url" is an instance field of the superclass
return Http.url(url).get(); return Http.url(url).cookies(cookies).get();
} }
@Override @Override

83
src/main/java/com/rarchives/ripme/ripper/rippers/PicstatioRipper.java Normal file
View File

@ -0,0 +1,83 @@
package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.utils.Http;
public class PicstatioRipper extends AbstractHTMLRipper {
public PicstatioRipper(URL url) throws IOException {
super(url);
}
private String getFullSizedImageFromURL(String fileName) {
try {
LOGGER.info("https://www.picstatio.com/wallpaper/" + fileName + "/download");
return Http.url("https://www.picstatio.com/wallpaper/" + fileName + "/download").get().select("p.text-center > span > a").attr("href");
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
@Override
public String getHost() {
return "picstatio";
}
@Override
public String getDomain() {
return "picstatio.com";
}
@Override
public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("https?://www.picstatio.com/([a-zA-Z1-9_-]*)/?$");
Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) {
return m.group(1);
}
throw new MalformedURLException("Expected picstatio URL format: " +
"www.picstatio.com//ID - got " + url + " instead");
}
@Override
public Document getFirstPage() throws IOException {
// "url" is an instance field of the superclass
return Http.url(url).get();
}
@Override
public Document getNextPage(Document doc) throws IOException {
if (doc.select("a.next_page") != null) {
return Http.url("https://www.picstatio.com" + doc.select("a.next_page").attr("href")).get();
}
throw new IOException("No more pages");
}
@Override
public List<String> getURLsFromPage(Document doc) {
List<String> result = new ArrayList<>();
for (Element e : doc.select("img.img")) {
String imageName = e.parent().attr("href");
LOGGER.info(getFullSizedImageFromURL(imageName.split("/")[2]));
result.add(getFullSizedImageFromURL(imageName.split("/")[2]));
}
return result;
}
@Override
public void downloadURL(URL url, int index) {
addURLToDownload(url, getPrefix(index));
}
}

44
src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java
View File

@ -8,6 +8,7 @@ import java.util.List;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import com.rarchives.ripme.ui.RipStatusMessage;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONObject; import org.json.JSONObject;
import org.json.JSONTokener; import org.json.JSONTokener;
@ -17,6 +18,10 @@ import com.rarchives.ripme.ui.UpdateUtils;
import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.RipUtils; import com.rarchives.ripme.utils.RipUtils;
import com.rarchives.ripme.utils.Utils; import com.rarchives.ripme.utils.Utils;
import org.jsoup.Jsoup;
import javax.swing.text.Document;
import javax.swing.text.Element;
public class RedditRipper extends AlbumRipper { public class RedditRipper extends AlbumRipper {
@ -35,6 +40,10 @@ public class RedditRipper extends AlbumRipper {
private long lastRequestTime = 0; private long lastRequestTime = 0;
private Boolean shouldAddURL() {
return (alreadyDownloadedUrls >= Utils.getConfigInteger("history.end_rip_after_already_seen", 1000000000) && !isThisATest());
}
@Override @Override
public boolean canRip(URL url) { public boolean canRip(URL url) {
return url.getHost().endsWith(DOMAIN); return url.getHost().endsWith(DOMAIN);
@ -61,6 +70,10 @@ public class RedditRipper extends AlbumRipper {
public void rip() throws IOException { public void rip() throws IOException {
URL jsonURL = getJsonURL(this.url); URL jsonURL = getJsonURL(this.url);
while (true) { while (true) {
if (shouldAddURL()) {
sendUpdate(RipStatusMessage.STATUS.DOWNLOAD_COMPLETE_HISTORY, "Already seen the last " + alreadyDownloadedUrls + " images ending rip");
break;
}
jsonURL = getAndParseAndReturnNext(jsonURL); jsonURL = getAndParseAndReturnNext(jsonURL);
if (jsonURL == null || isThisATest() || isStopped()) { if (jsonURL == null || isThisATest() || isStopped()) {
break; break;
@ -179,6 +192,32 @@ public class RedditRipper extends AlbumRipper {
} }
} }
private URL parseRedditVideoMPD(String vidURL) {
org.jsoup.nodes.Document doc = null;
try {
doc = Http.url(vidURL + "/DASHPlaylist.mpd").ignoreContentType().get();
int largestHeight = 0;
String baseURL = null;
// Loops over all the videos and finds the one with the largest height and sets baseURL to the base url of that video
for (org.jsoup.nodes.Element e : doc.select("MPD > Period > AdaptationSet > Representation")) {
String height = e.attr("height");
if (height.equals("")) {
height = "0";
}
if (largestHeight < Integer.parseInt(height)) {
largestHeight = Integer.parseInt(height);
baseURL = doc.select("MPD > Period > AdaptationSet > Representation[height=" + height + "]").select("BaseURL").text();
}
LOGGER.info("H " + e.attr("height") + " V " + e.attr("width"));
}
return new URL(vidURL + "/" + baseURL);
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
private void handleURL(String theUrl, String id) { private void handleURL(String theUrl, String id) {
URL originalURL; URL originalURL;
try { try {
@ -198,6 +237,11 @@ public class RedditRipper extends AlbumRipper {
savePath += id + "-" + m.group(1) + ".jpg"; savePath += id + "-" + m.group(1) + ".jpg";
addURLToDownload(urls.get(0), new File(savePath)); addURLToDownload(urls.get(0), new File(savePath));
} }
if (url.contains("v.redd.it")) {
String savePath = this.workingDir + File.separator;
savePath += id + "-" + url.split("/")[3] + ".mp4";
addURLToDownload(parseRedditVideoMPD(urls.get(0).toExternalForm()), new File(savePath));
}
else { else {
addURLToDownload(urls.get(0), id + "-", "", theUrl, null); addURLToDownload(urls.get(0), id + "-", "", theUrl, null);
} }

8
src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java
View File

@ -235,8 +235,12 @@ public class TumblrRipper extends AlbumRipper {
for (int j = 0; j < photos.length(); j++) { for (int j = 0; j < photos.length(); j++) {
photo = photos.getJSONObject(j); photo = photos.getJSONObject(j);
try { try {
if (Utils.getConfigBoolean("tumblr.get_raw_image", false)) { String imageUrl = photo.getJSONObject("original_size").getString("url");
String urlString = photo.getJSONObject("original_size").getString("url").replaceAll("https", "http"); // If the url is shorter than 65 chars long we skip it because it's those images don't support grabbing them in fullsize
if (Utils.getConfigBoolean("tumblr.get_raw_image", false) &&
imageUrl.replaceAll("https", "http").length() > 65) {
// We have to change the link to http because tumblr uses an invalid cert for data.tumblr.com
String urlString = imageUrl.replaceAll("https", "http");
urlString = urlString.replaceAll("https?://[a-sA-Z0-9_\\-\\.]*\\.tumblr", "http://data.tumblr"); urlString = urlString.replaceAll("https?://[a-sA-Z0-9_\\-\\.]*\\.tumblr", "http://data.tumblr");
urlString = urlString.replaceAll("_\\d+\\.", "_raw."); urlString = urlString.replaceAll("_\\d+\\.", "_raw.");
fileURL = new URL(urlString); fileURL = new URL(urlString);

39
src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
View File

@ -20,6 +20,8 @@ import com.rarchives.ripme.utils.Utils;
public class TwitterRipper extends AlbumRipper { public class TwitterRipper extends AlbumRipper {
int downloadUrls = 1;
private static final String DOMAIN = "twitter.com", private static final String DOMAIN = "twitter.com",
HOST = "twitter"; HOST = "twitter";
@ -123,15 +125,16 @@ public class TwitterRipper extends AlbumRipper {
.append("&include_entities=true") .append("&include_entities=true")
.append("&exclude_replies=true") .append("&exclude_replies=true")
.append("&trim_user=true") .append("&trim_user=true")
.append("&include_rts=false") .append("&count=" + 200)
.append("&count=" + 200); .append("&tweet_mode=extended");
break; break;
case SEARCH: case SEARCH:
req.append("https://api.twitter.com/1.1/search/tweets.json") req.append("https://api.twitter.com/1.1/search/tweets.json")
.append("?q=" + this.searchText) .append("?q=" + this.searchText)
.append("&include_entities=true") .append("&include_entities=true")
.append("&result_type=recent") .append("&result_type=recent")
.append("&count=100"); .append("&count=100")
.append("&tweet_mode=extended");
break; break;
} }
if (maxID > 0) { if (maxID > 0) {
@ -187,18 +190,32 @@ public class TwitterRipper extends AlbumRipper {
url = media.getString("media_url"); url = media.getString("media_url");
if (media.getString("type").equals("video")) { if (media.getString("type").equals("video")) {
JSONArray variants = media.getJSONObject("video_info").getJSONArray("variants"); JSONArray variants = media.getJSONObject("video_info").getJSONArray("variants");
int largestBitrate = 0;
String urlToDownload = null;
// Loop over all the video options and find the biggest video
for (int j = 0; j < medias.length(); j++) { for (int j = 0; j < medias.length(); j++) {
JSONObject variant = (JSONObject) variants.get(i); JSONObject variant = (JSONObject) variants.get(i);
if (variant.has("bitrate") && variant.getInt("bitrate") == 832000) { LOGGER.info(variant);
addURLToDownload(new URL(variant.getString("url"))); // If the video doesn't have a bitrate it's a m3u8 file we can't download
if (variant.has("bitrate")) {
if (variant.getInt("bitrate") > largestBitrate) {
largestBitrate = variant.getInt("bitrate");
urlToDownload = variant.getString("url");
}
}
}
if (urlToDownload != null) {
addURLToDownload(new URL(urlToDownload), getPrefix(downloadUrls));
downloadUrls++;
} else {
LOGGER.error("URLToDownload was null");
}
parsedCount++; parsedCount++;
break;
}
}
} else if (media.getString("type").equals("photo")) { } else if (media.getString("type").equals("photo")) {
if (url.contains(".twimg.com/")) { if (url.contains(".twimg.com/")) {
url += ":orig"; url += ":orig";
addURLToDownload(new URL(url)); addURLToDownload(new URL(url), getPrefix(downloadUrls));
downloadUrls++;
parsedCount++; parsedCount++;
} else { } else {
LOGGER.debug("Unexpected media_url: " + url); LOGGER.debug("Unexpected media_url: " + url);
@ -211,6 +228,10 @@ public class TwitterRipper extends AlbumRipper {
return parsedCount; return parsedCount;
} }
public String getPrefix(int index) {
return String.format("%03d_", index);
}
@Override @Override
public void rip() throws IOException { public void rip() throws IOException {
getAccessToken(); getAccessToken();

29
src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java
View File

@ -62,19 +62,21 @@ public class XhamsterRipper extends AbstractHTMLRipper {
@Override @Override
public boolean canRip(URL url) { public boolean canRip(URL url) {
Pattern p = Pattern.compile("^https?://[wmde.]*xhamster\\.com/photos/gallery/.*?(\\d+)$"); Pattern p = Pattern.compile("^https?://([\\w\\w]*\\.)?xhamster\\.com/photos/gallery/.*?(\\d+)$");
Matcher m = p.matcher(url.toExternalForm()); Matcher m = p.matcher(url.toExternalForm());
return m.matches(); return m.matches();
} }
@Override @Override
public Document getNextPage(Document doc) throws IOException { public Document getNextPage(Document doc) throws IOException {
if (!doc.select("a.next").first().attr("href").equals("")) { if (doc.select("a.next").first() != null) {
if (doc.select("a.next").first().attr("href").startsWith("http")) {
return Http.url(doc.select("a.next").first().attr("href")).get(); return Http.url(doc.select("a.next").first().attr("href")).get();
} else {
throw new IOException("No more pages");
} }
} }
throw new IOException("No more pages");
}
@Override @Override
public List<String> getURLsFromPage(Document doc) { public List<String> getURLsFromPage(Document doc) {
@ -100,4 +102,23 @@ public class XhamsterRipper extends AbstractHTMLRipper {
public void downloadURL(URL url, int index) { public void downloadURL(URL url, int index) {
addURLToDownload(url, getPrefix(index)); addURLToDownload(url, getPrefix(index));
} }
@Override
public String getAlbumTitle(URL url) throws MalformedURLException {
try {
// Attempt to use album title and username as GID
Document doc = getFirstPage();
Element user = doc.select("a.author").first();
String username = user.text();
String path = url.getPath();
Pattern p = Pattern.compile("^/photos/gallery/(.*)$");
Matcher m = p.matcher(path);
if (m.matches() && !username.isEmpty()) {
return getHost() + "_" + username + "_" + m.group(1);
}
} catch (IOException e) {
// Fall back to default album naming convention
}
return super.getAlbumTitle(url);
}
} }

40
src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java → src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java
View File

@ -1,19 +1,22 @@
package com.rarchives.ripme.ripper.rippers.video; package com.rarchives.ripme.ripper.rippers;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import com.rarchives.ripme.ripper.AbstractSingleFileRipper;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.jsoup.select.Elements; import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.VideoRipper;
import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Http;
public class XvideosRipper extends VideoRipper { public class XvideosRipper extends AbstractSingleFileRipper {
private static final String HOST = "xvideos"; private static final String HOST = "xvideos";
@ -21,11 +24,21 @@ public class XvideosRipper extends VideoRipper {
super(url); super(url);
} }
@Override
public Document getFirstPage() throws IOException {
return Http.url(this.url).get();
}
@Override @Override
public String getHost() { public String getHost() {
return HOST; return HOST;
} }
@Override
public String getDomain() {
return HOST + ".com";
}
@Override @Override
public boolean canRip(URL url) { public boolean canRip(URL url) {
Pattern p = Pattern.compile("^https?://[wm.]*xvideos\\.com/video[0-9]+.*$"); Pattern p = Pattern.compile("^https?://[wm.]*xvideos\\.com/video[0-9]+.*$");
@ -33,11 +46,6 @@ public class XvideosRipper extends VideoRipper {
return m.matches(); return m.matches();
} }
@Override
public URL sanitizeURL(URL url) throws MalformedURLException {
return url;
}
@Override @Override
public String getGID(URL url) throws MalformedURLException { public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^https?://[wm.]*xvideos\\.com/video([0-9]+).*$"); Pattern p = Pattern.compile("^https?://[wm.]*xvideos\\.com/video([0-9]+).*$");
@ -53,9 +61,8 @@ public class XvideosRipper extends VideoRipper {
} }
@Override @Override
public void rip() throws IOException { public List<String> getURLsFromPage(Document doc) {
LOGGER.info(" Retrieving " + this.url); List<String> results = new ArrayList<>();
Document doc = Http.url(this.url).get();
Elements scripts = doc.select("script"); Elements scripts = doc.select("script");
for (Element e : scripts) { for (Element e : scripts) {
if (e.html().contains("html5player.setVideoUrlHigh")) { if (e.html().contains("html5player.setVideoUrlHigh")) {
@ -64,13 +71,16 @@ public class XvideosRipper extends VideoRipper {
for (String line: lines) { for (String line: lines) {
if (line.contains("html5player.setVideoUrlHigh")) { if (line.contains("html5player.setVideoUrlHigh")) {
String videoURL = line.replaceAll("\t", "").replaceAll("html5player.setVideoUrlHigh\\(", "").replaceAll("\'", "").replaceAll("\\);", ""); String videoURL = line.replaceAll("\t", "").replaceAll("html5player.setVideoUrlHigh\\(", "").replaceAll("\'", "").replaceAll("\\);", "");
addURLToDownload(new URL(videoURL), HOST + "_" + getGID(this.url)); results.add(videoURL);
waitForThreads();
return;
} }
} }
} }
} }
throw new IOException("Unable to find video url at " + this.url.toExternalForm()); return results;
}
@Override
public void downloadURL(URL url, int index) {
addURLToDownload(url, getPrefix(index));
} }
} }

5
src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java
View File

@ -49,9 +49,8 @@ public class ZizkiRipper extends AbstractHTMLRipper {
public String getAlbumTitle(URL url) throws MalformedURLException { public String getAlbumTitle(URL url) throws MalformedURLException {
try { try {
// Attempt to use album title as GID // Attempt to use album title as GID
Element titleElement = getFirstPage().select("meta[name=description]").first(); Element titleElement = getFirstPage().select("h1.title").first();
String title = titleElement.attr("content"); String title = titleElement.text();
title = title.substring(title.lastIndexOf('/') + 1);
Element authorSpan = getFirstPage().select("span[class=creator]").first(); Element authorSpan = getFirstPage().select("span[class=creator]").first();
String author = authorSpan.select("a").first().text(); String author = authorSpan.select("a").first().text();

67
src/main/java/com/rarchives/ripme/ripper/rippers/video/VineRipper.java
View File

@ -1,67 +0,0 @@
package com.rarchives.ripme.ripper.rippers.video;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.VideoRipper;
import com.rarchives.ripme.utils.Http;
public class VineRipper extends VideoRipper {
private static final String HOST = "vine";
public VineRipper(URL url) throws IOException {
super(url);
}
@Override
public String getHost() {
return HOST;
}
@Override
public boolean canRip(URL url) {
// https://vine.co/v/hiqQrP0eUZx
Pattern p = Pattern.compile("^https?://[wm.]*vine\\.co/v/[a-zA-Z0-9]+.*$");
Matcher m = p.matcher(url.toExternalForm());
return m.matches();
}
@Override
public URL sanitizeURL(URL url) throws MalformedURLException {
return url;
}
@Override
public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^https?://[wm.]*vine\\.co/v/([a-zA-Z0-9]+).*$");
Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) {
return m.group(1);
}
throw new MalformedURLException(
"Expected vine format:"
+ "vine.co/v/####"
+ " Got: " + url);
}
@Override
public void rip() throws IOException {
LOGGER.info(" Retrieving " + this.url.toExternalForm());
Document doc = Http.url(this.url).get();
Elements props = doc.select("meta[property=twitter:player:stream]");
if (props.isEmpty()) {
throw new IOException("Could not find meta property 'twitter:player:stream' at " + url);
}
String vidUrl = props.get(0).attr("content");
addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url));
waitForThreads();
}
}

13
src/main/java/com/rarchives/ripme/ui/MainWindow.java
View File

@ -10,6 +10,7 @@ import java.awt.event.WindowAdapter;
import java.awt.event.WindowEvent; import java.awt.event.WindowEvent;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.lang.reflect.Array;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI; import java.net.URI;
import java.net.URL; import java.net.URL;
@ -142,6 +143,10 @@ public final class MainWindow implements Runnable, RipStatusHandler {
private ResourceBundle rb = Utils.getResourceBundle(null); private ResourceBundle rb = Utils.getResourceBundle(null);
// All the langs ripme has been translated into
private static String[] supportedLanges = new String[] {"en_US", "de_DE", "es_ES", "fr_CH", "kr_KR", "pt_PT",
"fi_FI", "in_ID", "nl_NL", "porrisavvo_FI", "ru_RU"};
private void updateQueueLabel() { private void updateQueueLabel() {
if (queueListModel.size() > 0) { if (queueListModel.size() > 0) {
optionQueue.setText(rb.getString("Queue") + " (" + queueListModel.size() + ")"); optionQueue.setText(rb.getString("Queue") + " (" + queueListModel.size() + ")");
@ -496,7 +501,7 @@ public final class MainWindow implements Runnable, RipStatusHandler {
configURLHistoryCheckbox = addNewCheckbox(rb.getString("remember.url.history"), "remember.url_history", true); configURLHistoryCheckbox = addNewCheckbox(rb.getString("remember.url.history"), "remember.url_history", true);
configLogLevelCombobox = new JComboBox<>(new String[] {"Log level: Error", "Log level: Warn", "Log level: Info", "Log level: Debug"}); configLogLevelCombobox = new JComboBox<>(new String[] {"Log level: Error", "Log level: Warn", "Log level: Info", "Log level: Debug"});
configSelectLangComboBox = new JComboBox<>(new String[] {"en_US", "de_DE", "es_ES", "fr_CH", "kr_KR", "pt_PT", "fi_FI", "in_ID", "porrisavvo_FI"}); configSelectLangComboBox = new JComboBox<>(supportedLanges);
configLogLevelCombobox.setSelectedItem(Utils.getConfigString("log.level", "Log level: Debug")); configLogLevelCombobox.setSelectedItem(Utils.getConfigString("log.level", "Log level: Debug"));
setLogLevel(configLogLevelCombobox.getSelectedItem().toString()); setLogLevel(configLogLevelCombobox.getSelectedItem().toString());
configSaveDirLabel = new JLabel(); configSaveDirLabel = new JLabel();
@ -1206,6 +1211,12 @@ public final class MainWindow implements Runnable, RipStatusHandler {
appendLog("Downloaded " + msg.getObject(), Color.GREEN); appendLog("Downloaded " + msg.getObject(), Color.GREEN);
} }
break; break;
case DOWNLOAD_COMPLETE_HISTORY:
if (LOGGER.isEnabledFor(Level.INFO)) {
appendLog("" + msg.getObject(), Color.GREEN);
}
break;
case DOWNLOAD_ERRORED: case DOWNLOAD_ERRORED:
if (LOGGER.isEnabledFor(Level.ERROR)) { if (LOGGER.isEnabledFor(Level.ERROR)) {
appendLog((String) msg.getObject(), Color.RED); appendLog((String) msg.getObject(), Color.RED);

1
src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java
View File

@ -10,6 +10,7 @@ public class RipStatusMessage {
DOWNLOAD_STARTED("Download Started"), DOWNLOAD_STARTED("Download Started"),
DOWNLOAD_COMPLETE("Download Complete"), DOWNLOAD_COMPLETE("Download Complete"),
DOWNLOAD_ERRORED("Download Errored"), DOWNLOAD_ERRORED("Download Errored"),
DOWNLOAD_COMPLETE_HISTORY("Download Complete History"),
RIP_COMPLETE("Rip Complete"), RIP_COMPLETE("Rip Complete"),
DOWNLOAD_WARN("Download problem"), DOWNLOAD_WARN("Download problem"),
TOTAL_BYTES("Total bytes"), TOTAL_BYTES("Total bytes"),

8
src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
View File

@ -20,7 +20,7 @@ import com.rarchives.ripme.utils.Utils;
public class UpdateUtils { public class UpdateUtils {
private static final Logger logger = Logger.getLogger(UpdateUtils.class); private static final Logger logger = Logger.getLogger(UpdateUtils.class);
private static final String DEFAULT_VERSION = "1.7.51"; private static final String DEFAULT_VERSION = "1.7.60";
private static final String REPO_NAME = "ripmeapp/ripme"; private static final String REPO_NAME = "ripmeapp/ripme";
private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json";
private static final String mainFileName = "ripme.jar"; private static final String mainFileName = "ripme.jar";
@ -128,9 +128,9 @@ public class UpdateUtils {
logger.info("Found newer version: " + latestVersion); logger.info("Found newer version: " + latestVersion);
int result = JOptionPane.showConfirmDialog( int result = JOptionPane.showConfirmDialog(
null, null,
"<html><font color=\"green\">New version (" + latestVersion + ") is available!</font>" String.format("<html><font color=\"green\">New version (%s) is available!</font>"
+ "<br><br>Recent changes:" + changeList + "<br><br>Recent changes: %s"
+ "<br><br>Do you want to download and run the newest version?</html>", + "<br><br>Do you want to download and run the newest version?</html>", latestVersion, changeList.replaceAll("\n", "")),
"RipMe Updater", "RipMe Updater",
JOptionPane.YES_NO_OPTION); JOptionPane.YES_NO_OPTION);
if (result != JOptionPane.YES_OPTION) { if (result != JOptionPane.YES_OPTION) {

30
src/main/java/com/rarchives/ripme/utils/RipUtils.java
View File

@ -3,9 +3,7 @@ package com.rarchives.ripme.utils;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.util.ArrayList; import java.util.*;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@ -14,7 +12,7 @@ import com.rarchives.ripme.ripper.rippers.EroShareRipper;
import com.rarchives.ripme.ripper.rippers.EromeRipper; import com.rarchives.ripme.ripper.rippers.EromeRipper;
import com.rarchives.ripme.ripper.rippers.ImgurRipper; import com.rarchives.ripme.ripper.rippers.ImgurRipper;
import com.rarchives.ripme.ripper.rippers.VidbleRipper; import com.rarchives.ripme.ripper.rippers.VidbleRipper;
import com.rarchives.ripme.ripper.rippers.video.GfycatRipper; import com.rarchives.ripme.ripper.rippers.GfycatRipper;
import org.apache.commons.lang.math.NumberUtils; import org.apache.commons.lang.math.NumberUtils;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
@ -56,6 +54,15 @@ public class RipUtils {
} catch (IOException e) { } catch (IOException e) {
logger.error("[!] Exception while loading album " + url, e); logger.error("[!] Exception while loading album " + url, e);
} }
} else if (url.getHost().endsWith("i.imgur.com") && url.toExternalForm().contains("gifv")) {
// links to imgur gifvs
try {
result.add(new URL(url.toExternalForm().replaceAll(".gifv", ".mp4")));
} catch (IOException e) {
logger.info("Couldn't get gifv from " + url);
}
return result;
} }
else if (url.getHost().endsWith("gfycat.com")) { else if (url.getHost().endsWith("gfycat.com")) {
try { try {
@ -88,6 +95,9 @@ public class RipUtils {
logger.warn("Exception while retrieving eroshare page:", e); logger.warn("Exception while retrieving eroshare page:", e);
} }
return result; return result;
} else if (url.toExternalForm().contains("v.redd.it")) {
result.add(url);
return result;
} }
else if (url.toExternalForm().contains("erome.com")) { else if (url.toExternalForm().contains("erome.com")) {
@ -279,4 +289,16 @@ public class RipUtils {
} }
return url; return url;
} }
/**
* Reads a cookie string (Key1=value1;key2=value2) from the config file and turns it into a hashmap
* @return Map of cookies containing session data.
*/
public static Map<String, String> getCookiesFromString(String line) {
Map<String,String> cookies = new HashMap<>();
for (String pair : line.split(";")) {
String[] kv = pair.split("=");
cookies.put(kv[0], kv[1]);
}
return cookies;
}
} }

339
src/main/java/com/rarchives/ripme/utils/Utils.java
View File

@ -1,87 +1,103 @@
package com.rarchives.ripme.utils; package com.rarchives.ripme.utils;
import java.io.*; import com.rarchives.ripme.ripper.AbstractRipper;
import java.lang.reflect.Constructor;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLDecoder;
import java.util.*;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import javax.sound.sampled.AudioSystem;
import javax.sound.sampled.Clip;
import javax.sound.sampled.Line;
import javax.sound.sampled.LineEvent;
import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.log4j.LogManager; import org.apache.log4j.LogManager;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator; import org.apache.log4j.PropertyConfigurator;
import com.rarchives.ripme.ripper.AbstractRipper; import javax.sound.sampled.AudioSystem;
import javax.sound.sampled.Clip;
import javax.sound.sampled.Line;
import javax.sound.sampled.LineEvent;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.lang.reflect.Array;
import java.lang.reflect.Constructor;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.MissingResourceException;
import java.util.ResourceBundle;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
/** /**
* Common utility functions used in various places throughout the project. * Common utility functions used in various places throughout the project.
*/ */
public class Utils { public class Utils {
private static final String RIP_DIRECTORY = "rips"; private static final String RIP_DIRECTORY = "rips";
private static final String configFile = "rip.properties"; private static final String CONFIG_FILE = "rip.properties";
private static final String OS = System.getProperty("os.name").toLowerCase(); private static final String OS = System.getProperty("os.name").toLowerCase();
private static final Logger logger = Logger.getLogger(Utils.class); private static final Logger LOGGER = Logger.getLogger(Utils.class);
private static final int SHORTENED_PATH_LENGTH = 12;
private static PropertiesConfiguration config; private static PropertiesConfiguration config;
private static HashMap<String, HashMap<String, String>> cookieCache;
static { static {
cookieCache = new HashMap<>();
try { try {
String configPath = getConfigFilePath(); String configPath = getConfigFilePath();
File f = new File(configPath); File file = new File(configPath);
if (!f.exists()) {
if (!file.exists()) {
// Use default bundled with .jar // Use default bundled with .jar
configPath = configFile; configPath = CONFIG_FILE;
} }
config = new PropertiesConfiguration(configPath); config = new PropertiesConfiguration(configPath);
logger.info("Loaded " + config.getPath()); LOGGER.info("Loaded " + config.getPath());
if (f.exists()) {
if (file.exists()) {
// Config was loaded from file // Config was loaded from file
if ( !config.containsKey("twitter.auth") if (!config.containsKey("twitter.auth") || !config.containsKey("twitter.max_requests")
|| !config.containsKey("twitter.max_requests") || !config.containsKey("tumblr.auth") || !config.containsKey("error.skip404")
|| !config.containsKey("tumblr.auth") || !config.containsKey("gw.api") || !config.containsKey("page.timeout")
|| !config.containsKey("error.skip404") || !config.containsKey("download.max_size")) {
|| !config.containsKey("gw.api")
|| !config.containsKey("page.timeout")
|| !config.containsKey("download.max_size")
) {
// Config is missing key fields // Config is missing key fields
// Need to reload the default config // Need to reload the default config
// See https://github.com/4pr0n/ripme/issues/158 // See https://github.com/4pr0n/ripme/issues/158
logger.warn("Config does not contain key fields, deleting old config"); LOGGER.warn("Config does not contain key fields, deleting old config");
f.delete(); file.delete();
config = new PropertiesConfiguration(configFile); config = new PropertiesConfiguration(CONFIG_FILE);
logger.info("Loaded " + config.getPath()); LOGGER.info("Loaded " + config.getPath());
} }
} }
} catch (Exception e) { } catch (Exception e) {
logger.error("[!] Failed to load properties file from " + configFile, e); LOGGER.error("[!] Failed to load properties file from " + CONFIG_FILE, e);
} }
} }
/** /**
* Get the root rips directory. * Get the root rips directory.
* @return *
* Root directory to save rips to. * @return Root directory to save rips to.
* @throws IOException
*/ */
public static File getWorkingDirectory() { public static File getWorkingDirectory() {
String currentDir = "."; String currentDir = ".";
try { try {
currentDir = new File(".").getCanonicalPath() + File.separator + RIP_DIRECTORY + File.separator; currentDir = new File(".").getCanonicalPath() + File.separator + RIP_DIRECTORY + File.separator;
} catch (IOException e) { } catch (IOException e) {
logger.error("Error while finding working dir: ", e); LOGGER.error("Error while finding working dir: ", e);
} }
if (config != null) { if (config != null) {
currentDir = getConfigString("rips.directory", currentDir); currentDir = getConfigString("rips.directory", currentDir);
} }
File workingDir = new File(currentDir); File workingDir = new File(currentDir);
if (!workingDir.exists()) { if (!workingDir.exists()) {
workingDir.mkdirs(); workingDir.mkdirs();
@ -100,36 +116,46 @@ public class Utils {
} }
public static String[] getConfigStringArray(String key) { public static String[] getConfigStringArray(String key) {
String[] s = config.getStringArray(key); String[] configStringArray = config.getStringArray(key);
if (s.length == 0) {
return null; return configStringArray.length == 0 ? null : configStringArray;
} else {
return s;
}
} }
public static int getConfigInteger(String key, int defaultValue) { public static int getConfigInteger(String key, int defaultValue) {
return config.getInt(key, defaultValue); return config.getInt(key, defaultValue);
} }
public static boolean getConfigBoolean(String key, boolean defaultValue) { public static boolean getConfigBoolean(String key, boolean defaultValue) {
return config.getBoolean(key, defaultValue); return config.getBoolean(key, defaultValue);
} }
public static List<String> getConfigList(String key) { public static List<String> getConfigList(String key) {
List<String> result = new ArrayList<>(); List<String> result = new ArrayList<>();
for (Object obj : config.getList(key, new ArrayList<String>())) { for (Object obj : config.getList(key, new ArrayList<String>())) {
if (obj instanceof String) { if (obj instanceof String) {
result.add( (String) obj); result.add((String) obj);
} }
} }
return result; return result;
} }
public static void setConfigBoolean(String key, boolean value) { config.setProperty(key, value); }
public static void setConfigString(String key, String value) { config.setProperty(key, value); } public static void setConfigBoolean(String key, boolean value) {
public static void setConfigInteger(String key, int value) { config.setProperty(key, value); } config.setProperty(key, value);
}
public static void setConfigString(String key, String value) {
config.setProperty(key, value);
}
public static void setConfigInteger(String key, int value) {
config.setProperty(key, value);
}
public static void setConfigList(String key, List<Object> list) { public static void setConfigList(String key, List<Object> list) {
config.clearProperty(key); config.clearProperty(key);
config.addProperty(key, list); config.addProperty(key, list);
} }
public static void setConfigList(String key, Enumeration<Object> enumeration) { public static void setConfigList(String key, Enumeration<Object> enumeration) {
config.clearProperty(key); config.clearProperty(key);
List<Object> list = new ArrayList<>(); List<Object> list = new ArrayList<>();
@ -142,9 +168,9 @@ public class Utils {
public static void saveConfig() { public static void saveConfig() {
try { try {
config.save(getConfigFilePath()); config.save(getConfigFilePath());
logger.info("Saved configuration to " + getConfigFilePath()); LOGGER.info("Saved configuration to " + getConfigFilePath());
} catch (ConfigurationException e) { } catch (ConfigurationException e) {
logger.error("Error while saving configuration: ", e); LOGGER.error("Error while saving configuration: ", e);
} }
} }
@ -176,7 +202,6 @@ public class Utils {
return System.getenv("LOCALAPPDATA") + File.separator + "ripme"; return System.getenv("LOCALAPPDATA") + File.separator + "ripme";
} }
/** /**
* Gets the directory of where the config file is stored on a UNIX machine. * Gets the directory of where the config file is stored on a UNIX machine.
*/ */
@ -197,13 +222,14 @@ public class Utils {
*/ */
private static boolean portableMode() { private static boolean portableMode() {
try { try {
File f = new File(new File(".").getCanonicalPath() + File.separator + configFile); File file = new File(new File(".").getCanonicalPath() + File.separator + CONFIG_FILE);
if(f.exists() && !f.isDirectory()) { if (file.exists() && !file.isDirectory()) {
return true; return true;
} }
} catch (IOException e) { } catch (IOException e) {
return false; return false;
} }
return false; return false;
} }
@ -229,6 +255,7 @@ public class Utils {
return "."; return ".";
} }
} }
/** /**
* Delete the url history file * Delete the url history file
*/ */
@ -248,26 +275,23 @@ public class Utils {
* Gets the path to the configuration file. * Gets the path to the configuration file.
*/ */
private static String getConfigFilePath() { private static String getConfigFilePath() {
return getConfigDir() + File.separator + configFile; return getConfigDir() + File.separator + CONFIG_FILE;
} }
/** /**
* Removes the current working directory (CWD) from a File. * Removes the current working directory (CWD) from a File.
* @param saveAs *
* The File path * @param saveAs The File path
* @return * @return saveAs in relation to the CWD
* saveAs in relation to the CWD
*/ */
public static String removeCWD(File saveAs) { public static String removeCWD(File saveAs) {
String prettySaveAs = saveAs.toString(); String prettySaveAs = saveAs.toString();
try { try {
prettySaveAs = saveAs.getCanonicalPath(); prettySaveAs = saveAs.getCanonicalPath();
String cwd = new File(".").getCanonicalPath() + File.separator; String cwd = new File(".").getCanonicalPath() + File.separator;
prettySaveAs = prettySaveAs.replace( prettySaveAs = prettySaveAs.replace(cwd, "." + File.separator);
cwd,
"." + File.separator);
} catch (Exception e) { } catch (Exception e) {
logger.error("Exception: ", e); LOGGER.error("Exception: ", e);
} }
return prettySaveAs; return prettySaveAs;
} }
@ -278,7 +302,6 @@ public class Utils {
* *
* @param url The URL to filter/strip * @param url The URL to filter/strip
* @param parameter The parameter to strip * @param parameter The parameter to strip
*
* @return The stripped URL * @return The stripped URL
*/ */
public static String stripURLParameter(String url, String parameter) { public static String stripURLParameter(String url, String parameter) {
@ -290,13 +313,13 @@ public class Utils {
} }
if (paramIndex > 0) { if (paramIndex > 0) {
int nextParam = url.indexOf("&", paramIndex+1); int nextParam = url.indexOf('&', paramIndex + 1);
if (nextParam != -1) { if (nextParam != -1) {
String c = "&"; String c = "&";
if (wasFirstParam) { if (wasFirstParam) {
c = "?"; c = "?";
} }
url = url.substring(0, paramIndex) + c + url.substring(nextParam+1, url.length()); url = url.substring(0, paramIndex) + c + url.substring(nextParam + 1, url.length());
} else { } else {
url = url.substring(0, paramIndex); url = url.substring(0, paramIndex);
} }
@ -307,10 +330,9 @@ public class Utils {
/** /**
* Removes the current working directory from a given filename * Removes the current working directory from a given filename
* @param file *
* Path to the file * @param file Path to the file
* @return * @return 'file' without the leading current working directory
* 'file' without the leading current working directory
*/ */
public static String removeCWD(String file) { public static String removeCWD(String file) {
return removeCWD(new File(file)); return removeCWD(new File(file));
@ -320,12 +342,11 @@ public class Utils {
* Get a list of all Classes within a package. * Get a list of all Classes within a package.
* Works with file system projects and jar files! * Works with file system projects and jar files!
* Borrowed from StackOverflow, but I don't have a link :[ * Borrowed from StackOverflow, but I don't have a link :[
* @param pkgname *
* The name of the package * @param pkgname The name of the package
* @return * @return List of classes within the package
* List of classes within the package
*/ */
public static ArrayList<Class<?>> getClassesForPackage(String pkgname) { public static List<Class<?>> getClassesForPackage(String pkgname) {
ArrayList<Class<?>> classes = new ArrayList<>(); ArrayList<Class<?>> classes = new ArrayList<>();
String relPath = pkgname.replace('.', '/'); String relPath = pkgname.replace('.', '/');
URL resource = ClassLoader.getSystemClassLoader().getResource(relPath); URL resource = ClassLoader.getSystemClassLoader().getResource(relPath);
@ -334,7 +355,8 @@ public class Utils {
} }
String fullPath = resource.getFile(); String fullPath = resource.getFile();
File directory = null; File directory;
try { try {
directory = new File(resource.toURI()); directory = new File(resource.toURI());
} catch (URISyntaxException e) { } catch (URISyntaxException e) {
@ -356,8 +378,7 @@ public class Utils {
} }
} }
} }
} } else {
else {
// Load from JAR // Load from JAR
try { try {
String jarPath = fullPath String jarPath = fullPath
@ -376,7 +397,7 @@ public class Utils {
try { try {
classes.add(Class.forName(className)); classes.add(Class.forName(className));
} catch (ClassNotFoundException e) { } catch (ClassNotFoundException e) {
logger.error("ClassNotFoundException loading " + className); LOGGER.error("ClassNotFoundException loading " + className);
jarFile.close(); // Resource leak fix? jarFile.close(); // Resource leak fix?
throw new RuntimeException("ClassNotFoundException loading " + className); throw new RuntimeException("ClassNotFoundException loading " + className);
} }
@ -384,20 +405,18 @@ public class Utils {
} }
jarFile.close(); // Eclipse said not closing it would have a resource leak jarFile.close(); // Eclipse said not closing it would have a resource leak
} catch (IOException e) { } catch (IOException e) {
logger.error("Error while loading jar file:", e); LOGGER.error("Error while loading jar file:", e);
throw new RuntimeException(pkgname + " (" + directory + ") does not appear to be a valid package", e); throw new RuntimeException(pkgname + " (" + directory + ") does not appear to be a valid package", e);
} }
} }
return classes; return classes;
} }
private static final int SHORTENED_PATH_LENGTH = 12;
/** /**
* Shortens the path to a file * Shortens the path to a file
* @param path *
* String of the path to the file * @param path String of the path to the file
* @return * @return The simplified path to the file.
* The simplified path to the file.
*/ */
public static String shortenPath(String path) { public static String shortenPath(String path) {
return shortenPath(new File(path)); return shortenPath(new File(path));
@ -405,10 +424,9 @@ public class Utils {
/** /**
* Shortens the path to a file * Shortens the path to a file
* @param file *
* File object that you want the shortened path of. * @param file File object that you want the shortened path of.
* @return * @return The simplified path to the file.
* The simplified path to the file.
*/ */
public static String shortenPath(File file) { public static String shortenPath(File file) {
String path = removeCWD(file); String path = removeCWD(file);
@ -422,10 +440,9 @@ public class Utils {
/** /**
* Sanitizes a string so that a filesystem can handle it * Sanitizes a string so that a filesystem can handle it
* @param text *
* The text to be sanitized. * @param text The text to be sanitized.
* @return * @return The sanitized text.
* The sanitized text.
*/ */
public static String filesystemSanitized(String text) { public static String filesystemSanitized(String text) {
text = text.replaceAll("[^a-zA-Z0-9.-]", "_"); text = text.replaceAll("[^a-zA-Z0-9.-]", "_");
@ -451,7 +468,7 @@ public class Utils {
public static String getOriginalDirectory(String path) { public static String getOriginalDirectory(String path) {
int index; int index;
if(isUnix() || isMacOS()) { if (isUnix() || isMacOS()) {
index = path.lastIndexOf('/'); index = path.lastIndexOf('/');
} else { } else {
// current OS is windows - nothing to do here // current OS is windows - nothing to do here
@ -459,17 +476,17 @@ public class Utils {
} }
String original = path; // needs to be checked if lowercase exists String original = path; // needs to be checked if lowercase exists
String lastPart = original.substring(index+1).toLowerCase(); // setting lowercase to check if it exists String lastPart = original.substring(index + 1).toLowerCase(); // setting lowercase to check if it exists
// Get a List of all Directories and check its lowercase // Get a List of all Directories and check its lowercase
// if file exists return it // if file exists return it
File f = new File(path.substring(0, index)); File file = new File(path.substring(0, index));
ArrayList<String> names = new ArrayList<String>(Arrays.asList(f.list())); ArrayList<String> names = new ArrayList<>(Arrays.asList(file.list()));
for (String s : names) { for (String name : names) {
if(s.toLowerCase().equals(lastPart)) { if (name.toLowerCase().equals(lastPart)) {
// Building Path of existing file // Building Path of existing file
return path.substring(0, index) + File.separator + s; return path.substring(0, index) + File.separator + name;
} }
} }
@ -478,14 +495,13 @@ public class Utils {
/** /**
* Converts an integer into a human readable string * Converts an integer into a human readable string
* @param bytes *
* Non-human readable integer. * @param bytes Non-human readable integer.
* @return * @return Human readable interpretation of a byte.
* Human readable interpretation of a byte.
*/ */
public static String bytesToHumanReadable(int bytes) { public static String bytesToHumanReadable(int bytes) {
float fbytes = (float) bytes; float fbytes = (float) bytes;
String[] mags = new String[] {"", "K", "M", "G", "T"}; String[] mags = new String[]{"", "K", "M", "G", "T"};
int magIndex = 0; int magIndex = 0;
while (fbytes >= 1024) { while (fbytes >= 1024) {
fbytes /= 1024; fbytes /= 1024;
@ -496,6 +512,7 @@ public class Utils {
/** /**
* Gets and returns a list of all the album rippers present in the "com.rarchives.ripme.ripper.rippers" package. * Gets and returns a list of all the album rippers present in the "com.rarchives.ripme.ripper.rippers" package.
*
* @return List<String> of all album rippers present. * @return List<String> of all album rippers present.
*/ */
public static List<String> getListOfAlbumRippers() throws Exception { public static List<String> getListOfAlbumRippers() throws Exception {
@ -508,6 +525,7 @@ public class Utils {
/** /**
* Gets and returns a list of all video rippers present in the "com.rarchives.rime.rippers.video" package * Gets and returns a list of all video rippers present in the "com.rarchives.rime.rippers.video" package
*
* @return List<String> of all the video rippers. * @return List<String> of all the video rippers.
*/ */
public static List<String> getListOfVideoRippers() throws Exception { public static List<String> getListOfVideoRippers() throws Exception {
@ -520,8 +538,8 @@ public class Utils {
/** /**
* Plays a sound from a file. * Plays a sound from a file.
* @param filename *
* Path to the sound file * @param filename Path to the sound file
*/ */
public static void playSound(String filename) { public static void playSound(String filename) {
URL resource = ClassLoader.getSystemClassLoader().getResource(filename); URL resource = ClassLoader.getSystemClassLoader().getResource(filename);
@ -535,7 +553,7 @@ public class Utils {
clip.open(AudioSystem.getAudioInputStream(resource)); clip.open(AudioSystem.getAudioInputStream(resource));
clip.start(); clip.start();
} catch (Exception e) { } catch (Exception e) {
logger.error("Failed to play sound " + filename, e); LOGGER.error("Failed to play sound " + filename, e);
} }
} }
@ -544,27 +562,25 @@ public class Utils {
*/ */
public static void configureLogger() { public static void configureLogger() {
LogManager.shutdown(); LogManager.shutdown();
String logFile; String logFile = getConfigBoolean("log.save", false) ? "log4j.file.properties" : "log4j.properties";
if (getConfigBoolean("log.save", false)) {
logFile = "log4j.file.properties"; try (InputStream stream = Utils.class.getClassLoader().getResourceAsStream(logFile)) {
}
else {
logFile = "log4j.properties";
}
InputStream stream = Utils.class.getClassLoader().getResourceAsStream(logFile);
if (stream == null) { if (stream == null) {
PropertyConfigurator.configure("src/main/resources/" + logFile); PropertyConfigurator.configure("src/main/resources/" + logFile);
} else { } else {
PropertyConfigurator.configure(stream); PropertyConfigurator.configure(stream);
} }
logger.info("Loaded " + logFile);
try { LOGGER.info("Loaded " + logFile);
stream.close(); } catch (IOException e) {
} catch (IOException e) { } LOGGER.error(e.getMessage(), e);
}
} }
/** /**
* Gets list of strings between two strings. * Gets list of strings between two strings.
*
* @param fullText Text to retrieve from. * @param fullText Text to retrieve from.
* @param start String that precedes the desired text * @param start String that precedes the desired text
* @param finish String that follows the desired text * @param finish String that follows the desired text
@ -572,29 +588,29 @@ public class Utils {
*/ */
public static List<String> between(String fullText, String start, String finish) { public static List<String> between(String fullText, String start, String finish) {
List<String> result = new ArrayList<>(); List<String> result = new ArrayList<>();
int i, j; int i = fullText.indexOf(start);
i = fullText.indexOf(start);
while (i >= 0) { while (i >= 0) {
i += start.length(); i += start.length();
j = fullText.indexOf(finish, i); int j = fullText.indexOf(finish, i);
if (j < 0) { if (j < 0) {
break; break;
} }
result.add(fullText.substring(i, j)); result.add(fullText.substring(i, j));
i = fullText.indexOf(start, j + finish.length()); i = fullText.indexOf(start, j + finish.length());
} }
return result; return result;
} }
/** /**
* Parses an URL query * Parses an URL query
* *
* @param query * @param query The query part of an URL
* The query part of an URL
* @return The map of all query parameters * @return The map of all query parameters
*/ */
public static Map<String,String> parseUrlQuery(String query) { public static Map<String, String> parseUrlQuery(String query) {
Map<String,String> res = new HashMap<>(); Map<String, String> res = new HashMap<>();
if (query.equals("")) { if (query.equals("")) {
return res; return res;
@ -622,10 +638,8 @@ public class Utils {
/** /**
* Parses an URL query and returns the requested parameter's value * Parses an URL query and returns the requested parameter's value
* *
* @param query * @param query The query part of an URL
* The query part of an URL * @param key The key whose value is requested
* @param key
* The key whose value is requested
* @return The associated value or null if key wasn't found * @return The associated value or null if key wasn't found
*/ */
public static String parseUrlQuery(String query, String key) { public static String parseUrlQuery(String query, String key) {
@ -655,18 +669,13 @@ public class Utils {
return null; return null;
} }
private static HashMap<String, HashMap<String, String>> cookieCache;
static {
cookieCache = new HashMap<String, HashMap<String, String>>();
}
/** /**
* Gets all the cookies from a certain host * Gets all the cookies from a certain host
*/ */
public static Map<String, String> getCookies(String host) { public static Map<String, String> getCookies(String host) {
HashMap<String, String> domainCookies = cookieCache.get(host); HashMap<String, String> domainCookies = cookieCache.get(host);
if (domainCookies == null) { if (domainCookies == null) {
domainCookies = new HashMap<String, String>(); domainCookies = new HashMap<>();
String cookiesConfig = getConfigString("cookies." + host, ""); String cookiesConfig = getConfigString("cookies." + host, "");
for (String pair : cookiesConfig.split(" ")) { for (String pair : cookiesConfig.split(" ")) {
pair = pair.trim(); pair = pair.trim();
@ -690,20 +699,68 @@ public class Utils {
if (langSelect == null) { if (langSelect == null) {
if (!getConfigString("lang", "").equals("")) { if (!getConfigString("lang", "").equals("")) {
String[] langCode = getConfigString("lang", "").split("_"); String[] langCode = getConfigString("lang", "").split("_");
logger.info("Setting locale to " + getConfigString("lang", "")); LOGGER.info("Setting locale to " + getConfigString("lang", ""));
return ResourceBundle.getBundle("LabelsBundle", new Locale(langCode[0], langCode[1]), new UTF8Control()); return ResourceBundle.getBundle("LabelsBundle", new Locale(langCode[0], langCode[1]), new UTF8Control());
} }
} else { } else {
String[] langCode = langSelect.split("_"); String[] langCode = langSelect.split("_");
logger.info("Setting locale to " + langSelect); LOGGER.info("Setting locale to " + langSelect);
return ResourceBundle.getBundle("LabelsBundle", new Locale(langCode[0], langCode[1]), new UTF8Control()); return ResourceBundle.getBundle("LabelsBundle", new Locale(langCode[0], langCode[1]), new UTF8Control());
} }
try { try {
logger.info("Setting locale to default"); LOGGER.info("Setting locale to default");
return ResourceBundle.getBundle("LabelsBundle", Locale.getDefault(), new UTF8Control()); return ResourceBundle.getBundle("LabelsBundle", Locale.getDefault(), new UTF8Control());
} catch (MissingResourceException e) { } catch (MissingResourceException e) {
logger.info("Setting locale to root"); LOGGER.info("Setting locale to root");
return ResourceBundle.getBundle("LabelsBundle", Locale.ROOT); return ResourceBundle.getBundle("LabelsBundle", Locale.ROOT);
} }
} }
/**
* Formats and reuturns the status text for rippers using the byte progress bar
*
* @param completionPercentage An int between 0 and 100 which repersents how close the download is to complete
* @param bytesCompleted How many bytes have been downloaded
* @param bytesTotal The total size of the file that is being downloaded
* @return Returns the formatted status text for rippers using the byte progress bar
*/
public static String getByteStatusText(int completionPercentage, int bytesCompleted, int bytesTotal) {
return String.valueOf(completionPercentage) +
"% - " +
Utils.bytesToHumanReadable(bytesCompleted) +
" / " +
Utils.bytesToHumanReadable(bytesTotal);
}
public static String getEXTFromMagic(byte[] magic) {
if (Arrays.equals(magic, new byte[]{-1, -40, -1, -37, 0, 0, 0, 0})) {
return "jpeg";
} else {
LOGGER.info("Unknown magic number " + Arrays.toString(magic));
}
return null;
}
// Checks if a file exists ignoring it's extension.
// Code from: https://stackoverflow.com/a/17698068
public static boolean fuzzyExists(File folder, String fileName) {
if (!folder.exists()) {
return false;
}
File[] listOfFiles = folder.listFiles();
if (listOfFiles == null) {
return false;
}
for (File file : listOfFiles) {
if (file.isFile()) {
String[] filename = file.getName().split("\\.(?=[^\\.]+$)"); //split filename from it's extension
if(filename[0].equalsIgnoreCase(fileName)) {
return true;
}
}
}
return false;
}
} }

37
src/main/resources/LabelsBundle_it_IT.properties Normal file
View File

@ -0,0 +1,37 @@
Log = Log
History = Cronologia
created = creato
modified = modificato
Queue = Coda
Configuration = Configurazione
# Keys for the Configuration menu
current.version = Versione Corrente
check.for.updates = Controlla Aggiornamenti
auto.update = Aggiornamento automatico?
max.download.threads = Thread massini
timeout.mill = Timeout (in millisecondi):
retry.download.count = Tentativi di download
overwrite.existing.files = Sovrascrivi file esistenti?
sound.when.rip.completes = Suono al terminie
preserve.order = Conserva ordine
save.logs = Salva log
notification.when.rip.starts = Notifica inizio
save.urls.only = Salva solo URL
save.album.titles = Salva titoli album
autorip.from.clipboard = Scarica da appunti
save.descriptions = Salva descrizioni
prefer.mp4.over.gif = Preferisci MP4 a GIF
restore.window.position = Ripristina posizione della finestra
remember.url.history = Riscorda la cronologia degli URL
loading.history.from = Carica cronologia da
# Misc UI keys
loading.history.from.configuration = Caricamento cronologia da configurazione
interrupted.while.waiting.to.rip.next.album = Interrotto mentre scaricavo album successivo
inactive = Inattivo
re-rip.checked = Re-rip selezionato
remove = Rimuovi
clear = Pulisci

37
src/main/resources/LabelsBundle_nl_NL.properties Normal file
View File

@ -0,0 +1,37 @@
Log = Logboek
History = Geschiedenis
created = gemaakt
modified = aangepast
Queue = Wachtrij
Configuration = Configuratie
# Keys for the Configuration menu
current.version = Huidige versie
check.for.updates = Controleer op updates
auto.update = Auto-update?
max.download.threads = Maximale downloadthreads
timeout.mill = Timeout (in milliseconden):
retry.download.count = Aantal keren opnieuw proberen te downloaden
overwrite.existing.files = Bestaande bestanden overschrijven?
sound.when.rip.completes = Geluid wanneer rip klaar is
preserve.order = Volgorde behouden
save.logs = Logbestanden opslaan
notification.when.rip.starts = Notificatie wanneer rip start
save.urls.only = Alleen URLs opslaan
save.album.titles = Album titels opslaan
autorip.from.clipboard = Rip automatisch van klembord
save.descriptions = Beschrijvingen opslaan
prefer.mp4.over.gif = Geef de voorkeur aan MP4 over GIF
restore.window.position = Vensterpositie herstellen
remember.url.history = URL geschiedenis onthouden
loading.history.from = Geschiedenis laden van
# Misc UI keys
loading.history.from.configuration = Geschiedenis laden van configuratie
interrupted.while.waiting.to.rip.next.album = Onderbroken tijdens het wachten om volgend album te rippen
inactive = Inactief
re-rip.checked = Re-rip Gecheckt
remove = Verwijderen
clear = Opruimen

37
src/main/resources/LabelsBundle_ru_RU.properties Normal file
View File

@ -0,0 +1,37 @@
Log = Лог
History = История
created = создано
modified = изменено
Queue = Очередь
Configuration = Настройки
# Keys for the Configuration menu
current.version = Текущая версия
check.for.updates = Проверить обновления
auto.update = Автообновление?
max.download.threads = Максимальное число потоков:
timeout.mill = Задержка (в миллисекундах):
retry.download.count = Число повторов
overwrite.existing.files = Перезаписать существующие файлы?
sound.when.rip.completes = Звук при завершении
preserve.order = Сохранять порядок
save.logs = Сохранять логи
notification.when.rip.starts = Уведомление при запуске
save.urls.only = Сохранять только ссылки
save.album.titles = Сохранять названия альбомов
autorip.from.clipboard = Автоскачивание из буфера
save.descriptions = Сохранять описания
prefer.mp4.over.gif = Предпочесть MP4 вместо GIF
restore.window.position = Восстановить положение окна
remember.url.history = Запоминать историю запросов
loading.history.from = Загрузить историю из
# Misc UI keys
loading.history.from.configuration = Загрузить историю из настроек
interrupted.while.waiting.to.rip.next.album = Прервано во время ожидания скачивания следующего альбома
inactive = Неактивно
re-rip.checked = Перекачать выбранное
remove = Удалить
clear = Очистить

45
src/test/java/com/rarchives/ripme/tst/UtilsTest.java Normal file
View File

@ -0,0 +1,45 @@
package com.rarchives.ripme.tst;
import junit.framework.TestCase;
import com.rarchives.ripme.utils.Utils;
import java.util.ArrayList;
import java.util.Arrays;
public class UtilsTest extends TestCase {
public void testGetEXTFromMagic() {
assertEquals("jpeg", Utils.getEXTFromMagic(new byte[]{-1, -40, -1, -37, 0, 0, 0, 0}));
}
public void testStripURLParameter() {
assertEquals("http://example.tld/image.ext",
Utils.stripURLParameter("http://example.tld/image.ext?param", "param"));
}
public void testShortenPath() {
String path = "/test/test/test/test/test/test/test/test/";
assertEquals("/test/test1", Utils.shortenPath("/test/test1"));
assertEquals("/test/test/t...st/test/test", Utils.shortenPath(path));
}
public void testBytesToHumanReadable() {
assertEquals("10.00iB", Utils.bytesToHumanReadable(10));
assertEquals("1.00KiB", Utils.bytesToHumanReadable(1024));
assertEquals("1.00MiB", Utils.bytesToHumanReadable(1024 * 1024));
assertEquals("1.00GiB", Utils.bytesToHumanReadable(1024 * 1024 * 1024));
}
public void testGetListOfAlbumRippers() throws Exception{
assert(!Utils.getListOfAlbumRippers().isEmpty());
}
public void testGetByteStatusText() {
assertEquals("5% - 500.00iB / 97.66KiB", Utils.getByteStatusText(5, 500, 100000));
}
public void testBetween() {
assertEquals(Arrays.asList(" is a "), Utils.between("This is a test", "This", "test"));
}
}

33
src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java Normal file
View File

@ -0,0 +1,33 @@
package com.rarchives.ripme.tst.ripper.rippers;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import com.rarchives.ripme.ripper.rippers.ArtStationRipper;
public class ArtStationRipperTest extends RippersTest {
public void testArtStationProjects() throws IOException {
List<URL> contentURLs = new ArrayList<>();
contentURLs.add(new URL("https://www.artstation.com/artwork/the-dwarf-mortar"));
contentURLs.add(new URL("https://www.artstation.com/artwork/K36GR"));
contentURLs.add(new URL("http://artstation.com/artwork/5JJQw"));
for (URL url : contentURLs) {
ArtStationRipper ripper = new ArtStationRipper(url);
testRipper(ripper);
}
}
public void testArtStationUserProfiles() throws IOException {
List<URL> contentURLs = new ArrayList<>();
contentURLs.add(new URL("https://www.artstation.com/heitoramatsu"));
contentURLs.add(new URL("https://artstation.com/kuvshinov_ilya"));
contentURLs.add(new URL("http://artstation.com/givemeapiggy"));
for (URL url : contentURLs) {
ArtStationRipper ripper = new ArtStationRipper(url);
testRipper(ripper);
}
}
}

9
src/test/java/com/rarchives/ripme/tst/ripper/rippers/BcfakesRipperTest.java
View File

@ -6,8 +6,9 @@ import java.net.URL;
import com.rarchives.ripme.ripper.rippers.BcfakesRipper; import com.rarchives.ripme.ripper.rippers.BcfakesRipper;
public class BcfakesRipperTest extends RippersTest { public class BcfakesRipperTest extends RippersTest {
public void testRip() throws IOException { // 21/06/2018 This test was disbaled as the site has experienced notable downtime
BcfakesRipper ripper = new BcfakesRipper(new URL("http://www.bcfakes.com/celebritylist/olivia-wilde/")); // public void testRip() throws IOException {
testRipper(ripper); // BcfakesRipper ripper = new BcfakesRipper(new URL("http://www.bcfakes.com/celebritylist/olivia-wilde/"));
} // testRipper(ripper);
// }
} }

5
src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java
View File

@ -10,4 +10,9 @@ public class CheveretoRipperTest extends RippersTest {
CheveretoRipper ripper = new CheveretoRipper(new URL("http://tag-fox.com/album/Thjb")); CheveretoRipper ripper = new CheveretoRipper(new URL("http://tag-fox.com/album/Thjb"));
testRipper(ripper); testRipper(ripper);
} }
public void testSubdirAlbum() throws IOException {
CheveretoRipper ripper = new CheveretoRipper(new URL("https://kenzato.uk/booru/album/TnEc"));
testRipper(ripper);
}
} }

30
src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java
View File

@ -2,24 +2,48 @@ package com.rarchives.ripme.tst.ripper.rippers;
import java.io.IOException; import java.io.IOException;
import java.net.URL; import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import com.rarchives.ripme.ripper.rippers.DeviantartRipper; import com.rarchives.ripme.ripper.rippers.DeviantartRipper;
import com.rarchives.ripme.utils.Http;
import org.jsoup.nodes.Document;
public class DeviantartRipperTest extends RippersTest { public class DeviantartRipperTest extends RippersTest {
public void testDeviantartAlbum() throws IOException { public void testDeviantartAlbum() throws IOException {
DeviantartRipper ripper = new DeviantartRipper(new URL("http://airgee.deviantart.com/gallery/")); DeviantartRipper ripper = new DeviantartRipper(new URL("https://www.deviantart.com/airgee/gallery/"));
testRipper(ripper); testRipper(ripper);
} }
public void testDeviantartNSFWAlbum() throws IOException { public void testDeviantartNSFWAlbum() throws IOException {
// NSFW gallery // NSFW gallery
DeviantartRipper ripper = new DeviantartRipper(new URL("http://faterkcx.deviantart.com/gallery/")); DeviantartRipper ripper = new DeviantartRipper(new URL("https://www.deviantart.com/faterkcx/gallery/"));
testRipper(ripper); testRipper(ripper);
} }
public void testGetGID() throws IOException { public void testGetGID() throws IOException {
URL url = new URL("http://airgee.deviantart.com/gallery/"); URL url = new URL("https://www.deviantart.com/airgee/gallery/");
DeviantartRipper ripper = new DeviantartRipper(url); DeviantartRipper ripper = new DeviantartRipper(url);
assertEquals("airgee", ripper.getGID(url)); assertEquals("airgee", ripper.getGID(url));
} }
public void testGetGalleryIDAndUsername() throws IOException {
URL url = new URL("https://www.deviantart.com/airgee/gallery/");
DeviantartRipper ripper = new DeviantartRipper(url);
Document doc = Http.url(url).get();
assertEquals("airgee", ripper.getUsername(doc));
assertEquals("714589", ripper.getGalleryID(doc));
}
public void testSanitizeURL() throws IOException {
List<URL> urls = new ArrayList<URL>();
urls.add(new URL("https://www.deviantart.com/airgee/"));
urls.add(new URL("https://www.deviantart.com/airgee"));
urls.add(new URL("https://www.deviantart.com/airgee/gallery/"));
for (URL url : urls) {
DeviantartRipper ripper = new DeviantartRipper(url);
assertEquals("https://www.deviantart.com/airgee/gallery/", ripper.sanitizeURL(url).toExternalForm());
}
}
} }

40
src/test/java/com/rarchives/ripme/tst/ripper/rippers/EromeRipperTest.java Normal file
View File

@ -0,0 +1,40 @@
package com.rarchives.ripme.tst.ripper.rippers;
import java.io.IOException;
import java.net.URL;
import com.rarchives.ripme.ripper.rippers.EromeRipper;
public class EromeRipperTest extends RippersTest {
public void testGetGIDProfilePage() throws IOException {
URL url = new URL("https://www.erome.com/Jay-Jenna");
EromeRipper ripper = new EromeRipper(url);
assertEquals("Jay-Jenna", ripper.getGID(url));
}
public void testGetGIDAlbum() throws IOException {
URL url = new URL("https://www.erome.com/a/KbDAM1XT");
EromeRipper ripper = new EromeRipper(url);
assertEquals("KbDAM1XT", ripper.getGID(url));
}
public void testGetAlbumsToQueue() throws IOException {
URL url = new URL("https://www.erome.com/Jay-Jenna");
EromeRipper ripper = new EromeRipper(url);
assert(2 >= ripper.getAlbumsToQueue(ripper.getFirstPage()).size());
}
public void testPageContainsAlbums() throws IOException {
URL url = new URL("https://www.erome.com/Jay-Jenna");
EromeRipper ripper = new EromeRipper(url);
assert(ripper.pageContainsAlbums(url));
assert(!ripper.pageContainsAlbums(new URL("https://www.erome.com/a/KbDAM1XT")));
}
public void testRip() throws IOException {
URL url = new URL("https://www.erome.com/a/4FqeUxil");
EromeRipper ripper = new EromeRipper(url);
testRipper(ripper);
}
}

8
src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java
View File

@ -16,4 +16,12 @@ public class FuraffinityRipperTest extends RippersTest {
FuraffinityRipper ripper = new FuraffinityRipper(url); FuraffinityRipper ripper = new FuraffinityRipper(url);
assertEquals("mustardgas", ripper.getGID(url)); assertEquals("mustardgas", ripper.getGID(url));
} }
public void testLogin() throws IOException {
URL url = new URL("https://www.furaffinity.net/gallery/mustardgas/");
FuraffinityRipper ripper = new FuraffinityRipper(url);
// Check if the first page contain the username of ripmes shared account
Boolean containsUsername = ripper.getFirstPage().html().contains("ripmethrowaway");
assert containsUsername;
}
} }

2
src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java
View File

@ -1,6 +1,6 @@
package com.rarchives.ripme.tst.ripper.rippers; package com.rarchives.ripme.tst.ripper.rippers;
import com.rarchives.ripme.ripper.rippers.video.GfycatRipper; import com.rarchives.ripme.ripper.rippers.GfycatRipper;
import java.io.IOException; import java.io.IOException;
import java.net.URL; import java.net.URL;

13
src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagearnRipperTest.java Normal file
View File

@ -0,0 +1,13 @@
package com.rarchives.ripme.tst.ripper.rippers;
import java.io.IOException;
import java.net.URL;
import com.rarchives.ripme.ripper.rippers.ImagearnRipper;
public class ImagearnRipperTest extends RippersTest {
public void testImagearnRip() throws IOException {
ImagearnRipper ripper = new ImagearnRipper(new URL("http://imagearn.com//gallery.php?id=578682"));
testRipper(ripper);
}
}

10
src/test/java/com/rarchives/ripme/tst/ripper/rippers/LoveromRipperTest.java Normal file
View File

@ -0,0 +1,10 @@
package com.rarchives.ripme.tst.ripper.rippers;
import com.rarchives.ripme.ripper.rippers.LoveromRipper;
import java.io.IOException;
import java.net.URL;
public class LoveromRipperTest extends RippersTest{
}

8
src/test/java/com/rarchives/ripme/tst/ripper/rippers/ManganeloRipperTest.java
View File

@ -7,7 +7,13 @@ import com.rarchives.ripme.ripper.rippers.ManganeloRipper;
public class ManganeloRipperTest extends RippersTest { public class ManganeloRipperTest extends RippersTest {
public void testRip() throws IOException { public void testRip() throws IOException {
ManganeloRipper ripper = new ManganeloRipper(new URL("http://manganelo.com/manga/black_clover")); ManganeloRipper ripper = new ManganeloRipper(new URL("https://manganelo.com/manga/demonic_housekeeper"));
testRipper(ripper); testRipper(ripper);
} }
public void testGetGID() throws IOException {
URL url = new URL("https://manganelo.com/manga/demonic_housekeeper");
ManganeloRipper ripper = new ManganeloRipper(url);
assertEquals("demonic_housekeeper", ripper.getGID(url));
}
} }

14
src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java
View File

@ -21,4 +21,18 @@ public class MyhentaicomicsRipperTest extends RippersTest {
// Test a tag // Test a tag
assertEquals("2409", ripper.getGID(new URL("http://myhentaicomics.com/index.php/tag/2409/"))); assertEquals("2409", ripper.getGID(new URL("http://myhentaicomics.com/index.php/tag/2409/")));
} }
public void testGetAlbumsToQueue() throws IOException {
URL url = new URL("https://myhentaicomics.com/index.php/tag/3167/");
MyhentaicomicsRipper ripper = new MyhentaicomicsRipper(url);
assertEquals(15, ripper.getAlbumsToQueue(ripper.getFirstPage()).size());
}
public void testPageContainsAlbums() throws IOException {
URL url = new URL("https://myhentaicomics.com/index.php/tag/3167/");
URL url2 = new URL("https://myhentaicomics.com/index.php/search?q=test");
MyhentaicomicsRipper ripper = new MyhentaicomicsRipper(url);
assertTrue(ripper.pageContainsAlbums(url));
assertTrue(ripper.pageContainsAlbums(url2));
}
} }

19
src/test/java/com/rarchives/ripme/tst/ripper/rippers/PicstatioRipperTest.java Normal file
View File

@ -0,0 +1,19 @@
package com.rarchives.ripme.tst.ripper.rippers;
import java.io.IOException;
import java.net.URL;
import com.rarchives.ripme.ripper.rippers.PicstatioRipper;
public class PicstatioRipperTest extends RippersTest {
public void testRip() throws IOException {
PicstatioRipper ripper = new PicstatioRipper(new URL("https://www.picstatio.com/aerial-view-wallpapers"));
testRipper(ripper);
}
public void testGID() throws IOException {
PicstatioRipper ripper = new PicstatioRipper(new URL("https://www.picstatio.com/aerial-view-wallpapers"));
assertEquals("aerial-view-wallpapers", ripper.getGID(new URL("https://www.picstatio.com/aerial-view-wallpapers")));
}
}

1
src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java
View File

@ -5,7 +5,6 @@ import java.io.IOException;
import java.net.URL; import java.net.URL;
import com.rarchives.ripme.ripper.rippers.RedditRipper; import com.rarchives.ripme.ripper.rippers.RedditRipper;
import com.rarchives.ripme.ripper.rippers.video.GfycatRipper;
public class RedditRipperTest extends RippersTest { public class RedditRipperTest extends RippersTest {
// https://github.com/RipMeApp/ripme/issues/253 - Disabled tests: RedditRipperTest#testRedditSubreddit*Rip is flaky // https://github.com/RipMeApp/ripme/issues/253 - Disabled tests: RedditRipperTest#testRedditSubreddit*Rip is flaky

26
src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java
View File

@ -7,10 +7,8 @@ import java.util.List;
import com.rarchives.ripme.ripper.VideoRipper; import com.rarchives.ripme.ripper.VideoRipper;
import com.rarchives.ripme.ripper.rippers.video.PornhubRipper; import com.rarchives.ripme.ripper.rippers.video.PornhubRipper;
import com.rarchives.ripme.ripper.rippers.video.TwitchVideoRipper;
import com.rarchives.ripme.ripper.rippers.video.VineRipper;
import com.rarchives.ripme.ripper.rippers.video.XhamsterRipper; import com.rarchives.ripme.ripper.rippers.video.XhamsterRipper;
import com.rarchives.ripme.ripper.rippers.video.XvideosRipper; import com.rarchives.ripme.ripper.rippers.XvideosRipper;
import com.rarchives.ripme.ripper.rippers.video.YoupornRipper; import com.rarchives.ripme.ripper.rippers.video.YoupornRipper;
import com.rarchives.ripme.ripper.rippers.video.YuvutuRipper; import com.rarchives.ripme.ripper.rippers.video.YuvutuRipper;
@ -59,16 +57,6 @@ public class VideoRippersTest extends RippersTest {
} }
} }
public void testXvideosRipper() throws IOException {
List<URL> contentURLs = new ArrayList<>();
contentURLs.add(new URL("https://www.xvideos.com/video19719109/ziggy_star_ultra_hard_anal_pounding"));
contentURLs.add(new URL("https://www.xvideos.com/video23515878/dee_s_pool_toys"));
for (URL url : contentURLs) {
XvideosRipper ripper = new XvideosRipper(url);
videoTestHelper(ripper);
}
}
public void testPornhubRipper() throws IOException { public void testPornhubRipper() throws IOException {
List<URL> contentURLs = new ArrayList<>(); List<URL> contentURLs = new ArrayList<>();
contentURLs.add(new URL("https://www.pornhub.com/view_video.php?viewkey=ph5a329fa707269")); contentURLs.add(new URL("https://www.pornhub.com/view_video.php?viewkey=ph5a329fa707269"));
@ -78,18 +66,6 @@ public class VideoRippersTest extends RippersTest {
} }
} }
// https://github.com/RipMeApp/ripme/issues/186
/*
public void testVineRipper() throws IOException {
List<URL> contentURLs = new ArrayList<>();
contentURLs.add(new URL("https://vine.co/v/hiqQrP0eUZx"));
for (URL url : contentURLs) {
VineRipper ripper = new VineRipper(url);
videoTestHelper(ripper);
}
}
*/
public void testYoupornRipper() throws IOException { public void testYoupornRipper() throws IOException {
List<URL> contentURLs = new ArrayList<>(); List<URL> contentURLs = new ArrayList<>();
contentURLs.add(new URL("http://www.youporn.com/watch/7669155/mrs-li-amateur-69-orgasm/?from=categ")); contentURLs.add(new URL("http://www.youporn.com/watch/7669155/mrs-li-amateur-69-orgasm/?from=categ"));

5
src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java
View File

@ -17,6 +17,11 @@ public class XhamsterRipperTest extends RippersTest {
testRipper(ripper); testRipper(ripper);
} }
public void testBrazilianXhamster() throws IOException {
XhamsterRipper ripper = new XhamsterRipper(new URL("https://pt.xhamster.com/photos/gallery/silvana-7105696"));
testRipper(ripper);
}
public void testGetGID() throws IOException { public void testGetGID() throws IOException {
URL url = new URL("https://xhamster.com/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664"); URL url = new URL("https://xhamster.com/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664");
XhamsterRipper ripper = new XhamsterRipper(url); XhamsterRipper ripper = new XhamsterRipper(url);

16
src/test/java/com/rarchives/ripme/tst/ripper/rippers/XvideosRipperTest.java Normal file
View File

@ -0,0 +1,16 @@
package com.rarchives.ripme.tst.ripper.rippers;
import java.io.IOException;
import java.net.URL;
import com.rarchives.ripme.ripper.rippers.XvideosRipper;
import com.rarchives.ripme.tst.ripper.rippers.RippersTest;
public class XvideosRipperTest extends RippersTest {
public void testXhamsterAlbum1() throws IOException {
XvideosRipper ripper = new XvideosRipper(new URL("https://www.xvideos.com/video23515878/dee_s_pool_toys"));
testRipper(ripper);
}
}