Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added video ripper support, xvideos for now #18

Browse Source
This commit is contained in:
4pr0n 2014-04-19 22:41:11 -07:00
parent fce0ecd8bf
commit 9946777e81
29 changed files with 599 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@ -4,7 +4,7 @@
<groupId>com.rarchives.ripme</groupId>
<artifactId>ripme</artifactId>
<packaging>jar</packaging>
<version>1.0.19</version>
<version>1.0.20</version>
<name>ripme</name>
<url>http://rip.rarchives.com</url>
<properties>

133
src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
View File

@ -6,10 +6,7 @@ import java.lang.reflect.Constructor;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Observable;
import org.apache.log4j.Logger;
@ -23,7 +20,7 @@ public abstract class AbstractRipper
extends Observable
implements RipperInterface, Runnable {
private static final Logger logger = Logger.getLogger(AbstractRipper.class);
protected static final Logger logger = Logger.getLogger(AbstractRipper.class);
protected static final String USER_AGENT =
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:27.0) Gecko/20100101 Firefox/27.0";
@ -33,9 +30,6 @@ public abstract class AbstractRipper
protected DownloadThreadPool threadPool;
protected RipStatusHandler observer = null;
protected Map<URL, File> itemsPending = Collections.synchronizedMap(new HashMap<URL, File>());
protected Map<URL, File> itemsCompleted = Collections.synchronizedMap(new HashMap<URL, File>());
protected Map<URL, String> itemsErrored = Collections.synchronizedMap(new HashMap<URL, String>());
protected boolean completed = true;
public abstract void rip() throws IOException;
@ -110,17 +104,7 @@ public abstract class AbstractRipper
* @param saveAs
* Path of the local file to save the content to.
*/
public void addURLToDownload(URL url, File saveAs) {
if (itemsPending.containsKey(url)
|| itemsCompleted.containsKey(url)
|| itemsErrored.containsKey(url)) {
// Item is already downloaded/downloading, skip it.
logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs));
return;
}
itemsPending.put(url, saveAs);
threadPool.addThread(new DownloadFileThread(url, saveAs, this));
}
public abstract void addURLToDownload(URL url, File saveAs);
/**
* Queues file to be downloaded and saved. With options.
@ -192,67 +176,30 @@ public abstract class AbstractRipper
* @param saveAs
* Where the downloaded file is stored.
*/
public void downloadCompleted(URL url, File saveAs) {
if (observer == null) {
return;
}
try {
String path = Utils.removeCWD(saveAs);
RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path);
itemsPending.remove(url);
itemsCompleted.put(url, saveAs);
observer.update(this, msg);
checkIfComplete();
} catch (Exception e) {
logger.error("Exception while updating observer: ", e);
}
}
public abstract void downloadCompleted(URL url, File saveAs);
/**
* Notifies observers that a file could not be downloaded (includes a reason).
* @param url
* @param reason
*/
public void downloadErrored(URL url, String reason) {
if (observer == null) {
return;
}
itemsPending.remove(url);
itemsErrored.put(url, reason);
observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason));
checkIfComplete();
}
public abstract void downloadErrored(URL url, String reason);
/**
* Notify observers that a download could not be completed,
* but was not technically an "error".
* @param url
* @param message
*/
public void downloadProblem(URL url, String message) {
if (observer == null) {
return;
}
itemsPending.remove(url);
itemsErrored.put(url, message);
observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " : " + message));
checkIfComplete();
}
public abstract void downloadProblem(URL url, String message);
/**
* Notifies observers and updates state if all files have been ripped.
*/
private void checkIfComplete() {
protected void checkIfComplete() {
if (observer == null) {
return;
}
if (!completed && itemsPending.isEmpty()) {
if (!completed) {
completed = true;
logger.info(" Rip completed!");
@ -274,26 +221,7 @@ public abstract class AbstractRipper
return workingDir;
}
/**
* Sets directory to save all ripped files to.
* @param url
* URL to define how the workin directory should be saved.
*/
public void setWorkingDir(URL url) throws IOException {
String path = Utils.getWorkingDirectory().getCanonicalPath();
if (!path.endsWith(File.separator)) {
path += File.separator;
}
String title = getAlbumTitle(this.url);
title = Utils.filesystemSafe(title);
path += title + File.separator;
this.workingDir = new File(path);
if (!this.workingDir.exists()) {
logger.info("[+] Creating directory: " + Utils.removeCWD(this.workingDir));
this.workingDir.mkdirs();
}
logger.debug("Set working directory to: " + this.workingDir);
}
public abstract void setWorkingDir(URL url) throws IOException;
public String getAlbumTitle(URL url) throws MalformedURLException {
return getHost() + "_" + getGID(url);
@ -309,9 +237,17 @@ public abstract class AbstractRipper
* If no compatible rippers can be found.
*/
public static AbstractRipper getRipper(URL url) throws Exception {
for (Constructor<?> constructor : getRipperConstructors()) {
for (Constructor<?> constructor : getRipperConstructors("com.rarchives.ripme.ripper.rippers")) {
try {
AbstractRipper ripper = (AbstractRipper) constructor.newInstance(url);
AlbumRipper ripper = (AlbumRipper) constructor.newInstance(url);
return ripper;
} catch (Exception e) {
// Incompatible rippers *will* throw exceptions during instantiation.
}
}
for (Constructor<?> constructor : getRipperConstructors("com.rarchives.ripme.ripper.rippers.video")) {
try {
VideoRipper ripper = (VideoRipper) constructor.newInstance(url);
return ripper;
} catch (Exception e) {
// Incompatible rippers *will* throw exceptions during instantiation.
@ -325,9 +261,9 @@ public abstract class AbstractRipper
* List of constructors for all eligible Rippers.
* @throws Exception
*/
private static List<Constructor<?>> getRipperConstructors() throws Exception {
private static List<Constructor<?>> getRipperConstructors(String pkg) throws Exception {
List<Constructor<?>> constructors = new ArrayList<Constructor<?>>();
for (Class<?> clazz : Utils.getClassesForPackage("com.rarchives.ripme.ripper.rippers")) {
for (Class<?> clazz : Utils.getClassesForPackage(pkg)) {
if (AbstractRipper.class.isAssignableFrom(clazz)) {
constructors.add( (Constructor<?>) clazz.getConstructor(URL.class) );
}
@ -347,28 +283,9 @@ public abstract class AbstractRipper
observer.update(this, new RipStatusMessage(status, message));
}
/**
* @return
* Integer between 0 and 100 defining the progress of the album rip.
*/
public int getCompletionPercentage() {
double total = itemsPending.size() + itemsErrored.size() + itemsCompleted.size();
return (int) (100 * ( (total - itemsPending.size()) / total));
}
public abstract int getCompletionPercentage();
/**
* @return
* Human-readable information on the status of the current rip.
*/
public String getStatusText() {
StringBuilder sb = new StringBuilder();
sb.append(getCompletionPercentage())
.append("% ")
.append("- Pending: " ).append(itemsPending.size())
.append(", Completed: ").append(itemsCompleted.size())
.append(", Errored: " ).append(itemsErrored.size());
return sb.toString();
}
public abstract String getStatusText();
/**
* Rips the album when the thread is invoked.
@ -382,4 +299,10 @@ public abstract class AbstractRipper
}
}
public void setBytesTotal(int bytes) {
// Do nothing
}
public void setBytesCompleted(int bytes) {
// Do nothing
}
}

146
src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java Normal file
View File

@ -0,0 +1,146 @@
package com.rarchives.ripme.ripper;
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import com.rarchives.ripme.ui.RipStatusMessage;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Utils;
public abstract class AlbumRipper extends AbstractRipper {
protected Map<URL, File> itemsPending = Collections.synchronizedMap(new HashMap<URL, File>());
protected Map<URL, File> itemsCompleted = Collections.synchronizedMap(new HashMap<URL, File>());
protected Map<URL, String> itemsErrored = Collections.synchronizedMap(new HashMap<URL, String>());
public AlbumRipper(URL url) throws IOException {
super(url);
}
public abstract boolean canRip(URL url);
public abstract URL sanitizeURL(URL url) throws MalformedURLException;
public abstract void rip() throws IOException;
public abstract String getHost();
public abstract String getGID(URL url) throws MalformedURLException;
@Override
public void addURLToDownload(URL url, File saveAs) {
if (itemsPending.containsKey(url)
|| itemsCompleted.containsKey(url)
|| itemsErrored.containsKey(url)) {
// Item is already downloaded/downloading, skip it.
logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs));
return;
}
itemsPending.put(url, saveAs);
threadPool.addThread(new DownloadFileThread(url, saveAs, this));
}
@Override
public void downloadCompleted(URL url, File saveAs) {
if (observer == null) {
return;
}
try {
String path = Utils.removeCWD(saveAs);
RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path);
itemsPending.remove(url);
itemsCompleted.put(url, saveAs);
observer.update(this, msg);
checkIfComplete();
} catch (Exception e) {
logger.error("Exception while updating observer: ", e);
}
}
@Override
public void downloadErrored(URL url, String reason) {
if (observer == null) {
return;
}
itemsPending.remove(url);
itemsErrored.put(url, reason);
observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason));
checkIfComplete();
}
@Override
public void downloadProblem(URL url, String message) {
if (observer == null) {
return;
}
itemsPending.remove(url);
itemsErrored.put(url, message);
observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " : " + message));
checkIfComplete();
}
/**
* Notifies observers and updates state if all files have been ripped.
*/
@Override
protected void checkIfComplete() {
if (observer == null) {
return;
}
if (itemsPending.isEmpty()) {
super.checkIfComplete();
}
}
/**
* Sets directory to save all ripped files to.
* @param url
* URL to define how the workin directory should be saved.
*/
@Override
public void setWorkingDir(URL url) throws IOException {
String path = Utils.getWorkingDirectory().getCanonicalPath();
if (!path.endsWith(File.separator)) {
path += File.separator;
}
String title = getAlbumTitle(this.url);
title = Utils.filesystemSafe(title);
path += title + File.separator;
this.workingDir = new File(path);
if (!this.workingDir.exists()) {
logger.info("[+] Creating directory: " + Utils.removeCWD(this.workingDir));
this.workingDir.mkdirs();
}
logger.debug("Set working directory to: " + this.workingDir);
}
/**
* @return
* Integer between 0 and 100 defining the progress of the album rip.
*/
@Override
public int getCompletionPercentage() {
double total = itemsPending.size() + itemsErrored.size() + itemsCompleted.size();
return (int) (100 * ( (total - itemsPending.size()) / total));
}
/**
* @return
* Human-readable information on the status of the current rip.
*/
@Override
public String getStatusText() {
StringBuilder sb = new StringBuilder();
sb.append(getCompletionPercentage())
.append("% ")
.append("- Pending: " ).append(itemsPending.size())
.append(", Completed: ").append(itemsCompleted.size())
.append(", Errored: " ).append(itemsErrored.size());
return sb.toString();
}
}

126
src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java Normal file
View File

@ -0,0 +1,126 @@
package com.rarchives.ripme.ripper;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import org.apache.log4j.Logger;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Utils;
/**
* Thread for downloading files.
* Includes retry logic, observer notifications, and other goodies.
*/
public class DownloadVideoThread extends Thread {
private static final Logger logger = Logger.getLogger(DownloadVideoThread.class);
private URL url;
private File saveAs;
private String prettySaveAs;
private AbstractRipper observer;
private int retries;
public DownloadVideoThread(URL url, File saveAs, AbstractRipper observer) {
super();
this.url = url;
this.saveAs = saveAs;
this.prettySaveAs = Utils.removeCWD(saveAs);
this.observer = observer;
this.retries = Utils.getConfigInteger("download.retries", 1);
}
/**
* Attempts to download the file. Retries as needed.
* Notifies observers upon completion/error/warn.
*/
public void run() {
try {
observer.stopCheck();
} catch (IOException e) {
observer.downloadErrored(url, "Download interrupted");
return;
}
if (saveAs.exists()) {
if (Utils.getConfigBoolean("file.overwrite", false)) {
logger.info("[!] Deleting existing file" + prettySaveAs);
saveAs.delete();
} else {
logger.info("[!] Skipping " + url + " -- file already exists: " + prettySaveAs);
observer.downloadProblem(url, "File already exists: " + prettySaveAs);
return;
}
}
int bytesTotal, bytesDownloaded = 0;
try {
bytesTotal = getTotalBytes(this.url);
} catch (IOException e) {
logger.error("Failed to get file size at " + this.url, e);
observer.downloadErrored(this.url, "Failed to get file size of " + this.url);
return;
}
observer.setBytesTotal(bytesTotal);
observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal);
logger.info("Size of file at " + this.url + " = " + bytesTotal + "b");
int tries = 0; // Number of attempts to download
do {
InputStream bis = null; OutputStream fos = null;
byte[] data = new byte[1024]; int bytesRead;
try {
logger.info(" Downloading file: " + url + (tries > 0 ? " Retry #" + tries : ""));
observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm());
tries += 1;
bis = new BufferedInputStream(this.url.openStream());
fos = new FileOutputStream(saveAs);
while ( (bytesRead = bis.read(data)) != -1) {
try {
observer.stopCheck();
} catch (IOException e) {
observer.downloadErrored(url, "Download interrupted");
return;
}
fos.write(data, 0, bytesRead);
bytesDownloaded += bytesRead;
observer.setBytesCompleted(bytesDownloaded);
observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded);
}
bis.close();
fos.close();
break; // Download successful: break out of infinite loop
} catch (IOException e) {
logger.error("[!] Exception while downloading file: " + url + " - " + e.getMessage(), e);
} finally {
// Close any open streams
try {
if (bis != null) { bis.close(); }
} catch (IOException e) { }
try {
if (fos != null) { fos.close(); }
} catch (IOException e) { }
}
if (tries > this.retries) {
logger.error("[!] Exceeded maximum retries (" + this.retries + ") for URL " + url);
observer.downloadErrored(url, "Failed to download " + url.toExternalForm());
return;
}
} while (true);
observer.downloadCompleted(url, saveAs);
logger.info("[+] Saved " + url + " as " + this.prettySaveAs);
}
private int getTotalBytes(URL url) throws IOException {
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("HEAD");
return conn.getContentLength();
}
}

121
src/main/java/com/rarchives/ripme/ripper/VideoRipper.java Normal file
View File

@ -0,0 +1,121 @@
package com.rarchives.ripme.ripper;
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import com.rarchives.ripme.ui.RipStatusMessage;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Utils;
public abstract class VideoRipper extends AbstractRipper {
private int bytesTotal = 1,
bytesCompleted = 1;
public VideoRipper(URL url) throws IOException {
super(url);
}
public abstract boolean canRip(URL url);
public abstract void rip() throws IOException;
public abstract String getHost();
public abstract String getGID(URL url) throws MalformedURLException;
@Override
public void setBytesTotal(int bytes) {
this.bytesTotal = bytes;
}
@Override
public void setBytesCompleted(int bytes) {
this.bytesCompleted = bytes;
}
@Override
public void addURLToDownload(URL url, File saveAs) {
threadPool.addThread(new DownloadVideoThread(url, saveAs, this));
}
@Override
public void setWorkingDir(URL url) throws IOException {
String path = Utils.getWorkingDirectory().getCanonicalPath();
if (!path.endsWith(File.separator)) {
path += File.separator;
}
path += "videos" + File.separator;
this.workingDir = new File(path);
if (!this.workingDir.exists()) {
logger.info("[+] Creating directory: " + Utils.removeCWD(this.workingDir));
this.workingDir.mkdirs();
}
logger.debug("Set working directory to: " + this.workingDir);
}
@Override
public int getCompletionPercentage() {
return (int) (100 * (bytesCompleted / (float) bytesTotal));
}
@Override
public void downloadCompleted(URL url, File saveAs) {
if (observer == null) {
return;
}
try {
String path = Utils.removeCWD(saveAs);
RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path);
observer.update(this, msg);
checkIfComplete();
} catch (Exception e) {
logger.error("Exception while updating observer: ", e);
}
}
@Override
public void downloadErrored(URL url, String reason) {
if (observer == null) {
return;
}
observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason));
checkIfComplete();
}
@Override
public void downloadProblem(URL url, String message) {
if (observer == null) {
return;
}
observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " : " + message));
checkIfComplete();
}
@Override
public String getStatusText() {
StringBuilder sb = new StringBuilder();
sb.append(getCompletionPercentage())
.append("% ")
.append(" - ")
.append(Utils.bytesToHumanReadable(bytesCompleted))
.append(" / ")
.append(Utils.bytesToHumanReadable(bytesTotal));
return sb.toString();
}
@Override
public URL sanitizeURL(URL url) throws MalformedURLException {
return url;
}
/**
* Notifies observers and updates state if all files have been ripped.
*/
@Override
protected void checkIfComplete() {
if (observer == null) {
return;
}
if (bytesCompleted >= bytesTotal) {
super.checkIfComplete();
}
}
}

4
src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
View File

@ -13,9 +13,9 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AbstractRipper;
import com.rarchives.ripme.ripper.AlbumRipper;
public class ChanRipper extends AbstractRipper {
public class ChanRipper extends AlbumRipper {
private static final Logger logger = Logger.getLogger(ChanRipper.class);

4
src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
View File

@ -18,10 +18,10 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AbstractRipper;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.utils.Utils;
public class DeviantartRipper extends AbstractRipper {
public class DeviantartRipper extends AlbumRipper {
private static final String DOMAIN = "deviantart.com",
HOST = "deviantart";

4
src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java
View File

@ -12,9 +12,9 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AbstractRipper;
import com.rarchives.ripme.ripper.AlbumRipper;
public class EightmusesRipper extends AbstractRipper {
public class EightmusesRipper extends AlbumRipper {
private static final String DOMAIN = "8muses.com",
HOST = "8muses";

4
src/main/java/com/rarchives/ripme/ripper/rippers/GonewildRipper.java
View File

@ -11,10 +11,10 @@ import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import com.rarchives.ripme.ripper.AbstractRipper;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.utils.Utils;
public class GonewildRipper extends AbstractRipper {
public class GonewildRipper extends AlbumRipper {
private static final String HOST = "gonewild";
private static final Logger logger = Logger.getLogger(GonewildRipper.class);

4
src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java
View File

@ -11,9 +11,9 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AbstractRipper;
import com.rarchives.ripme.ripper.AlbumRipper;
public class ImagearnRipper extends AbstractRipper {
public class ImagearnRipper extends AlbumRipper {
private static final String DOMAIN = "imagearn.com",
HOST = "imagearn";

4
src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java
View File

@ -11,9 +11,9 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AbstractRipper;
import com.rarchives.ripme.ripper.AlbumRipper;
public class ImagefapRipper extends AbstractRipper {
public class ImagefapRipper extends AlbumRipper {
private static final String DOMAIN = "imagefap.com",
HOST = "imagefap";

4
src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java
View File

@ -18,11 +18,11 @@ import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AbstractRipper;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Utils;
public class ImgurRipper extends AbstractRipper {
public class ImgurRipper extends AlbumRipper {
private static final String DOMAIN = "imgur.com",
HOST = "imgur";

4
src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java
View File

@ -13,9 +13,9 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AbstractRipper;
import com.rarchives.ripme.ripper.AlbumRipper;
public class InstagramRipper extends AbstractRipper {
public class InstagramRipper extends AlbumRipper {
private static final String DOMAIN = "instagram.com",
HOST = "instagram";

4
src/main/java/com/rarchives/ripme/ripper/rippers/KinkyshareRipper.java
View File

@ -11,9 +11,9 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AbstractRipper;
import com.rarchives.ripme.ripper.AlbumRipper;
public class KinkyshareRipper extends AbstractRipper {
public class KinkyshareRipper extends AlbumRipper {
private static final String HOST = "kinkyshare";
private static final Logger logger = Logger.getLogger(KinkyshareRipper.class);

4
src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java
View File

@ -11,10 +11,10 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AbstractRipper;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ripper.DownloadThreadPool;
public class MotherlessRipper extends AbstractRipper {
public class MotherlessRipper extends AlbumRipper {
private static final String DOMAIN = "motherless.com",
HOST = "motherless";

6
src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java
View File

@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.SocketTimeoutException;
import java.net.URL;
import java.util.List;
import java.util.regex.Matcher;
@ -14,12 +15,11 @@ import org.json.JSONTokener;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import com.rarchives.ripme.ripper.AbstractRipper;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.utils.RipUtils;
import com.rarchives.ripme.utils.Utils;
import java.net.SocketTimeoutException;
public class RedditRipper extends AbstractRipper {
public class RedditRipper extends AlbumRipper {
public RedditRipper(URL url) throws IOException {
super(url);

4
src/main/java/com/rarchives/ripme/ripper/rippers/SeeniveRipper.java
View File

@ -12,10 +12,10 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AbstractRipper;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ripper.DownloadThreadPool;
public class SeeniveRipper extends AbstractRipper {
public class SeeniveRipper extends AlbumRipper {
private static final String DOMAIN = "seenive.com",
HOST = "seenive";

4
src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java
View File

@ -12,10 +12,10 @@ import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import com.rarchives.ripme.ripper.AbstractRipper;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.utils.Utils;
public class TumblrRipper extends AbstractRipper {
public class TumblrRipper extends AlbumRipper {
private static final String DOMAIN = "tumblr.com",
HOST = "tumblr";

4
src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
View File

@ -16,10 +16,10 @@ import org.json.JSONTokener;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import com.rarchives.ripme.ripper.AbstractRipper;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.utils.Utils;
public class TwitterRipper extends AbstractRipper {
public class TwitterRipper extends AlbumRipper {
private static final String DOMAIN = "twitter.com",
HOST = "twitter";

4
src/main/java/com/rarchives/ripme/ripper/rippers/VineboxRipper.java
View File

@ -12,9 +12,9 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AbstractRipper;
import com.rarchives.ripme.ripper.AlbumRipper;
public class VineboxRipper extends AbstractRipper {
public class VineboxRipper extends AlbumRipper {
private static final String DOMAIN = "vinebox.co",
HOST = "vinebox";

4
src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java
View File

@ -18,9 +18,9 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AbstractRipper;
import com.rarchives.ripme.ripper.AlbumRipper;
public class VkRipper extends AbstractRipper {
public class VkRipper extends AlbumRipper {
private static final String DOMAIN = "vk.com",
HOST = "vk";

4
src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java
View File

@ -11,9 +11,9 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AbstractRipper;
import com.rarchives.ripme.ripper.AlbumRipper;
public class XhamsterRipper extends AbstractRipper {
public class XhamsterRipper extends AlbumRipper {
private static final String DOMAIN = "xhamster.com",
HOST = "xhamster";

84
src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java Normal file
View File

@ -0,0 +1,84 @@
package com.rarchives.ripme.ripper.rippers.video;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.VideoRipper;
public class XvideosRipper extends VideoRipper {
private static final String HOST = "xvideos";
private static final Logger logger = Logger.getLogger(XvideosRipper.class);
public XvideosRipper(URL url) throws IOException {
super(url);
}
@Override
public String getHost() {
return HOST;
}
@Override
public boolean canRip(URL url) {
Pattern p = Pattern.compile("^https?://[wm.]*xvideos\\.com/video[0-9]+.*$");
Matcher m = p.matcher(url.toExternalForm());
return m.matches();
}
@Override
public String getAlbumTitle(URL url) {
return "videos";
}
@Override
public URL sanitizeURL(URL url) throws MalformedURLException {
return url;
}
@Override
public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^https?://[wm.]*xvideos\\.com/video([0-9]+).*$");
Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) {
return m.group(1);
}
throw new MalformedURLException(
"Expected xvideo format:"
+ "xvideos.com/video####"
+ " Got: " + url);
}
@Override
public void rip() throws IOException {
logger.info(" Retrieving " + this.url.toExternalForm());
Document doc = Jsoup.connect(this.url.toExternalForm())
.userAgent(USER_AGENT)
.get();
Elements embeds = doc.select("embed");
if (embeds.size() == 0) {
throw new IOException("Could not find Embed code at " + url);
}
Element embed = embeds.get(0);
String vars = embed.attr("flashvars");
for (String var : vars.split("&")) {
if (var.startsWith("flv_url=")) {
String vidUrl = var.substring("flv_url=".length());
vidUrl = URLDecoder.decode(vidUrl, "UTF-8");
addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url));
}
}
waitForThreads();
}
}

7
src/main/java/com/rarchives/ripme/ui/MainWindow.java
View File

@ -721,6 +721,13 @@ public class MainWindow implements Runnable, RipStatusHandler {
}
});
mainFrame.pack();
break;
case COMPLETED_BYTES:
// Update completed bytes
break;
case TOTAL_BYTES:
// Update total bytes
break;
}
}

4
src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java
View File

@ -11,7 +11,9 @@ public class RipStatusMessage {
DOWNLOAD_COMPLETE("Download Complete"),
DOWNLOAD_ERRORED("Download Errored"),
RIP_COMPLETE("Rip Complete"),
DOWNLOAD_WARN("Download problem");
DOWNLOAD_WARN("Download problem"),
TOTAL_BYTES("Total bytes"),
COMPLETED_BYTES("Completed bytes");
String value;
STATUS(String value) {

2
src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
View File

@ -19,7 +19,7 @@ import org.jsoup.nodes.Document;
public class UpdateUtils {
private static final Logger logger = Logger.getLogger(UpdateUtils.class);
private static final String DEFAULT_VERSION = "1.0.19";
private static final String DEFAULT_VERSION = "1.0.20";
private static final String updateJsonURL = "http://rarchives.com/ripme.json";
private static final String updateJarURL = "http://rarchives.com/ripme.jar";
private static final String mainFileName = "ripme.jar";

11
src/main/java/com/rarchives/ripme/utils/Utils.java
View File

@ -247,4 +247,15 @@ public class Utils {
.replaceAll("__", "_")
.replaceAll("_+$", "");
}
public static String bytesToHumanReadable(int bytes) {
float fbytes = (float) bytes;
String[] mags = new String[] {"", "k", "m", "g", "t"};
int magIndex = 0;
while (fbytes >= 1024) {
fbytes /= 1024;
magIndex++;
}
return String.format("%.2f%sb", fbytes, mags[magIndex]);
}
}

3
src/test/java/com/rarchives/ripme/tst/AppTest.java
View File

@ -4,6 +4,8 @@ import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;
import com.rarchives.ripme.utils.Utils;
public class AppTest extends TestCase {
/**
* Create the test case
@ -25,6 +27,7 @@ public class AppTest extends TestCase {
* Rigourous Test :-)
*/
public void testApp() {
System.err.println(Utils.bytesToHumanReadable(1023 * 5000));
assertTrue( true );
}
}

32
src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java Normal file
View File

@ -0,0 +1,32 @@
package com.rarchives.ripme.tst.ripper.rippers;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import com.rarchives.ripme.ripper.rippers.video.XvideosRipper;
public class VideoRippersTest extends RippersTest {
public void testXvideosRipper() throws IOException {
if (false && !DOWNLOAD_CONTENT) {
return;
}
List<URL> contentURLs = new ArrayList<URL>();
contentURLs.add(new URL("http://www.xvideos.com/video1428195/stephanie_first_time_anal"));
contentURLs.add(new URL("http://www.xvideos.com/video7136868/vid-20140205-wa0011"));
for (URL url : contentURLs) {
try {
XvideosRipper ripper = new XvideosRipper(url);
ripper.rip();
assert(ripper.getWorkingDir().listFiles().length > 1);
deleteDir(ripper.getWorkingDir());
} catch (Exception e) {
e.printStackTrace();
fail("Error while ripping URL " + url + ": " + e.getMessage());
}
}
}
}