2014-02-25 10:28:22 +01:00
|
|
|
package com.rarchives.ripme.ripper;
|
|
|
|
|
2014-03-13 20:18:35 +01:00
|
|
|
import com.rarchives.ripme.ui.MainWindow;
|
|
|
|
import com.rarchives.ripme.ui.RipStatusHandler;
|
2014-02-26 08:44:22 +01:00
|
|
|
import java.io.File;
|
|
|
|
import java.io.IOException;
|
2014-03-02 04:37:09 +01:00
|
|
|
import java.lang.reflect.Constructor;
|
2014-02-25 10:28:22 +01:00
|
|
|
import java.net.MalformedURLException;
|
|
|
|
import java.net.URL;
|
2014-03-02 04:37:09 +01:00
|
|
|
import java.util.ArrayList;
|
2014-03-01 11:13:32 +01:00
|
|
|
import java.util.HashMap;
|
2014-03-02 04:37:09 +01:00
|
|
|
import java.util.List;
|
2014-03-01 11:13:32 +01:00
|
|
|
import java.util.Map;
|
|
|
|
import java.util.Observable;
|
|
|
|
import java.util.Observer;
|
2014-02-25 10:28:22 +01:00
|
|
|
|
2014-02-27 04:54:44 +01:00
|
|
|
import org.apache.log4j.Logger;
|
|
|
|
|
2014-03-01 11:13:32 +01:00
|
|
|
import com.rarchives.ripme.ui.RipStatusMessage;
|
|
|
|
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
|
2014-02-26 08:44:22 +01:00
|
|
|
import com.rarchives.ripme.utils.Utils;
|
2014-03-13 20:18:35 +01:00
|
|
|
import java.util.Collections;
|
2014-02-26 08:44:22 +01:00
|
|
|
|
2014-03-01 11:13:32 +01:00
|
|
|
public abstract class AbstractRipper
|
|
|
|
extends Observable
|
2014-03-01 11:43:47 +01:00
|
|
|
implements RipperInterface, Runnable {
|
2014-02-25 10:28:22 +01:00
|
|
|
|
2014-02-27 04:54:44 +01:00
|
|
|
private static final Logger logger = Logger.getLogger(AbstractRipper.class);
|
|
|
|
|
2014-03-09 08:59:36 +01:00
|
|
|
protected static final String USER_AGENT =
|
|
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:27.0) Gecko/20100101 Firefox/27.0";
|
|
|
|
|
2014-02-25 10:28:22 +01:00
|
|
|
protected URL url;
|
2014-02-27 04:54:44 +01:00
|
|
|
protected File workingDir;
|
|
|
|
protected DownloadThreadPool threadPool;
|
2014-03-13 20:18:35 +01:00
|
|
|
protected RipStatusHandler observer = null;
|
2014-03-01 11:13:32 +01:00
|
|
|
|
2014-03-13 20:18:35 +01:00
|
|
|
protected Map<URL, File> itemsPending = Collections.synchronizedMap(new HashMap<URL, File>());
|
|
|
|
protected Map<URL, File> itemsCompleted = Collections.synchronizedMap(new HashMap<URL, File>());
|
|
|
|
protected Map<URL, String> itemsErrored = Collections.synchronizedMap(new HashMap<URL, String>());
|
2014-03-02 03:08:16 +01:00
|
|
|
protected boolean completed = true;
|
2014-02-26 08:44:22 +01:00
|
|
|
|
|
|
|
public abstract void rip() throws IOException;
|
2014-02-27 04:54:44 +01:00
|
|
|
public abstract String getHost();
|
|
|
|
public abstract String getGID(URL url) throws MalformedURLException;
|
2014-02-25 10:28:22 +01:00
|
|
|
|
2014-02-26 08:44:22 +01:00
|
|
|
/**
|
2014-02-27 04:54:44 +01:00
|
|
|
* Ensures inheriting ripper can rip this URL, raises exception if not.
|
|
|
|
* Otherwise initializes working directory and thread pool.
|
|
|
|
*
|
2014-02-26 08:44:22 +01:00
|
|
|
* @param url
|
|
|
|
* URL to rip.
|
|
|
|
* @throws IOException
|
|
|
|
* If anything goes wrong.
|
|
|
|
*/
|
|
|
|
public AbstractRipper(URL url) throws IOException {
|
2014-02-25 10:28:22 +01:00
|
|
|
if (!canRip(url)) {
|
|
|
|
throw new MalformedURLException("Unable to rip url: " + url);
|
|
|
|
}
|
2014-02-27 04:54:44 +01:00
|
|
|
this.url = sanitizeURL(url);
|
|
|
|
setWorkingDir(url);
|
|
|
|
this.threadPool = new DownloadThreadPool();
|
|
|
|
}
|
2014-03-09 09:20:22 +01:00
|
|
|
|
2014-03-13 20:18:35 +01:00
|
|
|
public void setObserver(RipStatusHandler obs) {
|
2014-03-01 11:13:32 +01:00
|
|
|
this.observer = obs;
|
|
|
|
}
|
2014-02-27 04:54:44 +01:00
|
|
|
|
2014-02-28 12:04:03 +01:00
|
|
|
/**
|
|
|
|
* Queues image to be downloaded and saved.
|
|
|
|
* Uses filename from URL to decide filename.
|
|
|
|
* @param url
|
|
|
|
* URL to download
|
|
|
|
*/
|
2014-02-27 04:54:44 +01:00
|
|
|
public void addURLToDownload(URL url) {
|
2014-02-28 12:04:03 +01:00
|
|
|
// Use empty prefix and empty subdirectory
|
|
|
|
addURLToDownload(url, "", "");
|
2014-02-27 04:54:44 +01:00
|
|
|
}
|
|
|
|
|
2014-02-28 12:04:03 +01:00
|
|
|
/**
|
|
|
|
* Queues image to be downloaded and saved.
|
|
|
|
* Uses filename from URL (and 'prefix') to decide filename.
|
|
|
|
* @param url
|
|
|
|
* URL to download
|
|
|
|
* @param prefix
|
|
|
|
* Text to append to saved filename.
|
|
|
|
*/
|
2014-02-27 04:54:44 +01:00
|
|
|
public void addURLToDownload(URL url, String prefix) {
|
2014-02-28 12:04:03 +01:00
|
|
|
// Use empty subdirectory
|
|
|
|
addURLToDownload(url, prefix, "");
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Queues image to be downloaded and saved.
|
|
|
|
* @param url
|
|
|
|
* URL of the file
|
|
|
|
* @param saveAs
|
|
|
|
* Path of the local file to save the content to.
|
|
|
|
*/
|
|
|
|
public void addURLToDownload(URL url, File saveAs) {
|
2014-03-02 03:08:16 +01:00
|
|
|
if (itemsPending.containsKey(url)
|
|
|
|
|| itemsCompleted.containsKey(url)
|
|
|
|
|| itemsErrored.containsKey(url)) {
|
2014-03-01 11:13:32 +01:00
|
|
|
// Item is already downloaded/downloading, skip it.
|
2014-03-13 10:26:55 +01:00
|
|
|
logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs));
|
2014-03-01 11:13:32 +01:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
itemsPending.put(url, saveAs);
|
|
|
|
threadPool.addThread(new DownloadFileThread(url, saveAs, this));
|
2014-02-28 12:04:03 +01:00
|
|
|
}
|
|
|
|
|
2014-03-09 09:20:22 +01:00
|
|
|
/**
|
|
|
|
* Queues file to be downloaded and saved. With options.
|
|
|
|
* @param url
|
|
|
|
* URL to download.
|
|
|
|
* @param prefix
|
|
|
|
* Prefix to prepend to the saved filename.
|
|
|
|
* @param subdirectory
|
|
|
|
* Sub-directory of the working directory to save the images to.
|
|
|
|
*/
|
2014-02-28 12:04:03 +01:00
|
|
|
public void addURLToDownload(URL url, String prefix, String subdirectory) {
|
2014-02-27 04:54:44 +01:00
|
|
|
String saveAs = url.toExternalForm();
|
|
|
|
saveAs = saveAs.substring(saveAs.lastIndexOf('/')+1);
|
|
|
|
if (saveAs.indexOf('?') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('?')); }
|
|
|
|
if (saveAs.indexOf('#') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('#')); }
|
|
|
|
if (saveAs.indexOf('&') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('&')); }
|
2014-03-04 14:35:35 +01:00
|
|
|
if (saveAs.indexOf(':') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf(':')); }
|
2014-02-27 04:54:44 +01:00
|
|
|
File saveFileAs;
|
|
|
|
try {
|
2014-02-28 12:04:03 +01:00
|
|
|
if (!subdirectory.equals("")) {
|
|
|
|
subdirectory = File.separator + subdirectory;
|
|
|
|
}
|
|
|
|
saveFileAs = new File(
|
|
|
|
workingDir.getCanonicalPath()
|
|
|
|
+ subdirectory
|
|
|
|
+ File.separator
|
|
|
|
+ prefix
|
|
|
|
+ saveAs);
|
2014-02-27 04:54:44 +01:00
|
|
|
} catch (IOException e) {
|
2014-02-28 04:49:28 +01:00
|
|
|
logger.error("[!] Error creating save file path for URL '" + url + "':", e);
|
2014-02-27 04:54:44 +01:00
|
|
|
return;
|
|
|
|
}
|
2014-02-27 10:28:23 +01:00
|
|
|
logger.debug("Downloading " + url + " to " + saveFileAs);
|
2014-02-28 12:04:03 +01:00
|
|
|
if (!saveFileAs.getParentFile().exists()) {
|
|
|
|
logger.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent()));
|
|
|
|
saveFileAs.getParentFile().mkdirs();
|
|
|
|
}
|
2014-02-27 04:54:44 +01:00
|
|
|
addURLToDownload(url, saveFileAs);
|
|
|
|
}
|
2014-03-02 03:08:16 +01:00
|
|
|
|
2014-03-09 09:20:22 +01:00
|
|
|
/**
|
|
|
|
* Waits for downloading threads to complete.
|
|
|
|
*/
|
2014-03-02 03:08:16 +01:00
|
|
|
protected void waitForThreads() {
|
|
|
|
completed = false;
|
|
|
|
threadPool.waitForThreads();
|
2014-03-10 01:12:10 +01:00
|
|
|
checkIfComplete();
|
2014-03-02 03:08:16 +01:00
|
|
|
}
|
2014-02-25 10:28:22 +01:00
|
|
|
|
2014-03-09 09:20:22 +01:00
|
|
|
/**
|
|
|
|
* Notifies observers that source is being retrieved.
|
|
|
|
* @param url
|
|
|
|
* URL being retrieved
|
|
|
|
*/
|
2014-03-01 11:13:32 +01:00
|
|
|
public void retrievingSource(URL url) {
|
|
|
|
RipStatusMessage msg = new RipStatusMessage(STATUS.LOADING_RESOURCE, url);
|
|
|
|
observer.update(this, msg);
|
|
|
|
}
|
|
|
|
|
2014-03-09 09:20:22 +01:00
|
|
|
/**
|
|
|
|
* Notifies observers that a file download has completed.
|
|
|
|
* @param url
|
|
|
|
* URL that was completed.
|
|
|
|
* @param saveAs
|
|
|
|
* Where the downloaded file is stored.
|
|
|
|
*/
|
2014-03-01 11:13:32 +01:00
|
|
|
public void downloadCompleted(URL url, File saveAs) {
|
2014-03-02 04:37:09 +01:00
|
|
|
if (observer == null) {
|
|
|
|
return;
|
|
|
|
}
|
2014-03-01 11:13:32 +01:00
|
|
|
try {
|
2014-03-02 10:12:20 +01:00
|
|
|
String path = Utils.removeCWD(saveAs);
|
2014-03-01 11:13:32 +01:00
|
|
|
RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path);
|
2014-03-13 20:18:35 +01:00
|
|
|
itemsPending.remove(url);
|
|
|
|
itemsCompleted.put(url, saveAs);
|
|
|
|
observer.update(this, msg);
|
|
|
|
|
|
|
|
checkIfComplete();
|
2014-03-01 11:13:32 +01:00
|
|
|
} catch (Exception e) {
|
|
|
|
logger.error("Exception while updating observer: ", e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-03-09 09:20:22 +01:00
|
|
|
/**
|
|
|
|
* Notifies observers that a file could not be downloaded (includes a reason).
|
|
|
|
* @param url
|
|
|
|
* @param reason
|
|
|
|
*/
|
2014-03-01 11:13:32 +01:00
|
|
|
public void downloadErrored(URL url, String reason) {
|
2014-03-02 04:37:09 +01:00
|
|
|
if (observer == null) {
|
|
|
|
return;
|
|
|
|
}
|
2014-03-13 20:18:35 +01:00
|
|
|
itemsPending.remove(url);
|
|
|
|
itemsErrored.put(url, reason);
|
|
|
|
observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason));
|
|
|
|
|
|
|
|
checkIfComplete();
|
2014-03-01 11:13:32 +01:00
|
|
|
}
|
|
|
|
|
2014-03-09 09:20:22 +01:00
|
|
|
/**
|
|
|
|
* Notify observers that a download could not be completed,
|
|
|
|
* but was not technically an "error".
|
|
|
|
* @param url
|
|
|
|
* @param message
|
|
|
|
*/
|
2014-03-08 21:22:49 +01:00
|
|
|
public void downloadProblem(URL url, String message) {
|
|
|
|
if (observer == null) {
|
|
|
|
return;
|
|
|
|
}
|
2014-03-13 20:18:35 +01:00
|
|
|
|
|
|
|
itemsPending.remove(url);
|
|
|
|
itemsErrored.put(url, message);
|
|
|
|
observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " : " + message));
|
|
|
|
|
|
|
|
|
2014-03-10 01:12:10 +01:00
|
|
|
checkIfComplete();
|
2014-03-08 21:22:49 +01:00
|
|
|
}
|
|
|
|
|
2014-03-09 09:20:22 +01:00
|
|
|
/**
|
|
|
|
* Notifies observers and updates state if all files have been ripped.
|
|
|
|
*/
|
2014-03-01 11:13:32 +01:00
|
|
|
private void checkIfComplete() {
|
2014-03-11 09:29:46 +01:00
|
|
|
if (observer == null) {
|
|
|
|
return;
|
|
|
|
}
|
2014-03-13 20:18:35 +01:00
|
|
|
|
|
|
|
if (!completed && itemsPending.isEmpty()) {
|
|
|
|
completed = true;
|
|
|
|
logger.info(" Rip completed!");
|
|
|
|
|
|
|
|
RipStatusMessage msg = new RipStatusMessage(STATUS.RIP_COMPLETE, workingDir);
|
|
|
|
observer.update(this, msg);
|
2014-03-01 11:13:32 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-02-25 10:28:22 +01:00
|
|
|
public URL getURL() {
|
|
|
|
return url;
|
|
|
|
}
|
2014-03-03 09:44:07 +01:00
|
|
|
|
2014-03-09 09:20:22 +01:00
|
|
|
/**
|
|
|
|
* @return
|
|
|
|
* Path to the directory in which all files
|
|
|
|
* ripped via this ripper will be stored.
|
|
|
|
*/
|
2014-03-03 09:44:07 +01:00
|
|
|
public File getWorkingDir() {
|
|
|
|
return workingDir;
|
|
|
|
}
|
2014-02-25 10:28:22 +01:00
|
|
|
|
2014-03-09 09:20:22 +01:00
|
|
|
/**
|
|
|
|
* Sets directory to save all ripped files to.
|
|
|
|
* @param url
|
|
|
|
* URL to define how the workin directory should be saved.
|
|
|
|
*/
|
2014-02-27 04:54:44 +01:00
|
|
|
public void setWorkingDir(URL url) throws IOException {
|
|
|
|
String path = Utils.getWorkingDirectory().getCanonicalPath();
|
|
|
|
if (!path.endsWith(File.separator)) {
|
|
|
|
path += File.separator;
|
|
|
|
}
|
|
|
|
path += getHost() + "_" + getGID(this.url) + File.separator;
|
|
|
|
this.workingDir = new File(path);
|
|
|
|
if (!this.workingDir.exists()) {
|
2014-02-28 12:04:03 +01:00
|
|
|
logger.info("[+] Creating directory: " + Utils.removeCWD(this.workingDir));
|
2014-02-27 04:54:44 +01:00
|
|
|
this.workingDir.mkdirs();
|
|
|
|
}
|
|
|
|
logger.debug("Set working directory to: " + this.workingDir);
|
|
|
|
}
|
2014-03-01 11:13:32 +01:00
|
|
|
|
2014-02-27 10:28:23 +01:00
|
|
|
/**
|
|
|
|
* Finds, instantiates, and returns a compatible ripper for given URL.
|
|
|
|
* @param url
|
|
|
|
* URL to rip.
|
|
|
|
* @return
|
|
|
|
* Instantiated ripper ready to rip given URL.
|
|
|
|
* @throws Exception
|
|
|
|
* If no compatible rippers can be found.
|
|
|
|
*/
|
|
|
|
public static AbstractRipper getRipper(URL url) throws Exception {
|
2014-03-02 04:37:09 +01:00
|
|
|
for (Constructor<?> constructor : getRipperConstructors()) {
|
|
|
|
try {
|
|
|
|
AbstractRipper ripper = (AbstractRipper) constructor.newInstance(url);
|
|
|
|
return ripper;
|
|
|
|
} catch (Exception e) {
|
|
|
|
// Incompatible rippers *will* throw exceptions during instantiation.
|
|
|
|
}
|
|
|
|
}
|
2014-02-27 10:28:23 +01:00
|
|
|
throw new Exception("No compatible ripper found");
|
|
|
|
}
|
2014-03-04 14:35:35 +01:00
|
|
|
|
2014-03-09 09:20:22 +01:00
|
|
|
/**
|
|
|
|
* @return
|
|
|
|
* List of constructors for all eligible Rippers.
|
|
|
|
* @throws Exception
|
|
|
|
*/
|
2014-03-02 04:37:09 +01:00
|
|
|
private static List<Constructor<?>> getRipperConstructors() throws Exception {
|
|
|
|
List<Constructor<?>> constructors = new ArrayList<Constructor<?>>();
|
2014-03-04 14:35:35 +01:00
|
|
|
for (Class<?> clazz : Utils.getClassesForPackage("com.rarchives.ripme.ripper.rippers")) {
|
2014-03-03 11:29:02 +01:00
|
|
|
if (AbstractRipper.class.isAssignableFrom(clazz)) {
|
|
|
|
constructors.add( (Constructor<?>) clazz.getConstructor(URL.class) );
|
2014-03-02 04:37:09 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return constructors;
|
|
|
|
}
|
2014-03-01 11:13:32 +01:00
|
|
|
|
2014-03-09 09:20:22 +01:00
|
|
|
/**
|
|
|
|
* Sends an update message to the relevant observer(s) on this ripper.
|
|
|
|
* @param status
|
|
|
|
* @param message
|
|
|
|
*/
|
2014-03-01 11:13:32 +01:00
|
|
|
public void sendUpdate(STATUS status, Object message) {
|
2014-03-05 13:55:51 +01:00
|
|
|
if (observer == null) {
|
2014-03-02 04:37:09 +01:00
|
|
|
return;
|
|
|
|
}
|
2014-03-13 20:18:35 +01:00
|
|
|
observer.update(this, new RipStatusMessage(status, message));
|
2014-03-01 11:13:32 +01:00
|
|
|
}
|
2014-03-02 08:28:37 +01:00
|
|
|
|
2014-03-09 09:20:22 +01:00
|
|
|
/**
|
|
|
|
* @return
|
|
|
|
* Integer between 0 and 100 defining the progress of the album rip.
|
|
|
|
*/
|
2014-03-02 08:28:37 +01:00
|
|
|
public int getCompletionPercentage() {
|
|
|
|
double total = itemsPending.size() + itemsErrored.size() + itemsCompleted.size();
|
|
|
|
return (int) (100 * ( (total - itemsPending.size()) / total));
|
|
|
|
}
|
|
|
|
|
2014-03-09 09:20:22 +01:00
|
|
|
/**
|
|
|
|
* @return
|
|
|
|
* Human-readable information on the status of the current rip.
|
|
|
|
*/
|
2014-03-02 08:28:37 +01:00
|
|
|
public String getStatusText() {
|
|
|
|
StringBuilder sb = new StringBuilder();
|
|
|
|
sb.append(getCompletionPercentage())
|
2014-03-04 14:35:35 +01:00
|
|
|
.append("% ")
|
|
|
|
.append("- Pending: " ).append(itemsPending.size())
|
|
|
|
.append(", Completed: ").append(itemsCompleted.size())
|
|
|
|
.append(", Errored: " ).append(itemsErrored.size());
|
2014-03-02 08:28:37 +01:00
|
|
|
return sb.toString();
|
|
|
|
}
|
2014-03-01 11:13:32 +01:00
|
|
|
|
2014-03-09 09:20:22 +01:00
|
|
|
/**
|
|
|
|
* Rips the album when the thread is invoked.
|
|
|
|
*/
|
2014-03-01 11:43:47 +01:00
|
|
|
public void run() {
|
|
|
|
try {
|
|
|
|
rip();
|
|
|
|
} catch (IOException e) {
|
|
|
|
logger.error("Got exception while running ripper:", e);
|
2014-03-11 09:29:46 +01:00
|
|
|
waitForThreads();
|
2014-03-01 11:43:47 +01:00
|
|
|
}
|
|
|
|
}
|
2014-03-08 21:22:49 +01:00
|
|
|
|
2014-03-05 13:55:51 +01:00
|
|
|
}
|