2014-02-25 10:28:22 +01:00
|
|
|
package com.rarchives.ripme.ripper;
|
|
|
|
|
2014-02-26 08:44:22 +01:00
|
|
|
import java.io.File;
|
|
|
|
import java.io.IOException;
|
2014-02-25 10:28:22 +01:00
|
|
|
import java.net.MalformedURLException;
|
|
|
|
import java.net.URL;
|
|
|
|
|
2014-02-27 04:54:44 +01:00
|
|
|
import org.apache.log4j.Logger;
|
|
|
|
|
2014-02-26 08:44:22 +01:00
|
|
|
import com.rarchives.ripme.utils.Utils;
|
|
|
|
|
2014-02-25 10:28:22 +01:00
|
|
|
public abstract class AbstractRipper implements RipperInterface {
|
|
|
|
|
2014-02-27 04:54:44 +01:00
|
|
|
private static final Logger logger = Logger.getLogger(AbstractRipper.class);
|
|
|
|
|
2014-02-25 10:28:22 +01:00
|
|
|
protected URL url;
|
2014-02-27 04:54:44 +01:00
|
|
|
protected File workingDir;
|
|
|
|
protected DownloadThreadPool threadPool;
|
2014-02-26 08:44:22 +01:00
|
|
|
|
|
|
|
public abstract void rip() throws IOException;
|
2014-02-27 04:54:44 +01:00
|
|
|
public abstract String getHost();
|
|
|
|
public abstract String getGID(URL url) throws MalformedURLException;
|
2014-02-25 10:28:22 +01:00
|
|
|
|
2014-02-26 08:44:22 +01:00
|
|
|
/**
|
2014-02-27 04:54:44 +01:00
|
|
|
* Ensures inheriting ripper can rip this URL, raises exception if not.
|
|
|
|
* Otherwise initializes working directory and thread pool.
|
|
|
|
*
|
2014-02-26 08:44:22 +01:00
|
|
|
* @param url
|
|
|
|
* URL to rip.
|
|
|
|
* @throws IOException
|
|
|
|
* If anything goes wrong.
|
|
|
|
*/
|
|
|
|
public AbstractRipper(URL url) throws IOException {
|
2014-02-25 10:28:22 +01:00
|
|
|
if (!canRip(url)) {
|
|
|
|
throw new MalformedURLException("Unable to rip url: " + url);
|
|
|
|
}
|
2014-02-27 04:54:44 +01:00
|
|
|
this.url = sanitizeURL(url);
|
|
|
|
setWorkingDir(url);
|
|
|
|
this.threadPool = new DownloadThreadPool();
|
|
|
|
}
|
|
|
|
|
|
|
|
public void addURLToDownload(URL url) {
|
|
|
|
addURLToDownload(url, "");
|
|
|
|
}
|
|
|
|
|
|
|
|
public void addURLToDownload(URL url, String prefix) {
|
|
|
|
String saveAs = url.toExternalForm();
|
|
|
|
saveAs = saveAs.substring(saveAs.lastIndexOf('/')+1);
|
|
|
|
if (saveAs.indexOf('?') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('?')); }
|
|
|
|
if (saveAs.indexOf('#') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('#')); }
|
|
|
|
if (saveAs.indexOf('&') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('&')); }
|
|
|
|
File saveFileAs;
|
|
|
|
try {
|
|
|
|
saveFileAs = new File(workingDir.getCanonicalPath() + File.separator + prefix + saveAs);
|
|
|
|
} catch (IOException e) {
|
|
|
|
logger.error("Error creating save file path for URL '" + url + "':", e);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
logger.info("Downloading " + url + " to " + saveFileAs);
|
|
|
|
addURLToDownload(url, saveFileAs);
|
|
|
|
}
|
|
|
|
/**
|
|
|
|
* Add image to be downloaded and saved.
|
|
|
|
* @param url
|
|
|
|
* URL of the file
|
|
|
|
* @param saveAs
|
|
|
|
* Path of the local file to save the content to.
|
|
|
|
*/
|
|
|
|
public void addURLToDownload(URL url, File saveAs) {
|
|
|
|
threadPool.addThread(new DownloadFileThread(url, saveAs));
|
2014-02-25 10:28:22 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
public URL getURL() {
|
|
|
|
return url;
|
|
|
|
}
|
|
|
|
|
2014-02-27 04:54:44 +01:00
|
|
|
public void setWorkingDir(URL url) throws IOException {
|
|
|
|
String path = Utils.getWorkingDirectory().getCanonicalPath();
|
|
|
|
if (!path.endsWith(File.separator)) {
|
|
|
|
path += File.separator;
|
|
|
|
}
|
|
|
|
path += getHost() + "_" + getGID(this.url) + File.separator;
|
|
|
|
this.workingDir = new File(path);
|
|
|
|
if (!this.workingDir.exists()) {
|
|
|
|
logger.info("Creating working directory(s): " + this.workingDir);
|
|
|
|
this.workingDir.mkdirs();
|
|
|
|
}
|
|
|
|
logger.debug("Set working directory to: " + this.workingDir);
|
|
|
|
}
|
2014-02-26 08:44:22 +01:00
|
|
|
}
|