Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #365 from kevin51jiang/MoreDocumentation

Browse Source 
3 Metric Tons of Javadoc documentation for core Ripper stuff.
This commit is contained in:
cyian-1756 2017-12-29 04:21:26 -05:00 committed by GitHub
commit 97975a6257
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 223 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
src/main/java/com/rarchives/ripme/App.java
View File

@ -63,13 +63,21 @@ public class App {
SwingUtilities.invokeLater(mw);
}
}
/**
* Creates an abstract ripper and instructs it to rip.
* @param url URL to be ripped
* @throws Exception
*/
private static void rip(URL url) throws Exception {
AbstractRipper ripper = AbstractRipper.getRipper(url);
ripper.setup();
ripper.rip();
}
/**
* For dealing with command-line arguments.
* @param args Array of Command-line arguments
*/
private static void handleArguments(String[] args) {
CommandLine cl = getArgs(args);
if (cl.hasOption('h')) {
@ -109,7 +117,7 @@ public class App {
}
if (cl.hasOption('R')) {
loadHistory();
if (HISTORY.toList().size() == 0) {
if (HISTORY.toList().isEmpty()) {
logger.error("There are no history entries to re-rip. Rip some albums first");
System.exit(-1);
}
@ -173,14 +181,18 @@ public class App {
}
}
// this function will attempt to rip the provided url
/**
* Attempt to rip targetURL.
* @param targetURL URL to rip
* @param saveConfig Whether or not you want to save the config (?)
*/
private static void ripURL(String targetURL, boolean saveConfig) {
try {
URL url = new URL(targetURL);
rip(url);
List<String> history = Utils.getConfigList("download.history");
if (!history.contains(url.toExternalForm())) {
history.add(url.toExternalForm());
if (!history.contains(url.toExternalForm())) {//if you haven't already downloaded the file before
history.add(url.toExternalForm());//add it to history so you won't have to redownload
Utils.setConfigList("download.history", Arrays.asList(history.toArray()));
if (saveConfig) {
Utils.saveConfig();
@ -195,6 +207,10 @@ public class App {
}
}
/**
* Creates an Options object, returns it.
* @return Returns all acceptable command-line options.
*/
private static Options getOptions() {
Options opts = new Options();
opts.addOption("h", "help", false, "Print the help");
@ -213,6 +229,11 @@ public class App {
return opts;
}
/**
* Tries to parse commandline arguments.
* @param args Array of commandline arguments.
* @return CommandLine object containing arguments.
*/
private static CommandLine getArgs(String[] args) {
BasicParser parser = new BasicParser();
try {
@ -223,7 +244,10 @@ public class App {
return null;
}
}
/**
* Loads history from history file into memory.
*/
private static void loadHistory() {
File historyFile = new File(Utils.getConfigDir() + File.separator + "history.json");
HISTORY.clear();

30
src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java
View File

@ -140,6 +140,14 @@ public abstract class AbstractHTMLRipper extends AlbumRipper {
}
waitForThreads();
}
/**
* Gets the file name from the URL
* @param url
* URL that you want to get the filename from
* @return
* Filename of the URL
*/
private String fileNameFromURL(URL url) {
String saveAs = url.toExternalForm();
if (saveAs.substring(saveAs.length() - 1) == "/") { saveAs = saveAs.substring(0,saveAs.length() - 1) ;}
@ -150,6 +158,20 @@ public abstract class AbstractHTMLRipper extends AlbumRipper {
if (saveAs.indexOf(':') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf(':')); }
return saveAs;
}
/**
*
* @param url
* Target URL
* @param subdirectory
* Path to subdirectory where you want to save it
* @param text
* Text you want to save
* @param index
* Index in something like an album
* @return
* True if ripped successfully
* False if failed
*/
public boolean saveText(URL url, String subdirectory, String text, int index) {
String saveAs = fileNameFromURL(url);
return saveText(url,subdirectory,text,index,saveAs);
@ -189,6 +211,14 @@ public abstract class AbstractHTMLRipper extends AlbumRipper {
}
return true;
}
/**
* Gets prefix based on where in the index it is
* @param index
* The index in question
* @return
* Returns prefix for a file. (?)
*/
protected String getPrefix(int index) {
String prefix = "";
if (keepSortOrder() && Utils.getConfigBoolean("download.save_order", true)) {

1
src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java
View File

@ -20,6 +20,7 @@ public abstract class AbstractJSONRipper extends AlbumRipper {
}
protected abstract String getDomain();
@Override
public abstract String getHost();
protected abstract JSONObject getFirstPage() throws IOException;

81
src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
View File

@ -84,7 +84,14 @@ public abstract class AbstractRipper
}
}
}
/**
* Checks to see if Ripme has already downloaded a URL
* @param url URL to check if downloaded
* @return
* Returns true if previously downloaded.
* Returns false if not yet downloaded.
*/
private boolean hasDownloadedURL(String url) {
File file = new File(URLHistoryFile);
try {
@ -118,6 +125,15 @@ public abstract class AbstractRipper
this.url = sanitizeURL(url);
}
/**
* Sets ripper's:
* Working directory
* Logger (for debugging)
* FileAppender
* Threadpool
* @throws IOException
* Always be prepared.
*/
public void setup() throws IOException {
setWorkingDir(this.url);
Logger rootLogger = Logger.getRootLogger();
@ -155,9 +171,27 @@ public abstract class AbstractRipper
* @param cookies
* The cookies to send to the server while downloading this file.
* @return
* True if downloaded successfully
* False if failed to download
*/
protected abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String, String> cookies);
/**
* Queues image to be downloaded and saved.
* @param url
* URL of the file
* @param prefix
* Prefix for the downloaded file
* @param subdirectory
* Path to get to desired directory from working directory
* @param referrer
* The HTTP referrer to use while downloading this file.
* @param cookies
* The cookies to send to the server while downloading this file.
* @return
* True if downloaded successfully
* False if failed to download
*/
protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map<String, String> cookies) {
if (Utils.getConfigBoolean("remember.url_history", true) && !isThisATest()) {
if (hasDownloadedURL(url.toExternalForm())) {
@ -322,6 +356,11 @@ public abstract class AbstractRipper
}
}
/**
* Gets URL
* @return
* Returns URL that wants to be downloaded.
*/
public URL getURL() {
return url;
}
@ -335,8 +374,20 @@ public abstract class AbstractRipper
return workingDir;
}
@Override
public abstract void setWorkingDir(URL url) throws IOException;
/**
*
* @param url
* The URL you want to get the title of.
* @return
* host_URLid
* e.g. (for a reddit post)
* reddit_post_7mg2ur
* @throws MalformedURLException
* If any of those damned URLs gets malformed.
*/
public String getAlbumTitle(URL url) throws MalformedURLException {
return getHost() + "_" + getGID(url);
}
@ -391,7 +442,7 @@ public abstract class AbstractRipper
/**
* Sends an update message to the relevant observer(s) on this ripper.
* @param status
* @param status
* @param message
*/
public void sendUpdate(STATUS status, Object message) {
@ -400,9 +451,17 @@ public abstract class AbstractRipper
}
observer.update(this, new RipStatusMessage(status, message));
}
/**
* Get the completion percentage.
* @return
* Percentage complete
*/
public abstract int getCompletionPercentage();
/**
* @return
* Text for status
*/
public abstract String getStatusText();
/**
@ -423,7 +482,9 @@ public abstract class AbstractRipper
cleanup();
}
}
/**
* Tries to delete any empty directories
*/
private void cleanup() {
if (this.workingDir.list().length == 0) {
// No files, delete the dir
@ -434,7 +495,15 @@ public abstract class AbstractRipper
}
}
}
/**
* Pauses thread for a set amount of time.
* @param milliseconds
* Amount of time (in milliseconds) that the thread gets paused for
* @return
* True if paused successfully
* False if failed to pause/got interrupted.
*/
protected boolean sleep(int milliseconds) {
try {
logger.debug("Sleeping " + milliseconds + "ms");

21
src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java
View File

@ -13,6 +13,10 @@ import com.rarchives.ripme.ui.RipStatusMessage;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Utils;
/**'
* For ripping delicious albums off the interwebz.
*/
public abstract class AlbumRipper extends AbstractRipper {
private Map<URL, File> itemsPending = Collections.synchronizedMap(new HashMap<URL, File>());
@ -34,10 +38,17 @@ public abstract class AlbumRipper extends AbstractRipper {
}
@Override
/**
* Returns total amount of files attempted.
*/
public int getCount() {
return itemsCompleted.size() + itemsErrored.size();
}
@Override
/**
* Queues multiple URLs of single images to download from a single Album URL
*/
public boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies) {
// Only download one file if this is a test.
if (super.isThisATest() &&
@ -101,6 +112,9 @@ public abstract class AlbumRipper extends AbstractRipper {
}
@Override
/**
* Cleans up & tells user about successful download
*/
public void downloadCompleted(URL url, File saveAs) {
if (observer == null) {
return;
@ -119,6 +133,9 @@ public abstract class AlbumRipper extends AbstractRipper {
}
@Override
/**
* Cleans up & tells user about failed download.
*/
public void downloadErrored(URL url, String reason) {
if (observer == null) {
return;
@ -131,6 +148,10 @@ public abstract class AlbumRipper extends AbstractRipper {
}
@Override
/**
* Tells user that a single file in the album they wish to download has
* already been downloaded in the past.
*/
public void downloadExists(URL url, File file) {
if (observer == null) {
return;

15
src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java
View File

@ -23,17 +23,28 @@ public class DownloadThreadPool {
public DownloadThreadPool(String threadPoolName) {
initialize(threadPoolName);
}
/**
* Initializes the threadpool.
* @param threadPoolName Name of the threadpool.
*/
private void initialize(String threadPoolName) {
int threads = Utils.getConfigInteger("threads.size", 10);
logger.debug("Initializing " + threadPoolName + " thread pool with " + threads + " threads");
threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(threads);
}
/**
* For adding threads to execution pool.
* @param t
* Thread to be added.
*/
public void addThread(Thread t) {
threadPool.execute(t);
}
/**
* Tries to shutdown threadpool.
*/
public void waitForThreads() {
threadPool.shutdown();
try {

6
src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java
View File

@ -136,6 +136,12 @@ class DownloadVideoThread extends Thread {
logger.info("[+] Saved " + url + " as " + this.prettySaveAs);
}
/**
* @param url
* Target URL
* @return
* Returns connection length
*/
private int getTotalBytes(URL url) throws IOException {
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("HEAD");

2
src/main/java/com/rarchives/ripme/ripper/RipperInterface.java
View File

@ -7,6 +7,8 @@ import java.net.URL;
/**
* I have no idea why I made this interface. Everything is captured within the AbstractRipper.
* Oh well, here's to encapsulation and abstraction! (raises glass)
*
* (cheers!)
*/
interface RipperInterface {
void rip() throws IOException;

47
src/main/java/com/rarchives/ripme/ripper/VideoRipper.java
View File

@ -74,6 +74,12 @@ public abstract class VideoRipper extends AbstractRipper {
return addURLToDownload(url, saveAs);
}
/**
* Creates & sets working directory based on URL.
* @param url
* Target URL
*/
@Override
public void setWorkingDir(URL url) throws IOException {
String path = Utils.getWorkingDirectory().getCanonicalPath();
@ -88,12 +94,23 @@ public abstract class VideoRipper extends AbstractRipper {
}
logger.debug("Set working directory to: " + this.workingDir);
}
/**
* @return
* Returns % of video done downloading.
*/
@Override
public int getCompletionPercentage() {
return (int) (100 * (bytesCompleted / (float) bytesTotal));
}
/**
* Runs if download successfully completed.
* @param url
* Target URL
* @param saveAs
* Path to file, including filename.
*/
@Override
public void downloadCompleted(URL url, File saveAs) {
if (observer == null) {
@ -109,6 +126,14 @@ public abstract class VideoRipper extends AbstractRipper {
logger.error("Exception while updating observer: ", e);
}
}
/**
* Runs if the download errored somewhere.
* @param url
* Target URL
* @param reason
* Reason why the download failed.
*/
@Override
public void downloadErrored(URL url, String reason) {
if (observer == null) {
@ -117,6 +142,15 @@ public abstract class VideoRipper extends AbstractRipper {
observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason));
checkIfComplete();
}
/**
* Runs if user tries to redownload an already existing File.
* @param url
* Target URL
* @param file
* Existing file
*/
@Override
public void downloadExists(URL url, File file) {
if (observer == null) {
@ -126,6 +160,11 @@ public abstract class VideoRipper extends AbstractRipper {
checkIfComplete();
}
/**
* Gets the status and changes it to a human-readable form.
* @return
* Status of current download.
*/
@Override
public String getStatusText() {
StringBuilder sb = new StringBuilder();
@ -139,6 +178,10 @@ public abstract class VideoRipper extends AbstractRipper {
}
@Override
/**
* Sanitizes URL.
* Usually just returns itself.
*/
public URL sanitizeURL(URL url) throws MalformedURLException {
return url;
}