From 5b49514e3e4dc18e038cb45970da34d8601912c1 Mon Sep 17 00:00:00 2001 From: Kevin Jiang Date: Thu, 28 Dec 2017 00:04:23 -0500 Subject: [PATCH] 3 Metric Tons of Javadoc documentation for Core Ripper stuff. There should be no changes to the code. Just a ton of documentation. It should help newcomers get aquainted with code quicker. --- src/main/java/com/rarchives/ripme/App.java | 36 +++++++-- .../ripme/ripper/AbstractHTMLRipper.java | 30 +++++++ .../ripme/ripper/AbstractJSONRipper.java | 1 + .../ripme/ripper/AbstractRipper.java | 81 +++++++++++++++++-- .../rarchives/ripme/ripper/AlbumRipper.java | 21 +++++ .../ripme/ripper/DownloadThreadPool.java | 15 +++- .../ripme/ripper/DownloadVideoThread.java | 6 ++ .../ripme/ripper/RipperInterface.java | 2 + .../rarchives/ripme/ripper/VideoRipper.java | 47 ++++++++++- 9 files changed, 223 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index e046d5a6..0bb5f3f6 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -63,13 +63,21 @@ public class App { SwingUtilities.invokeLater(mw); } } - + /** + * Creates an abstract ripper and instructs it to rip. + * @param url URL to be ripped + * @throws Exception + */ private static void rip(URL url) throws Exception { AbstractRipper ripper = AbstractRipper.getRipper(url); ripper.setup(); ripper.rip(); } + /** + * For dealing with command-line arguments. + * @param args Array of Command-line arguments + */ private static void handleArguments(String[] args) { CommandLine cl = getArgs(args); if (cl.hasOption('h')) { @@ -109,7 +117,7 @@ public class App { } if (cl.hasOption('R')) { loadHistory(); - if (HISTORY.toList().size() == 0) { + if (HISTORY.toList().isEmpty()) { logger.error("There are no history entries to re-rip. Rip some albums first"); System.exit(-1); } @@ -173,14 +181,18 @@ public class App { } } - // this function will attempt to rip the provided url + /** + * Attempt to rip targetURL. + * @param targetURL URL to rip + * @param saveConfig Whether or not you want to save the config (?) + */ private static void ripURL(String targetURL, boolean saveConfig) { try { URL url = new URL(targetURL); rip(url); List history = Utils.getConfigList("download.history"); - if (!history.contains(url.toExternalForm())) { - history.add(url.toExternalForm()); + if (!history.contains(url.toExternalForm())) {//if you haven't already downloaded the file before + history.add(url.toExternalForm());//add it to history so you won't have to redownload Utils.setConfigList("download.history", Arrays.asList(history.toArray())); if (saveConfig) { Utils.saveConfig(); @@ -195,6 +207,10 @@ public class App { } } + /** + * Creates an Options object, returns it. + * @return Returns all acceptable command-line options. + */ private static Options getOptions() { Options opts = new Options(); opts.addOption("h", "help", false, "Print the help"); @@ -213,6 +229,11 @@ public class App { return opts; } + /** + * Tries to parse commandline arguments. + * @param args Array of commandline arguments. + * @return CommandLine object containing arguments. + */ private static CommandLine getArgs(String[] args) { BasicParser parser = new BasicParser(); try { @@ -223,7 +244,10 @@ public class App { return null; } } - + + /** + * Loads history from history file into memory. + */ private static void loadHistory() { File historyFile = new File(Utils.getConfigDir() + File.separator + "history.json"); HISTORY.clear(); diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 3b7496a0..25d51007 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -140,6 +140,14 @@ public abstract class AbstractHTMLRipper extends AlbumRipper { } waitForThreads(); } + + /** + * Gets the file name from the URL + * @param url + * URL that you want to get the filename from + * @return + * Filename of the URL + */ private String fileNameFromURL(URL url) { String saveAs = url.toExternalForm(); if (saveAs.substring(saveAs.length() - 1) == "/") { saveAs = saveAs.substring(0,saveAs.length() - 1) ;} @@ -150,6 +158,20 @@ public abstract class AbstractHTMLRipper extends AlbumRipper { if (saveAs.indexOf(':') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf(':')); } return saveAs; } + /** + * + * @param url + * Target URL + * @param subdirectory + * Path to subdirectory where you want to save it + * @param text + * Text you want to save + * @param index + * Index in something like an album + * @return + * True if ripped successfully + * False if failed + */ public boolean saveText(URL url, String subdirectory, String text, int index) { String saveAs = fileNameFromURL(url); return saveText(url,subdirectory,text,index,saveAs); @@ -189,6 +211,14 @@ public abstract class AbstractHTMLRipper extends AlbumRipper { } return true; } + + /** + * Gets prefix based on where in the index it is + * @param index + * The index in question + * @return + * Returns prefix for a file. (?) + */ protected String getPrefix(int index) { String prefix = ""; if (keepSortOrder() && Utils.getConfigBoolean("download.save_order", true)) { diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index a315bcdc..8c73bbb6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -20,6 +20,7 @@ public abstract class AbstractJSONRipper extends AlbumRipper { } protected abstract String getDomain(); + @Override public abstract String getHost(); protected abstract JSONObject getFirstPage() throws IOException; diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index a852e177..8209df68 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -84,7 +84,14 @@ public abstract class AbstractRipper } } } - + + /** + * Checks to see if Ripme has already downloaded a URL + * @param url URL to check if downloaded + * @return + * Returns true if previously downloaded. + * Returns false if not yet downloaded. + */ private boolean hasDownloadedURL(String url) { File file = new File(URLHistoryFile); try { @@ -118,6 +125,15 @@ public abstract class AbstractRipper this.url = sanitizeURL(url); } + /** + * Sets ripper's: + * Working directory + * Logger (for debugging) + * FileAppender + * Threadpool + * @throws IOException + * Always be prepared. + */ public void setup() throws IOException { setWorkingDir(this.url); Logger rootLogger = Logger.getRootLogger(); @@ -155,9 +171,27 @@ public abstract class AbstractRipper * @param cookies * The cookies to send to the server while downloading this file. * @return + * True if downloaded successfully + * False if failed to download */ protected abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies); + /** + * Queues image to be downloaded and saved. + * @param url + * URL of the file + * @param prefix + * Prefix for the downloaded file + * @param subdirectory + * Path to get to desired directory from working directory + * @param referrer + * The HTTP referrer to use while downloading this file. + * @param cookies + * The cookies to send to the server while downloading this file. + * @return + * True if downloaded successfully + * False if failed to download + */ protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies) { if (Utils.getConfigBoolean("remember.url_history", true) && !isThisATest()) { if (hasDownloadedURL(url.toExternalForm())) { @@ -322,6 +356,11 @@ public abstract class AbstractRipper } } + /** + * Gets URL + * @return + * Returns URL that wants to be downloaded. + */ public URL getURL() { return url; } @@ -335,8 +374,20 @@ public abstract class AbstractRipper return workingDir; } + @Override public abstract void setWorkingDir(URL url) throws IOException; + /** + * + * @param url + * The URL you want to get the title of. + * @return + * host_URLid + * e.g. (for a reddit post) + * reddit_post_7mg2ur + * @throws MalformedURLException + * If any of those damned URLs gets malformed. + */ public String getAlbumTitle(URL url) throws MalformedURLException { return getHost() + "_" + getGID(url); } @@ -391,7 +442,7 @@ public abstract class AbstractRipper /** * Sends an update message to the relevant observer(s) on this ripper. - * @param status + * @param status * @param message */ public void sendUpdate(STATUS status, Object message) { @@ -400,9 +451,17 @@ public abstract class AbstractRipper } observer.update(this, new RipStatusMessage(status, message)); } - + + /** + * Get the completion percentage. + * @return + * Percentage complete + */ public abstract int getCompletionPercentage(); - + /** + * @return + * Text for status + */ public abstract String getStatusText(); /** @@ -423,7 +482,9 @@ public abstract class AbstractRipper cleanup(); } } - + /** + * Tries to delete any empty directories + */ private void cleanup() { if (this.workingDir.list().length == 0) { // No files, delete the dir @@ -434,7 +495,15 @@ public abstract class AbstractRipper } } } - + + /** + * Pauses thread for a set amount of time. + * @param milliseconds + * Amount of time (in milliseconds) that the thread gets paused for + * @return + * True if paused successfully + * False if failed to pause/got interrupted. + */ protected boolean sleep(int milliseconds) { try { logger.debug("Sleeping " + milliseconds + "ms"); diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index a5b2c91e..a92f3870 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -13,6 +13,10 @@ import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Utils; + +/**' + * For ripping delicious albums off the interwebz. + */ public abstract class AlbumRipper extends AbstractRipper { private Map itemsPending = Collections.synchronizedMap(new HashMap()); @@ -34,10 +38,17 @@ public abstract class AlbumRipper extends AbstractRipper { } @Override + /** + * Returns total amount of files attempted. + */ public int getCount() { return itemsCompleted.size() + itemsErrored.size(); } + @Override + /** + * Queues multiple URLs of single images to download from a single Album URL + */ public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies) { // Only download one file if this is a test. if (super.isThisATest() && @@ -101,6 +112,9 @@ public abstract class AlbumRipper extends AbstractRipper { } @Override + /** + * Cleans up & tells user about successful download + */ public void downloadCompleted(URL url, File saveAs) { if (observer == null) { return; @@ -119,6 +133,9 @@ public abstract class AlbumRipper extends AbstractRipper { } @Override + /** + * Cleans up & tells user about failed download. + */ public void downloadErrored(URL url, String reason) { if (observer == null) { return; @@ -131,6 +148,10 @@ public abstract class AlbumRipper extends AbstractRipper { } @Override + /** + * Tells user that a single file in the album they wish to download has + * already been downloaded in the past. + */ public void downloadExists(URL url, File file) { if (observer == null) { return; diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java index 30e3c2b7..a811c98a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java @@ -23,17 +23,28 @@ public class DownloadThreadPool { public DownloadThreadPool(String threadPoolName) { initialize(threadPoolName); } - + + /** + * Initializes the threadpool. + * @param threadPoolName Name of the threadpool. + */ private void initialize(String threadPoolName) { int threads = Utils.getConfigInteger("threads.size", 10); logger.debug("Initializing " + threadPoolName + " thread pool with " + threads + " threads"); threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(threads); } - + /** + * For adding threads to execution pool. + * @param t + * Thread to be added. + */ public void addThread(Thread t) { threadPool.execute(t); } + /** + * Tries to shutdown threadpool. + */ public void waitForThreads() { threadPool.shutdown(); try { diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java index ee97ce60..437f18d0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java @@ -136,6 +136,12 @@ class DownloadVideoThread extends Thread { logger.info("[+] Saved " + url + " as " + this.prettySaveAs); } + /** + * @param url + * Target URL + * @return + * Returns connection length + */ private int getTotalBytes(URL url) throws IOException { HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod("HEAD"); diff --git a/src/main/java/com/rarchives/ripme/ripper/RipperInterface.java b/src/main/java/com/rarchives/ripme/ripper/RipperInterface.java index 57ff8418..550209c0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/RipperInterface.java +++ b/src/main/java/com/rarchives/ripme/ripper/RipperInterface.java @@ -7,6 +7,8 @@ import java.net.URL; /** * I have no idea why I made this interface. Everything is captured within the AbstractRipper. * Oh well, here's to encapsulation and abstraction! (raises glass) + * + * (cheers!) */ interface RipperInterface { void rip() throws IOException; diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 62be7858..13008cd9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -74,6 +74,12 @@ public abstract class VideoRipper extends AbstractRipper { return addURLToDownload(url, saveAs); } + + /** + * Creates & sets working directory based on URL. + * @param url + * Target URL + */ @Override public void setWorkingDir(URL url) throws IOException { String path = Utils.getWorkingDirectory().getCanonicalPath(); @@ -88,12 +94,23 @@ public abstract class VideoRipper extends AbstractRipper { } logger.debug("Set working directory to: " + this.workingDir); } - + + /** + * @return + * Returns % of video done downloading. + */ @Override public int getCompletionPercentage() { return (int) (100 * (bytesCompleted / (float) bytesTotal)); } - + + /** + * Runs if download successfully completed. + * @param url + * Target URL + * @param saveAs + * Path to file, including filename. + */ @Override public void downloadCompleted(URL url, File saveAs) { if (observer == null) { @@ -109,6 +126,14 @@ public abstract class VideoRipper extends AbstractRipper { logger.error("Exception while updating observer: ", e); } } + + /** + * Runs if the download errored somewhere. + * @param url + * Target URL + * @param reason + * Reason why the download failed. + */ @Override public void downloadErrored(URL url, String reason) { if (observer == null) { @@ -117,6 +142,15 @@ public abstract class VideoRipper extends AbstractRipper { observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason)); checkIfComplete(); } + + + /** + * Runs if user tries to redownload an already existing File. + * @param url + * Target URL + * @param file + * Existing file + */ @Override public void downloadExists(URL url, File file) { if (observer == null) { @@ -126,6 +160,11 @@ public abstract class VideoRipper extends AbstractRipper { checkIfComplete(); } + /** + * Gets the status and changes it to a human-readable form. + * @return + * Status of current download. + */ @Override public String getStatusText() { StringBuilder sb = new StringBuilder(); @@ -139,6 +178,10 @@ public abstract class VideoRipper extends AbstractRipper { } @Override + /** + * Sanitizes URL. + * Usually just returns itself. + */ public URL sanitizeURL(URL url) throws MalformedURLException { return url; }