From d11443e8ec6cbc663e999b25f0b7be00db7ef22d Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 5 Nov 2017 14:35:15 -0500 Subject: [PATCH 1/5] Can amlost skip already downloaded urls --- .../ripme/ripper/AbstractRipper.java | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 46f1f414..ef051b67 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -20,6 +20,10 @@ import com.rarchives.ripme.ui.RipStatusHandler; import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Utils; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.util.Scanner; public abstract class AbstractRipper extends Observable @@ -56,6 +60,44 @@ public abstract class AbstractRipper } } + private void writeDownloadedURL(String downloadedURL) throws IOException { + BufferedWriter bw = null; + FileWriter fw = null; + try { + File file = new File("history.downloaded"); + // if file doesnt exists, then create it + if (!file.exists()) { + file.createNewFile(); + } + fw = new FileWriter(file.getAbsoluteFile(), true); + bw = new BufferedWriter(fw); + bw.write(downloadedURL); + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + if (bw != null) + bw.close(); + if (fw != null) + fw.close(); + } catch (IOException ex) { + ex.printStackTrace(); + } + } + } + + private boolean hasDownloadedURL(String url) { + Scanner scanner = new Scanner("history.downloaded"); + while (scanner.hasNextLine()) { + final String lineFromFile = scanner.nextLine(); + if(lineFromFile.contains(url)) { + return true; + } + } + return false; + } + + /** * Ensures inheriting ripper can rip this URL, raises exception if not. * Otherwise initializes working directory and thread pool. @@ -113,6 +155,10 @@ public abstract class AbstractRipper protected abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies); protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies) { + if (!hasDownloadedURL(url.toExternalForm())) { + sendUpdate(STATUS.DOWNLOAD_WARN, "Already downloaded " + url.toExternalForm()); + return false; + } try { stopCheck(); } catch (IOException e) { @@ -146,6 +192,11 @@ public abstract class AbstractRipper logger.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent())); saveFileAs.getParentFile().mkdirs(); } + try { + writeDownloadedURL(url.toExternalForm() + "\n"); + } catch (IOException e) { + logger.debug("Unable to write URL history file"); + } return addURLToDownload(url, saveFileAs, referrer, cookies); } From c69613d8a54d13be8379f7865dbd65a9fda039a0 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 7 Nov 2017 04:11:13 -0500 Subject: [PATCH 2/5] Ripme now remembers downloaded URLs and doesn't try to redownload them --- .../ripme/ripper/AbstractRipper.java | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index ef051b67..d331b4a5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -1,8 +1,7 @@ package com.rarchives.ripme.ripper; import java.awt.Desktop; -import java.io.File; -import java.io.IOException; +import java.io.*; import java.lang.reflect.Constructor; import java.net.MalformedURLException; import java.net.URL; @@ -20,9 +19,8 @@ import com.rarchives.ripme.ui.RipStatusHandler; import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Utils; -import java.io.BufferedWriter; + import java.io.File; -import java.io.FileWriter; import java.util.Scanner; public abstract class AbstractRipper @@ -30,6 +28,7 @@ public abstract class AbstractRipper implements RipperInterface, Runnable { protected static final Logger logger = Logger.getLogger(AbstractRipper.class); + private final String URLHistoryFile = Utils.getConfigDir() + File.separator + "url_history.txt"; public static final String USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:36.0) Gecko/20100101 Firefox/36.0"; @@ -64,7 +63,7 @@ public abstract class AbstractRipper BufferedWriter bw = null; FileWriter fw = null; try { - File file = new File("history.downloaded"); + File file = new File(URLHistoryFile); // if file doesnt exists, then create it if (!file.exists()) { file.createNewFile(); @@ -86,13 +85,18 @@ public abstract class AbstractRipper } } - private boolean hasDownloadedURL(String url) { - Scanner scanner = new Scanner("history.downloaded"); - while (scanner.hasNextLine()) { - final String lineFromFile = scanner.nextLine(); - if(lineFromFile.contains(url)) { - return true; + public boolean hasDownloadedURL(String url) { + File file = new File(URLHistoryFile); + try { + Scanner scanner = new Scanner(file); + while (scanner.hasNextLine()) { + final String lineFromFile = scanner.nextLine(); + if (lineFromFile.equals(url)) { + return true; + } } + } catch (FileNotFoundException e) { + return false; } return false; } @@ -155,7 +159,7 @@ public abstract class AbstractRipper protected abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies); protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies) { - if (!hasDownloadedURL(url.toExternalForm())) { + if (hasDownloadedURL(url.toExternalForm())) { sendUpdate(STATUS.DOWNLOAD_WARN, "Already downloaded " + url.toExternalForm()); return false; } From 169547d745fb7fcca095d7b0c4e44d66e41e91d7 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 7 Nov 2017 04:27:36 -0500 Subject: [PATCH 3/5] Change method that didn't need to be public to private --- src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index d331b4a5..8a72482a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -85,7 +85,7 @@ public abstract class AbstractRipper } } - public boolean hasDownloadedURL(String url) { + private boolean hasDownloadedURL(String url) { File file = new File(URLHistoryFile); try { Scanner scanner = new Scanner(file); From 9898c149e1abac1e7f08fb30e2a266c6ef054edc Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 7 Nov 2017 04:49:39 -0500 Subject: [PATCH 4/5] Added Utils.getURLHistoryFile() func to get the URL history file path, clearing history now deletes url history file --- .../com/rarchives/ripme/ripper/AbstractRipper.java | 2 +- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 1 + src/main/java/com/rarchives/ripme/utils/Utils.java | 11 ++++++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 8a72482a..4bf1a7e0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -28,7 +28,7 @@ public abstract class AbstractRipper implements RipperInterface, Runnable { protected static final Logger logger = Logger.getLogger(AbstractRipper.class); - private final String URLHistoryFile = Utils.getConfigDir() + File.separator + "url_history.txt"; + private final String URLHistoryFile = Utils.getURLHistoryFile(); public static final String USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:36.0) Gecko/20100101 Firefox/36.0"; diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 984e750d..38c73889 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -665,6 +665,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { saveHistory(); }); historyButtonClear.addActionListener(event -> { + Utils.clearURLHistory(); HISTORY.clear(); try { historyTableModel.fireTableDataChanged(); diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 28be6639..7a34e6e3 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -20,7 +20,6 @@ import javax.sound.sampled.AudioSystem; import javax.sound.sampled.Clip; import javax.sound.sampled.Line; import javax.sound.sampled.LineEvent; -import javax.sound.sampled.LineListener; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; @@ -176,6 +175,16 @@ public class Utils { return "."; } } + // Delete the url history file + public static void clearURLHistory() { + File file = new File(getURLHistoryFile()); + file.delete(); + } + + // Return the path of the url history file + public static String getURLHistoryFile() { + return getConfigDir() + File.separator + "url_history.txt"; + } private static String getConfigFilePath() { return getConfigDir() + File.separator + configFile; From d5c1efd9390284c956575f287a56f68d99c993a1 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 7 Nov 2017 15:23:41 -0500 Subject: [PATCH 5/5] Now uses checkbox for remember.url_history setting --- .../rarchives/ripme/ripper/AbstractRipper.java | 16 ++++++++++------ .../java/com/rarchives/ripme/ui/MainWindow.java | 11 +++++++++++ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 4bf1a7e0..ed4cc085 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -159,9 +159,11 @@ public abstract class AbstractRipper protected abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies); protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies) { - if (hasDownloadedURL(url.toExternalForm())) { - sendUpdate(STATUS.DOWNLOAD_WARN, "Already downloaded " + url.toExternalForm()); - return false; + if (Utils.getConfigBoolean("remember.url_history", true)) { + if (hasDownloadedURL(url.toExternalForm())) { + sendUpdate(STATUS.DOWNLOAD_WARN, "Already downloaded " + url.toExternalForm()); + return false; + } } try { stopCheck(); @@ -196,10 +198,12 @@ public abstract class AbstractRipper logger.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent())); saveFileAs.getParentFile().mkdirs(); } - try { - writeDownloadedURL(url.toExternalForm() + "\n"); - } catch (IOException e) { + if (Utils.getConfigBoolean("remember.url_history", true)) { + try { + writeDownloadedURL(url.toExternalForm() + "\n"); + } catch (IOException e) { logger.debug("Unable to write URL history file"); + } } return addURLToDownload(url, saveFileAs, referrer, cookies); } diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 38c73889..7550e256 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -118,6 +118,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static JTextField configRetriesText; private static JCheckBox configAutoupdateCheckbox; private static JComboBox configLogLevelCombobox; + private static JCheckBox configURLHistoryCheckbox; private static JCheckBox configPlaySound; private static JCheckBox configSaveOrderCheckbox; private static JCheckBox configShowPopup; @@ -191,6 +192,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { Utils.setConfigBoolean("clipboard.autorip", configClipboardAutorip.isSelected()); Utils.setConfigBoolean("descriptions.save", configSaveDescriptions.isSelected()); Utils.setConfigBoolean("prefer.mp4", configPreferMp4.isSelected()); + Utils.setConfigBoolean("remember.url_history", configURLHistoryCheckbox.isSelected()); saveWindowPosition(mainFrame); saveHistory(); Utils.saveConfig(); @@ -489,6 +491,9 @@ public final class MainWindow implements Runnable, RipStatusHandler { configWindowPosition = new JCheckBox("Restore window position", Utils.getConfigBoolean("window.position", true)); configWindowPosition.setHorizontalAlignment(JCheckBox.RIGHT); configWindowPosition.setHorizontalTextPosition(JCheckBox.LEFT); + configURLHistoryCheckbox = new JCheckBox("Remember URL history", Utils.getConfigBoolean("remember.url_history", true)); + configURLHistoryCheckbox.setHorizontalAlignment(JCheckBox.RIGHT); + configURLHistoryCheckbox.setHorizontalTextPosition(JCheckBox.LEFT); configSaveDirLabel = new JLabel(); try { String workingDir = (Utils.shortenPath(Utils.getWorkingDirectory())); @@ -520,9 +525,11 @@ public final class MainWindow implements Runnable, RipStatusHandler { gbc.gridy = 9; gbc.gridx = 0; configurationPanel.add(configSaveDescriptions, gbc); gbc.gridx = 1; configurationPanel.add(configPreferMp4, gbc); gbc.gridy = 10; gbc.gridx = 0; configurationPanel.add(configWindowPosition, gbc); + gbc.gridx = 1; configurationPanel.add(configURLHistoryCheckbox, gbc); gbc.gridy = 11; gbc.gridx = 0; configurationPanel.add(configSaveDirLabel, gbc); gbc.gridx = 1; configurationPanel.add(configSaveDirButton, gbc); + emptyPanel = new JPanel(); emptyPanel.setPreferredSize(new Dimension(0, 0)); emptyPanel.setSize(0, 0); @@ -744,6 +751,10 @@ public final class MainWindow implements Runnable, RipStatusHandler { Utils.setConfigBoolean("urls_only.save", configSaveURLsOnly.isSelected()); Utils.configureLogger(); }); + configURLHistoryCheckbox.addActionListener(arg0 -> { + Utils.setConfigBoolean("remember.url_history", configURLHistoryCheckbox.isSelected()); + Utils.configureLogger(); + }); configSaveAlbumTitles.addActionListener(arg0 -> { Utils.setConfigBoolean("album_titles.save", configSaveAlbumTitles.isSelected()); Utils.configureLogger();