diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 46f1f414..ed4cc085 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -1,8 +1,7 @@ package com.rarchives.ripme.ripper; import java.awt.Desktop; -import java.io.File; -import java.io.IOException; +import java.io.*; import java.lang.reflect.Constructor; import java.net.MalformedURLException; import java.net.URL; @@ -21,11 +20,15 @@ import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Utils; +import java.io.File; +import java.util.Scanner; + public abstract class AbstractRipper extends Observable implements RipperInterface, Runnable { protected static final Logger logger = Logger.getLogger(AbstractRipper.class); + private final String URLHistoryFile = Utils.getURLHistoryFile(); public static final String USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:36.0) Gecko/20100101 Firefox/36.0"; @@ -56,6 +59,49 @@ public abstract class AbstractRipper } } + private void writeDownloadedURL(String downloadedURL) throws IOException { + BufferedWriter bw = null; + FileWriter fw = null; + try { + File file = new File(URLHistoryFile); + // if file doesnt exists, then create it + if (!file.exists()) { + file.createNewFile(); + } + fw = new FileWriter(file.getAbsoluteFile(), true); + bw = new BufferedWriter(fw); + bw.write(downloadedURL); + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + if (bw != null) + bw.close(); + if (fw != null) + fw.close(); + } catch (IOException ex) { + ex.printStackTrace(); + } + } + } + + private boolean hasDownloadedURL(String url) { + File file = new File(URLHistoryFile); + try { + Scanner scanner = new Scanner(file); + while (scanner.hasNextLine()) { + final String lineFromFile = scanner.nextLine(); + if (lineFromFile.equals(url)) { + return true; + } + } + } catch (FileNotFoundException e) { + return false; + } + return false; + } + + /** * Ensures inheriting ripper can rip this URL, raises exception if not. * Otherwise initializes working directory and thread pool. @@ -113,6 +159,12 @@ public abstract class AbstractRipper protected abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies); protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies) { + if (Utils.getConfigBoolean("remember.url_history", true)) { + if (hasDownloadedURL(url.toExternalForm())) { + sendUpdate(STATUS.DOWNLOAD_WARN, "Already downloaded " + url.toExternalForm()); + return false; + } + } try { stopCheck(); } catch (IOException e) { @@ -146,6 +198,13 @@ public abstract class AbstractRipper logger.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent())); saveFileAs.getParentFile().mkdirs(); } + if (Utils.getConfigBoolean("remember.url_history", true)) { + try { + writeDownloadedURL(url.toExternalForm() + "\n"); + } catch (IOException e) { + logger.debug("Unable to write URL history file"); + } + } return addURLToDownload(url, saveFileAs, referrer, cookies); } diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 984e750d..7550e256 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -118,6 +118,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static JTextField configRetriesText; private static JCheckBox configAutoupdateCheckbox; private static JComboBox configLogLevelCombobox; + private static JCheckBox configURLHistoryCheckbox; private static JCheckBox configPlaySound; private static JCheckBox configSaveOrderCheckbox; private static JCheckBox configShowPopup; @@ -191,6 +192,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { Utils.setConfigBoolean("clipboard.autorip", configClipboardAutorip.isSelected()); Utils.setConfigBoolean("descriptions.save", configSaveDescriptions.isSelected()); Utils.setConfigBoolean("prefer.mp4", configPreferMp4.isSelected()); + Utils.setConfigBoolean("remember.url_history", configURLHistoryCheckbox.isSelected()); saveWindowPosition(mainFrame); saveHistory(); Utils.saveConfig(); @@ -489,6 +491,9 @@ public final class MainWindow implements Runnable, RipStatusHandler { configWindowPosition = new JCheckBox("Restore window position", Utils.getConfigBoolean("window.position", true)); configWindowPosition.setHorizontalAlignment(JCheckBox.RIGHT); configWindowPosition.setHorizontalTextPosition(JCheckBox.LEFT); + configURLHistoryCheckbox = new JCheckBox("Remember URL history", Utils.getConfigBoolean("remember.url_history", true)); + configURLHistoryCheckbox.setHorizontalAlignment(JCheckBox.RIGHT); + configURLHistoryCheckbox.setHorizontalTextPosition(JCheckBox.LEFT); configSaveDirLabel = new JLabel(); try { String workingDir = (Utils.shortenPath(Utils.getWorkingDirectory())); @@ -520,9 +525,11 @@ public final class MainWindow implements Runnable, RipStatusHandler { gbc.gridy = 9; gbc.gridx = 0; configurationPanel.add(configSaveDescriptions, gbc); gbc.gridx = 1; configurationPanel.add(configPreferMp4, gbc); gbc.gridy = 10; gbc.gridx = 0; configurationPanel.add(configWindowPosition, gbc); + gbc.gridx = 1; configurationPanel.add(configURLHistoryCheckbox, gbc); gbc.gridy = 11; gbc.gridx = 0; configurationPanel.add(configSaveDirLabel, gbc); gbc.gridx = 1; configurationPanel.add(configSaveDirButton, gbc); + emptyPanel = new JPanel(); emptyPanel.setPreferredSize(new Dimension(0, 0)); emptyPanel.setSize(0, 0); @@ -665,6 +672,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { saveHistory(); }); historyButtonClear.addActionListener(event -> { + Utils.clearURLHistory(); HISTORY.clear(); try { historyTableModel.fireTableDataChanged(); @@ -743,6 +751,10 @@ public final class MainWindow implements Runnable, RipStatusHandler { Utils.setConfigBoolean("urls_only.save", configSaveURLsOnly.isSelected()); Utils.configureLogger(); }); + configURLHistoryCheckbox.addActionListener(arg0 -> { + Utils.setConfigBoolean("remember.url_history", configURLHistoryCheckbox.isSelected()); + Utils.configureLogger(); + }); configSaveAlbumTitles.addActionListener(arg0 -> { Utils.setConfigBoolean("album_titles.save", configSaveAlbumTitles.isSelected()); Utils.configureLogger(); diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 086fe634..7a34e6e3 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -175,6 +175,16 @@ public class Utils { return "."; } } + // Delete the url history file + public static void clearURLHistory() { + File file = new File(getURLHistoryFile()); + file.delete(); + } + + // Return the path of the url history file + public static String getURLHistoryFile() { + return getConfigDir() + File.separator + "url_history.txt"; + } private static String getConfigFilePath() { return getConfigDir() + File.separator + configFile;