From 441790bb3ce0b408628d55a03ca9b2d9fbfe0409 Mon Sep 17 00:00:00 2001 From: Erwin de Haan Date: Tue, 9 May 2017 23:02:24 +0200 Subject: [PATCH] ChanRipper: now downloads imgur links, etc. (#185) --- src/main/java/com/rarchives/ripme/App.java | 2 +- .../ripme/ripper/AbstractRipper.java | 37 ++++++++++++++-- .../ripme/ripper/rippers/ChanRipper.java | 25 +++++++++-- .../ripme/ripper/rippers/RedditRipper.java | 9 ++-- .../java/com/rarchives/ripme/ui/History.java | 12 +++++- .../com/rarchives/ripme/ui/HistoryEntry.java | 1 + .../ripme/ui/HistoryMenuMouseListener.java | 8 ++-- .../com/rarchives/ripme/ui/MainWindow.java | 42 +++++++++++++------ .../rarchives/ripme/ui/RipStatusMessage.java | 1 + .../com/rarchives/ripme/ui/UpdateUtils.java | 11 +++-- 10 files changed, 111 insertions(+), 37 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index e6d22aac..7ec0b483 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -216,7 +216,7 @@ public class App { CommandLine cl = parser.parse(getOptions(), args, false); return cl; } catch (ParseException e) { - logger.error("[!] Error while parsing command-line arguments: " + args, e); + logger.error("[!] Error while parsing command-line arguments: " + Arrays.toString(args), e); System.exit(-1); return null; } diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 5b269808..3d7c17db 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -20,6 +20,7 @@ import com.rarchives.ripme.ui.RipStatusHandler; import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Utils; +import java.lang.reflect.InvocationTargetException; public abstract class AbstractRipper extends Observable @@ -94,8 +95,22 @@ public abstract class AbstractRipper * URL of the file * @param saveAs * Path of the local file to save the content to. + * @return True on success, flase on failure. */ public abstract boolean addURLToDownload(URL url, File saveAs); + + /** + * Queues image to be downloaded and saved. + * @param url + * URL of the file + * @param saveAs + * Path of the local file to save the content to. + * @param referrer + * The HTTP referrer to use while downloading this file. + * @param cookies + * The cookies to send to the server while downloading this file. + * @return + */ public abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies); public boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies) { @@ -144,6 +159,7 @@ public abstract class AbstractRipper * Prefix to prepend to the saved filename. * @param subdirectory * Sub-directory of the working directory to save the images to. + * @return True on success, flase on failure. */ public boolean addURLToDownload(URL url, String prefix, String subdirectory) { return addURLToDownload(url, prefix, subdirectory, null, null); @@ -156,6 +172,7 @@ public abstract class AbstractRipper * URL to download * @param prefix * Text to append to saved filename. + * @return True on success, flase on failure. */ public boolean addURLToDownload(URL url, String prefix) { // Use empty subdirectory @@ -201,7 +218,7 @@ public abstract class AbstractRipper * Notify observers that a download could not be completed, * but was not technically an "error". * @param url - * @param message + * @param file */ public abstract void downloadExists(URL url, File file); @@ -281,7 +298,13 @@ public abstract class AbstractRipper AlbumRipper ripper = (AlbumRipper) constructor.newInstance(url); logger.debug("Found album ripper: " + ripper.getClass().getName()); return ripper; - } catch (Exception e) { + } catch (InstantiationException e) { + // Incompatible rippers *will* throw exceptions during instantiation. + } catch (IllegalAccessException e) { + // Incompatible rippers *will* throw exceptions during instantiation. + } catch (IllegalArgumentException e) { + // Incompatible rippers *will* throw exceptions during instantiation. + } catch (InvocationTargetException e) { // Incompatible rippers *will* throw exceptions during instantiation. } } @@ -290,7 +313,13 @@ public abstract class AbstractRipper VideoRipper ripper = (VideoRipper) constructor.newInstance(url); logger.debug("Found video ripper: " + ripper.getClass().getName()); return ripper; - } catch (Exception e) { + } catch (InstantiationException e) { + // Incompatible rippers *will* throw exceptions during instantiation. + } catch (IllegalAccessException e) { + // Incompatible rippers *will* throw exceptions during instantiation. + } catch (IllegalArgumentException e) { + // Incompatible rippers *will* throw exceptions during instantiation. + } catch (InvocationTargetException e) { // Incompatible rippers *will* throw exceptions during instantiation. } } @@ -298,6 +327,8 @@ public abstract class AbstractRipper } /** + * @param pkg + * The package name. * @return * List of constructors for all eligible Rippers. * @throws Exception diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java index b9289f28..30a26ca5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java @@ -11,11 +11,12 @@ import java.util.regex.Pattern; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ripper.rippers.ripperhelpers.ChanSite; import com.rarchives.ripme.utils.Http; +import com.rarchives.ripme.utils.RipUtils; + public class ChanRipper extends AbstractHTMLRipper { public static List explicit_domains = Arrays.asList( @@ -90,7 +91,11 @@ public class ChanRipper extends AbstractHTMLRipper { * For example the archives are all known. (Check 4chan-x) * Should be based on the software the specific chan uses. * FoolFuuka uses the same (url) layout as 4chan - * */ + * + * @param url + * @return + * The thread id in string form + * @throws java.net.MalformedURLException */ @Override public String getGID(URL url) throws MalformedURLException { Pattern p; @@ -181,8 +186,20 @@ public class ChanRipper extends AbstractHTMLRipper { } } } else { - //TODO also grab imgur/flickr albums (And all other supported rippers) Maybe add a setting? - } + //Copied code from RedditRipper, getFilesFromURL should also implement stuff like flickr albums + URL originalURL; + try { + originalURL = new URL(href); + } catch (MalformedURLException e) { + continue; + } + + List urls = RipUtils.getFilesFromURL(originalURL); + //for (int i = 0; i < urls.size(); i++) { + for(URL imageurl : urls){ + imageURLs.add(imageurl.toString()); + } + } if (isStopped()) { break; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index 067e7866..903fe3d1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -171,11 +171,12 @@ public class RedditRipper extends AlbumRipper { Pattern p = RipUtils.getURLRegex(); Matcher m = p.matcher(body); while (m.find()) { - String url = m.group(1); - while (url.endsWith(")")) { - url = url.substring(0, url.length() - 1); + String foundurl; + foundurl = m.group(1); + while (foundurl.endsWith(")")) { + foundurl = foundurl.substring(0, foundurl.length() - 1); } - handleURL(url, id); + handleURL(foundurl, id); } } diff --git a/src/main/java/com/rarchives/ripme/ui/History.java b/src/main/java/com/rarchives/ripme/ui/History.java index 79c31836..0ba751a3 100644 --- a/src/main/java/com/rarchives/ripme/ui/History.java +++ b/src/main/java/com/rarchives/ripme/ui/History.java @@ -17,7 +17,7 @@ import org.json.JSONObject; public class History { - private List list = new ArrayList(); + private final List list; private static final String[] COLUMNS = new String[] { "URL", "created", @@ -26,6 +26,10 @@ public class History { "" }; + public History() { + this.list = new ArrayList(); + } + public void add(HistoryEntry entry) { list.add(entry); } @@ -124,10 +128,14 @@ public class History { return jsonArray; } - public List toList() { + public List toList() { return list; } + public boolean isEmpty(){ + return list.isEmpty(); + } + public void toFile(String filename) throws IOException { OutputStream os = new FileOutputStream(filename); try { diff --git a/src/main/java/com/rarchives/ripme/ui/HistoryEntry.java b/src/main/java/com/rarchives/ripme/ui/HistoryEntry.java index 72d01ff9..53be0a21 100644 --- a/src/main/java/com/rarchives/ripme/ui/HistoryEntry.java +++ b/src/main/java/com/rarchives/ripme/ui/HistoryEntry.java @@ -47,6 +47,7 @@ public class HistoryEntry { return json; } + @Override public String toString() { return this.url; } diff --git a/src/main/java/com/rarchives/ripme/ui/HistoryMenuMouseListener.java b/src/main/java/com/rarchives/ripme/ui/HistoryMenuMouseListener.java index 1855cf96..30e81b86 100644 --- a/src/main/java/com/rarchives/ripme/ui/HistoryMenuMouseListener.java +++ b/src/main/java/com/rarchives/ripme/ui/HistoryMenuMouseListener.java @@ -24,7 +24,7 @@ public class HistoryMenuMouseListener extends MouseAdapter { @Override public void actionPerformed(ActionEvent ae) { for (int row = 0; row < tableComponent.getRowCount(); row++) { - tableComponent.setValueAt(new Boolean(true), row, 4); + tableComponent.setValueAt(true, row, 4); } } }; @@ -34,7 +34,7 @@ public class HistoryMenuMouseListener extends MouseAdapter { @Override public void actionPerformed(ActionEvent ae) { for (int row = 0; row < tableComponent.getRowCount(); row++) { - tableComponent.setValueAt(new Boolean(false), row, 4); + tableComponent.setValueAt(false, row, 4); } } }; @@ -46,7 +46,7 @@ public class HistoryMenuMouseListener extends MouseAdapter { @Override public void actionPerformed(ActionEvent ae) { for (int row : tableComponent.getSelectedRows()) { - tableComponent.setValueAt(new Boolean(true), row, 4); + tableComponent.setValueAt(true, row, 4); } } }; @@ -56,7 +56,7 @@ public class HistoryMenuMouseListener extends MouseAdapter { @Override public void actionPerformed(ActionEvent ae) { for (int row : tableComponent.getSelectedRows()) { - tableComponent.setValueAt(new Boolean(false), row, 4); + tableComponent.setValueAt(false, row, 4); } } }; diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index df2701d2..23938206 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -75,11 +75,13 @@ import org.apache.log4j.Logger; import com.rarchives.ripme.ripper.AbstractRipper; import com.rarchives.ripme.utils.RipUtils; import com.rarchives.ripme.utils.Utils; +import java.awt.AWTException; +import javax.swing.UnsupportedLookAndFeelException; /** * Everything UI-related starts and ends here. */ -public class MainWindow implements Runnable, RipStatusHandler { +public final class MainWindow implements Runnable, RipStatusHandler { private static final Logger logger = Logger.getLogger(MainWindow.class); @@ -255,7 +257,13 @@ public class MainWindow implements Runnable, RipStatusHandler { try { UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName()); - } catch (Exception e) { + } catch (ClassNotFoundException e) { + logger.error("[!] Exception setting system theme:", e); + } catch (InstantiationException e) { + logger.error("[!] Exception setting system theme:", e); + } catch (IllegalAccessException e) { + logger.error("[!] Exception setting system theme:", e); + } catch (UnsupportedLookAndFeelException e) { logger.error("[!] Exception setting system theme:", e); } @@ -271,12 +279,13 @@ public class MainWindow implements Runnable, RipStatusHandler { } catch (Exception e) { } JPanel ripPanel = new JPanel(new GridBagLayout()); ripPanel.setBorder(emptyBorder); - - gbc.gridx = 0; ripPanel.add(new JLabel("URL:", JLabel.RIGHT), gbc); - gbc.gridx = 1; ripPanel.add(ripTextfield, gbc); - gbc.gridx = 2; ripPanel.add(ripButton, gbc); - gbc.gridx = 3; ripPanel.add(stopButton, gbc); - + + gbc.gridx = 0; gbc.fill = GridBagConstraints.HORIZONTAL; ripPanel.add(new JLabel("URL:", JLabel.RIGHT), gbc); + gbc.gridx = 1; gbc.fill = GridBagConstraints.HORIZONTAL; ripPanel.add(ripTextfield, gbc); + gbc.gridx = 2; gbc.fill = GridBagConstraints.HORIZONTAL; ripPanel.add(ripButton, gbc); + gbc.gridx = 3; gbc.fill = GridBagConstraints.HORIZONTAL; ripPanel.add(stopButton, gbc); + gbc.fill = GridBagConstraints.BOTH; + statusLabel = new JLabel("Inactive"); statusLabel.setHorizontalAlignment(JLabel.CENTER); openButton = new JButton(); @@ -333,6 +342,7 @@ public class MainWindow implements Runnable, RipStatusHandler { public String getColumnName(int col) { return HISTORY.getColumnName(col); } + @Override public Class getColumnClass(int c) { return getValueAt(0, c).getClass(); } @@ -653,7 +663,7 @@ public class MainWindow implements Runnable, RipStatusHandler { historyButtonRerip.addActionListener(new ActionListener() { @Override public void actionPerformed(ActionEvent event) { - if (HISTORY.toList().size() == 0) { + if (HISTORY.isEmpty()) { JOptionPane.showMessageDialog(null, "There are no history entries to re-rip. Rip some albums first", "RipMe Error", @@ -673,7 +683,6 @@ public class MainWindow implements Runnable, RipStatusHandler { "Check an entry by clicking the checkbox to the right of the URL or Right-click a URL to check/uncheck all items", "RipMe Error", JOptionPane.ERROR_MESSAGE); - return; } } }); @@ -839,9 +848,13 @@ public class MainWindow implements Runnable, RipStatusHandler { private void setupTrayIcon() { mainFrame.addWindowListener(new WindowAdapter() { + @Override public void windowActivated(WindowEvent e) { trayMenuMain.setLabel("Hide"); } + @Override public void windowDeactivated(WindowEvent e) { trayMenuMain.setLabel("Show"); } + @Override public void windowDeiconified(WindowEvent e) { trayMenuMain.setLabel("Hide"); } + @Override public void windowIconified(WindowEvent e) { trayMenuMain.setLabel("Show"); } }); PopupMenu trayMenu = new PopupMenu(); @@ -947,7 +960,11 @@ public class MainWindow implements Runnable, RipStatusHandler { mainFrame.setAlwaysOnTop(false); } }); - } catch (Exception e) { + } catch (IOException e) { + //TODO implement proper stack trace handling this is really just intented as a placeholder until you implement proper error handling + e.printStackTrace(); + } catch (AWTException e) { + //TODO implement proper stack trace handling this is really just intented as a placeholder until you implement proper error handling e.printStackTrace(); } } @@ -1035,7 +1052,6 @@ public class MainWindow implements Runnable, RipStatusHandler { @SuppressWarnings("unchecked") private void ripNextAlbum() { isRipping = true; - // Save current state of queue to configuration. Utils.setConfigList("queue", (Enumeration) queueListModel.elements()); @@ -1045,7 +1061,7 @@ public class MainWindow implements Runnable, RipStatusHandler { return; } String nextAlbum = (String) queueListModel.remove(0); - if (queueListModel.size() == 0) { + if (queueListModel.isEmpty()) { optionQueue.setText("Queue"); } else { diff --git a/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java b/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java index 46c5de95..9d1c73d3 100644 --- a/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java +++ b/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java @@ -38,6 +38,7 @@ public class RipStatusMessage { return object; } + @Override public String toString() { return status.value + ": " + object.toString(); } diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 101f8691..ad3fda14 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -17,6 +17,7 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import com.rarchives.ripme.utils.Utils; +import java.util.Arrays; public class UpdateUtils { @@ -68,7 +69,7 @@ public class UpdateUtils { if (change.startsWith(UpdateUtils.getThisJarVersion() + ":")) { break; } - changeList.append("
+ " + change); + changeList.append("
+ ").append(change); } String latestVersion = json.getString("latestVersion"); @@ -111,7 +112,7 @@ public class UpdateUtils { int[] oldVersions = versionStringToInt(getThisJarVersion()); int[] newVersions = versionStringToInt(latestVersion); if (oldVersions.length < newVersions.length) { - System.err.println("Calculated: " + oldVersions + " < " + latestVersion); + System.err.println("Calculated: " + getThisJarVersion() + " < " + latestVersion); return true; } @@ -123,10 +124,7 @@ public class UpdateUtils { else if (newVersions[i] < oldVersions[i]) { logger.debug("oldVersion " + getThisJarVersion() + " > latestVersion " + latestVersion); return false; - } - else { - continue; - } + } } // At this point, the version numbers are exactly the same. @@ -202,6 +200,7 @@ public class UpdateUtils { logger.info("Executing: " + batchFile); Runtime.getRuntime().exec(batchExec); } catch (IOException e) { + //TODO implement proper stack trace handling this is really just intented as a placeholder until you implement proper error handling e.printStackTrace(); } }