Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

ChanRipper: now downloads imgur links, etc. (#185)

Browse Source
This commit is contained in:
Erwin de Haan 2017-05-09 23:02:24 +02:00 committed by metaprime
parent e6fa9263f1
commit 441790bb3c
10 changed files with 111 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/com/rarchives/ripme/App.java
View File

@ -216,7 +216,7 @@ public class App {
CommandLine cl = parser.parse(getOptions(), args, false);
return cl;
} catch (ParseException e) {
logger.error("[!] Error while parsing command-line arguments: " + args, e);
logger.error("[!] Error while parsing command-line arguments: " + Arrays.toString(args), e);
System.exit(-1);
return null;
}

37
src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
View File

@ -20,6 +20,7 @@ import com.rarchives.ripme.ui.RipStatusHandler;
import com.rarchives.ripme.ui.RipStatusMessage;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Utils;
import java.lang.reflect.InvocationTargetException;
public abstract class AbstractRipper
extends Observable
@ -94,8 +95,22 @@ public abstract class AbstractRipper
* URL of the file
* @param saveAs
* Path of the local file to save the content to.
* @return True on success, flase on failure.
*/
public abstract boolean addURLToDownload(URL url, File saveAs);
/**
* Queues image to be downloaded and saved.
* @param url
* URL of the file
* @param saveAs
* Path of the local file to save the content to.
* @param referrer
* The HTTP referrer to use while downloading this file.
* @param cookies
* The cookies to send to the server while downloading this file.
* @return
*/
public abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies);
public boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map<String,String> cookies) {
@ -144,6 +159,7 @@ public abstract class AbstractRipper
* Prefix to prepend to the saved filename.
* @param subdirectory
* Sub-directory of the working directory to save the images to.
* @return True on success, flase on failure.
*/
public boolean addURLToDownload(URL url, String prefix, String subdirectory) {
return addURLToDownload(url, prefix, subdirectory, null, null);
@ -156,6 +172,7 @@ public abstract class AbstractRipper
* URL to download
* @param prefix
* Text to append to saved filename.
* @return True on success, flase on failure.
*/
public boolean addURLToDownload(URL url, String prefix) {
// Use empty subdirectory
@ -201,7 +218,7 @@ public abstract class AbstractRipper
* Notify observers that a download could not be completed,
* but was not technically an "error".
* @param url
* @param message
* @param file
*/
public abstract void downloadExists(URL url, File file);
@ -281,7 +298,13 @@ public abstract class AbstractRipper
AlbumRipper ripper = (AlbumRipper) constructor.newInstance(url);
logger.debug("Found album ripper: " + ripper.getClass().getName());
return ripper;
} catch (Exception e) {
} catch (InstantiationException e) {
// Incompatible rippers *will* throw exceptions during instantiation.
} catch (IllegalAccessException e) {
// Incompatible rippers *will* throw exceptions during instantiation.
} catch (IllegalArgumentException e) {
// Incompatible rippers *will* throw exceptions during instantiation.
} catch (InvocationTargetException e) {
// Incompatible rippers *will* throw exceptions during instantiation.
}
}
@ -290,7 +313,13 @@ public abstract class AbstractRipper
VideoRipper ripper = (VideoRipper) constructor.newInstance(url);
logger.debug("Found video ripper: " + ripper.getClass().getName());
return ripper;
} catch (Exception e) {
} catch (InstantiationException e) {
// Incompatible rippers *will* throw exceptions during instantiation.
} catch (IllegalAccessException e) {
// Incompatible rippers *will* throw exceptions during instantiation.
} catch (IllegalArgumentException e) {
// Incompatible rippers *will* throw exceptions during instantiation.
} catch (InvocationTargetException e) {
// Incompatible rippers *will* throw exceptions during instantiation.
}
}
@ -298,6 +327,8 @@ public abstract class AbstractRipper
}
/**
* @param pkg
* The package name.
* @return
* List of constructors for all eligible Rippers.
* @throws Exception

25
src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
View File

@ -11,11 +11,12 @@ import java.util.regex.Pattern;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.ripper.rippers.ripperhelpers.ChanSite;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.RipUtils;
public class ChanRipper extends AbstractHTMLRipper {
public static List<ChanSite> explicit_domains = Arrays.asList(
@ -90,7 +91,11 @@ public class ChanRipper extends AbstractHTMLRipper {
* For example the archives are all known. (Check 4chan-x)
* Should be based on the software the specific chan uses.
* FoolFuuka uses the same (url) layout as 4chan
* */
*
* @param url
* @return
* The thread id in string form
* @throws java.net.MalformedURLException */
@Override
public String getGID(URL url) throws MalformedURLException {
Pattern p;
@ -181,8 +186,20 @@ public class ChanRipper extends AbstractHTMLRipper {
}
}
} else {
//TODO also grab imgur/flickr albums (And all other supported rippers) Maybe add a setting?
}
//Copied code from RedditRipper, getFilesFromURL should also implement stuff like flickr albums
URL originalURL;
try {
originalURL = new URL(href);
} catch (MalformedURLException e) {
continue;
}
List<URL> urls = RipUtils.getFilesFromURL(originalURL);
//for (int i = 0; i < urls.size(); i++) {
for(URL imageurl : urls){
imageURLs.add(imageurl.toString());
}
}
if (isStopped()) {
break;

9
src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java
View File

@ -171,11 +171,12 @@ public class RedditRipper extends AlbumRipper {
Pattern p = RipUtils.getURLRegex();
Matcher m = p.matcher(body);
while (m.find()) {
String url = m.group(1);
while (url.endsWith(")")) {
url = url.substring(0, url.length() - 1);
String foundurl;
foundurl = m.group(1);
while (foundurl.endsWith(")")) {
foundurl = foundurl.substring(0, foundurl.length() - 1);
}
handleURL(url, id);
handleURL(foundurl, id);
}
}

12
src/main/java/com/rarchives/ripme/ui/History.java
View File

@ -17,7 +17,7 @@ import org.json.JSONObject;
public class History {
private List<HistoryEntry> list = new ArrayList<HistoryEntry>();
private final List<HistoryEntry> list;
private static final String[] COLUMNS = new String[] {
"URL",
"created",
@ -26,6 +26,10 @@ public class History {
""
};
public History() {
this.list = new ArrayList<HistoryEntry>();
}
public void add(HistoryEntry entry) {
list.add(entry);
}
@ -124,10 +128,14 @@ public class History {
return jsonArray;
}
public List<HistoryEntry> toList() {
public List<HistoryEntry> toList() {
return list;
}
public boolean isEmpty(){
return list.isEmpty();
}
public void toFile(String filename) throws IOException {
OutputStream os = new FileOutputStream(filename);
try {

1
src/main/java/com/rarchives/ripme/ui/HistoryEntry.java
View File

@ -47,6 +47,7 @@ public class HistoryEntry {
return json;
}
@Override
public String toString() {
return this.url;
}

8
src/main/java/com/rarchives/ripme/ui/HistoryMenuMouseListener.java
View File

@ -24,7 +24,7 @@ public class HistoryMenuMouseListener extends MouseAdapter {
@Override
public void actionPerformed(ActionEvent ae) {
for (int row = 0; row < tableComponent.getRowCount(); row++) {
tableComponent.setValueAt(new Boolean(true), row, 4);
tableComponent.setValueAt(true, row, 4);
}
}
};
@ -34,7 +34,7 @@ public class HistoryMenuMouseListener extends MouseAdapter {
@Override
public void actionPerformed(ActionEvent ae) {
for (int row = 0; row < tableComponent.getRowCount(); row++) {
tableComponent.setValueAt(new Boolean(false), row, 4);
tableComponent.setValueAt(false, row, 4);
}
}
};
@ -46,7 +46,7 @@ public class HistoryMenuMouseListener extends MouseAdapter {
@Override
public void actionPerformed(ActionEvent ae) {
for (int row : tableComponent.getSelectedRows()) {
tableComponent.setValueAt(new Boolean(true), row, 4);
tableComponent.setValueAt(true, row, 4);
}
}
};
@ -56,7 +56,7 @@ public class HistoryMenuMouseListener extends MouseAdapter {
@Override
public void actionPerformed(ActionEvent ae) {
for (int row : tableComponent.getSelectedRows()) {
tableComponent.setValueAt(new Boolean(false), row, 4);
tableComponent.setValueAt(false, row, 4);
}
}
};

42
src/main/java/com/rarchives/ripme/ui/MainWindow.java
View File

@ -75,11 +75,13 @@ import org.apache.log4j.Logger;
import com.rarchives.ripme.ripper.AbstractRipper;
import com.rarchives.ripme.utils.RipUtils;
import com.rarchives.ripme.utils.Utils;
import java.awt.AWTException;
import javax.swing.UnsupportedLookAndFeelException;
/**
* Everything UI-related starts and ends here.
*/
public class MainWindow implements Runnable, RipStatusHandler {
public final class MainWindow implements Runnable, RipStatusHandler {
private static final Logger logger = Logger.getLogger(MainWindow.class);
@ -255,7 +257,13 @@ public class MainWindow implements Runnable, RipStatusHandler {
try {
UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName());
} catch (Exception e) {
} catch (ClassNotFoundException e) {
logger.error("[!] Exception setting system theme:", e);
} catch (InstantiationException e) {
logger.error("[!] Exception setting system theme:", e);
} catch (IllegalAccessException e) {
logger.error("[!] Exception setting system theme:", e);
} catch (UnsupportedLookAndFeelException e) {
logger.error("[!] Exception setting system theme:", e);
}
@ -271,12 +279,13 @@ public class MainWindow implements Runnable, RipStatusHandler {
} catch (Exception e) { }
JPanel ripPanel = new JPanel(new GridBagLayout());
ripPanel.setBorder(emptyBorder);
gbc.gridx = 0; ripPanel.add(new JLabel("URL:", JLabel.RIGHT), gbc);
gbc.gridx = 1; ripPanel.add(ripTextfield, gbc);
gbc.gridx = 2; ripPanel.add(ripButton, gbc);
gbc.gridx = 3; ripPanel.add(stopButton, gbc);
gbc.gridx = 0; gbc.fill = GridBagConstraints.HORIZONTAL; ripPanel.add(new JLabel("URL:", JLabel.RIGHT), gbc);
gbc.gridx = 1; gbc.fill = GridBagConstraints.HORIZONTAL; ripPanel.add(ripTextfield, gbc);
gbc.gridx = 2; gbc.fill = GridBagConstraints.HORIZONTAL; ripPanel.add(ripButton, gbc);
gbc.gridx = 3; gbc.fill = GridBagConstraints.HORIZONTAL; ripPanel.add(stopButton, gbc);
gbc.fill = GridBagConstraints.BOTH;
statusLabel = new JLabel("Inactive");
statusLabel.setHorizontalAlignment(JLabel.CENTER);
openButton = new JButton();
@ -333,6 +342,7 @@ public class MainWindow implements Runnable, RipStatusHandler {
public String getColumnName(int col) {
return HISTORY.getColumnName(col);
}
@Override
public Class<? extends Object> getColumnClass(int c) {
return getValueAt(0, c).getClass();
}
@ -653,7 +663,7 @@ public class MainWindow implements Runnable, RipStatusHandler {
historyButtonRerip.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent event) {
if (HISTORY.toList().size() == 0) {
if (HISTORY.isEmpty()) {
JOptionPane.showMessageDialog(null,
"There are no history entries to re-rip. Rip some albums first",
"RipMe Error",
@ -673,7 +683,6 @@ public class MainWindow implements Runnable, RipStatusHandler {
"Check an entry by clicking the checkbox to the right of the URL or Right-click a URL to check/uncheck all items",
"RipMe Error",
JOptionPane.ERROR_MESSAGE);
return;
}
}
});
@ -839,9 +848,13 @@ public class MainWindow implements Runnable, RipStatusHandler {
private void setupTrayIcon() {
mainFrame.addWindowListener(new WindowAdapter() {
@Override
public void windowActivated(WindowEvent e) { trayMenuMain.setLabel("Hide"); }
@Override
public void windowDeactivated(WindowEvent e) { trayMenuMain.setLabel("Show"); }
@Override
public void windowDeiconified(WindowEvent e) { trayMenuMain.setLabel("Hide"); }
@Override
public void windowIconified(WindowEvent e) { trayMenuMain.setLabel("Show"); }
});
PopupMenu trayMenu = new PopupMenu();
@ -947,7 +960,11 @@ public class MainWindow implements Runnable, RipStatusHandler {
mainFrame.setAlwaysOnTop(false);
}
});
} catch (Exception e) {
} catch (IOException e) {
//TODO implement proper stack trace handling this is really just intented as a placeholder until you implement proper error handling
e.printStackTrace();
} catch (AWTException e) {
//TODO implement proper stack trace handling this is really just intented as a placeholder until you implement proper error handling
e.printStackTrace();
}
}
@ -1035,7 +1052,6 @@ public class MainWindow implements Runnable, RipStatusHandler {
@SuppressWarnings("unchecked")
private void ripNextAlbum() {
isRipping = true;
// Save current state of queue to configuration.
Utils.setConfigList("queue", (Enumeration<Object>) queueListModel.elements());
@ -1045,7 +1061,7 @@ public class MainWindow implements Runnable, RipStatusHandler {
return;
}
String nextAlbum = (String) queueListModel.remove(0);
if (queueListModel.size() == 0) {
if (queueListModel.isEmpty()) {
optionQueue.setText("Queue");
}
else {

1
src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java
View File

@ -38,6 +38,7 @@ public class RipStatusMessage {
return object;
}
@Override
public String toString() {
return status.value + ": " + object.toString();
}

11
src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
View File

@ -17,6 +17,7 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import com.rarchives.ripme.utils.Utils;
import java.util.Arrays;
public class UpdateUtils {
@ -68,7 +69,7 @@ public class UpdateUtils {
if (change.startsWith(UpdateUtils.getThisJarVersion() + ":")) {
break;
}
changeList.append("<br> + " + change);
changeList.append("<br> + ").append(change);
}
String latestVersion = json.getString("latestVersion");
@ -111,7 +112,7 @@ public class UpdateUtils {
int[] oldVersions = versionStringToInt(getThisJarVersion());
int[] newVersions = versionStringToInt(latestVersion);
if (oldVersions.length < newVersions.length) {
System.err.println("Calculated: " + oldVersions + " < " + latestVersion);
System.err.println("Calculated: " + getThisJarVersion() + " < " + latestVersion);
return true;
}
@ -123,10 +124,7 @@ public class UpdateUtils {
else if (newVersions[i] < oldVersions[i]) {
logger.debug("oldVersion " + getThisJarVersion() + " > latestVersion " + latestVersion);
return false;
}
else {
continue;
}
}
}
// At this point, the version numbers are exactly the same.
@ -202,6 +200,7 @@ public class UpdateUtils {
logger.info("Executing: " + batchFile);
Runtime.getRuntime().exec(batchExec);
} catch (IOException e) {
//TODO implement proper stack trace handling this is really just intented as a placeholder until you implement proper error handling
e.printStackTrace();
}
}