Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

imgur albums rip via command-line. wowza

Browse Source
This commit is contained in:
4pr0n 2014-02-27 01:28:23 -08:00
parent 46e2948403
commit 85fd7f0847
11 changed files with 133 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.classpath
View File

@ -12,7 +12,7 @@
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" path="config"/>
<classpathentry kind="src" path="src/main/resources"/>
<classpathentry exported="true" kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/J2SE-1.5">
<attributes>
<attribute name="maven.pomderived" value="true"/>

1
config/rip.properties
View File

@ -1 +0,0 @@
threads.size = 5

17
pom.xml
View File

@ -4,9 +4,12 @@
<groupId>com.rarchives.ripme</groupId>
<artifactId>ripme</artifactId>
<packaging>jar</packaging>
<version>1.0-SNAPSHOT</version>
<version>1.0</version>
<name>ripme</name>
<url>http://maven.apache.org</url>
<url>http://rip.rarchives.com</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>junit</groupId>
@ -35,6 +38,11 @@
<artifactId>log4j</artifactId>
<version>1.2.17</version>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>1.2</version>
</dependency>
</dependencies>
<build>
<plugins>
@ -43,8 +51,11 @@
<configuration>
<archive>
<manifest>
<mainClass>fully.qualified.MainClass</mainClass>
<mainClass>com.rarchives.ripme.App</mainClass>
</manifest>
<manifestEntries>
<Class-Path>./config</Class-Path>
</manifestEntries>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>

79
src/main/java/com/rarchives/ripme/App.java
View File

@ -1,30 +1,85 @@
package com.rarchives.ripme;
import java.net.MalformedURLException;
import java.net.URL;
import org.apache.commons.cli.BasicParser;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.log4j.Logger;
import com.rarchives.ripme.ripper.rippers.ImgurRipper;
import com.rarchives.ripme.ripper.AbstractRipper;
import com.rarchives.ripme.utils.Utils;
/**
*
*/
public class App {
public static void main( String[] args ) throws Exception {
Logger logger = Logger.getLogger(App.class);
public static final Logger logger = Logger.getLogger(App.class);
public static void main(String[] args) throws MalformedURLException {
logger.debug("Initialized");
//URL url = new URL("http://www.imagefap.com/pictures/4117023/Mirror-flat-stomach-small-firm-tits");
URL url = new URL("http://imgur.com/a/Ox6jN");
CommandLine cl = handleArguments(args);
try {
ImgurRipper ir = new ImgurRipper(url);
ir.rip();
} catch (Exception e) {
logger.error("Caught exception:", e);
throw e;
URL url = new URL(cl.getOptionValue('u'));
rip(url);
} catch (MalformedURLException e) {
logger.error("Given URL is not valid. Expected URL format is http://domain.com/...");
System.exit(-1);
}
}
public static void initialize() {
public static void rip(URL url) {
try {
AbstractRipper ripper = AbstractRipper.getRipper(url);
ripper.rip();
} catch (Exception e) {
logger.error("Caught exception:", e);
System.exit(-1);
}
}
public static CommandLine handleArguments(String[] args) {
CommandLine cl = getArgs(args);
if (cl.hasOption('h')) {
HelpFormatter hf = new HelpFormatter();
hf.printHelp("asdf", getOptions());
System.exit(0);
}
if (cl.hasOption('w')) {
Utils.setConfigBoolean("file.overwrite", true);
}
if (!cl.hasOption('u')) {
System.err.println("\nRequired URL ('-u' or '--url') not provided");
System.err.println("\n\tExample: java -jar ripme.jar -u http://imgur.com/a/abcde");
System.exit(-1);
}
return cl;
}
public static Options getOptions() {
Options opts = new Options();
opts.addOption("h", "help", false, "Print the help");
opts.addOption("u", "url", true, "URL of album to rip");
opts.addOption("t", "threads", true, "Number of download threads per rip");
opts.addOption("w", "overwrite", false, "Overwrite existing files");
return opts;
}
public static CommandLine getArgs(String[] args) {
BasicParser parser = new BasicParser();
try {
CommandLine cl = parser.parse(getOptions(), args, false);
return cl;
} catch (ParseException e) {
logger.error("Error while parsing command-line arguments: " + args, e);
System.exit(-1);
return null;
}
}
}

26
src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
View File

@ -7,6 +7,8 @@ import java.net.URL;
import org.apache.log4j.Logger;
import com.rarchives.ripme.ripper.rippers.ImagefapRipper;
import com.rarchives.ripme.ripper.rippers.ImgurRipper;
import com.rarchives.ripme.utils.Utils;
public abstract class AbstractRipper implements RipperInterface {
@ -56,7 +58,7 @@ public abstract class AbstractRipper implements RipperInterface {
logger.error("Error creating save file path for URL '" + url + "':", e);
return;
}
logger.info("Downloading " + url + " to " + saveFileAs);
logger.debug("Downloading " + url + " to " + saveFileAs);
addURLToDownload(url, saveFileAs);
}
/**
@ -87,4 +89,26 @@ public abstract class AbstractRipper implements RipperInterface {
}
logger.debug("Set working directory to: " + this.workingDir);
}
/**
* Finds, instantiates, and returns a compatible ripper for given URL.
* @param url
* URL to rip.
* @return
* Instantiated ripper ready to rip given URL.
* @throws Exception
* If no compatible rippers can be found.
*/
public static AbstractRipper getRipper(URL url) throws Exception {
// I know what you're thinking. I'm disappointed too.
try {
AbstractRipper r = new ImagefapRipper(url);
return r;
} catch (IOException e) { }
try {
AbstractRipper r = new ImgurRipper(url);
return r;
} catch (IOException e) { }
throw new Exception("No compatible ripper found");
}
}

4
src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
View File

@ -36,7 +36,7 @@ public class DownloadFileThread extends Thread {
}
}
logger.debug("Downloading file from: " + url);
logger.info("Downloading file from: " + url);
try {
Response response;
response = Jsoup.connect(url.toExternalForm())
@ -49,7 +49,7 @@ public class DownloadFileThread extends Thread {
logger.error("Exception while downloading file: " + url, e);
return;
}
logger.debug("Download completed: " + url);
logger.info("Download completed: " + url);
}
}

4
src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java
View File

@ -32,7 +32,7 @@ public class ImgurRipper extends AbstractRipper {
}
public void processURL(URL url, String prefix) {
logger.info("Found URL: " + url);
logger.debug("Found URL: " + url);
addURLToDownload(url, prefix);
}
@ -80,7 +80,7 @@ public class ImgurRipper extends AbstractRipper {
private void ripAlbum(URL url) throws IOException {
int index = 0;
logger.debug("Retrieving " + url.toExternalForm());
logger.info("Retrieving " + url.toExternalForm());
Document doc = Jsoup.connect(url.toExternalForm()).get();
for (Element thumb : doc.select("div.image")) {
String image;

44
src/main/java/com/rarchives/ripme/utils/Utils.java
View File

@ -11,8 +11,18 @@ import org.apache.log4j.Logger;
public class Utils {
public static final String RIP_DIRECTORY = "rips";
private static final File configFile = new File("src/main/resources/rip.properties");
private static final Logger logger = Logger.getLogger(Utils.class);
private static Configuration config;
static {
try {
config = new PropertiesConfiguration(configFile);
} catch (ConfigurationException e) {
logger.error("Failed to load properties file from " + configFile, e);
}
}
public static File getWorkingDirectory() throws IOException {
String path = new File(".").getCanonicalPath() + File.separator;
path += RIP_DIRECTORY + File.separator;
@ -24,39 +34,19 @@ public class Utils {
}
public static String getConfigString(String key, String defaultValue) {
String value = defaultValue;
try {
Configuration config = new PropertiesConfiguration("config/rip.properties");
value = config.getString(key);
} catch (ConfigurationException e) {
logger.error("Failed to get configuration value for " + key
+ ", using default '" + value + "'");
}
return value;
return config.getString(key, defaultValue);
}
public static int getConfigInteger(String key, int defaultValue) {
int value = defaultValue;
try {
Configuration config = new PropertiesConfiguration(new File("./config/rip.properties"));
value = config.getInt(key, defaultValue);
} catch (Exception e) {
logger.error("Failed to get configuration value for " + key
+ ", using default '" + value + "'");
}
return value;
return config.getInt(key, defaultValue);
}
public static boolean getConfigBoolean(String key, boolean defaultValue) {
boolean value = defaultValue;
try {
Configuration config = new PropertiesConfiguration(new File("./config/rip.properties"));
value = config.getBoolean(key, defaultValue);
} catch (Exception e) {
logger.error("Failed to get configuration value for " + key
+ ", using default '" + value + "'");
}
return value;
return config.getBoolean(key, defaultValue);
}
public static void setConfigBoolean(String key, boolean value) {
config.setProperty(key, value);
}
}

5
config/log4j.properties → src/main/resources/log4j.properties
View File

@ -10,8 +10,9 @@ log4j.appender.FILE.layout.ConversionPattern = %d %-4r [%t] %-5p %c{2} %x - %m%n
# define the console appender
log4j.appender.stdout = org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target = System.out
log4j.appender.stdout.Threshold = info
log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern = %d %-4r [%t] %-5p %c{2} %x - %m%n
log4j.appender.stdout.layout.ConversionPattern = %m%n
# now map our console appender as a root logger, means all log messages will go to this appender
log4j.rootLogger = DEBUG, FILE, stdout
log4j.rootLogger = debug, stdout, FILE

2
src/main/resources/rip.properties Normal file
View File

@ -0,0 +1,2 @@
threads.size = 5
file.overwrite = false

3
src/test/java/com/rarchives/ripme/AppTest.java
View File

@ -4,9 +4,6 @@ import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;
/**
* Unit test for simple App.
*/
public class AppTest extends TestCase {
/**
* Create the test case