Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #4 from maesse/master

Browse Source 
Fixing reddit ripper, imgur image retrieval, UI bugs.
This commit is contained in:
4pr0n 2014-03-16 17:29:30 -07:00
commit 42c0473e3d
8 changed files with 185 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
View File

@ -1,5 +1,7 @@
package com.rarchives.ripme.ripper;
import com.rarchives.ripme.ui.MainWindow;
import com.rarchives.ripme.ui.RipStatusHandler;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
@ -17,6 +19,7 @@ import org.apache.log4j.Logger;
import com.rarchives.ripme.ui.RipStatusMessage;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Utils;
import java.util.Collections;
public abstract class AbstractRipper
extends Observable
@ -30,11 +33,11 @@ public abstract class AbstractRipper
protected URL url;
protected File workingDir;
protected DownloadThreadPool threadPool;
protected Observer observer = null;
protected RipStatusHandler observer = null;
protected Map<URL, File> itemsPending = new HashMap<URL, File>();
protected Map<URL, File> itemsCompleted = new HashMap<URL, File>();
protected Map<URL, String> itemsErrored = new HashMap<URL, String>();
protected Map<URL, File> itemsPending = Collections.synchronizedMap(new HashMap<URL, File>());
protected Map<URL, File> itemsCompleted = Collections.synchronizedMap(new HashMap<URL, File>());
protected Map<URL, String> itemsErrored = Collections.synchronizedMap(new HashMap<URL, String>());
protected boolean completed = true;
public abstract void rip() throws IOException;
@ -59,7 +62,7 @@ public abstract class AbstractRipper
this.threadPool = new DownloadThreadPool();
}
public void setObserver(Observer obs) {
public void setObserver(RipStatusHandler obs) {
this.observer = obs;
}
@ -162,7 +165,6 @@ public abstract class AbstractRipper
public void retrievingSource(URL url) {
RipStatusMessage msg = new RipStatusMessage(STATUS.LOADING_RESOURCE, url);
observer.update(this, msg);
observer.notifyAll();
}
/**
@ -179,13 +181,11 @@ public abstract class AbstractRipper
try {
String path = Utils.removeCWD(saveAs);
RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path);
synchronized(observer) {
itemsPending.remove(url);
itemsCompleted.put(url, saveAs);
observer.update(this, msg);
observer.notifyAll();
checkIfComplete();
}
} catch (Exception e) {
logger.error("Exception while updating observer: ", e);
}
@ -200,14 +200,12 @@ public abstract class AbstractRipper
if (observer == null) {
return;
}
synchronized(observer) {
itemsPending.remove(url);
itemsErrored.put(url, reason);
observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason));
observer.notifyAll();
checkIfComplete();
}
}
/**
* Notify observers that a download could not be completed,
@ -219,12 +217,12 @@ public abstract class AbstractRipper
if (observer == null) {
return;
}
synchronized(observer) {
itemsPending.remove(url);
itemsErrored.put(url, message);
observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " : " + message));
observer.notifyAll();
}
checkIfComplete();
}
@ -235,16 +233,13 @@ public abstract class AbstractRipper
if (observer == null) {
return;
}
synchronized (observer) {
if (!completed && itemsPending.size() == 0) {
if (!completed && itemsPending.isEmpty()) {
completed = true;
logger.info(" Rip completed!");
observer.update(this,
new RipStatusMessage(
STATUS.RIP_COMPLETE,
workingDir));
observer.notifyAll();
}
RipStatusMessage msg = new RipStatusMessage(STATUS.RIP_COMPLETE, workingDir);
observer.update(this, msg);
}
}
@ -325,10 +320,7 @@ public abstract class AbstractRipper
if (observer == null) {
return;
}
synchronized (observer) {
observer.update(this, new RipStatusMessage(status, message));
observer.notifyAll();
}
}
/**

2
src/main/java/com/rarchives/ripme/ripper/rippers/GonewildRipper.java
View File

@ -34,7 +34,7 @@ public class GonewildRipper extends AbstractRipper {
}
private Matcher getUsernameMatcher(URL url) {
Pattern p = Pattern.compile("^https?://[a-z]{0,3}\\.?reddit\\.com/(u|user)/([a-zA-Z0-9\\-]{3,})/?.*$");
Pattern p = Pattern.compile("^https?://[a-z]{0,3}\\.?gonewild\\.com/(u|user)/([a-zA-Z0-9\\-]{3,})/?.*$");
return p.matcher(url.toExternalForm());
}

20
src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java
View File

@ -16,6 +16,8 @@ import org.jsoup.nodes.Document;
import com.rarchives.ripme.ripper.AbstractRipper;
import com.rarchives.ripme.utils.RipUtils;
import com.rarchives.ripme.utils.Utils;
import java.net.SocketTimeoutException;
public class RedditRipper extends AbstractRipper {
@ -26,7 +28,7 @@ public class RedditRipper extends AbstractRipper {
private static final String HOST = "reddit";
private static final String DOMAIN = "reddit.com";
private static final Logger logger = Logger.getLogger(GonewildRipper.class);
private static final Logger logger = Logger.getLogger(RedditRipper.class);
private static final int SLEEP_TIME = 2000;
//private static final String USER_AGENT = "ripme by /u/4_pr0n github.com/4pr0n/ripme";
@ -67,6 +69,8 @@ public class RedditRipper extends AbstractRipper {
waitForThreads();
}
private URL getAndParseAndReturnNext(URL url) throws IOException {
JSONArray jsonArray = getJsonArrayFromURL(url), children;
JSONObject json, data;
@ -85,7 +89,7 @@ public class RedditRipper extends AbstractRipper {
parseJsonChild(children.getJSONObject(j));
}
if (data.has("after") && !data.isNull("after")) {
String nextURLString = url.toExternalForm();
String nextURLString = Utils.stripURLParameter(url.toExternalForm(), "after");
if (nextURLString.contains("?")) {
nextURLString = nextURLString.concat("&after=" + data.getString("after"));
}
@ -111,11 +115,21 @@ public class RedditRipper extends AbstractRipper {
}
lastRequestTime = System.currentTimeMillis();
int attempts = 0;
Document doc = null;
logger.info(" Retrieving " + url);
Document doc= Jsoup.connect(url.toExternalForm())
while(doc == null && attempts++ < 3) {
try {
doc= Jsoup.connect(url.toExternalForm())
.ignoreContentType(true)
.userAgent(USER_AGENT)
.get();
} catch(SocketTimeoutException ex) {
if(attempts >= 3) throw ex;
logger.warn(String.format("[!] Connection timed out (attempt %d)", attempts));
}
}
String jsonString = doc.body().html().replaceAll("&quot;", "\"");
Object jsonObj = new JSONTokener(jsonString).nextValue();

49
src/main/java/com/rarchives/ripme/ui/MainWindow.java
View File

@ -46,7 +46,7 @@ import com.rarchives.ripme.utils.Utils;
/**
* Everything UI-related starts and ends here.
*/
public class MainWindow implements Runnable {
public class MainWindow implements Runnable, RipStatusHandler {
private static final Logger logger = Logger.getLogger(MainWindow.class);
@ -101,7 +101,7 @@ public class MainWindow implements Runnable {
mainFrame.setVisible(true);
}
public static void status(String text) {
private void status(String text) {
statusLabel.setText(text);
mainFrame.pack();
}
@ -276,24 +276,14 @@ public class MainWindow implements Runnable {
}
private void appendLog(final String text, final Color color) {
try {
SwingUtilities.invokeAndWait(new Runnable() {
@Override
public void run() {
SimpleAttributeSet sas = new SimpleAttributeSet();
StyleConstants.setForeground(sas, color);
StyledDocument sd = logText.getStyledDocument();
try {
sd.insertString(sd.getLength(), text + "\n", sas);
} catch (BadLocationException e) { }
logText.setCaretPosition(logText.getText().length());
}
});
} catch (InterruptedException e) {
e.printStackTrace();
} catch (InvocationTargetException e) {
e.printStackTrace();
}
logText.setCaretPosition(sd.getLength());
}
private void loadHistory() {
@ -359,7 +349,7 @@ public class MainWindow implements Runnable {
try {
AbstractRipper ripper = AbstractRipper.getRipper(url);
ripTextfield.setText(ripper.getURL().toExternalForm());
ripper.setObserver(new RipStatusHandler());
ripper.setObserver((RipStatusHandler) this);
Thread t = new Thread(ripper);
t.start();
return t;
@ -376,13 +366,26 @@ public class MainWindow implements Runnable {
}
}
class RipStatusHandler implements Observer {
public void update(Observable observable, Object object) {
RipStatusMessage msg = (RipStatusMessage) object;
private class StatusEvent implements Runnable {
private final AbstractRipper ripper;
private final RipStatusMessage msg;
int completedPercent = ((AbstractRipper) observable).getCompletionPercentage();
public StatusEvent(AbstractRipper ripper, RipStatusMessage msg) {
this.ripper = ripper;
this.msg = msg;
}
public void run() {
handleEvent(this);
}
}
private void handleEvent(StatusEvent evt) {
RipStatusMessage msg = evt.msg;
int completedPercent = evt.ripper.getCompletionPercentage();
statusProgress.setValue(completedPercent);
status( ((AbstractRipper)observable).getStatusText() );
status( evt.ripper.getStatusText() );
switch(msg.getStatus()) {
case LOADING_RESOURCE:
@ -428,11 +431,17 @@ public class MainWindow implements Runnable {
mainFrame.pack();
}
}
public void update(AbstractRipper ripper, RipStatusMessage message) {
StatusEvent event = new StatusEvent(ripper, message);
SwingUtilities.invokeLater(event);
}
/** Simple TextPane that allows horizontal scrolling. */
class JTextPaneNoWrap extends JTextPane {
private static final long serialVersionUID = 1L;
@Override
public boolean getScrollableTracksViewportWidth() {
return false;
}

14
src/main/java/com/rarchives/ripme/ui/RipStatusHandler.java Normal file
View File

@ -0,0 +1,14 @@
package com.rarchives.ripme.ui;
import com.rarchives.ripme.ripper.AbstractRipper;
/**
*
* @author Mads
*/
public interface RipStatusHandler {
public void update(AbstractRipper ripper, RipStatusMessage message);
}

15
src/main/java/com/rarchives/ripme/utils/RipUtils.java
View File

@ -19,12 +19,14 @@ public class RipUtils {
List<URL> result = new ArrayList<URL>();
// Imgur album
if (url.getHost().equals("imgur.com") && url.toExternalForm().contains("imgur.com/a/")) {
if ((url.getHost().equals("m.imgur.com") || url.getHost().equals("imgur.com"))
&& url.toExternalForm().contains("imgur.com/a/")) {
try {
return ImgurRipper.getURLsFromAlbum(url);
} catch (IOException e) {
logger.error("[!] Exception while loading album " + url, e);
}
}
// Direct link to image
@ -40,6 +42,17 @@ public class RipUtils {
}
}
if(url.getHost().equals("imgur.com") ||
url.getHost().equals("m.imgur.com")){
try {
result.add(new URL(url.toExternalForm() + ".png"));
return result;
} catch (MalformedURLException ex) {
logger.error("[!] Exception while loading album " + url, ex);
}
}
logger.error("[!] Unable to rip URL: " + url);
return result;
}

22
src/main/java/com/rarchives/ripme/utils/Utils.java
View File

@ -84,6 +84,28 @@ public class Utils {
return prettySaveAs;
}
public static String stripURLParameter(String url, String parameter) {
int paramIndex = url.indexOf("?" + parameter);
boolean wasFirstParam = true;
if(paramIndex < 0) {
wasFirstParam = false;
paramIndex = url.indexOf("&" + parameter);
}
if(paramIndex > 0) {
int nextParam = url.indexOf("&", paramIndex+1);
if(nextParam != -1) {
String c = "&";
if(wasFirstParam) c = "?";
url = url.substring(0, paramIndex) + c + url.substring(nextParam+1, url.length());
} else {
url = url.substring(0, paramIndex);
}
}
return url;
}
/**
* Removes the current working directory from a given filename
* @param file

2
src/test/java/com/rarchives/ripme/tst/ripper/rippers/GonewildRipperTest.java
View File

@ -14,7 +14,7 @@ public class GonewildRipperTest extends RippersTest {
return;
}
List<URL> contentURLs = new ArrayList<URL>();
contentURLs.add(new URL("http://reddit.com/u/amle69"));
contentURLs.add(new URL("http://gonewild.com/u/amle69"));
for (URL url : contentURLs) {
try {
GonewildRipper ripper = new GonewildRipper(url);