Merge pull request #4 from maesse/master

Fixing reddit ripper, imgur image retrieval, UI bugs.

src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java

@@ -1,5 +1,7 @@
 package com.rarchives.ripme.ripper;
 
+import com.rarchives.ripme.ui.MainWindow;
+import com.rarchives.ripme.ui.RipStatusHandler;
 import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.Constructor;
@@ -17,6 +19,7 @@ import org.apache.log4j.Logger;
 import com.rarchives.ripme.ui.RipStatusMessage;
 import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
 import com.rarchives.ripme.utils.Utils;
+import java.util.Collections;
 
 public abstract class AbstractRipper 
                 extends Observable
@@ -30,11 +33,11 @@ public abstract class AbstractRipper
     protected URL url;
     protected File workingDir;
     protected DownloadThreadPool threadPool;
-    protected Observer observer = null;
+    protected RipStatusHandler observer = null;
 
-    protected Map<URL, File> itemsPending = new HashMap<URL, File>();
+    protected Map<URL, File> itemsPending = Collections.synchronizedMap(new HashMap<URL, File>());
-    protected Map<URL, File> itemsCompleted = new HashMap<URL, File>();
+    protected Map<URL, File> itemsCompleted = Collections.synchronizedMap(new HashMap<URL, File>());
-    protected Map<URL, String> itemsErrored = new HashMap<URL, String>();
+    protected Map<URL, String> itemsErrored = Collections.synchronizedMap(new HashMap<URL, String>());
     protected boolean completed = true;
 
     public abstract void rip() throws IOException;
@@ -59,7 +62,7 @@ public abstract class AbstractRipper
         this.threadPool = new DownloadThreadPool();
     }
 
-    public void setObserver(Observer obs) {
+    public void setObserver(RipStatusHandler obs) {
         this.observer = obs;
     }
 
@@ -162,7 +165,6 @@ public abstract class AbstractRipper
     public void retrievingSource(URL url) {
         RipStatusMessage msg = new RipStatusMessage(STATUS.LOADING_RESOURCE, url);
         observer.update(this,  msg);
-        observer.notifyAll();
     }
 
     /**
@@ -179,13 +181,11 @@ public abstract class AbstractRipper
         try {
             String path = Utils.removeCWD(saveAs);
             RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path);
-            synchronized(observer) {
             itemsPending.remove(url);
             itemsCompleted.put(url, saveAs);
             observer.update(this, msg);
-                observer.notifyAll();
+
             checkIfComplete();
-            }
         } catch (Exception e) {
             logger.error("Exception while updating observer: ", e);
         }
@@ -200,14 +200,12 @@ public abstract class AbstractRipper
         if (observer == null) {
             return;
         }
-        synchronized(observer) {
         itemsPending.remove(url);
         itemsErrored.put(url, reason);
         observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason));
-            observer.notifyAll();
+
         checkIfComplete();
     }
-    }
 
     /**
      * Notify observers that a download could not be completed,
@@ -219,12 +217,12 @@ public abstract class AbstractRipper
         if (observer == null) {
             return;
         }
-        synchronized(observer) {
+        
         itemsPending.remove(url);
         itemsErrored.put(url, message);
         observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " : " + message));
-            observer.notifyAll();
+            
-        }
+        
         checkIfComplete();
     }
 
@@ -235,16 +233,13 @@ public abstract class AbstractRipper
         if (observer == null) {
             return;
         }
-        synchronized (observer) {
+        
-            if (!completed && itemsPending.size() == 0) {
+        if (!completed && itemsPending.isEmpty()) {
             completed = true;
             logger.info("   Rip completed!");
-                observer.update(this,
+            
-                        new RipStatusMessage(
+            RipStatusMessage msg = new RipStatusMessage(STATUS.RIP_COMPLETE, workingDir);
-                                STATUS.RIP_COMPLETE,
+            observer.update(this, msg);
-                                workingDir));
-                observer.notifyAll();
-            }
         }
     }
 
@@ -325,10 +320,7 @@ public abstract class AbstractRipper
         if (observer == null) {
             return;
         }
-        synchronized (observer) {
         observer.update(this, new RipStatusMessage(status, message));
-            observer.notifyAll();
-        }
     }
     
     /**

src/main/java/com/rarchives/ripme/ripper/rippers/GonewildRipper.java

@@ -34,7 +34,7 @@ public class GonewildRipper extends AbstractRipper {
     }
 
     private Matcher getUsernameMatcher(URL url) {
-        Pattern p = Pattern.compile("^https?://[a-z]{0,3}\\.?reddit\\.com/(u|user)/([a-zA-Z0-9\\-]{3,})/?.*$");
+        Pattern p = Pattern.compile("^https?://[a-z]{0,3}\\.?gonewild\\.com/(u|user)/([a-zA-Z0-9\\-]{3,})/?.*$");
         return p.matcher(url.toExternalForm());
     }
 

src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java

@@ -16,6 +16,8 @@ import org.jsoup.nodes.Document;
 
 import com.rarchives.ripme.ripper.AbstractRipper;
 import com.rarchives.ripme.utils.RipUtils;
+import com.rarchives.ripme.utils.Utils;
+import java.net.SocketTimeoutException;
 
 public class RedditRipper extends AbstractRipper {
 
@@ -26,7 +28,7 @@ public class RedditRipper extends AbstractRipper {
     private static final String HOST   = "reddit";
     private static final String DOMAIN = "reddit.com";
 
-    private static final Logger logger = Logger.getLogger(GonewildRipper.class);
+    private static final Logger logger = Logger.getLogger(RedditRipper.class);
     private static final int SLEEP_TIME = 2000;
 
     //private static final String USER_AGENT = "ripme by /u/4_pr0n github.com/4pr0n/ripme";
@@ -67,6 +69,8 @@ public class RedditRipper extends AbstractRipper {
         waitForThreads();
     }
     
+    
+    
     private URL getAndParseAndReturnNext(URL url) throws IOException {
         JSONArray jsonArray = getJsonArrayFromURL(url), children;
         JSONObject json, data;
@@ -85,7 +89,7 @@ public class RedditRipper extends AbstractRipper {
                 parseJsonChild(children.getJSONObject(j));
             }
             if (data.has("after") && !data.isNull("after")) {
-                String nextURLString = url.toExternalForm();
+                String nextURLString = Utils.stripURLParameter(url.toExternalForm(), "after");
                 if (nextURLString.contains("?")) {
                     nextURLString = nextURLString.concat("&after=" + data.getString("after"));
                 }
@@ -111,11 +115,21 @@ public class RedditRipper extends AbstractRipper {
         }
         lastRequestTime = System.currentTimeMillis();
 
+        int attempts = 0;
+        Document doc = null;
         logger.info("    Retrieving " + url);
-        Document doc= Jsoup.connect(url.toExternalForm())
+        while(doc == null && attempts++ < 3) {
+            try {
+                doc= Jsoup.connect(url.toExternalForm())
                                         .ignoreContentType(true)
                                         .userAgent(USER_AGENT)
                                         .get();
+            } catch(SocketTimeoutException ex) {
+                if(attempts >= 3) throw ex;
+                logger.warn(String.format("[!] Connection timed out (attempt %d)", attempts));
+            }
+        }
+        
         String jsonString = doc.body().html().replaceAll("&quot;", "\"");
 
         Object jsonObj = new JSONTokener(jsonString).nextValue();

src/main/java/com/rarchives/ripme/ui/MainWindow.java

@@ -46,7 +46,7 @@ import com.rarchives.ripme.utils.Utils;
 /**
  * Everything UI-related starts and ends here.
  */
-public class MainWindow implements Runnable {
+public class MainWindow implements Runnable, RipStatusHandler {
 
     private static final Logger logger = Logger.getLogger(MainWindow.class);
     
@@ -101,7 +101,7 @@ public class MainWindow implements Runnable {
         mainFrame.setVisible(true);
     }
 
-    public static void status(String text) {
+    private void status(String text) {
         statusLabel.setText(text);
         mainFrame.pack();
     }
@@ -276,24 +276,14 @@ public class MainWindow implements Runnable {
     }
     
     private void appendLog(final String text, final Color color) {
-            try {
-                SwingUtilities.invokeAndWait(new Runnable() {
-                    @Override
-                    public void run() {
         SimpleAttributeSet sas = new SimpleAttributeSet();
         StyleConstants.setForeground(sas, color);
         StyledDocument sd = logText.getStyledDocument();
         try {
             sd.insertString(sd.getLength(), text + "\n", sas);
         } catch (BadLocationException e) { }
-                        logText.setCaretPosition(logText.getText().length());
+
-                    }
+        logText.setCaretPosition(sd.getLength());
-                });
-            } catch (InterruptedException e) {
-                e.printStackTrace();
-            } catch (InvocationTargetException e) {
-                e.printStackTrace();
-            }
     }
     
     private void loadHistory() {
@@ -359,7 +349,7 @@ public class MainWindow implements Runnable {
         try {
             AbstractRipper ripper = AbstractRipper.getRipper(url);
             ripTextfield.setText(ripper.getURL().toExternalForm());
-            ripper.setObserver(new RipStatusHandler());
+            ripper.setObserver((RipStatusHandler) this);
             Thread t = new Thread(ripper);
             t.start();
             return t;
@@ -376,13 +366,26 @@ public class MainWindow implements Runnable {
         }
     }
     
-    class RipStatusHandler implements Observer {
+    private class StatusEvent implements Runnable {
-        public void update(Observable observable, Object object) {
+        private final AbstractRipper ripper;
-            RipStatusMessage msg = (RipStatusMessage) object;
+        private final RipStatusMessage msg;
 
-            int completedPercent = ((AbstractRipper) observable).getCompletionPercentage();
+        public StatusEvent(AbstractRipper ripper, RipStatusMessage msg) {
+            this.ripper = ripper;
+            this.msg = msg;
+        }
+
+        public void run() {
+            handleEvent(this);
+        }
+    }
+    
+    private void handleEvent(StatusEvent evt) {
+        RipStatusMessage msg = evt.msg;
+
+        int completedPercent = evt.ripper.getCompletionPercentage();
         statusProgress.setValue(completedPercent);
-            status( ((AbstractRipper)observable).getStatusText() );
+        status( evt.ripper.getStatusText() );
 
         switch(msg.getStatus()) {
         case LOADING_RESOURCE:
@@ -428,11 +431,17 @@ public class MainWindow implements Runnable {
             mainFrame.pack();
         }
     }
+
+    public void update(AbstractRipper ripper, RipStatusMessage message) {
+        StatusEvent event = new StatusEvent(ripper, message);
+        SwingUtilities.invokeLater(event);
     }
     
     /** Simple TextPane that allows horizontal scrolling. */
     class JTextPaneNoWrap extends JTextPane {
         private static final long serialVersionUID = 1L;
+        
+        @Override
         public boolean getScrollableTracksViewportWidth() {
             return false;
         }

src/main/java/com/rarchives/ripme/ui/RipStatusHandler.java

@@ -0,0 +1,14 @@
+
+package com.rarchives.ripme.ui;
+
+import com.rarchives.ripme.ripper.AbstractRipper;
+
+/**
+ *
+ * @author Mads
+ */
+public interface RipStatusHandler {
+
+    public void update(AbstractRipper ripper, RipStatusMessage message);
+
+}

src/main/java/com/rarchives/ripme/utils/RipUtils.java

@@ -19,12 +19,14 @@ public class RipUtils {
         List<URL> result = new ArrayList<URL>();
 
         // Imgur album
-        if (url.getHost().equals("imgur.com") && url.toExternalForm().contains("imgur.com/a/")) {
+        if ((url.getHost().equals("m.imgur.com") || url.getHost().equals("imgur.com")) 
+                && url.toExternalForm().contains("imgur.com/a/")) {
             try {
                 return ImgurRipper.getURLsFromAlbum(url);
             } catch (IOException e) {
                 logger.error("[!] Exception while loading album " + url, e);
             }
+           
         }
 
         // Direct link to image
@@ -40,6 +42,17 @@ public class RipUtils {
             }
         }
         
+        if(url.getHost().equals("imgur.com") || 
+                url.getHost().equals("m.imgur.com")){
+            try {
+                result.add(new URL(url.toExternalForm() + ".png"));
+                return result;
+            } catch (MalformedURLException ex) {
+                logger.error("[!] Exception while loading album " + url, ex);
+            }
+            
+        }
+        
         logger.error("[!] Unable to rip URL: " + url);
         return result;
     }

src/main/java/com/rarchives/ripme/utils/Utils.java

@@ -84,6 +84,28 @@ public class Utils {
         return prettySaveAs;
     }
     
+    public static String stripURLParameter(String url, String parameter) {
+        int paramIndex = url.indexOf("?" + parameter);
+        boolean wasFirstParam = true;
+        if(paramIndex < 0) {
+            wasFirstParam = false;
+            paramIndex = url.indexOf("&" + parameter);
+        }
+        
+        if(paramIndex > 0) {
+            int nextParam = url.indexOf("&", paramIndex+1);
+            if(nextParam != -1) {
+                String c = "&";
+                if(wasFirstParam) c = "?";
+                url = url.substring(0, paramIndex) + c + url.substring(nextParam+1, url.length());
+            } else {
+                url = url.substring(0, paramIndex);
+            }
+        }
+        
+        return url;
+    }
+
     /**
      * Removes the current working directory from a given filename
      * @param file

src/test/java/com/rarchives/ripme/tst/ripper/rippers/GonewildRipperTest.java

@@ -14,7 +14,7 @@ public class GonewildRipperTest extends RippersTest {
             return;
         }
         List<URL> contentURLs = new ArrayList<URL>();
-        contentURLs.add(new URL("http://reddit.com/u/amle69"));
+        contentURLs.add(new URL("http://gonewild.com/u/amle69"));
         for (URL url : contentURLs) {
             try {
                 GonewildRipper ripper = new GonewildRipper(url);