Twitter ripper makes 8 more requests, stops

Previously 2 requests every time.
Now 10 requests (configurable) and stops when it hits the last tweet.

src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java

@@ -1,25 +1,23 @@
 package com.rarchives.ripme.ripper;
 
-import com.rarchives.ripme.ui.MainWindow;
-import com.rarchives.ripme.ui.RipStatusHandler;
 import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.Constructor;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Observable;
-import java.util.Observer;
 
 import org.apache.log4j.Logger;
 
+import com.rarchives.ripme.ui.RipStatusHandler;
 import com.rarchives.ripme.ui.RipStatusMessage;
 import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
 import com.rarchives.ripme.utils.Utils;
-import java.util.Collections;
 
 public abstract class AbstractRipper 
                 extends Observable

src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java

@@ -17,6 +17,7 @@ import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 
 import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
 import com.rarchives.ripme.utils.Utils;
 
 public class TwitterRipper extends AbstractRipper {
@@ -25,7 +26,7 @@ public class TwitterRipper extends AbstractRipper {
                                 HOST   = "twitter";
     private static final Logger logger = Logger.getLogger(TwitterRipper.class);
 
-    private static final int MAX_REQUESTS = 2;
+    private static final int MAX_REQUESTS = Utils.getConfigInteger("twitter.max_requests", 10);
     private static final int WAIT_TIME = 2000;
 
     // Base 64 of consumer key : consumer secret
@@ -116,30 +117,30 @@ public class TwitterRipper extends AbstractRipper {
         }
     }
     
-    private String getApiURL(String maxID) {
+    private String getApiURL(Long maxID) {
-        String req = "";
+        StringBuilder req = new StringBuilder();
         switch (albumType) {
         case ACCOUNT:
-            req = "https://api.twitter.com/1.1/statuses/user_timeline.json"
+            req.append("https://api.twitter.com/1.1/statuses/user_timeline.json")
-                + "?screen_name=" + this.accountName
+               .append("?screen_name=" + this.accountName)
-                + "&include_entities=true"
+               .append("&include_entities=true")
-                + "&exclude_replies=true"
+               .append("&exclude_replies=true")
-                + "&trim_user=true"
+               .append("&trim_user=true")
-                + "&include_rts=false"
+               .append("&include_rts=false")
-                + "&count=" + 200;
+               .append("&count=" + 200);
             break;
         case SEARCH:
-            req = "https://api.twitter.com/1.1/search/tweets.json"
+            req.append("https://api.twitter.com/1.1/search/tweets.json")
-                + "?q=" + this.searchText
+               .append("?q=" + this.searchText)
-                + "&include_entities=true"
+               .append("&include_entities=true")
-                + "&result_type=recent"
+               .append("&result_type=recent")
-                + "&count=100";
+               .append("&count=100");
             break;
         }
-        if (maxID != null) {
+        if (maxID > 0) {
-            req += "&max_id=" + maxID;
+            req.append("&max_id=" + Long.toString(maxID));
         }
-        return req;
+        return req.toString();
     }
     
     private List<JSONObject> getTweets(String url) throws IOException {
@@ -223,21 +224,29 @@ public class TwitterRipper extends AbstractRipper {
             break;
         }
 
-        String maxID = null;
+        Long lastMaxID = 0L;
         for (int i = 0; i < MAX_REQUESTS; i++) {
-            List<JSONObject> tweets = getTweets(getApiURL(maxID));
+            List<JSONObject> tweets = getTweets(getApiURL(lastMaxID - 1));
             if (tweets.size() == 0) {
                 logger.info("   No more tweets found.");
                 break;
             }
+            logger.debug("Twitter response #" + (i + 1) + " Tweets:\n" + tweets);
+            if (tweets.size() == 1 && 
+                    lastMaxID.equals(tweets.get(0).getString("id_str"))
+                ) {
+                break;
+            }
+            
             for (JSONObject tweet : tweets) {
-                maxID = tweet.getString("id_str");
+                lastMaxID = tweet.getLong("id");
                 parseTweet(tweet);
             }
 
             try {
                 Thread.sleep(WAIT_TIME);
-            } catch (InterruptedException e) {
+            }
+            catch (InterruptedException e) {
                 logger.error("[!] Interrupted while waiting to load more results", e);
                 break;
             }

src/main/resources/rip.properties

@@ -17,3 +17,5 @@ download.max_size = 104857600
 twitter.auth = VW9Ybjdjb1pkd2J0U3kwTUh2VXVnOm9GTzVQVzNqM29LQU1xVGhnS3pFZzhKbGVqbXU0c2lHQ3JrUFNNZm8=
 tumblr.auth = v5kUqGQXUtmF7K0itri1DGtgTs0VQpbSEbh1jxYgj9d2Sq18F8
 gw.api = gonewild
+
+twitter.max_requests = 10