From 8a22035b8982c15194ee01c227c61949bc17d15f Mon Sep 17 00:00:00 2001 From: 4pr0n Date: Thu, 3 Apr 2014 23:53:51 -0700 Subject: [PATCH] Twitter ripper makes 8 more requests, stops Previously 2 requests every time. Now 10 requests (configurable) and stops when it hits the last tweet. --- .../ripme/ripper/AbstractRipper.java | 6 +-- .../ripme/ripper/rippers/TwitterRipper.java | 53 +++++++++++-------- src/main/resources/rip.properties | 2 + 3 files changed, 35 insertions(+), 26 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 19917f97..626ef454 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -1,25 +1,23 @@ package com.rarchives.ripme.ripper; -import com.rarchives.ripme.ui.MainWindow; -import com.rarchives.ripme.ui.RipStatusHandler; import java.io.File; import java.io.IOException; import java.lang.reflect.Constructor; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Observable; -import java.util.Observer; import org.apache.log4j.Logger; +import com.rarchives.ripme.ui.RipStatusHandler; import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Utils; -import java.util.Collections; public abstract class AbstractRipper extends Observable diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java index 3f1f9197..fb214a76 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java @@ -17,6 +17,7 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import com.rarchives.ripme.ripper.AbstractRipper; +import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Utils; public class TwitterRipper extends AbstractRipper { @@ -25,7 +26,7 @@ public class TwitterRipper extends AbstractRipper { HOST = "twitter"; private static final Logger logger = Logger.getLogger(TwitterRipper.class); - private static final int MAX_REQUESTS = 2; + private static final int MAX_REQUESTS = Utils.getConfigInteger("twitter.max_requests", 10); private static final int WAIT_TIME = 2000; // Base 64 of consumer key : consumer secret @@ -116,30 +117,30 @@ public class TwitterRipper extends AbstractRipper { } } - private String getApiURL(String maxID) { - String req = ""; + private String getApiURL(Long maxID) { + StringBuilder req = new StringBuilder(); switch (albumType) { case ACCOUNT: - req = "https://api.twitter.com/1.1/statuses/user_timeline.json" - + "?screen_name=" + this.accountName - + "&include_entities=true" - + "&exclude_replies=true" - + "&trim_user=true" - + "&include_rts=false" - + "&count=" + 200; + req.append("https://api.twitter.com/1.1/statuses/user_timeline.json") + .append("?screen_name=" + this.accountName) + .append("&include_entities=true") + .append("&exclude_replies=true") + .append("&trim_user=true") + .append("&include_rts=false") + .append("&count=" + 200); break; case SEARCH: - req = "https://api.twitter.com/1.1/search/tweets.json" - + "?q=" + this.searchText - + "&include_entities=true" - + "&result_type=recent" - + "&count=100"; + req.append("https://api.twitter.com/1.1/search/tweets.json") + .append("?q=" + this.searchText) + .append("&include_entities=true") + .append("&result_type=recent") + .append("&count=100"); break; } - if (maxID != null) { - req += "&max_id=" + maxID; + if (maxID > 0) { + req.append("&max_id=" + Long.toString(maxID)); } - return req; + return req.toString(); } private List getTweets(String url) throws IOException { @@ -223,21 +224,29 @@ public class TwitterRipper extends AbstractRipper { break; } - String maxID = null; + Long lastMaxID = 0L; for (int i = 0; i < MAX_REQUESTS; i++) { - List tweets = getTweets(getApiURL(maxID)); + List tweets = getTweets(getApiURL(lastMaxID - 1)); if (tweets.size() == 0) { logger.info(" No more tweets found."); break; } + logger.debug("Twitter response #" + (i + 1) + " Tweets:\n" + tweets); + if (tweets.size() == 1 && + lastMaxID.equals(tweets.get(0).getString("id_str")) + ) { + break; + } + for (JSONObject tweet : tweets) { - maxID = tweet.getString("id_str"); + lastMaxID = tweet.getLong("id"); parseTweet(tweet); } try { Thread.sleep(WAIT_TIME); - } catch (InterruptedException e) { + } + catch (InterruptedException e) { logger.error("[!] Interrupted while waiting to load more results", e); break; } diff --git a/src/main/resources/rip.properties b/src/main/resources/rip.properties index 8ad70c86..add7fcf9 100644 --- a/src/main/resources/rip.properties +++ b/src/main/resources/rip.properties @@ -17,3 +17,5 @@ download.max_size = 104857600 twitter.auth = VW9Ybjdjb1pkd2J0U3kwTUh2VXVnOm9GTzVQVzNqM29LQU1xVGhnS3pFZzhKbGVqbXU0c2lHQ3JrUFNNZm8= tumblr.auth = v5kUqGQXUtmF7K0itri1DGtgTs0VQpbSEbh1jxYgj9d2Sq18F8 gw.api = gonewild + +twitter.max_requests = 10