diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java index b3f1b832..47c724cf 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java @@ -168,10 +168,10 @@ public class TwitterRipper extends AlbumRipper { return tweets; } - private void parseTweet(JSONObject tweet) throws MalformedURLException { + private boolean parseTweet(JSONObject tweet) throws MalformedURLException { if (!tweet.has("entities")) { logger.error("XXX Tweet doesn't have entitites"); - return; + return false; } JSONObject entities = tweet.getJSONObject("entities"); @@ -185,8 +185,12 @@ public class TwitterRipper extends AlbumRipper { url = media.getString("media_url"); if (url.contains(".twimg.com/")) { url += ":large"; + addURLToDownload(new URL(url)); + return true; + } + else { + logger.debug("Unexpected media_url: " + url); } - addURLToDownload(new URL(url)); } } @@ -204,6 +208,7 @@ public class TwitterRipper extends AlbumRipper { } } */ + return false; } @Override @@ -220,11 +225,8 @@ public class TwitterRipper extends AlbumRipper { } Long lastMaxID = 0L; + int parsedCount = 0; for (int i = 0; i < MAX_REQUESTS; i++) { - if (isStopped()) { - break; - } - List tweets = getTweets(getApiURL(lastMaxID - 1)); if (tweets.size() == 0) { logger.info(" No more tweets found."); @@ -234,12 +236,22 @@ public class TwitterRipper extends AlbumRipper { if (tweets.size() == 1 && lastMaxID.equals(tweets.get(0).getString("id_str")) ) { + logger.info(" No more tweet found."); break; } - + for (JSONObject tweet : tweets) { lastMaxID = tweet.getLong("id"); - parseTweet(tweet); + if (parseTweet(tweet)) { + parsedCount++; + } + if (isStopped() || (isThisATest() && parsedCount > 0) ) { + break; + } + } + + if (isStopped() || (isThisATest() && parsedCount > 0) ) { + break; } try {