Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Twitter ripper enhancements

Browse Source
This commit is contained in:
4pr0n 2015-02-06 11:39:51 -08:00
parent c2bb65bc60
commit bfd575906e
1 changed files with 21 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
View File

@ -168,10 +168,10 @@ public class TwitterRipper extends AlbumRipper {
return tweets; return tweets;
} }
private void parseTweet(JSONObject tweet) throws MalformedURLException { private boolean parseTweet(JSONObject tweet) throws MalformedURLException {
if (!tweet.has("entities")) { if (!tweet.has("entities")) {
logger.error("XXX Tweet doesn't have entitites"); logger.error("XXX Tweet doesn't have entitites");
return; return false;
} }
JSONObject entities = tweet.getJSONObject("entities"); JSONObject entities = tweet.getJSONObject("entities");
@ -185,8 +185,12 @@ public class TwitterRipper extends AlbumRipper {
url = media.getString("media_url"); url = media.getString("media_url");
if (url.contains(".twimg.com/")) { if (url.contains(".twimg.com/")) {
url += ":large"; url += ":large";
}
addURLToDownload(new URL(url)); addURLToDownload(new URL(url));
return true;
}
else {
logger.debug("Unexpected media_url: " + url);
}
} }
} }
@ -204,6 +208,7 @@ public class TwitterRipper extends AlbumRipper {
} }
} }
*/ */
return false;
} }
@Override @Override
@ -220,11 +225,8 @@ public class TwitterRipper extends AlbumRipper {
} }
Long lastMaxID = 0L; Long lastMaxID = 0L;
int parsedCount = 0;
for (int i = 0; i < MAX_REQUESTS; i++) { for (int i = 0; i < MAX_REQUESTS; i++) {
if (isStopped()) {
break;
}
List<JSONObject> tweets = getTweets(getApiURL(lastMaxID - 1)); List<JSONObject> tweets = getTweets(getApiURL(lastMaxID - 1));
if (tweets.size() == 0) { if (tweets.size() == 0) {
logger.info(" No more tweets found."); logger.info(" No more tweets found.");
@ -234,12 +236,22 @@ public class TwitterRipper extends AlbumRipper {
if (tweets.size() == 1 && if (tweets.size() == 1 &&
lastMaxID.equals(tweets.get(0).getString("id_str")) lastMaxID.equals(tweets.get(0).getString("id_str"))
) { ) {
logger.info(" No more tweet found.");
break; break;
} }
for (JSONObject tweet : tweets) { for (JSONObject tweet : tweets) {
lastMaxID = tweet.getLong("id"); lastMaxID = tweet.getLong("id");
parseTweet(tweet); if (parseTweet(tweet)) {
parsedCount++;
}
if (isStopped() || (isThisATest() && parsedCount > 0) ) {
break;
}
}
if (isStopped() || (isThisATest() && parsedCount > 0) ) {
break;
} }
try { try {