Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Twitter ripper enhancements

Browse Source
This commit is contained in:
4pr0n 2015-02-06 11:39:51 -08:00
parent c2bb65bc60
commit bfd575906e
1 changed files with 21 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
View File

@ -168,10 +168,10 @@ public class TwitterRipper extends AlbumRipper {
return tweets;
}
private void parseTweet(JSONObject tweet) throws MalformedURLException {
private boolean parseTweet(JSONObject tweet) throws MalformedURLException {
if (!tweet.has("entities")) {
logger.error("XXX Tweet doesn't have entitites");
return;
return false;
}
JSONObject entities = tweet.getJSONObject("entities");
@ -185,8 +185,12 @@ public class TwitterRipper extends AlbumRipper {
url = media.getString("media_url");
if (url.contains(".twimg.com/")) {
url += ":large";
addURLToDownload(new URL(url));
return true;
}
else {
logger.debug("Unexpected media_url: " + url);
}
addURLToDownload(new URL(url));
}
}
@ -204,6 +208,7 @@ public class TwitterRipper extends AlbumRipper {
}
}
*/
return false;
}
@Override
@ -220,11 +225,8 @@ public class TwitterRipper extends AlbumRipper {
}
Long lastMaxID = 0L;
int parsedCount = 0;
for (int i = 0; i < MAX_REQUESTS; i++) {
if (isStopped()) {
break;
}
List<JSONObject> tweets = getTweets(getApiURL(lastMaxID - 1));
if (tweets.size() == 0) {
logger.info(" No more tweets found.");
@ -234,12 +236,22 @@ public class TwitterRipper extends AlbumRipper {
if (tweets.size() == 1 &&
lastMaxID.equals(tweets.get(0).getString("id_str"))
) {
logger.info(" No more tweet found.");
break;
}
for (JSONObject tweet : tweets) {
lastMaxID = tweet.getLong("id");
parseTweet(tweet);
if (parseTweet(tweet)) {
parsedCount++;
}
if (isStopped() || (isThisATest() && parsedCount > 0) ) {
break;
}
}
if (isStopped() || (isThisATest() && parsedCount > 0) ) {
break;
}
try {