Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #478 from cyian-1756/igFix

Browse Source 
IG ripper can now rip from tags; fixed json parsing issues
This commit is contained in:
cyian-1756 2018-03-17 16:08:39 -04:00 committed by GitHub
commit 2beecc7829
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 61 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java
View File

@ -29,6 +29,8 @@ import com.rarchives.ripme.utils.Utils;
public class InstagramRipper extends AbstractHTMLRipper { public class InstagramRipper extends AbstractHTMLRipper {
String nextPageID = ""; String nextPageID = "";
private String qHash; private String qHash;
private boolean rippingTag = false;
private String tagName;
private String userID; private String userID;
@ -131,11 +133,13 @@ public class InstagramRipper extends AbstractHTMLRipper {
return m.group(1); return m.group(1);
} }
// p = Pattern.compile("^https?://www.instagram.com/explore/tags/([^/]+)/?"); p = Pattern.compile("^https?://www.instagram.com/explore/tags/([^/]+)/?");
// m = p.matcher(url.toExternalForm()); m = p.matcher(url.toExternalForm());
// if (m.matches()) { if (m.matches()) {
// return m.group(1); rippingTag = true;
// } tagName = m.group(1);
return m.group(1);
}
throw new MalformedURLException("Unable to find user in " + url); throw new MalformedURLException("Unable to find user in " + url);
} }
@ -146,6 +150,8 @@ public class InstagramRipper extends AbstractHTMLRipper {
" <body>", ""); " <body>", "");
t.replaceAll("</body>\n" + t.replaceAll("</body>\n" +
"</html>", ""); "</html>", "");
t = t.replaceAll("\n", "");
t = t.replaceAll("=\"\"", "");
return t; return t;
} }
@ -230,6 +236,7 @@ public class InstagramRipper extends AbstractHTMLRipper {
if (!url.toExternalForm().contains("/p/")) { if (!url.toExternalForm().contains("/p/")) {
JSONArray datas = new JSONArray(); JSONArray datas = new JSONArray();
if (!rippingTag) {
// This first try only works on data from the first page // This first try only works on data from the first page
try { try {
JSONArray profilePage = json.getJSONObject("entry_data").getJSONArray("ProfilePage"); JSONArray profilePage = json.getJSONObject("entry_data").getJSONArray("ProfilePage");
@ -240,17 +247,29 @@ public class InstagramRipper extends AbstractHTMLRipper {
datas = json.getJSONObject("data").getJSONObject("user") datas = json.getJSONObject("data").getJSONObject("user")
.getJSONObject("edge_owner_to_timeline_media").getJSONArray("edges"); .getJSONObject("edge_owner_to_timeline_media").getJSONArray("edges");
} }
} else {
try {
JSONArray tagPage = json.getJSONObject("entry_data").getJSONArray("TagPage");
datas = tagPage.getJSONObject(0).getJSONObject("graphql").getJSONObject("hashtag")
.getJSONObject("edge_hashtag_to_media").getJSONArray("edges");
} catch (JSONException e) {
datas = json.getJSONObject("data").getJSONObject("hashtag").getJSONObject("edge_hashtag_to_media")
.getJSONArray("edges");
}
}
for (int i = 0; i < datas.length(); i++) { for (int i = 0; i < datas.length(); i++) {
JSONObject data = (JSONObject) datas.get(i); JSONObject data = (JSONObject) datas.get(i);
data = data.getJSONObject("node"); data = data.getJSONObject("node");
Long epoch = data.getLong("taken_at_timestamp"); Long epoch = data.getLong("taken_at_timestamp");
Instant instant = Instant.ofEpochSecond(epoch); Instant instant = Instant.ofEpochSecond(epoch);
String image_date = DateTimeFormatter.ofPattern("yyyy_MM_dd_hh:mm_").format(ZonedDateTime.ofInstant(instant, ZoneOffset.UTC)); String image_date = DateTimeFormatter.ofPattern("yyyy_MM_dd_hh:mm_").format(ZonedDateTime.ofInstant(instant, ZoneOffset.UTC));
// It looks like tag pages don't have the __typename key
if (!rippingTag) {
if (data.getString("__typename").equals("GraphSidecar")) { if (data.getString("__typename").equals("GraphSidecar")) {
try { try {
Document slideShowDoc = Http.url(new URL("https://www.instagram.com/p/" + data.getString("shortcode"))).get(); Document slideShowDoc = Http.url(new URL("https://www.instagram.com/p/" + data.getString("shortcode"))).get();
List<String> toAdd = getPostsFromSinglePage(slideShowDoc); List<String> toAdd = getPostsFromSinglePage(slideShowDoc);
for (int slideShowInt=0; slideShowInt<toAdd.size(); slideShowInt++) { for (int slideShowInt = 0; slideShowInt < toAdd.size(); slideShowInt++) {
addURLToDownload(new URL(toAdd.get(slideShowInt)), image_date + data.getString("shortcode")); addURLToDownload(new URL(toAdd.get(slideShowInt)), image_date + data.getString("shortcode"));
} }
} catch (MalformedURLException e) { } catch (MalformedURLException e) {
@ -259,6 +278,7 @@ public class InstagramRipper extends AbstractHTMLRipper {
logger.error("Unable to download slide show"); logger.error("Unable to download slide show");
} }
} }
}
try { try {
if (!data.getBoolean("is_video")) { if (!data.getBoolean("is_video")) {
if (imageURLs.size() == 0) { if (imageURLs.size() == 0) {
@ -297,15 +317,12 @@ public class InstagramRipper extends AbstractHTMLRipper {
public Document getNextPage(Document doc) throws IOException { public Document getNextPage(Document doc) throws IOException {
Document toreturn; Document toreturn;
if (!nextPageID.equals("") && !isThisATest()) { if (!nextPageID.equals("") && !isThisATest()) {
if (url.toExternalForm().contains("/tags/")) { if (rippingTag) {
try { try {
// Sleep for a while to avoid a ban
sleep(2500); sleep(2500);
if (url.toExternalForm().substring(url.toExternalForm().length() - 1).equals("/")) { toreturn = Http.url("https://www.instagram.com/graphql/query/?query_hash=" + qHash +
toreturn = Http.url(url.toExternalForm() + "?max_id=" + nextPageID).ignoreContentType().get(); "&variables={\"tag_name\":\"" + tagName + "\",\"first\":4,\"after\":\"" + nextPageID + "\"}").ignoreContentType().get();
} else { // Sleep for a while to avoid a ban
toreturn = Http.url(url.toExternalForm() + "/?max_id=" + nextPageID).ignoreContentType().get();
}
logger.info(toreturn.html()); logger.info(toreturn.html());
return toreturn; return toreturn;
@ -337,6 +354,7 @@ public class InstagramRipper extends AbstractHTMLRipper {
} }
private boolean pageHasImages(Document doc) { private boolean pageHasImages(Document doc) {
logger.info("BAD DATA: " + stripHTMLTags(doc.html()));
JSONObject json = new JSONObject(stripHTMLTags(doc.html())); JSONObject json = new JSONObject(stripHTMLTags(doc.html()));
int numberOfImages = json.getJSONObject("data").getJSONObject("user") int numberOfImages = json.getJSONObject("data").getJSONObject("user")
.getJSONObject("edge_owner_to_timeline_media").getJSONArray("edges").length(); .getJSONObject("edge_owner_to_timeline_media").getJSONArray("edges").length();
@ -369,11 +387,19 @@ public class InstagramRipper extends AbstractHTMLRipper {
logger.info(e.getMessage()); logger.info(e.getMessage());
return null; return null;
} }
if (!rippingTag) {
Pattern jsP = Pattern.compile("o},queryId:.([a-zA-Z0-9]+)."); Pattern jsP = Pattern.compile("o},queryId:.([a-zA-Z0-9]+).");
Matcher m = jsP.matcher(sb.toString()); Matcher m = jsP.matcher(sb.toString());
if (m.find()) { if (m.find()) {
return m.group(1); return m.group(1);
} }
} else {
Pattern jsP = Pattern.compile("return e.tagMedia.byTagName.get\\(t\\).pagination},queryId:.([a-zA-Z0-9]+).");
Matcher m = jsP.matcher(sb.toString());
if (m.find()) {
return m.group(1);
}
}
logger.info("Could not find query_hash on " + jsFileURL); logger.info("Could not find query_hash on " + jsFileURL);
return null; return null;