From 16cfe8a9ddbc145133c11cffcd37b834e9a4ca65 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 19 Jun 2018 16:06:19 -0400 Subject: [PATCH 1/3] AbstractJSONRipper now respects hasASAPRipping --- .../java/com/rarchives/ripme/ripper/AbstractJSONRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index 291dd7df..4455270e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -69,7 +69,7 @@ public abstract class AbstractJSONRipper extends AlbumRipper { } } - if (imageURLs.isEmpty()) { + if (imageURLs.isEmpty() && !hasASAPRipping()) { throw new IOException("No images found at " + this.url); } From 867af57713655886305ecf47468dd889ef8a973e Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 19 Jun 2018 16:06:39 -0400 Subject: [PATCH 2/3] Got InstagramRipper working again --- .../ripme/ripper/rippers/InstagramRipper.java | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index cb410fb3..b38d37ea 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -76,6 +76,10 @@ public class InstagramRipper extends AbstractJSONRipper { return url.replaceAll("/[A-Z0-9]{8}/", "/"); } + @Override public boolean hasASAPRipping() { + return true; + } + private List getPostsFromSinglePage(JSONObject json) { List imageURLs = new ArrayList<>(); JSONArray datas; @@ -231,9 +235,21 @@ public class InstagramRipper extends AbstractJSONRipper { return imageURL; } + public String getAfter(JSONObject json) { + try { + return json.getJSONObject("entry_data").getJSONArray("ProfilePage").getJSONObject(0) + .getJSONObject("graphql").getJSONObject("user") + .getJSONObject("edge_owner_to_timeline_media").getJSONObject("page_info").getString("end_cursor"); + } catch (JSONException e) { + return json.getJSONObject("data").getJSONObject("user") + .getJSONObject("edge_owner_to_timeline_media").getJSONObject("page_info").getString("end_cursor"); + } + } + @Override public List getURLsFromJSON(JSONObject json) { List imageURLs = new ArrayList<>(); + nextPageID = getAfter(json); // get the rhx_gis value so we can get the next page later on if (rhx_gis == null) { @@ -251,7 +267,7 @@ public class InstagramRipper extends AbstractJSONRipper { .getJSONObject("edge_owner_to_timeline_media").getJSONArray("edges"); } catch (JSONException e) { datas = json.getJSONObject("data").getJSONObject("user") - .getJSONObject("edge_user_to_photos_of_you").getJSONArray("edges"); + .getJSONObject("edge_owner_to_timeline_media").getJSONArray("edges"); } } else { try { @@ -301,11 +317,10 @@ public class InstagramRipper extends AbstractJSONRipper { } } } catch (MalformedURLException e) { + LOGGER.info("Got MalformedURLException"); return imageURLs; } - nextPageID = data.getString("id"); - if (isThisATest()) { break; } @@ -369,10 +384,11 @@ public class InstagramRipper extends AbstractJSONRipper { try { // Sleep for a while to avoid a ban sleep(2500); - String vars = "{\"id\":\"" + userID + "\",\"first\":50,\"after\":\"" + nextPageID + "\"}"; + String vars = "{\"id\":\"" + userID + "\",\"first\":12,\"after\":\"" + nextPageID + "\"}"; String ig_gis = getIGGis(vars); LOGGER.info(ig_gis); + LOGGER.info("https://www.instagram.com/graphql/query/?query_hash=" + qHash + "&variables=" + vars); toreturn = getPage("https://www.instagram.com/graphql/query/?query_hash=" + qHash + "&variables=" + vars, ig_gis); if (!pageHasImages(toreturn)) { throw new IOException("No more pages"); @@ -394,7 +410,7 @@ public class InstagramRipper extends AbstractJSONRipper { private boolean pageHasImages(JSONObject json) { LOGGER.info(json); int numberOfImages = json.getJSONObject("data").getJSONObject("user") - .getJSONObject("edge_user_to_photos_of_you").getJSONArray("edges").length(); + .getJSONObject("edge_owner_to_timeline_media").getJSONArray("edges").length(); if (numberOfImages == 0) { return false; } @@ -453,26 +469,11 @@ public class InstagramRipper extends AbstractJSONRipper { return null; } if (!rippingTag) { - Pattern jsP = Pattern.compile("o},queryId:.([a-zA-Z0-9]+)."); + Pattern jsP = Pattern.compile("m=\"9ca88e465c3f866a76f7adee3871bdd8\",g=Object\\(c.b\\)\\(\\{pageSize:p.a,pagesToPreload:0,getState:function\\(e,t\\)\\{var o;return null===\\(o=e.profilePosts.byUserId.get\\(t\\)\\)\\|\\|void 0===o\\?void 0:o\\.pagination},queryId:.([a-zA-Z0-9]+)"); Matcher m = jsP.matcher(sb.toString()); if (m.find()) { return m.group(1); } - jsP = Pattern.compile("n.pagination:n},queryId:.([a-zA-Z0-9]+)."); - m = jsP.matcher(sb.toString()); - if (m.find()) { - return m.group(1); - } - jsP = Pattern.compile("0:n.pagination},queryId:.([a-zA-Z0-9]+)."); - m = jsP.matcher(sb.toString()); - if (m.find()) { - return m.group(1); - } - jsP = Pattern.compile("o.pagination},queryId:.([a-zA-Z0-9]+)."); - m = jsP.matcher(sb.toString()); - if (m.find()) { - return m.group(1); - } } else { Pattern jsP = Pattern.compile("return e.tagMedia.byTagName.get\\(t\\).pagination},queryId:.([a-zA-Z0-9]+)."); From 98d4c57f93cd35e3c65d04efff8fa0450d3f84eb Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 19 Jun 2018 16:22:22 -0400 Subject: [PATCH 3/3] Fixed ripping from single instagram page --- .../com/rarchives/ripme/ripper/rippers/InstagramRipper.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index b38d37ea..3077df94 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -249,7 +249,9 @@ public class InstagramRipper extends AbstractJSONRipper { @Override public List getURLsFromJSON(JSONObject json) { List imageURLs = new ArrayList<>(); - nextPageID = getAfter(json); + if (!url.toExternalForm().contains("/p/")) { + nextPageID = getAfter(json); + } // get the rhx_gis value so we can get the next page later on if (rhx_gis == null) {