Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #816 from mbc07/ArtStation_multi_page_fix

Browse Source 
ArtStationRipper: fix ripping from user profiles which many projects
This commit is contained in:
cyian-1756 2018-07-21 16:52:25 -04:00 committed by GitHub
commit f3fe3ebebd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 18 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java
View File

@ -21,6 +21,7 @@ public class ArtStationRipper extends AbstractJSONRipper {
private ParsedURL albumURL; private ParsedURL albumURL;
private String projectName; private String projectName;
private Integer projectIndex; private Integer projectIndex;
private Integer projectPageNumber;
public ArtStationRipper(URL url) throws IOException { public ArtStationRipper(URL url) throws IOException {
super(url); super(url);
@ -95,14 +96,25 @@ public class ArtStationRipper extends AbstractJSONRipper {
@Override @Override
protected JSONObject getNextPage(JSONObject doc) throws IOException { protected JSONObject getNextPage(JSONObject doc) throws IOException {
if (albumURL.getType() == URL_TYPE.USER_PORTFOLIO) { if (albumURL.getType() == URL_TYPE.USER_PORTFOLIO) {
// Initialize the index if it hasn't been initialized already // Initialize the page number if it hasn't been initialized already
if (projectIndex == null) { if (projectPageNumber == null) {
projectIndex = 1; projectPageNumber = 1;
} }
JSONObject albumContent = Http.url(albumURL.getLocation()).getJSON(); // Each page holds a maximum of 50 projects. Initialize the index if it hasn't
// been initialized already or increment page number and reset the index if all
// projects of the current page were already processed
if (projectIndex == null) {
projectIndex = 0;
} else if (projectIndex > 49) {
projectPageNumber++;
projectIndex = 0;
}
if (albumContent.getInt("total_count") > projectIndex) { Integer currentProject = ((projectPageNumber - 1) * 50) + (projectIndex + 1);
JSONObject albumContent = Http.url(albumURL.getLocation() + "?page=" + projectPageNumber).getJSON();
if (albumContent.getInt("total_count") > currentProject) {
// Get JSON of the next project and return it // Get JSON of the next project and return it
JSONObject projectInfo = albumContent.getJSONArray("data").getJSONObject(projectIndex); JSONObject projectInfo = albumContent.getJSONArray("data").getJSONObject(projectIndex);
ParsedURL projectURL = parseURL(new URL(projectInfo.getString("permalink"))); ParsedURL projectURL = parseURL(new URL(projectInfo.getString("permalink")));

1
src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java
View File

@ -23,6 +23,7 @@ public class ArtStationRipperTest extends RippersTest {
public void testArtStationUserProfiles() throws IOException { public void testArtStationUserProfiles() throws IOException {
List<URL> contentURLs = new ArrayList<>(); List<URL> contentURLs = new ArrayList<>();
contentURLs.add(new URL("https://www.artstation.com/heitoramatsu")); contentURLs.add(new URL("https://www.artstation.com/heitoramatsu"));
contentURLs.add(new URL("https://artstation.com/kuvshinov_ilya"));
contentURLs.add(new URL("http://artstation.com/givemeapiggy")); contentURLs.add(new URL("http://artstation.com/givemeapiggy"));
for (URL url : contentURLs) { for (URL url : contentURLs) {
ArtStationRipper ripper = new ArtStationRipper(url); ArtStationRipper ripper = new ArtStationRipper(url);