ImgScroll/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java

106 lines
3.1 KiB
Java
Raw Normal View History

package com.rarchives.ripme.ripper;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.List;
import org.json.JSONObject;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Utils;
2014-06-23 04:17:40 +02:00
/**
* Simplified ripper, designed for ripping from sites by parsing JSON.
*/
public abstract class AbstractJSONRipper extends AlbumRipper {
protected AbstractJSONRipper(URL url) throws IOException {
super(url);
}
protected abstract String getDomain();
public abstract String getHost();
protected abstract JSONObject getFirstPage() throws IOException;
protected JSONObject getNextPage(JSONObject doc) throws IOException {
throw new IOException("getNextPage not implemented");
}
protected abstract List<String> getURLsFromJSON(JSONObject json);
protected abstract void downloadURL(URL url, int index);
private DownloadThreadPool getThreadPool() {
return null;
}
protected boolean keepSortOrder() {
return true;
}
@Override
public boolean canRip(URL url) {
return url.getHost().endsWith(getDomain());
}
2017-06-19 19:32:57 +02:00
@Override
public URL sanitizeURL(URL url) throws MalformedURLException {
return url;
}
@Override
public void rip() throws IOException {
int index = 0;
logger.info("Retrieving " + this.url);
sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm());
JSONObject json = getFirstPage();
while (json != null) {
List<String> imageURLs = getURLsFromJSON(json);
// Remove all but 1 image
if (isThisATest()) {
while (imageURLs.size() > 1) {
imageURLs.remove(1);
}
}
if (imageURLs.size() == 0) {
throw new IOException("No images found at " + this.url);
}
for (String imageURL : imageURLs) {
if (isStopped()) {
break;
}
index += 1;
logger.debug("Found image url #" + index+ ": " + imageURL);
downloadURL(new URL(imageURL), index);
}
if (isStopped() || isThisATest()) {
break;
}
try {
sendUpdate(STATUS.LOADING_RESOURCE, "next page");
json = getNextPage(json);
} catch (IOException e) {
logger.info("Can't get next page: " + e.getMessage());
break;
}
}
// If they're using a thread pool, wait for it.
if (getThreadPool() != null) {
logger.debug("Waiting for threadpool " + getThreadPool().getClass().getName());
getThreadPool().waitForThreads();
}
waitForThreads();
}
protected String getPrefix(int index) {
String prefix = "";
if (keepSortOrder() && Utils.getConfigBoolean("download.save_order", true)) {
prefix = String.format("%03d_", index);
}
return prefix;
}
}