c5ea044f79
Ability to set log level, lots of debugging messages Turn on debug logging during tests, simplified test cases for HTML ripper Fix fusktator ripper, added test Fixed gifyo, added test Added tests for *all* rippers Adding a few album-guessing URLs
106 lines
3.1 KiB
Java
106 lines
3.1 KiB
Java
package com.rarchives.ripme.ripper;
|
|
|
|
import java.io.IOException;
|
|
import java.net.MalformedURLException;
|
|
import java.net.URL;
|
|
import java.util.List;
|
|
|
|
import org.json.JSONObject;
|
|
|
|
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
|
|
import com.rarchives.ripme.utils.Utils;
|
|
|
|
/**
|
|
* Simplified ripper, designed for ripping from sites by parsing JSON.
|
|
*/
|
|
public abstract class AbstractJSONRipper extends AlbumRipper {
|
|
|
|
public AbstractJSONRipper(URL url) throws IOException {
|
|
super(url);
|
|
}
|
|
|
|
public abstract String getDomain();
|
|
public abstract String getHost();
|
|
|
|
public abstract JSONObject getFirstPage() throws IOException;
|
|
public JSONObject getNextPage(JSONObject doc) throws IOException {
|
|
throw new IOException("getNextPage not implemented");
|
|
}
|
|
public abstract List<String> getURLsFromJSON(JSONObject json);
|
|
public abstract void downloadURL(URL url, int index);
|
|
public DownloadThreadPool getThreadPool() {
|
|
return null;
|
|
}
|
|
|
|
public boolean keepSortOrder() {
|
|
return true;
|
|
}
|
|
|
|
@Override
|
|
public boolean canRip(URL url) {
|
|
return url.getHost().endsWith(getDomain());
|
|
}
|
|
|
|
@Override
|
|
public URL sanitizeURL(URL url) throws MalformedURLException {
|
|
return url;
|
|
}
|
|
|
|
@Override
|
|
public void rip() throws IOException {
|
|
int index = 0;
|
|
logger.info("Retrieving " + this.url);
|
|
sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm());
|
|
JSONObject json = getFirstPage();
|
|
|
|
while (json != null) {
|
|
List<String> imageURLs = getURLsFromJSON(json);
|
|
// Remove all but 1 image
|
|
if (isThisATest()) {
|
|
while (imageURLs.size() > 1) {
|
|
imageURLs.remove(1);
|
|
}
|
|
}
|
|
|
|
if (imageURLs.size() == 0) {
|
|
throw new IOException("No images found at " + this.url);
|
|
}
|
|
|
|
for (String imageURL : imageURLs) {
|
|
if (isStopped()) {
|
|
break;
|
|
}
|
|
index += 1;
|
|
logger.debug("Found image url #" + index+ ": " + imageURL);
|
|
downloadURL(new URL(imageURL), index);
|
|
}
|
|
|
|
if (isStopped() || isThisATest()) {
|
|
break;
|
|
}
|
|
|
|
try {
|
|
sendUpdate(STATUS.LOADING_RESOURCE, "next page");
|
|
json = getNextPage(json);
|
|
} catch (IOException e) {
|
|
logger.info("Can't get next page: " + e.getMessage());
|
|
break;
|
|
}
|
|
}
|
|
|
|
// If they're using a thread pool, wait for it.
|
|
if (getThreadPool() != null) {
|
|
logger.debug("Waiting for threadpool " + getThreadPool().getClass().getName());
|
|
getThreadPool().waitForThreads();
|
|
}
|
|
waitForThreads();
|
|
}
|
|
|
|
public String getPrefix(int index) {
|
|
String prefix = "";
|
|
if (keepSortOrder() && Utils.getConfigBoolean("download.save_order", true)) {
|
|
prefix = String.format("%03d_", index);
|
|
}
|
|
return prefix;
|
|
}
|
|
} |