2014-06-22 02:08:42 +02:00
|
|
|
package com.rarchives.ripme.ripper;
|
|
|
|
|
|
|
|
import java.io.IOException;
|
|
|
|
import java.net.MalformedURLException;
|
|
|
|
import java.net.URL;
|
|
|
|
import java.util.List;
|
|
|
|
|
|
|
|
import org.jsoup.nodes.Document;
|
|
|
|
|
|
|
|
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
|
|
|
|
import com.rarchives.ripme.utils.Utils;
|
|
|
|
|
2014-06-23 04:12:29 +02:00
|
|
|
public abstract class AbstractHTMLRipper extends AlbumRipper {
|
2014-06-22 02:08:42 +02:00
|
|
|
|
2014-06-23 04:12:29 +02:00
|
|
|
public AbstractHTMLRipper(URL url) throws IOException {
|
2014-06-22 02:08:42 +02:00
|
|
|
super(url);
|
|
|
|
}
|
|
|
|
|
|
|
|
public abstract String getDomain();
|
|
|
|
public abstract String getHost();
|
|
|
|
|
|
|
|
public abstract Document getFirstPage() throws IOException;
|
|
|
|
public abstract Document getNextPage(Document doc) throws IOException;
|
|
|
|
public abstract List<String> getURLsFromPage(Document page);
|
|
|
|
public abstract void downloadURL(URL url, int index);
|
2014-06-23 04:12:29 +02:00
|
|
|
public DownloadThreadPool getThreadPool() {
|
|
|
|
return null;
|
|
|
|
}
|
2014-06-22 02:08:42 +02:00
|
|
|
|
|
|
|
public boolean keepSortOrder() {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public boolean canRip(URL url) {
|
|
|
|
return url.getHost().endsWith(getDomain());
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public URL sanitizeURL(URL url) throws MalformedURLException {
|
|
|
|
return url;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public void rip() throws IOException {
|
|
|
|
int index = 0;
|
|
|
|
logger.info("Retrieving " + this.url);
|
|
|
|
sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm());
|
|
|
|
Document doc = getFirstPage();
|
|
|
|
|
|
|
|
while (doc != null) {
|
|
|
|
List<String> imageURLs = getURLsFromPage(doc);
|
|
|
|
|
|
|
|
if (imageURLs.size() == 0) {
|
|
|
|
throw new IOException("No images found at " + this.url);
|
|
|
|
}
|
|
|
|
|
|
|
|
for (String imageURL : imageURLs) {
|
|
|
|
if (isStopped()) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
index += 1;
|
|
|
|
downloadURL(new URL(imageURL), index);
|
|
|
|
}
|
2014-06-23 04:12:29 +02:00
|
|
|
|
|
|
|
if (isStopped()) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2014-06-22 02:08:42 +02:00
|
|
|
try {
|
2014-06-23 04:12:29 +02:00
|
|
|
sendUpdate(STATUS.LOADING_RESOURCE, "next page");
|
2014-06-22 02:08:42 +02:00
|
|
|
doc = getNextPage(doc);
|
|
|
|
} catch (IOException e) {
|
|
|
|
logger.info("Can't get next page: " + e.getMessage());
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2014-06-23 04:12:29 +02:00
|
|
|
|
|
|
|
// If they're using a thread pool, wait for it.
|
|
|
|
if (getThreadPool() != null) {
|
|
|
|
getThreadPool().waitForThreads();
|
|
|
|
}
|
2014-06-22 02:08:42 +02:00
|
|
|
waitForThreads();
|
|
|
|
}
|
|
|
|
|
|
|
|
public String getPrefix(int index) {
|
|
|
|
String prefix = "";
|
|
|
|
if (keepSortOrder() && Utils.getConfigBoolean("download.save_order", true)) {
|
|
|
|
prefix = String.format("%03d_", index);
|
|
|
|
}
|
|
|
|
return prefix;
|
|
|
|
}
|
|
|
|
}
|