fixing rip completion check, various log lines

This commit is contained in:
4pr0n 2014-03-09 17:12:10 -07:00
parent c288afdd0a
commit 8e1dc5789b
10 changed files with 29 additions and 26 deletions

View File

@ -151,6 +151,7 @@ public abstract class AbstractRipper
protected void waitForThreads() { protected void waitForThreads() {
completed = false; completed = false;
threadPool.waitForThreads(); threadPool.waitForThreads();
checkIfComplete();
} }
/** /**
@ -223,14 +224,15 @@ public abstract class AbstractRipper
itemsErrored.put(url, message); itemsErrored.put(url, message);
observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " : " + message)); observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " : " + message));
observer.notifyAll(); observer.notifyAll();
checkIfComplete();
} }
checkIfComplete();
} }
/** /**
* Notifies observers and updates state if all files have been ripped. * Notifies observers and updates state if all files have been ripped.
*/ */
private void checkIfComplete() { private void checkIfComplete() {
synchronized (observer) {
if (!completed && itemsPending.size() == 0) { if (!completed && itemsPending.size() == 0) {
completed = true; completed = true;
logger.info(" Rip completed!"); logger.info(" Rip completed!");
@ -241,6 +243,7 @@ public abstract class AbstractRipper
observer.notifyAll(); observer.notifyAll();
} }
} }
}
public URL getURL() { public URL getURL() {
return url; return url;

View File

@ -60,7 +60,7 @@ public class ImagefapRipper extends AbstractRipper {
@Override @Override
public void rip() throws IOException { public void rip() throws IOException {
int index = 0; int index = 0;
logger.info("[ ] Retrieving " + this.url.toExternalForm()); logger.info(" Retrieving " + this.url.toExternalForm());
Document doc = Jsoup.connect(this.url.toExternalForm()).get(); Document doc = Jsoup.connect(this.url.toExternalForm()).get();
for (Element thumb : doc.select("#gallery img")) { for (Element thumb : doc.select("#gallery img")) {
if (!thumb.hasAttr("src") || !thumb.hasAttr("width")) { if (!thumb.hasAttr("src") || !thumb.hasAttr("width")) {

View File

@ -58,7 +58,7 @@ public class MotherlessRipper extends AbstractRipper {
@Override @Override
public void rip() throws IOException { public void rip() throws IOException {
int index = 0; int index = 0;
logger.info("[ ] Retrieving " + this.url.toExternalForm()); logger.info(" Retrieving " + this.url.toExternalForm());
Document doc = Jsoup.connect(this.url.toExternalForm()) Document doc = Jsoup.connect(this.url.toExternalForm())
.userAgent(USER_AGENT) .userAgent(USER_AGENT)
.get(); .get();

View File

@ -41,7 +41,7 @@ public class SeeniveRipper extends AbstractRipper {
@Override @Override
public void rip() throws IOException { public void rip() throws IOException {
String baseURL = this.url.toExternalForm(); String baseURL = this.url.toExternalForm();
logger.info("[ ] Retrieving " + baseURL); logger.info(" Retrieving " + baseURL);
Document doc = Jsoup.connect(baseURL) Document doc = Jsoup.connect(baseURL)
.header("Referer", baseURL) .header("Referer", baseURL)
.userAgent(USER_AGENT) .userAgent(USER_AGENT)

View File

@ -67,15 +67,15 @@ public class TumblrRipper extends AbstractRipper {
.ignoreContentType(true) .ignoreContentType(true)
.header("User-agent", USER_AGENT) .header("User-agent", USER_AGENT)
.get(); .get();
String jsonString = doc.body().html().replaceAll(""", "\"");
if (!handleJSON(jsonString)) {
// Returns false if an error occurs and we should stop.
break;
}
try { try {
Thread.sleep(1000); Thread.sleep(1000);
} catch (InterruptedException e) { } catch (InterruptedException e) {
logger.error("[!] Exception while waiting to load next album:", e); logger.error("[!] Interrupted while waiting to load next album:", e);
break;
}
String jsonString = doc.body().html().replaceAll(""", "\"");
if (!handleJSON(jsonString)) {
// Returns false if an error occurs and we should stop.
break; break;
} }
offset += 20; offset += 20;
@ -96,7 +96,7 @@ public class TumblrRipper extends AbstractRipper {
posts = json.getJSONObject("response").getJSONArray("posts"); posts = json.getJSONObject("response").getJSONArray("posts");
if (posts.length() == 0) { if (posts.length() == 0) {
logger.info(" Zero posts returned. Dropping out."); logger.info(" Zero posts returned.");
return false; return false;
} }

View File

@ -41,7 +41,7 @@ public class VineboxRipper extends AbstractRipper {
while (true) { while (true) {
page++; page++;
String urlPaged = this.url.toExternalForm() + "?page=" + page; String urlPaged = this.url.toExternalForm() + "?page=" + page;
logger.info("[ ] Retrieving " + urlPaged); logger.info(" Retrieving " + urlPaged);
try { try {
doc = Jsoup.connect(urlPaged).get(); doc = Jsoup.connect(urlPaged).get();
} catch (HttpStatusException e) { } catch (HttpStatusException e) {

View File

@ -15,8 +15,8 @@ public class TumblrRipperTest extends RippersTest {
} }
List<URL> contentURLs = new ArrayList<URL>(); List<URL> contentURLs = new ArrayList<URL>();
contentURLs.add(new URL("http://wrouinr.tumblr.com/archive")); contentURLs.add(new URL("http://wrouinr.tumblr.com/archive"));
contentURLs.add(new URL("http://topinstagirls.tumblr.com/tagged/berlinskaya")); //contentURLs.add(new URL("http://topinstagirls.tumblr.com/tagged/berlinskaya"));
contentURLs.add(new URL("http://fittingroomgirls.tumblr.com/post/78268776776")); //contentURLs.add(new URL("http://fittingroomgirls.tumblr.com/post/78268776776"));
for (URL url : contentURLs) { for (URL url : contentURLs) {
try { try {
TumblrRipper ripper = new TumblrRipper(url); TumblrRipper ripper = new TumblrRipper(url);