fixing rip completion check, various log lines
This commit is contained in:
parent
c288afdd0a
commit
8e1dc5789b
@ -151,6 +151,7 @@ public abstract class AbstractRipper
|
|||||||
protected void waitForThreads() {
|
protected void waitForThreads() {
|
||||||
completed = false;
|
completed = false;
|
||||||
threadPool.waitForThreads();
|
threadPool.waitForThreads();
|
||||||
|
checkIfComplete();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -223,14 +224,15 @@ public abstract class AbstractRipper
|
|||||||
itemsErrored.put(url, message);
|
itemsErrored.put(url, message);
|
||||||
observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " : " + message));
|
observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " : " + message));
|
||||||
observer.notifyAll();
|
observer.notifyAll();
|
||||||
checkIfComplete();
|
|
||||||
}
|
}
|
||||||
|
checkIfComplete();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Notifies observers and updates state if all files have been ripped.
|
* Notifies observers and updates state if all files have been ripped.
|
||||||
*/
|
*/
|
||||||
private void checkIfComplete() {
|
private void checkIfComplete() {
|
||||||
|
synchronized (observer) {
|
||||||
if (!completed && itemsPending.size() == 0) {
|
if (!completed && itemsPending.size() == 0) {
|
||||||
completed = true;
|
completed = true;
|
||||||
logger.info(" Rip completed!");
|
logger.info(" Rip completed!");
|
||||||
@ -241,6 +243,7 @@ public abstract class AbstractRipper
|
|||||||
observer.notifyAll();
|
observer.notifyAll();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public URL getURL() {
|
public URL getURL() {
|
||||||
return url;
|
return url;
|
||||||
|
@ -60,7 +60,7 @@ public class ImagefapRipper extends AbstractRipper {
|
|||||||
@Override
|
@Override
|
||||||
public void rip() throws IOException {
|
public void rip() throws IOException {
|
||||||
int index = 0;
|
int index = 0;
|
||||||
logger.info("[ ] Retrieving " + this.url.toExternalForm());
|
logger.info(" Retrieving " + this.url.toExternalForm());
|
||||||
Document doc = Jsoup.connect(this.url.toExternalForm()).get();
|
Document doc = Jsoup.connect(this.url.toExternalForm()).get();
|
||||||
for (Element thumb : doc.select("#gallery img")) {
|
for (Element thumb : doc.select("#gallery img")) {
|
||||||
if (!thumb.hasAttr("src") || !thumb.hasAttr("width")) {
|
if (!thumb.hasAttr("src") || !thumb.hasAttr("width")) {
|
||||||
|
@ -58,7 +58,7 @@ public class MotherlessRipper extends AbstractRipper {
|
|||||||
@Override
|
@Override
|
||||||
public void rip() throws IOException {
|
public void rip() throws IOException {
|
||||||
int index = 0;
|
int index = 0;
|
||||||
logger.info("[ ] Retrieving " + this.url.toExternalForm());
|
logger.info(" Retrieving " + this.url.toExternalForm());
|
||||||
Document doc = Jsoup.connect(this.url.toExternalForm())
|
Document doc = Jsoup.connect(this.url.toExternalForm())
|
||||||
.userAgent(USER_AGENT)
|
.userAgent(USER_AGENT)
|
||||||
.get();
|
.get();
|
||||||
|
@ -41,7 +41,7 @@ public class SeeniveRipper extends AbstractRipper {
|
|||||||
@Override
|
@Override
|
||||||
public void rip() throws IOException {
|
public void rip() throws IOException {
|
||||||
String baseURL = this.url.toExternalForm();
|
String baseURL = this.url.toExternalForm();
|
||||||
logger.info("[ ] Retrieving " + baseURL);
|
logger.info(" Retrieving " + baseURL);
|
||||||
Document doc = Jsoup.connect(baseURL)
|
Document doc = Jsoup.connect(baseURL)
|
||||||
.header("Referer", baseURL)
|
.header("Referer", baseURL)
|
||||||
.userAgent(USER_AGENT)
|
.userAgent(USER_AGENT)
|
||||||
|
@ -67,15 +67,15 @@ public class TumblrRipper extends AbstractRipper {
|
|||||||
.ignoreContentType(true)
|
.ignoreContentType(true)
|
||||||
.header("User-agent", USER_AGENT)
|
.header("User-agent", USER_AGENT)
|
||||||
.get();
|
.get();
|
||||||
String jsonString = doc.body().html().replaceAll(""", "\"");
|
|
||||||
if (!handleJSON(jsonString)) {
|
|
||||||
// Returns false if an error occurs and we should stop.
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
Thread.sleep(1000);
|
Thread.sleep(1000);
|
||||||
} catch (InterruptedException e) {
|
} catch (InterruptedException e) {
|
||||||
logger.error("[!] Exception while waiting to load next album:", e);
|
logger.error("[!] Interrupted while waiting to load next album:", e);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
String jsonString = doc.body().html().replaceAll(""", "\"");
|
||||||
|
if (!handleJSON(jsonString)) {
|
||||||
|
// Returns false if an error occurs and we should stop.
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
offset += 20;
|
offset += 20;
|
||||||
@ -96,7 +96,7 @@ public class TumblrRipper extends AbstractRipper {
|
|||||||
|
|
||||||
posts = json.getJSONObject("response").getJSONArray("posts");
|
posts = json.getJSONObject("response").getJSONArray("posts");
|
||||||
if (posts.length() == 0) {
|
if (posts.length() == 0) {
|
||||||
logger.info(" Zero posts returned. Dropping out.");
|
logger.info(" Zero posts returned.");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -41,7 +41,7 @@ public class VineboxRipper extends AbstractRipper {
|
|||||||
while (true) {
|
while (true) {
|
||||||
page++;
|
page++;
|
||||||
String urlPaged = this.url.toExternalForm() + "?page=" + page;
|
String urlPaged = this.url.toExternalForm() + "?page=" + page;
|
||||||
logger.info("[ ] Retrieving " + urlPaged);
|
logger.info(" Retrieving " + urlPaged);
|
||||||
try {
|
try {
|
||||||
doc = Jsoup.connect(urlPaged).get();
|
doc = Jsoup.connect(urlPaged).get();
|
||||||
} catch (HttpStatusException e) {
|
} catch (HttpStatusException e) {
|
||||||
|
@ -15,8 +15,8 @@ public class TumblrRipperTest extends RippersTest {
|
|||||||
}
|
}
|
||||||
List<URL> contentURLs = new ArrayList<URL>();
|
List<URL> contentURLs = new ArrayList<URL>();
|
||||||
contentURLs.add(new URL("http://wrouinr.tumblr.com/archive"));
|
contentURLs.add(new URL("http://wrouinr.tumblr.com/archive"));
|
||||||
contentURLs.add(new URL("http://topinstagirls.tumblr.com/tagged/berlinskaya"));
|
//contentURLs.add(new URL("http://topinstagirls.tumblr.com/tagged/berlinskaya"));
|
||||||
contentURLs.add(new URL("http://fittingroomgirls.tumblr.com/post/78268776776"));
|
//contentURLs.add(new URL("http://fittingroomgirls.tumblr.com/post/78268776776"));
|
||||||
for (URL url : contentURLs) {
|
for (URL url : contentURLs) {
|
||||||
try {
|
try {
|
||||||
TumblrRipper ripper = new TumblrRipper(url);
|
TumblrRipper ripper = new TumblrRipper(url);
|
||||||
|
Loading…
Reference in New Issue
Block a user