various bug fixes
This commit is contained in:
parent
2bfa377d5d
commit
aa78f60ce0
@ -99,7 +99,7 @@ public abstract class AbstractRipper
|
|||||||
|| itemsCompleted.containsKey(url)
|
|| itemsCompleted.containsKey(url)
|
||||||
|| itemsErrored.containsKey(url)) {
|
|| itemsErrored.containsKey(url)) {
|
||||||
// Item is already downloaded/downloading, skip it.
|
// Item is already downloaded/downloading, skip it.
|
||||||
logger.info("Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs));
|
logger.info(" Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
itemsPending.put(url, saveAs);
|
itemsPending.put(url, saveAs);
|
||||||
@ -232,6 +232,9 @@ public abstract class AbstractRipper
|
|||||||
* Notifies observers and updates state if all files have been ripped.
|
* Notifies observers and updates state if all files have been ripped.
|
||||||
*/
|
*/
|
||||||
private void checkIfComplete() {
|
private void checkIfComplete() {
|
||||||
|
if (observer == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
synchronized (observer) {
|
synchronized (observer) {
|
||||||
if (!completed && itemsPending.size() == 0) {
|
if (!completed && itemsPending.size() == 0) {
|
||||||
completed = true;
|
completed = true;
|
||||||
@ -359,6 +362,7 @@ public abstract class AbstractRipper
|
|||||||
rip();
|
rip();
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
logger.error("Got exception while running ripper:", e);
|
logger.error("Got exception while running ripper:", e);
|
||||||
|
waitForThreads();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -3,6 +3,8 @@ package com.rarchives.ripme.ripper.rippers;
|
|||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.MalformedURLException;
|
import java.net.MalformedURLException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
import java.util.regex.Matcher;
|
import java.util.regex.Matcher;
|
||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
@ -93,8 +95,18 @@ public class ImgurRipper extends AbstractRipper {
|
|||||||
|
|
||||||
private void ripAlbum(URL url, String subdirectory) throws IOException {
|
private void ripAlbum(URL url, String subdirectory) throws IOException {
|
||||||
int index = 0;
|
int index = 0;
|
||||||
logger.info(" Retrieving " + url.toExternalForm());
|
|
||||||
this.sendUpdate(STATUS.LOADING_RESOURCE, url.toExternalForm());
|
this.sendUpdate(STATUS.LOADING_RESOURCE, url.toExternalForm());
|
||||||
|
index = 0;
|
||||||
|
for (URL singleURL : getURLsFromAlbum(url)) {
|
||||||
|
index += 1;
|
||||||
|
processURL(singleURL, String.format("%03d_", index), subdirectory);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static List<URL> getURLsFromAlbum(URL url) throws IOException {
|
||||||
|
List<URL> result = new ArrayList<URL>();
|
||||||
|
|
||||||
|
logger.info(" Retrieving " + url.toExternalForm());
|
||||||
Document doc = Jsoup.connect(url.toExternalForm())
|
Document doc = Jsoup.connect(url.toExternalForm())
|
||||||
.userAgent(USER_AGENT)
|
.userAgent(USER_AGENT)
|
||||||
.get();
|
.get();
|
||||||
@ -114,10 +126,9 @@ public class ImgurRipper extends AbstractRipper {
|
|||||||
"http://i.imgur.com/"
|
"http://i.imgur.com/"
|
||||||
+ image.get("hash")
|
+ image.get("hash")
|
||||||
+ image.get("ext"));
|
+ image.get("ext"));
|
||||||
index += 1;
|
result.add(imageURL);
|
||||||
processURL(imageURL, String.format("%03d_", index), subdirectory);
|
|
||||||
}
|
}
|
||||||
return;
|
return result;
|
||||||
} catch (JSONException e) {
|
} catch (JSONException e) {
|
||||||
logger.debug("Error while parsing JSON at " + url + ", continuing", e);
|
logger.debug("Error while parsing JSON at " + url + ", continuing", e);
|
||||||
}
|
}
|
||||||
@ -136,10 +147,9 @@ public class ImgurRipper extends AbstractRipper {
|
|||||||
+ "/"
|
+ "/"
|
||||||
+ image.get("hash")
|
+ image.get("hash")
|
||||||
+ image.get("ext"));
|
+ image.get("ext"));
|
||||||
index += 1;
|
result.add(imageURL);
|
||||||
processURL(imageURL, String.format("%03d_", index), subdirectory);
|
|
||||||
}
|
}
|
||||||
return;
|
return result;
|
||||||
} catch (JSONException e) {
|
} catch (JSONException e) {
|
||||||
logger.debug("Error while parsing JSON at " + url + ", continuing", e);
|
logger.debug("Error while parsing JSON at " + url + ", continuing", e);
|
||||||
}
|
}
|
||||||
@ -164,9 +174,9 @@ public class ImgurRipper extends AbstractRipper {
|
|||||||
logger.error("[!] Unable to find image in div: " + thumb.toString());
|
logger.error("[!] Unable to find image in div: " + thumb.toString());
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
index += 1;
|
result.add(new URL(image));
|
||||||
processURL(new URL(image), String.format("%03d_", index), subdirectory);
|
|
||||||
}
|
}
|
||||||
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -10,13 +10,13 @@ import com.rarchives.ripme.ripper.rippers.TumblrRipper;
|
|||||||
public class TumblrRipperTest extends RippersTest {
|
public class TumblrRipperTest extends RippersTest {
|
||||||
|
|
||||||
public void testTumblrAlbums() throws IOException {
|
public void testTumblrAlbums() throws IOException {
|
||||||
if (false && !DOWNLOAD_CONTENT) {
|
if (!DOWNLOAD_CONTENT) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
List<URL> contentURLs = new ArrayList<URL>();
|
List<URL> contentURLs = new ArrayList<URL>();
|
||||||
contentURLs.add(new URL("http://wrouinr.tumblr.com/archive"));
|
contentURLs.add(new URL("http://wrouinr.tumblr.com/archive"));
|
||||||
//contentURLs.add(new URL("http://topinstagirls.tumblr.com/tagged/berlinskaya"));
|
contentURLs.add(new URL("http://topinstagirls.tumblr.com/tagged/berlinskaya"));
|
||||||
//contentURLs.add(new URL("http://fittingroomgirls.tumblr.com/post/78268776776"));
|
contentURLs.add(new URL("http://fittingroomgirls.tumblr.com/post/78268776776"));
|
||||||
for (URL url : contentURLs) {
|
for (URL url : contentURLs) {
|
||||||
try {
|
try {
|
||||||
TumblrRipper ripper = new TumblrRipper(url);
|
TumblrRipper ripper = new TumblrRipper(url);
|
||||||
|
Loading…
Reference in New Issue
Block a user