various bug fixes

This commit is contained in:
4pr0n 2014-03-11 01:29:46 -07:00
parent 2bfa377d5d
commit aa78f60ce0
3 changed files with 27 additions and 13 deletions

View File

@ -99,7 +99,7 @@ public abstract class AbstractRipper
|| itemsCompleted.containsKey(url) || itemsCompleted.containsKey(url)
|| itemsErrored.containsKey(url)) { || itemsErrored.containsKey(url)) {
// Item is already downloaded/downloading, skip it. // Item is already downloaded/downloading, skip it.
logger.info("Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); logger.info(" Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs));
return; return;
} }
itemsPending.put(url, saveAs); itemsPending.put(url, saveAs);
@ -232,6 +232,9 @@ public abstract class AbstractRipper
* Notifies observers and updates state if all files have been ripped. * Notifies observers and updates state if all files have been ripped.
*/ */
private void checkIfComplete() { private void checkIfComplete() {
if (observer == null) {
return;
}
synchronized (observer) { synchronized (observer) {
if (!completed && itemsPending.size() == 0) { if (!completed && itemsPending.size() == 0) {
completed = true; completed = true;
@ -359,6 +362,7 @@ public abstract class AbstractRipper
rip(); rip();
} catch (IOException e) { } catch (IOException e) {
logger.error("Got exception while running ripper:", e); logger.error("Got exception while running ripper:", e);
waitForThreads();
} }
} }

View File

@ -3,6 +3,8 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@ -93,8 +95,18 @@ public class ImgurRipper extends AbstractRipper {
private void ripAlbum(URL url, String subdirectory) throws IOException { private void ripAlbum(URL url, String subdirectory) throws IOException {
int index = 0; int index = 0;
logger.info(" Retrieving " + url.toExternalForm());
this.sendUpdate(STATUS.LOADING_RESOURCE, url.toExternalForm()); this.sendUpdate(STATUS.LOADING_RESOURCE, url.toExternalForm());
index = 0;
for (URL singleURL : getURLsFromAlbum(url)) {
index += 1;
processURL(singleURL, String.format("%03d_", index), subdirectory);
}
}
public static List<URL> getURLsFromAlbum(URL url) throws IOException {
List<URL> result = new ArrayList<URL>();
logger.info(" Retrieving " + url.toExternalForm());
Document doc = Jsoup.connect(url.toExternalForm()) Document doc = Jsoup.connect(url.toExternalForm())
.userAgent(USER_AGENT) .userAgent(USER_AGENT)
.get(); .get();
@ -114,10 +126,9 @@ public class ImgurRipper extends AbstractRipper {
"http://i.imgur.com/" "http://i.imgur.com/"
+ image.get("hash") + image.get("hash")
+ image.get("ext")); + image.get("ext"));
index += 1; result.add(imageURL);
processURL(imageURL, String.format("%03d_", index), subdirectory);
} }
return; return result;
} catch (JSONException e) { } catch (JSONException e) {
logger.debug("Error while parsing JSON at " + url + ", continuing", e); logger.debug("Error while parsing JSON at " + url + ", continuing", e);
} }
@ -136,10 +147,9 @@ public class ImgurRipper extends AbstractRipper {
+ "/" + "/"
+ image.get("hash") + image.get("hash")
+ image.get("ext")); + image.get("ext"));
index += 1; result.add(imageURL);
processURL(imageURL, String.format("%03d_", index), subdirectory);
} }
return; return result;
} catch (JSONException e) { } catch (JSONException e) {
logger.debug("Error while parsing JSON at " + url + ", continuing", e); logger.debug("Error while parsing JSON at " + url + ", continuing", e);
} }
@ -164,9 +174,9 @@ public class ImgurRipper extends AbstractRipper {
logger.error("[!] Unable to find image in div: " + thumb.toString()); logger.error("[!] Unable to find image in div: " + thumb.toString());
continue; continue;
} }
index += 1; result.add(new URL(image));
processURL(new URL(image), String.format("%03d_", index), subdirectory);
} }
return result;
} }
/** /**

View File

@ -10,13 +10,13 @@ import com.rarchives.ripme.ripper.rippers.TumblrRipper;
public class TumblrRipperTest extends RippersTest { public class TumblrRipperTest extends RippersTest {
public void testTumblrAlbums() throws IOException { public void testTumblrAlbums() throws IOException {
if (false && !DOWNLOAD_CONTENT) { if (!DOWNLOAD_CONTENT) {
return; return;
} }
List<URL> contentURLs = new ArrayList<URL>(); List<URL> contentURLs = new ArrayList<URL>();
contentURLs.add(new URL("http://wrouinr.tumblr.com/archive")); contentURLs.add(new URL("http://wrouinr.tumblr.com/archive"));
//contentURLs.add(new URL("http://topinstagirls.tumblr.com/tagged/berlinskaya")); contentURLs.add(new URL("http://topinstagirls.tumblr.com/tagged/berlinskaya"));
//contentURLs.add(new URL("http://fittingroomgirls.tumblr.com/post/78268776776")); contentURLs.add(new URL("http://fittingroomgirls.tumblr.com/post/78268776776"));
for (URL url : contentURLs) { for (URL url : contentURLs) {
try { try {
TumblrRipper ripper = new TumblrRipper(url); TumblrRipper ripper = new TumblrRipper(url);