Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fixed the Finebox ripper

Browse Source
This commit is contained in:
Erwin de Haan 2014-10-15 14:02:36 +02:00
parent 1ec8fd869d
commit 8b6c66da0c
5 changed files with 66 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
nbactions.xml Normal file
View File

@ -0,0 +1,17 @@
<?xml version="1.0" encoding="UTF-8"?>
<actions>
<action>
<actionName>run</actionName>
<packagings>
<packaging>jar</packaging>
</packagings>
<goals>
<goal>process-classes</goal>
<goal>org.codehaus.mojo:exec-maven-plugin:1.2.1:exec</goal>
</goals>
<properties>
<exec.args>-classpath %classpath com.rarchives.ripme.App</exec.args>
<exec.executable>java</exec.executable>
</properties>
</action>
</actions>

20
src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
View File

@ -94,14 +94,14 @@ public abstract class AbstractRipper
* @param saveAs
* Path of the local file to save the content to.
*/
public abstract void addURLToDownload(URL url, File saveAs);
public abstract void addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies);
public abstract boolean addURLToDownload(URL url, File saveAs);
public abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies);
public void addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map<String,String> cookies) {
public boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map<String,String> cookies) {
try {
stopCheck();
} catch (IOException e) {
return;
return false;
}
String saveAs = url.toExternalForm();
saveAs = saveAs.substring(saveAs.lastIndexOf('/')+1);
@ -122,14 +122,14 @@ public abstract class AbstractRipper
+ saveAs);
} catch (IOException e) {
logger.error("[!] Error creating save file path for URL '" + url + "':", e);
return;
return false;
}
logger.debug("Downloading " + url + " to " + saveFileAs);
if (!saveFileAs.getParentFile().exists()) {
logger.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent()));
saveFileAs.getParentFile().mkdirs();
}
addURLToDownload(url, saveFileAs, referrer, cookies);
return addURLToDownload(url, saveFileAs, referrer, cookies);
}
/**
@ -141,8 +141,8 @@ public abstract class AbstractRipper
* @param subdirectory
* Sub-directory of the working directory to save the images to.
*/
public void addURLToDownload(URL url, String prefix, String subdirectory) {
addURLToDownload(url, prefix, subdirectory, null, null);
public boolean addURLToDownload(URL url, String prefix, String subdirectory) {
return addURLToDownload(url, prefix, subdirectory, null, null);
}
/**
@ -153,9 +153,9 @@ public abstract class AbstractRipper
* @param prefix
* Text to append to saved filename.
*/
public void addURLToDownload(URL url, String prefix) {
public boolean addURLToDownload(URL url, String prefix) {
// Use empty subdirectory
addURLToDownload(url, prefix, "");
return addURLToDownload(url, prefix, "");
}
/**
* Waits for downloading threads to complete.

17
src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java
View File

@ -33,14 +33,14 @@ public abstract class AlbumRipper extends AbstractRipper {
return false;
}
public void addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies) {
public boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies) {
if (!allowDuplicates()
&& ( itemsPending.containsKey(url)
|| itemsCompleted.containsKey(url)
|| itemsErrored.containsKey(url) )) {
// Item is already downloaded/downloading, skip it.
logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs));
return;
return false;
}
if (Utils.getConfigBoolean("urls_only.save", false)) {
// Output URL to file
@ -68,11 +68,12 @@ public abstract class AlbumRipper extends AbstractRipper {
}
threadPool.addThread(dft);
}
return true;
}
@Override
public void addURLToDownload(URL url, File saveAs) {
addURLToDownload(url, saveAs, null, null);
public boolean addURLToDownload(URL url, File saveAs) {
return addURLToDownload(url, saveAs, null, null);
}
/**
@ -80,10 +81,12 @@ public abstract class AlbumRipper extends AbstractRipper {
* Uses filename from URL to decide filename.
* @param url
* URL to download
* @return
* True on success
*/
public void addURLToDownload(URL url) {
public boolean addURLToDownload(URL url) {
// Use empty prefix and empty subdirectory
addURLToDownload(url, "", "");
return addURLToDownload(url, "", "");
}
@Override
@ -146,6 +149,8 @@ public abstract class AlbumRipper extends AbstractRipper {
* Sets directory to save all ripped files to.
* @param url
* URL to define how the working directory should be saved.
* @throws
* IOException
*/
@Override
public void setWorkingDir(URL url) throws IOException {

37
src/main/java/com/rarchives/ripme/ripper/rippers/VineboxRipper.java → src/main/java/com/rarchives/ripme/ripper/rippers/FineboxRipper.java
View File

@ -13,31 +13,34 @@ import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http;
import org.jsoup.select.Elements;
public class VineboxRipper extends AlbumRipper {
public class FineboxRipper extends AlbumRipper {
private static final String DOMAIN = "vinebox.co",
HOST = "vinebox";
private static final String DOMAIN = "finebox.co",
DOMAIN_OLD = "vinebox.co",
HOST = "finebox";
public VineboxRipper(URL url) throws IOException {
public FineboxRipper(URL url) throws IOException {
super(url);
}
@Override
public boolean canRip(URL url) {
return url.getHost().endsWith(DOMAIN);
return url.getHost().endsWith(DOMAIN) || url.getHost().endsWith(DOMAIN_OLD);
}
@Override
public URL sanitizeURL(URL url) throws MalformedURLException {
return new URL("http://vinebox.co/u/" + getGID(url));
return new URL("http://"+DOMAIN+"/u/" + getGID(url));
}
@Override
public void rip() throws IOException {
int page = 0;
Document doc;
while (true) {
Boolean hasPagesLeft = true;
while (hasPagesLeft) {
page++;
String urlPaged = this.url.toExternalForm() + "?page=" + page;
logger.info("Retrieving " + urlPaged);
@ -47,10 +50,18 @@ public class VineboxRipper extends AlbumRipper {
} catch (HttpStatusException e) {
logger.debug("Hit end of pages at page " + page, e);
break;
}
for (Element element : doc.select("video")) {
addURLToDownload(new URL(element.attr("src")));
}
}
Elements videos = doc.select("video");
for (Element element : videos) {
String videourl = element.attr("src");
if(videourl.substring(0,4)!="http"){
videourl = "http://"+DOMAIN+ videourl;
}
if(!addURLToDownload(new URL(videourl))){
hasPagesLeft = false;
break;
}
}
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
@ -68,10 +79,10 @@ public class VineboxRipper extends AlbumRipper {
@Override
public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^https?://(www\\.)?vinebox\\.co/u/([a-zA-Z0-9]{1,}).*$");
Pattern p = Pattern.compile("^https?://(www\\.)?(v|f)inebox\\.co/u/([a-zA-Z0-9]{1,}).*$");
Matcher m = p.matcher(url.toExternalForm());
if (!m.matches()) {
throw new MalformedURLException("Expected format: http://vinebox.co/u/USERNAME");
throw new MalformedURLException("Expected format: http://"+DOMAIN+"/u/USERNAME");
}
return m.group(m.groupCount());
}

7
src/test/java/com/rarchives/ripme/tst/ripper/rippers/VineboxRipperTest.java → src/test/java/com/rarchives/ripme/tst/ripper/rippers/FineboxRipperTest.java
View File

@ -5,9 +5,9 @@ import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import com.rarchives.ripme.ripper.rippers.VineboxRipper;
import com.rarchives.ripme.ripper.rippers.FineboxRipper;
public class VineboxRipperTest extends RippersTest {
public class FineboxRipperTest extends RippersTest {
public void testVineboxAlbums() throws IOException {
if (DOWNLOAD_CONTENT) {
@ -15,9 +15,10 @@ public class VineboxRipperTest extends RippersTest {
}
List<URL> contentURLs = new ArrayList<URL>();
contentURLs.add(new URL("http://vinebox.co/u/wi57hMjc2Ka"));
contentURLs.add(new URL("http://finebox.co/u/wi57hMjc2Ka"));
for (URL url : contentURLs) {
try {
VineboxRipper ripper = new VineboxRipper(url);
FineboxRipper ripper = new FineboxRipper(url);
ripper.rip();
assert(ripper.getWorkingDir().listFiles().length > 1);
deleteDir(ripper.getWorkingDir());