Fixed the Finebox ripper

This commit is contained in:
Erwin de Haan 2014-10-15 14:02:36 +02:00
parent 1ec8fd869d
commit 8b6c66da0c
5 changed files with 66 additions and 32 deletions

17
nbactions.xml Normal file
View File

@ -0,0 +1,17 @@
<?xml version="1.0" encoding="UTF-8"?>
<actions>
<action>
<actionName>run</actionName>
<packagings>
<packaging>jar</packaging>
</packagings>
<goals>
<goal>process-classes</goal>
<goal>org.codehaus.mojo:exec-maven-plugin:1.2.1:exec</goal>
</goals>
<properties>
<exec.args>-classpath %classpath com.rarchives.ripme.App</exec.args>
<exec.executable>java</exec.executable>
</properties>
</action>
</actions>

View File

@ -94,14 +94,14 @@ public abstract class AbstractRipper
* @param saveAs * @param saveAs
* Path of the local file to save the content to. * Path of the local file to save the content to.
*/ */
public abstract void addURLToDownload(URL url, File saveAs); public abstract boolean addURLToDownload(URL url, File saveAs);
public abstract void addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies); public abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies);
public void addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map<String,String> cookies) { public boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map<String,String> cookies) {
try { try {
stopCheck(); stopCheck();
} catch (IOException e) { } catch (IOException e) {
return; return false;
} }
String saveAs = url.toExternalForm(); String saveAs = url.toExternalForm();
saveAs = saveAs.substring(saveAs.lastIndexOf('/')+1); saveAs = saveAs.substring(saveAs.lastIndexOf('/')+1);
@ -122,14 +122,14 @@ public abstract class AbstractRipper
+ saveAs); + saveAs);
} catch (IOException e) { } catch (IOException e) {
logger.error("[!] Error creating save file path for URL '" + url + "':", e); logger.error("[!] Error creating save file path for URL '" + url + "':", e);
return; return false;
} }
logger.debug("Downloading " + url + " to " + saveFileAs); logger.debug("Downloading " + url + " to " + saveFileAs);
if (!saveFileAs.getParentFile().exists()) { if (!saveFileAs.getParentFile().exists()) {
logger.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent())); logger.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent()));
saveFileAs.getParentFile().mkdirs(); saveFileAs.getParentFile().mkdirs();
} }
addURLToDownload(url, saveFileAs, referrer, cookies); return addURLToDownload(url, saveFileAs, referrer, cookies);
} }
/** /**
@ -141,8 +141,8 @@ public abstract class AbstractRipper
* @param subdirectory * @param subdirectory
* Sub-directory of the working directory to save the images to. * Sub-directory of the working directory to save the images to.
*/ */
public void addURLToDownload(URL url, String prefix, String subdirectory) { public boolean addURLToDownload(URL url, String prefix, String subdirectory) {
addURLToDownload(url, prefix, subdirectory, null, null); return addURLToDownload(url, prefix, subdirectory, null, null);
} }
/** /**
@ -153,9 +153,9 @@ public abstract class AbstractRipper
* @param prefix * @param prefix
* Text to append to saved filename. * Text to append to saved filename.
*/ */
public void addURLToDownload(URL url, String prefix) { public boolean addURLToDownload(URL url, String prefix) {
// Use empty subdirectory // Use empty subdirectory
addURLToDownload(url, prefix, ""); return addURLToDownload(url, prefix, "");
} }
/** /**
* Waits for downloading threads to complete. * Waits for downloading threads to complete.

View File

@ -33,14 +33,14 @@ public abstract class AlbumRipper extends AbstractRipper {
return false; return false;
} }
public void addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies) { public boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies) {
if (!allowDuplicates() if (!allowDuplicates()
&& ( itemsPending.containsKey(url) && ( itemsPending.containsKey(url)
|| itemsCompleted.containsKey(url) || itemsCompleted.containsKey(url)
|| itemsErrored.containsKey(url) )) { || itemsErrored.containsKey(url) )) {
// Item is already downloaded/downloading, skip it. // Item is already downloaded/downloading, skip it.
logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs));
return; return false;
} }
if (Utils.getConfigBoolean("urls_only.save", false)) { if (Utils.getConfigBoolean("urls_only.save", false)) {
// Output URL to file // Output URL to file
@ -68,11 +68,12 @@ public abstract class AlbumRipper extends AbstractRipper {
} }
threadPool.addThread(dft); threadPool.addThread(dft);
} }
return true;
} }
@Override @Override
public void addURLToDownload(URL url, File saveAs) { public boolean addURLToDownload(URL url, File saveAs) {
addURLToDownload(url, saveAs, null, null); return addURLToDownload(url, saveAs, null, null);
} }
/** /**
@ -80,10 +81,12 @@ public abstract class AlbumRipper extends AbstractRipper {
* Uses filename from URL to decide filename. * Uses filename from URL to decide filename.
* @param url * @param url
* URL to download * URL to download
* @return
* True on success
*/ */
public void addURLToDownload(URL url) { public boolean addURLToDownload(URL url) {
// Use empty prefix and empty subdirectory // Use empty prefix and empty subdirectory
addURLToDownload(url, "", ""); return addURLToDownload(url, "", "");
} }
@Override @Override
@ -146,6 +149,8 @@ public abstract class AlbumRipper extends AbstractRipper {
* Sets directory to save all ripped files to. * Sets directory to save all ripped files to.
* @param url * @param url
* URL to define how the working directory should be saved. * URL to define how the working directory should be saved.
* @throws
* IOException
*/ */
@Override @Override
public void setWorkingDir(URL url) throws IOException { public void setWorkingDir(URL url) throws IOException {

View File

@ -13,31 +13,34 @@ import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AlbumRipper; import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Http;
import org.jsoup.select.Elements;
public class VineboxRipper extends AlbumRipper { public class FineboxRipper extends AlbumRipper {
private static final String DOMAIN = "vinebox.co", private static final String DOMAIN = "finebox.co",
HOST = "vinebox"; DOMAIN_OLD = "vinebox.co",
HOST = "finebox";
public VineboxRipper(URL url) throws IOException { public FineboxRipper(URL url) throws IOException {
super(url); super(url);
} }
@Override @Override
public boolean canRip(URL url) { public boolean canRip(URL url) {
return url.getHost().endsWith(DOMAIN); return url.getHost().endsWith(DOMAIN) || url.getHost().endsWith(DOMAIN_OLD);
} }
@Override @Override
public URL sanitizeURL(URL url) throws MalformedURLException { public URL sanitizeURL(URL url) throws MalformedURLException {
return new URL("http://vinebox.co/u/" + getGID(url)); return new URL("http://"+DOMAIN+"/u/" + getGID(url));
} }
@Override @Override
public void rip() throws IOException { public void rip() throws IOException {
int page = 0; int page = 0;
Document doc; Document doc;
while (true) { Boolean hasPagesLeft = true;
while (hasPagesLeft) {
page++; page++;
String urlPaged = this.url.toExternalForm() + "?page=" + page; String urlPaged = this.url.toExternalForm() + "?page=" + page;
logger.info("Retrieving " + urlPaged); logger.info("Retrieving " + urlPaged);
@ -48,8 +51,16 @@ public class VineboxRipper extends AlbumRipper {
logger.debug("Hit end of pages at page " + page, e); logger.debug("Hit end of pages at page " + page, e);
break; break;
} }
for (Element element : doc.select("video")) { Elements videos = doc.select("video");
addURLToDownload(new URL(element.attr("src"))); for (Element element : videos) {
String videourl = element.attr("src");
if(videourl.substring(0,4)!="http"){
videourl = "http://"+DOMAIN+ videourl;
}
if(!addURLToDownload(new URL(videourl))){
hasPagesLeft = false;
break;
}
} }
try { try {
Thread.sleep(1000); Thread.sleep(1000);
@ -68,10 +79,10 @@ public class VineboxRipper extends AlbumRipper {
@Override @Override
public String getGID(URL url) throws MalformedURLException { public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^https?://(www\\.)?vinebox\\.co/u/([a-zA-Z0-9]{1,}).*$"); Pattern p = Pattern.compile("^https?://(www\\.)?(v|f)inebox\\.co/u/([a-zA-Z0-9]{1,}).*$");
Matcher m = p.matcher(url.toExternalForm()); Matcher m = p.matcher(url.toExternalForm());
if (!m.matches()) { if (!m.matches()) {
throw new MalformedURLException("Expected format: http://vinebox.co/u/USERNAME"); throw new MalformedURLException("Expected format: http://"+DOMAIN+"/u/USERNAME");
} }
return m.group(m.groupCount()); return m.group(m.groupCount());
} }

View File

@ -5,9 +5,9 @@ import java.net.URL;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import com.rarchives.ripme.ripper.rippers.VineboxRipper; import com.rarchives.ripme.ripper.rippers.FineboxRipper;
public class VineboxRipperTest extends RippersTest { public class FineboxRipperTest extends RippersTest {
public void testVineboxAlbums() throws IOException { public void testVineboxAlbums() throws IOException {
if (DOWNLOAD_CONTENT) { if (DOWNLOAD_CONTENT) {
@ -15,9 +15,10 @@ public class VineboxRipperTest extends RippersTest {
} }
List<URL> contentURLs = new ArrayList<URL>(); List<URL> contentURLs = new ArrayList<URL>();
contentURLs.add(new URL("http://vinebox.co/u/wi57hMjc2Ka")); contentURLs.add(new URL("http://vinebox.co/u/wi57hMjc2Ka"));
contentURLs.add(new URL("http://finebox.co/u/wi57hMjc2Ka"));
for (URL url : contentURLs) { for (URL url : contentURLs) {
try { try {
VineboxRipper ripper = new VineboxRipper(url); FineboxRipper ripper = new FineboxRipper(url);
ripper.rip(); ripper.rip();
assert(ripper.getWorkingDir().listFiles().length > 1); assert(ripper.getWorkingDir().listFiles().length > 1);
deleteDir(ripper.getWorkingDir()); deleteDir(ripper.getWorkingDir());