Fixed the Finebox ripper
This commit is contained in:
parent
1ec8fd869d
commit
8b6c66da0c
17
nbactions.xml
Normal file
17
nbactions.xml
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<actions>
|
||||||
|
<action>
|
||||||
|
<actionName>run</actionName>
|
||||||
|
<packagings>
|
||||||
|
<packaging>jar</packaging>
|
||||||
|
</packagings>
|
||||||
|
<goals>
|
||||||
|
<goal>process-classes</goal>
|
||||||
|
<goal>org.codehaus.mojo:exec-maven-plugin:1.2.1:exec</goal>
|
||||||
|
</goals>
|
||||||
|
<properties>
|
||||||
|
<exec.args>-classpath %classpath com.rarchives.ripme.App</exec.args>
|
||||||
|
<exec.executable>java</exec.executable>
|
||||||
|
</properties>
|
||||||
|
</action>
|
||||||
|
</actions>
|
@ -94,14 +94,14 @@ public abstract class AbstractRipper
|
|||||||
* @param saveAs
|
* @param saveAs
|
||||||
* Path of the local file to save the content to.
|
* Path of the local file to save the content to.
|
||||||
*/
|
*/
|
||||||
public abstract void addURLToDownload(URL url, File saveAs);
|
public abstract boolean addURLToDownload(URL url, File saveAs);
|
||||||
public abstract void addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies);
|
public abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies);
|
||||||
|
|
||||||
public void addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map<String,String> cookies) {
|
public boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map<String,String> cookies) {
|
||||||
try {
|
try {
|
||||||
stopCheck();
|
stopCheck();
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
String saveAs = url.toExternalForm();
|
String saveAs = url.toExternalForm();
|
||||||
saveAs = saveAs.substring(saveAs.lastIndexOf('/')+1);
|
saveAs = saveAs.substring(saveAs.lastIndexOf('/')+1);
|
||||||
@ -122,14 +122,14 @@ public abstract class AbstractRipper
|
|||||||
+ saveAs);
|
+ saveAs);
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
logger.error("[!] Error creating save file path for URL '" + url + "':", e);
|
logger.error("[!] Error creating save file path for URL '" + url + "':", e);
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
logger.debug("Downloading " + url + " to " + saveFileAs);
|
logger.debug("Downloading " + url + " to " + saveFileAs);
|
||||||
if (!saveFileAs.getParentFile().exists()) {
|
if (!saveFileAs.getParentFile().exists()) {
|
||||||
logger.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent()));
|
logger.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent()));
|
||||||
saveFileAs.getParentFile().mkdirs();
|
saveFileAs.getParentFile().mkdirs();
|
||||||
}
|
}
|
||||||
addURLToDownload(url, saveFileAs, referrer, cookies);
|
return addURLToDownload(url, saveFileAs, referrer, cookies);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -141,8 +141,8 @@ public abstract class AbstractRipper
|
|||||||
* @param subdirectory
|
* @param subdirectory
|
||||||
* Sub-directory of the working directory to save the images to.
|
* Sub-directory of the working directory to save the images to.
|
||||||
*/
|
*/
|
||||||
public void addURLToDownload(URL url, String prefix, String subdirectory) {
|
public boolean addURLToDownload(URL url, String prefix, String subdirectory) {
|
||||||
addURLToDownload(url, prefix, subdirectory, null, null);
|
return addURLToDownload(url, prefix, subdirectory, null, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -153,9 +153,9 @@ public abstract class AbstractRipper
|
|||||||
* @param prefix
|
* @param prefix
|
||||||
* Text to append to saved filename.
|
* Text to append to saved filename.
|
||||||
*/
|
*/
|
||||||
public void addURLToDownload(URL url, String prefix) {
|
public boolean addURLToDownload(URL url, String prefix) {
|
||||||
// Use empty subdirectory
|
// Use empty subdirectory
|
||||||
addURLToDownload(url, prefix, "");
|
return addURLToDownload(url, prefix, "");
|
||||||
}
|
}
|
||||||
/**
|
/**
|
||||||
* Waits for downloading threads to complete.
|
* Waits for downloading threads to complete.
|
||||||
|
@ -33,14 +33,14 @@ public abstract class AlbumRipper extends AbstractRipper {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies) {
|
public boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies) {
|
||||||
if (!allowDuplicates()
|
if (!allowDuplicates()
|
||||||
&& ( itemsPending.containsKey(url)
|
&& ( itemsPending.containsKey(url)
|
||||||
|| itemsCompleted.containsKey(url)
|
|| itemsCompleted.containsKey(url)
|
||||||
|| itemsErrored.containsKey(url) )) {
|
|| itemsErrored.containsKey(url) )) {
|
||||||
// Item is already downloaded/downloading, skip it.
|
// Item is already downloaded/downloading, skip it.
|
||||||
logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs));
|
logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs));
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
if (Utils.getConfigBoolean("urls_only.save", false)) {
|
if (Utils.getConfigBoolean("urls_only.save", false)) {
|
||||||
// Output URL to file
|
// Output URL to file
|
||||||
@ -68,11 +68,12 @@ public abstract class AlbumRipper extends AbstractRipper {
|
|||||||
}
|
}
|
||||||
threadPool.addThread(dft);
|
threadPool.addThread(dft);
|
||||||
}
|
}
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void addURLToDownload(URL url, File saveAs) {
|
public boolean addURLToDownload(URL url, File saveAs) {
|
||||||
addURLToDownload(url, saveAs, null, null);
|
return addURLToDownload(url, saveAs, null, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -80,10 +81,12 @@ public abstract class AlbumRipper extends AbstractRipper {
|
|||||||
* Uses filename from URL to decide filename.
|
* Uses filename from URL to decide filename.
|
||||||
* @param url
|
* @param url
|
||||||
* URL to download
|
* URL to download
|
||||||
|
* @return
|
||||||
|
* True on success
|
||||||
*/
|
*/
|
||||||
public void addURLToDownload(URL url) {
|
public boolean addURLToDownload(URL url) {
|
||||||
// Use empty prefix and empty subdirectory
|
// Use empty prefix and empty subdirectory
|
||||||
addURLToDownload(url, "", "");
|
return addURLToDownload(url, "", "");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -146,6 +149,8 @@ public abstract class AlbumRipper extends AbstractRipper {
|
|||||||
* Sets directory to save all ripped files to.
|
* Sets directory to save all ripped files to.
|
||||||
* @param url
|
* @param url
|
||||||
* URL to define how the working directory should be saved.
|
* URL to define how the working directory should be saved.
|
||||||
|
* @throws
|
||||||
|
* IOException
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public void setWorkingDir(URL url) throws IOException {
|
public void setWorkingDir(URL url) throws IOException {
|
||||||
|
@ -13,31 +13,34 @@ import org.jsoup.nodes.Element;
|
|||||||
import com.rarchives.ripme.ripper.AlbumRipper;
|
import com.rarchives.ripme.ripper.AlbumRipper;
|
||||||
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
|
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
|
||||||
import com.rarchives.ripme.utils.Http;
|
import com.rarchives.ripme.utils.Http;
|
||||||
|
import org.jsoup.select.Elements;
|
||||||
|
|
||||||
public class VineboxRipper extends AlbumRipper {
|
public class FineboxRipper extends AlbumRipper {
|
||||||
|
|
||||||
private static final String DOMAIN = "vinebox.co",
|
private static final String DOMAIN = "finebox.co",
|
||||||
HOST = "vinebox";
|
DOMAIN_OLD = "vinebox.co",
|
||||||
|
HOST = "finebox";
|
||||||
|
|
||||||
public VineboxRipper(URL url) throws IOException {
|
public FineboxRipper(URL url) throws IOException {
|
||||||
super(url);
|
super(url);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean canRip(URL url) {
|
public boolean canRip(URL url) {
|
||||||
return url.getHost().endsWith(DOMAIN);
|
return url.getHost().endsWith(DOMAIN) || url.getHost().endsWith(DOMAIN_OLD);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public URL sanitizeURL(URL url) throws MalformedURLException {
|
public URL sanitizeURL(URL url) throws MalformedURLException {
|
||||||
return new URL("http://vinebox.co/u/" + getGID(url));
|
return new URL("http://"+DOMAIN+"/u/" + getGID(url));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void rip() throws IOException {
|
public void rip() throws IOException {
|
||||||
int page = 0;
|
int page = 0;
|
||||||
Document doc;
|
Document doc;
|
||||||
while (true) {
|
Boolean hasPagesLeft = true;
|
||||||
|
while (hasPagesLeft) {
|
||||||
page++;
|
page++;
|
||||||
String urlPaged = this.url.toExternalForm() + "?page=" + page;
|
String urlPaged = this.url.toExternalForm() + "?page=" + page;
|
||||||
logger.info("Retrieving " + urlPaged);
|
logger.info("Retrieving " + urlPaged);
|
||||||
@ -47,10 +50,18 @@ public class VineboxRipper extends AlbumRipper {
|
|||||||
} catch (HttpStatusException e) {
|
} catch (HttpStatusException e) {
|
||||||
logger.debug("Hit end of pages at page " + page, e);
|
logger.debug("Hit end of pages at page " + page, e);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
for (Element element : doc.select("video")) {
|
Elements videos = doc.select("video");
|
||||||
addURLToDownload(new URL(element.attr("src")));
|
for (Element element : videos) {
|
||||||
}
|
String videourl = element.attr("src");
|
||||||
|
if(videourl.substring(0,4)!="http"){
|
||||||
|
videourl = "http://"+DOMAIN+ videourl;
|
||||||
|
}
|
||||||
|
if(!addURLToDownload(new URL(videourl))){
|
||||||
|
hasPagesLeft = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
Thread.sleep(1000);
|
Thread.sleep(1000);
|
||||||
} catch (InterruptedException e) {
|
} catch (InterruptedException e) {
|
||||||
@ -68,10 +79,10 @@ public class VineboxRipper extends AlbumRipper {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getGID(URL url) throws MalformedURLException {
|
public String getGID(URL url) throws MalformedURLException {
|
||||||
Pattern p = Pattern.compile("^https?://(www\\.)?vinebox\\.co/u/([a-zA-Z0-9]{1,}).*$");
|
Pattern p = Pattern.compile("^https?://(www\\.)?(v|f)inebox\\.co/u/([a-zA-Z0-9]{1,}).*$");
|
||||||
Matcher m = p.matcher(url.toExternalForm());
|
Matcher m = p.matcher(url.toExternalForm());
|
||||||
if (!m.matches()) {
|
if (!m.matches()) {
|
||||||
throw new MalformedURLException("Expected format: http://vinebox.co/u/USERNAME");
|
throw new MalformedURLException("Expected format: http://"+DOMAIN+"/u/USERNAME");
|
||||||
}
|
}
|
||||||
return m.group(m.groupCount());
|
return m.group(m.groupCount());
|
||||||
}
|
}
|
@ -5,9 +5,9 @@ import java.net.URL;
|
|||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import com.rarchives.ripme.ripper.rippers.VineboxRipper;
|
import com.rarchives.ripme.ripper.rippers.FineboxRipper;
|
||||||
|
|
||||||
public class VineboxRipperTest extends RippersTest {
|
public class FineboxRipperTest extends RippersTest {
|
||||||
|
|
||||||
public void testVineboxAlbums() throws IOException {
|
public void testVineboxAlbums() throws IOException {
|
||||||
if (DOWNLOAD_CONTENT) {
|
if (DOWNLOAD_CONTENT) {
|
||||||
@ -15,9 +15,10 @@ public class VineboxRipperTest extends RippersTest {
|
|||||||
}
|
}
|
||||||
List<URL> contentURLs = new ArrayList<URL>();
|
List<URL> contentURLs = new ArrayList<URL>();
|
||||||
contentURLs.add(new URL("http://vinebox.co/u/wi57hMjc2Ka"));
|
contentURLs.add(new URL("http://vinebox.co/u/wi57hMjc2Ka"));
|
||||||
|
contentURLs.add(new URL("http://finebox.co/u/wi57hMjc2Ka"));
|
||||||
for (URL url : contentURLs) {
|
for (URL url : contentURLs) {
|
||||||
try {
|
try {
|
||||||
VineboxRipper ripper = new VineboxRipper(url);
|
FineboxRipper ripper = new FineboxRipper(url);
|
||||||
ripper.rip();
|
ripper.rip();
|
||||||
assert(ripper.getWorkingDir().listFiles().length > 1);
|
assert(ripper.getWorkingDir().listFiles().length > 1);
|
||||||
deleteDir(ripper.getWorkingDir());
|
deleteDir(ripper.getWorkingDir());
|
Loading…
Reference in New Issue
Block a user