Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

1.0.70 Reddit ripper retrieves gfycat and vidble links

Browse Source 
As requested in #8
This commit is contained in:
4pr0n 2014-06-25 02:03:47 -07:00
parent 86ae583da6
commit 349804c968
5 changed files with 74 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@ -4,7 +4,7 @@
<groupId>com.rarchives.ripme</groupId>
<artifactId>ripme</artifactId>
<packaging>jar</packaging>
<version>1.0.69</version>
<version>1.0.70</version>
<name>ripme</name>
<url>http://rip.rarchives.com</url>
<properties>

69
src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java
View File

@ -3,23 +3,18 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils;
public class VidbleRipper extends AlbumRipper {
private static final String DOMAIN = "vidble.com",
HOST = "vidble";
private Document albumDoc = null;
public class VidbleRipper extends AbstractHTMLRipper {
public VidbleRipper(URL url) throws IOException {
super(url);
@ -27,7 +22,11 @@ public class VidbleRipper extends AlbumRipper {
@Override
public String getHost() {
return HOST;
return "vidble";
}
@Override
public String getDomain() {
return "vidble.com";
}
@Override
@ -46,44 +45,40 @@ public class VidbleRipper extends AlbumRipper {
}
@Override
public void rip() throws IOException {
logger.info("Retrieving " + this.url);
sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm());
if (albumDoc == null) {
albumDoc = Http.url(this.url).get();
public Document getFirstPage() throws IOException {
return Http.url(url).get();
}
Elements els = albumDoc.select("#ContentPlaceHolder1_thumbs");
if (els.size() == 0) {
throw new IOException("No thumbnails found at " + this.url);
@Override
public List<String> getURLsFromPage(Document doc) {
return getURLsFromPageStatic(doc);
}
int index = 0;
String thumbs = els.get(0).attr("value");
private static List<String> getURLsFromPageStatic(Document doc) {
List<String> imageURLs = new ArrayList<String>();
Elements els = doc.select("#ContentPlaceHolder1_thumbs");
String thumbs = els.first().attr("value");
for (String thumb : thumbs.split(",")) {
if (thumb.trim().equals("")) {
continue;
}
thumb = thumb.replaceAll("_[a-zA-Z]{3,5}", "");
String image = "http://vidble.com/" + thumb;
index += 1;
String prefix = "";
if (Utils.getConfigBoolean("download.save_order", true)) {
prefix = String.format("%03d_", index);
imageURLs.add("http://vidble.com/" + thumb);
}
addURLToDownload(new URL(image), prefix);
}
waitForThreads();
}
public boolean canRip(URL url) {
if (!url.getHost().endsWith(DOMAIN)) {
return false;
}
return true;
return imageURLs;
}
@Override
public URL sanitizeURL(URL url) throws MalformedURLException {
return url;
public void downloadURL(URL url, int index) {
addURLToDownload(url, getPrefix(index));
}
public static List<URL> getURLsFromPage(URL url) throws IOException {
List<URL> urls = new ArrayList<URL>();
Document doc = Http.url(url).get();
for (String stringURL : getURLsFromPageStatic(doc)) {
urls.add(new URL(stringURL));
}
return urls;
}
}

19
src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java
View File

@ -51,8 +51,20 @@ public class GfycatRipper extends VideoRipper {
@Override
public void rip() throws IOException {
logger.info(" Retrieving " + this.url.toExternalForm());
Document doc = Http.url(this.url).get();
String vidUrl = getVideoURL(this.url);
addURLToDownload(new URL(vidUrl), "gfycat_" + getGID(this.url));
waitForThreads();
}
/**
* Helper method for retrieving video URLs.
* @param url URL to gfycat page
* @return URL to video
* @throws IOException
*/
public static String getVideoURL(URL url) throws IOException {
logger.info("Retrieving " + url.toExternalForm());
Document doc = Http.url(url).get();
Elements videos = doc.select("source#mp4source");
if (videos.size() == 0) {
throw new IOException("Could not find source#mp4source at " + url);
@ -61,7 +73,6 @@ public class GfycatRipper extends VideoRipper {
if (vidUrl.startsWith("//")) {
vidUrl = "http:" + vidUrl;
}
addURLToDownload(new URL(vidUrl), "gfycat_" + getGID(this.url));
waitForThreads();
return vidUrl;
}
}

2
src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
View File

@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils;
public class UpdateUtils {
private static final Logger logger = Logger.getLogger(UpdateUtils.class);
private static final String DEFAULT_VERSION = "1.0.69";
private static final String DEFAULT_VERSION = "1.0.70";
private static final String updateJsonURL = "http://rarchives.com/ripme.json";
private static final String updateJarURL = "http://rarchives.com/ripme.jar";
private static final String mainFileName = "ripme.jar";

25
src/main/java/com/rarchives/ripme/utils/RipUtils.java
View File

@ -15,8 +15,10 @@ import org.jsoup.nodes.Element;
import com.rarchives.ripme.ripper.AbstractRipper;
import com.rarchives.ripme.ripper.rippers.ImgurRipper;
import com.rarchives.ripme.ripper.rippers.VidbleRipper;
import com.rarchives.ripme.ripper.rippers.ImgurRipper.ImgurAlbum;
import com.rarchives.ripme.ripper.rippers.ImgurRipper.ImgurImage;
import com.rarchives.ripme.ripper.rippers.video.GfycatRipper;
public class RipUtils {
private static final Logger logger = Logger.getLogger(RipUtils.class);
@ -25,19 +27,36 @@ public class RipUtils {
List<URL> result = new ArrayList<URL>();
// Imgur album
if ((url.getHost().equals("m.imgur.com") || url.getHost().equals("imgur.com"))
if ((url.getHost().endsWith("imgur.com"))
&& url.toExternalForm().contains("imgur.com/a/")) {
try {
ImgurAlbum imgurAlbum = ImgurRipper.getImgurAlbum(url);
for (ImgurImage imgurImage : imgurAlbum.images) {
result.add(imgurImage.url);
}
return result;
} catch (IOException e) {
logger.error("[!] Exception while loading album " + url, e);
}
return result;
}
else if (url.getHost().endsWith("gfycat.com")) {
try {
String videoURL = GfycatRipper.getVideoURL(url);
result.add(new URL(videoURL));
} catch (IOException e) {
// Do nothing
logger.warn("Exception while retrieving gfycat page:", e);
}
return result;
}
else if (url.toExternalForm().contains("vidble.com/album/")) {
try {
result.addAll(VidbleRipper.getURLsFromPage(url));
} catch (IOException e) {
// Do nothing
logger.warn("Exception while retrieving vidble page:", e);
}
return result;
}
// Direct link to image