From 349804c968ff3edda203bb7bf7bda5fb5ff0bf41 Mon Sep 17 00:00:00 2001 From: 4pr0n Date: Wed, 25 Jun 2014 02:03:47 -0700 Subject: [PATCH] 1.0.70 Reddit ripper retrieves gfycat and vidble links As requested in #8 --- pom.xml | 2 +- .../ripme/ripper/rippers/VidbleRipper.java | 73 +++++++++---------- .../ripper/rippers/video/GfycatRipper.java | 19 ++++- .../com/rarchives/ripme/ui/UpdateUtils.java | 2 +- .../com/rarchives/ripme/utils/RipUtils.java | 27 ++++++- 5 files changed, 74 insertions(+), 49 deletions(-) diff --git a/pom.xml b/pom.xml index 24c288fb..7aa6d0c8 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.0.69 + 1.0.70 ripme http://rip.rarchives.com diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java index b6402d7b..8cd8b957 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java @@ -3,23 +3,18 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.util.ArrayList; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.jsoup.nodes.Document; import org.jsoup.select.Elements; -import com.rarchives.ripme.ripper.AlbumRipper; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; +import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; -import com.rarchives.ripme.utils.Utils; -public class VidbleRipper extends AlbumRipper { - - private static final String DOMAIN = "vidble.com", - HOST = "vidble"; - - private Document albumDoc = null; +public class VidbleRipper extends AbstractHTMLRipper { public VidbleRipper(URL url) throws IOException { super(url); @@ -27,7 +22,11 @@ public class VidbleRipper extends AlbumRipper { @Override public String getHost() { - return HOST; + return "vidble"; + } + @Override + public String getDomain() { + return "vidble.com"; } @Override @@ -46,44 +45,40 @@ public class VidbleRipper extends AlbumRipper { } @Override - public void rip() throws IOException { - logger.info("Retrieving " + this.url); - sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm()); - if (albumDoc == null) { - albumDoc = Http.url(this.url).get(); - } - Elements els = albumDoc.select("#ContentPlaceHolder1_thumbs"); - if (els.size() == 0) { - throw new IOException("No thumbnails found at " + this.url); - } - int index = 0; - String thumbs = els.get(0).attr("value"); + public Document getFirstPage() throws IOException { + return Http.url(url).get(); + } + + @Override + public List getURLsFromPage(Document doc) { + return getURLsFromPageStatic(doc); + } + + private static List getURLsFromPageStatic(Document doc) { + List imageURLs = new ArrayList(); + Elements els = doc.select("#ContentPlaceHolder1_thumbs"); + String thumbs = els.first().attr("value"); for (String thumb : thumbs.split(",")) { if (thumb.trim().equals("")) { continue; } thumb = thumb.replaceAll("_[a-zA-Z]{3,5}", ""); - String image = "http://vidble.com/" + thumb; - index += 1; - String prefix = ""; - if (Utils.getConfigBoolean("download.save_order", true)) { - prefix = String.format("%03d_", index); - } - addURLToDownload(new URL(image), prefix); + imageURLs.add("http://vidble.com/" + thumb); } - waitForThreads(); - } - - public boolean canRip(URL url) { - if (!url.getHost().endsWith(DOMAIN)) { - return false; - } - return true; + return imageURLs; } @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return url; + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); } + public static List getURLsFromPage(URL url) throws IOException { + List urls = new ArrayList(); + Document doc = Http.url(url).get(); + for (String stringURL : getURLsFromPageStatic(doc)) { + urls.add(new URL(stringURL)); + } + return urls; + } } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java index f30ff832..1047ff01 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java @@ -51,8 +51,20 @@ public class GfycatRipper extends VideoRipper { @Override public void rip() throws IOException { - logger.info(" Retrieving " + this.url.toExternalForm()); - Document doc = Http.url(this.url).get(); + String vidUrl = getVideoURL(this.url); + addURLToDownload(new URL(vidUrl), "gfycat_" + getGID(this.url)); + waitForThreads(); + } + + /** + * Helper method for retrieving video URLs. + * @param url URL to gfycat page + * @return URL to video + * @throws IOException + */ + public static String getVideoURL(URL url) throws IOException { + logger.info("Retrieving " + url.toExternalForm()); + Document doc = Http.url(url).get(); Elements videos = doc.select("source#mp4source"); if (videos.size() == 0) { throw new IOException("Could not find source#mp4source at " + url); @@ -61,7 +73,6 @@ public class GfycatRipper extends VideoRipper { if (vidUrl.startsWith("//")) { vidUrl = "http:" + vidUrl; } - addURLToDownload(new URL(vidUrl), "gfycat_" + getGID(this.url)); - waitForThreads(); + return vidUrl; } } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 6327e870..a308a1dc 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.0.69"; + private static final String DEFAULT_VERSION = "1.0.70"; private static final String updateJsonURL = "http://rarchives.com/ripme.json"; private static final String updateJarURL = "http://rarchives.com/ripme.jar"; private static final String mainFileName = "ripme.jar"; diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index a06ad328..57bcd9fa 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -15,8 +15,10 @@ import org.jsoup.nodes.Element; import com.rarchives.ripme.ripper.AbstractRipper; import com.rarchives.ripme.ripper.rippers.ImgurRipper; +import com.rarchives.ripme.ripper.rippers.VidbleRipper; import com.rarchives.ripme.ripper.rippers.ImgurRipper.ImgurAlbum; import com.rarchives.ripme.ripper.rippers.ImgurRipper.ImgurImage; +import com.rarchives.ripme.ripper.rippers.video.GfycatRipper; public class RipUtils { private static final Logger logger = Logger.getLogger(RipUtils.class); @@ -25,19 +27,36 @@ public class RipUtils { List result = new ArrayList(); // Imgur album - if ((url.getHost().equals("m.imgur.com") || url.getHost().equals("imgur.com")) + if ((url.getHost().endsWith("imgur.com")) && url.toExternalForm().contains("imgur.com/a/")) { try { ImgurAlbum imgurAlbum = ImgurRipper.getImgurAlbum(url); for (ImgurImage imgurImage : imgurAlbum.images) { result.add(imgurImage.url); } - return result; } catch (IOException e) { logger.error("[!] Exception while loading album " + url, e); - return result; } - + return result; + } + else if (url.getHost().endsWith("gfycat.com")) { + try { + String videoURL = GfycatRipper.getVideoURL(url); + result.add(new URL(videoURL)); + } catch (IOException e) { + // Do nothing + logger.warn("Exception while retrieving gfycat page:", e); + } + return result; + } + else if (url.toExternalForm().contains("vidble.com/album/")) { + try { + result.addAll(VidbleRipper.getURLsFromPage(url)); + } catch (IOException e) { + // Do nothing + logger.warn("Exception while retrieving vidble page:", e); + } + return result; } // Direct link to image