From e5906db588b4650288a7b548c03fd095f887c1d0 Mon Sep 17 00:00:00 2001 From: 4pr0n Date: Wed, 23 Apr 2014 23:12:44 -0700 Subject: [PATCH] 1.0.28 - Added Vk video (single and albums) rippers #8 --- pom.xml | 2 +- .../ripme/ripper/rippers/VkRipper.java | 57 ++++++++++++- .../ripme/ripper/rippers/video/VkRipper.java | 83 +++++++++++++++++++ .../com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 4 files changed, 140 insertions(+), 4 deletions(-) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java diff --git a/pom.xml b/pom.xml index 3c563de6..1142b85e 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.0.27 + 1.0.28 ripme http://rip.rarchives.com diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java index a3654e9f..9af65715 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java @@ -32,7 +32,16 @@ public class VkRipper extends AlbumRipper { @Override public boolean canRip(URL url) { - return url.getHost().endsWith(DOMAIN); + if (!url.getHost().endsWith(DOMAIN)) { + return false; + } + // Ignore /video pages (but not /videos pages) + String u = url.toExternalForm(); + if (u.contains("/video") && !u.contains("videos")) { + // Single video page + return false; + } + return true; } @Override @@ -42,6 +51,50 @@ public class VkRipper extends AlbumRipper { @Override public void rip() throws IOException { + if (this.url.toExternalForm().contains("/videos")) { + ripVideos(); + } + else { + ripImages(); + } + } + + private void ripVideos() throws IOException { + String oid = getGID(this.url).replace("videos", ""); + String u = "http://vk.com/al_video.php"; + Map postData = new HashMap(); + postData.put("al", "1"); + postData.put("act", "load_videos_silent"); + postData.put("offset", "0"); + postData.put("oid", oid); + Document doc = Jsoup.connect(u) + .header("Referer", this.url.toExternalForm()) + .ignoreContentType(true) + .userAgent(USER_AGENT) + .timeout(5000) + .data(postData) + .post(); + String[] jsonStrings = doc.toString().split(""); + JSONObject json = new JSONObject(jsonStrings[jsonStrings.length - 1]); + JSONArray videos = json.getJSONArray("all"); + logger.info("Found " + videos.length() + " videos"); + for (int i = 0; i < videos.length(); i++) { + JSONArray jsonVideo = videos.getJSONArray(i); + int vidid = jsonVideo.getInt(1); + String videoURL = com.rarchives.ripme.ripper.rippers.video.VkRipper.getVideoURLAtPage( + "http://vk.com/video" + oid + "_" + vidid); + addURLToDownload(new URL(videoURL), String.format("%03d_", i + 1)); + try { + Thread.sleep(500); + } catch (InterruptedException e) { + logger.error("Interrupted while waiting to fetch next video URL", e); + break; + } + } + waitForThreads(); + } + + private void ripImages() throws IOException { Map photoIDsToURLs = new HashMap(); int offset = 0; while (true) { @@ -146,7 +199,7 @@ public class VkRipper extends AlbumRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://(www\\.)?vk\\.com/(photos|album)([a-zA-Z0-9_]{1,}).*$"); + Pattern p = Pattern.compile("^https?://(www\\.)?vk\\.com/(photos|album|videos)([a-zA-Z0-9_]{1,}).*$"); Matcher m = p.matcher(url.toExternalForm()); if (!m.matches()) { throw new MalformedURLException("Expected format: http://vk.com/album#### or vk.com/photos####"); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java new file mode 100644 index 00000000..93b51323 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java @@ -0,0 +1,83 @@ +package com.rarchives.ripme.ripper.rippers.video; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.log4j.Logger; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; + +import com.rarchives.ripme.ripper.VideoRipper; + +public class VkRipper extends VideoRipper { + + private static final String HOST = "vk"; + private static final Logger logger = Logger.getLogger(VkRipper.class); + + public VkRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return HOST; + } + + @Override + public boolean canRip(URL url) { + Pattern p = Pattern.compile("^https?://[wm.]*vk\\.com/video[0-9]+.*$"); + Matcher m = p.matcher(url.toExternalForm()); + return m.matches(); + } + + @Override + public URL sanitizeURL(URL url) throws MalformedURLException { + return url; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^https?://[wm.]*vk\\.com/video([0-9]+).*$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + + throw new MalformedURLException( + "Expected vk video URL format:" + + "vk.com/videos####" + + " Got: " + url); + } + + @Override + public void rip() throws IOException { + logger.info(" Retrieving " + this.url); + String videoURL = getVideoURLAtPage(this.url.toExternalForm()); + addURLToDownload(new URL(videoURL), HOST + "_" + getGID(this.url)); + waitForThreads(); + } + + public static String getVideoURLAtPage(String url) throws IOException { + Document doc = Jsoup.connect(url) + .userAgent(USER_AGENT) + .get(); + String html = doc.outerHtml(); + String videoURL = null; + for (String quality : new String[] {"1080", "720", "480", "240"}) { + quality = "url" + quality + "\\\":\\\""; + if (html.contains(quality)) { + videoURL = html.substring(html.indexOf(quality) + quality.length()); + videoURL = videoURL.substring(0, videoURL.indexOf("\"")); + videoURL = videoURL.replace("\\", ""); + break; + } + } + if (videoURL == null) { + throw new IOException("Could not find video URL at " + url); + } + return videoURL; + } +} \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index bdba9cbf..6fce27cc 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -19,7 +19,7 @@ import org.jsoup.nodes.Document; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.0.27"; + private static final String DEFAULT_VERSION = "1.0.28"; private static final String updateJsonURL = "http://rarchives.com/ripme.json"; private static final String updateJarURL = "http://rarchives.com/ripme.jar"; private static final String mainFileName = "ripme.jar";