From 67e947a0e580e4bd552eac768ba9a5e1a6387e9f Mon Sep 17 00:00:00 2001 From: 4pr0n Date: Sun, 20 Apr 2014 00:12:48 -0700 Subject: [PATCH] Added support for pornhub videos #18 --- .../ripme/ripper/DownloadVideoThread.java | 2 +- .../rarchives/ripme/ripper/VideoRipper.java | 5 + .../ripper/rippers/video/PornhubRipper.java | 99 +++++++++++++++++++ .../ripper/rippers/video/XvideosRipper.java | 5 - .../com/rarchives/ripme/ui/MainWindow.java | 1 + .../java/com/rarchives/ripme/utils/AES.java | 39 ++++++++ .../com/rarchives/ripme/utils/Base64.java | 76 ++++++++++++++ .../java/com/rarchives/ripme/tst/AppTest.java | 3 - .../tst/ripper/rippers/ImgurRipperTest.java | 2 +- .../tst/ripper/rippers/VideoRippersTest.java | 22 ++++- 10 files changed, 243 insertions(+), 11 deletions(-) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java create mode 100644 src/main/java/com/rarchives/ripme/utils/AES.java create mode 100644 src/main/java/com/rarchives/ripme/utils/Base64.java diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java index d50cf31b..665f307b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java @@ -74,7 +74,7 @@ public class DownloadVideoThread extends Thread { int tries = 0; // Number of attempts to download do { InputStream bis = null; OutputStream fos = null; - byte[] data = new byte[1024]; int bytesRead; + byte[] data = new byte[1024 * 256]; int bytesRead; try { logger.info(" Downloading file: " + url + (tries > 0 ? " Retry #" + tries : "")); observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm()); diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index ae52e0c3..3b4e06a8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -32,6 +32,11 @@ public abstract class VideoRipper extends AbstractRipper { this.bytesCompleted = bytes; } + @Override + public String getAlbumTitle(URL url) { + return "videos"; + } + @Override public void addURLToDownload(URL url, File saveAs) { threadPool.addThread(new DownloadVideoThread(url, saveAs, this)); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java new file mode 100644 index 00000000..76e8ef41 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java @@ -0,0 +1,99 @@ +package com.rarchives.ripme.ripper.rippers.video; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLDecoder; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.log4j.Logger; +import org.json.JSONException; +import org.json.JSONObject; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; + +import com.rarchives.ripme.ripper.VideoRipper; +import com.rarchives.ripme.utils.AES; + +public class PornhubRipper extends VideoRipper { + + private static final String HOST = "pornhub"; + private static final Logger logger = Logger.getLogger(PornhubRipper.class); + + public PornhubRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return HOST; + } + + @Override + public boolean canRip(URL url) { + Pattern p = Pattern.compile("^https?://[wm.]*pornhub\\.com/view_video.php\\?viewkey=[0-9]+.*$"); + Matcher m = p.matcher(url.toExternalForm()); + return m.matches(); + } + + @Override + public URL sanitizeURL(URL url) throws MalformedURLException { + return url; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^https?://[wm.]*pornhub\\.com/view_video.php\\?viewkey=([0-9]+).*$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + + throw new MalformedURLException( + "Expected pornhub format:" + + "pornhub.com/view_video.php?viewkey=####" + + " Got: " + url); + } + + @Override + public void rip() throws IOException { + logger.info(" Retrieving " + this.url.toExternalForm()); + Document doc = Jsoup.connect(this.url.toExternalForm()) + .userAgent(USER_AGENT) + .get(); + Pattern p = Pattern.compile("^.*var flashvars = (.*});.*$", Pattern.DOTALL); + Matcher m = p.matcher(doc.body().html()); + if (m.matches()) { + String title = null, + encryptedUrl = null; + try { + JSONObject json = new JSONObject(m.group(1)); + + title = json.getString("video_title"); + title = title.replaceAll("\\+", " "); + + encryptedUrl = null; + for (String quality : new String[] {"quality_1080p", "quality_720p", "quality_480p", "quality_240p"}) { + if (json.has(quality)) { + encryptedUrl = json.getString(quality); + break; + } + } + if (encryptedUrl == null) { + throw new IOException("Unable to find encrypted video URL at " + this.url); + } + encryptedUrl = URLDecoder.decode(encryptedUrl, "UTF-8"); + String vidUrl = AES.decrypt(encryptedUrl, title, 256); + addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url)); + } catch (JSONException e) { + logger.error("Error while parsing JSON at " + url, e); + throw e; + } catch (Exception e) { + logger.error("Error while retrieving video URL at " + url, e); + throw new IOException(e); + } + } + waitForThreads(); + } +} \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java index 468c6691..e70095eb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java @@ -36,11 +36,6 @@ public class XvideosRipper extends VideoRipper { return m.matches(); } - @Override - public String getAlbumTitle(URL url) { - return "videos"; - } - @Override public URL sanitizeURL(URL url) throws MalformedURLException { return url; diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index ad2f0550..22c91916 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -672,6 +672,7 @@ public class MainWindow implements Runnable, RipStatusHandler { int completedPercent = evt.ripper.getCompletionPercentage(); statusProgress.setValue(completedPercent); + statusProgress.setVisible(true); status( evt.ripper.getStatusText() ); switch(msg.getStatus()) { diff --git a/src/main/java/com/rarchives/ripme/utils/AES.java b/src/main/java/com/rarchives/ripme/utils/AES.java new file mode 100644 index 00000000..e6bc742c --- /dev/null +++ b/src/main/java/com/rarchives/ripme/utils/AES.java @@ -0,0 +1,39 @@ +package com.rarchives.ripme.utils; + +import java.util.Arrays; + +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +public class AES { + + public static String decrypt(String cipherText, String key, int nBits) throws Exception { + String res = null; + nBits = nBits / 8; + byte[] data = Base64.decode(cipherText); + byte[] k = Arrays.copyOf(key.getBytes(), nBits); + + Cipher cipher = Cipher.getInstance("AES/CTR/NoPadding"); + SecretKey secretKey = generateSecretKey(k, nBits); + byte[] nonceBytes = Arrays.copyOf(Arrays.copyOf(data, 8), nBits / 2); + IvParameterSpec nonce = new IvParameterSpec(nonceBytes); + cipher.init(Cipher.ENCRYPT_MODE, secretKey, nonce); + res = new String(cipher.doFinal(data, 8, data.length - 8)); + return res; + } + + private static SecretKey generateSecretKey(byte[] keyBytes, int nBits) throws Exception { + try { + SecretKey secretKey = new SecretKeySpec(keyBytes, "AES"); + Cipher cipher = Cipher.getInstance("AES/ECB/NoPadding"); + cipher.init(Cipher.ENCRYPT_MODE, secretKey); + keyBytes = cipher.doFinal(keyBytes); + } catch (Throwable e1) { + return null; + } + System.arraycopy(keyBytes, 0, keyBytes, nBits / 2, nBits / 2); + return new SecretKeySpec(keyBytes, "AES"); + } +} diff --git a/src/main/java/com/rarchives/ripme/utils/Base64.java b/src/main/java/com/rarchives/ripme/utils/Base64.java new file mode 100644 index 00000000..d38f545e --- /dev/null +++ b/src/main/java/com/rarchives/ripme/utils/Base64.java @@ -0,0 +1,76 @@ +package com.rarchives.ripme.utils; + +/** + * Base64 encoder/decoder + * From http://stackoverflow.com/a/4265472 + * + */ +public class Base64 { + private final static char[] ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".toCharArray(); + + private static int[] toInt = new int[128]; + + static { + for(int i=0; i< ALPHABET.length; i++){ + toInt[ALPHABET[i]]= i; + } + } + + /** + * Translates the specified byte array into Base64 string. + * + * @param buf the byte array (not null) + * @return the translated Base64 string (not null) + */ + public static String encode(byte[] buf){ + int size = buf.length; + char[] ar = new char[((size + 2) / 3) * 4]; + int a = 0; + int i=0; + while(i < size){ + byte b0 = buf[i++]; + byte b1 = (i < size) ? buf[i++] : 0; + byte b2 = (i < size) ? buf[i++] : 0; + + int mask = 0x3F; + ar[a++] = ALPHABET[(b0 >> 2) & mask]; + ar[a++] = ALPHABET[((b0 << 4) | ((b1 & 0xFF) >> 4)) & mask]; + ar[a++] = ALPHABET[((b1 << 2) | ((b2 & 0xFF) >> 6)) & mask]; + ar[a++] = ALPHABET[b2 & mask]; + } + switch(size % 3){ + case 1: ar[--a] = '='; + case 2: ar[--a] = '='; + } + return new String(ar); + } + + /** + * Translates the specified Base64 string into a byte array. + * + * @param s the Base64 string (not null) + * @return the byte array (not null) + */ + public static byte[] decode(String s){ + int delta = s.endsWith( "==" ) ? 2 : s.endsWith( "=" ) ? 1 : 0; + byte[] buffer = new byte[s.length()*3/4 - delta]; + int mask = 0xFF; + int index = 0; + for(int i=0; i< s.length(); i+=4){ + int c0 = toInt[s.charAt( i )]; + int c1 = toInt[s.charAt( i + 1)]; + buffer[index++]= (byte)(((c0 << 2) | (c1 >> 4)) & mask); + if(index >= buffer.length){ + return buffer; + } + int c2 = toInt[s.charAt( i + 2)]; + buffer[index++]= (byte)(((c1 << 4) | (c2 >> 2)) & mask); + if(index >= buffer.length){ + return buffer; + } + int c3 = toInt[s.charAt( i + 3 )]; + buffer[index++]= (byte)(((c2 << 6) | c3) & mask); + } + return buffer; + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/AppTest.java b/src/test/java/com/rarchives/ripme/tst/AppTest.java index 77368a89..630e46d9 100644 --- a/src/test/java/com/rarchives/ripme/tst/AppTest.java +++ b/src/test/java/com/rarchives/ripme/tst/AppTest.java @@ -4,8 +4,6 @@ import junit.framework.Test; import junit.framework.TestCase; import junit.framework.TestSuite; -import com.rarchives.ripme.utils.Utils; - public class AppTest extends TestCase { /** * Create the test case @@ -27,7 +25,6 @@ public class AppTest extends TestCase { * Rigourous Test :-) */ public void testApp() { - System.err.println(Utils.bytesToHumanReadable(1023 * 5000)); assertTrue( true ); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java index f957cba1..4b436f70 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java @@ -59,7 +59,7 @@ public class ImgurRipperTest extends RippersTest { } public void testImgurAlbums() throws IOException { - if (false && !DOWNLOAD_CONTENT) { + if (!DOWNLOAD_CONTENT) { return; } List contentURLs = new ArrayList(); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java index 847700a9..f2a4b132 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java @@ -5,12 +5,13 @@ import java.net.URL; import java.util.ArrayList; import java.util.List; +import com.rarchives.ripme.ripper.rippers.video.PornhubRipper; import com.rarchives.ripme.ripper.rippers.video.XvideosRipper; public class VideoRippersTest extends RippersTest { public void testXvideosRipper() throws IOException { - if (false && !DOWNLOAD_CONTENT) { + if (!DOWNLOAD_CONTENT) { return; } List contentURLs = new ArrayList(); @@ -28,5 +29,24 @@ public class VideoRippersTest extends RippersTest { } } } + + public void testPornhubRipper() throws IOException { + if (!DOWNLOAD_CONTENT) { + return; + } + List contentURLs = new ArrayList(); + contentURLs.add(new URL("http://www.pornhub.com/view_video.php?viewkey=993166542")); + for (URL url : contentURLs) { + try { + PornhubRipper ripper = new PornhubRipper(url); + ripper.rip(); + assert(ripper.getWorkingDir().listFiles().length > 1); + deleteDir(ripper.getWorkingDir()); + } catch (Exception e) { + e.printStackTrace(); + fail("Error while ripping URL " + url + ": " + e.getMessage()); + } + } + } }