From 7581d5d36634f0c138a25fed04bbab3b1b1d9768 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 1 Jul 2018 22:26:12 -0400 Subject: [PATCH 1/2] Xvideo ripper now uses AbstractHTMLRipper --- .../rippers/{video => }/XvideosRipper.java | 66 ++++++++++++++----- 1 file changed, 51 insertions(+), 15 deletions(-) rename src/main/java/com/rarchives/ripme/ripper/rippers/{video => }/XvideosRipper.java (57%) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java similarity index 57% rename from src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java rename to src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java index ef71a4bb..a94e7a21 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java @@ -1,31 +1,47 @@ -package com.rarchives.ripme.ripper.rippers.video; +package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.util.ArrayList; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Utils; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; -import com.rarchives.ripme.ripper.VideoRipper; import com.rarchives.ripme.utils.Http; -public class XvideosRipper extends VideoRipper { +public class XvideosRipper extends AbstractHTMLRipper { private static final String HOST = "xvideos"; + private int bytesTotal = 1; + private int bytesCompleted = 1; + public XvideosRipper(URL url) throws IOException { super(url); } + @Override + public Document getFirstPage() throws IOException { + return Http.url(this.url).get(); + } + @Override public String getHost() { return HOST; } + @Override + public String getDomain() { + return HOST + ".com"; + } + @Override public boolean canRip(URL url) { Pattern p = Pattern.compile("^https?://[wm.]*xvideos\\.com/video[0-9]+.*$"); @@ -33,11 +49,6 @@ public class XvideosRipper extends VideoRipper { return m.matches(); } - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return url; - } - @Override public String getGID(URL url) throws MalformedURLException { Pattern p = Pattern.compile("^https?://[wm.]*xvideos\\.com/video([0-9]+).*$"); @@ -53,9 +64,8 @@ public class XvideosRipper extends VideoRipper { } @Override - public void rip() throws IOException { - LOGGER.info(" Retrieving " + this.url); - Document doc = Http.url(this.url).get(); + public List getURLsFromPage(Document doc) { + List results = new ArrayList<>(); Elements scripts = doc.select("script"); for (Element e : scripts) { if (e.html().contains("html5player.setVideoUrlHigh")) { @@ -64,13 +74,39 @@ public class XvideosRipper extends VideoRipper { for (String line: lines) { if (line.contains("html5player.setVideoUrlHigh")) { String videoURL = line.replaceAll("\t", "").replaceAll("html5player.setVideoUrlHigh\\(", "").replaceAll("\'", "").replaceAll("\\);", ""); - addURLToDownload(new URL(videoURL), HOST + "_" + getGID(this.url)); - waitForThreads(); - return; + results.add(videoURL); } } } } - throw new IOException("Unable to find video url at " + this.url.toExternalForm()); + return results; } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } + + @Override + public String getStatusText() { + return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal); + } + + @Override + public int getCompletionPercentage() { + return (int) (100 * (bytesCompleted / (float) bytesTotal)); + } + + @Override + public void setBytesTotal(int bytes) { + this.bytesTotal = bytes; + } + + @Override + public void setBytesCompleted(int bytes) { + this.bytesCompleted = bytes; + } + + @Override + public boolean useByteProgessBar() {return true;} } \ No newline at end of file From f6bb268b4cb9be7a6c3e1202994a2d9365a1c2b7 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 1 Jul 2018 22:32:44 -0400 Subject: [PATCH 2/2] Added a unittest for the XvideosRipper --- .../tst/ripper/rippers/VideoRippersTest.java | 15 ++------------- .../tst/ripper/rippers/XvideosRipperTest.java | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 13 deletions(-) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/XvideosRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java index 05d7514e..02696755 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java @@ -7,9 +7,8 @@ import java.util.List; import com.rarchives.ripme.ripper.VideoRipper; import com.rarchives.ripme.ripper.rippers.video.PornhubRipper; -import com.rarchives.ripme.ripper.rippers.video.TwitchVideoRipper; import com.rarchives.ripme.ripper.rippers.video.XhamsterRipper; -import com.rarchives.ripme.ripper.rippers.video.XvideosRipper; +import com.rarchives.ripme.ripper.rippers.XvideosRipper; import com.rarchives.ripme.ripper.rippers.video.YoupornRipper; import com.rarchives.ripme.ripper.rippers.video.YuvutuRipper; @@ -57,17 +56,7 @@ public class VideoRippersTest extends RippersTest { videoTestHelper(ripper); } } - - public void testXvideosRipper() throws IOException { - List contentURLs = new ArrayList<>(); - contentURLs.add(new URL("https://www.xvideos.com/video19719109/ziggy_star_ultra_hard_anal_pounding")); - contentURLs.add(new URL("https://www.xvideos.com/video23515878/dee_s_pool_toys")); - for (URL url : contentURLs) { - XvideosRipper ripper = new XvideosRipper(url); - videoTestHelper(ripper); - } - } - + public void testPornhubRipper() throws IOException { List contentURLs = new ArrayList<>(); contentURLs.add(new URL("https://www.pornhub.com/view_video.php?viewkey=ph5a329fa707269")); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XvideosRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XvideosRipperTest.java new file mode 100644 index 00000000..543ad8ca --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XvideosRipperTest.java @@ -0,0 +1,16 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.XvideosRipper; +import com.rarchives.ripme.tst.ripper.rippers.RippersTest; + +public class XvideosRipperTest extends RippersTest { + + public void testXhamsterAlbum1() throws IOException { + XvideosRipper ripper = new XvideosRipper(new URL("https://www.xvideos.com/video23515878/dee_s_pool_toys")); + testRipper(ripper); + } + +}