From 27e6f3fe0c4ff255ab9a34f9fadca91ed88348e2 Mon Sep 17 00:00:00 2001 From: Kevin Jiang Date: Fri, 5 Jan 2018 17:01:31 -0500 Subject: [PATCH 1/5] Gfycat Tests & Fix for bad reddit submissions Added sanitization, and appropriate tests. Task #361 - Some gfycat doesn't work --- .../tst/ripper/rippers/GfycatRipperTest.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java new file mode 100644 index 00000000..5c099aca --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java @@ -0,0 +1,32 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import com.rarchives.ripme.ripper.rippers.video.GfycatRipper; +import java.io.IOException; +import java.net.URL; + + +public class GfycatRipperTest extends RippersTest { + + /** + * REDDIT TEST + * Tests a good GfycatURL (no "/gifs/detail") + * @throws IOException + */ + public void testRedditGfyGoodURL() throws IOException { + GfycatRipper ripper = new GfycatRipper(new URL("https://www.reddit.com/r/bottesting/comments/7msozf/good_link/")); + testRipper(ripper); + } + + + /** + * REDDIT TEST + * Tests a Bad URL with the "/gifs/detail" inside. + * @throws IOException + */ + public void testRedditGfyBadURL() throws IOException { + GfycatRipper ripper = new GfycatRipper(new URL("https://www.reddit.com/r/bottesting/comments/7msmhi/bad_link/")); + testRipper(ripper); + } + + +} From 8960b8a565b28d11bf03b373772c5406388023c7 Mon Sep 17 00:00:00 2001 From: Kevin Jiang Date: Fri, 5 Jan 2018 17:01:49 -0500 Subject: [PATCH 2/5] Gfycat Tests & Fix for bad reddit submissions Added sanitization, and appropriate tests. Task #361 - Some gfycat doesn't work --- .../rarchives/ripme/ripper/rippers/video/GfycatRipper.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java index dd0d20eb..35f42564 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java @@ -32,6 +32,8 @@ public class GfycatRipper extends VideoRipper { @Override public URL sanitizeURL(URL url) throws MalformedURLException { + url = new URL(url.toExternalForm().replace("/gifs/detail", "")); + return url; } @@ -64,6 +66,10 @@ public class GfycatRipper extends VideoRipper { */ public static String getVideoURL(URL url) throws IOException { logger.info("Retrieving " + url.toExternalForm()); + + //Sanitize the URL first + url = new URL(url.toExternalForm().replace("/gifs/detail", "")); + Document doc = Http.url(url).get(); Elements videos = doc.select("source#mp4Source"); if (videos.size() == 0) { From d0b93ea1009e348907044f6aefdf17e6e11e0bee Mon Sep 17 00:00:00 2001 From: Kevin Jiang Date: Fri, 5 Jan 2018 17:11:24 -0500 Subject: [PATCH 3/5] Gfycat Tests & Fix for bad reddit submissions Added and restructured Gfycat tests. - Rips from reddit go in Reddit Test Suite - Direct rips from Gfycat go in the Gfycat Test Suite Task #361 - Some gfycat doesn't work --- .../tst/ripper/rippers/GfycatRipperTest.java | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java index 5c099aca..ca73f138 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java @@ -8,25 +8,19 @@ import java.net.URL; public class GfycatRipperTest extends RippersTest { /** - * REDDIT TEST - * Tests a good GfycatURL (no "/gifs/detail") + * Rips correctly formatted URL directly from Gfycat * @throws IOException */ - public void testRedditGfyGoodURL() throws IOException { - GfycatRipper ripper = new GfycatRipper(new URL("https://www.reddit.com/r/bottesting/comments/7msozf/good_link/")); + public void GfycatGoodURL() throws IOException{ + GfycatRipper ripper = new GfycatRipper(new URL("https://gfycat.com/TemptingExcellentIchthyosaurs")); testRipper(ripper); } - - /** - * REDDIT TEST - * Tests a Bad URL with the "/gifs/detail" inside. + * Rips badly formatted URL directly from Gfycat * @throws IOException */ - public void testRedditGfyBadURL() throws IOException { - GfycatRipper ripper = new GfycatRipper(new URL("https://www.reddit.com/r/bottesting/comments/7msmhi/bad_link/")); + public void GfycatBadURL() throws IOException { + GfycatRipper ripper = new GfycatRipper(new URL("https://gfycat.com/gifs/detail/limitedtestyamericancrow")); testRipper(ripper); } - - -} +} \ No newline at end of file From 8cee44125226a82d63c22b406494b472a033d33c Mon Sep 17 00:00:00 2001 From: Kevin Jiang Date: Fri, 5 Jan 2018 17:11:46 -0500 Subject: [PATCH 4/5] Gfycat Tests & Fix for bad reddit submissions Added and restructured Gfycat tests. - Rips from reddit go in Reddit Test Suite - Direct rips from Gfycat go in the Gfycat Test Suite Task #361 - Some gfycat doesn't work --- .../tst/ripper/rippers/RedditRipperTest.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java index f20a21f2..f4ea70a1 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java @@ -5,6 +5,7 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.RedditRipper; +import com.rarchives.ripme.ripper.rippers.video.GfycatRipper; public class RedditRipperTest extends RippersTest { // https://github.com/RipMeApp/ripme/issues/253 - Disabled tests: RedditRipperTest#testRedditSubreddit*Rip is flaky @@ -22,4 +23,25 @@ public class RedditRipperTest extends RippersTest { RedditRipper ripper = new RedditRipper(new URL("http://www.reddit.com/r/UnrealGirls/comments/1ziuhl/in_class_veronique_popa/")); testRipper(ripper); } + + /** + * GFYCAT TEST + * Tests a good GfycatURL (no "/gifs/detail") + * @throws IOException + */ + public void testRedditGfyGoodURL() throws IOException { + GfycatRipper ripper = new GfycatRipper(new URL("https://www.reddit.com/r/bottesting/comments/7msozf/good_link/")); + testRipper(ripper); + } + + + /** + * GFYCAT TEST + * Tests a Bad URL with the "/gifs/detail" inside. + * @throws IOException + */ + public void testRedditGfyBadURL() throws IOException { + GfycatRipper ripper = new GfycatRipper(new URL("https://www.reddit.com/r/bottesting/comments/7msmhi/bad_link/")); + testRipper(ripper); + } } From 16efd06e3fcd11461f99d6b0aaf26e06d5ce9b06 Mon Sep 17 00:00:00 2001 From: Kevin Jiang Date: Fri, 5 Jan 2018 17:38:30 -0500 Subject: [PATCH 5/5] Gfycat Tests & Fix for bad reddit submissions Final few test tweaks. Everything's working now. Task #361 - Some gfycat doesn't work --- .../rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java index f4ea70a1..fdd61cf9 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java @@ -30,7 +30,7 @@ public class RedditRipperTest extends RippersTest { * @throws IOException */ public void testRedditGfyGoodURL() throws IOException { - GfycatRipper ripper = new GfycatRipper(new URL("https://www.reddit.com/r/bottesting/comments/7msozf/good_link/")); + RedditRipper ripper = new RedditRipper(new URL("https://www.reddit.com/r/bottesting/comments/7msozf/good_link/")); testRipper(ripper); } @@ -41,7 +41,7 @@ public class RedditRipperTest extends RippersTest { * @throws IOException */ public void testRedditGfyBadURL() throws IOException { - GfycatRipper ripper = new GfycatRipper(new URL("https://www.reddit.com/r/bottesting/comments/7msmhi/bad_link/")); + RedditRipper ripper = new RedditRipper(new URL("https://www.reddit.com/r/bottesting/comments/7msmhi/bad_link/")); testRipper(ripper); } }