From fc3912585b98a85df3514d4879bda72fadc19850 Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 02:04:32 -0800 Subject: [PATCH 01/43] Split BasicRippersTest.java into per-ripper test files. (#240) --- .../java/com/rarchives/ripme/tst/AppTest.java | 8 +- .../tst/ripper/rippers/BasicRippersTest.java | 301 ------------------ .../ripper/rippers/DeviantartRipperTest.java | 19 ++ .../ripper/rippers/EightmusesRipperTest.java | 13 + .../tst/ripper/rippers/FineboxRipperTest.java | 22 ++ .../rippers/FivehundredpxRipperTest.java | 13 + .../tst/ripper/rippers/FlickrRipperTest.java | 15 + .../ripper/rippers/FuraffinityRipperTest.java | 15 + .../ripper/rippers/FuskatorRipperTest.java | 15 + .../rippers/GirlsOfDesireRipperTest.java | 13 + .../rippers/HentaifoundryRipperTest.java | 13 + .../ripper/rippers/ImagearnRipperTest.java | 13 + .../ripper/rippers/ImagebamRipperTest.java | 13 + .../ripper/rippers/ImagestashRipperTest.java | 15 + .../ripper/rippers/ImagevenueRipperTest.java | 13 + .../tst/ripper/rippers/ImgboxRipperTest.java | 13 + .../ripper/rippers/ModelmayhemRipperTest.java | 13 + .../ripper/rippers/MotherlessRipperTest.java | 13 + .../tst/ripper/rippers/NfsfwRipperTest.java | 13 + .../ripper/rippers/PhotobucketRipperTest.java | 21 ++ .../ripper/rippers/PichunterRipperTest.java | 20 ++ .../tst/ripper/rippers/PornhubRipperTest.java | 13 + .../tst/ripper/rippers/RedditRipperTest.java | 24 ++ .../rippers/SankakuComplexRipperTest.java | 19 ++ .../ripper/rippers/ShesFreakyRipperTest.java | 13 + .../ripper/rippers/TapasticRipperTest.java | 13 + .../ripper/rippers/TeenplanetRipperTest.java | 13 + .../rippers/TheyiffgalleryRipperTest.java | 13 + .../tst/ripper/rippers/TumblrRipperTest.java | 24 ++ .../tst/ripper/rippers/TwitterRipperTest.java | 20 ++ .../rippers/TwodgalleriesRipperTest.java | 16 + .../tst/ripper/rippers/VidbleRipperTest.java | 14 + .../tst/ripper/rippers/VineRipperTest.java | 15 + .../tst/ripper/rippers/VkRipperTest.java | 27 ++ .../ripper/rippers/XhamsterRipperTest.java | 14 + 35 files changed, 527 insertions(+), 305 deletions(-) delete mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/FineboxRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/FivehundredpxRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/FlickrRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuskatorRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/GirlsOfDesireRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaifoundryRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagearnRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagestashRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagevenueRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgboxRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelmayhemRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/NfsfwRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/PhotobucketRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/PichunterRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/SankakuComplexRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/ShesFreakyRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/TapasticRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/TeenplanetRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/TheyiffgalleryRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwitterRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwodgalleriesRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/VidbleRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/VineRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/AppTest.java b/src/test/java/com/rarchives/ripme/tst/AppTest.java index 630e46d9..fb9bc680 100644 --- a/src/test/java/com/rarchives/ripme/tst/AppTest.java +++ b/src/test/java/com/rarchives/ripme/tst/AppTest.java @@ -10,21 +10,21 @@ public class AppTest extends TestCase { * * @param testName name of the test case */ - public AppTest( String testName ) { - super( testName ); + public AppTest(String testName) { + super(testName); } /** * @return the suite of tests being tested */ public static Test suite() { - return new TestSuite( AppTest.class ); + return new TestSuite(AppTest.class); } /** * Rigourous Test :-) */ public void testApp() { - assertTrue( true ); + assertTrue(true); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java deleted file mode 100644 index 751242c6..00000000 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java +++ /dev/null @@ -1,301 +0,0 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import java.io.IOException; -import java.net.URL; - -import com.rarchives.ripme.ripper.AbstractRipper; -import com.rarchives.ripme.ripper.rippers.DeviantartRipper; -import com.rarchives.ripme.ripper.rippers.EightmusesRipper; -import com.rarchives.ripme.ripper.rippers.FivehundredpxRipper; -import com.rarchives.ripme.ripper.rippers.FuraffinityRipper; -import com.rarchives.ripme.ripper.rippers.GirlsOfDesireRipper; -import com.rarchives.ripme.ripper.rippers.HentaifoundryRipper; -import com.rarchives.ripme.ripper.rippers.ImagearnRipper; -import com.rarchives.ripme.ripper.rippers.ImagebamRipper; -import com.rarchives.ripme.ripper.rippers.ImagevenueRipper; -import com.rarchives.ripme.ripper.rippers.ImgboxRipper; -import com.rarchives.ripme.ripper.rippers.ModelmayhemRipper; -import com.rarchives.ripme.ripper.rippers.MotherlessRipper; -import com.rarchives.ripme.ripper.rippers.NfsfwRipper; -import com.rarchives.ripme.ripper.rippers.PhotobucketRipper; -import com.rarchives.ripme.ripper.rippers.PornhubRipper; -import com.rarchives.ripme.ripper.rippers.ShesFreakyRipper; -import com.rarchives.ripme.ripper.rippers.TapasticRipper; -import com.rarchives.ripme.ripper.rippers.TeenplanetRipper; -import com.rarchives.ripme.ripper.rippers.TwitterRipper; -import com.rarchives.ripme.ripper.rippers.TwodgalleriesRipper; -import com.rarchives.ripme.ripper.rippers.VidbleRipper; -import com.rarchives.ripme.ripper.rippers.VineRipper; -import com.rarchives.ripme.ripper.rippers.VkRipper; -import com.rarchives.ripme.ripper.rippers.XhamsterRipper; -import com.rarchives.ripme.ripper.rippers.PichunterRipper; -import com.rarchives.ripme.ripper.rippers.TheyiffgalleryRipper; - - -/** - * Simple test cases for various rippers. - * These tests only require a URL, no other special validation. - */ -public class BasicRippersTest extends RippersTest { - - public void testDeviantartAlbum() throws IOException { - DeviantartRipper ripper = new DeviantartRipper(new URL("http://airgee.deviantart.com/gallery/")); - testRipper(ripper); - } - public void testDeviantartNSFWAlbum() throws IOException { - // NSFW gallery - DeviantartRipper ripper = new DeviantartRipper(new URL("http://faterkcx.deviantart.com/gallery/")); - testRipper(ripper); - } - - public void testEightmusesAlbum() throws IOException { - EightmusesRipper ripper = new EightmusesRipper(new URL("https://www.8muses.com/album/jab-comics/a-model-life")); - testRipper(ripper); - } - - /* - public void testVineboxAlbum() throws IOException { - FineboxRipper ripper = new FineboxRipper(new URL("http://vinebox.co/u/wi57hMjc2Ka")); - testRipper(ripper); - } - */ - - /* - public void testFineboxAlbum() throws IOException { - FineboxRipper ripper = new FineboxRipper(new URL("http://finebox.co/u/wi57hMjc2Ka")); - testRipper(ripper); - } - */ - - /* - public void testRedditSubredditRip() throws IOException { - RedditRipper ripper = new RedditRipper(new URL("http://www.reddit.com/r/nsfw_oc")); - testRipper(ripper); - } - public void testRedditSubredditTopRip() throws IOException { - RedditRipper ripper = new RedditRipper(new URL("http://www.reddit.com/r/nsfw_oc/top?t=all")); - testRipper(ripper); - } - public void testRedditPostRip() throws IOException { - RedditRipper ripper = new RedditRipper(new URL("http://www.reddit.com/r/UnrealGirls/comments/1ziuhl/in_class_veronique_popa/")); - testRipper(ripper); - } - - public void testTumblrFullRip() throws IOException { - TumblrRipper ripper = new TumblrRipper(new URL("http://wrouinr.tumblr.com/archive")); - testRipper(ripper); - } - public void testTumblrTagRip() throws IOException { - TumblrRipper ripper = new TumblrRipper(new URL("http://topinstagirls.tumblr.com/tagged/berlinskaya")); - testRipper(ripper); - } - public void testTumblrPostRip() throws IOException { - TumblrRipper ripper = new TumblrRipper(new URL("http://sadbaffoon.tumblr.com/post/132045920789/what-a-hoe")); - testRipper(ripper); - } - */ - - public void testTwitterUserRip() throws IOException { - TwitterRipper ripper = new TwitterRipper(new URL("https://twitter.com/danngamber01/media")); - testRipper(ripper); - } - /* - public void testTwitterSearchRip() throws IOException { - TwitterRipper ripper = new TwitterRipper(new URL("https://twitter.com/search?q=from%3ADaisyfairymfc%20filter%3Aimages&src=typd")); - testRipper(ripper); - } - */ - - public void test500pxAlbum() throws IOException { - FivehundredpxRipper ripper = new FivehundredpxRipper(new URL("https://marketplace.500px.com/alexander_hurman")); - testRipper(ripper); - } - - /* - public void testFlickrAlbum() throws IOException { - FlickrRipper ripper = new FlickrRipper(new URL("https://www.flickr.com/photos/leavingallbehind/sets/72157621895942720/")); - testRipper(ripper); - } - */ - - /* - public void testFuraffinityAlbum() throws IOException { - FuraffinityRipper ripper = new FuraffinityRipper(new URL("https://www.furaffinity.net/gallery/mustardgas/")); - testRipper(ripper); - } - */ - - /* - public void testFuskatorAlbum() throws IOException { - FuskatorRipper ripper = new FuskatorRipper(new URL("http://fuskator.com/full/emJa1U6cqbi/index.html")); - testRipper(ripper); - } - */ - - public void testGirlsofdesireAlbum() throws IOException { - GirlsOfDesireRipper ripper = new GirlsOfDesireRipper(new URL("http://www.girlsofdesire.org/galleries/krillia/")); - testRipper(ripper); - } - - public void testHentaifoundryRip() throws IOException { - HentaifoundryRipper ripper = new HentaifoundryRipper(new URL("http://www.hentai-foundry.com/pictures/user/personalami")); - testRipper(ripper); - } - - public void testImagearnRip() throws IOException { - AbstractRipper ripper = new ImagearnRipper(new URL("http://imagearn.com//gallery.php?id=578682")); - testRipper(ripper); - } - - public void testImagebamRip() throws IOException { - AbstractRipper ripper = new ImagebamRipper(new URL("http://www.imagebam.com/gallery/488cc796sllyf7o5srds8kpaz1t4m78i")); - testRipper(ripper); - } - - /* - public void testImagestashRip() throws IOException { - AbstractRipper ripper = new ImagestashRipper(new URL("https://imagestash.org/tag/everydayuncensor")); - testRipper(ripper); - } - */ - - public void testImagevenueRip() throws IOException { - AbstractRipper ripper = new ImagevenueRipper(new URL("http://img120.imagevenue.com/galshow.php?gal=gallery_1373818527696_191lo")); - testRipper(ripper); - } - - public void testImgboxRip() throws IOException { - AbstractRipper ripper = new ImgboxRipper(new URL("https://imgbox.com/g/FJPF7t26FD")); - testRipper(ripper); - } - - /* - public void testMinusUserRip() throws IOException { - AbstractRipper ripper = new MinusRipper(new URL("http://vampyr3.minus.com/")); - testRipper(ripper); - deleteSubdirs(ripper.getWorkingDir()); - deleteDir(ripper.getWorkingDir()); - } - public void testMinusUserAlbumRip() throws IOException { - AbstractRipper ripper = new MinusRipper(new URL("http://vampyr3.minus.com/mw7ztQ6xzP7ae")); - testRipper(ripper); - } - public void testMinusUserUploadsRip() throws IOException { - AbstractRipper ripper = new MinusRipper(new URL("http://vampyr3.minus.com/uploads")); - testRipper(ripper); - } - public void testMinusAlbumRip() throws IOException { - AbstractRipper ripper = new MinusRipper(new URL("http://minus.com/mw7ztQ6xzP7ae")); - testRipper(ripper); - } - */ - - public void testModelmayhemRip() throws IOException { - AbstractRipper ripper = new ModelmayhemRipper(new URL("https://www.modelmayhem.com/portfolio/520206/viewall")); - testRipper(ripper); - } - - public void testPichunterRip() throws IOException { - // A non-photoset - AbstractRipper ripper = new PichunterRipper(new URL("https://www.pichunter.com/models/Madison_Ivy")); - testRipper(ripper); - // a photo set - ripper = new PichunterRipper(new URL("http://www.pichunter.com/gallery/3270642/Its_not_only_those_who")); - testRipper(ripper); - - } - - public void testMotherlessAlbumRip() throws IOException { - MotherlessRipper ripper = new MotherlessRipper(new URL("http://motherless.com/G4DAA18D")); - testRipper(ripper); - } - - public void testNfsfwRip() throws IOException { - AbstractRipper ripper = new NfsfwRipper(new URL("http://nfsfw.com/gallery/v/Kitten/")); - testRipper(ripper); - } - - /* - // https://github.com/RipMeApp/ripme/issues/229 : Disabled test (temporary) : BasicRippersTest#testPhotobucketRip (timing out) - public void testPhotobucketRip() throws IOException { - AbstractRipper ripper = new PhotobucketRipper(new URL("http://s844.photobucket.com/user/SpazzySpizzy/library/Album%20Covers?sort=3&page=1")); - testRipper(ripper); - deleteSubdirs(ripper.getWorkingDir()); - deleteDir(ripper.getWorkingDir()); - } - */ - - public void testPornhubRip() throws IOException { - AbstractRipper ripper = new PornhubRipper(new URL("https://www.pornhub.com/album/15680522")); - testRipper(ripper); - } - - public void testTheyiffgallery() throws IOException { - AbstractRipper ripper = new TheyiffgalleryRipper(new URL("https://theyiffgallery.com/index?/category/4303")); - testRipper(ripper); - } - - /* - public void testSankakuChanRip() throws IOException { - AbstractRipper ripper = new SankakuComplexRipper(new URL("https://chan.sankakucomplex.com/?tags=cleavage")); - testRipper(ripper); - } - public void testSankakuIdolRip() throws IOException { - AbstractRipper ripper = new SankakuComplexRipper(new URL("https://idol.sankakucomplex.com/?tags=meme_%28me%21me%21me%21%29_%28cosplay%29")); - testRipper(ripper); - } - */ - - public void testShesFreakyRip() throws IOException { - AbstractRipper ripper = new ShesFreakyRipper(new URL("http://www.shesfreaky.com/gallery/nicee-snow-bunny-579NbPjUcYa.html")); - testRipper(ripper); - } - - public void testTapasticRip() throws IOException { - AbstractRipper ripper = new TapasticRipper(new URL("http://tapastic.com/episode/2139")); - testRipper(ripper); - } - - public void testTeenplanetRip() throws IOException { - AbstractRipper ripper = new TeenplanetRipper(new URL("http://teenplanet.org/galleries/the-perfect-side-of-me-6588.html")); - testRipper(ripper); - } - - /* - public void testTwodgalleriesRip() throws IOException { - AbstractRipper ripper = new TwodgalleriesRipper(new URL("http://www.2dgalleries.com/artist/regis-loisel-6477")); - testRipper(ripper); - } - */ - - public void testVidbleRip() throws IOException { - AbstractRipper ripper = new VidbleRipper(new URL("http://www.vidble.com/album/y1oyh3zd")); - testRipper(ripper); - } - - /* - public void testVineRip() throws IOException { - AbstractRipper ripper = new VineRipper(new URL("https://vine.co/u/954440445776334848")); - testRipper(ripper); - } - */ - - public void testVkSubalbumRip() throws IOException { - VkRipper ripper = new VkRipper(new URL("http://vk.com/album45506334_0")); - testRipper(ripper); - } - public void testVkRootAlbumRip() throws IOException { - VkRipper ripper = new VkRipper(new URL("https://vk.com/album45506334_0")); - testRipper(ripper); - } - public void testVkPhotosRip() throws IOException { - VkRipper ripper = new VkRipper(new URL("https://vk.com/photos45506334")); - testRipper(ripper); - } - - // https://github.com/RipMeApp/ripme/issues/206 : Disabled test : BasicRippersTest#testXhamsterAlbums -- fix and re-enable - // public void testXhamsterAlbums() throws IOException { - // XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.com/photos/gallery/sexy-preggo-girls-9026608")); - // testRipper(ripper); - // } -} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java new file mode 100644 index 00000000..a3fdfd81 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java @@ -0,0 +1,19 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.DeviantartRipper; + +public class DeviantartRipperTest extends RippersTest { + public void testDeviantartAlbum() throws IOException { + DeviantartRipper ripper = new DeviantartRipper(new URL("http://airgee.deviantart.com/gallery/")); + testRipper(ripper); + } + + public void testDeviantartNSFWAlbum() throws IOException { + // NSFW gallery + DeviantartRipper ripper = new DeviantartRipper(new URL("http://faterkcx.deviantart.com/gallery/")); + testRipper(ripper); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java new file mode 100644 index 00000000..068f9a81 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.EightmusesRipper; + +public class EightmusesRipperTest extends RippersTest { + public void testEightmusesAlbum() throws IOException { + EightmusesRipper ripper = new EightmusesRipper(new URL("https://www.8muses.com/album/jab-comics/a-model-life")); + testRipper(ripper); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FineboxRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FineboxRipperTest.java new file mode 100644 index 00000000..6c55efd1 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FineboxRipperTest.java @@ -0,0 +1,22 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.FineboxRipper; + +public class FineboxRipperTest extends RippersTest { + /* + public void testVineboxAlbum() throws IOException { + FineboxRipper ripper = new FineboxRipper(new URL("http://vinebox.co/u/wi57hMjc2Ka")); + testRipper(ripper); + } + */ + + /* + public void testFineboxAlbum() throws IOException { + FineboxRipper ripper = new FineboxRipper(new URL("http://finebox.co/u/wi57hMjc2Ka")); + testRipper(ripper); + } + */ +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FivehundredpxRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FivehundredpxRipperTest.java new file mode 100644 index 00000000..538d493c --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FivehundredpxRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.FivehundredpxRipper; + +public class FivehundredpxRipperTest extends RippersTest { + public void test500pxAlbum() throws IOException { + FivehundredpxRipper ripper = new FivehundredpxRipper(new URL("https://marketplace.500px.com/alexander_hurman")); + testRipper(ripper); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FlickrRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FlickrRipperTest.java new file mode 100644 index 00000000..5172d5a4 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FlickrRipperTest.java @@ -0,0 +1,15 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.FlickrRipper; + +public class FlickrRipperTest extends RippersTest { + /* + public void testFlickrAlbum() throws IOException { + FlickrRipper ripper = new FlickrRipper(new URL("https://www.flickr.com/photos/leavingallbehind/sets/72157621895942720/")); + testRipper(ripper); + } + */ +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java new file mode 100644 index 00000000..a88c7a28 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java @@ -0,0 +1,15 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.FuraffinityRipper; + +public class FuraffinityRipperTest extends RippersTest { + /* + public void testFuraffinityAlbum() throws IOException { + FuraffinityRipper ripper = new FuraffinityRipper(new URL("https://www.furaffinity.net/gallery/mustardgas/")); + testRipper(ripper); + } + */ +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuskatorRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuskatorRipperTest.java new file mode 100644 index 00000000..290cc4aa --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuskatorRipperTest.java @@ -0,0 +1,15 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.FuskatorRipper; + +public class FuskatorRipperTest extends RippersTest { + /* + public void testFuskatorAlbum() throws IOException { + FuskatorRipper ripper = new FuskatorRipper(new URL("http://fuskator.com/full/emJa1U6cqbi/index.html")); + testRipper(ripper); + } + */ +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GirlsOfDesireRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GirlsOfDesireRipperTest.java new file mode 100644 index 00000000..fa51a262 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GirlsOfDesireRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.GirlsOfDesireRipper; + +public class GirlsOfDesireRipperTest extends RippersTest { + public void testGirlsofdesireAlbum() throws IOException { + GirlsOfDesireRipper ripper = new GirlsOfDesireRipper(new URL("http://www.girlsofdesire.org/galleries/krillia/")); + testRipper(ripper); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaifoundryRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaifoundryRipperTest.java new file mode 100644 index 00000000..9dadbd27 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaifoundryRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.HentaifoundryRipper; + +public class HentaifoundryRipperTest extends RippersTest { + public void testHentaifoundryRip() throws IOException { + HentaifoundryRipper ripper = new HentaifoundryRipper(new URL("http://www.hentai-foundry.com/pictures/user/personalami")); + testRipper(ripper); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagearnRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagearnRipperTest.java new file mode 100644 index 00000000..b9710635 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagearnRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.ImagearnRipper; + +public class ImagearnRipperTest extends RippersTest { + public void testImagearnRip() throws IOException { + ImagearnRipper ripper = new ImagearnRipper(new URL("http://imagearn.com//gallery.php?id=578682")); + testRipper(ripper); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java new file mode 100644 index 00000000..0c3e0761 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.ImagebamRipper; + +public class ImagebamRipperTest extends RippersTest { + public void testImagebamRip() throws IOException { + ImagebamRipper ripper = new ImagebamRipper(new URL("http://www.imagebam.com/gallery/488cc796sllyf7o5srds8kpaz1t4m78i")); + testRipper(ripper); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagestashRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagestashRipperTest.java new file mode 100644 index 00000000..f6776ef2 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagestashRipperTest.java @@ -0,0 +1,15 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.ImagestashRipper; + +public class ImagestashRipperTest extends RippersTest { + /* + public void testImagestashRip() throws IOException { + ImagestashRipper ripper = new ImagestashRipper(new URL("https://imagestash.org/tag/everydayuncensor")); + testRipper(ripper); + } + */ +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagevenueRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagevenueRipperTest.java new file mode 100644 index 00000000..135a7b0a --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagevenueRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.ImagevenueRipper; + +public class ImagevenueRipperTest extends RippersTest { + public void testImagevenueRip() throws IOException { + ImagevenueRipper ripper = new ImagevenueRipper(new URL("http://img120.imagevenue.com/galshow.php?gal=gallery_1373818527696_191lo")); + testRipper(ripper); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgboxRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgboxRipperTest.java new file mode 100644 index 00000000..db8e1680 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgboxRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.ImgboxRipper; + +public class ImgboxRipperTest extends RippersTest { + public void testImgboxRip() throws IOException { + ImgboxRipper ripper = new ImgboxRipper(new URL("https://imgbox.com/g/FJPF7t26FD")); + testRipper(ripper); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelmayhemRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelmayhemRipperTest.java new file mode 100644 index 00000000..621d77c3 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelmayhemRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.ModelmayhemRipper; + +public class ModelmayhemRipperTest extends RippersTest { + public void testModelmayhemRip() throws IOException { + ModelmayhemRipper ripper = new ModelmayhemRipper(new URL("https://www.modelmayhem.com/portfolio/520206/viewall")); + testRipper(ripper); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java new file mode 100644 index 00000000..391cc3c9 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.MotherlessRipper; + +public class MotherlessRipperTest extends RippersTest { + public void testMotherlessAlbumRip() throws IOException { + MotherlessRipper ripper = new MotherlessRipper(new URL("http://motherless.com/G4DAA18D")); + testRipper(ripper); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NfsfwRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NfsfwRipperTest.java new file mode 100644 index 00000000..bc0d4908 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NfsfwRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.NfsfwRipper; + +public class NfsfwRipperTest extends RippersTest { + public void testNfsfwRip() throws IOException { + NfsfwRipper ripper = new NfsfwRipper(new URL("http://nfsfw.com/gallery/v/Kitten/")); + testRipper(ripper); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PhotobucketRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PhotobucketRipperTest.java new file mode 100644 index 00000000..dff101a0 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PhotobucketRipperTest.java @@ -0,0 +1,21 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.PhotobucketRipper; + +public class PhotobucketRipperTest extends RippersTest { + /* + // https://github.com/RipMeApp/ripme/issues/229 : Disabled test (temporary) : BasicRippersTest#testPhotobucketRip (timing out) + public void testPhotobucketRip() throws IOException { + PhotobucketRipper ripper = new PhotobucketRipper(new URL("http://s844.photobucket.com/user/SpazzySpizzy/library/Album%20Covers?sort=3&page=1")); + testRipper(ripper); + deleteSubdirs(ripper.getWorkingDir()); + deleteDir(ripper.getWorkingDir()); + } + */ +} + + + diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PichunterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PichunterRipperTest.java new file mode 100644 index 00000000..43d70884 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PichunterRipperTest.java @@ -0,0 +1,20 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.PichunterRipper; + +public class PichunterRipperTest extends RippersTest { + public void testPichunterModelPageRip() throws IOException { + // A non-photoset + PichunterRipper ripper = new PichunterRipper(new URL("https://www.pichunter.com/models/Madison_Ivy")); + testRipper(ripper); + } + + public void testPichunterGalleryRip() throws IOException { + // a photo set + PichunterRipper ripper = new PichunterRipper(new URL("http://www.pichunter.com/gallery/3270642/Its_not_only_those_who")); + testRipper(ripper); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java new file mode 100644 index 00000000..96685cbc --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.PornhubRipper; + +public class PornhubRipperTest extends RippersTest { + public void testPornhubRip() throws IOException { + PornhubRipper ripper = new PornhubRipper(new URL("https://www.pornhub.com/album/15680522")); + testRipper(ripper); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java new file mode 100644 index 00000000..49936e17 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java @@ -0,0 +1,24 @@ + +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.RedditRipper; + +public class RedditRipperTest extends RippersTest { + /* + public void testRedditSubredditRip() throws IOException { + RedditRipper ripper = new RedditRipper(new URL("http://www.reddit.com/r/nsfw_oc")); + testRipper(ripper); + } + public void testRedditSubredditTopRip() throws IOException { + RedditRipper ripper = new RedditRipper(new URL("http://www.reddit.com/r/nsfw_oc/top?t=all")); + testRipper(ripper); + } + public void testRedditPostRip() throws IOException { + RedditRipper ripper = new RedditRipper(new URL("http://www.reddit.com/r/UnrealGirls/comments/1ziuhl/in_class_veronique_popa/")); + testRipper(ripper); + } + */ +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SankakuComplexRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SankakuComplexRipperTest.java new file mode 100644 index 00000000..e0027a00 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SankakuComplexRipperTest.java @@ -0,0 +1,19 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.SankakuComplexRipper; + +public class SankakuComplexRipperTest extends RippersTest { + /* + public void testSankakuChanRip() throws IOException { + SankakuComplexRipper ripper = new SankakuComplexRipper(new URL("https://chan.sankakucomplex.com/?tags=cleavage")); + testRipper(ripper); + } + public void testSankakuIdolRip() throws IOException { + SankakuComplexRipper ripper = new SankakuComplexRipper(new URL("https://idol.sankakucomplex.com/?tags=meme_%28me%21me%21me%21%29_%28cosplay%29")); + testRipper(ripper); + } + */ +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ShesFreakyRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ShesFreakyRipperTest.java new file mode 100644 index 00000000..aca763ce --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ShesFreakyRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.ShesFreakyRipper; + +public class ShesFreakyRipperTest extends RippersTest { + public void testShesFreakyRip() throws IOException { + ShesFreakyRipper ripper = new ShesFreakyRipper(new URL("http://www.shesfreaky.com/gallery/nicee-snow-bunny-579NbPjUcYa.html")); + testRipper(ripper); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TapasticRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TapasticRipperTest.java new file mode 100644 index 00000000..efef40cd --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TapasticRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.TapasticRipper; + +public class TapasticRipperTest extends RippersTest { + public void testTapasticRip() throws IOException { + TapasticRipper ripper = new TapasticRipper(new URL("http://tapastic.com/episode/2139")); + testRipper(ripper); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TeenplanetRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TeenplanetRipperTest.java new file mode 100644 index 00000000..aa43103b --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TeenplanetRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.TeenplanetRipper; + +public class TeenplanetRipperTest extends RippersTest { + public void testTeenplanetRip() throws IOException { + TeenplanetRipper ripper = new TeenplanetRipper(new URL("http://teenplanet.org/galleries/the-perfect-side-of-me-6588.html")); + testRipper(ripper); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TheyiffgalleryRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TheyiffgalleryRipperTest.java new file mode 100644 index 00000000..3c7a8fbd --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TheyiffgalleryRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.TheyiffgalleryRipper; + +public class TheyiffgalleryRipperTest extends RippersTest { + public void testTheyiffgallery() throws IOException { + TheyiffgalleryRipper ripper = new TheyiffgalleryRipper(new URL("https://theyiffgallery.com/index?/category/4303")); + testRipper(ripper); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java new file mode 100644 index 00000000..2c0dc405 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java @@ -0,0 +1,24 @@ + +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.TumblrRipper; + +public class TumblrRipperTest extends RippersTest { + /* + public void testTumblrFullRip() throws IOException { + TumblrRipper ripper = new TumblrRipper(new URL("http://wrouinr.tumblr.com/archive")); + testRipper(ripper); + } + public void testTumblrTagRip() throws IOException { + TumblrRipper ripper = new TumblrRipper(new URL("http://topinstagirls.tumblr.com/tagged/berlinskaya")); + testRipper(ripper); + } + public void testTumblrPostRip() throws IOException { + TumblrRipper ripper = new TumblrRipper(new URL("http://sadbaffoon.tumblr.com/post/132045920789/what-a-hoe")); + testRipper(ripper); + } + */ +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwitterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwitterRipperTest.java new file mode 100644 index 00000000..c29a482d --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwitterRipperTest.java @@ -0,0 +1,20 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.TwitterRipper; + +public class TwitterRipperTest extends RippersTest { + public void testTwitterUserRip() throws IOException { + TwitterRipper ripper = new TwitterRipper(new URL("https://twitter.com/danngamber01/media")); + testRipper(ripper); + } + + /* + public void testTwitterSearchRip() throws IOException { + TwitterRipper ripper = new TwitterRipper(new URL("https://twitter.com/search?q=from%3ADaisyfairymfc%20filter%3Aimages&src=typd")); + testRipper(ripper); + } + */ +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwodgalleriesRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwodgalleriesRipperTest.java new file mode 100644 index 00000000..09af04ee --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwodgalleriesRipperTest.java @@ -0,0 +1,16 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.TwodgalleriesRipper; + +public class TwodgalleriesRipperTest extends RippersTest { + /* + public void testTwodgalleriesRip() throws IOException { + TwodgalleriesRipper ripper = new TwodgalleriesRipper(new URL("http://www.2dgalleries.com/artist/regis-loisel-6477")); + testRipper(ripper); + } + */ +} + diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VidbleRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VidbleRipperTest.java new file mode 100644 index 00000000..ccbeb8bb --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VidbleRipperTest.java @@ -0,0 +1,14 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.VidbleRipper; + +public class VidbleRipperTest extends RippersTest { + public void testVidbleRip() throws IOException { + VidbleRipper ripper = new VidbleRipper(new URL("http://www.vidble.com/album/y1oyh3zd")); + testRipper(ripper); + } +} + diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VineRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VineRipperTest.java new file mode 100644 index 00000000..ae8ac402 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VineRipperTest.java @@ -0,0 +1,15 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.VineRipper; + +public class VineRipperTest extends RippersTest { + /* + public void testVineRip() throws IOException { + VineRipper ripper = new VineRipper(new URL("https://vine.co/u/954440445776334848")); + testRipper(ripper); + } + */ +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java new file mode 100644 index 00000000..77c5f44f --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java @@ -0,0 +1,27 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.VkRipper; + +public class VkRipperTest extends RippersTest { + // Not supported (error): https://vk.com/helga_model (Profile Page) + // Not supported (rips nothing): https://vk.com/albums45506334 (all albums under a Profile Page) + // EXAMPLE: https://vk.com/photos45506334 (all photos for a model) + // EXAMPLE: https://vk.com/album45506334_0 (a single album - profile pictures) + // EXAMPLE: https://vk.com/album45506334_00?rev=1 (a single album - wall pictures) + // EXAMPLE: https://vk.com/album45506334_101886701 (a single album - custom) + public void testVkAlbumHttpRip() throws IOException { + VkRipper ripper = new VkRipper(new URL("http://vk.com/album45506334_0")); + testRipper(ripper); + } + public void testVkAlbumHttpsRip() throws IOException { + VkRipper ripper = new VkRipper(new URL("https://vk.com/album45506334_0")); + testRipper(ripper); + } + public void testVkPhotosRip() throws IOException { + VkRipper ripper = new VkRipper(new URL("https://vk.com/photos45506334")); + testRipper(ripper); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java new file mode 100644 index 00000000..74e5818f --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java @@ -0,0 +1,14 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.XhamsterRipper; + +public class XhamsterRipperTest extends RippersTest { + // https://github.com/RipMeApp/ripme/issues/206 : Disabled test : BasicRippersTest#testXhamsterAlbums -- fix and re-enable + // public void testXhamsterAlbums() throws IOException { + // XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.com/photos/gallery/sexy-preggo-girls-9026608")); + // testRipper(ripper); + // } +} From dffe9a40aa536758ef9edc9dd58e47b6c14f4d77 Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 02:02:26 -0800 Subject: [PATCH 02/43] ImageStash.com is permanently down: remove ImagestashRipper --- .../ripper/rippers/ImagestashRipper.java | 87 ------------------- .../ripper/rippers/ImagestashRipperTest.java | 15 ---- 2 files changed, 102 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/ImagestashRipper.java delete mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagestashRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagestashRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagestashRipper.java deleted file mode 100644 index c8cc3d74..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagestashRipper.java +++ /dev/null @@ -1,87 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.json.JSONArray; -import org.json.JSONObject; - -import com.rarchives.ripme.ripper.AbstractJSONRipper; -import com.rarchives.ripme.utils.Http; - -public class ImagestashRipper extends AbstractJSONRipper { - - private int page = 1; - - public ImagestashRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return "imagestash"; - } - @Override - public String getDomain() { - return "imagestash.org"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^.*imagestash.org/tag/([a-zA-Z0-9\\-_]+)$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - throw new MalformedURLException( - "Expected imagestash.org tag formats: " - + "imagestash.org/tag/tagname" - + " Got: " + url); - } - - @Override - public JSONObject getFirstPage() throws IOException { - String baseURL = "https://imagestash.org/images?tags=" - + getGID(url) - + "&page=" + page; - return Http.url(baseURL).getJSON(); - } - - @Override - public JSONObject getNextPage(JSONObject json) throws IOException { - int count = json.getInt("count"), - offset = json.getInt("offset"), - total = json.getInt("total"); - if (count + offset >= total || json.getJSONArray("images").length() == 0) { - throw new IOException("No more images"); - } - sleep(1000); - page++; - return getFirstPage(); - } - - @Override - public List getURLsFromJSON(JSONObject json) { - List imageURLs = new ArrayList<>(); - JSONArray images = json.getJSONArray("images"); - for (int i = 0; i < images.length(); i++) { - JSONObject image = images.getJSONObject(i); - String imageURL = image.getString("src"); - if (imageURL.startsWith("/")) { - imageURL = "https://imagestash.org" + imageURL; - } - imageURLs.add(imageURL); - } - return imageURLs; - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - } -} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagestashRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagestashRipperTest.java deleted file mode 100644 index f6776ef2..00000000 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagestashRipperTest.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import java.io.IOException; -import java.net.URL; - -import com.rarchives.ripme.ripper.rippers.ImagestashRipper; - -public class ImagestashRipperTest extends RippersTest { - /* - public void testImagestashRip() throws IOException { - ImagestashRipper ripper = new ImagestashRipper(new URL("https://imagestash.org/tag/everydayuncensor")); - testRipper(ripper); - } - */ -} From cbf84e2c28e91e0702b1e22a5e0c50899832039f Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 02:02:50 -0800 Subject: [PATCH 03/43] Ignore .class files in vscode --- .vscode/settings.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index bde22831..5a048090 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,7 +2,8 @@ "files.exclude": { "target/**": true, "**/.git": true, - "**/.DS_Store": true + "**/.DS_Store": true, + "**/*.class": true }, "java.configuration.updateBuildConfiguration": "automatic" } From 13f772ef381fcc296d471c85287e736fb33b709f Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 02:03:39 -0800 Subject: [PATCH 04/43] Add comments linking to issues tracking disabled tests. --- .../rarchives/ripme/tst/ripper/rippers/FineboxRipperTest.java | 2 ++ .../rarchives/ripme/tst/ripper/rippers/FlickrRipperTest.java | 1 + .../ripme/tst/ripper/rippers/FuraffinityRipperTest.java | 1 + 3 files changed, 4 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FineboxRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FineboxRipperTest.java index 6c55efd1..6e3dfe4d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FineboxRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FineboxRipperTest.java @@ -6,6 +6,8 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.FineboxRipper; public class FineboxRipperTest extends RippersTest { + // https://github.com/RipMeApp/ripme/issues/242 + /* public void testVineboxAlbum() throws IOException { FineboxRipper ripper = new FineboxRipper(new URL("http://vinebox.co/u/wi57hMjc2Ka")); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FlickrRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FlickrRipperTest.java index 5172d5a4..82ca58d1 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FlickrRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FlickrRipperTest.java @@ -6,6 +6,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.FlickrRipper; public class FlickrRipperTest extends RippersTest { + // https://github.com/RipMeApp/ripme/issues/243 /* public void testFlickrAlbum() throws IOException { FlickrRipper ripper = new FlickrRipper(new URL("https://www.flickr.com/photos/leavingallbehind/sets/72157621895942720/")); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java index a88c7a28..36a4d91c 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java @@ -6,6 +6,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.FuraffinityRipper; public class FuraffinityRipperTest extends RippersTest { + // https://github.com/RipMeApp/ripme/issues/183 /* public void testFuraffinityAlbum() throws IOException { FuraffinityRipper ripper = new FuraffinityRipper(new URL("https://www.furaffinity.net/gallery/mustardgas/")); From be3c09a83e547e730e3bbe904db596c8abacbf2f Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 02:21:06 -0800 Subject: [PATCH 05/43] Add comment linking to issue about MotherlessRipperTest flakiness --- .../rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java index 391cc3c9..6b00f170 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java @@ -6,6 +6,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.MotherlessRipper; public class MotherlessRipperTest extends RippersTest { + // https://github.com/RipMeApp/ripme/issues/238 - MotherlessRipperTest is flaky on Travis CI public void testMotherlessAlbumRip() throws IOException { MotherlessRipper ripper = new MotherlessRipper(new URL("http://motherless.com/G4DAA18D")); testRipper(ripper); From a1a87053095457cd129ab5c8871423ba06b8fb23 Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 02:32:51 -0800 Subject: [PATCH 06/43] Close #229: re-enable PhotobucketRipperTest, pages are now loading just fine (#247) --- .../ripme/tst/ripper/rippers/PhotobucketRipperTest.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PhotobucketRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PhotobucketRipperTest.java index dff101a0..4c4f9781 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PhotobucketRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PhotobucketRipperTest.java @@ -6,16 +6,10 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.PhotobucketRipper; public class PhotobucketRipperTest extends RippersTest { - /* - // https://github.com/RipMeApp/ripme/issues/229 : Disabled test (temporary) : BasicRippersTest#testPhotobucketRip (timing out) public void testPhotobucketRip() throws IOException { PhotobucketRipper ripper = new PhotobucketRipper(new URL("http://s844.photobucket.com/user/SpazzySpizzy/library/Album%20Covers?sort=3&page=1")); testRipper(ripper); deleteSubdirs(ripper.getWorkingDir()); deleteDir(ripper.getWorkingDir()); } - */ } - - - From 98af20828ffe1e744f0b4ca71f12eb6705d1386b Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 02:33:05 -0800 Subject: [PATCH 07/43] Close #226: re-enable ImagefapRipperTest (#246) --- .../ripme/tst/ripper/rippers/ImagefapRipperTest.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java index bd2f5a49..95649015 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java @@ -8,13 +8,9 @@ import java.util.Map; import com.rarchives.ripme.ripper.rippers.ImagefapRipper; public class ImagefapRipperTest extends RippersTest { - public void testImagefapAlbums() throws IOException { Map testURLs = new HashMap<>(); - /* - Temporarily disabled test. See issue https://github.com/RipMeApp/ripme/issues/226 - // Album with specific title testURLs.put(new URL("http://www.imagefap.com/pictures/4649440/Frozen-%28Elsa-and-Anna%29?view=2"), "Frozen (Elsa and Anna)"); @@ -22,12 +18,10 @@ public class ImagefapRipperTest extends RippersTest { // New URL format testURLs.put(new URL("http://www.imagefap.com/gallery.php?pgid=fffd68f659befa5535cf78f014e348f1"), "imagefap_fffd68f659befa5535cf78f014e348f1"); - */ for (URL url : testURLs.keySet()) { ImagefapRipper ripper = new ImagefapRipper(url); testRipper(ripper); } } - } From 657ee130ca1bbd1cdcd1b397b4bd9bfdeb854c70 Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 02:33:23 -0800 Subject: [PATCH 08/43] Re-enable FuskatorRipperTest since there doesn't seem to be anything wrong. (#244) --- .../rarchives/ripme/tst/ripper/rippers/FuskatorRipperTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuskatorRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuskatorRipperTest.java index 290cc4aa..952887e5 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuskatorRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuskatorRipperTest.java @@ -6,10 +6,8 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.FuskatorRipper; public class FuskatorRipperTest extends RippersTest { - /* public void testFuskatorAlbum() throws IOException { FuskatorRipper ripper = new FuskatorRipper(new URL("http://fuskator.com/full/emJa1U6cqbi/index.html")); testRipper(ripper); } - */ } From 686ad8e3d673f0445bebbe27cf843c41c014b37e Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 03:06:52 -0800 Subject: [PATCH 09/43] Add issue links to tests. --- .../rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java | 1 + .../rarchives/ripme/tst/ripper/rippers/TwitterRipperTest.java | 3 ++- .../ripme/tst/ripper/rippers/TwodgalleriesRipperTest.java | 1 + .../rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java | 2 ++ .../com/rarchives/ripme/tst/ripper/rippers/VineRipperTest.java | 1 + .../com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java | 2 ++ 6 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java index 2c0dc405..eb616c4a 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java @@ -7,6 +7,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.TumblrRipper; public class TumblrRipperTest extends RippersTest { + // https://github.com/RipMeApp/ripme/issues/250 /* public void testTumblrFullRip() throws IOException { TumblrRipper ripper = new TumblrRipper(new URL("http://wrouinr.tumblr.com/archive")); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwitterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwitterRipperTest.java index c29a482d..5a40ef02 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwitterRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwitterRipperTest.java @@ -11,9 +11,10 @@ public class TwitterRipperTest extends RippersTest { testRipper(ripper); } + // https://github.com/RipMeApp/ripme/issues/251 /* public void testTwitterSearchRip() throws IOException { - TwitterRipper ripper = new TwitterRipper(new URL("https://twitter.com/search?q=from%3ADaisyfairymfc%20filter%3Aimages&src=typd")); + TwitterRipper ripper = new TwitterRipper(new URL("https://twitter.com/search?f=tweets&q=from%3Aalinalixxx%20filter%3Aimages&src=typd")); testRipper(ripper); } */ diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwodgalleriesRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwodgalleriesRipperTest.java index 09af04ee..a11efbdb 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwodgalleriesRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwodgalleriesRipperTest.java @@ -6,6 +6,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.TwodgalleriesRipper; public class TwodgalleriesRipperTest extends RippersTest { + // https://github.com/RipMeApp/ripme/issues/182 /* public void testTwodgalleriesRip() throws IOException { TwodgalleriesRipper ripper = new TwodgalleriesRipper(new URL("http://www.2dgalleries.com/artist/regis-loisel-6477")); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java index e855fbe4..73c818d0 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java @@ -45,6 +45,7 @@ public class VideoRippersTest extends RippersTest { } } + // https://github.com/RipMeApp/ripme/issues/187 /* public void testPornhubRipper() throws IOException { List contentURLs = new ArrayList<>(); @@ -56,6 +57,7 @@ public class VideoRippersTest extends RippersTest { } */ + // https://github.com/RipMeApp/ripme/issues/186 /* public void testVineRipper() throws IOException { List contentURLs = new ArrayList<>(); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VineRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VineRipperTest.java index ae8ac402..343a72b0 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VineRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VineRipperTest.java @@ -6,6 +6,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.VineRipper; public class VineRipperTest extends RippersTest { + // https://github.com/RipMeApp/ripme/issues/181 /* public void testVineRip() throws IOException { VineRipper ripper = new VineRipper(new URL("https://vine.co/u/954440445776334848")); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java index 77c5f44f..cc15ff99 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java @@ -6,8 +6,10 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.VkRipper; public class VkRipperTest extends RippersTest { + // https://github.com/RipMeApp/ripme/issues/252 // Not supported (error): https://vk.com/helga_model (Profile Page) // Not supported (rips nothing): https://vk.com/albums45506334 (all albums under a Profile Page) + // EXAMPLE: https://vk.com/photos45506334 (all photos for a model) // EXAMPLE: https://vk.com/album45506334_0 (a single album - profile pictures) // EXAMPLE: https://vk.com/album45506334_00?rev=1 (a single album - wall pictures) From 827547451d5b6962b1322a9764f90a4bf71039c4 Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 03:07:20 -0800 Subject: [PATCH 10/43] Re-enable SankakuComplexRipperTest (#249) --- .../ripme/tst/ripper/rippers/SankakuComplexRipperTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SankakuComplexRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SankakuComplexRipperTest.java index e0027a00..59a6101e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SankakuComplexRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SankakuComplexRipperTest.java @@ -6,7 +6,6 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.SankakuComplexRipper; public class SankakuComplexRipperTest extends RippersTest { - /* public void testSankakuChanRip() throws IOException { SankakuComplexRipper ripper = new SankakuComplexRipper(new URL("https://chan.sankakucomplex.com/?tags=cleavage")); testRipper(ripper); @@ -15,5 +14,4 @@ public class SankakuComplexRipperTest extends RippersTest { SankakuComplexRipper ripper = new SankakuComplexRipper(new URL("https://idol.sankakucomplex.com/?tags=meme_%28me%21me%21me%21%29_%28cosplay%29")); testRipper(ripper); } - */ } From b296137b9369447169dd4187864505cd73a6df38 Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 03:07:36 -0800 Subject: [PATCH 11/43] Fix #242: Remove broken and not valuable FineboxRipper (#245) --- .../ripme/ripper/rippers/FineboxRipper.java | 91 ------------------- .../tst/ripper/rippers/FineboxRipperTest.java | 24 ----- 2 files changed, 115 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/FineboxRipper.java delete mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/FineboxRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FineboxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FineboxRipper.java deleted file mode 100644 index a89e2d1b..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FineboxRipper.java +++ /dev/null @@ -1,91 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.jsoup.HttpStatusException; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; - -import com.rarchives.ripme.ripper.AlbumRipper; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; -import com.rarchives.ripme.utils.Http; -import org.jsoup.select.Elements; - -public class FineboxRipper extends AlbumRipper { - - private static final String DOMAIN = "finebox.co", - DOMAIN_OLD = "vinebox.co", - HOST = "finebox"; - - public FineboxRipper(URL url) throws IOException { - super(url); - } - - @Override - public boolean canRip(URL url) { - return url.getHost().endsWith(DOMAIN) || url.getHost().endsWith(DOMAIN_OLD); - } - - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return new URL("http://"+DOMAIN+"/u/" + getGID(url)); - } - - @Override - public void rip() throws IOException { - int page = 0; - Document doc; - Boolean hasPagesLeft = true; - while (hasPagesLeft) { - page++; - String urlPaged = this.url.toExternalForm() + "?page=" + page; - logger.info("Retrieving " + urlPaged); - sendUpdate(STATUS.LOADING_RESOURCE, urlPaged); - try { - doc = Http.url(this.url).get(); - } catch (HttpStatusException e) { - logger.debug("Hit end of pages at page " + page, e); - break; - } - Elements videos = doc.select("video"); - for (Element element : videos) { - String videourl = element.select("source").attr("src"); - if (!videourl.startsWith("http")) { - videourl = "http://" + DOMAIN + videourl; - } - logger.info("URL to download: " + videourl); - if (!addURLToDownload(new URL(videourl))) { - hasPagesLeft = false; - break; - } - } - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - logger.error("[!] Interrupted while waiting to load next page", e); - break; - } - } - waitForThreads(); - } - - @Override - public String getHost() { - return HOST; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://(www\\.)?([vf])inebox\\.co/u/([a-zA-Z0-9]+).*$"); - Matcher m = p.matcher(url.toExternalForm()); - if (!m.matches()) { - throw new MalformedURLException("Expected format: http://"+DOMAIN+"/u/USERNAME"); - } - return m.group(m.groupCount()); - } - -} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FineboxRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FineboxRipperTest.java deleted file mode 100644 index 6e3dfe4d..00000000 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FineboxRipperTest.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import java.io.IOException; -import java.net.URL; - -import com.rarchives.ripme.ripper.rippers.FineboxRipper; - -public class FineboxRipperTest extends RippersTest { - // https://github.com/RipMeApp/ripme/issues/242 - - /* - public void testVineboxAlbum() throws IOException { - FineboxRipper ripper = new FineboxRipper(new URL("http://vinebox.co/u/wi57hMjc2Ka")); - testRipper(ripper); - } - */ - - /* - public void testFineboxAlbum() throws IOException { - FineboxRipper ripper = new FineboxRipper(new URL("http://finebox.co/u/wi57hMjc2Ka")); - testRipper(ripper); - } - */ -} From 164115bbdb7f20e377dfbd7209c73bee4cc608b7 Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 03:17:08 -0800 Subject: [PATCH 12/43] Disable ShesFreakyRipperTest (#254) --- .../ripme/tst/ripper/rippers/ShesFreakyRipperTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ShesFreakyRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ShesFreakyRipperTest.java index aca763ce..3efa31b4 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ShesFreakyRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ShesFreakyRipperTest.java @@ -6,8 +6,11 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.ShesFreakyRipper; public class ShesFreakyRipperTest extends RippersTest { + // https://github.com/RipMeApp/ripme/issues/254 + /* public void testShesFreakyRip() throws IOException { ShesFreakyRipper ripper = new ShesFreakyRipper(new URL("http://www.shesfreaky.com/gallery/nicee-snow-bunny-579NbPjUcYa.html")); testRipper(ripper); } + */ } From f31f943ae709f13f29e49f97a8843956de01e9e4 Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 03:39:53 -0800 Subject: [PATCH 13/43] Disabled flaky SankakuComplexRipperTest (#257) --- .../ripme/tst/ripper/rippers/SankakuComplexRipperTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SankakuComplexRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SankakuComplexRipperTest.java index 59a6101e..6bbc8890 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SankakuComplexRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SankakuComplexRipperTest.java @@ -6,6 +6,8 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.SankakuComplexRipper; public class SankakuComplexRipperTest extends RippersTest { + // https://github.com/RipMeApp/ripme/issues/257 + /* public void testSankakuChanRip() throws IOException { SankakuComplexRipper ripper = new SankakuComplexRipper(new URL("https://chan.sankakucomplex.com/?tags=cleavage")); testRipper(ripper); @@ -14,4 +16,5 @@ public class SankakuComplexRipperTest extends RippersTest { SankakuComplexRipper ripper = new SankakuComplexRipper(new URL("https://idol.sankakucomplex.com/?tags=meme_%28me%21me%21me%21%29_%28cosplay%29")); testRipper(ripper); } + */ } From d64282ed7b3cd617ba64695a11ff9852bbfbb605 Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 03:45:49 -0800 Subject: [PATCH 14/43] Fix #115: Remove GifyoRipper (gifyo.com no longer exists) --- .../ripme/ripper/rippers/GifyoRipper.java | 113 ------------------ 1 file changed, 113 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/GifyoRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GifyoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GifyoRipper.java deleted file mode 100644 index e844e782..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GifyoRipper.java +++ /dev/null @@ -1,113 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.jsoup.Connection.Method; -import org.jsoup.Connection.Response; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; -import com.rarchives.ripme.utils.Http; - -public class GifyoRipper extends AbstractHTMLRipper { - - private int page = 0; - private Map cookies = new HashMap<>(); - - public GifyoRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return "gifyo"; - } - @Override - public String getDomain() { - return "gifyo.com"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://[w.]*gifyo.com/([a-zA-Z0-9\\-_]+)/?$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - throw new MalformedURLException("Gifyo user not found in " + url + ", expected http://gifyo.com/username"); - } - - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return new URL("http://gifyo.com/" + getGID(url) + "/"); - } - - @Override - public Document getFirstPage() throws IOException { - Response resp = Http.url(this.url) - .ignoreContentType() - .response(); - cookies = resp.cookies(); - - Document doc = resp.parse(); - if (doc.html().contains("profile is private")) { - sendUpdate(STATUS.RIP_ERRORED, "User has private profile"); - throw new IOException("User has private profile"); - } - return doc; - } - - @Override - public Document getNextPage(Document doc) throws IOException { - page++; - Map postData = new HashMap<>(); - postData.put("cmd", "refreshData"); - postData.put("view", "gif"); - postData.put("layout", "grid"); - postData.put("page", Integer.toString(page)); - Response resp = Http.url(this.url) - .ignoreContentType() - .data(postData) - .cookies(cookies) - .method(Method.POST) - .response(); - cookies.putAll(resp.cookies()); - Document nextDoc = resp.parse(); - if (nextDoc.select("div.gif img").size() == 0) { - throw new IOException("No more images found"); - } - sleep(2000); - return nextDoc; - } - - @Override - public List getURLsFromPage(Document doc) { - List imageURLs = new ArrayList<>(); - for (Element image : doc.select("img.profile_gif")) { - String imageUrl = image.attr("data-animated"); - if (imageUrl.startsWith("//")) { - imageUrl = "http:" + imageUrl; - } - imageUrl = imageUrl.replace("/medium/", "/large/"); - imageUrl = imageUrl.replace("_s.gif", ".gif"); - imageURLs.add(imageUrl); - } - logger.debug("Found " + imageURLs.size() + " images"); - return imageURLs; - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url); - } -} From cbee58c2bae30a5ba2ae191830ec08d08ba9fb2e Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 03:47:44 -0800 Subject: [PATCH 15/43] Revert "Close #229: re-enable PhotobucketRipperTest, pages are now loading just fine (#247)" This reverts commit a1a87053095457cd129ab5c8871423ba06b8fb23. --- .../ripme/tst/ripper/rippers/PhotobucketRipperTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PhotobucketRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PhotobucketRipperTest.java index 4c4f9781..dff101a0 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PhotobucketRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PhotobucketRipperTest.java @@ -6,10 +6,16 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.PhotobucketRipper; public class PhotobucketRipperTest extends RippersTest { + /* + // https://github.com/RipMeApp/ripme/issues/229 : Disabled test (temporary) : BasicRippersTest#testPhotobucketRip (timing out) public void testPhotobucketRip() throws IOException { PhotobucketRipper ripper = new PhotobucketRipper(new URL("http://s844.photobucket.com/user/SpazzySpizzy/library/Album%20Covers?sort=3&page=1")); testRipper(ripper); deleteSubdirs(ripper.getWorkingDir()); deleteDir(ripper.getWorkingDir()); } + */ } + + + From 30319d1fd29e98544744716801e5617ad12fbd12 Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 04:01:22 -0800 Subject: [PATCH 16/43] Re-enable Reddit tests. (#248) * Re-enable Reddit tests. * Subreddit tests are flaky. --- .../rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java index 49936e17..f20a21f2 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java @@ -7,6 +7,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.RedditRipper; public class RedditRipperTest extends RippersTest { + // https://github.com/RipMeApp/ripme/issues/253 - Disabled tests: RedditRipperTest#testRedditSubreddit*Rip is flaky /* public void testRedditSubredditRip() throws IOException { RedditRipper ripper = new RedditRipper(new URL("http://www.reddit.com/r/nsfw_oc")); @@ -16,9 +17,9 @@ public class RedditRipperTest extends RippersTest { RedditRipper ripper = new RedditRipper(new URL("http://www.reddit.com/r/nsfw_oc/top?t=all")); testRipper(ripper); } + */ public void testRedditPostRip() throws IOException { RedditRipper ripper = new RedditRipper(new URL("http://www.reddit.com/r/UnrealGirls/comments/1ziuhl/in_class_veronique_popa/")); testRipper(ripper); } - */ } From 7ee18bea79bb3d96ecb75bdd4f75c5d50a816019 Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 04:07:47 -0800 Subject: [PATCH 17/43] Remove ButttoucherRipper (butttoucher.com is permanently down) --- .../ripper/rippers/ButttoucherRipper.java | 70 ------------------- .../com/rarchives/ripme/utils/RipUtils.java | 1 - 2 files changed, 71 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/ButttoucherRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ButttoucherRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ButttoucherRipper.java deleted file mode 100644 index 77a25902..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ButttoucherRipper.java +++ /dev/null @@ -1,70 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; - -public class ButttoucherRipper extends AbstractHTMLRipper { - - public ButttoucherRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return "butttoucher"; - } - @Override - public String getDomain() { - return "butttoucher.com"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p; Matcher m; - - p = Pattern.compile("^.*butttoucher.com/users/([a-zA-Z0-9_\\-]+).*$"); - m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - throw new MalformedURLException( - "Expected butttoucher.com gallery format: " - + "butttoucher.com/users/" - + " Got: " + url); - } - - @Override - public Document getFirstPage() throws IOException { - return Http.url(this.url).get(); - } - - @Override - public List getURLsFromPage(Document page) { - List thumbs = new ArrayList<>(); - for (Element thumb : page.select(".thumb img")) { - if (!thumb.hasAttr("src")) { - continue; - } - String smallImage = thumb.attr("src"); - thumbs.add(smallImage.replace("m.", ".")); - } - return thumbs; - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - } - -} \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index 6f3835aa..2788b0bc 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -155,7 +155,6 @@ public class RipUtils { if (url == null) url = urlFromDeviantartDirectoryName(dir); if (url == null) url = urlFromRedditDirectoryName(dir); if (url == null) url = urlFromSiteDirectoryName(dir, "bfcakes", "http://www.bcfakes.com/celebritylist/", ""); - if (url == null) url = urlFromSiteDirectoryName(dir, "butttoucher", "http://butttoucher.com/users/", ""); if (url == null) url = urlFromSiteDirectoryName(dir, "cheeby", "http://cheeby.com/u/", ""); if (url == null) url = urlFromSiteDirectoryName(dir, "datwin", "http://datw.in/", ""); if (url == null) url = urlFromSiteDirectoryName(dir, "drawcrowd", "http://drawcrowd.com/", ""); From 46e2ce074a43f35f60d32986bc9ee832c3d4b0fb Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 04:10:38 -0800 Subject: [PATCH 18/43] Remove CheebyRipper (cheeby.com permanently down) --- .../ripme/ripper/rippers/CheebyRipper.java | 180 ------------------ .../com/rarchives/ripme/utils/RipUtils.java | 1 - 2 files changed, 181 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/CheebyRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/CheebyRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/CheebyRipper.java deleted file mode 100644 index e7f8ef3a..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/CheebyRipper.java +++ /dev/null @@ -1,180 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.File; -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; -import com.rarchives.ripme.utils.Http; - -public class CheebyRipper extends AbstractHTMLRipper { - - private int offset = 0; - private Map albumSets = new HashMap<>(); - - public CheebyRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return "cheeby"; - } - @Override - public String getDomain() { - return "cheeby.com"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://[w.]*cheeby.com/u/([a-zA-Z0-9\\-_]{3,}).*$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - throw new MalformedURLException("cheeby user not found in " + url + ", expected http://cheeby.com/u/username"); - } - - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return new URL("http://cheeby.com/u/" + getGID(url) + "/pics"); - } - - @Override - public Document getFirstPage() throws IOException { - String url = this.url + "?limit=10&offset=0"; - return Http.url(url) - .get(); - } - - @Override - public Document getNextPage(Document doc) throws IOException { - sleep(500); - offset += 1; - String url = this.url + "?p=" + offset; - Document nextDoc = Http.url(url).get(); - if (nextDoc.select("div.i a img").size() == 0) { - throw new IOException("No more images to fetch"); - } - return nextDoc; - } - - @Override - public void downloadURL(URL url, int index) { - // Not implmeneted here - } - - @Override - public List getURLsFromPage(Document page) { - // Not implemented here - return null; - } - - private List getImagesFromPage(Document page) { - List imageURLs = new ArrayList<>(); - for (Element image : page.select("div.i a img")) { - // Get image URL - String imageURL = image.attr("src"); - imageURL = imageURL.replace("s.", "."); - - // Get "album" from image link - String href = image.parent().attr("href"); - while (href.endsWith("/")) { - href = href.substring(0, href.length() - 2); - } - String[] hrefs = href.split("/"); - String prefix = hrefs[hrefs.length - 1]; - - // Keep track of how many images are in this album - int albumSetCount = 0; - if (albumSets.containsKey(prefix)) { - albumSetCount = albumSets.get(prefix); - } - albumSetCount++; - albumSets.put(prefix, albumSetCount); - - imageURLs.add(new Image(imageURL, prefix, albumSetCount)); - - } - return imageURLs; - } - - @Override - public void rip() throws IOException { - logger.info("Retrieving " + this.url); - sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm()); - Document doc = getFirstPage(); - - while (doc != null) { - List images = getImagesFromPage(doc); - - if (images.size() == 0) { - throw new IOException("No images found at " + doc.location()); - } - - for (Image image : images) { - if (isStopped()) { - break; - } - // Don't create subdirectory if "album" only has 1 image - if (albumSets.get(image.prefix) > 1) { - addURLToDownload(new URL(image.url), getPrefix(image.index), image.prefix); - } - else { - addURLToDownload(new URL(image.url)); - } - } - - if (isStopped()) { - break; - } - - try { - sendUpdate(STATUS.LOADING_RESOURCE, "next page"); - doc = getNextPage(doc); - } catch (IOException e) { - logger.info("Can't get next page: " + e.getMessage()); - break; - } - } - - // If they're using a thread pool, wait for it. - if (getThreadPool() != null) { - getThreadPool().waitForThreads(); - } - waitForThreads(); - - // Delete empty subdirectories - for (String prefix : albumSets.keySet()) { - if (prefix.trim().equals("")) { - continue; - } - File f = new File(this.workingDir, prefix); - if (f.list() != null && f.list().length == 0) { - logger.info("Deleting empty directory: " + f.getAbsolutePath()); - f.delete(); - } - } - } - - private class Image { - String url, prefix; - int index; - Image(String url, String prefix, int index) { - this.url = url; - this.prefix = prefix; - this.index = index; - } - } -} diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index 2788b0bc..61538ef3 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -155,7 +155,6 @@ public class RipUtils { if (url == null) url = urlFromDeviantartDirectoryName(dir); if (url == null) url = urlFromRedditDirectoryName(dir); if (url == null) url = urlFromSiteDirectoryName(dir, "bfcakes", "http://www.bcfakes.com/celebritylist/", ""); - if (url == null) url = urlFromSiteDirectoryName(dir, "cheeby", "http://cheeby.com/u/", ""); if (url == null) url = urlFromSiteDirectoryName(dir, "datwin", "http://datw.in/", ""); if (url == null) url = urlFromSiteDirectoryName(dir, "drawcrowd", "http://drawcrowd.com/", ""); if (url == null) url = urlFromSiteDirectoryName(dir.replace("-", "/"), "ehentai", "http://g.e-hentai.org/g/", ""); From 14b35a0072b4509d7fecdbd34f2c8fc9056e9b9e Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 04:13:12 -0800 Subject: [PATCH 19/43] Remove DatwinRipper (datw.in permanently down) --- .../ripme/ripper/rippers/DatwinRipper.java | 65 ------------------- .../com/rarchives/ripme/utils/RipUtils.java | 1 - 2 files changed, 66 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/DatwinRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DatwinRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DatwinRipper.java deleted file mode 100644 index 38708230..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DatwinRipper.java +++ /dev/null @@ -1,65 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; - -public class DatwinRipper extends AbstractHTMLRipper { - - public DatwinRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return "datwin"; - } - @Override - public String getDomain() { - return "datw.in"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^.*datw.in/([a-zA-Z0-9\\-_]+).*$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - throw new MalformedURLException( - "Expected datw.in gallery formats: " - + "datw.in/..." - + " Got: " + url); - } - - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } - - @Override - public List getURLsFromPage(Document doc) { - List imageURLs = new ArrayList<>(); - for (Element thumb : doc.select("img.attachment-thumbnail")) { - String image = thumb.attr("src"); - image = image.replaceAll("-\\d{1,3}x\\d{1,3}", ""); - imageURLs.add(image); - } - return imageURLs; - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - } -} diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index 61538ef3..7d33d097 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -155,7 +155,6 @@ public class RipUtils { if (url == null) url = urlFromDeviantartDirectoryName(dir); if (url == null) url = urlFromRedditDirectoryName(dir); if (url == null) url = urlFromSiteDirectoryName(dir, "bfcakes", "http://www.bcfakes.com/celebritylist/", ""); - if (url == null) url = urlFromSiteDirectoryName(dir, "datwin", "http://datw.in/", ""); if (url == null) url = urlFromSiteDirectoryName(dir, "drawcrowd", "http://drawcrowd.com/", ""); if (url == null) url = urlFromSiteDirectoryName(dir.replace("-", "/"), "ehentai", "http://g.e-hentai.org/g/", ""); if (url == null) url = urlFromSiteDirectoryName(dir, "fapproved", "http://fapproved.com/users/", ""); From 0afe0377f2cc21c15237447a0e230dc13fe09d3f Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 04:19:04 -0800 Subject: [PATCH 20/43] Removed FapprovedRipper (permanently down) --- .../ripme/ripper/rippers/FapprovedRipper.java | 101 ------------------ .../com/rarchives/ripme/utils/RipUtils.java | 1 - 2 files changed, 102 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/FapprovedRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FapprovedRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FapprovedRipper.java deleted file mode 100644 index e630b3a3..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FapprovedRipper.java +++ /dev/null @@ -1,101 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; - -public class FapprovedRipper extends AbstractHTMLRipper { - - private int pageIndex = 1; - private String username = null; - - public FapprovedRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return "fapproved"; - } - @Override - public String getDomain() { - return "fapproved.com"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://[w.]*fapproved.com/users/([a-zA-Z0-9\\-_]{3,}).*$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - username = m.group(1); - return username; - } - throw new MalformedURLException("Fapproved user not found in " + url + ", expected http://fapproved.com/users/username/images"); - } - - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return new URL("http://fapproved.com/users/" + getGID(url)); - } - - @Override - public Document getFirstPage() throws IOException { - pageIndex = 1; - String pageURL = getPageURL(pageIndex); - return Http.url(pageURL) - .ignoreContentType() - .get(); - } - - @Override - public Document getNextPage(Document doc) throws IOException { - if ((doc.select("div.pagination li.next.disabled").size() != 0) - || (doc.select("div.pagination").size() == 0)) { - throw new IOException("No more pages found"); - } - sleep(1000); - pageIndex++; - String pageURL = getPageURL(pageIndex); - return Http.url(pageURL) - .ignoreContentType() - .get(); - } - - private String getPageURL(int index) throws IOException { - if (username == null) { - username = getGID(this.url); - } - return "http://fapproved.com/users/" + username + "/images?page=" + pageIndex; - } - - @Override - public List getURLsFromPage(Document page) { - List imageURLs = new ArrayList<>(); - for (Element image : page.select("div.actual-image img")) { - String imageURL = image.attr("src"); - if (imageURL.startsWith("//")) { - imageURL = "http:" + imageURL; - } - else if (imageURL.startsWith("/")) { - imageURL = "http://fapproved.com" + imageURL; - } - imageURLs.add(imageURL); - } - return imageURLs; - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - } -} diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index 7d33d097..b7b8c239 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -157,7 +157,6 @@ public class RipUtils { if (url == null) url = urlFromSiteDirectoryName(dir, "bfcakes", "http://www.bcfakes.com/celebritylist/", ""); if (url == null) url = urlFromSiteDirectoryName(dir, "drawcrowd", "http://drawcrowd.com/", ""); if (url == null) url = urlFromSiteDirectoryName(dir.replace("-", "/"), "ehentai", "http://g.e-hentai.org/g/", ""); - if (url == null) url = urlFromSiteDirectoryName(dir, "fapproved", "http://fapproved.com/users/", ""); if (url == null) url = urlFromSiteDirectoryName(dir, "vinebox", "http://finebox.co/u/", ""); if (url == null) url = urlFromSiteDirectoryName(dir, "imgbox", "http://imgbox.com/g/", ""); if (url == null) url = urlFromSiteDirectoryName(dir, "modelmayhem", "http://www.modelmayhem.com/", ""); From d1aad83d28497c9afc800995499ee8dc8ed55cdb Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 04:23:21 -0800 Subject: [PATCH 21/43] Removed MediacrushRipper (permanently down) --- .../ripper/rippers/MediacrushRipper.java | 166 ------------------ 1 file changed, 166 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/MediacrushRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MediacrushRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MediacrushRipper.java deleted file mode 100644 index 687ca509..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MediacrushRipper.java +++ /dev/null @@ -1,166 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.awt.Desktop; -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import javax.net.ssl.SSLException; -import javax.swing.JOptionPane; - -import org.json.JSONArray; -import org.json.JSONObject; - -import com.rarchives.ripme.ripper.AbstractJSONRipper; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; -import com.rarchives.ripme.utils.Http; - -public class MediacrushRipper extends AbstractJSONRipper { - - /** Ordered list of preferred formats, sorted by preference (low-to-high) */ - private static final Map PREFERRED_FORMATS = new HashMap<>(); - static { - PREFERRED_FORMATS.put("mp4", 0); - PREFERRED_FORMATS.put("wemb",1); - PREFERRED_FORMATS.put("ogv", 2); - PREFERRED_FORMATS.put("mp3", 3); - PREFERRED_FORMATS.put("ogg", 4); - PREFERRED_FORMATS.put("gif", 5); - PREFERRED_FORMATS.put("png", 6); - PREFERRED_FORMATS.put("jpg", 7); - PREFERRED_FORMATS.put("jpeg",8); - } - - public MediacrushRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return "mediacrush"; - } - @Override - public String getDomain() { - return "mediacru.sh"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("https?://[wm.]*mediacru\\.sh/([a-zA-Z0-9]+).*"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - throw new MalformedURLException("Could not find mediacru.sh page ID from " + url - + " expected format: http://mediacru.sh/pageid"); - } - - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - String u = url.toExternalForm(); - // Strip trailing "/" characters - while (u.endsWith("/")) { - u = u.substring(0, u.length() - 1); - } - // Append .json - if (!u.endsWith(".json")) { - u += ".json"; - } - return new URL(u); - } - - @Override - public JSONObject getFirstPage() throws IOException { - try { - String jsonString = Http.url(url) - .ignoreContentType() - .connection() - .execute().body(); - jsonString = jsonString.replace(""", "\""); - return new JSONObject(jsonString); - } catch (SSLException re) { - // Check for >1024 bit encryption but in older versions of Java - // It's the bug. Suggest downloading the latest version. - int selection = JOptionPane.showOptionDialog(null, - "You need to upgrade to the latest Java (7+) to rip this album.\n" - + "Do you want to open java.com and download the latest version?", - "RipMe - Java Error", - JOptionPane.OK_CANCEL_OPTION, - JOptionPane.ERROR_MESSAGE, - null, - new String[] {"Go to java.com", "Cancel"}, - 0); - sendUpdate(STATUS.RIP_ERRORED, "Your version of Java can't handle some secure websites"); - if (selection == 0) { - URL javaUrl = new URL("https://www.java.com/en/download/"); - try { - Desktop.getDesktop().browse(javaUrl.toURI()); - } catch (URISyntaxException use) { } - } - throw new IOException("Cannot rip due to limitations in Java installation, consider upgrading Java", re.getCause()); - } - catch (Exception e) { - throw new IOException("Unexpected error: " + e.getMessage(), e); - } - } - - @Override - public List getURLsFromJSON(JSONObject json) { - List imageURLs = new ArrayList<>(); - // Iterate over all files - JSONArray files = json.getJSONArray("files"); - for (int i = 0; i < files.length(); i++) { - JSONObject file = (JSONObject) files.get(i); - // Find preferred file format - JSONArray subfiles = file.getJSONArray("files"); - String preferredUrl = getPreferredUrl(subfiles); - if (preferredUrl == null) { - logger.warn("Could not find 'file' inside of " + file); - sendUpdate(STATUS.DOWNLOAD_ERRORED, "Could not find file inside of " + file); - continue; - } - - imageURLs.add(preferredUrl); - } - return imageURLs; - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - } - - /** - * Iterates over list if "file" objects and returns the preferred - * image format. - * @param subfiles Array of "files" (JSONObjects) which contain - * @return Preferred media format. - */ - private String getPreferredUrl(JSONArray subfiles) { - String preferredUrl = null; - int preferredIndex = Integer.MAX_VALUE; - // Iterate over all media types - for (int j = 0; j < subfiles.length(); j++) { - JSONObject subfile = subfiles.getJSONObject(j); - String thisurl = subfile.getString("url"); - String extension = thisurl.substring(thisurl.lastIndexOf(".") + 1); - if (!PREFERRED_FORMATS.containsKey(extension)) { - continue; - } - // Keep track of the most-preferred format - int thisindex = PREFERRED_FORMATS.get(extension); - if (preferredUrl == null || thisindex < preferredIndex) { - preferredIndex = thisindex; - preferredUrl = thisurl; - } - } - return preferredUrl; - } -} From 24a05f736e6fa6f62f474340388f920907212067 Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 04:34:05 -0800 Subject: [PATCH 22/43] Re #206: Update XhamsterRipperTest (cannot re-enable, still flaky in CI) --- .../tst/ripper/rippers/XhamsterRipperTest.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java index 74e5818f..fa6f1b24 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java @@ -6,9 +6,17 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.XhamsterRipper; public class XhamsterRipperTest extends RippersTest { - // https://github.com/RipMeApp/ripme/issues/206 : Disabled test : BasicRippersTest#testXhamsterAlbums -- fix and re-enable - // public void testXhamsterAlbums() throws IOException { - // XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.com/photos/gallery/sexy-preggo-girls-9026608")); - // testRipper(ripper); - // } + // https://github.com/RipMeApp/ripme/issues/206 - Disabled tests: XhamsterRipperTest test cases are flaky in Travis CI + + /* + public void testXhamsterAlbum1() throws IOException { + XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.com/photos/gallery/sexy-preggo-girls-9026608")); + testRipper(ripper); + } + + public void testXhamsterAlbum2() throws IOException { + XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.com/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664")); + testRipper(ripper); + } + */ } From 1000fb78e8aaf37d92419eccac9cd958f301f043 Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 04:44:45 -0800 Subject: [PATCH 23/43] Removed SupertangasRipper (no longer any static content to download) --- .../ripper/rippers/SupertangasRipper.java | 95 ------------------- 1 file changed, 95 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/SupertangasRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SupertangasRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SupertangasRipper.java deleted file mode 100644 index d8fb6b26..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SupertangasRipper.java +++ /dev/null @@ -1,95 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.jsoup.HttpStatusException; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -import com.rarchives.ripme.ripper.AlbumRipper; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; -import com.rarchives.ripme.utils.Http; - -/** - * Appears to be broken as of 2015-02-11. - * Looks like supertangas changed their site completely. - */ -public class SupertangasRipper extends AlbumRipper { - - private static final String DOMAIN = "supertangas.com", - HOST = "supertangas"; - - public SupertangasRipper(URL url) throws IOException { - super(url); - } - - @Override - public boolean canRip(URL url) { - return url.getHost().endsWith(DOMAIN); - } - - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return url; - } - - @Override - public void rip() throws IOException { - int page = 0; - String baseURL = "http://www.supertangas.com/fotos/?level=search&exact=1&searchterms=" + this.getGID(this.url); - Document doc; - while (true) { - page++; - String theURL = baseURL; - if (page > 1) { - theURL += "&plog_page=" + page; - } - try { - logger.info(" Retrieving " + theURL); - sendUpdate(STATUS.LOADING_RESOURCE, theURL); - doc = Http.url(theURL).get(); - } catch (HttpStatusException e) { - logger.debug("Hit end of pages at page " + page, e); - break; - } - Elements images = doc.select("li.thumbnail a"); - if (images.size() == 0) { - break; - } - for (Element imageElement : images) { - String image = imageElement.attr("href"); - image = image.replaceAll("/fotos/", "/fotos/images/"); - addURLToDownload(new URL(image)); - } - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - logger.error("[!] Interrupted while waiting to load next page", e); - break; - } - } - waitForThreads(); - } - - @Override - public String getHost() { - return HOST; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - // http://www.supertangas.com/fotos/?level=search&exact=1&searchterms=Tahiticora%20(France) - Pattern p = Pattern.compile("^https?://[w.]*supertangas\\.com/fotos/\\?.*&searchterms=([a-zA-Z0-9%()+]+).*$"); - Matcher m = p.matcher(url.toExternalForm()); - if (!m.matches()) { - throw new MalformedURLException("Expected format: http://supertangas.com/fotos/?level=search&exact=1&searchterms=..."); - } - return m.group(m.groupCount()); - } - -} From b9245a04324f31d02c48a8ba28427866d1b6323a Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 21 Nov 2017 09:22:35 -0500 Subject: [PATCH 24/43] Added hentaicafe ripper unit test --- .../tst/ripper/rippers/HentaicafeRipperTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaicafeRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaicafeRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaicafeRipperTest.java new file mode 100644 index 00000000..eae13f73 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaicafeRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.HentaiCafeRipper; + +public class HentaicafeRipperTest extends RippersTest { + public void testHentaiCafeAlbum() throws IOException { + HentaiCafeRipper ripper = new HentaiCafeRipper(new URL("https://hentai.cafe/kikuta-the-oni-in-the-room/")); + testRipper(ripper); + } +} From e69b32d8a5c6bd0b977d103413509477750fb874 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 21 Nov 2017 09:24:45 -0500 Subject: [PATCH 25/43] Added unit test for myhentaicomics --- .../ripper/rippers/MyhentaicomicsRipperTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java new file mode 100644 index 00000000..193567f2 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.MyhentaicomicsRipper; + +public class MyhentaicomicsRipperTest extends RippersTest { + public void testHentaiCafeAlbum() throws IOException { + MyhentaicomicsRipper ripper = new MyhentaicomicsRipper(new URL("http://myhentaicomics.com/index.php/Nienna-Lost-Tales")); + testRipper(ripper); + } +} \ No newline at end of file From 2beffeaf0d1280a1339c732a15c8bb0282a20164 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 21 Nov 2017 09:27:13 -0500 Subject: [PATCH 26/43] Added unit test for Ehentai --- .../ripme/tst/ripper/rippers/EhentaiRipperTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java new file mode 100644 index 00000000..9c97813e --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.EHentaiRipper; + +public class EhentaiRipperTest extends RippersTest { + public void testHentaiCafeAlbum() throws IOException { + EHentaiRipper ripper = new EHentaiRipper(new URL("https://e-hentai.org/g/1144492/e823bdf9a5/")); + testRipper(ripper); + } +} \ No newline at end of file From 3a0d6c1213bcd5d1ba17d600f3dcbd4a92b8a5f4 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 21 Nov 2017 09:31:07 -0500 Subject: [PATCH 27/43] Fixed class naming error --- .../ripme/tst/ripper/rippers/EhentaiRipperTest.java | 2 +- .../ripper/rippers/MyhentaicomicsRipperTest.java | 2 +- .../tst/ripper/rippers/SinnercomicsRipperTest.java | 13 +++++++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinnercomicsRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java index 9c97813e..00a3f8b6 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java @@ -6,7 +6,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.EHentaiRipper; public class EhentaiRipperTest extends RippersTest { - public void testHentaiCafeAlbum() throws IOException { + public void testEHentaiAlbum() throws IOException { EHentaiRipper ripper = new EHentaiRipper(new URL("https://e-hentai.org/g/1144492/e823bdf9a5/")); testRipper(ripper); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java index 193567f2..6714195d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java @@ -6,7 +6,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.MyhentaicomicsRipper; public class MyhentaicomicsRipperTest extends RippersTest { - public void testHentaiCafeAlbum() throws IOException { + public void testMyhentaicomicsAlbum() throws IOException { MyhentaicomicsRipper ripper = new MyhentaicomicsRipper(new URL("http://myhentaicomics.com/index.php/Nienna-Lost-Tales")); testRipper(ripper); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinnercomicsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinnercomicsRipperTest.java new file mode 100644 index 00000000..d1ce6b33 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinnercomicsRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.SinnercomicsRipper; + +public class SinnercomicsRipperTest extends RippersTest { + public void testSinnercomicsAlbum() throws IOException { + SinnercomicsRipper ripper = new SinnercomicsRipper(new URL("https://sinnercomics.com/comic/beyond-the-hotel-page-01/")); + testRipper(ripper); + } +} \ No newline at end of file From 6993007fd6b81014a97209337ca962154b14d97e Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 21 Nov 2017 09:34:03 -0500 Subject: [PATCH 28/43] Added unit test for porncomix ripper --- .../tst/ripper/rippers/PorncomixRipperTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixRipperTest.java new file mode 100644 index 00000000..7abe6e1e --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.PorncomixRipper; + +public class PorncomixRipperTest extends RippersTest { + public void testPorncomixAlbum() throws IOException { + PorncomixRipper ripper = new PorncomixRipper(new URL("http://www.porncomix.info/lust-unleashed-desire-to-submit/")); + testRipper(ripper); + } +} \ No newline at end of file From 9b35916eca90e350cb9375b3fc4c44eb4a56cf62 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 21 Nov 2017 09:48:44 -0500 Subject: [PATCH 29/43] Added hentai2read unit test --- .../tst/ripper/rippers/Hentai2readRipperTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java new file mode 100644 index 00000000..144606fc --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.Hentai2readRipper; + +public class Hentai2readRipperTest extends RippersTest { + public void testHentai2readAlbum() throws IOException { + Hentai2readRipper ripper = new Hentai2readRipper(new URL("https://hentai2read.com/sm_school_memorial/")); + testRipper(ripper); + } +} \ No newline at end of file From 0de0db71831c7c17612de2f89ec669973daa61c7 Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 12:08:08 -0800 Subject: [PATCH 30/43] Add AerisdiesRipperTest (#260) --- .../ripper/rippers/AerisdiesRipperTest.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java new file mode 100644 index 00000000..fb991ec7 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java @@ -0,0 +1,20 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.AerisdiesRipper;; + +public class AerisdiesRipperTest extends RippersTest { + public void testAlbum() throws IOException { + AerisdiesRipper ripper = new AerisdiesRipper(new URL("http://www.aerisdies.com/html/lb/alb_1097_1.html")); + testRipper(ripper); + } + + public void testSubAlbum() throws IOException { + AerisdiesRipper ripper = new AerisdiesRipper(new URL("http://www.aerisdies.com/html/lb/alb_3692_1.html")); + testRipper(ripper); + } + + // TODO: Add a test for an album with a title. +} From 7b912dfa6402e825fdbe6083e195adb9629d6f36 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 21 Nov 2017 15:12:13 -0500 Subject: [PATCH 31/43] Improved webtoons folder naming (#265) --- .../java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java index 57fda38f..f7b74133 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java @@ -50,7 +50,7 @@ public class WebtoonsRipper extends AbstractHTMLRipper { Pattern pat = Pattern.compile("https?://www.webtoons.com/[a-zA-Z]+/[a-zA-Z]+/([a-zA-Z0-9_-]*)/[a-zA-Z0-9_-]+/\\S*"); Matcher mat = pat.matcher(url.toExternalForm()); if (mat.matches()) { - return mat.group(1); + return getHost() + "_" + mat.group(1); } return super.getAlbumTitle(url); From 4f12914da4ffc3b6c3744e0342457e3a64810482 Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 12:22:23 -0800 Subject: [PATCH 32/43] 1.7.0: Improved webtoons folder naming; Added code coverage with Coveralls.io; improved unit tests and code coverage; removed rippers for dead sites --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index b661de5a..5034fd34 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.6.13 + 1.7.0 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index d5a5baa4..c51eaa08 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.6.13", + "latestVersion": "1.7.0", "changeList": [ + "1.7.0: Improved webtoons folder naming; Added code coverage with Coveralls.io; improved unit tests and code coverage; removed rippers for dead sites", "1.6.13: Added support for ripping from instagram tags; Instagram regex now matches all ripable urls; improved regex for pichunter", "1.6.12: Fix InstagramRipper with timestamps; Pichunter galleries support; logging improvements", "1.6.11: dded pichunter.com ripper; Changed instagram output name format; added tehyiffgallery ripper; Fixed xchan ripper; Fixed regession in chanRipper folder naming", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 448d0f77..92152c4c 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.6.13"; + private static final String DEFAULT_VERSION = "1.7.0"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From ab600e1fa8870e9705f99b2ebec67649ed642991 Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 12:29:57 -0800 Subject: [PATCH 33/43] Add WebtoonsRipperTest (#268) --- .../tst/ripper/rippers/WebtoonsRipperTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java new file mode 100644 index 00000000..20cf17b8 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.WebtoonsRipper; + +public class WebtoonsRipperTest extends RippersTest { + public void testWebtoonsAlbum() throws IOException { + WebtoonsRipper ripper = new WebtoonsRipper(new URL("http://www.webtoons.com/en/drama/my-boo/ep-33/viewer?title_no=1185&episode_no=33")); + testRipper(ripper); + } +} From f630123a6eb23cc73122a225bea447a84dbbb4b2 Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 12:46:58 -0800 Subject: [PATCH 34/43] Fix WordpressComicRipper's ripper for freeadultcomix.com (#270) --- .../rarchives/ripme/ripper/rippers/WordpressComicRipper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java index e8405fd8..7809db7a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java @@ -275,9 +275,10 @@ public class WordpressComicRipper extends AbstractHTMLRipper { } // freeadultcomix gets it own if because it needs to add http://freeadultcomix.com to the start of each link + // TODO review the above comment which no longer applies -- see if there's a refactoring we should do here. if (url.toExternalForm().contains("freeadultcomix.com")) { for (Element elem : doc.select("div.single-post > p > img.aligncenter")) { - result.add("http://freeadultcomix.com" + elem.attr("src")); + result.add(elem.attr("src")); } } From bbb4770a11a23beb6db1982549e7f0480a1f8c2e Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 11:45:05 -0800 Subject: [PATCH 35/43] Add WordpressComicRipperTest --- .../ripper/rippers/WordpressComicRipper.java | 2 +- .../rippers/WordpressComicRipperTest.java | 84 +++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java index 7809db7a..fbb628a9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java @@ -22,7 +22,7 @@ public class WordpressComicRipper extends AbstractHTMLRipper { super(url); } - // Test links: + // Test links (see also WordpressComicRipperTest.java) // http://www.totempole666.com/comic/first-time-for-everything-00-cover/ // http://buttsmithy.com/archives/comic/p1 // http://themonsterunderthebed.net/?comic=test-post diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java new file mode 100644 index 00000000..6aa0cd77 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java @@ -0,0 +1,84 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.WordpressComicRipper; + +public class WordpressComicRipperTest extends RippersTest { + // Test links (see also WordpressComicRipper.java) + // http://www.totempole666.com/comic/first-time-for-everything-00-cover/ + // http://buttsmithy.com/archives/comic/p1 + // http://themonsterunderthebed.net/?comic=test-post + // http://prismblush.com/comic/hella-trap-pg-01/ + // http://www.konradokonski.com/sawdust/ + // http://www.konradokonski.com/wiory/ + // http://freeadultcomix.com/finders-feepaid-in-full-sparrow/ + // http://comics-xxx.com/republic-rendezvous-palcomix-star-wars-xxx/ + // http://tnbtu.com/comic/01-00/ + // http://shipinbottle.pepsaga.com/?p=281 + + public void test_totempole666() throws IOException { + WordpressComicRipper ripper = new WordpressComicRipper( + new URL("http://www.totempole666.com/comic/first-time-for-everything-00-cover/")); + testRipper(ripper); + } + + public void test_buttsmithy() throws IOException { + WordpressComicRipper ripper = new WordpressComicRipper( + new URL("http://buttsmithy.com/archives/comic/p1")); + testRipper(ripper); + } + + public void test_themonsterunderthebed() throws IOException { + WordpressComicRipper ripper = new WordpressComicRipper( + new URL("http://themonsterunderthebed.net/?comic=test-post")); + testRipper(ripper); + } + + public void test_prismblush() throws IOException { + WordpressComicRipper ripper = new WordpressComicRipper( + new URL("http://prismblush.com/comic/hella-trap-pg-01/")); + testRipper(ripper); + } + + /* + // https://github.com/RipMeApp/ripme/issues/266 - WordpressRipper: konradokonski.com previously supported but now cannot rip + + public void test_konradokonski_1() throws IOException { + WordpressComicRipper ripper = new WordpressComicRipper( + new URL("http://www.konradokonski.com/sawdust/")); + testRipper(ripper); + } + + public void test_konradokonski_2() throws IOException { + WordpressComicRipper ripper = new WordpressComicRipper( + new URL("http://www.konradokonski.com/wiory/")); + testRipper(ripper); + } + */ + + public void test_freeadultcomix() throws IOException { + WordpressComicRipper ripper = new WordpressComicRipper( + new URL("http://freeadultcomix.com/finders-feepaid-in-full-sparrow/")); + testRipper(ripper); + } + + public void test_comicsxxx() throws IOException { + WordpressComicRipper ripper = new WordpressComicRipper( + new URL("http://comics-xxx.com/republic-rendezvous-palcomix-star-wars-xxx/")); + testRipper(ripper); + } + + public void test_tnbtu() throws IOException { + WordpressComicRipper ripper = new WordpressComicRipper( + new URL("http://tnbtu.com/comic/01-00/")); + testRipper(ripper); + } + + public void test_pepsaga() throws IOException { + WordpressComicRipper ripper = new WordpressComicRipper( + new URL("http://shipinbottle.pepsaga.com/?p=281")); + testRipper(ripper); + } +} From bb0678644c813775f456779bf99ef9117e4111d4 Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 11:47:12 -0800 Subject: [PATCH 36/43] comics-xxx.com no longer exists --- .../ripme/ripper/rippers/WordpressComicRipper.java | 2 -- .../ripme/tst/ripper/rippers/WordpressComicRipperTest.java | 7 ------- 2 files changed, 9 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java index fbb628a9..b7a41be8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java @@ -30,7 +30,6 @@ public class WordpressComicRipper extends AbstractHTMLRipper { // http://www.konradokonski.com/sawdust/ // http://www.konradokonski.com/wiory/ // http://freeadultcomix.com/finders-feepaid-in-full-sparrow/ - // http://comics-xxx.com/republic-rendezvous-palcomix-star-wars-xxx/ // http://tnbtu.com/comic/01-00/ // http://shipinbottle.pepsaga.com/?p=281 @@ -42,7 +41,6 @@ public class WordpressComicRipper extends AbstractHTMLRipper { "www.konradokonski.com", "freeadultcomix.com", "thisis.delvecomic.com", - "comics-xxx.com", "tnbtu.com", "shipinbottle.pepsaga.com" ); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java index 6aa0cd77..a3094bbe 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java @@ -14,7 +14,6 @@ public class WordpressComicRipperTest extends RippersTest { // http://www.konradokonski.com/sawdust/ // http://www.konradokonski.com/wiory/ // http://freeadultcomix.com/finders-feepaid-in-full-sparrow/ - // http://comics-xxx.com/republic-rendezvous-palcomix-star-wars-xxx/ // http://tnbtu.com/comic/01-00/ // http://shipinbottle.pepsaga.com/?p=281 @@ -64,12 +63,6 @@ public class WordpressComicRipperTest extends RippersTest { testRipper(ripper); } - public void test_comicsxxx() throws IOException { - WordpressComicRipper ripper = new WordpressComicRipper( - new URL("http://comics-xxx.com/republic-rendezvous-palcomix-star-wars-xxx/")); - testRipper(ripper); - } - public void test_tnbtu() throws IOException { WordpressComicRipper ripper = new WordpressComicRipper( new URL("http://tnbtu.com/comic/01-00/")); From 9632fcd554e9737484cf0621d8fbb8b1c6691d84 Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 11:56:30 -0800 Subject: [PATCH 37/43] Add test for thisis.delvecomic.com --- .../ripme/ripper/rippers/WordpressComicRipper.java | 1 + .../ripme/tst/ripper/rippers/WordpressComicRipperTest.java | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java index b7a41be8..6c19e704 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java @@ -30,6 +30,7 @@ public class WordpressComicRipper extends AbstractHTMLRipper { // http://www.konradokonski.com/sawdust/ // http://www.konradokonski.com/wiory/ // http://freeadultcomix.com/finders-feepaid-in-full-sparrow/ + // http://thisis.delvecomic.com/NewWP/comic/in-too-deep/ // http://tnbtu.com/comic/01-00/ // http://shipinbottle.pepsaga.com/?p=281 diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java index a3094bbe..532009fc 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java @@ -14,6 +14,7 @@ public class WordpressComicRipperTest extends RippersTest { // http://www.konradokonski.com/sawdust/ // http://www.konradokonski.com/wiory/ // http://freeadultcomix.com/finders-feepaid-in-full-sparrow/ + // http://thisis.delvecomic.com/NewWP/comic/in-too-deep/ // http://tnbtu.com/comic/01-00/ // http://shipinbottle.pepsaga.com/?p=281 @@ -63,6 +64,12 @@ public class WordpressComicRipperTest extends RippersTest { testRipper(ripper); } + public void test_delvecomic() throws IOException { + WordpressComicRipper ripper = new WordpressComicRipper( + new URL("http://thisis.delvecomic.com/NewWP/comic/in-too-deep/")); + testRipper(ripper); + } + public void test_tnbtu() throws IOException { WordpressComicRipper ripper = new WordpressComicRipper( new URL("http://tnbtu.com/comic/01-00/")); From 0b3f259873353666629d2d3290a38c12de7320cd Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 11 Nov 2017 08:32:56 -0500 Subject: [PATCH 38/43] Started on fixing furaffinity ripper --- .../ripper/rippers/FuraffinityRipper.java | 59 ++++++------------- 1 file changed, 17 insertions(+), 42 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java index 9c4b89aa..a12912a5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java @@ -55,15 +55,12 @@ public class FuraffinityRipper extends AbstractHTMLRipper { } @Override public boolean hasDescriptionSupport() { - return true; + return false; } @Override public Document getFirstPage() throws IOException { - if (cookies == null || cookies.size() == 0) { - login(); - } - return Http.url(url).cookies(cookies).get(); + return Http.url(url).get(); } private void login() throws IOException { @@ -84,7 +81,6 @@ public class FuraffinityRipper extends AbstractHTMLRipper { Response doLogin = Http.url(urlBase + "/login/?ref=" + url) .referrer(urlBase + "/login/") - .cookies(cookies) .data(formData) .method(Method.POST) .response(); @@ -101,7 +97,7 @@ public class FuraffinityRipper extends AbstractHTMLRipper { String nextUrl = urlBase + nextPageUrl.first().attr("action"); sleep(500); - Document nextPage = Http.url(nextUrl).cookies(cookies).get(); + Document nextPage = Http.url(nextUrl).get(); Elements hrefs = nextPage.select("div#no-images"); if (hrefs.size() != 0) { @@ -110,12 +106,21 @@ public class FuraffinityRipper extends AbstractHTMLRipper { return nextPage; } + private String getImageFromPost(String url) { + try { + logger.info("found url " + Http.url(url).get().select("meta[property=og:image]").attr("content")); + return Http.url(url).get().select("meta[property=og:image]").attr("content"); + } catch (IOException e) { + return ""; + } + } + @Override public List getURLsFromPage(Document page) { List urls = new ArrayList<>(); Elements urlElements = page.select("figure.t-image > b > u > a"); for (Element e : urlElements) { - urls.add(urlBase + e.select("a").first().attr("href")); + urls.add(getImageFromPost(urlBase + e.select("a").first().attr("href"))); } return urls; } @@ -138,7 +143,6 @@ public class FuraffinityRipper extends AbstractHTMLRipper { // Fetch the image page Response resp = Http.url(page) .referrer(this.url) - .cookies(cookies) .response(); cookies.putAll(resp.cookies()); @@ -210,8 +214,7 @@ public class FuraffinityRipper extends AbstractHTMLRipper { } @Override public void downloadURL(URL url, int index) { - furaffinityThreadPool.addThread(new FuraffinityDocumentThread(url)); - sleep(250); + addURLToDownload(url, getPrefix(index)); } @Override @@ -222,6 +225,7 @@ public class FuraffinityRipper extends AbstractHTMLRipper { if (m.matches()) { return m.group(1); } + throw new MalformedURLException("Expected furaffinity.net URL format: " + "www.furaffinity.net/gallery/username - got " + url + " instead"); @@ -235,37 +239,8 @@ public class FuraffinityRipper extends AbstractHTMLRipper { this.url = url; } - @Override - public void run() { - try { - Document doc = Http.url(url).cookies(cookies).get(); - // Find image - Elements donwloadLink = doc.select("div.alt1 b a[href^=//d.facdn.net/]"); - if (donwloadLink.size() == 0) { - logger.warn("Could not download " + this.url); - return; - } - String link = "http:" + donwloadLink.first().attr("href"); - logger.info("Found URL " + link); - String[] fileNameSplit = link.split("/"); - String fileName = fileNameSplit[fileNameSplit.length -1]; - fileName = fileName.replaceAll("[0-9]*\\.", ""); - String[] fileExtSplit = link.split("\\."); - String fileExt = fileExtSplit[fileExtSplit.length -1]; - fileName = fileName.replaceAll(fileExt, ""); - File saveAS; - fileName = fileName.replace("[0-9]*\\.", ""); - saveAS = new File( - workingDir.getCanonicalPath() - + File.separator - + fileName - + "." - + fileExt); - addURLToDownload(new URL(link),saveAS,"",cookies); - } catch (IOException e) { - logger.error("[!] Exception while loading/parsing " + this.url, e); - } - } + } + } From eb7fcac481c8ff41e5ecd073018c425dd9fb66fa Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 12:52:57 -0800 Subject: [PATCH 39/43] Fix #183: Re-enable FuraffinityRipperTest --- .../ripme/tst/ripper/rippers/FuraffinityRipperTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java index 36a4d91c..4285993b 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java @@ -6,11 +6,8 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.FuraffinityRipper; public class FuraffinityRipperTest extends RippersTest { - // https://github.com/RipMeApp/ripme/issues/183 - /* public void testFuraffinityAlbum() throws IOException { FuraffinityRipper ripper = new FuraffinityRipper(new URL("https://www.furaffinity.net/gallery/mustardgas/")); testRipper(ripper); } - */ } From 5da06eb53bee4d0049ed3755511d425ff2960f4e Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 12:35:05 -0800 Subject: [PATCH 40/43] Issue #269: Disabled test - WordpressRipperTest: various domains flaky in CI --- .../tst/ripper/rippers/WordpressComicRipperTest.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java index 532009fc..a4c90a9a 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java @@ -18,23 +18,32 @@ public class WordpressComicRipperTest extends RippersTest { // http://tnbtu.com/comic/01-00/ // http://shipinbottle.pepsaga.com/?p=281 + /* + // https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI public void test_totempole666() throws IOException { WordpressComicRipper ripper = new WordpressComicRipper( new URL("http://www.totempole666.com/comic/first-time-for-everything-00-cover/")); testRipper(ripper); } + */ + /* + // https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI public void test_buttsmithy() throws IOException { WordpressComicRipper ripper = new WordpressComicRipper( new URL("http://buttsmithy.com/archives/comic/p1")); testRipper(ripper); } + */ + /* + // https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI public void test_themonsterunderthebed() throws IOException { WordpressComicRipper ripper = new WordpressComicRipper( new URL("http://themonsterunderthebed.net/?comic=test-post")); testRipper(ripper); } + */ public void test_prismblush() throws IOException { WordpressComicRipper ripper = new WordpressComicRipper( @@ -58,11 +67,14 @@ public class WordpressComicRipperTest extends RippersTest { } */ + /* + // https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI public void test_freeadultcomix() throws IOException { WordpressComicRipper ripper = new WordpressComicRipper( new URL("http://freeadultcomix.com/finders-feepaid-in-full-sparrow/")); testRipper(ripper); } + */ public void test_delvecomic() throws IOException { WordpressComicRipper ripper = new WordpressComicRipper( From 8840a2baa1b3f8e8d97e091f742eeb7752d9d996 Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 13:03:48 -0800 Subject: [PATCH 41/43] 1.7.1: Fix WordpressComicRipper's ripper for freeadultcomix.com; FuraffinityRipper can now rip public albums --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 5034fd34..6ef1c93b 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.0 + 1.7.1 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index c51eaa08..cb358704 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.0", + "latestVersion": "1.7.1", "changeList": [ + "1.7.1: Fix WordpressComicRipper's ripper for freeadultcomix.com; FuraffinityRipper can now rip public albums", "1.7.0: Improved webtoons folder naming; Added code coverage with Coveralls.io; improved unit tests and code coverage; removed rippers for dead sites", "1.6.13: Added support for ripping from instagram tags; Instagram regex now matches all ripable urls; improved regex for pichunter", "1.6.12: Fix InstagramRipper with timestamps; Pichunter galleries support; logging improvements", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 92152c4c..d2d885aa 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.0"; + private static final String DEFAULT_VERSION = "1.7.1"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 9d9cf61961fd9022155efc57c8f8b647ee103196 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 21 Nov 2017 16:07:25 -0500 Subject: [PATCH 42/43] Added support for ripping single pages from instagram (#239) * Added support for ripping from single pages * Removed whitespace * Instagram folder naming improvments * Added GID tests for instagram single pages * Added some album download tests for instagram * Commented out flaky unit test --- .../ripme/ripper/rippers/InstagramRipper.java | 66 +++++++++++++++---- .../ripper/rippers/InstagramRipperTest.java | 12 ++-- 2 files changed, 61 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index 564227f6..337b658d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -50,6 +50,42 @@ public class InstagramRipper extends AbstractHTMLRipper { return san_url; } + private List getPostsFromSinglePage(Document Doc) { + List imageURLs = new ArrayList<>(); + JSONArray datas; + try { + JSONObject json = getJSONFromPage(Doc); + if (json.getJSONObject("entry_data").getJSONArray("PostPage") + .getJSONObject(0).getJSONObject("graphql").getJSONObject("shortcode_media") + .has("edge_sidecar_to_children")) { + datas = json.getJSONObject("entry_data").getJSONArray("PostPage") + .getJSONObject(0).getJSONObject("graphql").getJSONObject("shortcode_media") + .getJSONObject("edge_sidecar_to_children").getJSONArray("edges"); + for (int i = 0; i < datas.length(); i++) { + JSONObject data = (JSONObject) datas.get(i); + data = data.getJSONObject("node"); + if (data.has("is_video") && data.getBoolean("is_video")) { + imageURLs.add(data.getString("video_url")); + } else { + imageURLs.add(data.getString("display_url")); + } + } + } else { + JSONObject data = json.getJSONObject("entry_data").getJSONArray("PostPage") + .getJSONObject(0).getJSONObject("graphql").getJSONObject("shortcode_media"); + if (data.getBoolean("is_video")) { + imageURLs.add(data.getString("video_url")); + } else { + imageURLs.add(data.getString("display_url")); + } + } + return imageURLs; + } catch (IOException e) { + logger.error("Unable to get JSON from page " + url.toExternalForm()); + return null; + } + } + @Override public String getGID(URL url) throws MalformedURLException { Pattern p = Pattern.compile("^https?://instagram.com/([^/]+)/?"); @@ -64,7 +100,19 @@ public class InstagramRipper extends AbstractHTMLRipper { return m.group(1); } - p = Pattern.compile("^https?://www.instagram.com/p/[a-zA-Z0-9_-]+/\\?taken-by=([^/]+)/?"); + p = Pattern.compile("^https?://www.instagram.com/p/([a-zA-Z0-9_-]+)/\\?taken-by=([^/]+)/?"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(2) + "_" + m.group(1); + } + + p = Pattern.compile("^https?://www.instagram.com/p/([a-zA-Z0-9_-]+)/?"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + + p = Pattern.compile("^https?://www.instagram.com/p/([a-zA-Z0-9_-]+)/?(?:\\?hl=\\S*)?/?"); m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); @@ -148,9 +196,8 @@ public class InstagramRipper extends AbstractHTMLRipper { logger.warn("Unable to exact json from page"); } - Pattern p = Pattern.compile("^.*instagram.com/p/([a-zA-Z0-9\\-_.]+)/?"); - Matcher m = p.matcher(url.toExternalForm()); - if (!m.matches()) { + + if (!url.toExternalForm().contains("/p/")) { JSONArray datas = new JSONArray(); try { JSONArray profilePage = json.getJSONObject("entry_data").getJSONArray("ProfilePage"); @@ -216,16 +263,9 @@ public class InstagramRipper extends AbstractHTMLRipper { } } else { // We're ripping from a single page logger.info("Ripping from single page"); - if (!doc.select("meta[property=og:video]").attr("content").equals("")) { - String videoURL = doc.select("meta[property=og:video]").attr("content"); - // We're ripping a page with a video on it - imageURLs.add(videoURL); - } else { - // We're ripping a picture - imageURLs.add(doc.select("meta[property=og:image]").attr("content")); - } - + imageURLs = getPostsFromSinglePage(doc); } + return imageURLs; } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/InstagramRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/InstagramRipperTest.java index fd989b92..cf9c05e2 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/InstagramRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/InstagramRipperTest.java @@ -15,6 +15,10 @@ public class InstagramRipperTest extends RippersTest { Map testURLs = new HashMap<>(); testURLs.put(new URL("http://instagram.com/Test_User"), "Test_User"); testURLs.put(new URL("http://instagram.com/_test_user_"), "_test_user_"); + testURLs.put(new URL("https://www.instagram.com/p/BZ4egP7njW5/?hl=en"), "BZ4egP7njW5"); + testURLs.put(new URL("https://www.instagram.com/p/BZ4egP7njW5"), "BZ4egP7njW5"); + testURLs.put(new URL("https://www.instagram.com/p/BaNPpaHn2zU/?taken-by=hilaryduff"), "hilaryduff_BaNPpaHn2zU"); + testURLs.put(new URL("https://www.instagram.com/p/BaNPpaHn2zU/"), "BaNPpaHn2zU"); for (URL url : testURLs.keySet()) { InstagramRipper ripper = new InstagramRipper(url); ripper.setup(); @@ -23,15 +27,15 @@ public class InstagramRipperTest extends RippersTest { } } - /* public void testInstagramAlbums() throws IOException { List contentURLs = new ArrayList<>(); - contentURLs.add(new URL("http://instagram.com/anacheri")); + // This unit test is a bit flaky + //contentURLs.add(new URL("https://www.instagram.com/Test_User/")); + contentURLs.add(new URL("https://www.instagram.com/p/BZ4egP7njW5/?hl=en")); + contentURLs.add(new URL("https://www.instagram.com/p/BaNPpaHn2zU/")); for (URL url : contentURLs) { InstagramRipper ripper = new InstagramRipper(url); testRipper(ripper); } } - */ - } From f6d58ee7482a980c8ed583e2019136771cebdac6 Mon Sep 17 00:00:00 2001 From: metaprime Date: Tue, 21 Nov 2017 13:08:22 -0800 Subject: [PATCH 43/43] 1.7.2: InstagramRipper: Added support for ripping individual posts --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 6ef1c93b..662deb74 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.1 + 1.7.2 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index cb358704..c0179943 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.1", + "latestVersion": "1.7.2", "changeList": [ + "1.7.2: InstagramRipper: Added support for ripping individual posts", "1.7.1: Fix WordpressComicRipper's ripper for freeadultcomix.com; FuraffinityRipper can now rip public albums", "1.7.0: Improved webtoons folder naming; Added code coverage with Coveralls.io; improved unit tests and code coverage; removed rippers for dead sites", "1.6.13: Added support for ripping from instagram tags; Instagram regex now matches all ripable urls; improved regex for pichunter", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index d2d885aa..53841875 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.1"; + private static final String DEFAULT_VERSION = "1.7.2"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar";