From 9214ce14bbbe194b0d0b2de8b8277f0860bdb8f6 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 17 Nov 2017 17:15:56 -0500 Subject: [PATCH 01/18] Improved regex for pichunter.com --- .../ripme/ripper/rippers/PichunterRipper.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PichunterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PichunterRipper.java index 5b4b15fb..d9f63626 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PichunterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PichunterRipper.java @@ -32,23 +32,23 @@ public class PichunterRipper extends AbstractHTMLRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("https?://www.pichunter.com/(|tags|models|sites)/([a-zA-Z0-9_-]+)/?"); + Pattern p = Pattern.compile("https?://www.pichunter.com/(|tags|models|sites)/(\\S*)/?"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(2); } - p = Pattern.compile("https?://www.pichunter.com/(tags|models|sites)/([a-zA-Z0-9_-]+)/photos/\\d+/?"); + p = Pattern.compile("https?://www.pichunter.com/(tags|models|sites)/(\\S*)/photos/\\d+/?"); m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(2); } - p = Pattern.compile("https?://www.pichunter.com/tags/all/([a-zA-Z0-9_-]+)/\\d+/?"); + p = Pattern.compile("https?://www.pichunter.com/tags/all/(\\S*)/\\d+/?"); m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); } - p = Pattern.compile("https?://www.pichunter.com/gallery/\\d+/([a-zA-Z0-9_-]+)/?"); + p = Pattern.compile("https?://www.pichunter.com/gallery/\\d+/(\\S*)/?"); m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); @@ -58,7 +58,7 @@ public class PichunterRipper extends AbstractHTMLRipper { } private boolean isPhotoSet(URL url) { - Pattern p = Pattern.compile("https?://www.pichunter.com/gallery/\\d+/([a-zA-Z0-9_-]+)/?"); + Pattern p = Pattern.compile("https?://www.pichunter.com/gallery/\\d+/(\\S*)/?"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return true; From 5c185c05df8789a25c4bd88746c78d8b87a05810 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 18 Nov 2017 03:52:17 -0500 Subject: [PATCH 02/18] Improved instagram regex and added sanitizeURL --- .../rarchives/ripme/ripper/rippers/InstagramRipper.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index c085059c..6a33e71d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -43,6 +43,13 @@ public class InstagramRipper extends AbstractHTMLRipper { return (url.getHost().endsWith("instagram.com")); } + @Override + public URL sanitizeURL(URL url) throws MalformedURLException { + URL san_url = new URL(url.toExternalForm().replaceAll("\\?hl=\\S*", "")); + logger.info("sanitized URL is " + san_url.toExternalForm()); + return san_url; + } + @Override public String getGID(URL url) throws MalformedURLException { Pattern p = Pattern.compile("^https?://instagram.com/([^/]+)/?"); @@ -51,7 +58,7 @@ public class InstagramRipper extends AbstractHTMLRipper { return m.group(1); } - p = Pattern.compile("^https?://www.instagram.com/([^/]+)/?"); + p = Pattern.compile("^https?://www.instagram.com/([^/]+)/?(?:\\?hl=\\S*)?/?"); m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); From e8185aaf3ec63b70dbbf15d91d0aadcd8562f5b7 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 18 Nov 2017 07:22:25 -0500 Subject: [PATCH 03/18] Added support for ripping from tags --- .../ripme/ripper/rippers/InstagramRipper.java | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index c085059c..de089c02 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -62,6 +62,13 @@ public class InstagramRipper extends AbstractHTMLRipper { if (m.matches()) { return m.group(1); } + + p = Pattern.compile("^https?://www.instagram.com/explore/tags/([^/]+)/?"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Unable to find user in " + url); } @@ -134,11 +141,18 @@ public class InstagramRipper extends AbstractHTMLRipper { logger.warn("Unable to exact json from page"); } - Pattern p = Pattern.compile("^.*instagram\\.com/([a-zA-Z0-9\\-_.]+)/?"); + Pattern p = Pattern.compile("^.*instagram.com/p/([a-zA-Z0-9\\-_.]+)/?"); Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - JSONArray profilePage = json.getJSONObject("entry_data").getJSONArray("ProfilePage"); - JSONArray datas = profilePage.getJSONObject(0).getJSONObject("user").getJSONObject("media").getJSONArray("nodes"); + if (!m.matches()) { + JSONArray datas = new JSONArray(); + try { + JSONArray profilePage = json.getJSONObject("entry_data").getJSONArray("ProfilePage"); + datas = profilePage.getJSONObject(0).getJSONObject("user").getJSONObject("media").getJSONArray("nodes"); + } catch (JSONException e) { + // Handle hashtag pages + datas = json.getJSONObject("entry_data").getJSONArray("TagPage").getJSONObject(0) + .getJSONObject("tag").getJSONObject("media").getJSONArray("nodes"); + } for (int i = 0; i < datas.length(); i++) { JSONObject data = (JSONObject) datas.get(i); Long epoch = data.getLong("date"); @@ -168,6 +182,16 @@ public class InstagramRipper extends AbstractHTMLRipper { } // Rip the next page if (!nextPageID.equals("") && !isThisATest()) { + if (url.toExternalForm().contains("/tags/")) { + try { + // Sleep for a while to avoid a ban + sleep(2500); + getURLsFromPage(Http.url(url.toExternalForm() + "?max_id=" + nextPageID).get()); + } catch (IOException e) { + return imageURLs; + } + + } try { // Sleep for a while to avoid a ban sleep(2500); @@ -177,6 +201,7 @@ public class InstagramRipper extends AbstractHTMLRipper { } } } else { // We're ripping from a single page + logger.info("Ripping from single page"); if (!doc.select("meta[property=og:video]").attr("content").equals("")) { String videoURL = doc.select("meta[property=og:video]").attr("content"); // We're ripping a page with a video on it From bcf3f9d2a763a72f5e35e2e7cb1fc307d250b663 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 18 Nov 2017 09:07:25 -0500 Subject: [PATCH 04/18] Instagram ripper now handles urls with and without a trailing / --- .../rarchives/ripme/ripper/rippers/InstagramRipper.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index de089c02..7292684d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -186,7 +186,12 @@ public class InstagramRipper extends AbstractHTMLRipper { try { // Sleep for a while to avoid a ban sleep(2500); - getURLsFromPage(Http.url(url.toExternalForm() + "?max_id=" + nextPageID).get()); + if (url.toExternalForm().substring(url.toExternalForm().length() - 1).equals("/")) { + getURLsFromPage(Http.url(url.toExternalForm() + "?max_id=" + nextPageID).get()); + } else { + getURLsFromPage(Http.url(url.toExternalForm() + "/?max_id=" + nextPageID).get()); + } + } catch (IOException e) { return imageURLs; } @@ -199,6 +204,8 @@ public class InstagramRipper extends AbstractHTMLRipper { } catch (IOException e) { return imageURLs; } + } else { + logger.warn("Can't get net page"); } } else { // We're ripping from a single page logger.info("Ripping from single page"); From 08d5e665f4c288ce8654aee509a46f0a3db40d12 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 18 Nov 2017 09:23:01 -0500 Subject: [PATCH 05/18] 1.6.13: Added support for ripping from instagram tags; Instagram regex now matches all ripable urls; improved regex for pichunter --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index e3935459..bd16b6af 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.6.12 + 1.6.13 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index e736fb5b..d5a5baa4 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.6.12", + "latestVersion": "1.6.13", "changeList": [ + "1.6.13: Added support for ripping from instagram tags; Instagram regex now matches all ripable urls; improved regex for pichunter", "1.6.12: Fix InstagramRipper with timestamps; Pichunter galleries support; logging improvements", "1.6.11: dded pichunter.com ripper; Changed instagram output name format; added tehyiffgallery ripper; Fixed xchan ripper; Fixed regession in chanRipper folder naming", "1.6.10: Added viewcomic ripper; Fixed webtoons malformed url error message; Fixed chan ripper thread title; Fixed Modelmayhem ripper", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 31408758..448d0f77 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.6.12"; + private static final String DEFAULT_VERSION = "1.6.13"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 0475db5fab9bf74a3da810da9fbbb8c67237a8e2 Mon Sep 17 00:00:00 2001 From: metaprime Date: Sat, 18 Nov 2017 20:31:50 -0800 Subject: [PATCH 06/18] Add Coveralls code coverage badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 85f08b81..890f52d3 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![Build Status](https://travis-ci.org/RipMeApp/ripme.svg?branch=master)](https://travis-ci.org/RipMeApp/ripme) [![Join the chat at https://gitter.im/RipMeApp/Lobby](https://badges.gitter.im/RipMeApp/Lobby.svg)](https://gitter.im/RipMeApp/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![Coverage Status](https://coveralls.io/repos/github/RipMeApp/ripme/badge.svg?branch=master)](https://coveralls.io/github/RipMeApp/ripme?branch=master) Album ripper for various websites. Runs on your computer. Requires Java 8. From a95cc719d2d2021bed43eb0f9619e2ff9fb451e9 Mon Sep 17 00:00:00 2001 From: metaprime Date: Sat, 18 Nov 2017 20:40:28 -0800 Subject: [PATCH 07/18] pom.xml: Coveralls plugins --- pom.xml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pom.xml b/pom.xml index bd16b6af..b661de5a 100644 --- a/pom.xml +++ b/pom.xml @@ -88,6 +88,25 @@ 1.8 + + org.eluder.coveralls + coveralls-maven-plugin + 4.3.0 + + + + org.jacoco + jacoco-maven-plugin + 0.7.6.201602180812 + + + prepare-agent + + prepare-agent + + + + From 1e47413fdbbf7beefb53578d08dd47e54910ba14 Mon Sep 17 00:00:00 2001 From: metaprime Date: Sat, 18 Nov 2017 22:52:42 -0800 Subject: [PATCH 08/18] Issue #226: Temporarily disable ImagefapRipperTest because fap.to is unreachable. --- .../ripme/tst/ripper/rippers/ImagefapRipperTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java index d5a1aca3..bd2f5a49 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java @@ -11,6 +11,10 @@ public class ImagefapRipperTest extends RippersTest { public void testImagefapAlbums() throws IOException { Map testURLs = new HashMap<>(); + + /* + Temporarily disabled test. See issue https://github.com/RipMeApp/ripme/issues/226 + // Album with specific title testURLs.put(new URL("http://www.imagefap.com/pictures/4649440/Frozen-%28Elsa-and-Anna%29?view=2"), "Frozen (Elsa and Anna)"); @@ -18,6 +22,7 @@ public class ImagefapRipperTest extends RippersTest { // New URL format testURLs.put(new URL("http://www.imagefap.com/gallery.php?pgid=fffd68f659befa5535cf78f014e348f1"), "imagefap_fffd68f659befa5535cf78f014e348f1"); + */ for (URL url : testURLs.keySet()) { ImagefapRipper ripper = new ImagefapRipper(url); From eda92001afe4c6afaecb1c100e5d95434f93905b Mon Sep 17 00:00:00 2001 From: metaprime Date: Sat, 18 Nov 2017 23:06:13 -0800 Subject: [PATCH 09/18] Update LICENSE and .gitignore --- .gitignore | 2 +- LICENSE.md => LICENSE.txt | 0 README.md | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename LICENSE.md => LICENSE.txt (100%) diff --git a/.gitignore b/.gitignore index a640323f..434f0343 100644 --- a/.gitignore +++ b/.gitignore @@ -112,7 +112,7 @@ rips/ .history ripme.jar.update *.swp -rip.properties +*.properties history.json *.iml .settings/ diff --git a/LICENSE.md b/LICENSE.txt similarity index 100% rename from LICENSE.md rename to LICENSE.txt diff --git a/README.md b/README.md index 85f08b81..e6ab0355 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# RipMe +# RipMe [![Licensed under the MIT License](https://img.shields.io/badge/License-MIT-blue.svg)](https://github.com/RipMeApp/ripme/blob/master/LICENSE.txt) [![Build Status](https://travis-ci.org/RipMeApp/ripme.svg?branch=master)](https://travis-ci.org/RipMeApp/ripme) [![Join the chat at https://gitter.im/RipMeApp/Lobby](https://badges.gitter.im/RipMeApp/Lobby.svg)](https://gitter.im/RipMeApp/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) From e78f961c88f62ab636f4475d2828186dfb8e83a1 Mon Sep 17 00:00:00 2001 From: metaprime Date: Sat, 18 Nov 2017 23:11:44 -0800 Subject: [PATCH 10/18] Move badges around on README --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 7b623ad2..5ee22a36 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ -# RipMe [![Licensed under the MIT License](https://img.shields.io/badge/License-MIT-blue.svg)](https://github.com/RipMeApp/ripme/blob/master/LICENSE.txt) +# RipMe [![Licensed under the MIT License](https://img.shields.io/badge/License-MIT-blue.svg)](https://github.com/RipMeApp/ripme/blob/master/LICENSE.txt) [![Join the chat at https://gitter.im/RipMeApp/Lobby](https://badges.gitter.im/RipMeApp/Lobby.svg)](https://gitter.im/RipMeApp/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Build Status](https://travis-ci.org/RipMeApp/ripme.svg?branch=master)](https://travis-ci.org/RipMeApp/ripme) -[![Join the chat at https://gitter.im/RipMeApp/Lobby](https://badges.gitter.im/RipMeApp/Lobby.svg)](https://gitter.im/RipMeApp/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Coverage Status](https://coveralls.io/repos/github/RipMeApp/ripme/badge.svg?branch=master)](https://coveralls.io/github/RipMeApp/ripme?branch=master) Album ripper for various websites. Runs on your computer. Requires Java 8. From e60765bad376d1dcf3ab2d902983ff4a7470b0aa Mon Sep 17 00:00:00 2001 From: metaprime Date: Sat, 18 Nov 2017 23:14:26 -0800 Subject: [PATCH 11/18] Update .travis.yml to upload code coverage to Coveralls --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 8ca274d2..e8a2c98a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,3 +2,5 @@ language: java jdk: - oraclejdk8 - openjdk8 +after_success: + - mvn clean test jacoco:report coveralls:report From 7ee8fef59a5b73073b80669dae8d63feb21e156d Mon Sep 17 00:00:00 2001 From: metaprime Date: Sun, 19 Nov 2017 12:47:41 -0800 Subject: [PATCH 12/18] Update README with donation links. (#228) --- README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5ee22a36..13e69cd0 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,20 @@ [![Build Status](https://travis-ci.org/RipMeApp/ripme.svg?branch=master)](https://travis-ci.org/RipMeApp/ripme) [![Coverage Status](https://coveralls.io/repos/github/RipMeApp/ripme/badge.svg?branch=master)](https://coveralls.io/github/RipMeApp/ripme?branch=master) -Album ripper for various websites. Runs on your computer. Requires Java 8. +# Contribute + +RipMe is maintained with ♥️ and in our limited free time by **[@MetaPrime](https://github.com/metaprime)** and **[@cyian-1756](https://github.com/cyian-1756)**. If you'd like to contribute but aren't good with code, help keep us happy with a small contribution! + +[![Tip with PayPal](https://img.shields.io/badge/PayPal-Buy_us...-lightgrey.svg)](https://www.paypal.me/ripmeap) +[![Tip with PayPal](https://img.shields.io/badge/coffee-%245-green.svg)](https://www.paypal.com/paypalme/ripmeapp/send?amount=5.00¤cyCode=USD&locale.x=en_US&country.x=US) +[![Tip with PayPal](https://img.shields.io/badge/beer-%2410-yellow.svg)](https://www.paypal.com/paypalme/ripmeapp/send?amount=10.00¤cyCode=USD&locale.x=en_US&country.x=US) +[![Tip with PayPal](https://img.shields.io/badge/lunch-%2420-orange.svg)](https://www.paypal.com/paypalme/ripmeapp/send?amount=20.00¤cyCode=USD&locale.x=en_US&country.x=US) +[![Tip with PayPal](https://img.shields.io/badge/dinner-%2450-red.svg)](https://www.paypal.com/paypalme/ripmeapp/send?amount=20.00¤cyCode=USD&locale.x=en_US&country.x=US) +[![Tip with PayPal](https://img.shields.io/badge/custom_amount-...-lightgrey.svg)](https://www.paypal.me/ripmeap) + +# About + +RipMe is an album ripper for various websites. Runs on your computer. Requires Java 8. ![Screenshot](http://i.imgur.com/kWzhsIu.png) From 28280b12759a49d9652f0705be4bacc0a8b46f9e Mon Sep 17 00:00:00 2001 From: metaprime Date: Sun, 19 Nov 2017 12:54:35 -0800 Subject: [PATCH 13/18] Fix typo in paypal links. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 13e69cd0..c4058231 100644 --- a/README.md +++ b/README.md @@ -7,12 +7,12 @@ RipMe is maintained with ♥️ and in our limited free time by **[@MetaPrime](https://github.com/metaprime)** and **[@cyian-1756](https://github.com/cyian-1756)**. If you'd like to contribute but aren't good with code, help keep us happy with a small contribution! -[![Tip with PayPal](https://img.shields.io/badge/PayPal-Buy_us...-lightgrey.svg)](https://www.paypal.me/ripmeap) +[![Tip with PayPal](https://img.shields.io/badge/PayPal-Buy_us...-lightgrey.svg)](https://www.paypal.me/ripmeapp) [![Tip with PayPal](https://img.shields.io/badge/coffee-%245-green.svg)](https://www.paypal.com/paypalme/ripmeapp/send?amount=5.00¤cyCode=USD&locale.x=en_US&country.x=US) [![Tip with PayPal](https://img.shields.io/badge/beer-%2410-yellow.svg)](https://www.paypal.com/paypalme/ripmeapp/send?amount=10.00¤cyCode=USD&locale.x=en_US&country.x=US) [![Tip with PayPal](https://img.shields.io/badge/lunch-%2420-orange.svg)](https://www.paypal.com/paypalme/ripmeapp/send?amount=20.00¤cyCode=USD&locale.x=en_US&country.x=US) [![Tip with PayPal](https://img.shields.io/badge/dinner-%2450-red.svg)](https://www.paypal.com/paypalme/ripmeapp/send?amount=20.00¤cyCode=USD&locale.x=en_US&country.x=US) -[![Tip with PayPal](https://img.shields.io/badge/custom_amount-...-lightgrey.svg)](https://www.paypal.me/ripmeap) +[![Tip with PayPal](https://img.shields.io/badge/custom_amount-...-lightgrey.svg)](https://www.paypal.me/ripmeapp) # About From 92da8b490994d65346f7c44fb81c6e2890bcf3fb Mon Sep 17 00:00:00 2001 From: metaprime Date: Sun, 19 Nov 2017 12:55:35 -0800 Subject: [PATCH 14/18] Fix typo in paypal link. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c4058231..ba15506d 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ RipMe is maintained with ♥️ and in our limited free time by **[@MetaPrime](h [![Tip with PayPal](https://img.shields.io/badge/coffee-%245-green.svg)](https://www.paypal.com/paypalme/ripmeapp/send?amount=5.00¤cyCode=USD&locale.x=en_US&country.x=US) [![Tip with PayPal](https://img.shields.io/badge/beer-%2410-yellow.svg)](https://www.paypal.com/paypalme/ripmeapp/send?amount=10.00¤cyCode=USD&locale.x=en_US&country.x=US) [![Tip with PayPal](https://img.shields.io/badge/lunch-%2420-orange.svg)](https://www.paypal.com/paypalme/ripmeapp/send?amount=20.00¤cyCode=USD&locale.x=en_US&country.x=US) -[![Tip with PayPal](https://img.shields.io/badge/dinner-%2450-red.svg)](https://www.paypal.com/paypalme/ripmeapp/send?amount=20.00¤cyCode=USD&locale.x=en_US&country.x=US) +[![Tip with PayPal](https://img.shields.io/badge/dinner-%2450-red.svg)](https://www.paypal.com/paypalme/ripmeapp/send?amount=50.00¤cyCode=USD&locale.x=en_US&country.x=US) [![Tip with PayPal](https://img.shields.io/badge/custom_amount-...-lightgrey.svg)](https://www.paypal.me/ripmeapp) # About From 66a2641a6b6a676ad10028c637d9f6a8dbd669d2 Mon Sep 17 00:00:00 2001 From: metaprime Date: Sun, 19 Nov 2017 13:15:40 -0800 Subject: [PATCH 15/18] Disabled BasicRippersTest#testPhotobucketRip --- .../rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java index d6827407..751242c6 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BasicRippersTest.java @@ -215,12 +215,15 @@ public class BasicRippersTest extends RippersTest { testRipper(ripper); } + /* + // https://github.com/RipMeApp/ripme/issues/229 : Disabled test (temporary) : BasicRippersTest#testPhotobucketRip (timing out) public void testPhotobucketRip() throws IOException { AbstractRipper ripper = new PhotobucketRipper(new URL("http://s844.photobucket.com/user/SpazzySpizzy/library/Album%20Covers?sort=3&page=1")); testRipper(ripper); deleteSubdirs(ripper.getWorkingDir()); deleteDir(ripper.getWorkingDir()); } + */ public void testPornhubRip() throws IOException { AbstractRipper ripper = new PornhubRipper(new URL("https://www.pornhub.com/album/15680522")); @@ -290,6 +293,7 @@ public class BasicRippersTest extends RippersTest { testRipper(ripper); } + // https://github.com/RipMeApp/ripme/issues/206 : Disabled test : BasicRippersTest#testXhamsterAlbums -- fix and re-enable // public void testXhamsterAlbums() throws IOException { // XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.com/photos/gallery/sexy-preggo-girls-9026608")); // testRipper(ripper); From 9e37d5de27081235046ba25aeccbfc91726b2734 Mon Sep 17 00:00:00 2001 From: metaprime Date: Sun, 19 Nov 2017 19:17:05 -0800 Subject: [PATCH 16/18] Remove utils/AES.java (#234) --- .../java/com/rarchives/ripme/utils/AES.java | 57 ------------------- 1 file changed, 57 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/utils/AES.java diff --git a/src/main/java/com/rarchives/ripme/utils/AES.java b/src/main/java/com/rarchives/ripme/utils/AES.java deleted file mode 100644 index db849f46..00000000 --- a/src/main/java/com/rarchives/ripme/utils/AES.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.rarchives.ripme.utils; - -import java.lang.reflect.Field; -import java.util.Arrays; - -import javax.crypto.Cipher; -import javax.crypto.SecretKey; -import javax.crypto.spec.IvParameterSpec; -import javax.crypto.spec.SecretKeySpec; - -class AES { - - /** - * Hack to get JCE Unlimited Strenght so we can use weird AES encryption stuff. - * From http://stackoverflow.com/a/20286961 - */ - static { - try { - Field field = Class.forName("javax.crypto.JceSecurity").getDeclaredField("isRestricted"); - if (!field.isAccessible()) { - field.setAccessible(true); - field.set(null, java.lang.Boolean.FALSE); - } - } catch (Exception ex) { - // Assume it's fine. - } - } - - public static String decrypt(String cipherText, String key, int nBits) throws Exception { - String res = null; - nBits = nBits / 8; - byte[] data = Base64.decode(cipherText); - byte[] k = Arrays.copyOf(key.getBytes(), nBits); - - Cipher cipher = Cipher.getInstance("AES/CTR/NoPadding"); - SecretKey secretKey = generateSecretKey(k, nBits); - byte[] nonceBytes = Arrays.copyOf(Arrays.copyOf(data, 8), nBits / 2); - IvParameterSpec nonce = new IvParameterSpec(nonceBytes); - cipher.init(Cipher.ENCRYPT_MODE, secretKey, nonce); - res = new String(cipher.doFinal(data, 8, data.length - 8)); - return res; - } - - private static SecretKey generateSecretKey(byte[] keyBytes, int nBits) throws Exception { - try { - SecretKey secretKey = new SecretKeySpec(keyBytes, "AES"); - Cipher cipher = Cipher.getInstance("AES/ECB/NoPadding"); - cipher.init(Cipher.ENCRYPT_MODE, secretKey); - keyBytes = cipher.doFinal(keyBytes); - } catch (Throwable e1) { - e1.printStackTrace(); - return null; - } - System.arraycopy(keyBytes, 0, keyBytes, nBits / 2, nBits / 2); - return new SecretKeySpec(keyBytes, "AES"); - } -} From 2735491b3f246478fb791ccf06169886fff72839 Mon Sep 17 00:00:00 2001 From: metaprime Date: Sun, 19 Nov 2017 19:17:14 -0800 Subject: [PATCH 17/18] Remove nbactions.xml (#236) --- nbactions.xml | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 nbactions.xml diff --git a/nbactions.xml b/nbactions.xml deleted file mode 100644 index fa676bd9..00000000 --- a/nbactions.xml +++ /dev/null @@ -1,17 +0,0 @@ - - - - run - - jar - - - process-classes - org.codehaus.mojo:exec-maven-plugin:1.2.1:exec - - - -classpath %classpath com.rarchives.ripme.App - java - - - From 5510b407ad6b3490ca7b43df72ec69897745505d Mon Sep 17 00:00:00 2001 From: metaprime Date: Sun, 19 Nov 2017 19:42:12 -0800 Subject: [PATCH 18/18] Fix loggers (#235) --- src/main/java/com/rarchives/ripme/App.java | 11 +++++++---- .../rarchives/ripme/ripper/rippers/E621Ripper.java | 14 +++++--------- .../ripme/ripper/rippers/PahealRipper.java | 9 +++++---- .../ripme/ripper/rippers/XbooruRipper.java | 7 ++++--- src/main/java/com/rarchives/ripme/utils/Http.java | 2 +- 5 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index 326dc5f8..e046d5a6 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -34,9 +34,15 @@ import com.rarchives.ripme.utils.Utils; */ public class App { - public static Logger logger; + public static final Logger logger; private static final History HISTORY = new History(); + static { + //initialize logger + Utils.configureLogger(); + logger = Logger.getLogger(App.class); + } + public static void main(String[] args) throws MalformedURLException { CommandLine cl = getArgs(args); if (args.length > 0 && cl.hasOption('v')){ @@ -44,11 +50,8 @@ public class App { System.exit(0); } - //initialize logger - Utils.configureLogger(); System.setProperty("apple.laf.useScreenMenuBar", "true"); System.setProperty("com.apple.mrj.application.apple.menu.about.name", "RipMe"); - logger = Logger.getLogger(App.class); logger.info("Initialized ripme v" + UpdateUtils.getThisJarVersion()); if (args.length > 0) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java index 231533e2..e381661c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java @@ -1,4 +1,3 @@ - package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.ripper.AbstractHTMLRipper; @@ -12,19 +11,16 @@ import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.log4j.Logger; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; -/** - * - * @author - */ public class E621Ripper extends AbstractHTMLRipper{ + private static final Logger logger = Logger.getLogger(E621Ripper.class); + private static Pattern gidPattern=null; private static Pattern gidPattern2=null; private static Pattern gidPatternPool=null; @@ -89,7 +85,7 @@ public class E621Ripper extends AbstractHTMLRipper{ addURLToDownload(new URL(page.getElementById("image").absUrl("src")),Utils.getConfigBoolean("download.save_order",true)?url.getRef()+"-":""); } catch (IOException ex) { - Logger.getLogger(E621Ripper.class.getName()).log(Level.SEVERE, null, ex); + logger.error(ex); } } })); @@ -121,7 +117,7 @@ public class E621Ripper extends AbstractHTMLRipper{ return Utils.filesystemSafe(prefix+new URI(getTerm(url)).getPath()); } catch (URISyntaxException ex) { - Logger.getLogger(PahealRipper.class.getName()).log(Level.SEVERE, null, ex); + logger.error(ex); } throw new MalformedURLException("Expected e621.net URL format: e621.net/post/index/1/searchterm - got "+url+" instead"); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PahealRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PahealRipper.java index 22060b88..d2421f37 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PahealRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PahealRipper.java @@ -13,15 +13,16 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.log4j.Logger; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; public class PahealRipper extends AbstractHTMLRipper { + private static final Logger logger = Logger.getLogger(PahealRipper.class); + private static Map cookies = null; private static Pattern gidPattern = null; @@ -93,7 +94,7 @@ public class PahealRipper extends AbstractHTMLRipper { + ext); addURLToDownload(url, outFile); } catch (IOException | URISyntaxException ex) { - Logger.getLogger(PahealRipper.class.getName()).log(Level.SEVERE, null, ex); + logger.error("Error while downloading URL " + url, ex); } } @@ -115,7 +116,7 @@ public class PahealRipper extends AbstractHTMLRipper { try { return Utils.filesystemSafe(new URI(getTerm(url)).getPath()); } catch (URISyntaxException ex) { - Logger.getLogger(PahealRipper.class.getName()).log(Level.SEVERE, null, ex); + logger.error(ex); } throw new MalformedURLException("Expected paheal.net URL format: rule34.paheal.net/post/list/searchterm - got " + url + " instead"); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XbooruRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XbooruRipper.java index e987a294..6b697f70 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XbooruRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XbooruRipper.java @@ -10,14 +10,15 @@ import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.log4j.Logger; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; public class XbooruRipper extends AbstractHTMLRipper { + private static final Logger logger = Logger.getLogger(XbooruRipper.class); + private static Pattern gidPattern = null; public XbooruRipper(URL url) throws IOException { @@ -87,7 +88,7 @@ public class XbooruRipper extends AbstractHTMLRipper { try { return Utils.filesystemSafe(new URI(getTerm(url)).getPath()); } catch (URISyntaxException ex) { - Logger.getLogger(PahealRipper.class.getName()).log(Level.SEVERE, null, ex); + logger.error(ex); } throw new MalformedURLException("Expected xbooru.com URL format: xbooru.com/index.php?tags=searchterm - got " + url + " instead"); diff --git a/src/main/java/com/rarchives/ripme/utils/Http.java b/src/main/java/com/rarchives/ripme/utils/Http.java index 01b54c6f..885a194d 100644 --- a/src/main/java/com/rarchives/ripme/utils/Http.java +++ b/src/main/java/com/rarchives/ripme/utils/Http.java @@ -23,7 +23,7 @@ import com.rarchives.ripme.ripper.AbstractRipper; public class Http { private static final int TIMEOUT = Utils.getConfigInteger("page.timeout", 5 * 1000); - private static final Logger logger = Logger.getLogger(AbstractRipper.class); + private static final Logger logger = Logger.getLogger(Http.class); private int retries; private String url;