From ea7573a975eada546826703b9b8735b49845c0cb Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 31 May 2018 04:37:36 -0400 Subject: [PATCH 01/10] Added maven plugin to make builds reproducible; build.sh and build.bat now strip the compiled jar of any non-reproducable metadata; Removed Executable Bit from build.bat --- build.bat | 3 ++- build.sh | 4 +++- pom.xml | 5 +++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/build.bat b/build.bat index 7e7c3221..7c2aa6c3 100755 --- a/build.bat +++ b/build.bat @@ -1 +1,2 @@ -mvn clean compile assembly:single \ No newline at end of file +mvn clean compile assembly:single +mvn io.github.zlika:reproducible-build-maven-plugin:0.6:strip-jar \ No newline at end of file diff --git a/build.sh b/build.sh index a3ec0242..2f044cde 100755 --- a/build.sh +++ b/build.sh @@ -1,2 +1,4 @@ #!/usr/bin/env bash -mvn clean compile assembly:single \ No newline at end of file +mvn clean compile assembly:single +# Strip the jar of any non-reproducible metadata such as timestamps +mvn io.github.zlika:reproducible-build-maven-plugin:0.6:strip-jar \ No newline at end of file diff --git a/pom.xml b/pom.xml index c94f1d56..0900a8e7 100644 --- a/pom.xml +++ b/pom.xml @@ -61,6 +61,11 @@ + + io.github.zlika + reproducible-build-maven-plugin + 0.6 + maven-assembly-plugin From 3f1e78a47847a120129ef99ef85ac0722dd3b293 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 31 May 2018 04:53:38 -0400 Subject: [PATCH 02/10] patch.py now strips jars when building --- patch.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/patch.py b/patch.py index ca63cbf3..4c417515 100644 --- a/patch.py +++ b/patch.py @@ -77,6 +77,8 @@ subprocess.call(['git', 'commit', '-m', commitMessage]) subprocess.call(['git', 'tag', nextVersion]) print("Building ripme") subprocess.call(["mvn", "clean", "compile", "assembly:single"]) +print("Stripping jar") +subprocess.call(["mvn", "io.github.zlika:reproducible-build-maven-plugin:0.6:strip-jar"]) print("Hashing .jar file") openedFile = open("./target/ripme-{}-jar-with-dependencies.jar".format(nextVersion), "rb") readFile = openedFile.read() From 79c501ce7ac62c393c8f61677514a91c0a077f03 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 31 May 2018 04:58:13 -0400 Subject: [PATCH 03/10] Added tag skip feature to features list --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7cc8ffa8..826ab948 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ For information about running the `.jar` file, see [the How To Run wiki](https:/ * Built in updater * Can rip images from tumblr in the size they were uploaded in [See here for how to enable](https://github.com/RipMeApp/ripme/wiki/Config-options#tumblrget_raw_image) * Skips already downloaded images by default +* Can auto skip e-hentai and nhentai albums containing certain tags [See here for how to enable](https://github.com/RipMeApp/ripme/wiki/Config-options#nhentaiblacklisttags) ## [List of Supported Sites](https://github.com/ripmeapp/ripme/wiki/Supported-Sites) From 5cbc152e1c04b09dc28e1f60bc617efabfc60ced Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 2 Jun 2018 15:32:21 -0400 Subject: [PATCH 04/10] Change log is now displayed when updating from command line --- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 95af45c6..8882c9af 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -68,8 +68,9 @@ public class UpdateUtils { if (change.startsWith(UpdateUtils.getThisJarVersion() + ":")) { break; } - changeList.append("
+ ").append(change); + changeList.append("\n").append(change); } + logger.info("Change log: \n" + changeList.toString()); String latestVersion = ripmeJson.getString("latestVersion"); if (UpdateUtils.isNewerVersion(latestVersion)) { From 10d256a6e5ef42399e8a922e29fe32b98a6eeb4c Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 2 Jun 2018 15:39:27 -0400 Subject: [PATCH 05/10] Cut down on repeated code --- .../com/rarchives/ripme/ui/UpdateUtils.java | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 8882c9af..ebfa1428 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -39,6 +39,20 @@ public class UpdateUtils { } return thisVersion; } + + private static String getChangeList(JSONObject rj) { + JSONArray jsonChangeList = rj.getJSONArray("changeList"); + StringBuilder changeList = new StringBuilder(); + for (int i = 0; i < jsonChangeList.length(); i++) { + String change = jsonChangeList.getString(i); + if (change.startsWith(UpdateUtils.getThisJarVersion() + ":")) { + break; + } + changeList.append("\n").append(change); + } + return changeList.toString(); + } + public static void updateProgramCLI() { logger.info("Checking for update..."); @@ -61,16 +75,10 @@ public class UpdateUtils { } String jsonString = doc.body().html().replaceAll(""", "\""); ripmeJson = new JSONObject(jsonString); - JSONArray jsonChangeList = ripmeJson.getJSONArray("changeList"); - StringBuilder changeList = new StringBuilder(); - for (int i = 0; i < jsonChangeList.length(); i++) { - String change = jsonChangeList.getString(i); - if (change.startsWith(UpdateUtils.getThisJarVersion() + ":")) { - break; - } - changeList.append("\n").append(change); - } - logger.info("Change log: \n" + changeList.toString()); + + String changeList = getChangeList(ripmeJson); + + logger.info("Change log: \n" + changeList); String latestVersion = ripmeJson.getString("latestVersion"); if (UpdateUtils.isNewerVersion(latestVersion)) { @@ -112,15 +120,8 @@ public class UpdateUtils { } String jsonString = doc.body().html().replaceAll(""", "\""); ripmeJson = new JSONObject(jsonString); - JSONArray jsonChangeList = ripmeJson.getJSONArray("changeList"); - StringBuilder changeList = new StringBuilder(); - for (int i = 0; i < jsonChangeList.length(); i++) { - String change = jsonChangeList.getString(i); - if (change.startsWith(UpdateUtils.getThisJarVersion() + ":")) { - break; - } - changeList.append("
+ ").append(change); - } + + String changeList = getChangeList(ripmeJson); String latestVersion = ripmeJson.getString("latestVersion"); if (UpdateUtils.isNewerVersion(latestVersion)) { @@ -128,7 +129,7 @@ public class UpdateUtils { int result = JOptionPane.showConfirmDialog( null, "New version (" + latestVersion + ") is available!" - + "

Recent changes:" + changeList.toString() + + "

Recent changes:" + changeList + "

Do you want to download and run the newest version?", "RipMe Updater", JOptionPane.YES_NO_OPTION); From b0bcf506aa0dfac1b7cdced99c55117e245b93ba Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 2 Jun 2018 16:02:52 -0400 Subject: [PATCH 06/10] Can now disable the update hash check --- .../com/rarchives/ripme/ui/UpdateUtils.java | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index ebfa1428..9b78a7c8 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -229,17 +229,20 @@ public class UpdateUtils { try (FileOutputStream out = new FileOutputStream(updateFileName)) { out.write(response.bodyAsBytes()); } - String updateHash = createSha256(new File(updateFileName)); - logger.info("Download of new version complete; saved to " + updateFileName); - logger.info("Checking hash of update"); + // Only check the hash if the user hasn't disabled hash checking + if (Utils.getConfigBoolean("security.check_update_hash", true)) { + String updateHash = createSha256(new File(updateFileName)); + logger.info("Download of new version complete; saved to " + updateFileName); + logger.info("Checking hash of update"); - if (!ripmeJson.getString("currentHash").equals(updateHash)) { - logger.error("Error: Update has bad hash"); - logger.debug("Expected hash: " + ripmeJson.getString("currentHash")); - logger.debug("Actual hash: " + updateHash); - throw new IOException("Got bad file hash"); - } else { - logger.info("Hash is good"); + if (!ripmeJson.getString("currentHash").equals(updateHash)) { + logger.error("Error: Update has bad hash"); + logger.debug("Expected hash: " + ripmeJson.getString("currentHash")); + logger.debug("Actual hash: " + updateHash); + throw new IOException("Got bad file hash"); + } else { + logger.info("Hash is good"); + } } if (shouldLaunch) { // Setup updater script From 1adc0a9bf75b204af1c6909de3e75f3db4afb865 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 2 Jun 2018 16:08:49 -0400 Subject: [PATCH 07/10] Added config option to make ripme always try to update (for testing purposes) --- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 9b78a7c8..868f00ac 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -159,6 +159,10 @@ public class UpdateUtils { } private static boolean isNewerVersion(String latestVersion) { + // If we're testing the update utils we want the program to always try to update + if (Utils.getConfigBoolean("testing.always_try_to_update", false)) { + return true; + } int[] oldVersions = versionStringToInt(getThisJarVersion()); int[] newVersions = versionStringToInt(latestVersion); if (oldVersions.length < newVersions.length) { From 9af46aad206a481324e245e802922454431c8e12 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 2 Jun 2018 16:14:48 -0400 Subject: [PATCH 08/10] Ripme now logs when it's forcing an update for testing --- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 868f00ac..b4659d9c 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -161,6 +161,7 @@ public class UpdateUtils { private static boolean isNewerVersion(String latestVersion) { // If we're testing the update utils we want the program to always try to update if (Utils.getConfigBoolean("testing.always_try_to_update", false)) { + logger.info("isNewerVersion is returning true because the key \"testing.always_try_to_update\" is true"); return true; } int[] oldVersions = versionStringToInt(getThisJarVersion()); From 416e5817e8d3d37148c93e223b195de130819eb6 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 2 Jun 2018 18:40:52 -0400 Subject: [PATCH 09/10] Added the ability to rip from vsco profiles --- .../ripme/ripper/rippers/VscoRipper.java | 103 +++++++++++++----- 1 file changed, 73 insertions(+), 30 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java index 00eed1f4..dc289514 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java @@ -6,10 +6,11 @@ import com.rarchives.ripme.utils.Http; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; -import java.util.ArrayList; -import java.util.List; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; + +import org.json.JSONObject; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -19,7 +20,11 @@ import org.jsoup.select.Elements; /** * For ripping VSCO pictures. */ -public class VscoRipper extends AbstractHTMLRipper{ +public class VscoRipper extends AbstractHTMLRipper { + + int pageNumber = 1; + JSONObject profileJSON; + private static final String DOMAIN = "vsco.co", HOST = "vsco"; @@ -76,30 +81,20 @@ public class VscoRipper extends AbstractHTMLRipper{ logger.debug("Failed to convert " + url.toString() + " to external form."); } - } else {//want to rip a member profile - /* - String baseURL = "https://vsco.co"; - - - //Find all the relative links, adds Base URL, then adds them to an ArrayList - List relativeLinks = new ArrayList<>(); - Elements links = page.getElementsByTag("a"); - - - for(Element link : links){ - System.out.println(link.toString()); - //if link includes "/media/", add it to the list - if (link.attr("href").contains("/media")) { - try { - String relativeURL = vscoImageToURL(link.attr("href")); - toRip.add(baseURL + relativeURL); - } catch (IOException ex) { - logger.debug("Could not add \"" + link.toString() + "\" to list for ripping."); - } + } else { + String username = getUserName(); + String userTkn = getUserTkn(username); + String siteID = getSiteID(userTkn, username); + while (true) { + profileJSON = getProfileJSON(userTkn, username, Integer.toString(pageNumber), siteID); + for (int i = 0; i < profileJSON.getJSONArray("media").length(); i++) { + toRip.add("https://" + profileJSON.getJSONArray("media").getJSONObject(i).getString("responsive_url")); } + if (pageNumber * 1000 > profileJSON.getInt("total")) { + return toRip; + } + pageNumber++; } - */ - logger.debug("Sorry, RipMe currently only supports ripping single images."); } @@ -107,6 +102,59 @@ public class VscoRipper extends AbstractHTMLRipper{ return toRip; } + private String getUserTkn(String username) { + String userinfoPage = "https://vsco.co/content/Static/userinfo"; + String referer = "https://vsco.co/" + username + "/images/1"; + Map cookies = new HashMap<>(); + cookies.put("vs_anonymous_id", UUID.randomUUID().toString()); + try { + Element doc = Http.url(userinfoPage).cookies(cookies).referrer(referer).ignoreContentType().get().body(); + String json = doc.text().replaceAll("define\\(", ""); + json = json.replaceAll("\\)", ""); + return new JSONObject(json).getString("tkn"); + } catch (IOException e) { + logger.error("Could not get user tkn"); + return null; + } + } + + private String getUserName() { + Pattern p = Pattern.compile("^https?://vsco.co/([a-zA-Z0-9]+)/images/[0-9]+"); + Matcher m = p.matcher(url.toExternalForm()); + + if (m.matches()) { + String user = m.group(1); + return user; + } + return null; + } + + private JSONObject getProfileJSON(String tkn, String username, String page, String siteId) { + String size = "1000"; + String purl = "https://vsco.co/ajxp/" + tkn + "/2.0/medias?site_id=" + siteId + "&page=" + page + "&size=" + size; + Map cookies = new HashMap<>(); + cookies.put("vs", tkn); + try { + JSONObject j = Http.url(purl).cookies(cookies).getJSON(); + return j; + } catch (IOException e) { + logger.error("Could not profile images"); + return null; + } + } + + private String getSiteID(String tkn, String username) { + Map cookies = new HashMap<>(); + cookies.put("vs", tkn); + try { + JSONObject j = Http.url("https://vsco.co/ajxp/" + tkn + "/2.0/sites?subdomain=" + username).cookies(cookies).getJSON(); + return Integer.toString(j.getJSONArray("sites").getJSONObject(0).getInt("id")); + } catch (IOException e) { + logger.error("Could not get site id"); + return null; + } + } + private String vscoImageToURL(String url) throws IOException{ Document page = Jsoup.connect(url).userAgent(USER_AGENT) .get(); @@ -177,11 +225,6 @@ public class VscoRipper extends AbstractHTMLRipper{ return Http.url(url).get(); } - @Override - public Document getNextPage(Document doc) throws IOException { - return super.getNextPage(doc); - } - @Override public void downloadURL(URL url, int index) { addURLToDownload(url, getPrefix(index)); From 7f4df8b567c15b12af3f83c77f77fa0125585b8a Mon Sep 17 00:00:00 2001 From: Gaboso Date: Sat, 2 Jun 2018 20:49:35 -0400 Subject: [PATCH 10/10] Updating the VideoRipper class: - ordered imports alphabetically; - removed signature of the canRip method, which is already in AbstractRipper; - split variable declaration in different lines; - corrected javadocs alignment; - replaced unnecessary use of StringBuilder. - added a few empty lines to increase the readability of the code --- .../rarchives/ripme/ripper/VideoRipper.java | 100 +++++++++--------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 6b1032e5..e289bc01 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -1,5 +1,9 @@ package com.rarchives.ripme.ripper; +import com.rarchives.ripme.ui.RipStatusMessage; +import com.rarchives.ripme.ui.RipStatusMessage.STATUS; +import com.rarchives.ripme.utils.Utils; + import java.io.File; import java.io.FileWriter; import java.io.IOException; @@ -7,29 +11,26 @@ import java.net.MalformedURLException; import java.net.URL; import java.util.Map; -import com.rarchives.ripme.ui.RipStatusMessage; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; -import com.rarchives.ripme.utils.Utils; -import com.sun.org.apache.xpath.internal.operations.Bool; - public abstract class VideoRipper extends AbstractRipper { - private int bytesTotal = 1, - bytesCompleted = 1; + private int bytesTotal = 1; + private int bytesCompleted = 1; protected VideoRipper(URL url) throws IOException { super(url); } - public abstract boolean canRip(URL url); public abstract void rip() throws IOException; + public abstract String getHost(); + public abstract String getGID(URL url) throws MalformedURLException; @Override public void setBytesTotal(int bytes) { this.bytesTotal = bytes; } + @Override public void setBytesCompleted(int bytes) { this.bytesCompleted = bytes; @@ -56,8 +57,7 @@ public abstract class VideoRipper extends AbstractRipper { logger.error("Error while writing to " + urlFile, e); return false; } - } - else { + } else { if (isThisATest()) { // Tests shouldn't download the whole video // Just change this.url to the download URL so the test knows we found it. @@ -71,52 +71,54 @@ public abstract class VideoRipper extends AbstractRipper { } @Override - public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { + public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { return addURLToDownload(url, saveAs); } - /** * Creates & sets working directory based on URL. - * @param url - * Target URL + * + * @param url Target URL */ @Override public void setWorkingDir(URL url) throws IOException { String path = Utils.getWorkingDirectory().getCanonicalPath(); + if (!path.endsWith(File.separator)) { path += File.separator; } + path += "videos" + File.separator; - this.workingDir = new File(path); - if (!this.workingDir.exists()) { - logger.info("[+] Creating directory: " + Utils.removeCWD(this.workingDir)); - this.workingDir.mkdirs(); + workingDir = new File(path); + + if (!workingDir.exists()) { + logger.info("[+] Creating directory: " + Utils.removeCWD(workingDir)); + workingDir.mkdirs(); } - logger.debug("Set working directory to: " + this.workingDir); + + logger.debug("Set working directory to: " + workingDir); } - + /** - * @return - * Returns % of video done downloading. + * @return Returns % of video done downloading. */ @Override public int getCompletionPercentage() { return (int) (100 * (bytesCompleted / (float) bytesTotal)); } - + /** * Runs if download successfully completed. - * @param url - * Target URL - * @param saveAs - * Path to file, including filename. + * + * @param url Target URL + * @param saveAs Path to file, including filename. */ @Override public void downloadCompleted(URL url, File saveAs) { if (observer == null) { return; } + try { String path = Utils.removeCWD(saveAs); RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); @@ -127,62 +129,58 @@ public abstract class VideoRipper extends AbstractRipper { logger.error("Exception while updating observer: ", e); } } - + /** * Runs if the download errored somewhere. - * @param url - * Target URL - * @param reason - * Reason why the download failed. + * + * @param url Target URL + * @param reason Reason why the download failed. */ @Override public void downloadErrored(URL url, String reason) { if (observer == null) { return; } + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason)); checkIfComplete(); } - - + /** * Runs if user tries to redownload an already existing File. - * @param url - * Target URL - * @param file - * Existing file + * + * @param url Target URL + * @param file Existing file */ @Override public void downloadExists(URL url, File file) { if (observer == null) { return; } + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file)); checkIfComplete(); } /** * Gets the status and changes it to a human-readable form. - * @return - * Status of current download. + * + * @return Status of current download. */ @Override public String getStatusText() { - StringBuilder sb = new StringBuilder(); - sb.append(getCompletionPercentage()) - .append("% ") - .append(" - ") - .append(Utils.bytesToHumanReadable(bytesCompleted)) - .append(" / ") - .append(Utils.bytesToHumanReadable(bytesTotal)); - return sb.toString(); + return String.valueOf(getCompletionPercentage()) + + "% - " + + Utils.bytesToHumanReadable(bytesCompleted) + + " / " + + Utils.bytesToHumanReadable(bytesTotal); } - @Override /** * Sanitizes URL. * Usually just returns itself. */ + @Override public URL sanitizeURL(URL url) throws MalformedURLException { return url; } @@ -195,8 +193,10 @@ public abstract class VideoRipper extends AbstractRipper { if (observer == null) { return; } + if (bytesCompleted >= bytesTotal) { super.checkIfComplete(); } } -} + +} \ No newline at end of file