From 9a8888d445b45216e7442c4f9bad79e1a660054f Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 31 May 2018 16:42:06 -0400 Subject: [PATCH 1/5] Ripper now can download videos again and no longer uses the same api key every time --- .../ripme/ripper/rippers/TumblrRipper.java | 63 +++++++++++-------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java index 89884854..41da5d13 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java @@ -23,8 +23,8 @@ import com.rarchives.ripme.utils.Utils; public class TumblrRipper extends AlbumRipper { private static final String DOMAIN = "tumblr.com", - HOST = "tumblr", - IMAGE_PATTERN = "([^\\s]+(\\.(?i)(jpg|png|gif|bmp))$)"; + HOST = "tumblr", + IMAGE_PATTERN = "([^\\s]+(\\.(?i)(jpg|png|gif|bmp))$)"; private enum ALBUM_TYPE { SUBDOMAIN, @@ -37,11 +37,8 @@ public class TumblrRipper extends AlbumRipper { private static final String TUMBLR_AUTH_CONFIG_KEY = "tumblr.auth"; private static boolean useDefaultApiKey = false; // fall-back for bad user-specified key - private static final List APIKEYS = Arrays.asList("JFNLu3CbINQjRdUvZibXW9VpSEVYYtiPJ86o8YmvgLZIoKyuNX", - "FQrwZMCxVnzonv90rgNUJcAk4FpnoS0mYuSuGYqIpM2cFgp9L4", - "qpdkY6nMknksfvYAhf2xIHp0iNRLkMlcWShxqzXyFJRxIsZ1Zz"); - private static int genNum = new Random().nextInt(APIKEYS.size()); - private static final String API_KEY = APIKEYS.get(genNum); // Select random API key from APIKEYS + private static String API_KEY = null; + /** * Gets the API key. @@ -49,6 +46,10 @@ public class TumblrRipper extends AlbumRipper { * @return Tumblr API key */ public static String getApiKey() { + if (API_KEY == null) { + API_KEY = pickRandomApiKey(); + } + if (useDefaultApiKey || Utils.getConfigString(TUMBLR_AUTH_CONFIG_KEY, "JFNLu3CbINQjRdUvZibXW9VpSEVYYtiPJ86o8YmvgLZIoKyuNX").equals("JFNLu3CbINQjRdUvZibXW9VpSEVYYtiPJ86o8YmvgLZIoKyuNX")) { logger.info("Using api key: " + API_KEY); return API_KEY; @@ -60,9 +61,19 @@ public class TumblrRipper extends AlbumRipper { } + private static String pickRandomApiKey() { + final List APIKEYS = Arrays.asList("JFNLu3CbINQjRdUvZibXW9VpSEVYYtiPJ86o8YmvgLZIoKyuNX", + "FQrwZMCxVnzonv90rgNUJcAk4FpnoS0mYuSuGYqIpM2cFgp9L4", + "qpdkY6nMknksfvYAhf2xIHp0iNRLkMlcWShxqzXyFJRxIsZ1Zz"); + int genNum = new Random().nextInt(APIKEYS.size()); + logger.info(genNum); + final String API_KEY = APIKEYS.get(genNum); // Select random API key from APIKEYS + return API_KEY; + } + public TumblrRipper(URL url) throws IOException { super(url); - if (API_KEY == null) { + if (getApiKey() == null) { throw new IOException("Could not find tumblr authentication key in configuration"); } } @@ -100,7 +111,7 @@ public class TumblrRipper extends AlbumRipper { checkURL += "/info?api_key=" + getApiKey(); try { JSONObject json = Http.url(checkURL) - .getJSON(); + .getJSON(); int status = json.getJSONObject("meta").getInt("status"); return status == 200; } catch (IOException e) { @@ -245,11 +256,11 @@ public class TumblrRipper extends AlbumRipper { } } else if (post.has("video_url")) { try { - fileURL = new URL(post.getString("video_url").replaceAll("http", "https")); + fileURL = new URL(post.getString("video_url").replaceAll("http:", "https:")); addURLToDownload(fileURL); } catch (Exception e) { - logger.error("[!] Error while parsing video in " + post, e); - return true; + logger.error("[!] Error while parsing video in " + post, e); + return true; } } if (albumType == ALBUM_TYPE.POST) { @@ -263,24 +274,24 @@ public class TumblrRipper extends AlbumRipper { StringBuilder sb = new StringBuilder(); if (albumType == ALBUM_TYPE.POST) { sb.append("http://api.tumblr.com/v2/blog/") - .append(subdomain) - .append("/posts?id=") - .append(postNumber) - .append("&api_key=") - .append(getApiKey()); + .append(subdomain) + .append("/posts?id=") + .append(postNumber) + .append("&api_key=") + .append(getApiKey()); return sb.toString(); } sb.append("http://api.tumblr.com/v2/blog/") - .append(subdomain) - .append("/posts/") - .append(mediaType) - .append("?api_key=") - .append(getApiKey()) - .append("&offset=") - .append(offset); + .append(subdomain) + .append("/posts/") + .append(mediaType) + .append("?api_key=") + .append(getApiKey()) + .append("&offset=") + .append(offset); if (albumType == ALBUM_TYPE.TAG) { - sb.append("&tag=") - .append(tagName); + sb.append("&tag=") + .append(tagName); } return sb.toString(); } From 30a2d015aa1a865f1a666711c3b887ec997f53fc Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 1 Jun 2018 16:00:32 -0400 Subject: [PATCH 2/5] Added -j flag for updating ripme --- src/main/java/com/rarchives/ripme/App.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index a4da1cae..81dafce5 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -241,6 +241,10 @@ public class App { ripURL(url, !cl.hasOption("n")); } + if (cl.hasOption('j')) { + UpdateUtils.updateProgramCLI(); + } + } /** @@ -290,6 +294,7 @@ public class App { opts.addOption("v", "version", false, "Show current version"); opts.addOption("s", "socks-server", true, "Use socks server ([user:password]@host[:port])"); opts.addOption("p", "proxy-server", true, "Use HTTP Proxy server ([user:password]@host[:port])"); + opts.addOption("j", "update", false, "Update ripme"); return opts; } From 7dc8b23aae9adb83ed0b6232806dec03e840e9ae Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 1 Jun 2018 16:01:13 -0400 Subject: [PATCH 3/5] Can now update ripme using the -j flag --- .../com/rarchives/ripme/ui/MainWindow.java | 4 +- .../com/rarchives/ripme/ui/UpdateUtils.java | 160 ++++++++++++------ 2 files changed, 108 insertions(+), 56 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 34904e2c..8d881fdf 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -198,7 +198,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { if (!configurationPanel.isVisible()) { optionConfiguration.doClick(); } - Runnable r = () -> UpdateUtils.updateProgram(configUpdateLabel); + Runnable r = () -> UpdateUtils.updateProgramGUI(configUpdateLabel); new Thread(r).start(); } @@ -786,7 +786,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { } }); configUpdateButton.addActionListener(arg0 -> { - Thread t = new Thread(() -> UpdateUtils.updateProgram(configUpdateLabel)); + Thread t = new Thread(() -> UpdateUtils.updateProgramGUI(configUpdateLabel)); t.start(); }); configLogLevelCombobox.addActionListener(arg0 -> { diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 5aba9bf4..61717142 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -20,7 +20,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.50"; + private static final String DEFAULT_VERSION = "1.7.49"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; @@ -39,8 +39,57 @@ public class UpdateUtils { } return thisVersion; } + public static void updateProgramCLI() { + logger.info("Checking for update..."); - public static void updateProgram(JLabel configUpdateLabel) { + Document doc = null; + try { + logger.debug("Retrieving " + UpdateUtils.updateJsonURL); + doc = Jsoup.connect(UpdateUtils.updateJsonURL) + .timeout(10 * 1000) + .ignoreContentType(true) + .get(); + } catch (IOException e) { + logger.error("Error while fetching update: ", e); + JOptionPane.showMessageDialog(null, + "Error while fetching update: " + e.getMessage() + "", + "RipMe Updater", + JOptionPane.ERROR_MESSAGE); + return; + } finally { + logger.info("Current version: " + getThisJarVersion()); + } + String jsonString = doc.body().html().replaceAll(""", "\""); + ripmeJson = new JSONObject(jsonString); + JSONArray jsonChangeList = ripmeJson.getJSONArray("changeList"); + StringBuilder changeList = new StringBuilder(); + for (int i = 0; i < jsonChangeList.length(); i++) { + String change = jsonChangeList.getString(i); + if (change.startsWith(UpdateUtils.getThisJarVersion() + ":")) { + break; + } + changeList.append("
+ ").append(change); + } + + String latestVersion = ripmeJson.getString("latestVersion"); + if (UpdateUtils.isNewerVersion(latestVersion)) { + logger.info("Found newer version: " + latestVersion); + logger.info("Downloading new version..."); + logger.info("New version found, downloading..."); + try { + UpdateUtils.downloadJarAndLaunch(getUpdateJarURL(latestVersion), false); + } catch (IOException e) { + logger.error("Error while updating: ", e); + } + } else { + logger.debug("This version (" + UpdateUtils.getThisJarVersion() + + ") is the same or newer than the website's version (" + latestVersion + ")"); + logger.info("v" + UpdateUtils.getThisJarVersion() + " is the latest version"); + logger.debug("Running latest version: " + UpdateUtils.getThisJarVersion()); + } + } + + public static void updateProgramGUI(JLabel configUpdateLabel) { configUpdateLabel.setText("Checking for update..."); Document doc = null; @@ -90,7 +139,7 @@ public class UpdateUtils { configUpdateLabel.setText("Downloading new version..."); logger.info("New version found, downloading..."); try { - UpdateUtils.downloadJarAndLaunch(getUpdateJarURL(latestVersion)); + UpdateUtils.downloadJarAndLaunch(getUpdateJarURL(latestVersion), true); } catch (IOException e) { JOptionPane.showMessageDialog(null, "Error while updating: " + e.getMessage(), @@ -166,7 +215,7 @@ public class UpdateUtils { return null; } - private static void downloadJarAndLaunch(String updateJarURL) + private static void downloadJarAndLaunch(String updateJarURL, Boolean shouldLaunch) throws IOException { Response response; response = Jsoup.connect(updateJarURL) @@ -190,58 +239,61 @@ public class UpdateUtils { } else { logger.info("Hash is good"); } + if (shouldLaunch) { + // Setup updater script + final String batchFile, script; + final String[] batchExec; + String os = System.getProperty("os.name").toLowerCase(); + if (os.contains("win")) { + // Windows + batchFile = "update_ripme.bat"; + String batchPath = new File(batchFile).getAbsolutePath(); + script = "@echo off\r\n" + + "timeout 1" + "\r\n" + + "copy " + updateFileName + " " + mainFileName + "\r\n" + + "del " + updateFileName + "\r\n" + + "ripme.jar" + "\r\n" + + "del " + batchPath + "\r\n"; + batchExec = new String[]{batchPath}; - // Setup updater script - final String batchFile, script; - final String[] batchExec; - String os = System.getProperty("os.name").toLowerCase(); - if (os.contains("win")) { - // Windows - batchFile = "update_ripme.bat"; - String batchPath = new File(batchFile).getAbsolutePath(); - script = "@echo off\r\n" - + "timeout 1" + "\r\n" - + "copy " + updateFileName + " " + mainFileName + "\r\n" - + "del " + updateFileName + "\r\n" - + "ripme.jar" + "\r\n" - + "del " + batchPath + "\r\n"; - batchExec = new String[] { batchPath }; - - } - else { - // Mac / Linux - batchFile = "update_ripme.sh"; - String batchPath = new File(batchFile).getAbsolutePath(); - script = "#!/bin/sh\n" - + "sleep 1" + "\n" - + "cd " + new File(mainFileName).getAbsoluteFile().getParent() + "\n" - + "cp -f " + updateFileName + " " + mainFileName + "\n" - + "rm -f " + updateFileName + "\n" - + "java -jar \"" + new File(mainFileName).getAbsolutePath() + "\" &\n" - + "sleep 1" + "\n" - + "rm -f " + batchPath + "\n"; - batchExec = new String[] { "sh", batchPath }; - } - - // Create updater script - try (BufferedWriter bw = new BufferedWriter(new FileWriter(batchFile))) { - bw.write(script); - bw.flush(); - } - - logger.info("Saved update script to " + batchFile); - // Run updater script on exit - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - try { - logger.info("Executing: " + batchFile); - Runtime.getRuntime().exec(batchExec); - } catch (IOException e) { - //TODO implement proper stack trace handling this is really just intented as a placeholder until you implement proper error handling - e.printStackTrace(); + } else { + // Mac / Linux + batchFile = "update_ripme.sh"; + String batchPath = new File(batchFile).getAbsolutePath(); + script = "#!/bin/sh\n" + + "sleep 1" + "\n" + + "cd " + new File(mainFileName).getAbsoluteFile().getParent() + "\n" + + "cp -f " + updateFileName + " " + mainFileName + "\n" + + "rm -f " + updateFileName + "\n" + + "java -jar \"" + new File(mainFileName).getAbsolutePath() + "\" &\n" + + "sleep 1" + "\n" + + "rm -f " + batchPath + "\n"; + batchExec = new String[]{"sh", batchPath}; } - })); - logger.info("Exiting older version, should execute update script (" + batchFile + ") during exit"); - System.exit(0); + + // Create updater script + try (BufferedWriter bw = new BufferedWriter(new FileWriter(batchFile))) { + bw.write(script); + bw.flush(); + } + + logger.info("Saved update script to " + batchFile); + // Run updater script on exit + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { + logger.info("Executing: " + batchFile); + Runtime.getRuntime().exec(batchExec); + } catch (IOException e) { + //TODO implement proper stack trace handling this is really just intented as a placeholder until you implement proper error handling + e.printStackTrace(); + } + })); + logger.info("Exiting older version, should execute update script (" + batchFile + ") during exit"); + System.exit(0); + } else { + new File(mainFileName).delete(); + new File(updateFileName).renameTo(new File(mainFileName)); + } } } From 2bb7508e9b5f5663a5de98fff94e28ab283b974a Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 1 Jun 2018 18:17:33 -0400 Subject: [PATCH 4/5] Undid DEFAULT_VERSION change --- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 61717142..95af45c6 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -20,7 +20,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.49"; + private static final String DEFAULT_VERSION = "1.7.50"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From d8d2e50551778c191e5a763096ea13e8f386685b Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 1 Jun 2018 18:20:48 -0400 Subject: [PATCH 5/5] currentHash now matches the hash of ripme version 1.7.50 --- ripme.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ripme.json b/ripme.json index 41071c2e..b2a8e044 100644 --- a/ripme.json +++ b/ripme.json @@ -1,5 +1,5 @@ { - "currentHash": "689d4a5b270b62be7defbdbb696de79c8a223ab7497391e1d610eaf1cb860510", + "currentHash": "f6e1e6c931abfbeffdd37dabb65f83e4335ca11ccc017f31e1d835ee6e6bec7a", "changeList": [ "1.7.50: Ripme now checks file hash before running update; fixed update bug which cased ripme to report every update as new", "1.7.49: Fixed -n flag; Added ability to change locale at runtime and from gui; Update kr_KR translation; Removed support for tnbtu.com; No longer writes url to url_history file is save urls only is checked",