From 368646145d31b8e6d6647965e93c955c83b29e69 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 22 Jul 2018 19:02:06 -0400 Subject: [PATCH 1/4] Can now get file type from magic number --- .../rarchives/ripme/ripper/DownloadFileThread.java | 13 +++++++++++++ src/main/java/com/rarchives/ripme/utils/Utils.java | 10 ++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 4f189c3c..c699522e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -6,9 +6,11 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.lang.reflect.Array; import java.net.HttpURLConnection; import java.net.URL; import java.net.URLConnection; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -181,6 +183,17 @@ class DownloadFileThread extends Thread { saveAs = new File(saveAs.toString() + "." + fileExt); } else { logger.error("Was unable to get content type from stream"); + // Try to get the file type from the magic number + byte[] magicBytes = new byte[8]; + bis.read(magicBytes,0, 5); + bis.reset(); + fileExt = Utils.getEXTFromMagic(magicBytes); + if (fileExt != null) { + saveAs = new File(saveAs.toString() + "." + fileExt); + } else { + logger.error("Was unable to get content type using magic number"); + logger.error("Magic number was: " + Arrays.toString(magicBytes)); + } } } // If we're resuming a download we append data to the existing file diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index e57acf77..77a062d0 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -15,6 +15,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; +import java.lang.reflect.Array; import java.lang.reflect.Constructor; import java.net.URISyntaxException; import java.net.URL; @@ -731,4 +732,13 @@ public class Utils { Utils.bytesToHumanReadable(bytesTotal); } + public static String getEXTFromMagic(byte[] magic) { + if (Arrays.equals(magic, new byte[]{-1, -40, -1, -37, 0, 0, 0, 0})) { + return "jpeg"; + } else { + LOGGER.info("Unknown magic number " + Arrays.toString(magic)); + } + return null; + } + } \ No newline at end of file From 1d60bf695667a297e495ec86804abbf9a0686fdc Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 22 Jul 2018 19:31:11 -0400 Subject: [PATCH 2/4] Added a fuzzyExist func to check if a file exist when we don't know it's extension (For use with rippers that use getFileExtFromMIME) --- .../ripme/ripper/DownloadFileThread.java | 2 +- .../java/com/rarchives/ripme/utils/Utils.java | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index c699522e..f0d78ddb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -79,7 +79,7 @@ class DownloadFileThread extends Thread { observer.downloadErrored(url, "Download interrupted"); return; } - if (saveAs.exists() && !observer.tryResumeDownload()) { + if (saveAs.exists() && !observer.tryResumeDownload() || Utils.fuzzyExists(new File(saveAs.getParent()), saveAs.getName())) { if (Utils.getConfigBoolean("file.overwrite", false)) { logger.info("[!] Deleting existing file" + prettySaveAs); saveAs.delete(); diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 77a062d0..31af93a0 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -741,4 +741,26 @@ public class Utils { return null; } + // Checks if a file exists ignoring it's extension. + // Code from: https://stackoverflow.com/a/17698068 + public static boolean fuzzyExists(File folder, String fileName) { + if (!folder.exists()) { + return false; + } + File[] listOfFiles = folder.listFiles(); + if (listOfFiles == null) { + return false; + } + + for (File file : listOfFiles) { + if (file.isFile()) { + String[] filename = file.getName().split("\\.(?=[^\\.]+$)"); //split filename from it's extension + if(filename[0].equalsIgnoreCase(fileName)) { + return true; + } + } + } + return false; + } + } \ No newline at end of file From cb1fec52f551c1091c2c5940fff2ee2117f163cd Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Mon, 23 Jul 2018 03:12:03 -0400 Subject: [PATCH 3/4] saveAs.exists() imrpovements --- .../java/com/rarchives/ripme/ripper/DownloadFileThread.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index f0d78ddb..55b8ffde 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -79,7 +79,8 @@ class DownloadFileThread extends Thread { observer.downloadErrored(url, "Download interrupted"); return; } - if (saveAs.exists() && !observer.tryResumeDownload() || Utils.fuzzyExists(new File(saveAs.getParent()), saveAs.getName())) { + if (saveAs.exists() && !observer.tryResumeDownload() && !getFileExtFromMIME || + Utils.fuzzyExists(new File(saveAs.getParent()), saveAs.getName()) && getFileExtFromMIME && !observer.tryResumeDownload()) { if (Utils.getConfigBoolean("file.overwrite", false)) { logger.info("[!] Deleting existing file" + prettySaveAs); saveAs.delete(); From 58cdea1ee22ffff0166117a8625e80ed3404564d Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Mon, 23 Jul 2018 05:02:25 -0400 Subject: [PATCH 4/4] Added some unit tests --- .../com/rarchives/ripme/tst/UtilsTest.java | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/UtilsTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/UtilsTest.java b/src/test/java/com/rarchives/ripme/tst/UtilsTest.java new file mode 100644 index 00000000..d0789b39 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/UtilsTest.java @@ -0,0 +1,34 @@ +package com.rarchives.ripme.tst; + +import junit.framework.TestCase; +import com.rarchives.ripme.utils.Utils; + +public class UtilsTest extends TestCase { + + public void testGetEXTFromMagic() { + assertEquals("jpeg", Utils.getEXTFromMagic(new byte[]{-1, -40, -1, -37, 0, 0, 0, 0})); + } + + public void testStripURLParameter() { + assertEquals("http://example.tld/image.ext", + Utils.stripURLParameter("http://example.tld/image.ext?param", "param")); + } + + public void testShortenPath() { + String path = "/test/test/test/test/test/test/test/test/"; + assertEquals("/test/test1", Utils.shortenPath("/test/test1")); + assertEquals("/test/test/t...st/test/test", Utils.shortenPath(path)); + } + + public void testBytesToHumanReadable() { + assertEquals("10.00iB", Utils.bytesToHumanReadable(10)); + assertEquals("1.00KiB", Utils.bytesToHumanReadable(1024)); + assertEquals("1.00MiB", Utils.bytesToHumanReadable(1024 * 1024)); + assertEquals("1.00GiB", Utils.bytesToHumanReadable(1024 * 1024 * 1024)); + } + + public void testGetListOfAlbumRippers() throws Exception{ + assert(!Utils.getListOfAlbumRippers().isEmpty()); + } + +}