Merge pull request #823 from cyian-1756/magicNumbers

Can now get file type from magic number; No longer erroneously redownloads file if getFileExtFromMIME is true

src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java

@@ -6,9 +6,11 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.lang.reflect.Array;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.net.URLConnection;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -77,7 +79,8 @@ class DownloadFileThread extends Thread {
             observer.downloadErrored(url, "Download interrupted");
             return;
         }
-        if (saveAs.exists() && !observer.tryResumeDownload()) {
+        if (saveAs.exists() && !observer.tryResumeDownload() && !getFileExtFromMIME ||
+                Utils.fuzzyExists(new File(saveAs.getParent()), saveAs.getName()) && getFileExtFromMIME && !observer.tryResumeDownload()) {
             if (Utils.getConfigBoolean("file.overwrite", false)) {
                 logger.info("[!] Deleting existing file" + prettySaveAs);
                 saveAs.delete();
@@ -181,6 +184,17 @@ class DownloadFileThread extends Thread {
                         saveAs = new File(saveAs.toString() + "." + fileExt);
                     } else {
                         logger.error("Was unable to get content type from stream");
+                        // Try to get the file type from the magic number
+                        byte[] magicBytes = new byte[8];
+                        bis.read(magicBytes,0, 5);
+                        bis.reset();
+                        fileExt = Utils.getEXTFromMagic(magicBytes);
+                        if (fileExt != null) {
+                            saveAs = new File(saveAs.toString() + "." + fileExt);
+                        } else {
+                            logger.error("Was unable to get content type using magic number");
+                            logger.error("Magic number was: " + Arrays.toString(magicBytes));
+                        }
                     }
                 }
                 // If we're resuming a download we append data to the existing file

src/main/java/com/rarchives/ripme/utils/Utils.java

@@ -15,6 +15,7 @@ import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Array;
 import java.lang.reflect.Constructor;
 import java.net.URISyntaxException;
 import java.net.URL;
@@ -731,4 +732,35 @@ public class Utils {
                 Utils.bytesToHumanReadable(bytesTotal);
     }
 
+    public static String getEXTFromMagic(byte[] magic) {
+        if (Arrays.equals(magic, new byte[]{-1, -40, -1, -37, 0, 0, 0, 0})) {
+            return "jpeg";
+        } else {
+            LOGGER.info("Unknown magic number " + Arrays.toString(magic));
+        }
+        return null;
+    }
+
+    // Checks if a file exists ignoring it's extension.
+    // Code from: https://stackoverflow.com/a/17698068
+    public static boolean fuzzyExists(File folder, String fileName) {
+        if (!folder.exists()) {
+            return false;
+        }
+        File[] listOfFiles = folder.listFiles();
+        if (listOfFiles == null) {
+            return false;
+        }
+
+        for (File file : listOfFiles) {
+            if (file.isFile()) {
+                String[] filename = file.getName().split("\\.(?=[^\\.]+$)"); //split filename from it's extension
+                if(filename[0].equalsIgnoreCase(fileName)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
 }

src/test/java/com/rarchives/ripme/tst/UtilsTest.java

@@ -0,0 +1,34 @@
+package com.rarchives.ripme.tst;
+
+import junit.framework.TestCase;
+import com.rarchives.ripme.utils.Utils;
+
+public class UtilsTest extends TestCase {
+
+    public void testGetEXTFromMagic() {
+        assertEquals("jpeg", Utils.getEXTFromMagic(new byte[]{-1, -40, -1, -37, 0, 0, 0, 0}));
+    }
+
+    public void testStripURLParameter() {
+        assertEquals("http://example.tld/image.ext",
+                Utils.stripURLParameter("http://example.tld/image.ext?param", "param"));
+    }
+
+    public void testShortenPath() {
+        String path = "/test/test/test/test/test/test/test/test/";
+        assertEquals("/test/test1", Utils.shortenPath("/test/test1"));
+        assertEquals("/test/test/t...st/test/test", Utils.shortenPath(path));
+    }
+
+    public void testBytesToHumanReadable() {
+        assertEquals("10.00iB", Utils.bytesToHumanReadable(10));
+        assertEquals("1.00KiB", Utils.bytesToHumanReadable(1024));
+        assertEquals("1.00MiB", Utils.bytesToHumanReadable(1024 * 1024));
+        assertEquals("1.00GiB", Utils.bytesToHumanReadable(1024 * 1024 * 1024));
+    }
+
+    public void testGetListOfAlbumRippers() throws Exception{
+        assert(!Utils.getListOfAlbumRippers().isEmpty());
+    }
+
+}