Merge pull request #559 from cyian-1756/ContentType-changes

addURLToDownload can now set the file extension to the files MIME type

src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java

@@ -192,7 +192,8 @@ public abstract class AbstractRipper
      *      True if downloaded successfully
      *      False if failed to download
      */
-    protected abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String, String> cookies);
+    protected abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String, String> cookies,
+                                                Boolean getFileExtFromMIME);
 
     /**
      * Queues image to be downloaded and saved.
@@ -212,7 +213,7 @@ public abstract class AbstractRipper
      *      True if downloaded successfully
      *      False if failed to download
      */
-    protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map<String, String> cookies, String fileName, String extension) {
+    protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map<String, String> cookies, String fileName, String extension, Boolean getFileExtFromMIME) {
         // Don't re-add the url if it was downloaded in a previous rip
         if (Utils.getConfigBoolean("remember.url_history", true) && !isThisATest()) {
             if (hasDownloadedURL(url.toExternalForm())) {
@@ -257,7 +258,11 @@ public abstract class AbstractRipper
                 logger.debug("Unable to write URL history file");
             }
         }
-        return addURLToDownload(url, saveFileAs, referrer, cookies);
+        return addURLToDownload(url, saveFileAs, referrer, cookies, getFileExtFromMIME);
+    }
+
+    protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map<String,String> cookies, String fileName, String extension) {
+        return addURLToDownload(url, prefix, subdirectory, referrer, cookies, fileName, extension, false);
     }
 
     protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map<String, String> cookies, String fileName) {

src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java

@@ -50,7 +50,7 @@ public abstract class AlbumRipper extends AbstractRipper {
     /**
      * Queues multiple URLs of single images to download from a single Album URL
      */
-    public boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies) {
+    public boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies, Boolean getFileExtFromMIME) {
         // Only download one file if this is a test.
         if (super.isThisATest() &&
                 (itemsPending.size() > 0 || itemsCompleted.size() > 0 || itemsErrored.size() > 0)) {
@@ -82,7 +82,7 @@ public abstract class AlbumRipper extends AbstractRipper {
         }
         else {
             itemsPending.put(url, saveAs);
-            DownloadFileThread dft = new DownloadFileThread(url,  saveAs,  this);
+            DownloadFileThread dft = new DownloadFileThread(url,  saveAs,  this, getFileExtFromMIME);
             if (referrer != null) {
                 dft.setReferrer(referrer);
             }
@@ -96,7 +96,7 @@ public abstract class AlbumRipper extends AbstractRipper {
 
     @Override
     public boolean addURLToDownload(URL url, File saveAs) {
-        return addURLToDownload(url, saveAs, null, null);
+        return addURLToDownload(url, saveAs, null, null, false);
     }
 
     /**

src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java

@@ -8,6 +8,7 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.HttpURLConnection;
 import java.net.URL;
+import java.net.URLConnection;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -36,10 +37,11 @@ class DownloadFileThread extends Thread {
     private String prettySaveAs;
     private AbstractRipper observer;
     private int retries;
+    private Boolean getFileExtFromMIME;
 
     private final int TIMEOUT;
 
-    public DownloadFileThread(URL url, File saveAs, AbstractRipper observer) {
+    public DownloadFileThread(URL url, File saveAs, AbstractRipper observer, Boolean getFileExtFromMIME) {
         super();
         this.url = url;
         this.saveAs = saveAs;
@@ -47,6 +49,7 @@ class DownloadFileThread extends Thread {
         this.observer = observer;
         this.retries = Utils.getConfigInteger("download.retries", 1);
         this.TIMEOUT = Utils.getConfigInteger("download.timeout", 60000);
+        this.getFileExtFromMIME = getFileExtFromMIME;
     }
 
     public void setReferrer(String referrer) {
@@ -143,9 +146,15 @@ class DownloadFileThread extends Thread {
                     observer.downloadErrored(url, "Imgur image is 404: " + url.toExternalForm());
                     return;
                 }
-
                 // Save file
                 bis = new BufferedInputStream(huc.getInputStream());
+
+                // Check if we should get the file ext from the MIME type
+                if (getFileExtFromMIME) {
+                    String fileExt = URLConnection.guessContentTypeFromStream(bis).replaceAll("image/", "");
+                    saveAs = new File(saveAs.toString() + "." + fileExt);
+                }
+
                 fos = new FileOutputStream(saveAs);
                 IOUtils.copy(bis, fos);
                 break; // Download successful: break out of infinite loop

src/main/java/com/rarchives/ripme/ripper/VideoRipper.java

@@ -10,6 +10,7 @@ import java.util.Map;
 import com.rarchives.ripme.ui.RipStatusMessage;
 import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
 import com.rarchives.ripme.utils.Utils;
+import com.sun.org.apache.xpath.internal.operations.Bool;
 
 public abstract class VideoRipper extends AbstractRipper {
 
@@ -70,7 +71,7 @@ public abstract class VideoRipper extends AbstractRipper {
     }
 
     @Override
-    public boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies) {
+    public boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies, Boolean getFileExtFromMIME) {
         return addURLToDownload(url, saveAs);
     }
 

src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java

@@ -332,7 +332,7 @@ public class FivehundredpxRipper extends AbstractJSONRipper {
         String[] fields = u.split("/");
         String prefix = getPrefix(index) + fields[fields.length - 3];
         File saveAs = new File(getWorkingDir() + File.separator + prefix + ".jpg");
-        addURLToDownload(url,  saveAs,  "", null);
+        addURLToDownload(url,  saveAs,  "", null, false);
     }
 
 }

src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java

@@ -98,6 +98,10 @@ public class TsuminoRipper extends AbstractHTMLRipper {
     @Override
     public void downloadURL(URL url, int index) {
         sleep(1000);
-        addURLToDownload(url, getPrefix(index), "", null, null, null, "png");
+        /*
+        There is no way to tell if an image returned from tsumino.com is a png to jpg. The content-type header is always
+        "image/jpeg" even when the image is a png. The file ext is not included in the url.
+         */
+        addURLToDownload(url, getPrefix(index), "", null, null, null, null, true);
     }
 }