From d604d4059659fca6653d70266feaa049b6d7d1b5 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 5 May 2018 05:36:00 -0400 Subject: [PATCH] Added option for addURLToDownload which sets the file extension to the files MIME type --- .../com/rarchives/ripme/ripper/AbstractRipper.java | 11 ++++++++--- .../com/rarchives/ripme/ripper/AlbumRipper.java | 6 +++--- .../rarchives/ripme/ripper/DownloadFileThread.java | 13 +++++++++++-- .../com/rarchives/ripme/ripper/VideoRipper.java | 3 ++- .../ripme/ripper/rippers/FivehundredpxRipper.java | 2 +- .../ripme/ripper/rippers/TsuminoRipper.java | 6 +++++- 6 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 4438ae2a..20889495 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -192,7 +192,8 @@ public abstract class AbstractRipper * True if downloaded successfully * False if failed to download */ - protected abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies); + protected abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies, + Boolean getFileExtFromMIME); /** * Queues image to be downloaded and saved. @@ -212,7 +213,7 @@ public abstract class AbstractRipper * True if downloaded successfully * False if failed to download */ - protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName, String extension) { + protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName, String extension, Boolean getFileExtFromMIME) { // Don't re-add the url if it was downloaded in a previous rip if (Utils.getConfigBoolean("remember.url_history", true) && !isThisATest()) { if (hasDownloadedURL(url.toExternalForm())) { @@ -257,7 +258,11 @@ public abstract class AbstractRipper logger.debug("Unable to write URL history file"); } } - return addURLToDownload(url, saveFileAs, referrer, cookies); + return addURLToDownload(url, saveFileAs, referrer, cookies, getFileExtFromMIME); + } + + protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName, String extension) { + return addURLToDownload(url, prefix, subdirectory, referrer, cookies, fileName, extension, false); } protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName) { diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 1726343a..f700f012 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -50,7 +50,7 @@ public abstract class AlbumRipper extends AbstractRipper { /** * Queues multiple URLs of single images to download from a single Album URL */ - public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies) { + public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { // Only download one file if this is a test. if (super.isThisATest() && (itemsPending.size() > 0 || itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { @@ -82,7 +82,7 @@ public abstract class AlbumRipper extends AbstractRipper { } else { itemsPending.put(url, saveAs); - DownloadFileThread dft = new DownloadFileThread(url, saveAs, this); + DownloadFileThread dft = new DownloadFileThread(url, saveAs, this, getFileExtFromMIME); if (referrer != null) { dft.setReferrer(referrer); } @@ -96,7 +96,7 @@ public abstract class AlbumRipper extends AbstractRipper { @Override public boolean addURLToDownload(URL url, File saveAs) { - return addURLToDownload(url, saveAs, null, null); + return addURLToDownload(url, saveAs, null, null, false); } /** diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index c62d58a6..42dedffe 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -8,6 +8,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URL; +import java.net.URLConnection; import java.util.HashMap; import java.util.Map; @@ -36,10 +37,11 @@ class DownloadFileThread extends Thread { private String prettySaveAs; private AbstractRipper observer; private int retries; + private Boolean getFileExtFromMIME; private final int TIMEOUT; - public DownloadFileThread(URL url, File saveAs, AbstractRipper observer) { + public DownloadFileThread(URL url, File saveAs, AbstractRipper observer, Boolean getFileExtFromMIME) { super(); this.url = url; this.saveAs = saveAs; @@ -47,6 +49,7 @@ class DownloadFileThread extends Thread { this.observer = observer; this.retries = Utils.getConfigInteger("download.retries", 1); this.TIMEOUT = Utils.getConfigInteger("download.timeout", 60000); + this.getFileExtFromMIME = getFileExtFromMIME; } public void setReferrer(String referrer) { @@ -143,9 +146,15 @@ class DownloadFileThread extends Thread { observer.downloadErrored(url, "Imgur image is 404: " + url.toExternalForm()); return; } - // Save file bis = new BufferedInputStream(huc.getInputStream()); + + // Check if we should get the file ext from the MIME type + if (getFileExtFromMIME) { + String fileExt = URLConnection.guessContentTypeFromStream(bis).replaceAll("image/", ""); + saveAs = new File(saveAs.toString() + "." + fileExt); + } + fos = new FileOutputStream(saveAs); IOUtils.copy(bis, fos); break; // Download successful: break out of infinite loop diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 13008cd9..29200d5a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -10,6 +10,7 @@ import java.util.Map; import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Utils; +import com.sun.org.apache.xpath.internal.operations.Bool; public abstract class VideoRipper extends AbstractRipper { @@ -70,7 +71,7 @@ public abstract class VideoRipper extends AbstractRipper { } @Override - public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies) { + public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { return addURLToDownload(url, saveAs); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java index 7e532943..93aedba2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java @@ -332,7 +332,7 @@ public class FivehundredpxRipper extends AbstractJSONRipper { String[] fields = u.split("/"); String prefix = getPrefix(index) + fields[fields.length - 3]; File saveAs = new File(getWorkingDir() + File.separator + prefix + ".jpg"); - addURLToDownload(url, saveAs, "", null); + addURLToDownload(url, saveAs, "", null, false); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java index 6030c9f0..7d35fc1d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java @@ -98,6 +98,10 @@ public class TsuminoRipper extends AbstractHTMLRipper { @Override public void downloadURL(URL url, int index) { sleep(1000); - addURLToDownload(url, getPrefix(index), "", null, null, null, "png"); + /* + There is no way to tell if an image returned from tsumino.com is a png to jpg. The content-type header is always + "image/jpeg" even when the image is a png. The file ext is not included in the url. + */ + addURLToDownload(url, getPrefix(index), "", null, null, null, null, true); } }