Merge pull request #467 from cyian-1756/8musesFileNameFixes

Added another overload to addURLToDownload which allows the ripper to set the name of the file; Fixed 8muses filename length issue
This commit is contained in:
cyian-1756 2018-03-14 12:37:33 -04:00 committed by GitHub
commit c239d235e9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 34 additions and 9 deletions

View File

@ -174,7 +174,7 @@ public abstract class AbstractRipper
* URL of the file * URL of the file
* @param saveAs * @param saveAs
* Path of the local file to save the content to. * Path of the local file to save the content to.
* @return True on success, flase on failure. * @return True on success, false on failure.
*/ */
public abstract boolean addURLToDownload(URL url, File saveAs); public abstract boolean addURLToDownload(URL url, File saveAs);
@ -206,11 +206,13 @@ public abstract class AbstractRipper
* The HTTP referrer to use while downloading this file. * The HTTP referrer to use while downloading this file.
* @param cookies * @param cookies
* The cookies to send to the server while downloading this file. * The cookies to send to the server while downloading this file.
* @param fileName
* The name that file will be written to
* @return * @return
* True if downloaded successfully * True if downloaded successfully
* False if failed to download * False if failed to download
*/ */
protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map<String, String> cookies) { protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map<String, String> cookies, String fileName) {
// Don't re-add the url if it was downloaded in a previous rip // Don't re-add the url if it was downloaded in a previous rip
if (Utils.getConfigBoolean("remember.url_history", true) && !isThisATest()) { if (Utils.getConfigBoolean("remember.url_history", true) && !isThisATest()) {
if (hasDownloadedURL(url.toExternalForm())) { if (hasDownloadedURL(url.toExternalForm())) {
@ -225,9 +227,18 @@ public abstract class AbstractRipper
logger.debug("Ripper has been stopped"); logger.debug("Ripper has been stopped");
return false; return false;
} }
logger.debug("url: " + url + ", prefix: " + prefix + ", subdirectory" + subdirectory + ", referrer: " + referrer + ", cookies: " + cookies); logger.debug("url: " + url + ", prefix: " + prefix + ", subdirectory" + subdirectory + ", referrer: " + referrer + ", cookies: " + cookies + ", fileName: " + fileName);
String saveAs = url.toExternalForm(); String saveAs;
saveAs = saveAs.substring(saveAs.lastIndexOf('/')+1); if (fileName != null) {
saveAs = fileName;
// Get the extension of the file
String extension = url.toExternalForm().substring(url.toExternalForm().lastIndexOf(".") + 1);
saveAs = saveAs + "." + extension;
} else {
saveAs = url.toExternalForm();
saveAs = saveAs.substring(saveAs.lastIndexOf('/')+1);
}
if (saveAs.indexOf('?') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('?')); } if (saveAs.indexOf('?') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('?')); }
if (saveAs.indexOf('#') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('#')); } if (saveAs.indexOf('#') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('#')); }
if (saveAs.indexOf('&') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('&')); } if (saveAs.indexOf('&') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('&')); }
@ -274,7 +285,11 @@ public abstract class AbstractRipper
* @return True on success, flase on failure. * @return True on success, flase on failure.
*/ */
protected boolean addURLToDownload(URL url, String prefix, String subdirectory) { protected boolean addURLToDownload(URL url, String prefix, String subdirectory) {
return addURLToDownload(url, prefix, subdirectory, null, null); return addURLToDownload(url, prefix, subdirectory, null, null, null);
}
protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map<String, String> cookies) {
return addURLToDownload(url, prefix, subdirectory, referrer, cookies, null);
} }
/** /**
@ -290,6 +305,8 @@ public abstract class AbstractRipper
// Use empty subdirectory // Use empty subdirectory
return addURLToDownload(url, prefix, ""); return addURLToDownload(url, prefix, "");
} }
/** /**
* Waits for downloading threads to complete. * Waits for downloading threads to complete.
*/ */

View File

@ -11,6 +11,7 @@ import java.util.Map;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import com.rarchives.ripme.utils.Utils;
import org.jsoup.Connection.Response; import org.jsoup.Connection.Response;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
@ -125,7 +126,11 @@ public class EightmusesRipper extends AbstractHTMLRipper {
logger.info("Retrieving full-size image location from " + imageHref); logger.info("Retrieving full-size image location from " + imageHref);
image = getFullSizeImage(imageHref); image = getFullSizeImage(imageHref);
URL imageUrl = new URL(image); URL imageUrl = new URL(image);
addURLToDownload(imageUrl, getPrefix(x), getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies); if (Utils.getConfigBoolean("8muses.use_short_names", false)) {
addURLToDownload(imageUrl, getPrefixShort(x), getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies, "");
} else {
addURLToDownload(imageUrl, getPrefixLong(x), getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies);
}
// X is our page index // X is our page index
x++; x++;
@ -178,8 +183,11 @@ public class EightmusesRipper extends AbstractHTMLRipper {
addURLToDownload(url, getPrefix(index), "", this.url.toExternalForm(), cookies); addURLToDownload(url, getPrefix(index), "", this.url.toExternalForm(), cookies);
} }
@Override public String getPrefixLong(int index) {
public String getPrefix(int index) {
return String.format("%03d_", index); return String.format("%03d_", index);
} }
public String getPrefixShort(int index) {
return String.format("%03d", index);
}
} }