Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Can amlost skip already downloaded urls

Browse Source
This commit is contained in:
cyian-1756 2017-11-05 14:35:15 -05:00
parent b2b5f4175b
commit d11443e8ec
1 changed files with 51 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
View File

@ -20,6 +20,10 @@ import com.rarchives.ripme.ui.RipStatusHandler;
import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Utils; import com.rarchives.ripme.utils.Utils;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.util.Scanner;
public abstract class AbstractRipper public abstract class AbstractRipper
extends Observable extends Observable
@ -56,6 +60,44 @@ public abstract class AbstractRipper
} }
} }
private void writeDownloadedURL(String downloadedURL) throws IOException {
BufferedWriter bw = null;
FileWriter fw = null;
try {
File file = new File("history.downloaded");
// if file doesnt exists, then create it
if (!file.exists()) {
file.createNewFile();
}
fw = new FileWriter(file.getAbsoluteFile(), true);
bw = new BufferedWriter(fw);
bw.write(downloadedURL);
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (bw != null)
bw.close();
if (fw != null)
fw.close();
} catch (IOException ex) {
ex.printStackTrace();
}
}
}
private boolean hasDownloadedURL(String url) {
Scanner scanner = new Scanner("history.downloaded");
while (scanner.hasNextLine()) {
final String lineFromFile = scanner.nextLine();
if(lineFromFile.contains(url)) {
return true;
}
}
return false;
}
/** /**
* Ensures inheriting ripper can rip this URL, raises exception if not. * Ensures inheriting ripper can rip this URL, raises exception if not.
* Otherwise initializes working directory and thread pool. * Otherwise initializes working directory and thread pool.
@ -113,6 +155,10 @@ public abstract class AbstractRipper
protected abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String, String> cookies); protected abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String, String> cookies);
protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map<String, String> cookies) { protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map<String, String> cookies) {
if (!hasDownloadedURL(url.toExternalForm())) {
sendUpdate(STATUS.DOWNLOAD_WARN, "Already downloaded " + url.toExternalForm());
return false;
}
try { try {
stopCheck(); stopCheck();
} catch (IOException e) { } catch (IOException e) {
@ -146,6 +192,11 @@ public abstract class AbstractRipper
logger.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent())); logger.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent()));
saveFileAs.getParentFile().mkdirs(); saveFileAs.getParentFile().mkdirs();
} }
try {
writeDownloadedURL(url.toExternalForm() + "\n");
} catch (IOException e) {
logger.debug("Unable to write URL history file");
}
return addURLToDownload(url, saveFileAs, referrer, cookies); return addURLToDownload(url, saveFileAs, referrer, cookies);
} }