Added abilty to blacklist tags on tsumino

This commit is contained in:
cyian-1756 2018-09-13 07:17:55 -04:00
parent d35fd511db
commit 26e27a9ed5
2 changed files with 60 additions and 3 deletions

View File

@ -12,6 +12,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage;
import com.rarchives.ripme.utils.Utils;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONObject; import org.json.JSONObject;
import org.jsoup.Connection; import org.jsoup.Connection;
@ -21,6 +22,7 @@ import org.jsoup.nodes.Document;
import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Http;
import org.jsoup.nodes.Element;
public class TsuminoRipper extends AbstractHTMLRipper { public class TsuminoRipper extends AbstractHTMLRipper {
private Map<String,String> cookies = new HashMap<>(); private Map<String,String> cookies = new HashMap<>();
@ -29,6 +31,41 @@ public class TsuminoRipper extends AbstractHTMLRipper {
super(url); super(url);
} }
private List<String> getTags(Document doc) {
List<String> tags = new ArrayList<>();
LOGGER.info("Getting tags");
for (Element tag : doc.select("div#Tag > a")) {
LOGGER.info("Found tag " + tag.text());
tags.add(tag.text().toLowerCase());
}
return tags;
}
/**
* Checks for blacklisted tags on page. If it finds one it returns it, if not it return null
*
* @param doc
* @return String
*/
public String checkTags(Document doc, String[] blackListedTags) {
// If the user hasn't blacklisted any tags we return null;
if (blackListedTags == null) {
return null;
}
LOGGER.info("Blacklisted tags " + blackListedTags[0]);
List<String> tagsOnPage = getTags(doc);
for (String tag : blackListedTags) {
for (String pageTag : tagsOnPage) {
// We replace all dashes in the tag with spaces because the tags we get from the site are separated using
// dashes
if (tag.trim().toLowerCase().equals(pageTag.toLowerCase())) {
return tag.toLowerCase();
}
}
}
return null;
}
private JSONArray getPageUrls() { private JSONArray getPageUrls() {
String postURL = "http://www.tsumino.com/Read/Load"; String postURL = "http://www.tsumino.com/Read/Load";
try { try {
@ -86,7 +123,14 @@ public class TsuminoRipper extends AbstractHTMLRipper {
public Document getFirstPage() throws IOException { public Document getFirstPage() throws IOException {
Connection.Response resp = Http.url(url).response(); Connection.Response resp = Http.url(url).response();
cookies.putAll(resp.cookies()); cookies.putAll(resp.cookies());
return resp.parse(); Document doc = resp.parse();
String blacklistedTag = checkTags(doc, Utils.getConfigStringArray("tsumino.blacklist.tags"));
if (blacklistedTag != null) {
sendUpdate(RipStatusMessage.STATUS.DOWNLOAD_WARN, "Skipping " + url.toExternalForm() + " as it " +
"contains the blacklisted tag \"" + blacklistedTag + "\"");
return null;
}
return doc;
} }
@Override @Override

View File

@ -2,13 +2,26 @@ package com.rarchives.ripme.tst.ripper.rippers;
import java.io.IOException; import java.io.IOException;
import java.net.URL; import java.net.URL;
import java.util.List;
import com.rarchives.ripme.ripper.rippers.TsuminoRipper; import com.rarchives.ripme.ripper.rippers.TsuminoRipper;
public class TsuminoRipperTest extends RippersTest { public class TsuminoRipperTest extends RippersTest {
public void testPahealRipper() throws IOException { public void testTsuminoRipper() throws IOException {
// a photo set
TsuminoRipper ripper = new TsuminoRipper(new URL("http://www.tsumino.com/Book/Info/42882/chaldea-maid-")); TsuminoRipper ripper = new TsuminoRipper(new URL("http://www.tsumino.com/Book/Info/42882/chaldea-maid-"));
testRipper(ripper); testRipper(ripper);
} }
public void testTagBlackList() throws IOException {
TsuminoRipper ripper = new TsuminoRipper(new URL("http://www.tsumino.com/Book/Info/42882/chaldea-maid-"));
String[] tags1 = {"test", "one", "Blowjob"};
String blacklistedTag = ripper.checkTags(ripper.getFirstPage(), tags1);
assertEquals("blowjob", blacklistedTag);
// Test a tag with spaces
String[] tags2 = {"test", "one", "Full Color"};
blacklistedTag = ripper.checkTags(ripper.getFirstPage(), tags2);
assertEquals("full color", blacklistedTag);
}
} }