Added abilty to blacklist tags on tsumino
This commit is contained in:
parent
d35fd511db
commit
26e27a9ed5
@ -12,6 +12,7 @@ import java.util.regex.Matcher;
|
|||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
import com.rarchives.ripme.ui.RipStatusMessage;
|
import com.rarchives.ripme.ui.RipStatusMessage;
|
||||||
|
import com.rarchives.ripme.utils.Utils;
|
||||||
import org.json.JSONArray;
|
import org.json.JSONArray;
|
||||||
import org.json.JSONObject;
|
import org.json.JSONObject;
|
||||||
import org.jsoup.Connection;
|
import org.jsoup.Connection;
|
||||||
@ -21,6 +22,7 @@ import org.jsoup.nodes.Document;
|
|||||||
|
|
||||||
import com.rarchives.ripme.ripper.AbstractHTMLRipper;
|
import com.rarchives.ripme.ripper.AbstractHTMLRipper;
|
||||||
import com.rarchives.ripme.utils.Http;
|
import com.rarchives.ripme.utils.Http;
|
||||||
|
import org.jsoup.nodes.Element;
|
||||||
|
|
||||||
public class TsuminoRipper extends AbstractHTMLRipper {
|
public class TsuminoRipper extends AbstractHTMLRipper {
|
||||||
private Map<String,String> cookies = new HashMap<>();
|
private Map<String,String> cookies = new HashMap<>();
|
||||||
@ -29,6 +31,41 @@ public class TsuminoRipper extends AbstractHTMLRipper {
|
|||||||
super(url);
|
super(url);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private List<String> getTags(Document doc) {
|
||||||
|
List<String> tags = new ArrayList<>();
|
||||||
|
LOGGER.info("Getting tags");
|
||||||
|
for (Element tag : doc.select("div#Tag > a")) {
|
||||||
|
LOGGER.info("Found tag " + tag.text());
|
||||||
|
tags.add(tag.text().toLowerCase());
|
||||||
|
}
|
||||||
|
return tags;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Checks for blacklisted tags on page. If it finds one it returns it, if not it return null
|
||||||
|
*
|
||||||
|
* @param doc
|
||||||
|
* @return String
|
||||||
|
*/
|
||||||
|
public String checkTags(Document doc, String[] blackListedTags) {
|
||||||
|
// If the user hasn't blacklisted any tags we return null;
|
||||||
|
if (blackListedTags == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
LOGGER.info("Blacklisted tags " + blackListedTags[0]);
|
||||||
|
List<String> tagsOnPage = getTags(doc);
|
||||||
|
for (String tag : blackListedTags) {
|
||||||
|
for (String pageTag : tagsOnPage) {
|
||||||
|
// We replace all dashes in the tag with spaces because the tags we get from the site are separated using
|
||||||
|
// dashes
|
||||||
|
if (tag.trim().toLowerCase().equals(pageTag.toLowerCase())) {
|
||||||
|
return tag.toLowerCase();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
private JSONArray getPageUrls() {
|
private JSONArray getPageUrls() {
|
||||||
String postURL = "http://www.tsumino.com/Read/Load";
|
String postURL = "http://www.tsumino.com/Read/Load";
|
||||||
try {
|
try {
|
||||||
@ -86,7 +123,14 @@ public class TsuminoRipper extends AbstractHTMLRipper {
|
|||||||
public Document getFirstPage() throws IOException {
|
public Document getFirstPage() throws IOException {
|
||||||
Connection.Response resp = Http.url(url).response();
|
Connection.Response resp = Http.url(url).response();
|
||||||
cookies.putAll(resp.cookies());
|
cookies.putAll(resp.cookies());
|
||||||
return resp.parse();
|
Document doc = resp.parse();
|
||||||
|
String blacklistedTag = checkTags(doc, Utils.getConfigStringArray("tsumino.blacklist.tags"));
|
||||||
|
if (blacklistedTag != null) {
|
||||||
|
sendUpdate(RipStatusMessage.STATUS.DOWNLOAD_WARN, "Skipping " + url.toExternalForm() + " as it " +
|
||||||
|
"contains the blacklisted tag \"" + blacklistedTag + "\"");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return doc;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -2,13 +2,26 @@ package com.rarchives.ripme.tst.ripper.rippers;
|
|||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
import com.rarchives.ripme.ripper.rippers.TsuminoRipper;
|
import com.rarchives.ripme.ripper.rippers.TsuminoRipper;
|
||||||
|
|
||||||
public class TsuminoRipperTest extends RippersTest {
|
public class TsuminoRipperTest extends RippersTest {
|
||||||
public void testPahealRipper() throws IOException {
|
public void testTsuminoRipper() throws IOException {
|
||||||
// a photo set
|
|
||||||
TsuminoRipper ripper = new TsuminoRipper(new URL("http://www.tsumino.com/Book/Info/42882/chaldea-maid-"));
|
TsuminoRipper ripper = new TsuminoRipper(new URL("http://www.tsumino.com/Book/Info/42882/chaldea-maid-"));
|
||||||
testRipper(ripper);
|
testRipper(ripper);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void testTagBlackList() throws IOException {
|
||||||
|
TsuminoRipper ripper = new TsuminoRipper(new URL("http://www.tsumino.com/Book/Info/42882/chaldea-maid-"));
|
||||||
|
String[] tags1 = {"test", "one", "Blowjob"};
|
||||||
|
String blacklistedTag = ripper.checkTags(ripper.getFirstPage(), tags1);
|
||||||
|
assertEquals("blowjob", blacklistedTag);
|
||||||
|
|
||||||
|
// Test a tag with spaces
|
||||||
|
String[] tags2 = {"test", "one", "Full Color"};
|
||||||
|
blacklistedTag = ripper.checkTags(ripper.getFirstPage(), tags2);
|
||||||
|
assertEquals("full color", blacklistedTag);
|
||||||
|
|
||||||
|
}
|
||||||
}
|
}
|
Loading…
Reference in New Issue
Block a user