Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

1.0.91 - Cleaning up ChanRipper, fix to work with anonib

Browse Source
This commit is contained in:
4pr0n 2014-11-14 06:03:13 -08:00
parent fa16954781
commit 707f64c516
3 changed files with 26 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@ -4,7 +4,7 @@
<groupId>com.rarchives.ripme</groupId> <groupId>com.rarchives.ripme</groupId>
<artifactId>ripme</artifactId> <artifactId>ripme</artifactId>
<packaging>jar</packaging> <packaging>jar</packaging>
<version>1.0.90</version> <version>1.0.91</version>
<name>ripme</name> <name>ripme</name>
<url>http://rip.rarchives.com</url> <url>http://rip.rarchives.com</url>
<properties> <properties>

43
src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
View File

@ -18,18 +18,12 @@ import java.util.Arrays;
public class ChanRipper extends AbstractHTMLRipper { public class ChanRipper extends AbstractHTMLRipper {
//ArrayList<String> explicit_domains = new ArrayList<String>();
public static List<ChanSite> explicit_domains = Arrays.asList( public static List<ChanSite> explicit_domains = Arrays.asList(
//Tested (main boards) new ChanSite(Arrays.asList("boards.4chan.org"), Arrays.asList("4cdn.org")),
//Untested (main boards) new ChanSite(Arrays.asList("archive.moe"), Arrays.asList("data.archive.moe")),
new ChanSite(Arrays.asList("anon-ib.com")), new ChanSite(Arrays.asList("4archive.org"), Arrays.asList("imgur.com")),
new ChanSite(Arrays.asList("boards.4chan.org"),Arrays.asList("4cdn.org")), new ChanSite(Arrays.asList("archive.4plebs.org"), Arrays.asList("img.4plebs.org")),
//Tested (archives) new ChanSite(Arrays.asList("fgts.jp"), Arrays.asList("dat.fgts.jp"))
new ChanSite(Arrays.asList("archive.moe"),Arrays.asList("data.archive.moe")), //4chan archive (successor of foolz archive) Archives: [ a / biz / c / co / diy / gd / i / int / jp / m / mlp / out / po / q / s4s / sci / sp / tg / tv / v / vg / vp / vr / wsg ]
//Untested (archives)new ChanSite(Arrays.asList("anon-ib.com")),
new ChanSite(Arrays.asList("4archive.org"),Arrays.asList("imgur.com")), //4chan archive (on demand)
new ChanSite(Arrays.asList("archive.4plebs.org"),Arrays.asList("img.4plebs.org")), //4chan archive Archives: [ adv / f / hr / o / pol / s4s / tg / trv / tv / x ] Boards: [ plebs ]
new ChanSite(Arrays.asList("fgts.jp"),Arrays.asList("dat.fgts.jp")) //4chan archive Archives: [ asp / cm / h / hc / hm / n / p / r / s / soc / y ]
); );
public static List<String> url_piece_blacklist = Arrays.asList( public static List<String> url_piece_blacklist = Arrays.asList(
"=http", "=http",
@ -44,15 +38,13 @@ public class ChanRipper extends AbstractHTMLRipper {
public ChanRipper(URL url) throws IOException { public ChanRipper(URL url) throws IOException {
super(url); super(url);
for (ChanSite _chanSite : explicit_domains) { for (ChanSite _chanSite : explicit_domains) {
for (String host : _chanSite.domains) { if (_chanSite.domains.contains(url.getHost())) {
if (url.getHost().equals(host)) {
chanSite = _chanSite; chanSite = _chanSite;
generalChanSite = false; generalChanSite = false;
} }
} }
} if (chanSite == null) {
if(chanSite==null){ chanSite = new ChanSite(Arrays.asList(url.getHost()));
chanSite = new ChanSite(Arrays.asList("url.getHost()"));
} }
} }
@ -70,15 +62,11 @@ public class ChanRipper extends AbstractHTMLRipper {
@Override @Override
public boolean canRip(URL url) { public boolean canRip(URL url) {
//explicit_domains testing
for (ChanSite _chanSite : explicit_domains) { for (ChanSite _chanSite : explicit_domains) {
for (String host : _chanSite.domains) { if (_chanSite.domains.contains(url.getHost())) {
if (url.getHost().equals(host)) {
return true; return true;
} }
} }
}
//It'll fail further down the road.
return url.toExternalForm().contains("/res/") // Most chans return url.toExternalForm().contains("/res/") // Most chans
|| url.toExternalForm().contains("/thread/"); // 4chan, archive.moe || url.toExternalForm().contains("/thread/"); // 4chan, archive.moe
} }
@ -92,7 +80,7 @@ public class ChanRipper extends AbstractHTMLRipper {
Pattern p; Matcher m; Pattern p; Matcher m;
String u = url.toExternalForm(); String u = url.toExternalForm();
if (u.contains("/thread/")||u.contains("/res/")) { if (u.contains("/thread/") || u.contains("/res/")) {
p = Pattern.compile("^.*\\.[a-z]{1,3}/[a-zA-Z0-9]+/(thread|res)/([0-9]+)(\\.html|\\.php)?.*$"); p = Pattern.compile("^.*\\.[a-z]{1,3}/[a-zA-Z0-9]+/(thread|res)/([0-9]+)(\\.html|\\.php)?.*$");
m = p.matcher(u); m = p.matcher(u);
if (m.matches()) { if (m.matches()) {
@ -125,23 +113,24 @@ public class ChanRipper extends AbstractHTMLRipper {
if (!link.hasAttr("href")) { if (!link.hasAttr("href")) {
continue; continue;
} }
String href = link.attr("href"); String href = link.attr("href").trim();
//Check all blacklist items //Check all blacklist items
for(String blacklist_item : url_piece_blacklist){ for (String blacklist_item : url_piece_blacklist) {
if (href.contains(blacklist_item)){ if (href.contains(blacklist_item)) {
logger.debug("Skipping link that contains '"+blacklist_item+"': " + href); logger.debug("Skipping link that contains '"+blacklist_item+"': " + href);
continue elementloop; continue elementloop;
} }
} }
Boolean self_hosted = false; Boolean self_hosted = false;
if(!generalChanSite){ if (!generalChanSite) {
for(String cdnDomain : chanSite.cdnDomains){ for (String cdnDomain : chanSite.cdnDomains) {
if (href.contains(cdnDomain)){ if (href.contains(cdnDomain)){
self_hosted = true; self_hosted = true;
} }
} }
} }
if(self_hosted||generalChanSite){ if(self_hosted||generalChanSite){
p = Pattern.compile("^.*\\.(jpg|jpeg|png|gif|apng|webp|tif|tiff|webm)$", Pattern.CASE_INSENSITIVE); p = Pattern.compile("^.*\\.(jpg|jpeg|png|gif|apng|webp|tif|tiff|webm)$", Pattern.CASE_INSENSITIVE);
m = p.matcher(href); m = p.matcher(href);

2
src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
View File

@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils;
public class UpdateUtils { public class UpdateUtils {
private static final Logger logger = Logger.getLogger(UpdateUtils.class); private static final Logger logger = Logger.getLogger(UpdateUtils.class);
private static final String DEFAULT_VERSION = "1.0.90"; private static final String DEFAULT_VERSION = "1.0.91";
private static final String updateJsonURL = "http://rarchives.com/ripme.json"; private static final String updateJsonURL = "http://rarchives.com/ripme.json";
private static final String updateJarURL = "http://rarchives.com/ripme.jar"; private static final String updateJarURL = "http://rarchives.com/ripme.jar";
private static final String mainFileName = "ripme.jar"; private static final String mainFileName = "ripme.jar";