Added Natalie.mu news album ripper.
Rips from both album and news pages.
This commit is contained in:
Erwin de Haan
parent
4871142252
commit
0ba8786e0f
@ -0,0 +1,134 @@
|
|||||||
|
package com.rarchives.ripme.ripper.rippers;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.regex.Matcher;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
import org.jsoup.nodes.Document;
|
||||||
|
import org.jsoup.nodes.Element;
|
||||||
|
|
||||||
|
import com.rarchives.ripme.ripper.AbstractHTMLRipper;
|
||||||
|
import com.rarchives.ripme.ripper.rippers.ripperhelpers.ChanSite;
|
||||||
|
import com.rarchives.ripme.utils.Http;
|
||||||
|
|
||||||
|
public class NatalieMuRipper extends AbstractHTMLRipper {
|
||||||
|
|
||||||
|
public int news_id = 0;
|
||||||
|
|
||||||
|
public NatalieMuRipper(URL url) throws IOException {
|
||||||
|
super(url);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getHost() {
|
||||||
|
String host = this.url.getHost();
|
||||||
|
host = host.substring(0, host.lastIndexOf('.'));
|
||||||
|
if (host.contains(".")) {
|
||||||
|
// Host has subdomain (www)
|
||||||
|
host = host.substring(host.lastIndexOf('.') + 1);
|
||||||
|
}
|
||||||
|
String board = this.url.toExternalForm().split("/")[3];
|
||||||
|
return host + "_" + board;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean canRip(URL url) {
|
||||||
|
//urls like:
|
||||||
|
// http://cdn2.natalie.mu/music/gallery/show/news_id/xxxxxx/image_id/xxxxxx
|
||||||
|
// http://cdn2.natalie.mu/music/news/140411
|
||||||
|
return url.toExternalForm().contains("natalie.mu") // Most chans
|
||||||
|
&& (url.toExternalForm().contains("/news_id/")
|
||||||
|
|| url.toExternalForm().contains("/news/")); // 4chan, archive.moe
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* For example the achrives are all known. (Check 4chan-x)
|
||||||
|
* Should be based on the software the specific chan uses.
|
||||||
|
* FoolFuuka uses the same (url) layout as 4chan
|
||||||
|
* */
|
||||||
|
@Override
|
||||||
|
public String getGID(URL url) throws MalformedURLException {
|
||||||
|
Pattern p; Matcher m;
|
||||||
|
|
||||||
|
String u = url.toExternalForm();
|
||||||
|
if (u.contains("/news_id/")) {
|
||||||
|
p = Pattern.compile("/news_id/([0-9]+)/");
|
||||||
|
m = p.matcher(u);
|
||||||
|
if (m.find()) {
|
||||||
|
return m.group(1);
|
||||||
|
}
|
||||||
|
} else if (u.contains("/news/")) {
|
||||||
|
p = Pattern.compile("/news/([0-9]+)/?");
|
||||||
|
m = p.matcher(u);
|
||||||
|
if (m.find()) {
|
||||||
|
return m.group(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new MalformedURLException(
|
||||||
|
"Expected natalie.mu URL formats: "
|
||||||
|
+ "http://natalie.mu/music/news/xxxxxx or http://natalie.mu/music/gallery/show/news_id/xxxxxx/image_id/yyyyyy"
|
||||||
|
+ " Got: " + u);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getDomain() {
|
||||||
|
return this.url.getHost();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Document getFirstPage() throws IOException {
|
||||||
|
return Http.url(this.url).get();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<String> getURLsFromPage(Document page) {
|
||||||
|
List<String> imageURLs = new ArrayList<String>();
|
||||||
|
Pattern p; Matcher m;
|
||||||
|
//select all album thumbnails
|
||||||
|
for (Element span : page.select(".NA_articleGallery span")) {
|
||||||
|
if (!span.hasAttr("style")) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
String style = span.attr("style").trim();
|
||||||
|
|
||||||
|
p = Pattern.compile("background-image: url\\((.*list_thumb_inbox.*)\\);", Pattern.CASE_INSENSITIVE);
|
||||||
|
m = p.matcher(style);
|
||||||
|
if (m.find()) {
|
||||||
|
String imgUrl = m.group(1);
|
||||||
|
if (imgUrl.startsWith("//")) {
|
||||||
|
imgUrl = "http:" + imgUrl;
|
||||||
|
}
|
||||||
|
if (imgUrl.startsWith("/")) {
|
||||||
|
imgUrl = "http://" + this.url.getHost() + imgUrl;
|
||||||
|
}
|
||||||
|
//convert thumbnail url into fullsize url
|
||||||
|
imgUrl = imgUrl.replace("list_thumb_inbox","xlarge");
|
||||||
|
// Don't download the same URL twice
|
||||||
|
if (imageURLs.contains(imgUrl)) {
|
||||||
|
logger.debug("Already attempted: " + imgUrl);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
imageURLs.add(imgUrl);
|
||||||
|
if (isThisATest()) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isStopped()) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return imageURLs;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void downloadURL(URL url, int index) {
|
||||||
|
addURLToDownload(url, getPrefix(index), "", this.url.toString(), null);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,65 @@
|
|||||||
|
package com.rarchives.ripme.tst.ripper.rippers;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import com.rarchives.ripme.ripper.rippers.NatalieMuRipper;
|
||||||
|
|
||||||
|
public class NatalieMuRipperTest extends RippersTest {
|
||||||
|
|
||||||
|
public void testNatalieMuURLFailures() throws IOException {
|
||||||
|
List<URL> failURLs = new ArrayList<URL>();
|
||||||
|
// URLs that should not work
|
||||||
|
for (URL url : failURLs) {
|
||||||
|
try {
|
||||||
|
new NatalieMuRipper(url);
|
||||||
|
fail("Instantiated ripper for URL that should not work: " + url);
|
||||||
|
} catch (Exception e) {
|
||||||
|
// Expected
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testNatalieMuURLPasses() throws IOException {
|
||||||
|
List<URL> passURLs = new ArrayList<URL>();
|
||||||
|
// URLs that should work
|
||||||
|
passURLs.add(new URL("http://natalie.mu/music/news/140367"));
|
||||||
|
passURLs.add(new URL("http://cdn2.natalie.mu/music/news/140411"));
|
||||||
|
passURLs.add(new URL("http://cdn2.natalie.mu/music/gallery/show/news_id/140411/image_id/369655"));
|
||||||
|
passURLs.add(new URL("http://natalie.mu/music/gallery/show/news_id/139146/image_id/365218"));
|
||||||
|
for (URL url : passURLs) {
|
||||||
|
NatalieMuRipper ripper = new NatalieMuRipper(url);
|
||||||
|
ripper.setup();
|
||||||
|
assert(ripper.canRip(url));
|
||||||
|
assertNotNull("Ripper for " + url + " did not have a valid working directory.",
|
||||||
|
ripper.getWorkingDir());
|
||||||
|
deleteDir(ripper.getWorkingDir());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testNatalieMuRipper() throws IOException {
|
||||||
|
List<URL> contentURLs = new ArrayList<URL>();
|
||||||
|
// URLs that should return more than 1 image
|
||||||
|
contentURLs.add(new URL("http://natalie.mu/music/news/140367"));
|
||||||
|
contentURLs.add(new URL("http://cdn2.natalie.mu/music/news/140411"));
|
||||||
|
contentURLs.add(new URL("http://cdn2.natalie.mu/music/gallery/show/news_id/140411/image_id/369655"));
|
||||||
|
contentURLs.add(new URL("http://natalie.mu/music/gallery/show/news_id/139146/image_id/365218"));
|
||||||
|
|
||||||
|
// Most *chans have volatile threads & can't be trusted for integration testing.
|
||||||
|
|
||||||
|
//contentURLs.add(new URL("http://boards.4chan.org/r/res/12225949"));
|
||||||
|
//contentURLs.add(new URL("http://7chan.org/gif/res/23795.html"));
|
||||||
|
//contentURLs.add(new URL("http://unichan2.org/b/res/518004.html"));
|
||||||
|
|
||||||
|
// xchan has an HTTPS certificaiton error...
|
||||||
|
//contentURLs.add(new URL("http://xchan.pw/porn/res/437.html"));
|
||||||
|
for (URL url : contentURLs) {
|
||||||
|
NatalieMuRipper ripper = new NatalieMuRipper(url);
|
||||||
|
testRipper(ripper);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user