Merge pull request #3 from RipMeApp/master

Merging with master
This commit is contained in:
randomcommitter 2018-04-20 12:49:27 +01:00 committed by GitHub
commit 75f9c448ea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 75 additions and 60 deletions

View File

@ -4,7 +4,7 @@
<groupId>com.rarchives.ripme</groupId>
<artifactId>ripme</artifactId>
<packaging>jar</packaging>
<version>1.7.35</version>
<version>1.7.37</version>
<name>ripme</name>
<url>http://rip.rarchives.com</url>
<properties>

View File

@ -1,6 +1,8 @@
{
"latestVersion": "1.7.35",
"latestVersion": "1.7.37",
"changeList": [
"1.7.37: MInor code clean up; Added socks proxy support; Added support for 8muses.download; Hentaifoundry no longer errors when there are no more pages; Fix bug that causes tumblr to replace https with httpss when downloading resized images",
"1.7.36: Fixed Instagram ripper; Fixed hentai2read ripper test; Fixed tnbtu.com ripper",
"1.7.35: Fixed instagram ripper; hentai2read ripper now properly names folders",
"1.7.34: Added Blackbrickroadofoz Ripper; Fixed webtoons regex",
"1.7.33: Instagram ripper no longer errors out when downloading from more than 1 page",

View File

@ -55,14 +55,8 @@ public class BlackbrickroadofozRipper extends AbstractHTMLRipper {
throw new IOException("No more pages");
}
String nextPage = elem.attr("href");
// Some times this returns a empty string
// This for stops that
if (nextPage == "") {
throw new IOException("No more pages");
}
else {
return Http.url(nextPage).get();
}
return Http.url(nextPage).get();
}
@Override

View File

@ -59,7 +59,7 @@ public class CfakeRipper extends AbstractHTMLRipper {
String nextPage = elem.attr("href");
// Some times this returns a empty string
// This for stops that
if (nextPage == "") {
if (nextPage.equals("")) {
return null;
}
else {

View File

@ -4,7 +4,6 @@ import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

View File

@ -86,7 +86,7 @@ public class EroShareRipper extends AbstractHTMLRipper {
throw new IOException("No more pages");
}
nextUrl = elem.attr("href");
if (nextUrl == "") {
if (nextUrl.equals("")) {
throw new IOException("No more pages");
}
return Http.url("eroshae.com" + nextUrl).get();

View File

@ -13,7 +13,6 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.rarchives.ripme.utils.Utils;
import org.jsoup.Connection.Method;
import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
@ -23,7 +22,6 @@ import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.ripper.DownloadThreadPool;
import com.rarchives.ripme.utils.Base64;
import com.rarchives.ripme.utils.Http;
public class FuraffinityRipper extends AbstractHTMLRipper {
@ -162,10 +160,6 @@ public class FuraffinityRipper extends AbstractHTMLRipper {
if (!subdirectory.equals("")) {
subdirectory = File.separator + subdirectory;
}
int o = url.toString().lastIndexOf('/')-1;
String test = url.toString().substring(url.toString().lastIndexOf('/',o)+1);
test = test.replace("/",""); // This is probably not the best way to do this.
test = test.replace("\\",""); // CLOSE ENOUGH!
saveFileAs = new File(
workingDir.getCanonicalPath()
+ subdirectory

View File

@ -33,7 +33,7 @@ public class Hentai2readRipper extends AbstractHTMLRipper {
@Override
public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("https?://hentai2read\\.com/([a-zA-Z0-9_-]*)/\\d/?");
Pattern p = Pattern.compile("https?://hentai2read\\.com/([a-zA-Z0-9_-]*)/(\\d)?/?");
Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) {
return m.group(1);

View File

@ -52,16 +52,16 @@ public class HentaifoundryRipper extends AbstractHTMLRipper {
Response resp = Http.url("http://www.hentai-foundry.com/").response();
cookies = resp.cookies();
resp = Http.url("http://www.hentai-foundry.com/?enterAgree=1&size=1500")
.referrer("http://www.hentai-foundry.com/")
.cookies(cookies)
.response();
.referrer("http://www.hentai-foundry.com/")
.cookies(cookies)
.response();
// The only cookie that seems to matter in getting around the age wall is the phpsession cookie
cookies.putAll(resp.cookies());
sleep(500);
resp = Http.url(url)
.referrer("http://www.hentai-foundry.com/")
.cookies(cookies)
.response();
.referrer("http://www.hentai-foundry.com/")
.cookies(cookies)
.response();
cookies.putAll(resp.cookies());
return resp.parse();
}
@ -74,12 +74,16 @@ public class HentaifoundryRipper extends AbstractHTMLRipper {
}
Elements els = doc.select("li.next > a");
Element first = els.first();
String nextURL = first.attr("href");
nextURL = "http://www.hentai-foundry.com" + nextURL;
return Http.url(nextURL)
.referrer(url)
.cookies(cookies)
.get();
try {
String nextURL = first.attr("href");
nextURL = "http://www.hentai-foundry.com" + nextURL;
return Http.url(nextURL)
.referrer(url)
.cookies(cookies)
.get();
} catch (NullPointerException e) {
throw new IOException("No more pages");
}
}
@Override
@ -97,13 +101,6 @@ public class HentaifoundryRipper extends AbstractHTMLRipper {
}
Document imagePage;
try {
Response resp = Http.url("http://www.hentai-foundry.com/").response();
cookies = resp.cookies();
resp = Http.url("http://www.hentai-foundry.com/?enterAgree=1&size=1500")
.referrer("http://www.hentai-foundry.com/")
.cookies(cookies)
.response();
cookies.putAll(resp.cookies());
logger.info("grabbing " + "http://www.hentai-foundry.com" + thumb.attr("href"));
imagePage = Http.url("http://www.hentai-foundry.com" + thumb.attr("href")).cookies(cookies).get();

View File

@ -374,7 +374,7 @@ public class InstagramRipper extends AbstractHTMLRipper {
try {
// Sleep for a while to avoid a ban
sleep(2500);
String vars = "{\"id\":\"" + userID + "\",\"first\":100,\"after\":\"" + nextPageID + "\"}";
String vars = "{\"id\":\"" + userID + "\",\"first\":50,\"after\":\"" + nextPageID + "\"}";
String ig_gis = getIGGis(vars);
logger.info(ig_gis);
toreturn = Http.url("https://www.instagram.com/graphql/query/?query_hash=" + qHash + "&variables=" + vars

View File

@ -50,7 +50,7 @@ public class LusciousRipper extends AbstractHTMLRipper {
// This is here for pages with mp4s instead of images
String video_image = "";
video_image = page.select("div > video > source").attr("src");
if (video_image != "") {
if (!video_image.equals("")) {
urls.add(video_image);
}
return urls;

View File

@ -73,7 +73,7 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper {
if (m.matches()) {
nextUrl = "http://myhentaicomics.com" + m.group(0);
}
if (nextUrl == "") {
if (nextUrl.equals("")) {
throw new IOException("No more pages");
}
// Sleep for half a sec to avoid getting IP banned
@ -100,7 +100,7 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper {
Element elem = nextAlbumPage.select("a.ui-icon-right").first();
String nextPage = elem.attr("href");
pageNumber = pageNumber + 1;
if (nextPage == "") {
if (nextPage.equals("")) {
logger.info("Got " + pageNumber + " pages");
break;
}
@ -220,7 +220,7 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper {
// If true the page is a page of albums
if (doc.toString().contains("class=\"g-item g-album\"")) {
// This if checks that there is more than 1 page
if (doc.select("a.ui-icon-right").last().attr("href") != "") {
if (!doc.select("a.ui-icon-right").last().attr("href").equals("")) {
// There is more than one page so we call getListOfPages
List<String> pagesToRip = getListOfPages(doc);
logger.debug("Pages to rip = " + pagesToRip);

View File

@ -57,7 +57,7 @@ public class SinfestRipper extends AbstractHTMLRipper {
String nextPage = elem.parent().attr("href");
// Some times this returns a empty string
// This for stops that
if (nextPage == "") {
if (nextPage.equals("")) {
return null;
}
else {

View File

@ -230,7 +230,7 @@ public class TumblrRipper extends AlbumRipper {
urlString = urlString.replaceAll("_\\d+\\.", "_raw.");
fileURL = new URL(urlString);
} else {
fileURL = new URL(photo.getJSONObject("original_size").getString("url").replaceAll("http", "https"));
fileURL = new URL(photo.getJSONObject("original_size").getString("url").replaceAll("http:", "https:"));
}
m = p.matcher(fileURL.toString());
if (m.matches()) {

View File

@ -4,14 +4,12 @@ import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.utils.Http;

View File

@ -44,7 +44,8 @@ public class WordpressComicRipper extends AbstractHTMLRipper {
"freeadultcomix.com",
"thisis.delvecomic.com",
"tnbtu.com",
"shipinbottle.pepsaga.com"
"shipinbottle.pepsaga.com",
"8muses.download"
);
@Override
@ -135,6 +136,12 @@ public class WordpressComicRipper extends AbstractHTMLRipper {
if (shipinbottleMat.matches()) {
return true;
}
Pattern eight_musesPat = Pattern.compile("https?://8muses.download/([a-zA-Z0-9_-]+)/?$");
Matcher eight_musesMat = eight_musesPat.matcher(url.toExternalForm());
if (eight_musesMat.matches()) {
return true;
}
}
@ -209,6 +216,11 @@ public class WordpressComicRipper extends AbstractHTMLRipper {
return getHost() + "_" + "Ship_in_bottle";
}
Pattern eight_musesPat = Pattern.compile("https?://8muses.download/([a-zA-Z0-9_-]+)/?$");
Matcher eight_musesMat = eight_musesPat.matcher(url.toExternalForm());
if (eight_musesMat.matches()) {
return getHost() + "_" + eight_musesMat.group(1);
}
return super.getAlbumTitle(url);
}
@ -247,7 +259,7 @@ public class WordpressComicRipper extends AbstractHTMLRipper {
nextPage = elem.attr("href");
}
if (nextPage == "") {
if (nextPage.equals("")) {
throw new IOException("No more pages");
} else {
return Http.url(nextPage).get();
@ -315,6 +327,12 @@ public class WordpressComicRipper extends AbstractHTMLRipper {
}
}
if (url.toExternalForm().contains("8muses.download")) {
for (Element elem : doc.select("div.popup-gallery > figure > a")) {
result.add(elem.attr("href"));
}
}
return result;
}
@ -327,8 +345,14 @@ public class WordpressComicRipper extends AbstractHTMLRipper {
|| getHost().contains("themonsterunderthebed.net")) {
addURLToDownload(url, pageTitle + "_");
}
// If we're ripping a site where we can't get the page number/title we just rip normally
addURLToDownload(url, getPrefix(index));
if (getHost().contains("tnbtu.com")) {
// We need to set the referrer header for tnbtu
addURLToDownload(url, getPrefix(index), "","http://www.tnbtu.com/comic", null);
} else {
// If we're ripping a site where we can't get the page number/title we just rip normally
addURLToDownload(url, getPrefix(index));
}
}
@Override

View File

@ -72,7 +72,7 @@ public class XhamsterRipper extends AbstractHTMLRipper {
@Override
public Document getNextPage(Document doc) throws IOException {
if (doc.select("a.next").first().attr("href") != "") {
if (!doc.select("a.next").first().attr("href").equals("")) {
return Http.url(doc.select("a.next").first().attr("href")).get();
} else {
throw new IOException("No more pages");

View File

@ -3,7 +3,6 @@ package com.rarchives.ripme.ripper.rippers.video;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

View File

@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils;
public class UpdateUtils {
private static final Logger logger = Logger.getLogger(UpdateUtils.class);
private static final String DEFAULT_VERSION = "1.7.35";
private static final String DEFAULT_VERSION = "1.7.37";
private static final String REPO_NAME = "ripmeapp/ripme";
private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json";
private static final String mainFileName = "ripme.jar";

View File

@ -3,7 +3,7 @@ package com.rarchives.ripme.tst.ripper.rippers;
import java.io.IOException;
import java.net.URL;
import com.rarchives.ripme.ripper.rippers.AerisdiesRipper;;
import com.rarchives.ripme.ripper.rippers.AerisdiesRipper;
public class AerisdiesRipperTest extends RippersTest {
public void testAlbum() throws IOException {

View File

@ -9,5 +9,7 @@ public class Hentai2readRipperTest extends RippersTest {
public void testHentai2readAlbum() throws IOException {
Hentai2readRipper ripper = new Hentai2readRipper(new URL("https://hentai2read.com/sm_school_memorial/"));
testRipper(ripper);
ripper = new Hentai2readRipper(new URL("https://hentai2read.com/sm_school_memorial/1/"));
testRipper(ripper);
}
}

View File

@ -83,6 +83,12 @@ public class WordpressComicRipperTest extends RippersTest {
new URL("http://tnbtu.com/comic/01-00/"));
testRipper(ripper);
}
public void test_Eightmuses_download() throws IOException {
WordpressComicRipper ripper = new WordpressComicRipper(
new URL("https://8muses.download/lustomic-playkittens-josh-samuel-porn-comics-8-muses/"));
testRipper(ripper);
}
// https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI
// public void test_pepsaga() throws IOException {
// WordpressComicRipper ripper = new WordpressComicRipper(