Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #296 from metaprime/hushpix-consent

Browse Source 
Format CheveretoRipper; implement AGREE_CONSENT cookie for hushpix; add CheveretoRipperTest
This commit is contained in:
metaprime 2017-11-29 03:37:53 -08:00 committed by GitHub
commit 5c95fedd5c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 117 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java
View File

@ -6,6 +6,8 @@ import java.net.URL;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@ -16,12 +18,18 @@ import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Http;
public class CheveretoRipper extends AbstractHTMLRipper { public class CheveretoRipper extends AbstractHTMLRipper {
private static final Map<String, String> CONSENT_COOKIE;
static {
CONSENT_COOKIE = new TreeMap<String, String>();
CONSENT_COOKIE.put("AGREE_CONSENT", "1");
}
public CheveretoRipper(URL url) throws IOException { public CheveretoRipper(URL url) throws IOException {
super(url); super(url);
} }
private static List<String> explicit_domains_1 = Arrays.asList("hushpix.com", "tag-fox.com"); private static List<String> explicit_domains_1 = Arrays.asList("hushpix.com", "tag-fox.com");
@Override @Override
public String getHost() { public String getHost() {
return url.toExternalForm().split("/")[2]; return url.toExternalForm().split("/")[2];
@ -60,7 +68,6 @@ public class CheveretoRipper extends AbstractHTMLRipper {
return super.getAlbumTitle(url); return super.getAlbumTitle(url);
} }
@Override @Override
public String getGID(URL url) throws MalformedURLException { public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("(?:https?://)?(?:www\\.)?[a-z1-9-]*\\.[a-z1-9]*/album/([a-zA-Z1-9]*)/?$"); Pattern p = Pattern.compile("(?:https?://)?(?:www\\.)?[a-z1-9-]*\\.[a-z1-9]*/album/([a-zA-Z1-9]*)/?$");
@ -75,7 +82,7 @@ public class CheveretoRipper extends AbstractHTMLRipper {
@Override @Override
public Document getFirstPage() throws IOException { public Document getFirstPage() throws IOException {
// "url" is an instance field of the superclass // "url" is an instance field of the superclass
return Http.url(url).get(); return Http.url(url).cookies(CONSENT_COOKIE).get();
} }
@Override @Override
@ -92,9 +99,8 @@ public class CheveretoRipper extends AbstractHTMLRipper {
// This for stops that // This for stops that
if (nextPage == "") { if (nextPage == "") {
return null; return null;
} } else {
else { return Http.url(nextPage).cookies(CONSENT_COOKIE).get();
return Http.url(nextPage).get();
} }
} }
@ -115,6 +121,4 @@ public class CheveretoRipper extends AbstractHTMLRipper {
public void downloadURL(URL url, int index) { public void downloadURL(URL url, int index) {
addURLToDownload(url, getPrefix(index)); addURLToDownload(url, getPrefix(index));
} }
}
}

18
src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java Normal file
View File

@ -0,0 +1,18 @@
package com.rarchives.ripme.tst.ripper.rippers;
import java.io.IOException;
import java.net.URL;
import com.rarchives.ripme.ripper.rippers.CheveretoRipper;
public class CheveretoRipperTest extends RippersTest {
public void testHushpix() throws IOException {
CheveretoRipper ripper = new CheveretoRipper(new URL("https://hushpix.com/album/gKcu"));
testRipper(ripper);
}
public void testTagFox() throws IOException {
CheveretoRipper ripper = new CheveretoRipper(new URL("http://tag-fox.com/album/Thjb"));
testRipper(ripper);
}
}