Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #986 from cyian-1756/xhamsterFixes

Browse Source 
Fixed xhamster ripper
This commit is contained in:
cyian-1756 2018-10-06 09:07:16 -05:00 committed by GitHub
commit 89680cf48d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 30 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
View File

@ -214,6 +214,7 @@ class DownloadFileThread extends Thread {
}
byte[] data = new byte[1024 * 256];
int bytesRead;
boolean shouldSkipFileDownload = huc.getContentLength() / 10000000 >= 10;
while ( (bytesRead = bis.read(data)) != -1) {
try {
observer.stopCheck();
@ -228,7 +229,7 @@ class DownloadFileThread extends Thread {
observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded);
}
// If this is a test and we're downloading a large file
if (AbstractRipper.isThisATest() && bytesTotal / 10000000 >= 10) {
if (AbstractRipper.isThisATest() && shouldSkipFileDownload) {
logger.debug("Not downloading whole file because it is over 10mb and this is a test");
bis.close();
fos.close();

42
src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java
View File

@ -26,6 +26,12 @@ public class XhamsterRipper extends AbstractHTMLRipper {
super(url);
}
private int index = 1;
@Override public boolean hasASAPRipping() {
return true;
}
@Override
public String getHost() {
return "xhamster";
@ -144,24 +150,21 @@ public class XhamsterRipper extends AbstractHTMLRipper {
@Override
public List<String> getURLsFromPage(Document doc) {
LOGGER.debug("Checking for urls");
List<String> result = new ArrayList<>();
if (!isVideoUrl(url)) {
for (Element thumb : doc.select("div.picture_view > div.pictures_block > div.items > div.item-container > a > div.thumb_container > div.img > img")) {
String image = thumb.attr("src");
// replace thumbnail urls with the urls to the full sized images
image = image.replaceAll(
"https://upt.xhcdn\\.",
"http://up.xhamster.");
image = image.replaceAll("ept\\.xhcdn", "ep.xhamster");
image = image.replaceAll(
"_160\\.",
"_1000.");
// Xhamster has bad cert management and uses invalid certs for some cdns, so we change all our requests to http
image = image.replaceAll("https", "http");
result.add(image);
}
for (Element page : doc.select("div.items > div.item-container > a.item")) {
String pageWithImageUrl = page.attr("href");
try {
String image = Http.url(new URL(pageWithImageUrl)).get().select("div.picture_container > a > img").attr("src");
downloadFile(image);
} catch (IOException e) {
LOGGER.error("Was unable to load page " + pageWithImageUrl);
}
}
} else {
result.add(doc.select("div.player-container > a").attr("href"));
String imgUrl = doc.select("div.player-container > a").attr("href");
downloadFile(imgUrl);
}
return result;
}
@ -170,6 +173,15 @@ public class XhamsterRipper extends AbstractHTMLRipper {
public void downloadURL(URL url, int index) {
addURLToDownload(url, getPrefix(index));
}
private void downloadFile(String url) {
try {
addURLToDownload(new URL(url), getPrefix(index));
index = index + 1;
} catch (MalformedURLException e) {
LOGGER.error("The url \"" + url + "\" is malformed");
}
}
@Override
public String getAlbumTitle(URL url) throws MalformedURLException {

2
src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java
View File

@ -35,7 +35,7 @@ public class XhamsterRipperTest extends RippersTest {
}
public void testGetNextPage() throws IOException {
XhamsterRipper ripper = new XhamsterRipper(new URL("https://pt.xhamster.com/photos/gallery/silvana-7105696"));
XhamsterRipper ripper = new XhamsterRipper(new URL("https://pt.xhamster.com/photos/gallery/mega-compil-6-10728626"));
Document doc = ripper.getFirstPage();
try {
ripper.getNextPage(doc);