From 630f84eab315f6465de7e07be689f414448bcb0f Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 6 Oct 2018 09:03:52 -0400 Subject: [PATCH 1/5] Fixed xhamster ripper --- .../ripme/ripper/rippers/XhamsterRipper.java | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index 20f6487e..e7fda66e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -124,20 +124,16 @@ public class XhamsterRipper extends AbstractHTMLRipper { @Override public List getURLsFromPage(Document doc) { + LOGGER.debug("Checking for urls"); List result = new ArrayList<>(); - for (Element thumb : doc.select("div.picture_view > div.pictures_block > div.items > div.item-container > a > div.thumb_container > div.img > img")) { - String image = thumb.attr("src"); - // replace thumbnail urls with the urls to the full sized images - image = image.replaceAll( - "https://upt.xhcdn\\.", - "http://up.xhamster."); - image = image.replaceAll("ept\\.xhcdn", "ep.xhamster"); - image = image.replaceAll( - "_160\\.", - "_1000."); - // Xhamster has bad cert management and uses invalid certs for some cdns, so we change all our requests to http - image = image.replaceAll("https", "http"); - result.add(image); + for (Element page : doc.select("div.items > div.item-container > a.item")) { + String pageWithImageUrl = page.attr("href"); + try { + String image = Http.url(new URL(pageWithImageUrl)).get().select("div.picture_container > a > img").attr("src"); + result.add(image); + } catch (IOException e) { + LOGGER.error("Was unable to load page " + pageWithImageUrl); + } } return result; } From 7cca4ff5b26425642e81bd57a96d0eb501eee1f2 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 6 Oct 2018 09:11:22 -0400 Subject: [PATCH 2/5] Fixed sytax issue --- .../java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index 0a900a49..5ec4f04c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -155,6 +155,7 @@ public class XhamsterRipper extends AbstractHTMLRipper { } catch (IOException e) { LOGGER.error("Was unable to load page " + pageWithImageUrl); } + } } else { result.add(doc.select("div.player-container > a").attr("href")); } From dd9a81a4f54595680e6f5dacc8bf713f577381fd Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 6 Oct 2018 09:47:44 -0400 Subject: [PATCH 3/5] Fixed bug which caused large files to be fully downloaded when running tests --- .../java/com/rarchives/ripme/ripper/DownloadFileThread.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 85a8f64d..2f8a1503 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -214,6 +214,7 @@ class DownloadFileThread extends Thread { } byte[] data = new byte[1024 * 256]; int bytesRead; + boolean shouldSkipFileDownload = huc.getContentLength() / 10000000 >= 10; while ( (bytesRead = bis.read(data)) != -1) { try { observer.stopCheck(); @@ -228,7 +229,7 @@ class DownloadFileThread extends Thread { observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded); } // If this is a test and we're downloading a large file - if (AbstractRipper.isThisATest() && bytesTotal / 10000000 >= 10) { + if (AbstractRipper.isThisATest() && shouldSkipFileDownload) { logger.debug("Not downloading whole file because it is over 10mb and this is a test"); bis.close(); fos.close(); From ec64de1c9b352581c6ba97cc1dbc764e5f14047b Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 6 Oct 2018 09:50:12 -0400 Subject: [PATCH 4/5] Added ASAP ripping --- .../ripme/ripper/rippers/XhamsterRipper.java | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index 5ec4f04c..b844b4f5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -26,6 +26,12 @@ public class XhamsterRipper extends AbstractHTMLRipper { super(url); } + private int index = 1; + + @Override public boolean hasASAPRipping() { + return true; + } + @Override public String getHost() { return "xhamster"; @@ -151,13 +157,14 @@ public class XhamsterRipper extends AbstractHTMLRipper { String pageWithImageUrl = page.attr("href"); try { String image = Http.url(new URL(pageWithImageUrl)).get().select("div.picture_container > a > img").attr("src"); - result.add(image); + downloadFile(image); } catch (IOException e) { LOGGER.error("Was unable to load page " + pageWithImageUrl); } } } else { - result.add(doc.select("div.player-container > a").attr("href")); + String imgUrl = doc.select("div.player-container > a").attr("href"); + downloadFile(imgUrl); } return result; } @@ -166,6 +173,15 @@ public class XhamsterRipper extends AbstractHTMLRipper { public void downloadURL(URL url, int index) { addURLToDownload(url, getPrefix(index)); } + + private void downloadFile(String url) { + try { + addURLToDownload(new URL(url), getPrefix(index)); + index = index + 1; + } catch (MalformedURLException e) { + LOGGER.error("The url \"" + url + "\" is malformed"); + } + } @Override public String getAlbumTitle(URL url) throws MalformedURLException { From fb2c6583b01d66148ad85059f1a397c731d23a98 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 6 Oct 2018 09:50:26 -0400 Subject: [PATCH 5/5] replaced unit test url --- .../rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java index 7847a36b..3ccccb49 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java @@ -35,7 +35,7 @@ public class XhamsterRipperTest extends RippersTest { } public void testGetNextPage() throws IOException { - XhamsterRipper ripper = new XhamsterRipper(new URL("https://pt.xhamster.com/photos/gallery/silvana-7105696")); + XhamsterRipper ripper = new XhamsterRipper(new URL("https://pt.xhamster.com/photos/gallery/mega-compil-6-10728626")); Document doc = ripper.getFirstPage(); try { ripper.getNextPage(doc);