Merge pull request #275 from cyian-1756/xhamster_rewrite

Rewrote xhamster ripper to use the html only mobile site and not the JS heavy beta site

src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java

@@ -3,91 +3,46 @@ package com.rarchives.ripme.ripper.rippers;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 
-import com.rarchives.ripme.ripper.AlbumRipper;
+import com.rarchives.ripme.ripper.AbstractHTMLRipper;
 import com.rarchives.ripme.utils.Http;
-import com.rarchives.ripme.utils.Utils;
 
-public class XhamsterRipper extends AlbumRipper {
+public class XhamsterRipper extends AbstractHTMLRipper {
-
-    private static final String HOST = "xhamster";
 
     public XhamsterRipper(URL url) throws IOException {
         super(url);
     }
 
     @Override
-    public boolean canRip(URL url) {
+    public String getHost() {
-        Pattern p = Pattern.compile("^https?://[wmde.]*xhamster\\.com/photos/gallery/.*[0-9]+$");
+        return "xhamster";
-        Matcher m = p.matcher(url.toExternalForm());
+    }
-        return m.matches();
+
+    @Override
+    public String getDomain() {
+        return "xhamster.com";
     }
 
     @Override
     public URL sanitizeURL(URL url) throws MalformedURLException {
-        return url;
+        String URLToReturn = url.toExternalForm();
-    }
+        URLToReturn = URLToReturn.replaceAll("m.xhamster.com", "xhamster.com");
-
+        URLToReturn = URLToReturn.replaceAll("\\w\\w.xhamster.com", "xhamster.com");
-    @Override
+        URL san_url = new URL(URLToReturn.replaceAll("xhamster.com", "m.xhamster.com"));
-    public void rip() throws IOException {
+        logger.info("sanitized URL is " + san_url.toExternalForm());
-        int index = 0;
+        return san_url;
-        String nextURL = this.url.toExternalForm();
-        while (nextURL != null) {
-            logger.info("    Retrieving " + nextURL);
-            Document doc = Http.url(nextURL).get();
-            for (Element thumb : doc.select("table.iListing div.img img")) {
-                if (!thumb.hasAttr("src")) {
-                    continue;
-                }
-                String image = thumb.attr("src");
-                // replace thumbnail urls with the urls to the full sized images
-                image = image.replaceAll(
-                        "https://upt.xhcdn\\.",
-                        "http://up.xhamster.");
-                image = image.replaceAll("ept\\.xhcdn", "ep.xhamster");
-                image = image.replaceAll(
-                        "_160\\.",
-                        "_1000.");
-                // Xhamster has shitty cert management and uses the wrong cert for their ep.xhamster Domain
-                // so we change all https requests to http
-                image = image.replaceAll(
-                            "https://",
-                            "http://");
-                index += 1;
-                String prefix = "";
-                if (Utils.getConfigBoolean("download.save_order", true)) {
-                    prefix = String.format("%03d_", index);
-                }
-                addURLToDownload(new URL(image), prefix);
-                if (isThisATest()) {
-                    break;
-                }
-            }
-            if (isThisATest()) {
-                break;
-            }
-            nextURL = null;
-            for (Element element : doc.select("a.last")) {
-                nextURL = element.attr("href");
-                break;
-            }
-        }
-        waitForThreads();
-    }
-
-    @Override
-    public String getHost() {
-        return HOST;
     }
 
     @Override
     public String getGID(URL url) throws MalformedURLException {
-        Pattern p = Pattern.compile("^https?://[wmde.]*xhamster\\.com/photos/gallery/.*?(\\d+)$");
+        Pattern p = Pattern.compile("^https?://[\\w\\w.]*xhamster\\.com/photos/gallery/.*?(\\d+)$");
         Matcher m = p.matcher(url.toExternalForm());
         if (m.matches()) {
             return m.group(1);
@@ -98,4 +53,54 @@ public class XhamsterRipper extends AlbumRipper {
                         + " Got: " + url);
     }
 
+
+    @Override
+    public Document getFirstPage() throws IOException {
+        // "url" is an instance field of the superclass
+        return Http.url(url).get();
+    }
+
+    @Override
+    public boolean canRip(URL url) {
+        Pattern p = Pattern.compile("^https?://[wmde.]*xhamster\\.com/photos/gallery/.*?(\\d+)$");
+        Matcher m = p.matcher(url.toExternalForm());
+        if (m.matches()) {
+            return true;
+        }
+        return false;
+    }
+
+    @Override
+    public Document getNextPage(Document doc) throws IOException {
+        if (doc.select("a.next").first().attr("href") != "") {
+            return Http.url(doc.select("a.next").first().attr("href")).get();
+        } else {
+            throw new IOException("No more pages");
+        }
+    }
+
+    @Override
+    public List<String> getURLsFromPage(Document doc) {
+        List<String> result = new ArrayList<>();
+        for (Element thumb : doc.select("div.picture_view > div.pictures_block > div.items > div.item-container > a > div.thumb_container > div.img > img")) {
+            String image = thumb.attr("src");
+            // replace thumbnail urls with the urls to the full sized images
+            image = image.replaceAll(
+                    "https://upt.xhcdn\\.",
+                    "http://up.xhamster.");
+            image = image.replaceAll("ept\\.xhcdn", "ep.xhamster");
+            image = image.replaceAll(
+                    "_160\\.",
+                    "_1000.");
+            // Xhamster has bad cert management and uses invalid certs for some cdns, so we change all our requests to http
+            image = image.replaceAll("https", "http");
+            result.add(image);
+        }
+        return result;
+    }
+
+    @Override
+    public void downloadURL(URL url, int index) {
+        addURLToDownload(url, getPrefix(index));
+    }
 }

src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java

@@ -6,9 +6,7 @@ import java.net.URL;
 import com.rarchives.ripme.ripper.rippers.XhamsterRipper;
 
 public class XhamsterRipperTest extends RippersTest {
-    // https://github.com/RipMeApp/ripme/issues/206 - Disabled tests: XhamsterRipperTest test cases are flaky in Travis CI
 
-    /*
     public void testXhamsterAlbum1() throws IOException {
         XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.com/photos/gallery/sexy-preggo-girls-9026608"));
         testRipper(ripper);
@@ -18,5 +16,4 @@ public class XhamsterRipperTest extends RippersTest {
         XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.com/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664"));
         testRipper(ripper);
     }
-    */
 }