Code style changes

This commit is contained in:
kas-luthor 2017-02-22 10:31:55 +01:00
parent 2f4793e9e3
commit ab91254226
2 changed files with 131 additions and 114 deletions

View File

@ -26,71 +26,79 @@ import org.jsoup.select.Elements;
* *
* @author * @author
*/ */
public class E621Ripper extends AbstractHTMLRipper{ public class E621Ripper extends AbstractHTMLRipper {
public static final int POOL_IMAGES_PER_PAGE = 24; public static final int POOL_IMAGES_PER_PAGE = 24;
private DownloadThreadPool e621ThreadPool = new DownloadThreadPool("e621"); private DownloadThreadPool e621ThreadPool = new DownloadThreadPool("e621");
public E621Ripper(URL url) throws IOException { public E621Ripper(URL url) throws IOException {
super(url); super(url);
} }
@Override @Override
public DownloadThreadPool getThreadPool() { public DownloadThreadPool getThreadPool() {
return e621ThreadPool; return e621ThreadPool;
} }
@Override @Override
public String getDomain() { public String getDomain() {
return "e621.net"; return "e621.net";
} }
@Override @Override
public String getHost() { public String getHost() {
return "e621"; return "e621";
} }
@Override @Override
public Document getFirstPage() throws IOException { public Document getFirstPage() throws IOException {
if (url.getPath().startsWith("/pool/show/")) if (url.getPath().startsWith("/pool/show/")) {
return Http.url("https://e621.net/pool/show/" + getTerm(url)).get(); return Http.url("https://e621.net/pool/show/" + getTerm(url)).get();
else } else {
return Http.url("https://e621.net/post/index/1/" + getTerm(url)).get(); return Http.url("https://e621.net/post/index/1/" + getTerm(url)).get();
}
} }
@Override @Override
public List<String> getURLsFromPage(Document page) { public List<String> getURLsFromPage(Document page) {
Elements elements = page.select("#post-list .thumb a,#pool-show .thumb a"); Elements elements = page.select("#post-list .thumb a,#pool-show .thumb a");
List<String> res = new ArrayList<String>(elements.size()); List<String> res = new ArrayList<String>(elements.size());
if (page.getElementById("pool-show") != null) { if (page.getElementById("pool-show") != null) {
int index = 0; int index = 0;
Element e = page.getElementById("paginator"); Element e = page.getElementById("paginator");
if (e != null && (e = e.getElementsByClass("current").first()) != null) if (e != null) {
index = (Integer.parseInt(e.text()) - 1) * POOL_IMAGES_PER_PAGE; e = e.getElementsByClass("current").first();
if (e != null) {
for (Element e_ : elements) index = (Integer.parseInt(e.text()) - 1) * POOL_IMAGES_PER_PAGE;
}
}
for (Element e_ : elements) {
res.add(e_.absUrl("href") + "#" + ++index); res.add(e_.absUrl("href") + "#" + ++index);
}
} else { } else {
for (Element e : elements) for (Element e : elements) {
res.add(e.absUrl("href") + "#" + e.child(0).attr("id").substring(1)); res.add(e.absUrl("href") + "#" + e.child(0).attr("id").substring(1));
}
} }
return res; return res;
} }
@Override @Override
public Document getNextPage(Document page) throws IOException { public Document getNextPage(Document page) throws IOException {
for (Element e : page.select("#paginator a")) { for (Element e : page.select("#paginator a")) {
if (e.attr("rel").equals("next")) if (e.attr("rel").equals("next")) {
return Http.url(e.absUrl("href")).get(); return Http.url(e.absUrl("href")).get();
}
} }
return null; return null;
} }
@Override @Override
public void downloadURL(final URL url, int index) { public void downloadURL(final URL url, int index) {
e621ThreadPool.addThread(new Thread(new Runnable() { e621ThreadPool.addThread(new Thread(new Runnable() {
@ -98,70 +106,76 @@ public static final int POOL_IMAGES_PER_PAGE = 24;
try { try {
Document page = Http.url(url).get(); Document page = Http.url(url).get();
Element e = page.getElementById("image"); Element e = page.getElementById("image");
if (e != null) if (e != null) {
addURLToDownload(new URL(e.absUrl("src")), Utils.getConfigBoolean("download.save_order", true) ? url.getRef() + "-" : ""); addURLToDownload(new URL(e.absUrl("src")), Utils.getConfigBoolean("download.save_order", true) ? url.getRef() + "-" : "");
else if ((e = page.select(".content object>param[name=\"movie\"]").first()) != null) } else if ((e = page.select(".content object>param[name=\"movie\"]").first()) != null) {
addURLToDownload(new URL(e.absUrl("value")), Utils.getConfigBoolean("download.save_order", true) ? url.getRef() + "-" : ""); addURLToDownload(new URL(e.absUrl("value")), Utils.getConfigBoolean("download.save_order", true) ? url.getRef() + "-" : "");
else } else {
Logger.getLogger(E621Ripper.class.getName()).log(Level.WARNING, "Unsupported media type - please report to program author: " + url.toString()); Logger.getLogger(E621Ripper.class.getName()).log(Level.WARNING, "Unsupported media type - please report to program author: " + url.toString());
}
} catch (IOException ex) { } catch (IOException ex) {
Logger.getLogger(E621Ripper.class.getName()).log(Level.SEVERE, null, ex); Logger.getLogger(E621Ripper.class.getName()).log(Level.SEVERE, null, ex);
} }
} }
})); }));
} }
private String getTerm(URL url) throws MalformedURLException { private String getTerm(URL url) throws MalformedURLException {
String query = url.getQuery(); String query = url.getQuery();
if (query != null) if (query != null) {
return Utils.parseUrlQuery(query, "tags"); return Utils.parseUrlQuery(query, "tags");
}
if (query == null) { if (query == null) {
if ((query = url.getPath()).startsWith("/post/index/")) { if ((query = url.getPath()).startsWith("/post/index/")) {
query = query.substring(12); query = query.substring(12);
int pos = query.indexOf('/'); int pos = query.indexOf('/');
if (pos == -1) if (pos == -1) {
return null; return null;
}
// skip page number // skip page number
query = query.substring(pos + 1); query = query.substring(pos + 1);
if (query.endsWith("/")) if (query.endsWith("/")) {
query = query.substring(0, query.length() - 1); query = query.substring(0, query.length() - 1);
}
try { try {
return URLDecoder.decode(query, "UTF-8"); return URLDecoder.decode(query, "UTF-8");
} catch (UnsupportedEncodingException e) { } catch (UnsupportedEncodingException e) {
// Shouldn't happen since UTF-8 is required to be supported // Shouldn't happen since UTF-8 is required to be supported
throw new RuntimeException(e); throw new RuntimeException(e);
} }
} else if (query.startsWith("/pool/show/")) { } else if (query.startsWith("/pool/show/")) {
query = query.substring(11); query = query.substring(11);
if (query.endsWith("/")) if (query.endsWith("/")) {
query = query.substring(0, query.length() - 1); query = query.substring(0, query.length() - 1);
}
return query; return query;
} }
} }
return null; return null;
} }
@Override @Override
public String getGID(URL url) throws MalformedURLException { public String getGID(URL url) throws MalformedURLException {
String prefix = ""; String prefix = "";
if (url.getPath().startsWith("/pool/show/")) if (url.getPath().startsWith("/pool/show/")) {
prefix = "pool_"; prefix = "pool_";
else } else {
prefix = "term_"; prefix = "term_";
}
return Utils.filesystemSafe(prefix + getTerm(url)); return Utils.filesystemSafe(prefix + getTerm(url));
} }
} }

View File

@ -390,70 +390,73 @@ public class Utils {
} }
return result; return result;
} }
/** /**
* Parses an URL query * Parses an URL query
* *
* @param query * @param query
* The query part of an URL * The query part of an URL
* @return The map of all query parameters * @return The map of all query parameters
*/ */
public static Map<String,String> parseUrlQuery(String query) { public static Map<String,String> parseUrlQuery(String query) {
Map<String,String> res = new HashMap<String, String>(); Map<String,String> res = new HashMap<String, String>();
if (query.equals("")) if (query.equals("")){
return res; return res;
}
String[] parts = query.split("&");
int pos; String[] parts = query.split("&");
int pos;
try {
for (String part : parts) { try {
if ((pos = part.indexOf('=')) >= 0) for (String part : parts) {
res.put(URLDecoder.decode(part.substring(0, pos), "UTF-8"), if ((pos = part.indexOf('=')) >= 0){
URLDecoder.decode(part.substring(pos + 1), "UTF-8")); res.put(URLDecoder.decode(part.substring(0, pos), "UTF-8"), URLDecoder.decode(part.substring(pos + 1), "UTF-8"));
else }else{
res.put(URLDecoder.decode(part, "UTF-8"), ""); res.put(URLDecoder.decode(part, "UTF-8"), "");
} }
} catch (UnsupportedEncodingException e) { }
// Shouldn't happen since UTF-8 is required to be supported } catch (UnsupportedEncodingException e) {
throw new RuntimeException(e); // Shouldn't happen since UTF-8 is required to be supported
} throw new RuntimeException(e);
}
return res;
} return res;
}
/**
* Parses an URL query and returns the requested parameter's value /**
* * Parses an URL query and returns the requested parameter's value
* @param query *
* The query part of an URL * @param query
* @param key * The query part of an URL
* The key whose value is requested * @param key
* @return The associated value or null if key wasn't found * The key whose value is requested
*/ * @return The associated value or null if key wasn't found
public static String parseUrlQuery(String query, String key) { */
if (query.equals("")) public static String parseUrlQuery(String query, String key) {
return null; if (query.equals("")){
return null;
String[] parts = query.split("&"); }
int pos;
String[] parts = query.split("&");
try { int pos;
for (String part : parts) {
if ((pos = part.indexOf('=')) >= 0) { try {
if (URLDecoder.decode(part.substring(0, pos), "UTF-8").equals(key)) for (String part : parts) {
return URLDecoder.decode(part.substring(pos + 1), "UTF-8"); if ((pos = part.indexOf('=')) >= 0) {
if (URLDecoder.decode(part.substring(0, pos), "UTF-8").equals(key)){
} else if (URLDecoder.decode(part, "UTF-8").equals(key)) { return URLDecoder.decode(part.substring(pos + 1), "UTF-8");
return ""; }
}
} } else if (URLDecoder.decode(part, "UTF-8").equals(key)) {
} catch (UnsupportedEncodingException e) { return "";
// Shouldn't happen since UTF-8 is required to be supported }
throw new RuntimeException(e); }
} } catch (UnsupportedEncodingException e) {
// Shouldn't happen since UTF-8 is required to be supported
return null; throw new RuntimeException(e);
} }
return null;
}
} }