Code style changes
This commit is contained in:
parent
2f4793e9e3
commit
ab91254226
@ -26,71 +26,79 @@ import org.jsoup.select.Elements;
|
|||||||
*
|
*
|
||||||
* @author
|
* @author
|
||||||
*/
|
*/
|
||||||
public class E621Ripper extends AbstractHTMLRipper{
|
public class E621Ripper extends AbstractHTMLRipper {
|
||||||
public static final int POOL_IMAGES_PER_PAGE = 24;
|
public static final int POOL_IMAGES_PER_PAGE = 24;
|
||||||
|
|
||||||
private DownloadThreadPool e621ThreadPool = new DownloadThreadPool("e621");
|
private DownloadThreadPool e621ThreadPool = new DownloadThreadPool("e621");
|
||||||
|
|
||||||
public E621Ripper(URL url) throws IOException {
|
public E621Ripper(URL url) throws IOException {
|
||||||
super(url);
|
super(url);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public DownloadThreadPool getThreadPool() {
|
public DownloadThreadPool getThreadPool() {
|
||||||
return e621ThreadPool;
|
return e621ThreadPool;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getDomain() {
|
public String getDomain() {
|
||||||
return "e621.net";
|
return "e621.net";
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getHost() {
|
public String getHost() {
|
||||||
return "e621";
|
return "e621";
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Document getFirstPage() throws IOException {
|
public Document getFirstPage() throws IOException {
|
||||||
if (url.getPath().startsWith("/pool/show/"))
|
if (url.getPath().startsWith("/pool/show/")) {
|
||||||
return Http.url("https://e621.net/pool/show/" + getTerm(url)).get();
|
return Http.url("https://e621.net/pool/show/" + getTerm(url)).get();
|
||||||
else
|
} else {
|
||||||
return Http.url("https://e621.net/post/index/1/" + getTerm(url)).get();
|
return Http.url("https://e621.net/post/index/1/" + getTerm(url)).get();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<String> getURLsFromPage(Document page) {
|
public List<String> getURLsFromPage(Document page) {
|
||||||
Elements elements = page.select("#post-list .thumb a,#pool-show .thumb a");
|
Elements elements = page.select("#post-list .thumb a,#pool-show .thumb a");
|
||||||
List<String> res = new ArrayList<String>(elements.size());
|
List<String> res = new ArrayList<String>(elements.size());
|
||||||
|
|
||||||
if (page.getElementById("pool-show") != null) {
|
if (page.getElementById("pool-show") != null) {
|
||||||
int index = 0;
|
int index = 0;
|
||||||
|
|
||||||
Element e = page.getElementById("paginator");
|
Element e = page.getElementById("paginator");
|
||||||
if (e != null && (e = e.getElementsByClass("current").first()) != null)
|
if (e != null) {
|
||||||
index = (Integer.parseInt(e.text()) - 1) * POOL_IMAGES_PER_PAGE;
|
e = e.getElementsByClass("current").first();
|
||||||
|
if (e != null) {
|
||||||
for (Element e_ : elements)
|
index = (Integer.parseInt(e.text()) - 1) * POOL_IMAGES_PER_PAGE;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (Element e_ : elements) {
|
||||||
res.add(e_.absUrl("href") + "#" + ++index);
|
res.add(e_.absUrl("href") + "#" + ++index);
|
||||||
|
}
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
for (Element e : elements)
|
for (Element e : elements) {
|
||||||
res.add(e.absUrl("href") + "#" + e.child(0).attr("id").substring(1));
|
res.add(e.absUrl("href") + "#" + e.child(0).attr("id").substring(1));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Document getNextPage(Document page) throws IOException {
|
public Document getNextPage(Document page) throws IOException {
|
||||||
for (Element e : page.select("#paginator a")) {
|
for (Element e : page.select("#paginator a")) {
|
||||||
if (e.attr("rel").equals("next"))
|
if (e.attr("rel").equals("next")) {
|
||||||
return Http.url(e.absUrl("href")).get();
|
return Http.url(e.absUrl("href")).get();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void downloadURL(final URL url, int index) {
|
public void downloadURL(final URL url, int index) {
|
||||||
e621ThreadPool.addThread(new Thread(new Runnable() {
|
e621ThreadPool.addThread(new Thread(new Runnable() {
|
||||||
@ -98,70 +106,76 @@ public static final int POOL_IMAGES_PER_PAGE = 24;
|
|||||||
try {
|
try {
|
||||||
Document page = Http.url(url).get();
|
Document page = Http.url(url).get();
|
||||||
Element e = page.getElementById("image");
|
Element e = page.getElementById("image");
|
||||||
|
|
||||||
if (e != null)
|
if (e != null) {
|
||||||
addURLToDownload(new URL(e.absUrl("src")), Utils.getConfigBoolean("download.save_order", true) ? url.getRef() + "-" : "");
|
addURLToDownload(new URL(e.absUrl("src")), Utils.getConfigBoolean("download.save_order", true) ? url.getRef() + "-" : "");
|
||||||
else if ((e = page.select(".content object>param[name=\"movie\"]").first()) != null)
|
} else if ((e = page.select(".content object>param[name=\"movie\"]").first()) != null) {
|
||||||
addURLToDownload(new URL(e.absUrl("value")), Utils.getConfigBoolean("download.save_order", true) ? url.getRef() + "-" : "");
|
addURLToDownload(new URL(e.absUrl("value")), Utils.getConfigBoolean("download.save_order", true) ? url.getRef() + "-" : "");
|
||||||
else
|
} else {
|
||||||
Logger.getLogger(E621Ripper.class.getName()).log(Level.WARNING, "Unsupported media type - please report to program author: " + url.toString());
|
Logger.getLogger(E621Ripper.class.getName()).log(Level.WARNING, "Unsupported media type - please report to program author: " + url.toString());
|
||||||
|
}
|
||||||
|
|
||||||
} catch (IOException ex) {
|
} catch (IOException ex) {
|
||||||
Logger.getLogger(E621Ripper.class.getName()).log(Level.SEVERE, null, ex);
|
Logger.getLogger(E621Ripper.class.getName()).log(Level.SEVERE, null, ex);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
private String getTerm(URL url) throws MalformedURLException {
|
private String getTerm(URL url) throws MalformedURLException {
|
||||||
String query = url.getQuery();
|
String query = url.getQuery();
|
||||||
|
|
||||||
if (query != null)
|
if (query != null) {
|
||||||
return Utils.parseUrlQuery(query, "tags");
|
return Utils.parseUrlQuery(query, "tags");
|
||||||
|
}
|
||||||
|
|
||||||
if (query == null) {
|
if (query == null) {
|
||||||
if ((query = url.getPath()).startsWith("/post/index/")) {
|
if ((query = url.getPath()).startsWith("/post/index/")) {
|
||||||
query = query.substring(12);
|
query = query.substring(12);
|
||||||
|
|
||||||
int pos = query.indexOf('/');
|
int pos = query.indexOf('/');
|
||||||
if (pos == -1)
|
if (pos == -1) {
|
||||||
return null;
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
// skip page number
|
// skip page number
|
||||||
query = query.substring(pos + 1);
|
query = query.substring(pos + 1);
|
||||||
|
|
||||||
if (query.endsWith("/"))
|
if (query.endsWith("/")) {
|
||||||
query = query.substring(0, query.length() - 1);
|
query = query.substring(0, query.length() - 1);
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
return URLDecoder.decode(query, "UTF-8");
|
return URLDecoder.decode(query, "UTF-8");
|
||||||
} catch (UnsupportedEncodingException e) {
|
} catch (UnsupportedEncodingException e) {
|
||||||
// Shouldn't happen since UTF-8 is required to be supported
|
// Shouldn't happen since UTF-8 is required to be supported
|
||||||
throw new RuntimeException(e);
|
throw new RuntimeException(e);
|
||||||
}
|
}
|
||||||
|
|
||||||
} else if (query.startsWith("/pool/show/")) {
|
} else if (query.startsWith("/pool/show/")) {
|
||||||
query = query.substring(11);
|
query = query.substring(11);
|
||||||
|
|
||||||
if (query.endsWith("/"))
|
if (query.endsWith("/")) {
|
||||||
query = query.substring(0, query.length() - 1);
|
query = query.substring(0, query.length() - 1);
|
||||||
|
}
|
||||||
|
|
||||||
return query;
|
return query;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getGID(URL url) throws MalformedURLException {
|
public String getGID(URL url) throws MalformedURLException {
|
||||||
String prefix = "";
|
String prefix = "";
|
||||||
if (url.getPath().startsWith("/pool/show/"))
|
if (url.getPath().startsWith("/pool/show/")) {
|
||||||
prefix = "pool_";
|
prefix = "pool_";
|
||||||
else
|
} else {
|
||||||
prefix = "term_";
|
prefix = "term_";
|
||||||
|
}
|
||||||
|
|
||||||
return Utils.filesystemSafe(prefix + getTerm(url));
|
return Utils.filesystemSafe(prefix + getTerm(url));
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
@ -390,70 +390,73 @@ public class Utils {
|
|||||||
}
|
}
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Parses an URL query
|
* Parses an URL query
|
||||||
*
|
*
|
||||||
* @param query
|
* @param query
|
||||||
* The query part of an URL
|
* The query part of an URL
|
||||||
* @return The map of all query parameters
|
* @return The map of all query parameters
|
||||||
*/
|
*/
|
||||||
public static Map<String,String> parseUrlQuery(String query) {
|
public static Map<String,String> parseUrlQuery(String query) {
|
||||||
Map<String,String> res = new HashMap<String, String>();
|
Map<String,String> res = new HashMap<String, String>();
|
||||||
|
|
||||||
if (query.equals(""))
|
if (query.equals("")){
|
||||||
return res;
|
return res;
|
||||||
|
}
|
||||||
String[] parts = query.split("&");
|
|
||||||
int pos;
|
String[] parts = query.split("&");
|
||||||
|
int pos;
|
||||||
try {
|
|
||||||
for (String part : parts) {
|
try {
|
||||||
if ((pos = part.indexOf('=')) >= 0)
|
for (String part : parts) {
|
||||||
res.put(URLDecoder.decode(part.substring(0, pos), "UTF-8"),
|
if ((pos = part.indexOf('=')) >= 0){
|
||||||
URLDecoder.decode(part.substring(pos + 1), "UTF-8"));
|
res.put(URLDecoder.decode(part.substring(0, pos), "UTF-8"), URLDecoder.decode(part.substring(pos + 1), "UTF-8"));
|
||||||
else
|
}else{
|
||||||
res.put(URLDecoder.decode(part, "UTF-8"), "");
|
res.put(URLDecoder.decode(part, "UTF-8"), "");
|
||||||
}
|
}
|
||||||
} catch (UnsupportedEncodingException e) {
|
}
|
||||||
// Shouldn't happen since UTF-8 is required to be supported
|
} catch (UnsupportedEncodingException e) {
|
||||||
throw new RuntimeException(e);
|
// Shouldn't happen since UTF-8 is required to be supported
|
||||||
}
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
return res;
|
|
||||||
}
|
return res;
|
||||||
|
}
|
||||||
/**
|
|
||||||
* Parses an URL query and returns the requested parameter's value
|
/**
|
||||||
*
|
* Parses an URL query and returns the requested parameter's value
|
||||||
* @param query
|
*
|
||||||
* The query part of an URL
|
* @param query
|
||||||
* @param key
|
* The query part of an URL
|
||||||
* The key whose value is requested
|
* @param key
|
||||||
* @return The associated value or null if key wasn't found
|
* The key whose value is requested
|
||||||
*/
|
* @return The associated value or null if key wasn't found
|
||||||
public static String parseUrlQuery(String query, String key) {
|
*/
|
||||||
if (query.equals(""))
|
public static String parseUrlQuery(String query, String key) {
|
||||||
return null;
|
if (query.equals("")){
|
||||||
|
return null;
|
||||||
String[] parts = query.split("&");
|
}
|
||||||
int pos;
|
|
||||||
|
String[] parts = query.split("&");
|
||||||
try {
|
int pos;
|
||||||
for (String part : parts) {
|
|
||||||
if ((pos = part.indexOf('=')) >= 0) {
|
try {
|
||||||
if (URLDecoder.decode(part.substring(0, pos), "UTF-8").equals(key))
|
for (String part : parts) {
|
||||||
return URLDecoder.decode(part.substring(pos + 1), "UTF-8");
|
if ((pos = part.indexOf('=')) >= 0) {
|
||||||
|
if (URLDecoder.decode(part.substring(0, pos), "UTF-8").equals(key)){
|
||||||
} else if (URLDecoder.decode(part, "UTF-8").equals(key)) {
|
return URLDecoder.decode(part.substring(pos + 1), "UTF-8");
|
||||||
return "";
|
}
|
||||||
}
|
|
||||||
}
|
} else if (URLDecoder.decode(part, "UTF-8").equals(key)) {
|
||||||
} catch (UnsupportedEncodingException e) {
|
return "";
|
||||||
// Shouldn't happen since UTF-8 is required to be supported
|
}
|
||||||
throw new RuntimeException(e);
|
}
|
||||||
}
|
} catch (UnsupportedEncodingException e) {
|
||||||
|
// Shouldn't happen since UTF-8 is required to be supported
|
||||||
return null;
|
throw new RuntimeException(e);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user