Gurkengewuerz/ImgScroll
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #746 from cyian-1756/byteProgressBar

Browse Source 
All rippers can now use a byte progress bar, added the ability to resume some file downloads
This commit is contained in:
cyian-1756 2018-06-28 06:52:06 -04:00 committed by GitHub
commit f698bdacde
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 163 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
View File

@ -613,4 +613,9 @@ public abstract class AbstractRipper
protected boolean isThisATest() {
return thisIsATest;
}
// If true ripme uses a byte progress bar
protected boolean useByteProgessBar() { return false;}
// If true ripme will try to resume a broken download for this ripper
protected boolean tryResumeDownload() { return false;}
}

55
src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
View File

@ -59,18 +59,25 @@ class DownloadFileThread extends Thread {
this.cookies = cookies;
}
/**
* Attempts to download the file. Retries as needed.
* Notifies observers upon completion/error/warn.
*/
public void run() {
long fileSize = 0;
int bytesTotal = 0;
int bytesDownloaded = 0;
if (saveAs.exists() && observer.tryResumeDownload()) {
fileSize = saveAs.length();
}
try {
observer.stopCheck();
} catch (IOException e) {
observer.downloadErrored(url, "Download interrupted");
return;
}
if (saveAs.exists()) {
if (saveAs.exists() && !observer.tryResumeDownload()) {
if (Utils.getConfigBoolean("file.overwrite", false)) {
logger.info("[!] Deleting existing file" + prettySaveAs);
saveAs.delete();
@ -80,7 +87,6 @@ class DownloadFileThread extends Thread {
return;
}
}
URL urlToDownload = this.url;
boolean redirected = false;
int tries = 0; // Number of attempts to download
@ -114,11 +120,20 @@ class DownloadFileThread extends Thread {
cookie += key + "=" + cookies.get(key);
}
huc.setRequestProperty("Cookie", cookie);
if (observer.tryResumeDownload()) {
if (fileSize != 0) {
huc.setRequestProperty("Range", "bytes=" + fileSize + "-");
}
}
logger.debug("Request properties: " + huc.getRequestProperties());
huc.connect();
int statusCode = huc.getResponseCode();
logger.debug("Status code: " + statusCode);
if (statusCode != 206 && observer.tryResumeDownload() && saveAs.exists()) {
// TODO find a better way to handle servers that don't support resuming downloads then just erroring out
throw new IOException("Server doesn't support resuming downloads");
}
if (statusCode / 100 == 3) { // 3xx Redirect
if (!redirected) {
// Don't increment retries on the first redirect
@ -146,6 +161,15 @@ class DownloadFileThread extends Thread {
observer.downloadErrored(url, "Imgur image is 404: " + url.toExternalForm());
return;
}
// If the ripper is using the bytes progress bar set bytesTotal to huc.getContentLength()
if (observer.useByteProgessBar()) {
bytesTotal = huc.getContentLength();
observer.setBytesTotal(bytesTotal);
observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal);
logger.debug("Size of file at " + this.url + " = " + bytesTotal + "b");
}
// Save file
bis = new BufferedInputStream(huc.getInputStream());
@ -154,9 +178,30 @@ class DownloadFileThread extends Thread {
String fileExt = URLConnection.guessContentTypeFromStream(bis).replaceAll("image/", "");
saveAs = new File(saveAs.toString() + "." + fileExt);
}
fos = new FileOutputStream(saveAs);
IOUtils.copy(bis, fos);
// If we're resuming a download we append data to the existing file
if (statusCode == 206) {
fos = new FileOutputStream(saveAs, true);
} else {
fos = new FileOutputStream(saveAs);
}
byte[] data = new byte[1024 * 256];
int bytesRead;
while ( (bytesRead = bis.read(data)) != -1) {
try {
observer.stopCheck();
} catch (IOException e) {
observer.downloadErrored(url, "Download interrupted");
return;
}
fos.write(data, 0, bytesRead);
if (observer.useByteProgessBar()) {
bytesDownloaded += bytesRead;
observer.setBytesCompleted(bytesDownloaded);
observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded);
}
}
bis.close();
fos.close();
break; // Download successful: break out of infinite loop
} catch (HttpStatusException hse) {
logger.debug("HTTP status exception", hse);

3
src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java
View File

@ -76,7 +76,8 @@ class DownloadVideoThread extends Thread {
int tries = 0; // Number of attempts to download
do {
InputStream bis = null; OutputStream fos = null;
byte[] data = new byte[1024 * 256]; int bytesRead;
byte[] data = new byte[1024 * 256];
int bytesRead;
try {
logger.info(" Downloading file: " + url + (tries > 0 ? " Retry #" + tries : ""));
observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm());

71
src/main/java/com/rarchives/ripme/ripper/rippers/video/GfycatRipper.java → src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java
View File

@ -1,18 +1,26 @@
package com.rarchives.ripme.ripper.rippers.video;
package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.utils.Utils;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.VideoRipper;
import com.rarchives.ripme.utils.Http;
public class GfycatRipper extends VideoRipper {
public class GfycatRipper extends AbstractHTMLRipper {
private int bytesTotal = 1;
private int bytesCompleted = 1;
private static final String HOST = "gfycat.com";
@ -20,9 +28,14 @@ public class GfycatRipper extends VideoRipper {
super(url);
}
@Override
public String getDomain() {
return "gfycat.com";
}
@Override
public String getHost() {
return HOST;
return "gfycat";
}
@Override
@ -37,6 +50,16 @@ public class GfycatRipper extends VideoRipper {
return url;
}
@Override
public Document getFirstPage() throws IOException {
return Http.url(url).get();
}
@Override
public void downloadURL(URL url, int index) {
addURLToDownload(url, getPrefix(index));
}
@Override
public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^https?://[wm.]*gfycat\\.com/([a-zA-Z0-9]+).*$");
@ -52,10 +75,15 @@ public class GfycatRipper extends VideoRipper {
}
@Override
public void rip() throws IOException {
String vidUrl = getVideoURL(this.url);
addURLToDownload(new URL(vidUrl), "gfycat_" + getGID(this.url));
waitForThreads();
public List<String> getURLsFromPage(Document doc) {
List<String> result = new ArrayList<>();
Elements videos = doc.select("source#mp4Source");
String vidUrl = videos.first().attr("src");
if (vidUrl.startsWith("//")) {
vidUrl = "http:" + vidUrl;
}
result.add(vidUrl);
return result;
}
/**
@ -66,10 +94,10 @@ public class GfycatRipper extends VideoRipper {
*/
public static String getVideoURL(URL url) throws IOException {
LOGGER.info("Retrieving " + url.toExternalForm());
//Sanitize the URL first
url = new URL(url.toExternalForm().replace("/gifs/detail", ""));
Document doc = Http.url(url).get();
Elements videos = doc.select("source#mp4Source");
if (videos.isEmpty()) {
@ -81,4 +109,27 @@ public class GfycatRipper extends VideoRipper {
}
return vidUrl;
}
@Override
public String getStatusText() {
return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal);
}
@Override
public int getCompletionPercentage() {
return (int) (100 * (bytesCompleted / (float) bytesTotal));
}
@Override
public void setBytesTotal(int bytes) {
this.bytesTotal = bytes;
}
@Override
public void setBytesCompleted(int bytes) {
this.bytesCompleted = bytes;
}
@Override
public boolean useByteProgessBar() {return true;}
}

27
src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java
View File

@ -8,6 +8,7 @@ import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.rarchives.ripme.utils.Utils;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
@ -16,6 +17,9 @@ import com.rarchives.ripme.utils.Http;
public class GfycatporntubeRipper extends AbstractHTMLRipper {
private int bytesTotal = 1;
private int bytesCompleted = 1;
public GfycatporntubeRipper(URL url) throws IOException {
super(url);
}
@ -58,4 +62,27 @@ public class GfycatporntubeRipper extends AbstractHTMLRipper {
public void downloadURL(URL url, int index) {
addURLToDownload(url, getPrefix(index));
}
@Override
public String getStatusText() {
return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal);
}
@Override
public int getCompletionPercentage() {
return (int) (100 * (bytesCompleted / (float) bytesTotal));
}
@Override
public void setBytesTotal(int bytes) {
this.bytesTotal = bytes;
}
@Override
public void setBytesCompleted(int bytes) {
this.bytesCompleted = bytes;
}
@Override
public boolean useByteProgessBar() {return true;}
}

67
src/main/java/com/rarchives/ripme/ripper/rippers/video/VineRipper.java
View File

@ -1,67 +0,0 @@
package com.rarchives.ripme.ripper.rippers.video;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import com.rarchives.ripme.ripper.VideoRipper;
import com.rarchives.ripme.utils.Http;
public class VineRipper extends VideoRipper {
private static final String HOST = "vine";
public VineRipper(URL url) throws IOException {
super(url);
}
@Override
public String getHost() {
return HOST;
}
@Override
public boolean canRip(URL url) {
// https://vine.co/v/hiqQrP0eUZx
Pattern p = Pattern.compile("^https?://[wm.]*vine\\.co/v/[a-zA-Z0-9]+.*$");
Matcher m = p.matcher(url.toExternalForm());
return m.matches();
}
@Override
public URL sanitizeURL(URL url) throws MalformedURLException {
return url;
}
@Override
public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^https?://[wm.]*vine\\.co/v/([a-zA-Z0-9]+).*$");
Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) {
return m.group(1);
}
throw new MalformedURLException(
"Expected vine format:"
+ "vine.co/v/####"
+ " Got: " + url);
}
@Override
public void rip() throws IOException {
LOGGER.info(" Retrieving " + this.url.toExternalForm());
Document doc = Http.url(this.url).get();
Elements props = doc.select("meta[property=twitter:player:stream]");
if (props.isEmpty()) {
throw new IOException("Could not find meta property 'twitter:player:stream' at " + url);
}
String vidUrl = props.get(0).attr("content");
addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url));
waitForThreads();
}
}

2
src/main/java/com/rarchives/ripme/utils/RipUtils.java
View File

@ -12,7 +12,7 @@ import com.rarchives.ripme.ripper.rippers.EroShareRipper;
import com.rarchives.ripme.ripper.rippers.EromeRipper;
import com.rarchives.ripme.ripper.rippers.ImgurRipper;
import com.rarchives.ripme.ripper.rippers.VidbleRipper;
import com.rarchives.ripme.ripper.rippers.video.GfycatRipper;
import com.rarchives.ripme.ripper.rippers.GfycatRipper;
import org.apache.commons.lang.math.NumberUtils;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;

16
src/main/java/com/rarchives/ripme/utils/Utils.java
View File

@ -715,4 +715,20 @@ public class Utils {
}
}
/**
* Formats and reuturns the status text for rippers using the byte progress bar
*
* @param completionPercentage An int between 0 and 100 which repersents how close the download is to complete
* @param bytesCompleted How many bytes have been downloaded
* @param bytesTotal The total size of the file that is being downloaded
* @return Returns the formatted status text for rippers using the byte progress bar
*/
public static String getByteStatusText(int completionPercentage, int bytesCompleted, int bytesTotal) {
return String.valueOf(completionPercentage) +
"% - " +
Utils.bytesToHumanReadable(bytesCompleted) +
" / " +
Utils.bytesToHumanReadable(bytesTotal);
}
}

2
src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java
View File

@ -1,6 +1,6 @@
package com.rarchives.ripme.tst.ripper.rippers;
import com.rarchives.ripme.ripper.rippers.video.GfycatRipper;
import com.rarchives.ripme.ripper.rippers.GfycatRipper;
import java.io.IOException;
import java.net.URL;

1
src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java
View File

@ -5,7 +5,6 @@ import java.io.IOException;
import java.net.URL;
import com.rarchives.ripme.ripper.rippers.RedditRipper;
import com.rarchives.ripme.ripper.rippers.video.GfycatRipper;
public class RedditRipperTest extends RippersTest {
// https://github.com/RipMeApp/ripme/issues/253 - Disabled tests: RedditRipperTest#testRedditSubreddit*Rip is flaky

13
src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java
View File

@ -8,7 +8,6 @@ import java.util.List;
import com.rarchives.ripme.ripper.VideoRipper;
import com.rarchives.ripme.ripper.rippers.video.PornhubRipper;
import com.rarchives.ripme.ripper.rippers.video.TwitchVideoRipper;
import com.rarchives.ripme.ripper.rippers.video.VineRipper;
import com.rarchives.ripme.ripper.rippers.video.XhamsterRipper;
import com.rarchives.ripme.ripper.rippers.video.XvideosRipper;
import com.rarchives.ripme.ripper.rippers.video.YoupornRipper;
@ -78,18 +77,6 @@ public class VideoRippersTest extends RippersTest {
}
}
// https://github.com/RipMeApp/ripme/issues/186
/*
public void testVineRipper() throws IOException {
List<URL> contentURLs = new ArrayList<>();
contentURLs.add(new URL("https://vine.co/v/hiqQrP0eUZx"));
for (URL url : contentURLs) {
VineRipper ripper = new VineRipper(url);
videoTestHelper(ripper);
}
}
*/
public void testYoupornRipper() throws IOException {
List<URL> contentURLs = new ArrayList<>();
contentURLs.add(new URL("http://www.youporn.com/watch/7669155/mrs-li-amateur-69-orgasm/?from=categ"));