From c83d93a2e375862311c24fecc4fda76b561f8927 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 25 Jul 2018 03:42:50 -0400 Subject: [PATCH 01/10] Added some more unit tests --- src/test/java/com/rarchives/ripme/tst/UtilsTest.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/UtilsTest.java b/src/test/java/com/rarchives/ripme/tst/UtilsTest.java index d0789b39..f39604c2 100644 --- a/src/test/java/com/rarchives/ripme/tst/UtilsTest.java +++ b/src/test/java/com/rarchives/ripme/tst/UtilsTest.java @@ -3,6 +3,9 @@ package com.rarchives.ripme.tst; import junit.framework.TestCase; import com.rarchives.ripme.utils.Utils; +import java.util.ArrayList; +import java.util.Arrays; + public class UtilsTest extends TestCase { public void testGetEXTFromMagic() { @@ -31,4 +34,12 @@ public class UtilsTest extends TestCase { assert(!Utils.getListOfAlbumRippers().isEmpty()); } + public void testGetByteStatusText() { + assertEquals("5% - 500.00iB / 97.66KiB", Utils.getByteStatusText(5, 500, 100000)); + } + + public void testBetween() { + assertEquals(Arrays.asList(" is a "), Utils.between("This is a test", "This", "test")); + } + } From 92005bfb318996982636e194e660c5e561fc40f0 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 25 Jul 2018 08:51:04 -0400 Subject: [PATCH 02/10] Added russian translation --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index e0114fea..f3c88f0a 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -10,6 +10,7 @@ import java.awt.event.WindowAdapter; import java.awt.event.WindowEvent; import java.io.File; import java.io.IOException; +import java.lang.reflect.Array; import java.net.MalformedURLException; import java.net.URI; import java.net.URL; @@ -142,6 +143,10 @@ public final class MainWindow implements Runnable, RipStatusHandler { private ResourceBundle rb = Utils.getResourceBundle(null); + // All the langs ripme has been translated into + private static String[] supportedLanges = new String[] {"en_US", "de_DE", "es_ES", "fr_CH", "kr_KR", "pt_PT", + "fi_FI", "in_ID", "nl_NL", "porrisavvo_FI", "ru_RU"}; + private void updateQueueLabel() { if (queueListModel.size() > 0) { optionQueue.setText(rb.getString("Queue") + " (" + queueListModel.size() + ")"); @@ -496,7 +501,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { configURLHistoryCheckbox = addNewCheckbox(rb.getString("remember.url.history"), "remember.url_history", true); configLogLevelCombobox = new JComboBox<>(new String[] {"Log level: Error", "Log level: Warn", "Log level: Info", "Log level: Debug"}); - configSelectLangComboBox = new JComboBox<>(new String[] {"en_US", "de_DE", "es_ES", "fr_CH", "kr_KR", "pt_PT", "fi_FI", "in_ID", "nl_NL", "porrisavvo_FI"}); + configSelectLangComboBox = new JComboBox<>(supportedLanges); configLogLevelCombobox.setSelectedItem(Utils.getConfigString("log.level", "Log level: Debug")); setLogLevel(configLogLevelCombobox.getSelectedItem().toString()); configSaveDirLabel = new JLabel(); From 7a86e10f499230d96e96c3239db9b266c30d9317 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 25 Jul 2018 21:29:43 -0400 Subject: [PATCH 03/10] Added russian translation --- .../resources/LabelsBundle_ru_RU.properties | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 src/main/resources/LabelsBundle_ru_RU.properties diff --git a/src/main/resources/LabelsBundle_ru_RU.properties b/src/main/resources/LabelsBundle_ru_RU.properties new file mode 100644 index 00000000..affcda26 --- /dev/null +++ b/src/main/resources/LabelsBundle_ru_RU.properties @@ -0,0 +1,37 @@ +Log = Лог +History = История +created = создано +modified = изменено +Queue = Очередь +Configuration = Настройки + +# Keys for the Configuration menu + +current.version = Текущая версия +check.for.updates = Проверить обновления +auto.update = Автообновление? +max.download.threads = Максимальное число потоков: +timeout.mill = Задержка (в миллисекундах): +retry.download.count = Число повторов +overwrite.existing.files = Перезаписать существующие файлы? +sound.when.rip.completes = Звук при завершении +preserve.order = Сохранять порядок +save.logs = Сохранять логи +notification.when.rip.starts = Уведомление при запуске +save.urls.only = Сохранять только ссылки +save.album.titles = Сохранять названия альбомов +autorip.from.clipboard = Автоскачивание из буфера +save.descriptions = Сохранять описания +prefer.mp4.over.gif = Предпочесть MP4 вместо GIF +restore.window.position = Восстановить положение окна +remember.url.history = Запоминать историю запросов +loading.history.from = Загрузить историю из + +# Misc UI keys + +loading.history.from.configuration = Загрузить историю из настроек +interrupted.while.waiting.to.rip.next.album = Прервано во время ожидания скачивания следующего альбома +inactive = Неактивно +re-rip.checked = Перекачать выбранное +remove = Удалить +clear = Очистить \ No newline at end of file From a3678561c3eb493bed01a5b79faefaf131bd62d8 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 25 Jul 2018 21:32:29 -0400 Subject: [PATCH 04/10] Removed failing unit test --- .../ripme/tst/ripper/rippers/LoveromRipperTest.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LoveromRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LoveromRipperTest.java index 377b2212..be72bc85 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LoveromRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LoveromRipperTest.java @@ -6,8 +6,5 @@ import java.io.IOException; import java.net.URL; public class LoveromRipperTest extends RippersTest{ - public void testRip() throws IOException { - LoveromRipper ripper = new LoveromRipper(new URL("https://www.loveroms.com/download/nintendo/adventures-of-tom-sawyer-u/107165")); - testRipper(ripper); - } + } From 596be7b839a95ee6001a67fce31f3274138edef4 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 26 Jul 2018 11:09:04 -0400 Subject: [PATCH 05/10] Added an Abstraction for rippers that only ever download a single file and want to use the bytes progress bar --- .../ripper/AbstractSingleFileRipper.java | 43 +++++++++++++++++++ .../ripme/ripper/rippers/GfycatRipper.java | 31 +------------ .../ripper/rippers/GfycatporntubeRipper.java | 32 +------------- .../ripme/ripper/rippers/XvideosRipper.java | 32 ++------------ 4 files changed, 50 insertions(+), 88 deletions(-) create mode 100644 src/main/java/com/rarchives/ripme/ripper/AbstractSingleFileRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractSingleFileRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractSingleFileRipper.java new file mode 100644 index 00000000..f1f8be41 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractSingleFileRipper.java @@ -0,0 +1,43 @@ +package com.rarchives.ripme.ripper; + +import com.rarchives.ripme.utils.Utils; + +import java.io.IOException; +import java.net.URL; + + +/** + * This is just an extension of AbstractHTMLRipper that auto overrides a few things + * to help cut down on copy pasted code + */ +public abstract class AbstractSingleFileRipper extends AbstractHTMLRipper { + private int bytesTotal = 1; + private int bytesCompleted = 1; + + protected AbstractSingleFileRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getStatusText() { + return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal); + } + + @Override + public int getCompletionPercentage() { + return (int) (100 * (bytesCompleted / (float) bytesTotal)); + } + + @Override + public void setBytesTotal(int bytes) { + this.bytesTotal = bytes; + } + + @Override + public void setBytesCompleted(int bytes) { + this.bytesCompleted = bytes; + } + + @Override + public boolean useByteProgessBar() {return true;} +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java index a09d68ab..44ff857d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java @@ -9,18 +9,14 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Utils; +import com.rarchives.ripme.ripper.AbstractSingleFileRipper; import org.jsoup.nodes.Document; import org.jsoup.select.Elements; import com.rarchives.ripme.utils.Http; -public class GfycatRipper extends AbstractHTMLRipper { - - private int bytesTotal = 1; - private int bytesCompleted = 1; +public class GfycatRipper extends AbstractSingleFileRipper { private static final String HOST = "gfycat.com"; @@ -109,27 +105,4 @@ public class GfycatRipper extends AbstractHTMLRipper { } return vidUrl; } - - @Override - public String getStatusText() { - return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal); - } - - @Override - public int getCompletionPercentage() { - return (int) (100 * (bytesCompleted / (float) bytesTotal)); - } - - @Override - public void setBytesTotal(int bytes) { - this.bytesTotal = bytes; - } - - @Override - public void setBytesCompleted(int bytes) { - this.bytesCompleted = bytes; - } - - @Override - public boolean useByteProgessBar() {return true;} } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java index 55150d9e..fd8c292a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java @@ -8,17 +8,12 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.rarchives.ripme.utils.Utils; +import com.rarchives.ripme.ripper.AbstractSingleFileRipper; import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; -public class GfycatporntubeRipper extends AbstractHTMLRipper { - - private int bytesTotal = 1; - private int bytesCompleted = 1; +public class GfycatporntubeRipper extends AbstractSingleFileRipper { public GfycatporntubeRipper(URL url) throws IOException { super(url); @@ -62,27 +57,4 @@ public class GfycatporntubeRipper extends AbstractHTMLRipper { public void downloadURL(URL url, int index) { addURLToDownload(url, getPrefix(index)); } - - @Override - public String getStatusText() { - return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal); - } - - @Override - public int getCompletionPercentage() { - return (int) (100 * (bytesCompleted / (float) bytesTotal)); - } - - @Override - public void setBytesTotal(int bytes) { - this.bytesTotal = bytes; - } - - @Override - public void setBytesCompleted(int bytes) { - this.bytesCompleted = bytes; - } - - @Override - public boolean useByteProgessBar() {return true;} } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java index a94e7a21..ff38b347 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java @@ -8,21 +8,18 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Utils; + +import com.rarchives.ripme.ripper.AbstractSingleFileRipper; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import com.rarchives.ripme.utils.Http; -public class XvideosRipper extends AbstractHTMLRipper { +public class XvideosRipper extends AbstractSingleFileRipper { private static final String HOST = "xvideos"; - private int bytesTotal = 1; - private int bytesCompleted = 1; - public XvideosRipper(URL url) throws IOException { super(url); } @@ -86,27 +83,4 @@ public class XvideosRipper extends AbstractHTMLRipper { public void downloadURL(URL url, int index) { addURLToDownload(url, getPrefix(index)); } - - @Override - public String getStatusText() { - return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal); - } - - @Override - public int getCompletionPercentage() { - return (int) (100 * (bytesCompleted / (float) bytesTotal)); - } - - @Override - public void setBytesTotal(int bytes) { - this.bytesTotal = bytes; - } - - @Override - public void setBytesCompleted(int bytes) { - this.bytesCompleted = bytes; - } - - @Override - public boolean useByteProgessBar() {return true;} } \ No newline at end of file From 0a0b6d299a334c2027c0d13697f42b98024ee6ba Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 29 Jul 2018 10:37:15 -0400 Subject: [PATCH 06/10] Added kenzato.uk to CheveretoRippers --- .../ripme/ripper/rippers/CheveretoRipper.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java index cb0b765f..005ba5c7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java @@ -28,7 +28,7 @@ public class CheveretoRipper extends AbstractHTMLRipper { super(url); } - private static List explicit_domains_1 = Arrays.asList("tag-fox.com"); + private static List explicit_domains = Arrays.asList("tag-fox.com", "kenzato.uk"); @Override public String getHost() { @@ -43,12 +43,8 @@ public class CheveretoRipper extends AbstractHTMLRipper { @Override public boolean canRip(URL url) { String url_name = url.toExternalForm(); - if (explicit_domains_1.contains(url_name.split("/")[2])) { - Pattern pa = Pattern.compile("(?:https?://)?(?:www\\.)?[a-z1-9-]*\\.[a-z1-9]*/album/([a-zA-Z1-9]*)/?$"); - Matcher ma = pa.matcher(url.toExternalForm()); - if (ma.matches()) { + if (explicit_domains.contains(url_name.split("/")[2])) { return true; - } } return false; } @@ -70,7 +66,7 @@ public class CheveretoRipper extends AbstractHTMLRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("(?:https?://)?(?:www\\.)?[a-z1-9-]*\\.[a-z1-9]*/album/([a-zA-Z1-9]*)/?$"); + Pattern p = Pattern.compile("(?:https?://)?(?:www\\.)?[a-z1-9-]*\\.[a-z1-9]*(?:[a-zA-Z1-9]*)/album/([a-zA-Z1-9]*)/?$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); From 7196c594535181bc113c0425c9d7b68c87d591cb Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 29 Jul 2018 10:40:22 -0400 Subject: [PATCH 07/10] Added unit test for kenzato.uk --- .../ripme/tst/ripper/rippers/CheveretoRipperTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java index 4f456b8d..6fd20060 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java @@ -10,4 +10,9 @@ public class CheveretoRipperTest extends RippersTest { CheveretoRipper ripper = new CheveretoRipper(new URL("http://tag-fox.com/album/Thjb")); testRipper(ripper); } + + public void testSubdirAlbum() throws IOException { + CheveretoRipper ripper = new CheveretoRipper(new URL("https://kenzato.uk/booru/album/TnEc")); + testRipper(ripper); + } } From 771159de5689d7ee5e2ed1dababcfb4733da051c Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 29 Jul 2018 10:48:46 -0400 Subject: [PATCH 08/10] Removed outdated todo --- src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 01444642..b24017f7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -224,7 +224,6 @@ public abstract class AbstractHTMLRipper extends AlbumRipper { if (!subdirectory.equals("")) { // Not sure about this part subdirectory = File.separator + subdirectory; } - // TODO Get prefix working again, probably requires reworking a lot of stuff! (Might be fixed now) saveFileAs = new File( workingDir.getCanonicalPath() + subdirectory From 77a64a1dd79dbffe09506014fcd189cafc83b48d Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 31 Jul 2018 16:50:19 -0400 Subject: [PATCH 09/10] Timeout now applies to both connection and read --- .../java/com/rarchives/ripme/ripper/DownloadFileThread.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 55b8ffde..e704740e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -8,6 +8,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.lang.reflect.Array; import java.net.HttpURLConnection; +import java.net.SocketTimeoutException; import java.net.URL; import java.net.URLConnection; import java.util.Arrays; @@ -110,6 +111,7 @@ class DownloadFileThread extends Thread { } huc.setInstanceFollowRedirects(true); huc.setConnectTimeout(TIMEOUT); + huc.setReadTimeout(TIMEOUT); huc.setRequestProperty("accept", "*/*"); if (!referrer.equals("")) { huc.setRequestProperty("Referer", referrer); // Sic @@ -222,6 +224,9 @@ class DownloadFileThread extends Thread { bis.close(); fos.close(); break; // Download successful: break out of infinite loop + } catch (SocketTimeoutException timeoutEx) { + logger.error(url.toExternalForm() + " timedout!"); + break; } catch (HttpStatusException hse) { logger.debug("HTTP status exception", hse); logger.error("[!] HTTP status " + hse.getStatusCode() + " while downloading from " + urlToDownload); From d71addb8a72a9836d741a827eb2011b53529ad4e Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 31 Jul 2018 16:55:21 -0400 Subject: [PATCH 10/10] improved logging message and added comment for timeouts --- .../java/com/rarchives/ripme/ripper/DownloadFileThread.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index e704740e..f481919a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -110,6 +110,8 @@ class DownloadFileThread extends Thread { huc = (HttpURLConnection) urlToDownload.openConnection(); } huc.setInstanceFollowRedirects(true); + // It is important to set both ConnectTimeout and ReadTimeout. If you don't then ripme will wait forever + // for the server to send data after connecting. huc.setConnectTimeout(TIMEOUT); huc.setReadTimeout(TIMEOUT); huc.setRequestProperty("accept", "*/*"); @@ -225,7 +227,9 @@ class DownloadFileThread extends Thread { fos.close(); break; // Download successful: break out of infinite loop } catch (SocketTimeoutException timeoutEx) { - logger.error(url.toExternalForm() + " timedout!"); + // Handle the timeout + logger.error("[!] " + url.toExternalForm() + " timedout!"); + // Download failed, break out of loop break; } catch (HttpStatusException hse) { logger.debug("HTTP status exception", hse);