diff --git a/.gitignore b/.gitignore index e7813bc7..c36c4a16 100644 --- a/.gitignore +++ b/.gitignore @@ -120,3 +120,5 @@ history.json .classpath *.txt bin/ + +reddit_meta.db \ No newline at end of file diff --git a/html/css/simple-grid.css b/html/css/simple-grid.css new file mode 100644 index 00000000..bd0931a6 --- /dev/null +++ b/html/css/simple-grid.css @@ -0,0 +1,198 @@ +/** +*** SIMPLE GRID +*** (C) ZACH COLE 2016 +**/ + +@import url(https://fonts.googleapis.com/css?family=Lato:400,300,300italic,400italic,700,700italic); + +/* UNIVERSAL */ + +html, +body { + height: 100%; + width: 100%; + margin: 0; + padding: 0; + left: 0; + top: 0; + font-size: 100%; +} + +/* POSITIONING */ + +.left { + text-align: left; +} + +.right { + text-align: right; +} + +.center { + text-align: center; + margin-left: auto; + margin-right: auto; +} + +.justify { + text-align: justify; +} + +/* ==== GRID SYSTEM ==== */ + +.container { + width: 90%; + margin-left: auto; + margin-right: auto; +} + +.row { + position: relative; + width: 100%; +} + +.row [class^="col"] { + float: left; + margin: 0.5rem 2%; + min-height: 0.125rem; +} + +.col-1, +.col-2, +.col-3, +.col-4, +.col-5, +.col-6, +.col-7, +.col-8, +.col-9, +.col-10, +.col-11, +.col-12 { + width: 96%; +} + +.col-1-sm { + width: 4.33%; +} + +.col-2-sm { + width: 12.66%; +} + +.col-3-sm { + width: 21%; +} + +.col-4-sm { + width: 29.33%; +} + +.col-5-sm { + width: 37.66%; +} + +.col-6-sm { + width: 46%; +} + +.col-7-sm { + width: 54.33%; +} + +.col-8-sm { + width: 62.66%; +} + +.col-9-sm { + width: 71%; +} + +.col-10-sm { + width: 79.33%; +} + +.col-11-sm { + width: 87.66%; +} + +.col-12-sm { + width: 96%; +} + +.row::after { + content: ""; + display: table; + clear: both; +} + +.hidden-sm { + display: none; +} + +@media only screen and (min-width: 33.75em) { /* 540px */ + .container { + width: 80%; + } +} + +@media only screen and (min-width: 45em) { /* 720px */ + .col-1 { + width: 4.33%; + } + + .col-2 { + width: 12.66%; + } + + .col-3 { + width: 21%; + } + + .col-4 { + width: 29.33%; + } + + .col-5 { + width: 37.66%; + } + + .col-6 { + width: 46%; + } + + .col-7 { + width: 54.33%; + } + + .col-8 { + width: 62.66%; + } + + .col-9 { + width: 71%; + } + + .col-10 { + width: 79.33%; + } + + .col-11 { + width: 87.66%; + } + + .col-12 { + width: 96%; + } + + .hidden-sm { + display: block; + } +} + +@media only screen and (min-width: 60em) { /* 960px */ + .container { + width: 75%; + max-width: 60rem; + } +} diff --git a/html/css/style.css b/html/css/style.css new file mode 100644 index 00000000..a97bf5a3 --- /dev/null +++ b/html/css/style.css @@ -0,0 +1,1912 @@ +@import "https://fonts.googleapis.com/css?family=Raleway"; + +.category-container { + display: flex; + justify-content: center; + flex-wrap: wrap; + margin-bottom: 50px +} + +.category-row { + margin-left: 10px; + padding: 10px +} + +.category-small { + width: 40px +} + +.category-medium { + width: 150px +} + +.category-row > * { + padding: 10px; + font-size: 1.1em +} + +.category-row > .btn { + margin-right: 10px +} + +.category-row .delete-btn { + position: relative; + display: inline-block; + overflow: hidden; + opacity: .5 +} + +.category-row .delete-btn.active { + opacity: 1; + transition: 1s +} + +.category-row .static-value { + text-align: center; + color: red; + display: inline-block; + font-size: 1.5em; + width: 2em +} + +.btn { + display: inline-block; + font-weight: 400; + line-height: 1.25; + text-align: center; + white-space: nowrap; + vertical-align: middle; + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; + border: 1px solid transparent; + padding: .5rem 1rem; + font-size: 1rem; + border-radius: .25rem; + -webkit-transition: all .2s ease-in-out; + -o-transition: all .2s ease-in-out; + transition: all .2s ease-in-out +} + +.btn.active, .btn:active { + background-image: none; + outline: 0; + -webkit-box-shadow: inset 0 3px 5px rgba(0, 0, 0, .125); + box-shadow: inset 0 3px 5px rgba(0, 0, 0, .125) +} + +.btn-primary { + color: #fff; + background-color: #0275d8; + border-color: #0275d8 +} + +.btn-primary:hover { + color: #fff; + background-color: #025aa5; + border-color: #01549b +} + +.btn-info { + color: #fff; + background-color: #5bc0de; + border-color: #5bc0de +} + +.btn-info.active { + color: #fff; + background-color: #31b0d5; + border-color: #269abc +} + +.btn-info.active:hover { + color: #fff; + background-color: #269abc; + border-color: #1b6d85 +} + +.btn-info:hover { + color: #fff; + background-color: #31b0d5; + border-color: #2aabd2 +} + +.btn-success { + color: #fff; + background-color: #5cb85c; + border-color: #5cb85c +} + +.btn-success:hover { + color: #fff; + background-color: #449d44; + border-color: #419641 +} + +.btn-danger { + color: #fff; + background-color: #d9534f; + border-color: #d43f3a +} + +.btn-danger:hover { + color: #fff; + background-color: #c9302c; + border-color: #ac2925 +} + +.list-page-container { + margin: auto; + color: #fff; + width: 100%; + height: 100%; + font-size: 1.7em +} + +.tree-container { + position: relative; + width: 90%; + font-size: 1em; + min-height: 3em +} + +.list-container .category-tree { + display: flex; + flex-wrap: wrap; + padding-top: 1em; + padding-bottom: 3em; + padding-left: 1em +} + +.tree-search-container { + display: flex; + justify-content: center; + padding-top: 1em; + padding-bottom: 1em +} + +.tree-search-middle { + width: 90% +} + +.tree-search { + padding: .7em; + box-sizing: border-box; + width: 100%; + display: inline-block +} + +.category-tree { + position: relative +} + +.category-tree > .list-category { + padding-top: 1em +} + +.category-tree > .list-category:first-child { + margin-top: 0 !important +} + +.list-category { + position: relative; + margin-top: -2.5em; + padding-top: .5em; + padding-left: 1em; + background-color: #1b252f +} + +.link-builder-container { + position: fixed; + display: flex; + align-items: center; + bottom: 0; + width: 100%; + z-index: 2; + background-color: #ff0592; + background-color: #1b252f +} + +.link-builder-filler { + position: relative; + margin-top: -2.5em; + height: 200px; + background-color: #1b252f +} + +.list-spinner { + position: relative; + margin-top: -2.5em; + height: 200px; + background-color: #1b252f +} + +.link-builder-close { + position: absolute; + display: flex; + right: 0; + top: 0; + bottom: 0; + font-weight: 700; + font-size: 1.1em; + justify-content: center; + align-items: center; + width: 5%; + min-width: 1.5em +} + +.link-builder-close-icon { + cursor: pointer; + width: 2em; + height: 2em; + line-height: 2em; + text-align: center +} + +.link-builder { + margin: auto; + width: 90%; + max-width: calc(100% - 3em); + padding-top: .5em; + padding-bottom: .5em +} + +.link-builder-icon { + height: 3em +} + +.link-builder-link { + display: flex; + align-items: center; + box-sizing: border-box; + background-color: #fff; + font-size: .8em; + line-height: 3em; + width: 100%; + color: #14171e +} + +.link-builder-link span { + overflow: hidden; + padding-left: .8em; + text-overflow: ellipsis; + width: 100% +} + +.link-builder-link:not(:first-child) { + margin-top: .8em +} + +.list-element-container { + position: relative +} + +.list-element-container.expanded { + padding-bottom: 3em !important +} + +.list-element-container.show-all { + padding-bottom: 2.5em !important +} + +.list-element-container:not(:empty) { + padding-top: .5em; + padding-bottom: .5em; + padding-left: 1em; + height: calc(100% - 100px); + overflow-y: hidden; + display: flex; + flex-wrap: wrap +} + +.list-show-more { + position: absolute; + bottom: 2.5em; + display: flex; + justify-content: center; + align-items: center; + font-size: 1em; + height: 1.5em; + left: 0; + right: 0; + width: 100% +} + +.list-element-container.expanded .list-show-more { + bottom: 1.5em +} + +.list-show-more-background { + position: absolute; + width: 100%; + height: 100%; + z-index: 0; + border-radius: .3rem; + background-color: #1b252f; + left: 0; + right: 0; + bottom: 0; + top: 0; + opacity: .8 +} + +.list-show-more-icon { + color: red; + color: #ea9215; + color: #ff0592; + cursor: pointer; + z-index: 1; + height: 1em +} + +.list-element { + min-width: 12em +} + +.list-checkbox { + font-size: 2em +} + +.list-header:hover .list-checkbox { + visibility: visible +} + +.list-element h3 { + display: inline-block; + font-size: .8em; + margin: 0; + margin-bottom: .5em; + margin-top: .5em; + font-weight: 400 +} + +.list-category h2 { + display: inline-block; + font-size: 1.1em; + margin: 0; + padding-bottom: .2em +} + +.list-header .fa { + padding-left: .5em; + padding-right: .5em +} + +.list-header .list-external-link { + margin-left: .3em; + margin-right: .5em +} + +.tree-checkbox { + cursor: pointer; + display: inline-block; + position: relative; + border-radius: .3rem; + margin-right: .7em; + height: 1em; + width: 1em; + background-color: #fff +} + +.tree-checkbox.checked > div { + border-bottom: 4px solid green; + border-right: 4px solid green; + transform: rotate(45deg); + height: 100%; + width: 50%; + position: absolute; + left: 10px; + bottom: 4px +} + +@media (min-width: 1280px) and (max-width: 1920px) { + .list-page-container { + font-size: 1.4em + } +} + +@media (max-width: 1280px) { + .list-page-container { + font-size: 1.3em + } +} + +.list-buttons { + display: flex; + justify-content: center +} + +.list-button { + cursor: pointer; + width: 2em; + text-align: center +} + +.list-button.active { + color: #ff0592 +} + +body { + background-color: #1b252f; + font-size: 14px; + font-family: Raleway, sans-serif; + width: 100%; + margin: 0 +} + +.center-bar { + position: absolute; + bottom: 0; + top: 0; + margin: auto; + color: #fff; + width: 100%; + height: 100%; + font-size: 5em +} + +.block { + padding: 6px; + display: inline-block +} + +.btn.large { + cursor: pointer; + padding: 1.3125rem 2.625rem; + font-size: 2.1875rem; + border-radius: .525rem +} + +.center-center { + height: 100%; + display: flex; + justify-content: center; + align-items: center +} + +.btn-info { + color: #fff; + background-color: #5bc0de; + border-color: #5bc0de +} + +.btn-info:hover { + color: #fff; + background-color: #31b0d5; + border-color: #2aabd2 +} + +.btn-success { + color: #fff; + background-color: #5cb85c; + border-color: #5cb85c +} + +.btn-success:hover { + color: #fff; + background-color: #449d44; + border-color: #419641 +} + +.btn-danger { + color: #fff; + background-color: #d9534f; + border-color: #d43f3a +} + +.btn-danger:hover { + color: #fff; + background-color: #c9302c; + border-color: #ac2925 +} + +@media (min-width: 1920px) { + .main-content { + padding: 0 5%; + max-width: 2000px + } + + .page-info-header { + padding: 0 5%; + max-width: 2000px + } + + .slideshow .info-bar { + font-size: 2em + } + + .slideshow .info-bar .right { + width: 100px !important + } + + .header { + font-size: 4em + } + + .slideshow .info-bar .middle { + width: calc(100% - 200px) + } + + .slideshow .info-bar .left { + width: 125px !important + } +} + +@media (min-width: 1280px) and (max-width: 1920px) { + .top img { + min-width: 2000px + } + + .top { + height: 300px !important + } + + .header { + font-size: 3.5em + } + + .main-content { + padding: 0 5% + } + + .page-info-header { + padding: 0 5% + } + + .grid .info-bar { + font-size: 1.5em !important + } + + .slideshow .info-bar { + font-size: 1.5em; + height: 80px !important + } + + div.arrow-bar.down { + height: 20px !important + } + + .info-bar .left { + font-size: 1.25em !important + } + + .external-link { + font-size: 1.1em !important + } + + .header h1 { + font-size: .9em !important + } +} + +@media (min-width: 800px) and (max-width: 1280px) { + .header h1 { + font-size: .8em !important + } + + .main-content { + padding: 0 5% + } + + .page-info-header { + padding: 0 5% + } + + .top img { + min-width: 1280px + } + + .top { + height: 200px !important + } + + .header { + font-size: 3em + } + + .grid .info-bar { + font-size: 1.4em !important + } + + .slideshow .info-bar { + font-size: 1.5em; + height: 70px !important + } + + div.arrow-bar.down { + height: 10px !important + } + + .info-bar .left { + font-size: 1.25em !important + } + + .external-link { + font-size: 1.1em !important + } +} + +@media (max-width: 799px) { + .top { + display: none + } + + .page-info-content { + padding: 4.5em 1em 1em 1em !important; + font-size: .5em !important + } + + .header { + visibility: hidden + } + + .error-bar { + font-size: 1.5em !important; + height: 100vh !important + } + + .grid .info-bar { + font-size: 1.3em !important + } + + .info-bar .left { + font-size: 1.25em !important + } + + .external-link { + font-size: 1.1em !important + } + + .center-bar { + font-size: 2em !important + } + + .center-bar .btn.large { + padding: .75rem 1.5rem; + font-size: 1.15rem; + border-radius: .3rem + } + + .center-bar .block { + padding: 4px + } + + .slideshow .info-bar { + font-size: 1.3em !important; + height: 55px !important + } + + video { + pointer-events: none + } + + div.arrow-bar { + visibility: hidden + } + + .media-element.grid:hover .media-overlay.gradient { + visibility: hidden !important + } + + .media-element.grid .border { + border-left: 0 !important; + border-right: 0 !important + } + + .stack { + width: 18px; + height: 13px + } + + .external-link a { + padding-left: 6px + } +} + +body { + -webkit-tap-highlight-color: transparent; + background-color: #1b252f; + font-size: 14px; + font-family: Raleway; + width: 100%; + margin: 0 +} + +body.slideshow { + height: 1px; + overflow-y: hidden +} + +.top { + height: 500px; + position: relative; + width: 100%; + overflow: hidden +} + +.hidden { + display: none !important +} + +.invisible { + visibility: hidden !important +} + +.top .image-wrapper { + position: absolute; + z-index: -1; + width: 100% +} + +.top img { + width: 100% +} + +.hide-top { + height: 100%; + width: 100%; + position: absolute; + background-color: #1b252f; + z-index: -1 +} + +.top-overlay { + height: 100%; + background: linear-gradient(to top, #1b252f 0, #1b252f 5%, rgba(24, 27, 30, 0) 100%); + text-align: center; + width: 100%; + display: table +} + +.header { + position: absolute; + color: #fff; + width: 100%; + height: 100% +} + +.header h1 { + display: flex; + flex-wrap: wrap; + margin: 0; + padding: 10px; + box-sizing: border-box; + line-height: 1.3em; + height: 100%; + width: 100%; + justify-content: center; + align-items: center; + font-size: 1em; + font-weight: 400 +} + +.header h1 a { + padding-left: .5em +} + +.header-single-item { + display: inline-block; + cursor: pointer; + margin-left: .3em; + margin-right: .3em +} + +div.media-element.grid .arrows { + display: none +} + +div.close-icon { + position: absolute; + font-size: 1.3em; + right: 0; + top: 10px; + width: 50px; + font-weight: 700; + text-align: center +} + +div.arrows { + z-index: 1; + position: absolute; + transition: opacity 1s; + webkit-transition: opacity 1s; + color: #fff; + left: 0; + height: 100%; + width: 100%; + font-size: 1.3em +} + +div.arrow-bar { + pointer-events: all; + z-index: 1; + position: absolute +} + +div.arrow-bar.right { + right: 0; + width: 50px; + height: 100% +} + +div.arrow-bar.left { + left: 0; + width: 50px; + height: 100% +} + +div.arrow-bar.up { + top: 0; + width: 100%; + height: 50px +} + +div.arrow-bar.down { + bottom: 0; + width: 100%; + height: 30px +} + +div.arrow { + position: absolute; + height: 30px; + width: 30px; + left: 0; + right: 0; + top: 0; + bottom: 0; + margin: auto; + text-align: center +} + +.arrow-bar:hover i { + transform: scale(1.2) +} + +.screen-filler { + position: absolute; + z-index: -1; + width: 100%; + height: 105vh +} + +.media-element.slideshow .padded { + height: 200%; + width: 100%; + top: 0; + left: 0; + z-index: 2; + background-color: #000; + position: fixed +} + +.media-element img, .media-element video { + -webkit-user-select: none; + -moz-user-select: none; + -khtml-user-select: none; + -ms-user-select: none; + user-select: none; + -webkit-tap-highlight-color: transparent; + -webkit-tap-highlight-color: transparent; + display: block; + width: 100% +} + +.media-element.slideshow .border { + height: 100vh; + width: 100vw; + position: relative +} + +.media-element.slideshow .overflow { + display: flex; + justify-content: center; + align-items: center; + height: 100%; + width: 100% +} + +.fill-size { + height: 100%; + width: 100% +} + +.media-element.slideshow img, .media-element.slideshow video { + object-fit: contain; + z-index: -1; + width: 100%; + height: 100%; + max-height: 100% +} + +.main-content { + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; + z-index: 1; + margin: auto; + width: 100% +} + +#media-container { + width: 100% +} + +.wrapper { + position: relative +} + +.media-element { + position: absolute; + vertical-align: top +} + +.media-element.grid .padded { + margin: auto +} + +.media-element.grid .border { + border: 1px solid #fff +} + +.media-element.grid .overflow { + display: flex; + align-items: center; + position: relative; + width: 100%; + overflow: hidden +} + +.media-element.loaded .overflow { + background-color: #000 +} + +.fill-width { + width: 100% +} + +.media-element .link { + margin-left: 60px +} + +.media-overlay { + visibility: hidden; + position: absolute; + pointer-events: none; + bottom: 0; + height: 100%; + width: 100% +} + +.media-element.grid:hover .media-overlay.gradient { + visibility: visible +} + +.media-element.grid .media-overlay.visible { + background: linear-gradient(to top, rgba(0, 0, 0, .13) 50%, rgba(0, 0, 0, 0) 100%); + height: 50px; + visibility: visible !important +} + +.media-element.grid .media-overlay.gradient { + background: linear-gradient(to top, rgba(27, 37, 47, .3) 5%, rgba(27, 37, 47, .1) 80%) +} + +.media-element.slideshow .media-overlay { + visibility: visible +} + +.disliked-image { + position: absolute; + height: 100%; + background-color: #1b252f; + width: 100%; + z-index: 0; + top: 0; + bottom: 0; + color: #fff; + display: flex; + height: 100%; + align-items: center; + justify-content: center; + font-size: 6em +} + +.disliked-image .i { + height: 100px; + width: 100px +} + +.play-video-icon { + position: absolute; + height: 100%; + top: 0; + bottom: 0; + font-size: 5em; + color: #fff; + width: 100% +} + +.play-video-icon i.fa-play { + padding: 10px +} + +.media-overlay .info .text a { + pointer-events: all +} + +.media-element:hover { + cursor: pointer +} + +.media-element.hide-controls .arrows { + opacity: 0 +} + +.spinner-container.bottom { + height: 400px; + position: relative +} + +.spinner-container.floating { + background-color: #1b252f; + height: 100vh; + width: 100%; + top: 0; + bottom: 0; + left: 0; + right: 0; + margin: auto; + position: fixed; + pointer-events: none; + opacity: 0; + transition: opacity .5s ease; + z-index: 2 +} + +.spinner-container.inline { + z-index: 2; + position: absolute; + top: 0; + left: 0; + right: 0; + pointer-events: none; + background-color: #1b252f; + height: 100%; + opacity: 0 +} + +.spinner-container.inline .spinner-top { + position: absolute; + top: calc(50vh - 6em); + left: 0; + right: 0 +} + +.spinner-container.slideshow { + background-color: #000 +} + +.spinner-container.on-top { + opacity: 1 +} + +.home-icon { + color: #fff; + font-size: 2.5em; + position: fixed; + left: 15px; + top: 15px; + z-index: 1; + cursor: pointer; + pointer-events: all +} + +.home-icon a { + padding: 5px +} + +.settings-position { + color: #fff; + font-size: 2.5em; + position: fixed; + right: 15px; + top: 15px; + z-index: 1; + pointer-events: all +} + +.settings-icon { + padding: 5px +} + +.info { + height: 100% +} + +.info-bar { + position: absolute; + color: #fff; + width: 100%; + height: 55px; + font-size: 1.7em +} + +.info-bar.info-bottom { + bottom: 0 +} + +.info-bar.info-top { + top: 0 +} + +.info-bar.immersive-info-bar { + display: none +} + +.error-bar { + color: #fff; + height: 300px; + font-size: 2em; + text-align: center +} + +.error-bar .subheader { + padding-top: 20px +} + +.error-bar .btn { + font-size: 1.1em +} + +.center-bar { + position: absolute; + bottom: 0; + top: 0; + margin: auto; + color: #fff; + width: 100%; + height: 100%; + font-size: 5em +} + +.block { + padding: 6px; + display: inline-block +} + +.btn.large { + cursor: pointer; + padding: 1.3125rem 2.625rem; + font-size: 2.1875rem; + border-radius: .525rem +} + +.btn-lg { + padding: .75rem 1.5rem; + font-size: 1.25rem; + border-radius: .3rem +} + +.slideshow .info-bar.info-bottom { + background: linear-gradient(to top, rgba(0, 0, 0, .13) 50%, rgba(0, 0, 0, 0) 100%); + height: 100px +} + +.slideshow .info-bar.info-top { + background: linear-gradient(to bottom, rgba(0, 0, 0, .2) 0, rgba(0, 0, 0, .2) 50%, rgba(0, 0, 0, 0) 100%); + height: 100px +} + +.info-bar .left { + position: absolute; + left: 0; + bottom: 0; + height: 100%; + pointer-events: all +} + +.grid .info-bar .left { + width: 75px +} + +.slideshow .info-bar .left { + width: 75px +} + +.info-bar .right { + position: absolute; + right: 0; + bottom: 0; + width: 60px; + height: 100%; + pointer-events: all +} + +.info-bar .middle { + position: absolute; + left: 0; + right: 0; + margin: auto; + bottom: 0; + height: 100%; + width: calc(100% - 150px); + pointer-events: all +} + +.center-center2 { + display: flex; + justify-content: center; + align-items: center +} + +.center-center { + height: 100%; + display: flex; + justify-content: center; + align-items: center +} + +.evenly-distribute { + height: 100%; + display: flex; + align-items: center +} + +.admin-search { + display: flex; + justify-content: center; + padding-top: 50px +} + +.admin-search .feed-element { + width: 90% +} + +.center-container { + margin: 100px +} + +.slideshow .heart { + z-index: 3 +} + +.grid .heart { + z-index: 0 !important +} + +.heart { + padding: 5px +} + +.heart.active { + color: red +} + +.media-element.slideshow .heart i { + z-index: 3 +} + +.media-element.slideshow .stack { + z-index: 3 +} + +.stack { + position: relative; + width: 17px; + height: 13px; + pointer-events: all +} + +.stack figure { + margin: auto +} + +.stack div { + border: 1px solid #fff; + height: 100%; + width: 100%; + position: absolute +} + +.stack .one { + transform: translate(20%, -20%) +} + +.stack .three { + transform: translate(-20%, 20%) +} + +.animated { + -webkit-animation-duration: 10s; + animation-duration: 10s; + -webkit-animation-fill-mode: both; + animation-fill-mode: both +} + +@-webkit-keyframes fadeIn { + 0% { + opacity: 0 + } + 100% { + opacity: 1 + } +} + +@keyframes fadeIn { + 0% { + opacity: 0 + } + 100% { + opacity: 1 + } +} + +.fadeIn { + -webkit-animation-name: fadeIn; + animation-name: fadeIn +} + +.center-container { + max-width: 2000px; + width: 100% +} + +.admin-container .half { + display: inline-block; + width: 50% +} + +.admin-container .third { + display: inline-block; + width: 33% +} + +.admin-container .fourth { + display: inline-block; + width: 25% +} + +.admin-container .fifth { + display: inline-block; + width: 20% +} + +.inline { + display: inline-block +} + +.segment-dropdown { + display: flex +} + +.flex-rows { + display: flex; + flex-wrap: wrap; + align-items: center; + justify-content: flex-start +} + +.input-container { + box-sizing: border-box; + display: inline-block +} + +.feed-element { + width: 40%; + padding: 10px; + margin-left: 20px; + display: inline-block +} + +.input-with-padding { + padding: 10px; + margin-left: 20px +} + +.input-with-padding.highlight { + background-color: #8b8bf3 +} + +a { + color: #fff; + text-decoration: none +} + +.admin-container { + margin: auto; + color: #fff; + width: 100%; + height: 100%; + font-size: 1.6em +} + +.external-link { + position: absolute; + top: 3px; + bottom: 0; + margin: auto; + font-size: .8em +} + +.external-link a { + padding-left: 10px +} + +.settings-menu { + margin-top: 6px; + padding: 6px; + background-color: #1b252f; + box-shadow: 5px 5px 5px rgba(0, 0, 0, .3), -5px 5px 5px rgba(0, 0, 0, .3); + width: 240px; + position: absolute; + right: 15px; + font-size: 1.2rem +} + +.settings-checkbox { + display: inline-block; + float: right +} + +.settings-checkbox .tree-checkbox { + background-color: inherit +} + +.settings-checkbox .checked > div { + border-bottom: 4px solid red !important; + border-right: 4px solid red !important +} + +.settings-position i { + cursor: pointer +} + +.settings-menu .btn { + width: 100%; + margin-bottom: 10px; + text-align: left +} + +.settings-button { + cursor: pointer +} + +.settings-half { + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; + text-align: center; + display: inline-block; + width: 25% !important; + font-size: 1.1em; + padding-top: .3em; + padding-bottom: .4em; + transition: background-color .2s +} + +.settings-half:hover { + background-color: #881616 +} + +.settings-half.active { + background-color: red +} + +.settings-half.left { + margin-left: 25%; + border-radius: .3rem 0 0 .3rem +} + +.settings-half.right { + border-radius: 0 .3rem .3rem 0 +} + +.settings-item { + padding-top: .3em; + padding-left: .6em; + padding-bottom: 10px +} + +.reset-item { + padding-right: .6em +} + +.warning { + color: red +} + +.scroll-slider-container { + position: fixed; + width: calc(100vw - 150px); + max-width: 90vw; + height: 60px; + top: 40px; + left: 0; + right: 0; + z-index: 3; + opacity: 1; + margin: auto; + color: #fff; + transition: opacity 1s, visibility 1s +} + +.scroll-slider-hidden { + opacity: 0 +} + +.scroll-slider-background { + position: absolute; + z-index: -1; + height: 100%; + width: 100%; + background-color: #000; + background-color: #1b252f; + background-color: #1980e6; + background-color: #425a71; + background-color: #0d365d; + opacity: 1; + border-radius: .3rem +} + +.scroll-slider-icons { + vertical-align: top; + display: inline-block; + height: 100%; + width: 125px +} + +.scroll-slider-slider { + vertical-align: top; + display: inline-block; + width: calc(100% - 125px); + height: 100% +} + +.scroll-slider-element { + cursor: pointer; + background-color: #fff; + height: 50%; + width: 100%; + margin-left: 25px; + margin-right: 15px; + overflow: hidden; + border-radius: .3rem +} + +.scroll-slider-completed { + height: 100%; + background-color: #00f; + background-color: #1b252f; + background-color: red; + background-color: #ce1919 +} + +.scroll-slider-icons i { + cursor: pointer; + margin: 5px; + padding: 10px; + font-size: 1.8em +} + +.scroll-slider-icons i.close { + font-size: 2.3em +} + +.inline-block { + display: inline-block +} + +.block { + padding: 6px; + display: inline-block +} + +.center-center { + height: 100%; + display: flex; + justify-content: center; + align-items: center +} + +.btn { + cursor: pointer +} + +.btn-color-1 { + background-color: #9350d0 +} + +.btn-color-2 { + background-color: #f75f00 +} + +.btn-color-3 { + background-color: #3469ff +} + +.btn-color-4 { + background-color: #c70404 +} + +.btn-color-5 { + background-color: #74d136 +} + +.btn-color-6 { + background-color: #b33265 +} + +.btn-color-pink { + background-color: #ff0592 +} + +.btn-color-pink:hover { + background-color: #ef0087 +} + +.btn.not-active { + opacity: .5 +} + +.font-color-pink { + color: #ff0592 +} + +@keyframes spin-animation { + 0% { + transform: rotate(0) + } + 100% { + transform: rotate(359deg) + } +} + +.loading-spinner-container { + color: #fff; + font-size: 3em; + width: 3em; + left: 100px; + position: absolute +} + +.loading-spinner-circle { + animation: spin-animation 2s infinite linear; + border-radius: 50%; + border-left: .1em solid #fff; + border-right: .1em solid rgba(255, 255, 255, .5); + border-top: .1em solid rgba(255, 255, 255, .5); + border-bottom: .1em solid rgba(255, 255, 255, .5); + width: .7em; + height: .7em +} + +.page-info-header { + overflow: hidden; + transition: height 1s ease; + font-size: 2em; + color: #fff; + width: 100%; + margin: auto; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box +} + +.page-info-content { + padding-bottom: 2em; + padding-left: 2em; + padding-right: 2em; + font-size: .8em +} + +.page-info-content a { + color: red +} + +.page-info-content h2 { + font-size: 1em; + font-weight: 400; + display: inline +} + +.page-info-line { + margin-top: .2em +} + +.page-info-content p { + margin: 0 +} + +.page-info-header.visible { + height: 400px +} + +.grid { + display: grid; + grid-gap: 10px; + grid-auto-rows: 10px; + margin-left: 2em; + margin-right: 2em; +} + +@media only screen and (min-width: 1800px) { + .grid { + grid-template-columns: repeat(auto-fill, minmax(550px, 1fr)); + } +} + +@media only screen and (min-width: 1256px) and (max-width: 1799px){ + .grid { + grid-template-columns: repeat(auto-fill, minmax(400px, 1fr)); + } +} + +@media only screen and (min-width: 1100px) and (max-width: 1255px){ + .grid { + grid-template-columns: repeat(auto-fill, minmax(300px, 1fr)); + } +} + +@media only screen and (min-width: 800px) and (max-width: 1099px){ + .grid { + grid-template-columns: repeat(auto-fill, minmax(250px, 1fr)); + } +} + +@media only screen and (min-width: 0px) and (max-width: 799px){ + .grid { + grid-template-columns: repeat(auto-fill, minmax(200px, 1fr)); + } +} + +.item { + background-color: #333333; + height: 100%; +} + +.item:hover { + cursor: pointer; +} + +.item img { + width: 100%; + object-fit: cover; + height: 100%; +} + +.item video { + -webkit-user-select: none; + -moz-user-select: none; + -khtml-user-select: none; + -ms-user-select: none; + user-select: none; + -webkit-tap-highlight-color: transparent; + -webkit-tap-highlight-color: transparent; + display: block; + width: 100%; + height: 100%; + object-fit: unset; +} + +.h100 { + height: 100%; +} + +.footer { + color: white; + font-family: Raleway; + text-align: center; + font-weight: bold; + font-size: 1.1em; +} + +.footer-container { + margin-top: 5em; +} + +.load-bar { + position: relative; + margin-top: 20px; + width: 100%; + height: 6px; + background-color: #f2f2f2; +} + +.bar { + content: ""; + display: inline; + position: absolute; + width: 0; + height: 100%; + left: 50%; + text-align: center; +} + +.bar:nth-child(1) { + background-color: #1b252f; + animation: loading 3s linear infinite; +} + +.bar:nth-child(2) { + background-color: #f2f2f2; + animation: loading 3s linear 2s infinite; +} + +@keyframes loading { + from { + left: 50%; + width: 0; + z-index: 100; + } + 50% { + left: 0; + width: 100%; + z-index: 10; + } + to { + left: 0; + width: 100%; + } +} + +.clearfix::after { + content: ""; + clear: both; + display: table; +} + +.reddit-selected { + color: limegreen !important; + font-weight: bold; +} + +#checkme { + background: #1b252f; + width: 100%; + height: 2em; +} + +.overlay { + color: white; + position: absolute; + bottom: 8px; + left: 50%; + transform: translate(-50%, -50%); + font-size: 1.3em!important; + text-shadow: 5px 5px 11px rgba(150, 150, 150, 1); +} + +.content { + position: relative; +} + +.hoverme { + filter: brightness(85%); + -webkit-transition: -webkit-filter 100ms linear; +} + +.featherlight .featherlight-content { + padding: 5px 5px 0 !important; + border-bottom: 5px solid transparent !important; + overflow: hidden !important; +} + +.featherlight:last-of-type { + background: rgba(0,0,0,0.98) !important; +} + +@media only screen and (min-width: 800px) { + .featherlight .featherlight-content{ + height: 100%; + } + + .featherlight .featherlight-content img{ + height: 100% !important; + } + + .featherlight .featherlight-content video{ + height: 100% !important; + } +} diff --git a/html/images/icons/android-chrome-192x192.png b/html/images/icons/android-chrome-192x192.png new file mode 100644 index 00000000..d805cc47 Binary files /dev/null and b/html/images/icons/android-chrome-192x192.png differ diff --git a/html/images/icons/android-chrome-512x512.png b/html/images/icons/android-chrome-512x512.png new file mode 100644 index 00000000..38c35297 Binary files /dev/null and b/html/images/icons/android-chrome-512x512.png differ diff --git a/html/images/icons/apple-touch-icon.png b/html/images/icons/apple-touch-icon.png new file mode 100644 index 00000000..9857eaad Binary files /dev/null and b/html/images/icons/apple-touch-icon.png differ diff --git a/html/images/icons/browserconfig.xml b/html/images/icons/browserconfig.xml new file mode 100644 index 00000000..f9c2e67f --- /dev/null +++ b/html/images/icons/browserconfig.xml @@ -0,0 +1,9 @@ + + + + + + #2b5797 + + + diff --git a/html/images/icons/favicon-16x16.png b/html/images/icons/favicon-16x16.png new file mode 100644 index 00000000..a5019a3a Binary files /dev/null and b/html/images/icons/favicon-16x16.png differ diff --git a/html/images/icons/favicon-32x32.png b/html/images/icons/favicon-32x32.png new file mode 100644 index 00000000..cb71786d Binary files /dev/null and b/html/images/icons/favicon-32x32.png differ diff --git a/html/images/icons/favicon.ico b/html/images/icons/favicon.ico new file mode 100644 index 00000000..21c20c0a Binary files /dev/null and b/html/images/icons/favicon.ico differ diff --git a/html/images/icons/mstile-150x150.png b/html/images/icons/mstile-150x150.png new file mode 100644 index 00000000..f815dd97 Binary files /dev/null and b/html/images/icons/mstile-150x150.png differ diff --git a/html/images/icons/safari-pinned-tab.svg b/html/images/icons/safari-pinned-tab.svg new file mode 100644 index 00000000..25cab55e --- /dev/null +++ b/html/images/icons/safari-pinned-tab.svg @@ -0,0 +1,32 @@ + + + + +Created by potrace 1.11, written by Peter Selinger 2001-2013 + + + + + + diff --git a/html/images/icons/site.webmanifest b/html/images/icons/site.webmanifest new file mode 100644 index 00000000..b20abb7c --- /dev/null +++ b/html/images/icons/site.webmanifest @@ -0,0 +1,19 @@ +{ + "name": "", + "short_name": "", + "icons": [ + { + "src": "/android-chrome-192x192.png", + "sizes": "192x192", + "type": "image/png" + }, + { + "src": "/android-chrome-512x512.png", + "sizes": "512x512", + "type": "image/png" + } + ], + "theme_color": "#ffffff", + "background_color": "#ffffff", + "display": "standalone" +} diff --git a/html/images/top-big.jpg b/html/images/top-big.jpg new file mode 100644 index 00000000..eaef6d4c Binary files /dev/null and b/html/images/top-big.jpg differ diff --git a/html/index.html b/html/index.html new file mode 100644 index 00000000..5366799e --- /dev/null +++ b/html/index.html @@ -0,0 +1,289 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ImgScroll + + +
+ +
+
+
+
+ +
+
+

ImgScroll

+
+
+
+
+
+
+
+ files: + +
+
+ subreddits: + +
+
+
+
+
+
+
+ +
+
+
+
+ + +
+
+
+ + + + \ No newline at end of file diff --git a/html/manifest.json b/html/manifest.json new file mode 100644 index 00000000..5e65b41c --- /dev/null +++ b/html/manifest.json @@ -0,0 +1,20 @@ +{ + "name": "ImgScroll", + "short_name": "ImgScroll", + "icons": [ + { + "src": "/images/icons/android-chrome-192x192.png", + "sizes": "192x192", + "type": "image/png" + }, + { + "src": "/images/icons/android-chrome-512x512.png", + "sizes": "512x512", + "type": "image/png" + } + ], + "start_url": "/", + "display": "standalone", + "background_color": "#2b5797", + "theme_color": "#2b5797" +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index ab237191..2aa1d0c9 100644 --- a/pom.xml +++ b/pom.xml @@ -58,6 +58,16 @@ httpmime 4.3.3 + + org.xerial + sqlite-jdbc + 3.25.2 + + + org.eclipse.jetty + jetty-server + 9.4.3.v20170317 + diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index 20d31d78..cf2abfac 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -1,35 +1,37 @@ package com.rarchives.ripme.ripper.rippers; +import com.rarchives.ripme.ripper.AlbumRipper; +import com.rarchives.ripme.ui.RipStatusMessage; +import com.rarchives.ripme.ui.UpdateUtils; +import com.rarchives.ripme.utils.Http; +import com.rarchives.ripme.utils.RipUtils; +import com.rarchives.ripme.utils.Utils; +import org.json.JSONArray; +import org.json.JSONObject; +import org.json.JSONTokener; + import java.io.File; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashSet; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.rarchives.ripme.ui.RipStatusMessage; -import org.json.JSONArray; -import org.json.JSONObject; -import org.json.JSONTokener; - -import com.rarchives.ripme.ripper.AlbumRipper; -import com.rarchives.ripme.ui.UpdateUtils; -import com.rarchives.ripme.utils.Http; -import com.rarchives.ripme.utils.RipUtils; -import com.rarchives.ripme.utils.Utils; -import org.jsoup.Jsoup; - -import javax.swing.text.Document; -import javax.swing.text.Element; - public class RedditRipper extends AlbumRipper { public RedditRipper(URL url) throws IOException { super(url); } - private static final String HOST = "reddit"; + private static final String HOST = "reddit"; private static final String DOMAIN = "reddit.com"; private static final String REDDIT_USER_AGENT = "RipMe:github.com/RipMeApp/ripme:" + UpdateUtils.getThisJarVersion() + " (by /u/metaprime and /u/ineedmorealts)"; @@ -40,6 +42,13 @@ public class RedditRipper extends AlbumRipper { private long lastRequestTime = 0; + private static Connection connection = null; + private Statement statement; + private PreparedStatement preparedStatement; + private HashSet knownIds = new HashSet<>(); + + private int downloadCount = 0; + private Boolean shouldAddURL() { return (alreadyDownloadedUrls >= Utils.getConfigInteger("history.end_rip_after_already_seen", 1000000000) && !isThisATest()); } @@ -69,21 +78,42 @@ public class RedditRipper extends AlbumRipper { @Override public void rip() throws IOException { URL jsonURL = getJsonURL(this.url); - while (true) { - if (shouldAddURL()) { - sendUpdate(RipStatusMessage.STATUS.DOWNLOAD_COMPLETE_HISTORY, "Already seen the last " + alreadyDownloadedUrls + " images ending rip"); - break; + + try { + // create a database connection + connection = DriverManager.getConnection("jdbc:sqlite:reddit_meta.db"); + statement = connection.createStatement(); + statement.setQueryTimeout(30); // set timeout to 30 sec. + + ResultSet rs = statement.executeQuery("select id from metalist"); + while (rs.next()) { + knownIds.add(rs.getString("id")); } - jsonURL = getAndParseAndReturnNext(jsonURL); - if (jsonURL == null || isThisATest() || isStopped()) { - break; + + while (true) { + if (shouldAddURL()) { + sendUpdate(RipStatusMessage.STATUS.DOWNLOAD_COMPLETE_HISTORY, "Already seen the last " + alreadyDownloadedUrls + " images ending rip"); + break; + } + jsonURL = getAndParseAndReturnNext(jsonURL); + if (jsonURL == null || isThisATest() || isStopped() || downloadCount >= Utils.getConfigInteger("download.max_per_session", 50)) { + break; + } + } + waitForThreads(); + } catch (SQLException e) { + LOGGER.warn("[!] Reddit Meta Database error", e); + } finally { + try { + if (connection != null) + connection.close(); + } catch (SQLException e) { + LOGGER.warn("[!] Reddit Meta Database error", e); } } - waitForThreads(); } - private URL getAndParseAndReturnNext(URL url) throws IOException { JSONArray jsonArray = getJsonArrayFromURL(url), children; JSONObject json, data; @@ -105,8 +135,7 @@ public class RedditRipper extends AlbumRipper { String nextURLString = Utils.stripURLParameter(url.toExternalForm(), "after"); if (nextURLString.contains("?")) { nextURLString = nextURLString.concat("&after=" + data.getString("after")); - } - else { + } else { nextURLString = nextURLString.concat("?after=" + data.getString("after")); } nextURL = new URL(nextURLString); @@ -136,10 +165,10 @@ public class RedditRipper extends AlbumRipper { lastRequestTime = System.currentTimeMillis(); String jsonString = Http.url(url) - .ignoreContentType() - .userAgent(REDDIT_USER_AGENT) - .response() - .body(); + .ignoreContentType() + .userAgent(REDDIT_USER_AGENT) + .response() + .body(); Object jsonObj = new JSONTokener(jsonString).nextValue(); JSONArray jsonArray = new JSONArray(); @@ -158,21 +187,20 @@ public class RedditRipper extends AlbumRipper { JSONObject data = child.getJSONObject("data"); if (kind.equals("t1")) { // Comment - handleBody(data.getString("body"), data.getString("id"), ""); - } - else if (kind.equals("t3")) { + handleBody(data.getString("body"), data.getString("id"), "", data.getInt("created"), data.getInt("ups"), data.getString("subreddit")); + } else if (kind.equals("t3")) { // post if (data.getBoolean("is_self")) { // TODO Parse self text - handleBody(data.getString("selftext"), data.getString("id"), data.getString("title")); + handleBody(data.getString("selftext"), data.getString("id"), data.getString("title"), data.getInt("created"), data.getInt("ups"), data.getString("subreddit")); } else { // Get link - handleURL(data.getString("url"), data.getString("id"), data.getString("title")); + handleURL(data.getString("url"), data.getString("id"), data.getString("title"), data.getInt("created"), data.getInt("ups"), data.getString("subreddit")); } if (data.has("replies") && data.get("replies") instanceof JSONObject) { JSONArray replies = data.getJSONObject("replies") - .getJSONObject("data") - .getJSONArray("children"); + .getJSONObject("data") + .getJSONArray("children"); for (int i = 0; i < replies.length(); i++) { parseJsonChild(replies.getJSONObject(i)); } @@ -180,7 +208,7 @@ public class RedditRipper extends AlbumRipper { } } - private void handleBody(String body, String id, String title) { + private void handleBody(String body, String id, String title, int created, int ups, String sub) { Pattern p = RipUtils.getURLRegex(); Matcher m = p.matcher(body); while (m.find()) { @@ -188,7 +216,7 @@ public class RedditRipper extends AlbumRipper { while (url.endsWith(")")) { url = url.substring(0, url.length() - 1); } - handleURL(url, id, title); + handleURL(url, id, title, created, ups, sub); } } @@ -218,13 +246,28 @@ public class RedditRipper extends AlbumRipper { } - private void handleURL(String theUrl, String id, String title) { + private void handleURL(String theUrl, String id, String title, int created, int ups, String sub) { URL originalURL; try { originalURL = new URL(theUrl); } catch (MalformedURLException e) { return; } + + if (ups < Utils.getConfigInteger("reddit.min_ups", 100)) { + LOGGER.info("[i] Skipping " + id + " not reached minimum upvotes: " + ups + "/" + Utils.getConfigInteger("reddit.min_ups", 100)); + return; + } + + downloadCount++; + + if (knownIds.contains(id)) { + LOGGER.info("[i] Skipping ID " + id); + return; + } + + knownIds.add(id); + String subdirectory = ""; if (Utils.getConfigBoolean("reddit.use_sub_dirs", true)) { if (Utils.getConfigBoolean("album_titles.save", true)) { @@ -250,9 +293,8 @@ public class RedditRipper extends AlbumRipper { String savePath = this.workingDir + File.separator; savePath += id + "-" + url.split("/")[3] + title + ".mp4"; addURLToDownload(parseRedditVideoMPD(urls.get(0).toExternalForm()), new File(savePath)); - } - else { - addURLToDownload(urls.get(0), id + title, "", theUrl, null); + } else { + addURLToDownload(urls.get(0), id + "-" + title, "", theUrl, null); } } else if (urls.size() > 1) { for (int i = 0; i < urls.size(); i++) { @@ -263,6 +305,20 @@ public class RedditRipper extends AlbumRipper { addURLToDownload(urls.get(i), prefix, subdirectory, theUrl, null); } } + + try { + preparedStatement = connection.prepareStatement("INSERT INTO metalist values (?, ?, ?, ?, ?)"); + + preparedStatement.setString(1, id); + preparedStatement.setString(2, sub); + preparedStatement.setInt(3, created); + preparedStatement.setString(4, title); + preparedStatement.setString(5, "file"); + + preparedStatement.executeUpdate(); + } catch (SQLException e) { + LOGGER.warn("[!] FAILED TO INSERT META DATA", e); + } } @Override diff --git a/src/main/java/de/gurkengewuerz/ripmewrapper/ImageCrawler.java b/src/main/java/de/gurkengewuerz/ripmewrapper/ImageCrawler.java new file mode 100644 index 00000000..4f027794 --- /dev/null +++ b/src/main/java/de/gurkengewuerz/ripmewrapper/ImageCrawler.java @@ -0,0 +1,71 @@ +package de.gurkengewuerz.ripmewrapper; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.HashSet; +import java.util.List; +import java.util.TimerTask; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Created by gurkengewuerz.de on 09.10.2018. + */ +public class ImageCrawler extends TimerTask { + + @Override + public void run() { + Webserver.CRAWLER_IS_RUNNING = true; + Logger.getLogger(ImageCrawler.class.getName()).log(Level.INFO, ("Run Timer")); + try { + Connection connection = DriverManager.getConnection("jdbc:sqlite:reddit_meta.db"); + Statement statement = connection.createStatement(); + statement.setQueryTimeout(30); // set timeout to 30 sec. + + HashSet findIds = new HashSet<>(); + ResultSet rs = statement.executeQuery("SELECT metalist.id FROM metalist LEFT JOIN filelist ON metalist.id = filelist.id WHERE filelist.path IS NULL"); + while (rs.next()) { + findIds.add(rs.getString("id")); + } + + + List pathList = Files.find(Paths.get(System.getProperty("user.dir")), 100, + (path, basicFileAttributes) -> { + File file = path.toFile(); + return !file.isDirectory(); + }).map(String::valueOf) + .collect(Collectors.toList()); + + PreparedStatement ps = connection.prepareStatement("INSERT INTO filelist VALUES (NULL, ?, ?)"); + for (String s : pathList) { + String id = s.substring(s.lastIndexOf(File.separator) + 1).split("-")[0]; + if (!findIds.contains(id)) continue; + ps.setString(1, id); + ps.setString(2, s); + ps.executeUpdate(); + findIds.remove(id); + } + ps.close(); + + ps = connection.prepareStatement("DELETE FROM metalist WHERE id = ?"); + for (String id : findIds) { + ps.setString(1, id); + ps.executeUpdate(); + } + ps.close(); + + connection.close(); + } catch (Exception e) { + Logger.getLogger(ImageCrawler.class.getName()).log(Level.SEVERE, null, e); + } finally { + Webserver.CRAWLER_IS_RUNNING = false; + } + } +} diff --git a/src/main/java/de/gurkengewuerz/ripmewrapper/Webserver.java b/src/main/java/de/gurkengewuerz/ripmewrapper/Webserver.java new file mode 100644 index 00000000..5951fcdd --- /dev/null +++ b/src/main/java/de/gurkengewuerz/ripmewrapper/Webserver.java @@ -0,0 +1,72 @@ +package de.gurkengewuerz.ripmewrapper; + +import de.gurkengewuerz.ripmewrapper.handler.APIHandler; +import de.gurkengewuerz.ripmewrapper.handler.StaticHandler; +import org.eclipse.jetty.server.ConnectionFactory; +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.handler.ContextHandler; +import org.eclipse.jetty.server.handler.ContextHandlerCollection; +import org.eclipse.jetty.server.session.SessionHandler; + +import java.io.File; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.util.Timer; + +/** + * Created by gurkengewuerz.de on 09.10.2018. + */ +public class Webserver { + + public static boolean CRAWLER_IS_RUNNING = false; + + public static void main(String... args) throws Exception { + Connection connection = DriverManager.getConnection("jdbc:sqlite:reddit_meta.db"); + Statement statement = connection.createStatement(); + statement.setQueryTimeout(30); // set timeout to 30 sec. + + statement.executeUpdate("CREATE TABLE IF NOT EXISTS metalist (id string, subreddit string, created integer, title string, file string)"); + statement.executeUpdate("CREATE TABLE IF NOT EXISTS filelist (iid INTEGER PRIMARY KEY AUTOINCREMENT, id string, path string)"); + connection.close(); + + + Server server = new Server(3030); + + for (Connector y : server.getConnectors()) { + for (ConnectionFactory x : y.getConnectionFactories()) { + if (x instanceof HttpConnectionFactory) { + ((HttpConnectionFactory) x).getHttpConfiguration().setSendServerVersion(false); + } + } + } + + ContextHandler context_root = new ContextHandler("/"); + context_root.setContextPath("/"); + context_root.setHandler(new StaticHandler()); + + ContextHandler context_api = new ContextHandler("/api"); + context_api.setContextPath("/api"); + SessionHandler s = new SessionHandler(); + s.setHandler(new APIHandler()); + context_api.setHandler(s); + + ContextHandlerCollection contexts = new ContextHandlerCollection(); + contexts.setHandlers(new Handler[]{context_root, context_api}); + server.setHandler(contexts); + + Timer timer = new Timer(); + timer.schedule(new ImageCrawler(), 0, 5 * 60 * 1000); + + server.start(); + server.dumpStdErr(); + server.join(); + } + + public static File getStaticdir() { + return new File(System.getProperty("user.dir") + File.separator + "html" + File.separator); + } +} diff --git a/src/main/java/de/gurkengewuerz/ripmewrapper/handler/APIHandler.java b/src/main/java/de/gurkengewuerz/ripmewrapper/handler/APIHandler.java new file mode 100644 index 00000000..f0a548b9 --- /dev/null +++ b/src/main/java/de/gurkengewuerz/ripmewrapper/handler/APIHandler.java @@ -0,0 +1,169 @@ +package de.gurkengewuerz.ripmewrapper.handler; + +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.handler.AbstractHandler; +import org.json.JSONArray; +import org.json.JSONObject; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Created by gurkengewuerz.de on 09.10.2018. + */ +public class APIHandler extends AbstractHandler { + + @Override + public void handle(String s, Request request, HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) throws IOException { + try { + Connection connection = DriverManager.getConnection("jdbc:sqlite:reddit_meta.db"); + Statement statement = connection.createStatement(); + statement.setQueryTimeout(30); // set timeout to 30 sec. + + + JSONObject returnObject = null; + JSONArray returnArray = null; + + request.setCharacterEncoding("UTF-8"); + httpServletResponse.setCharacterEncoding("UTF-8"); + + if (s.equals("/get")) { + httpServletResponse.setStatus(HttpServletResponse.SC_OK); + returnArray = new JSONArray(); + + if (request.getParameter("subreddits") != null) { + String[] subreddits = request.getParameter("subreddits").split(","); + + int created = request.getParameter("offset") == null ? Integer.MAX_VALUE : Integer.valueOf(request.getParameter("offset")); + + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < subreddits.length; i++) { + builder.append("?,"); + } + + + PreparedStatement ps = connection.prepareStatement( + "SELECT iid, metalist.id, subreddit, created, path FROM filelist LEFT JOIN metalist ON filelist.id = metalist.id WHERE subreddit IN (" + builder.deleteCharAt(builder.length() - 1).toString() + ") AND created < ? ORDER BY created DESC LIMIT 10" + ); + + int index = 1; + for (String o : subreddits) { + ps.setString(index++, o); + } + + ps.setInt(index, created); + + + ResultSet rs = ps.executeQuery(); + while (rs.next()) { + JSONObject data = new JSONObject(); + data.put("iid", rs.getInt("iid")); + data.put("id", rs.getString("id")); + data.put("subreddit", rs.getString("subreddit")); + data.put("created", rs.getInt("created")); + data.put("webm", rs.getString("path").endsWith(".webm")); + data.put("mp4", rs.getString("path").endsWith(".mp4")); + returnArray.put(data); + } + rs.close(); + } + } else if (s.equals("/img")) { + httpServletResponse.setStatus(HttpServletResponse.SC_OK); + + PreparedStatement ps = connection.prepareStatement("SELECT path FROM filelist WHERE iid = ? AND id = ?"); + + ps.setString(1, request.getParameter("iid")); + ps.setString(2, request.getParameter("id")); + + ResultSet rs = ps.executeQuery(); + + + if (rs.next()) { + String path = rs.getString("path"); + + + byte[] encoded = Files.readAllBytes(Paths.get(path)); + httpServletResponse.setContentType(Files.probeContentType(Paths.get(path)));// or png or gif, etc + httpServletResponse.setContentLength(encoded.length); + httpServletResponse.getOutputStream().write(encoded); + } + + request.setHandled(true); + return; + } else if (s.equals("/info")) { + httpServletResponse.setStatus(HttpServletResponse.SC_OK); + returnObject = new JSONObject(); + + ResultSet rs = statement.executeQuery("SELECT DISTINCT subreddit FROM metalist;"); + JSONArray ja = new JSONArray(); + while (rs.next()) { + ja.put(rs.getString("subreddit")); + } + returnObject.put("subreddits", ja); + returnObject.put("subreddits_count", ja.length()); + + rs = statement.executeQuery("SELECT COUNT(*) c_all FROM filelist;"); + if (rs.next()) { + returnObject.put("files", rs.getInt("c_all")); + } + + + if (request.getParameter("subreddits") != null) { + String[] subreddits = request.getParameter("subreddits").split(","); + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < subreddits.length; i++) { + builder.append("?,"); + } + + PreparedStatement ps = connection.prepareStatement( + "SELECT COUNT(*) c FROM filelist LEFT JOIN metalist ON filelist.id = metalist.id WHERE subreddit IN (" + builder.deleteCharAt(builder.length() - 1).toString() + ")" + ); + + int index = 1; + for (String o : subreddits) { + ps.setString(index++, o); + } + + rs = ps.executeQuery(); + if (rs.next()) { + returnObject.put("files_subreddits", rs.getInt("c")); + } + } + } else { + httpServletResponse.setStatus(HttpServletResponse.SC_FORBIDDEN); + } + + httpServletResponse.setContentType("application/json; charset=utf-8"); + + PrintWriter out = httpServletResponse.getWriter(); + if (returnObject != null) { + out.write(returnObject.toString()); + } else if (returnArray != null) { + out.write(returnArray.toString()); + } else { + returnObject = new JSONObject(); + returnObject.put("error", "not found"); + out.write(returnObject.toString()); + } + + request.setHandled(true); + + connection.close(); + } catch (SQLException e) { + Logger.getLogger(getClass().getName()).log(Level.SEVERE, null, e); + } + } + +} diff --git a/src/main/java/de/gurkengewuerz/ripmewrapper/handler/StaticHandler.java b/src/main/java/de/gurkengewuerz/ripmewrapper/handler/StaticHandler.java new file mode 100644 index 00000000..65b358de --- /dev/null +++ b/src/main/java/de/gurkengewuerz/ripmewrapper/handler/StaticHandler.java @@ -0,0 +1,140 @@ +package de.gurkengewuerz.ripmewrapper.handler; + +import de.gurkengewuerz.ripmewrapper.Webserver; +import org.apache.commons.io.FilenameUtils; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.handler.AbstractHandler; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Created by gurkengewuerz.de on 09.10.2018. + */ +public class StaticHandler extends AbstractHandler { + + private String get403() throws IOException { + File f = new File(Webserver.getStaticdir(), "403.html"); + if (f.exists() && f.canRead()) return getFile(f, false); + return "

403 Forbidden


403 forbidden"; + } + + private String get404() throws IOException { + File f = new File(Webserver.getStaticdir(), "404.html"); + if (f.exists() && f.canRead()) return getFile(f, false); + return "

404 Not Found


404 not found"; + } + + private byte[] getBytes(File f) throws IOException { + return Files.readAllBytes(f.toPath()); + } + + private String getContent(File f) throws IOException { + return new String(getBytes(f), "utf-8"); + } + + private String getFile(File f, boolean get) throws IOException { + String data = getContent(f); + if (get) { + if (data.startsWith("")) { + return get403(); + } + } + data = data.replace("", ""); + + Pattern pattern = Pattern.compile(""); + Matcher matcher = pattern.matcher(data); + List matches = new ArrayList<>(); + while (matcher.find()) { + matches.add(matcher.group()); + } + for (String match : matches) { + Pattern pPattern = Pattern.compile("