From 89be8ac4296a72df4f877348892e7a6b27f1850f Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 25 Mar 2024 13:30:40 +0100 Subject: [PATCH 01/82] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 46bee72..7636dc1 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +# This project has been sunset as Crunchyroll moved to a DRM-only system. See [#362](https://github.com/crunchy-labs/crunchy-cli/issues/362). + # crunchy-cli 👇 A Command-line downloader for [Crunchyroll](https://www.crunchyroll.com). From ba8028737dbd5e99f7cc7d40d432e0f34505d9fa Mon Sep 17 00:00:00 2001 From: Amelia Date: Wed, 3 Apr 2024 06:49:51 -0700 Subject: [PATCH 02/82] Update missing fonts (#360) * Update missing fonts * Compile fix --- crunchy-cli-core/src/utils/download.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index 7f87583..736f9e6 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -1015,7 +1015,7 @@ fn get_video_stats(path: &Path) -> Result<(NaiveTime, f64)> { } // all subtitle fonts (extracted from javascript) -const FONTS: [(&str, &str); 66] = [ +const FONTS: [(&str, &str); 68] = [ ("Adobe Arabic", "AdobeArabic-Bold.woff2"), ("Andale Mono", "andalemo.woff2"), ("Arial", "arial.woff2"), @@ -1073,6 +1073,8 @@ const FONTS: [(&str, &str); 66] = [ ("Impact", "impact.woff2"), ("Mangal", "MANGAL.woff2"), ("Meera Inimai", "MeeraInimai-Regular.woff2"), + ("Noto Sans Tamil", "NotoSansTamil.woff2"), + ("Noto Sans Telugu", "NotoSansTelegu.woff2"), ("Noto Sans Thai", "NotoSansThai.woff2"), ("Rubik", "Rubik-Regular.woff2"), ("Rubik Black", "Rubik-Black.woff2"), From e694046b07fbcabf40714e62d1ac7a98f511fcdd Mon Sep 17 00:00:00 2001 From: bytedream Date: Wed, 3 Apr 2024 15:48:15 +0200 Subject: [PATCH 03/82] Move to new, DRM-free, endpoint --- Cargo.lock | 404 ++++++++++------------- crunchy-cli-core/Cargo.toml | 6 +- crunchy-cli-core/src/archive/command.rs | 2 +- crunchy-cli-core/src/download/command.rs | 2 +- crunchy-cli-core/src/search/format.rs | 36 +- crunchy-cli-core/src/utils/download.rs | 57 ++-- crunchy-cli-core/src/utils/format.rs | 31 +- crunchy-cli-core/src/utils/video.rs | 38 +-- 8 files changed, 245 insertions(+), 331 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 113b430..b7617b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,17 +17,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher", - "cpufeatures", -] - [[package]] name = "aho-corasick" version = "1.1.2" @@ -156,13 +145,19 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" + [[package]] name = "base64-serde" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba368df5de76a5bea49aaf0cf1b39ccfbbef176924d1ba5db3e4135216cbe3c7" dependencies = [ - "base64", + "base64 0.21.7", "serde", ] @@ -178,15 +173,6 @@ version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" -[[package]] -name = "block-padding" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" -dependencies = [ - "generic-array", -] - [[package]] name = "bumpalo" version = "3.15.4" @@ -199,15 +185,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" -[[package]] -name = "cbc" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" -dependencies = [ - "cipher", -] - [[package]] name = "cc" version = "1.0.90" @@ -228,9 +205,9 @@ checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" [[package]] name = "chrono" -version = "0.4.35" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eaf5903dcbc0a39312feb77df2ff4c76387d591b9fc7b04a238dcf8bb62639a" +checksum = "8a0d04d43504c61aa6c7531f1871dd0d418d91130162063b789da00fd7057a5e" dependencies = [ "android-tzdata", "iana-time-zone", @@ -241,16 +218,6 @@ dependencies = [ "windows-targets 0.52.4", ] -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common", - "inout", -] - [[package]] name = "clap" version = "4.5.2" @@ -373,15 +340,6 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" -[[package]] -name = "cpufeatures" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" -dependencies = [ - "libc", -] - [[package]] name = "crunchy-cli" version = "3.3.1" @@ -432,20 +390,17 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.8.6" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f99fcd7627d214fd57cd1d030e8c859a773e19aa29fb0d15017aa84efaba353" +checksum = "05fcd99a09d001333ab482412473aaa03c84f980d451845b4c43d58986b6eb64" dependencies = [ - "aes", "async-trait", - "cbc", "chrono", "crunchyroll-rs-internal", "dash-mpd", "futures-util", "jsonwebtoken", "lazy_static", - "m3u8-rs", "regex", "reqwest", "rustls", @@ -455,30 +410,20 @@ dependencies = [ "smart-default", "tokio", "tower-service", - "webpki-roots 0.26.1", + "webpki-roots", ] [[package]] name = "crunchyroll-rs-internal" -version = "0.8.6" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2dd269b2df82ebbec9e8164e9950c6ad14a01cfcbb85eceeb3f3ef26c7da90c" +checksum = "d406a27eca9ceab379b601101cd96b0f7bfff34e93487ca58d54118a8b4fbf91" dependencies = [ "darling", "quote", "syn", ] -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - [[package]] name = "ctrlc" version = "3.4.4" @@ -526,11 +471,11 @@ dependencies = [ [[package]] name = "dash-mpd" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18c18f28b58beade78e0f61a846a63a122cb92c5f5ed6bad29d7ad13287c7526" +checksum = "6cafa2c33eff2857e1a14c38aa9a432aa565a01e77804a541fce7aec3affb8f8" dependencies = [ - "base64", + "base64 0.22.0", "base64-serde", "chrono", "fs-err", @@ -614,15 +559,6 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" -[[package]] -name = "encoding_rs" -version = "0.8.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" -dependencies = [ - "cfg-if", -] - [[package]] name = "equivalent" version = "1.0.1" @@ -761,16 +697,6 @@ dependencies = [ "slab", ] -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getrandom" version = "0.2.12" @@ -790,25 +716,6 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" -[[package]] -name = "h2" -version = "0.3.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap 2.2.5", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "hashbrown" version = "0.12.3" @@ -841,9 +748,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "0.2.12" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ "bytes", "fnv", @@ -852,12 +759,24 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.6" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" dependencies = [ "bytes", "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", "pin-project-lite", ] @@ -867,61 +786,76 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - [[package]] name = "hyper" -version = "0.14.28" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +checksum = "186548d73ac615b32a73aafe38fb4f56c0d340e110e5a200bcadbaf2e199263a" dependencies = [ "bytes", "futures-channel", - "futures-core", "futures-util", - "h2", "http", "http-body", "httparse", - "httpdate", "itoa", "pin-project-lite", - "socket2", + "smallvec", "tokio", - "tower-service", - "tracing", "want", ] [[package]] name = "hyper-rustls" -version = "0.24.2" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" dependencies = [ "futures-util", "http", "hyper", + "hyper-util", "rustls", + "rustls-pki-types", "tokio", "tokio-rustls", + "tower-service", ] [[package]] name = "hyper-tls" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", + "http-body-util", "hyper", + "hyper-util", "native-tls", "tokio", "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "pin-project-lite", + "socket2", + "tokio", + "tower", + "tower-service", + "tracing", ] [[package]] @@ -1008,16 +942,6 @@ dependencies = [ "unicode-width", ] -[[package]] -name = "inout" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" -dependencies = [ - "block-padding", - "generic-array", -] - [[package]] name = "instant" version = "0.1.12" @@ -1059,11 +983,11 @@ dependencies = [ [[package]] name = "jsonwebtoken" -version = "9.2.0" +version = "9.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7ea04a7c5c055c175f189b6dc6ba036fd62306b58c66c9f6389036c503a3f4" +checksum = "b9ae10193d25051e74945f1ea2d0b42e03cc3b890f7e4cc5faa44997d808193f" dependencies = [ - "base64", + "base64 0.21.7", "js-sys", "ring", "serde", @@ -1105,16 +1029,6 @@ version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" -[[package]] -name = "m3u8-rs" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f03cd3335fb5f2447755d45cda9c70f76013626a9db44374973791b0926a86c3" -dependencies = [ - "chrono", - "nom", -] - [[package]] name = "memchr" version = "2.7.1" @@ -1127,6 +1041,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1303,6 +1227,26 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.13" @@ -1399,9 +1343,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.3" +version = "1.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ "aho-corasick", "memchr", @@ -1428,38 +1372,39 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "reqwest" -version = "0.11.25" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eea5a9eb898d3783f17c6407670e3592fd174cb81a10e51d4c37f49450b9946" +checksum = "2d66674f2b6fb864665eea7a3c1ac4e3dfacd2fda83cf6f935a612e01b0e3338" dependencies = [ - "base64", + "base64 0.21.7", "bytes", "cookie", "cookie_store", - "encoding_rs", "futures-core", "futures-util", - "h2", "http", "http-body", + "http-body-util", "hyper", "hyper-rustls", "hyper-tls", + "hyper-util", "ipnet", "js-sys", "log", "mime", + "mime_guess", "native-tls", "once_cell", "percent-encoding", "pin-project-lite", "rustls", "rustls-pemfile 1.0.4", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", - "system-configuration", "tokio", "tokio-native-tls", "tokio-rustls", @@ -1471,7 +1416,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 0.25.4", + "webpki-roots", "winreg", ] @@ -1517,14 +1462,16 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.10" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" +checksum = "99008d7ad0bbbea527ec27bddbc0e432c5b87d8175178cee68d2eec9c4a1813c" dependencies = [ "log", "ring", + "rustls-pki-types", "rustls-webpki", - "sct", + "subtle", + "zeroize", ] [[package]] @@ -1546,7 +1493,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64", + "base64 0.21.7", ] [[package]] @@ -1555,7 +1502,7 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f48172685e6ff52a556baa527774f61fcaa884f59daf3375c62a3f1cd2549dab" dependencies = [ - "base64", + "base64 0.21.7", "rustls-pki-types", ] @@ -1567,11 +1514,12 @@ checksum = "5ede67b28608b4c60685c7d54122d4400d90f62b40caee7700e700380a390fa8" [[package]] name = "rustls-webpki" -version = "0.101.7" +version = "0.102.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" dependencies = [ "ring", + "rustls-pki-types", "untrusted", ] @@ -1590,16 +1538,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "security-framework" version = "2.9.2" @@ -1687,11 +1625,11 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.6.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15d167997bd841ec232f5b2b8e0e26606df2e7caa4c31b95ea9ca52b200bd270" +checksum = "ee80b0e361bbf88fd2f6e242ccd19cfda072cb0faa6ae694ecee08199938569a" dependencies = [ - "base64", + "base64 0.21.7", "chrono", "hex", "indexmap 1.9.3", @@ -1705,9 +1643,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.6.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "865f9743393e638991566a8b7a479043c2c8da94a33e0a31f18214c9cae0a64d" +checksum = "6561dc161a9224638a31d876ccdfefbc1df91d3f3a8342eddb35f055d48c7655" dependencies = [ "darling", "proc-macro2", @@ -1736,6 +1674,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + [[package]] name = "smart-default" version = "0.7.1" @@ -1775,6 +1719,12 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + [[package]] name = "syn" version = "2.0.52" @@ -1801,27 +1751,6 @@ dependencies = [ "libc", ] -[[package]] -name = "system-configuration" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658bc6ee10a9b4fcf576e9b0819d95ec16f4d2c02d39fd83ac1c8789785c4a42" -dependencies = [ - "bitflags 2.4.2", - "core-foundation", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "tempfile" version = "3.10.1" @@ -1836,18 +1765,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.57" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" +checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.57" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" +checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ "proc-macro2", "quote", @@ -1902,9 +1831,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.36.0" +version = "1.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" +checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" dependencies = [ "backtrace", "bytes", @@ -1940,11 +1869,12 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.24.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" dependencies = [ "rustls", + "rustls-pki-types", "tokio", ] @@ -1974,6 +1904,28 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" @@ -1986,6 +1938,7 @@ version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -2018,10 +1971,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] -name = "typenum" -version = "1.17.0" +name = "unicase" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" +dependencies = [ + "version_check", +] [[package]] name = "unicode-bidi" @@ -2189,12 +2145,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki-roots" -version = "0.25.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" - [[package]] name = "webpki-roots" version = "0.26.1" @@ -2387,3 +2337,9 @@ dependencies = [ "linux-raw-sys", "rustix", ] + +[[package]] +name = "zeroize" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 0242606..7617c58 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -16,20 +16,20 @@ anyhow = "1.0" async-speed-limit = "0.4" clap = { version = "4.5", features = ["derive", "string"] } chrono = "0.4" -crunchyroll-rs = { version = "0.8.6", features = ["dash-stream", "experimental-stabilizations", "tower"] } +crunchyroll-rs = { version = "0.10.0", features = ["experimental-stabilizations", "tower"] } ctrlc = "3.4" dialoguer = { version = "0.11", default-features = false } dirs = "5.0" derive_setters = "0.1" futures-util = { version = "0.3", features = ["io"] } fs2 = "0.4" -http = "0.2" +http = "1.1" indicatif = "0.17" lazy_static = "1.4" log = { version = "0.4", features = ["std"] } num_cpus = "1.16" regex = "1.10" -reqwest = { version = "0.11", default-features = false, features = ["socks", "stream"] } +reqwest = { version = "0.12", default-features = false, features = ["socks", "stream"] } serde = "1.0" serde_json = "1.0" serde_plain = "1.0" diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index f7bea0e..0dc0b86 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -487,7 +487,7 @@ async fn get_format( single_format.audio == Locale::ja_JP || stream.subtitles.len() > 1, ) }); - let cc = stream.closed_captions.get(s).cloned().map(|l| (l, false)); + let cc = stream.captions.get(s).cloned().map(|l| (l, false)); subtitles .into_iter() diff --git a/crunchy-cli-core/src/download/command.rs b/crunchy-cli-core/src/download/command.rs index 843f5cd..47b29c9 100644 --- a/crunchy-cli-core/src/download/command.rs +++ b/crunchy-cli-core/src/download/command.rs @@ -389,7 +389,7 @@ async fn get_format( .get(subtitle_locale) .cloned() // use closed captions as fallback if no actual subtitles are found - .or_else(|| stream.closed_captions.get(subtitle_locale).cloned()) + .or_else(|| stream.captions.get(subtitle_locale).cloned()) } else { None }; diff --git a/crunchy-cli-core/src/search/format.rs b/crunchy-cli-core/src/search/format.rs index 156bd95..10eefd8 100644 --- a/crunchy-cli-core/src/search/format.rs +++ b/crunchy-cli-core/src/search/format.rs @@ -163,37 +163,15 @@ impl From<&Concert> for FormatConcert { struct FormatStream { pub locale: Locale, pub dash_url: String, - pub drm_dash_url: String, - pub hls_url: String, - pub drm_hls_url: String, + pub is_drm: bool, } impl From<&Stream> for FormatStream { fn from(value: &Stream) -> Self { - let (dash_url, drm_dash_url, hls_url, drm_hls_url) = - value.variants.get(&Locale::Custom("".to_string())).map_or( - ( - "".to_string(), - "".to_string(), - "".to_string(), - "".to_string(), - ), - |v| { - ( - v.adaptive_dash.clone().unwrap_or_default().url, - v.drm_adaptive_dash.clone().unwrap_or_default().url, - v.adaptive_hls.clone().unwrap_or_default().url, - v.drm_adaptive_hls.clone().unwrap_or_default().url, - ) - }, - ); - Self { locale: value.audio_locale.clone(), - dash_url, - drm_dash_url, - hls_url, - drm_hls_url, + dash_url: value.url.clone(), + is_drm: value.session.uses_stream_limits, } } } @@ -441,7 +419,7 @@ impl Format { if !stream_empty { for (_, episodes) in tree.iter_mut() { for (episode, streams) in episodes { - streams.push(episode.stream().await?) + streams.push(episode.stream_maybe_without_drm().await?) } } } else { @@ -510,7 +488,7 @@ impl Format { } if !stream_empty { for (movie, streams) in tree.iter_mut() { - streams.push(movie.stream().await?) + streams.push(movie.stream_maybe_without_drm().await?) } } else { for (_, streams) in tree.iter_mut() { @@ -548,7 +526,7 @@ impl Format { let stream_empty = self.check_pattern_count_empty(Scope::Stream); let music_video = must_match_if_true!(!music_video_empty => media_collection|MediaCollection::MusicVideo(music_video) => music_video.clone()).unwrap_or_default(); - let stream = must_match_if_true!(!stream_empty => media_collection|MediaCollection::MusicVideo(music_video) => music_video.stream().await?).unwrap_or_default(); + let stream = must_match_if_true!(!stream_empty => media_collection|MediaCollection::MusicVideo(music_video) => music_video.stream_maybe_without_drm().await?).unwrap_or_default(); let music_video_map = self.serializable_to_json_map(FormatMusicVideo::from(&music_video)); let stream_map = self.serializable_to_json_map(FormatStream::from(&stream)); @@ -570,7 +548,7 @@ impl Format { let stream_empty = self.check_pattern_count_empty(Scope::Stream); let concert = must_match_if_true!(!concert_empty => media_collection|MediaCollection::Concert(concert) => concert.clone()).unwrap_or_default(); - let stream = must_match_if_true!(!stream_empty => media_collection|MediaCollection::Concert(concert) => concert.stream().await?).unwrap_or_default(); + let stream = must_match_if_true!(!stream_empty => media_collection|MediaCollection::Concert(concert) => concert.stream_maybe_without_drm().await?).unwrap_or_default(); let concert_map = self.serializable_to_json_map(FormatConcert::from(&concert)); let stream_map = self.serializable_to_json_map(FormatStream::from(&stream)); diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index 736f9e6..8a1fe66 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -4,7 +4,7 @@ use crate::utils::os::{cache_dir, is_special_file, temp_directory, temp_named_pi use crate::utils::rate_limit::RateLimiterService; use anyhow::{bail, Result}; use chrono::NaiveTime; -use crunchyroll_rs::media::{SkipEvents, SkipEventsEvent, Subtitle, VariantData, VariantSegment}; +use crunchyroll_rs::media::{SkipEvents, SkipEventsEvent, StreamData, StreamSegment, Subtitle}; use crunchyroll_rs::Locale; use indicatif::{ProgressBar, ProgressDrawTarget, ProgressFinish, ProgressStyle}; use log::{debug, warn, LevelFilter}; @@ -117,8 +117,8 @@ struct FFmpegMeta { } pub struct DownloadFormat { - pub video: (VariantData, Locale), - pub audios: Vec<(VariantData, Locale)>, + pub video: (StreamData, Locale), + pub audios: Vec<(StreamData, Locale)>, pub subtitles: Vec<(Subtitle, bool)>, pub metadata: DownloadFormatMetadata, } @@ -671,20 +671,17 @@ impl Downloader { &self, dst: &Path, ) -> Result<(Option<(PathBuf, u64)>, Option<(PathBuf, u64)>)> { - let mut all_variant_data = vec![]; + let mut all_stream_data = vec![]; for format in &self.formats { - all_variant_data.push(&format.video.0); - all_variant_data.extend(format.audios.iter().map(|(a, _)| a)) + all_stream_data.push(&format.video.0); + all_stream_data.extend(format.audios.iter().map(|(a, _)| a)) } let mut estimated_required_space: u64 = 0; - for variant_data in all_variant_data { - // nearly no overhead should be generated with this call(s) as we're using dash as - // stream provider and generating the dash segments does not need any fetching of - // additional (http) resources as hls segments would - let segments = variant_data.segments().await?; + for stream_data in all_stream_data { + let segments = stream_data.segments(); // sum the length of all streams up - estimated_required_space += estimate_variant_file_size(variant_data, &segments); + estimated_required_space += estimate_variant_file_size(stream_data, &segments); } let tmp_stat = fs2::statvfs(temp_directory()).unwrap(); @@ -730,29 +727,21 @@ impl Downloader { Ok((tmp_required, dst_required)) } - async fn download_video( - &self, - variant_data: &VariantData, - message: String, - ) -> Result { + async fn download_video(&self, stream_data: &StreamData, message: String) -> Result { let tempfile = tempfile(".mp4")?; let (mut file, path) = tempfile.into_parts(); - self.download_segments(&mut file, message, variant_data) + self.download_segments(&mut file, message, stream_data) .await?; Ok(path) } - async fn download_audio( - &self, - variant_data: &VariantData, - message: String, - ) -> Result { + async fn download_audio(&self, stream_data: &StreamData, message: String) -> Result { let tempfile = tempfile(".m4a")?; let (mut file, path) = tempfile.into_parts(); - self.download_segments(&mut file, message, variant_data) + self.download_segments(&mut file, message, stream_data) .await?; Ok(path) @@ -806,15 +795,15 @@ impl Downloader { &self, writer: &mut impl Write, message: String, - variant_data: &VariantData, + stream_data: &StreamData, ) -> Result<()> { - let segments = variant_data.segments().await?; + let segments = stream_data.segments(); let total_segments = segments.len(); let count = Arc::new(Mutex::new(0)); let progress = if log::max_level() == LevelFilter::Info { - let estimated_file_size = estimate_variant_file_size(variant_data, &segments); + let estimated_file_size = estimate_variant_file_size(stream_data, &segments); let progress = ProgressBar::new(estimated_file_size) .with_style( @@ -832,7 +821,7 @@ impl Downloader { }; let cpus = self.download_threads; - let mut segs: Vec> = Vec::with_capacity(cpus); + let mut segs: Vec> = Vec::with_capacity(cpus); for _ in 0..cpus { segs.push(vec![]) } @@ -858,7 +847,7 @@ impl Downloader { let download = || async move { for (i, segment) in thread_segments.into_iter().enumerate() { let mut retry_count = 0; - let mut buf = loop { + let buf = loop { let request = thread_client .get(&segment.url) .timeout(Duration::from_secs(60)); @@ -884,11 +873,9 @@ impl Downloader { retry_count += 1; }; - buf = VariantSegment::decrypt(&mut buf, segment.key)?.to_vec(); - let mut c = thread_count.lock().await; debug!( - "Downloaded and decrypted segment [{}/{} {:.2}%] {}", + "Downloaded segment [{}/{} {:.2}%] {}", num + (i * cpus) + 1, total_segments, ((*c + 1) as f64 / total_segments as f64) * 100f64, @@ -928,7 +915,7 @@ impl Downloader { if let Some(p) = &progress { let progress_len = p.length().unwrap(); - let estimated_segment_len = (variant_data.bandwidth / 8) + let estimated_segment_len = (stream_data.bandwidth / 8) * segments.get(pos as usize).unwrap().length.as_secs(); let bytes_len = bytes.len() as u64; @@ -977,8 +964,8 @@ impl Downloader { } } -fn estimate_variant_file_size(variant_data: &VariantData, segments: &[VariantSegment]) -> u64 { - (variant_data.bandwidth / 8) * segments.iter().map(|s| s.length.as_secs()).sum::() +fn estimate_variant_file_size(stream_data: &StreamData, segments: &[StreamSegment]) -> u64 { + (stream_data.bandwidth / 8) * segments.iter().map(|s| s.length.as_secs()).sum::() } /// Get the length and fps of a video. diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index 7146a55..df79d64 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -2,9 +2,9 @@ use crate::utils::filter::real_dedup_vec; use crate::utils::locale::LanguageTagging; use crate::utils::log::tab_info; use crate::utils::os::{is_special_file, sanitize}; -use anyhow::Result; +use anyhow::{bail, Result}; use chrono::{Datelike, Duration}; -use crunchyroll_rs::media::{Resolution, SkipEvents, Stream, Subtitle, VariantData}; +use crunchyroll_rs::media::{Resolution, SkipEvents, Stream, StreamData, Subtitle}; use crunchyroll_rs::{Concert, Episode, Locale, MediaCollection, Movie, MusicVideo}; use log::{debug, info}; use std::cmp::Ordering; @@ -167,12 +167,17 @@ impl SingleFormat { pub async fn stream(&self) -> Result { let stream = match &self.source { - MediaCollection::Episode(e) => e.stream().await?, - MediaCollection::Movie(m) => m.stream().await?, - MediaCollection::MusicVideo(mv) => mv.stream().await?, - MediaCollection::Concert(c) => c.stream().await?, + MediaCollection::Episode(e) => e.stream_maybe_without_drm().await?, + MediaCollection::Movie(m) => m.stream_maybe_without_drm().await?, + MediaCollection::MusicVideo(mv) => mv.stream_maybe_without_drm().await?, + MediaCollection::Concert(c) => c.stream_maybe_without_drm().await?, _ => unreachable!(), }; + + if stream.session.uses_stream_limits { + bail!("Found a stream which probably uses DRM. DRM downloads aren't supported") + } + Ok(stream) } @@ -331,9 +336,7 @@ impl Iterator for SingleFormatCollectionIterator { type Item = Vec; fn next(&mut self) -> Option { - let Some((_, episodes)) = self.0 .0.iter_mut().next() else { - return None; - }; + let (_, episodes) = self.0 .0.iter_mut().next()?; let value = episodes.pop_first().unwrap().1; if episodes.is_empty() { @@ -377,7 +380,7 @@ pub struct Format { impl Format { #[allow(clippy::type_complexity)] pub fn from_single_formats( - mut single_formats: Vec<(SingleFormat, VariantData, Vec<(Subtitle, bool)>)>, + mut single_formats: Vec<(SingleFormat, StreamData, Vec<(Subtitle, bool)>)>, ) -> Self { let locales: Vec<(Locale, Vec)> = single_formats .iter() @@ -397,10 +400,10 @@ impl Format { title: first_format.title, description: first_format.description, locales, - resolution: first_stream.resolution.clone(), - width: first_stream.resolution.width, - height: first_stream.resolution.height, - fps: first_stream.fps, + resolution: first_stream.resolution().unwrap(), + width: first_stream.resolution().unwrap().width, + height: first_stream.resolution().unwrap().height, + fps: first_stream.fps().unwrap(), release_year: first_format.release_year, release_month: first_format.release_month, release_day: first_format.release_day, diff --git a/crunchy-cli-core/src/utils/video.rs b/crunchy-cli-core/src/utils/video.rs index 0ae4ba4..7f7d73e 100644 --- a/crunchy-cli-core/src/utils/video.rs +++ b/crunchy-cli-core/src/utils/video.rs @@ -1,17 +1,17 @@ use anyhow::{bail, Result}; -use crunchyroll_rs::media::{Resolution, Stream, VariantData}; +use crunchyroll_rs::media::{Resolution, Stream, StreamData}; use crunchyroll_rs::Locale; pub async fn variant_data_from_stream( stream: &Stream, resolution: &Resolution, subtitle: Option, -) -> Result> { +) -> Result> { // sometimes Crunchyroll marks episodes without real subtitles that they have subtitles and // reports that only hardsub episode are existing. the following lines are trying to prevent // potential errors which might get caused by this incorrect reporting // (https://github.com/crunchy-labs/crunchy-cli/issues/231) - let mut hardsub_locales = stream.streaming_hardsub_locales(); + let mut hardsub_locales: Vec = stream.hard_subs.keys().cloned().collect(); let (hardsub_locale, mut contains_hardsub) = if !hardsub_locales .contains(&Locale::Custom("".to_string())) && !hardsub_locales.contains(&Locale::Custom(":".to_string())) @@ -29,39 +29,29 @@ pub async fn variant_data_from_stream( (subtitle, hardsubs_requested) }; - let mut streaming_data = match stream.dash_streaming_data(hardsub_locale).await { + let (mut videos, mut audios) = match stream.stream_data(hardsub_locale).await { Ok(data) => data, Err(e) => { // the error variant is only `crunchyroll_rs::error::Error::Input` when the requested // hardsub is not available if let crunchyroll_rs::error::Error::Input { .. } = e { contains_hardsub = false; - stream.dash_streaming_data(None).await? + stream.stream_data(None).await? } else { bail!(e) } } - }; - streaming_data - .0 - .sort_by(|a, b| a.bandwidth.cmp(&b.bandwidth).reverse()); - streaming_data - .1 - .sort_by(|a, b| a.bandwidth.cmp(&b.bandwidth).reverse()); + } + .unwrap(); + videos.sort_by(|a, b| a.bandwidth.cmp(&b.bandwidth).reverse()); + audios.sort_by(|a, b| a.bandwidth.cmp(&b.bandwidth).reverse()); let video_variant = match resolution.height { - u64::MAX => Some(streaming_data.0.into_iter().next().unwrap()), - u64::MIN => Some(streaming_data.0.into_iter().last().unwrap()), - _ => streaming_data - .0 + u64::MAX => Some(videos.into_iter().next().unwrap()), + u64::MIN => Some(videos.into_iter().last().unwrap()), + _ => videos .into_iter() - .find(|v| resolution.height == v.resolution.height), + .find(|v| resolution.height == v.resolution().unwrap().height), }; - Ok(video_variant.map(|v| { - ( - v, - streaming_data.1.first().unwrap().clone(), - contains_hardsub, - ) - })) + Ok(video_variant.map(|v| (v, audios.first().unwrap().clone(), contains_hardsub))) } From f16cd25ea4a40f48b1744c67de09549756ec8505 Mon Sep 17 00:00:00 2001 From: Amelia Frost Date: Wed, 3 Apr 2024 16:09:33 +0200 Subject: [PATCH 04/82] Fix for some chapters being sent by CR as floats (#351) * Fix for some chapters being sent by CR as floats. See: https://github.com/crunchy-labs/crunchyroll-rs/commit/3f3a80f7f7205e8ecb67e15fcf82b988eb88d9b2 * Compile fix for error[E0277]: cannot multiply `f32` by `u32` * Format Co-authored-by: bytedream --- crunchy-cli-core/src/utils/download.rs | 42 ++++++++++++++------------ 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index 8a1fe66..d7904f3 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -1214,41 +1214,45 @@ fn write_ffmpeg_chapters( ) -> Result<()> { let video_len = video_len .signed_duration_since(NaiveTime::MIN) - .num_seconds() as u32; - events.sort_by(|(_, event_a), (_, event_b)| event_a.start.cmp(&event_b.start)); + .num_milliseconds() as f32 + / 1000.0; + events.sort_by(|(_, event_a), (_, event_b)| event_a.start.total_cmp(&event_b.start)); writeln!(file, ";FFMETADATA1")?; - let mut last_end_time = 0; + let mut last_end_time = 0.0; for (name, event) in events { - // include an extra 'Episode' chapter if the start of the current chapter is more than 10 - // seconds later than the end of the last chapter. - // this is done before writing the actual chapter of this loop to keep the chapter - // chronologically in order - if event.start as i32 - last_end_time as i32 > 10 { + /* + - Convert from seconds to milliseconds for the correct timescale + - Include an extra 'Episode' chapter if the start of the current chapter is more than 10 + seconds later than the end of the last chapter. + This is done before writing the actual chapter of this loop to keep the chapter + chronologically in order + */ + if event.start - last_end_time > 10.0 { writeln!(file, "[CHAPTER]")?; - writeln!(file, "TIMEBASE=1/1")?; - writeln!(file, "START={}", last_end_time)?; - writeln!(file, "END={}", event.start)?; + writeln!(file, "TIMEBASE=1/1000")?; + writeln!(file, "START={}", (last_end_time * 1000.0) as u32)?; + writeln!(file, "END={}", (event.start * 1000.0) as u32)?; writeln!(file, "title=Episode")?; } writeln!(file, "[CHAPTER]")?; - writeln!(file, "TIMEBASE=1/1")?; - writeln!(file, "START={}", event.start)?; - writeln!(file, "END={}", event.end)?; + writeln!(file, "TIMEBASE=1/1000")?; + writeln!(file, "START={}", (event.start * 1000.0) as u32)?; + writeln!(file, "END={}", (event.end * 1000.0) as u32)?; writeln!(file, "title={}", name)?; last_end_time = event.end; } - // only add a traling chapter if the gab between the end of the last chapter and the total video + // only add a trailing chapter if the gap between the end of the last chapter and the total video // length is greater than 10 seconds - if video_len as i32 - last_end_time as i32 > 10 { + if video_len - last_end_time > 10.0 { writeln!(file, "[CHAPTER]")?; - writeln!(file, "TIMEBASE=1/1")?; - writeln!(file, "START={}", last_end_time)?; - writeln!(file, "END={}", video_len)?; + writeln!(file, "TIMEBASE=1/1000")?; + writeln!(file, "START={}", (last_end_time * 1000.0) as u32)?; + writeln!(file, "END={}", (video_len * 1000.0) as u32)?; writeln!(file, "title=Episode")?; } From 111e461b302544f027f31d8b2453a268a7ab0013 Mon Sep 17 00:00:00 2001 From: bytedream Date: Wed, 3 Apr 2024 16:20:24 +0200 Subject: [PATCH 05/82] Update dependencies and version --- Cargo.lock | 134 +++++++++++++++++------------------- Cargo.toml | 4 +- crunchy-cli-core/Cargo.toml | 4 +- 3 files changed, 66 insertions(+), 76 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b7617b1..52bba41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,9 +19,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aho-corasick" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] @@ -91,9 +91,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.80" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1" +checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" [[package]] name = "async-speed-limit" @@ -109,9 +109,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.77" +version = "0.1.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" +checksum = "a507401cad91ec6a857ed5513a2073c82a9b9048762b885bb98655b306964681" dependencies = [ "proc-macro2", "quote", @@ -120,15 +120,15 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" [[package]] name = "backtrace" -version = "0.3.69" +version = "0.3.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" dependencies = [ "addr2line", "cc", @@ -169,9 +169,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" [[package]] name = "bumpalo" @@ -181,9 +181,9 @@ checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" [[package]] name = "bytes" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "cc" @@ -220,9 +220,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.2" +version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b230ab84b0ffdf890d5a10abdbc8b83ae1c4918275daea1ab8801f71536b2651" +checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" dependencies = [ "clap_builder", "clap_derive", @@ -237,7 +237,7 @@ dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim 0.11.0", + "strsim 0.11.1", ] [[package]] @@ -251,9 +251,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.0" +version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "307bc0538d5f0f83b8248db3087aa92fe504e4691294d0c96c0eabc33f47ba47" +checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" dependencies = [ "heck", "proc-macro2", @@ -342,7 +342,7 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "crunchy-cli" -version = "3.3.1" +version = "3.3.2" dependencies = [ "chrono", "clap", @@ -355,7 +355,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.3.1" +version = "3.3.2" dependencies = [ "anyhow", "async-speed-limit", @@ -577,9 +577,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" [[package]] name = "fnv" @@ -730,9 +730,9 @@ checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" [[package]] name = "heck" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" @@ -920,9 +920,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.5" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -968,9 +968,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "js-sys" @@ -1008,13 +1008,12 @@ checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libredox" -version = "0.0.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "libc", - "redox_syscall", ] [[package]] @@ -1031,9 +1030,9 @@ checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "memchr" -version = "2.7.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "mime" @@ -1099,7 +1098,7 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "cfg-if", "cfg_aliases", "libc", @@ -1167,7 +1166,7 @@ version = "0.10.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "cfg-if", "foreign-types", "libc", @@ -1204,9 +1203,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.101" +version = "0.9.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dda2b0f344e78efc2facf7d195d098df0dd72151b26ab98da807afc26c198dff" +checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" dependencies = [ "cc", "libc", @@ -1249,9 +1248,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" [[package]] name = "pin-utils" @@ -1279,9 +1278,9 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "proc-macro2" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" dependencies = [ "unicode-ident", ] @@ -1321,20 +1320,11 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "redox_syscall" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "redox_users" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" +checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" dependencies = [ "getrandom", "libredox", @@ -1366,9 +1356,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" [[package]] name = "reqwest" @@ -1449,11 +1439,11 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.38.31" +version = "0.38.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "errno", "libc", "linux-raw-sys", @@ -1508,9 +1498,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.3.1" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ede67b28608b4c60685c7d54122d4400d90f62b40caee7700e700380a390fa8" +checksum = "ecd36cc4259e3e4514335c4a138c6b43171a8d61d8f5c9348f9fc7529416f247" [[package]] name = "rustls-webpki" @@ -1540,9 +1530,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.9.2" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6" dependencies = [ "bitflags 1.3.2", "core-foundation", @@ -1553,9 +1543,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.9.1" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +checksum = "41f3cc463c0ef97e11c3461a9d3787412d30e8e7eb907c79180c4a57bf7c04ef" dependencies = [ "core-foundation-sys", "libc", @@ -1583,9 +1573,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.114" +version = "1.0.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" +checksum = "12dc5c46daa8e9fdf4f5e71b6cf9a53f2487da0e86e55808e2d35539666497dd" dependencies = [ "itoa", "ryu", @@ -1633,7 +1623,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.2.5", + "indexmap 2.2.6", "serde", "serde_derive", "serde_json", @@ -1715,9 +1705,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "strsim" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "subtle" @@ -1727,9 +1717,9 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" -version = "2.0.52" +version = "2.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07" +checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 681fe73..159b3cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.3.1" +version = "3.3.2" edition = "2021" license = "MIT" @@ -14,7 +14,7 @@ openssl-tls = ["dep:native-tls-crate", "native-tls-crate/openssl", "crunchy-cli- openssl-tls-static = ["dep:native-tls-crate", "native-tls-crate/openssl", "crunchy-cli-core/openssl-tls-static"] [dependencies] -tokio = { version = "1.36", features = ["macros", "rt-multi-thread", "time"], default-features = false } +tokio = { version = "1.37", features = ["macros", "rt-multi-thread", "time"], default-features = false } native-tls-crate = { package = "native-tls", version = "0.2.11", optional = true } diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 7617c58..f3e8b5f 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.3.1" +version = "3.3.2" edition = "2021" license = "MIT" @@ -36,7 +36,7 @@ serde_plain = "1.0" shlex = "1.3" sys-locale = "0.3" tempfile = "3.10" -tokio = { version = "1.36", features = ["io-util", "macros", "net", "rt-multi-thread", "time"] } +tokio = { version = "1.37", features = ["io-util", "macros", "net", "rt-multi-thread", "time"] } tokio-util = "0.7" tower-service = "0.3" rustls-native-certs = { version = "0.7", optional = true } From c0f334684679665a06cb6dc247017f63d52e2090 Mon Sep 17 00:00:00 2001 From: bytedream Date: Wed, 3 Apr 2024 16:46:49 +0200 Subject: [PATCH 06/82] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7636dc1..5463e30 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -# This project has been sunset as Crunchyroll moved to a DRM-only system. See [#362](https://github.com/crunchy-labs/crunchy-cli/issues/362). +> ~~This project has been sunset as Crunchyroll moved to a DRM-only system. See [#362](https://github.com/crunchy-labs/crunchy-cli/issues/362).~~ +> +> Well there is one endpoint which still has DRM-free streams, I guess I still have a bit time until (finally) everything is DRM-only. # crunchy-cli From af8ab248261c7d3b56930363fa6956ed01a8be9d Mon Sep 17 00:00:00 2001 From: bytedream Date: Wed, 3 Apr 2024 17:13:44 +0200 Subject: [PATCH 07/82] Update search command url help --- crunchy-cli-core/src/search/command.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crunchy-cli-core/src/search/command.rs b/crunchy-cli-core/src/search/command.rs index 226e242..f683e24 100644 --- a/crunchy-cli-core/src/search/command.rs +++ b/crunchy-cli-core/src/search/command.rs @@ -88,9 +88,7 @@ pub struct Search { /// /// stream.locale → Stream locale/language /// stream.dash_url → Stream url in DASH format - /// stream.drm_dash_url → Stream url in DRM protected DASH format - /// stream.hls_url → Stream url in HLS format - /// stream.drm_hls_url → Stream url in DRM protected HLS format + /// stream.is_drm → If `stream.is_drm` is DRM encrypted /// /// subtitle.locale → Subtitle locale/language /// subtitle.url → Url to the subtitle From 8c1868f2fd9fb6348a09313f8cdf6f41239d58b3 Mon Sep 17 00:00:00 2001 From: bytedream Date: Wed, 3 Apr 2024 17:14:07 +0200 Subject: [PATCH 08/82] Update dependencies and version --- Cargo.lock | 65 +++++++++++++++++++++++++++++++++---- Cargo.toml | 2 +- crunchy-cli-core/Cargo.toml | 6 ++-- 3 files changed, 63 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 52bba41..54d364c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -342,7 +342,7 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "crunchy-cli" -version = "3.3.2" +version = "3.3.3" dependencies = [ "chrono", "clap", @@ -355,7 +355,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.3.2" +version = "3.3.3" dependencies = [ "anyhow", "async-speed-limit", @@ -390,9 +390,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05fcd99a09d001333ab482412473aaa03c84f980d451845b4c43d58986b6eb64" +checksum = "b9dba87354272cbe34eea8c27cab75b5104d7334aa6374db4807bd145f77a5ac" dependencies = [ "async-trait", "chrono", @@ -415,9 +415,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d406a27eca9ceab379b601101cd96b0f7bfff34e93487ca58d54118a8b4fbf91" +checksum = "7f7727afdfa43dcc8981a83c299a2e416262053402dc0586b15bfe0488f05e23" dependencies = [ "darling", "quote", @@ -559,6 +559,15 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -716,6 +725,25 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +[[package]] +name = "h2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51ee2dd2e4f378392eeff5d51618cd9a63166a2513846bbc55f21cfacd9199d4" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap 2.2.6", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -795,6 +823,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", + "h2", "http", "http-body", "httparse", @@ -1370,8 +1399,10 @@ dependencies = [ "bytes", "cookie", "cookie_store", + "encoding_rs", "futures-core", "futures-util", + "h2", "http", "http-body", "http-body-util", @@ -1395,6 +1426,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper", + "system-configuration", "tokio", "tokio-native-tls", "tokio-rustls", @@ -1741,6 +1773,27 @@ dependencies = [ "libc", ] +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tempfile" version = "3.10.1" diff --git a/Cargo.toml b/Cargo.toml index 159b3cd..3a86a3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.3.2" +version = "3.3.3" edition = "2021" license = "MIT" diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index f3e8b5f..9d175d8 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.3.2" +version = "3.3.3" edition = "2021" license = "MIT" @@ -16,7 +16,7 @@ anyhow = "1.0" async-speed-limit = "0.4" clap = { version = "4.5", features = ["derive", "string"] } chrono = "0.4" -crunchyroll-rs = { version = "0.10.0", features = ["experimental-stabilizations", "tower"] } +crunchyroll-rs = { version = "0.10.1", features = ["experimental-stabilizations", "tower"] } ctrlc = "3.4" dialoguer = { version = "0.11", default-features = false } dirs = "5.0" @@ -29,7 +29,7 @@ lazy_static = "1.4" log = { version = "0.4", features = ["std"] } num_cpus = "1.16" regex = "1.10" -reqwest = { version = "0.12", default-features = false, features = ["socks", "stream"] } +reqwest = { version = "0.12", features = ["socks", "stream"] } serde = "1.0" serde_json = "1.0" serde_plain = "1.0" From 6b6d24a575ab4a4e873b93d11127d201f0efebb8 Mon Sep 17 00:00:00 2001 From: bytedream Date: Thu, 4 Apr 2024 21:01:34 +0200 Subject: [PATCH 09/82] Update dependencies and version --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 2 +- crunchy-cli-core/Cargo.toml | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 54d364c..f584cee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -342,7 +342,7 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "crunchy-cli" -version = "3.3.3" +version = "3.3.4" dependencies = [ "chrono", "clap", @@ -355,7 +355,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.3.3" +version = "3.3.4" dependencies = [ "anyhow", "async-speed-limit", @@ -390,9 +390,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9dba87354272cbe34eea8c27cab75b5104d7334aa6374db4807bd145f77a5ac" +checksum = "0010e5dded0388e3a1e69105c2e65637d092eff049ba10f132f216c8e26a2473" dependencies = [ "async-trait", "chrono", @@ -415,9 +415,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7727afdfa43dcc8981a83c299a2e416262053402dc0586b15bfe0488f05e23" +checksum = "7e5226275711b3d1dc6afdc5e2241a45bb5d4dc1a813902265d628ccfe1ab67d" dependencies = [ "darling", "quote", @@ -727,9 +727,9 @@ checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "h2" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51ee2dd2e4f378392eeff5d51618cd9a63166a2513846bbc55f21cfacd9199d4" +checksum = "816ec7294445779408f36fe57bc5b7fc1cf59664059096c65f905c1c61f58069" dependencies = [ "bytes", "fnv", diff --git a/Cargo.toml b/Cargo.toml index 3a86a3d..221db3f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.3.3" +version = "3.3.4" edition = "2021" license = "MIT" diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 9d175d8..0bbf493 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.3.3" +version = "3.3.4" edition = "2021" license = "MIT" @@ -16,7 +16,7 @@ anyhow = "1.0" async-speed-limit = "0.4" clap = { version = "4.5", features = ["derive", "string"] } chrono = "0.4" -crunchyroll-rs = { version = "0.10.1", features = ["experimental-stabilizations", "tower"] } +crunchyroll-rs = { version = "0.10.2", features = ["experimental-stabilizations", "tower"] } ctrlc = "3.4" dialoguer = { version = "0.11", default-features = false } dirs = "5.0" From c40ea8b1320eda6b2243147ec345161d7b81b53a Mon Sep 17 00:00:00 2001 From: bytedream Date: Fri, 5 Apr 2024 22:31:39 +0200 Subject: [PATCH 10/82] Update dependencies and version --- Cargo.lock | 35 +++++++++++++---------------------- Cargo.toml | 2 +- crunchy-cli-core/Cargo.toml | 4 ++-- 3 files changed, 16 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f584cee..7d0b7e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -342,7 +342,7 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "crunchy-cli" -version = "3.3.4" +version = "3.3.5" dependencies = [ "chrono", "clap", @@ -355,7 +355,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.3.4" +version = "3.3.5" dependencies = [ "anyhow", "async-speed-limit", @@ -390,9 +390,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0010e5dded0388e3a1e69105c2e65637d092eff049ba10f132f216c8e26a2473" +checksum = "2eae6c95ec38118d02ef2ca738e245d8afc404f05e502051013dc37e0295bb32" dependencies = [ "async-trait", "chrono", @@ -415,9 +415,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e5226275711b3d1dc6afdc5e2241a45bb5d4dc1a813902265d628ccfe1ab67d" +checksum = "2f589713700c948db9a976d3f83816ab12efebdf759044a7bb963dad62000c12" dependencies = [ "darling", "quote", @@ -1391,11 +1391,11 @@ checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" [[package]] name = "reqwest" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d66674f2b6fb864665eea7a3c1ac4e3dfacd2fda83cf6f935a612e01b0e3338" +checksum = "3e6cc1e89e689536eb5aeede61520e874df5a4707df811cd5da4aa5fbb2aae19" dependencies = [ - "base64 0.21.7", + "base64 0.22.0", "bytes", "cookie", "cookie_store", @@ -1420,7 +1420,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rustls", - "rustls-pemfile 1.0.4", + "rustls-pemfile", "rustls-pki-types", "serde", "serde_json", @@ -1503,21 +1503,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" dependencies = [ "openssl-probe", - "rustls-pemfile 2.1.1", + "rustls-pemfile", "rustls-pki-types", "schannel", "security-framework", ] -[[package]] -name = "rustls-pemfile" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" -dependencies = [ - "base64 0.21.7", -] - [[package]] name = "rustls-pemfile" version = "2.1.1" @@ -2362,9 +2353,9 @@ checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" [[package]] name = "winreg" -version = "0.50.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" dependencies = [ "cfg-if", "windows-sys 0.48.0", diff --git a/Cargo.toml b/Cargo.toml index 221db3f..c49244b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.3.4" +version = "3.3.5" edition = "2021" license = "MIT" diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 0bbf493..07973e1 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.3.4" +version = "3.3.5" edition = "2021" license = "MIT" @@ -16,7 +16,7 @@ anyhow = "1.0" async-speed-limit = "0.4" clap = { version = "4.5", features = ["derive", "string"] } chrono = "0.4" -crunchyroll-rs = { version = "0.10.2", features = ["experimental-stabilizations", "tower"] } +crunchyroll-rs = { version = "0.10.3", features = ["experimental-stabilizations", "tower"] } ctrlc = "3.4" dialoguer = { version = "0.11", default-features = false } dirs = "5.0" From 4b74299733732ee07f94dcc503d1c64920c888f2 Mon Sep 17 00:00:00 2001 From: bytedream Date: Fri, 5 Apr 2024 22:53:53 +0200 Subject: [PATCH 11/82] Only run ci action on branch push --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6991cad..70f4400 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,8 @@ name: ci on: push: + branches: + - '*' pull_request: workflow_dispatch: From 25cde6163c8c0a5b13459274f4239b1b7f99181a Mon Sep 17 00:00:00 2001 From: bytedream Date: Sat, 6 Apr 2024 21:23:21 +0200 Subject: [PATCH 12/82] Add account scope for search command --- crunchy-cli-core/src/lib.rs | 2 - crunchy-cli-core/src/search/command.rs | 13 ++++-- crunchy-cli-core/src/search/format.rs | 59 ++++++++++++++++++++++++-- 3 files changed, 65 insertions(+), 9 deletions(-) diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index 8067c42..483cb63 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -225,8 +225,6 @@ async fn execute_executor(executor: impl Execute, ctx: Context) { if let Some(crunchy_error) = err.downcast_mut::() { if let Error::Block { message, .. } = crunchy_error { *message = "Triggered Cloudflare bot protection. Try again later or use a VPN or proxy to spoof your location".to_string() - } else if let Error::Request { message, .. } = crunchy_error { - *message = "You've probably hit a rate limit. Try again later, generally after 10-20 minutes the rate limit is over and you can continue to use the cli".to_string() } error!("An error occurred: {}", crunchy_error) diff --git a/crunchy-cli-core/src/search/command.rs b/crunchy-cli-core/src/search/command.rs index f683e24..c29ce34 100644 --- a/crunchy-cli-core/src/search/command.rs +++ b/crunchy-cli-core/src/search/command.rs @@ -8,6 +8,7 @@ use crunchyroll_rs::common::StreamExt; use crunchyroll_rs::search::QueryResults; use crunchyroll_rs::{Episode, Locale, MediaCollection, MovieListing, MusicVideo, Series}; use log::warn; +use std::sync::Arc; #[derive(Debug, clap::Parser)] #[clap(about = "Search in videos")] @@ -87,11 +88,16 @@ pub struct Search { /// concert.premium_only → If the concert is only available with Crunchyroll premium /// /// stream.locale → Stream locale/language - /// stream.dash_url → Stream url in DASH format - /// stream.is_drm → If `stream.is_drm` is DRM encrypted + /// stream.dash_url → Stream url in DASH format. You need to set the `Authorization` header to `Bearer ` when requesting this url + /// stream.is_drm → If `stream.dash_url` is DRM encrypted /// /// subtitle.locale → Subtitle locale/language /// subtitle.url → Url to the subtitle + /// + /// account.token → Access token to make request to restricted endpoints. This token is only valid for a max. of 5 minutes + /// account.id → Internal ID of the user account + /// account.profile_name → Profile name of the account + /// account.email → Email address of the account #[arg(short, long, verbatim_doc_comment)] #[arg(default_value = "S{{season.number}}E{{episode.number}} - {{episode.title}}")] output: String, @@ -143,13 +149,14 @@ impl Execute for Search { output }; + let crunchy_arc = Arc::new(ctx.crunchy); for (media_collection, url_filter) in input { let filter_options = FilterOptions { audio: self.audio.clone(), url_filter, }; - let format = Format::new(self.output.clone(), filter_options)?; + let format = Format::new(self.output.clone(), filter_options, crunchy_arc.clone())?; println!("{}", format.parse(media_collection).await?); } diff --git a/crunchy-cli-core/src/search/format.rs b/crunchy-cli-core/src/search/format.rs index 10eefd8..7ea84d8 100644 --- a/crunchy-cli-core/src/search/format.rs +++ b/crunchy-cli-core/src/search/format.rs @@ -2,13 +2,15 @@ use crate::search::filter::FilterOptions; use anyhow::{bail, Result}; use crunchyroll_rs::media::{Stream, Subtitle}; use crunchyroll_rs::{ - Concert, Episode, Locale, MediaCollection, Movie, MovieListing, MusicVideo, Season, Series, + Concert, Crunchyroll, Episode, Locale, MediaCollection, Movie, MovieListing, MusicVideo, + Season, Series, }; use regex::Regex; use serde::Serialize; use serde_json::{Map, Value}; use std::collections::HashMap; use std::ops::Range; +use std::sync::Arc; #[derive(Default, Serialize)] struct FormatSeries { @@ -191,6 +193,27 @@ impl From<&Subtitle> for FormatSubtitle { } } +#[derive(Default, Serialize)] +struct FormatAccount { + pub token: String, + pub id: String, + pub profile_name: String, + pub email: String, +} + +impl FormatAccount { + pub async fn async_from(value: &Crunchyroll) -> Result { + let account = value.account().await?; + + Ok(Self { + token: value.access_token().await, + id: account.account_id, + profile_name: account.profile_name, + email: account.email, + }) + } +} + #[derive(Clone, Debug, Eq, PartialEq, Hash)] enum Scope { Series, @@ -202,6 +225,7 @@ enum Scope { Concert, Stream, Subtitle, + Account, } macro_rules! must_match_if_true { @@ -230,10 +254,15 @@ pub struct Format { pattern_count: HashMap, input: String, filter_options: FilterOptions, + crunchyroll: Arc, } impl Format { - pub fn new(input: String, filter_options: FilterOptions) -> Result { + pub fn new( + input: String, + filter_options: FilterOptions, + crunchyroll: Arc, + ) -> Result { let scope_regex = Regex::new(r"(?m)\{\{\s*(?P\w+)\.(?P\w+)\s*}}").unwrap(); let mut pattern = vec![]; let mut pattern_count = HashMap::new(); @@ -260,6 +289,7 @@ impl Format { Scope::Concert => FormatConcert Scope::Stream => FormatStream Scope::Subtitle => FormatSubtitle + Scope::Account => FormatAccount ); for capture in scope_regex.captures_iter(&input) { @@ -277,6 +307,7 @@ impl Format { "concert" => Scope::Concert, "stream" => Scope::Stream, "subtitle" => Scope::Subtitle, + "account" => Scope::Account, _ => bail!("'{}.{}' is not a valid keyword", scope, field), }; @@ -302,6 +333,7 @@ impl Format { pattern_count, input, filter_options, + crunchyroll, }) } @@ -316,6 +348,7 @@ impl Format { Scope::Episode, Scope::Stream, Scope::Subtitle, + Scope::Account, ])?; self.parse_series(media_collection).await @@ -326,17 +359,28 @@ impl Format { Scope::Movie, Scope::Stream, Scope::Subtitle, + Scope::Account, ])?; self.parse_movie_listing(media_collection).await } MediaCollection::MusicVideo(_) => { - self.check_scopes(vec![Scope::MusicVideo, Scope::Stream, Scope::Subtitle])?; + self.check_scopes(vec![ + Scope::MusicVideo, + Scope::Stream, + Scope::Subtitle, + Scope::Account, + ])?; self.parse_music_video(media_collection).await } MediaCollection::Concert(_) => { - self.check_scopes(vec![Scope::Concert, Scope::Stream, Scope::Subtitle])?; + self.check_scopes(vec![ + Scope::Concert, + Scope::Stream, + Scope::Subtitle, + Scope::Account, + ])?; self.parse_concert(media_collection).await } @@ -349,6 +393,7 @@ impl Format { let episode_empty = self.check_pattern_count_empty(Scope::Episode); let stream_empty = self.check_pattern_count_empty(Scope::Stream) && self.check_pattern_count_empty(Scope::Subtitle); + let account_empty = self.check_pattern_count_empty(Scope::Account); #[allow(clippy::type_complexity)] let mut tree: Vec<(Season, Vec<(Episode, Vec)>)> = vec![]; @@ -431,6 +476,11 @@ impl Format { } let mut output = vec![]; + let account_map = if !account_empty { + self.serializable_to_json_map(FormatAccount::async_from(&self.crunchyroll).await?) + } else { + Map::default() + }; let series_map = self.serializable_to_json_map(FormatSeries::from(&series)); for (season, episodes) in tree { let season_map = self.serializable_to_json_map(FormatSeason::from(&season)); @@ -442,6 +492,7 @@ impl Format { output.push( self.replace_all( HashMap::from([ + (Scope::Account, &account_map), (Scope::Series, &series_map), (Scope::Season, &season_map), (Scope::Episode, &episode_map), From fe49161e933e739784393a5ba92e29291025b803 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 8 Apr 2024 00:37:19 +0200 Subject: [PATCH 13/82] End ffmpeg progress always with at least 100% --- crunchy-cli-core/src/utils/download.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index d7904f3..cee89e2 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -1285,21 +1285,11 @@ async fn ffmpeg_progress( let reader = BufReader::new(stats); let mut lines = reader.lines(); + let mut frame = 0; loop { select! { - // when gracefully canceling this future, set the progress to 100% (finished). sometimes - // ffmpeg is too fast or already finished when the reading process of 'stats' starts - // which causes the progress to be stuck at 0% _ = cancellation_token.cancelled() => { - if let Some(p) = &progress { - p.set_position(total_frames) - } - debug!( - "Processed frame [{}/{} 100%]", - total_frames, - total_frames - ); - return Ok(()) + break } line = lines.next_line() => { let Some(line) = line? else { @@ -1314,7 +1304,7 @@ async fn ffmpeg_progress( let Some(frame_str) = frame_cap.name("frame") else { break }; - let frame: u64 = frame_str.as_str().parse()?; + frame = frame_str.as_str().parse()?; if let Some(p) = &progress { p.set_position(frame) @@ -1330,5 +1320,15 @@ async fn ffmpeg_progress( } } + // when this future is gracefully cancelled or if ffmpeg is too fast or already finished when + // reading process of 'stats' starts (which causes the progress to be stuck at 0%), the progress + // is manually set to 100% here + if frame < total_frames { + if let Some(p) = &progress { + p.set_position(frame) + } + debug!("Processed frame [{}/{} 100%]", total_frames, total_frames); + } + Ok(()) } From 1a511e12f95e7e4e76d97a4624306387909c6590 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 8 Apr 2024 13:57:06 +0200 Subject: [PATCH 14/82] Add archive start sync flag --- Cargo.lock | 122 +++++ crunchy-cli-core/Cargo.toml | 2 + crunchy-cli-core/src/archive/command.rs | 35 +- crunchy-cli-core/src/download/command.rs | 4 +- crunchy-cli-core/src/lib.rs | 33 +- crunchy-cli-core/src/utils/download.rs | 653 +++++++++++++++++------ crunchy-cli-core/src/utils/os.rs | 20 +- crunchy-cli-core/src/utils/video.rs | 2 +- 8 files changed, 692 insertions(+), 179 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7d0b7e1..c1f24f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -179,6 +179,18 @@ version = "3.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" +[[package]] +name = "bytemuck" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d6d68c57235a3a081186990eca2867354726650f42f7516ca50c28d6281fd15" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.6.0" @@ -369,6 +381,8 @@ dependencies = [ "fs2", "futures-util", "http", + "image", + "image_hasher", "indicatif", "lazy_static", "log", @@ -936,6 +950,32 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "image" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd54d660e773627692c524beaad361aca785a4f9f5730ce91f42aabe5bce3d11" +dependencies = [ + "bytemuck", + "byteorder", + "num-traits", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image_hasher" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481465fe767d92494987319b0b447a5829edf57f09c52bf8639396abaaeaf78" +dependencies = [ + "base64 0.22.0", + "image", + "rustdct", + "serde", + "transpose", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -1143,12 +1183,30 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num-complex" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23c6602fda94a57c990fe0df199a035d83576b496aa29f4e634a8ac6004e68a6" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.18" @@ -1305,6 +1363,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "primal-check" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9df7f93fd637f083201473dab4fee2db4c429d32e55e3299980ab3957ab916a0" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro2" version = "1.0.79" @@ -1469,6 +1536,30 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustdct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b61555105d6a9bf98797c063c362a1d24ed8ab0431655e38f1cf51e52089551" +dependencies = [ + "rustfft", +] + +[[package]] +name = "rustfft" +version = "6.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43806561bc506d0c5d160643ad742e3161049ac01027b5e6d7524091fd401d86" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", + "version_check", +] + [[package]] name = "rustix" version = "0.38.32" @@ -1720,6 +1811,12 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "strsim" version = "0.10.0" @@ -1998,6 +2095,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -2377,3 +2484,18 @@ name = "zeroize" version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" + +[[package]] +name = "zune-core" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" + +[[package]] +name = "zune-jpeg" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec866b44a2a1fd6133d363f073ca1b179f438f99e7e5bfb1e33f7181facfe448" +dependencies = [ + "zune-core", +] diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 07973e1..bd47aba 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -24,6 +24,8 @@ derive_setters = "0.1" futures-util = { version = "0.3", features = ["io"] } fs2 = "0.4" http = "1.1" +image = { version = "0.25", features = ["jpeg"], default-features = false } +image_hasher = "2.0" indicatif = "0.17" lazy_static = "1.4" log = { version = "0.4", features = ["std"] } diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index 0dc0b86..64ad66a 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -10,7 +10,7 @@ use crate::utils::locale::{all_locale_in_locales, resolve_locales, LanguageTaggi use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file}; use crate::utils::parse::parse_url; -use crate::utils::video::variant_data_from_stream; +use crate::utils::video::stream_data_from_stream; use crate::Execute; use anyhow::bail; use anyhow::Result; @@ -89,6 +89,17 @@ pub struct Archive { #[arg(value_parser = crate::utils::clap::clap_parse_resolution)] pub(crate) resolution: Resolution, + #[arg(help = "Tries to sync the timing of all downloaded audios to match one video")] + #[arg( + long_help = "Tries to sync the timing of all downloaded audios to match one video. \ + This is done by downloading the first few segments/frames of all video tracks that differ in length and comparing them frame by frame. \ + The value of this flag determines how accurate the syncing is, generally speaking everything over 15 begins to be more inaccurate and everything below 6 is too accurate (and won't succeed). \ + If you want to provide a custom value to this flag, you have to set it with an equals (e.g. `--sync-start=10` instead of `--sync-start 10`). \ + When the syncing fails, the command is continued as if `--sync-start` wasn't provided for this episode + " + )] + #[arg(long, require_equals = true, num_args = 0..=1, default_missing_value = "7.5")] + pub(crate) sync_start: Option, #[arg( help = "Sets the behavior of the stream merging. Valid behaviors are 'auto', 'audio' and 'video'" )] @@ -216,8 +227,19 @@ impl Execute for Archive { } } - if self.include_chapters && !matches!(self.merge, MergeBehavior::Audio) { - bail!("`--include-chapters` can only be used if `--merge` is set to 'audio'") + if self.include_chapters + && !matches!(self.merge, MergeBehavior::Audio) + && self.sync_start.is_none() + { + bail!("`--include-chapters` can only be used if `--merge` is set to 'audio' or `--sync-start` is set") + } + + if !matches!(self.merge, MergeBehavior::Auto) && self.sync_start.is_some() { + bail!("`--sync-start` can only be used if `--merge` is set to `auto`") + } + + if self.sync_start.is_some() && self.ffmpeg_preset.is_none() { + warn!("Using `--sync-start` without `--ffmpeg-preset` might produce worse sync results than with `--ffmpeg-preset` set") } if self.output.contains("{resolution}") @@ -294,6 +316,7 @@ impl Execute for Archive { .audio_sort(Some(self.audio.clone())) .subtitle_sort(Some(self.subtitle.clone())) .no_closed_caption(self.no_closed_caption) + .sync_start_value(self.sync_start) .threads(self.threads) .audio_locale_output_map( zip(self.audio.clone(), self.output_audio_locales.clone()).collect(), @@ -450,7 +473,7 @@ async fn get_format( for single_format in single_formats { let stream = single_format.stream().await?; let Some((video, audio, _)) = - variant_data_from_stream(&stream, &archive.resolution, None).await? + stream_data_from_stream(&stream, &archive.resolution, None).await? else { if single_format.is_episode() { bail!( @@ -569,7 +592,9 @@ async fn get_format( video: (video, single_format.audio.clone()), audios: vec![(audio, single_format.audio.clone())], subtitles, - metadata: DownloadFormatMetadata { skip_events: None }, + metadata: DownloadFormatMetadata { + skip_events: single_format.skip_events().await?, + }, }, )); } diff --git a/crunchy-cli-core/src/download/command.rs b/crunchy-cli-core/src/download/command.rs index 47b29c9..fd29030 100644 --- a/crunchy-cli-core/src/download/command.rs +++ b/crunchy-cli-core/src/download/command.rs @@ -8,7 +8,7 @@ use crate::utils::locale::{resolve_locales, LanguageTagging}; use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file}; use crate::utils::parse::parse_url; -use crate::utils::video::variant_data_from_stream; +use crate::utils::video::stream_data_from_stream; use crate::Execute; use anyhow::bail; use anyhow::Result; @@ -351,7 +351,7 @@ async fn get_format( try_peer_hardsubs: bool, ) -> Result<(DownloadFormat, Format)> { let stream = single_format.stream().await?; - let Some((video, audio, contains_hardsub)) = variant_data_from_stream( + let Some((video, audio, contains_hardsub)) = stream_data_from_stream( &stream, &download.resolution, if try_peer_hardsubs { diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index 483cb63..d8d5ec5 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -184,16 +184,29 @@ pub async fn main(args: &[String]) { .unwrap_or_default() .starts_with(".crunchy-cli_") { - let result = fs::remove_file(file.path()); - debug!( - "Ctrl-c removed temporary file {} {}", - file.path().to_string_lossy(), - if result.is_ok() { - "successfully" - } else { - "not successfully" - } - ) + if file.file_type().map_or(true, |ft| ft.is_file()) { + let result = fs::remove_file(file.path()); + debug!( + "Ctrl-c removed temporary file {} {}", + file.path().to_string_lossy(), + if result.is_ok() { + "successfully" + } else { + "not successfully" + } + ) + } else { + let result = fs::remove_dir_all(file.path()); + debug!( + "Ctrl-c removed temporary directory {} {}", + file.path().to_string_lossy(), + if result.is_ok() { + "successfully" + } else { + "not successfully" + } + ) + } } } } diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index cee89e2..fe0e84a 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -1,11 +1,15 @@ use crate::utils::ffmpeg::FFmpegPreset; use crate::utils::filter::real_dedup_vec; -use crate::utils::os::{cache_dir, is_special_file, temp_directory, temp_named_pipe, tempfile}; +use crate::utils::log::progress; +use crate::utils::os::{ + cache_dir, is_special_file, temp_directory, temp_named_pipe, tempdir, tempfile, +}; use crate::utils::rate_limit::RateLimiterService; use anyhow::{bail, Result}; -use chrono::NaiveTime; +use chrono::{NaiveTime, TimeDelta}; use crunchyroll_rs::media::{SkipEvents, SkipEventsEvent, StreamData, StreamSegment, Subtitle}; use crunchyroll_rs::Locale; +use image_hasher::{Hasher, HasherConfig, ImageHash}; use indicatif::{ProgressBar, ProgressDrawTarget, ProgressFinish, ProgressStyle}; use log::{debug, warn, LevelFilter}; use regex::Regex; @@ -59,6 +63,7 @@ pub struct DownloadBuilder { force_hardsub: bool, download_fonts: bool, no_closed_caption: bool, + sync_start_value: Option, threads: usize, ffmpeg_threads: Option, audio_locale_output_map: HashMap, @@ -78,6 +83,7 @@ impl DownloadBuilder { force_hardsub: false, download_fonts: false, no_closed_caption: false, + sync_start_value: None, threads: num_cpus::get(), ffmpeg_threads: None, audio_locale_output_map: HashMap::new(), @@ -99,6 +105,8 @@ impl DownloadBuilder { download_fonts: self.download_fonts, no_closed_caption: self.no_closed_caption, + sync_start_value: self.sync_start_value, + download_threads: self.threads, ffmpeg_threads: self.ffmpeg_threads, @@ -110,10 +118,23 @@ impl DownloadBuilder { } } -struct FFmpegMeta { +struct FFmpegVideoMeta { path: TempPath, - language: Locale, - title: String, + length: TimeDelta, + start_time: Option, +} + +struct FFmpegAudioMeta { + path: TempPath, + locale: Locale, + start_time: Option, +} + +struct FFmpegSubtitleMeta { + path: TempPath, + locale: Locale, + cc: bool, + start_time: Option, } pub struct DownloadFormat { @@ -141,6 +162,8 @@ pub struct Downloader { download_fonts: bool, no_closed_caption: bool, + sync_start_value: Option, + download_threads: usize, ffmpeg_threads: Option, @@ -216,13 +239,16 @@ impl Downloader { } } + let mut video_offset = None; + let mut audio_offsets = HashMap::new(); + let mut subtitle_offsets = HashMap::new(); let mut videos = vec![]; let mut audios = vec![]; let mut subtitles = vec![]; let mut fonts = vec![]; let mut chapters = None; - let mut max_len = NaiveTime::MIN; - let mut max_frames = 0f64; + let mut max_len = TimeDelta::min_value(); + let mut max_frames = 0; let fmt_space = self .formats .iter() @@ -234,115 +260,252 @@ impl Downloader { .max() .unwrap(); - for (i, format) in self.formats.iter().enumerate() { - let video_path = self - .download_video( - &format.video.0, - format!("{:<1$}", format!("Downloading video #{}", i + 1), fmt_space), - ) - .await?; - for (variant_data, locale) in format.audios.iter() { - let audio_path = self - .download_audio( - variant_data, - format!("{:<1$}", format!("Downloading {} audio", locale), fmt_space), + if self.formats.len() > 1 && self.sync_start_value.is_some() { + let all_segments_count: Vec = self + .formats + .iter() + .map(|f| f.video.0.segments().len()) + .collect(); + let sync_segments = 11.max( + all_segments_count.iter().max().unwrap() - all_segments_count.iter().min().unwrap(), + ); + let mut sync_vids = vec![]; + for (i, format) in self.formats.iter().enumerate() { + let path = self + .download_video( + &format.video.0, + format!("Downloading video #{} sync segments", i + 1), + Some(sync_segments), ) .await?; - audios.push(FFmpegMeta { - path: audio_path, - language: locale.clone(), - title: if i == 0 { - locale.to_human_readable() - } else { - format!("{} [Video: #{}]", locale.to_human_readable(), i + 1) - }, + sync_vids.push(SyncVideo { + path, + length: len_from_segments(&format.video.0.segments()), + available_frames: (len_from_segments( + &format.video.0.segments()[0..sync_segments], + ) + .num_milliseconds() as f64 + * format.video.0.fps().unwrap() + / 1000.0) as u64, + idx: i, }) } - let (len, fps) = get_video_stats(&video_path)?; + let _progress_handler = + progress!("Syncing video start times (this might take some time)"); + let mut offsets = sync_videos(sync_vids, self.sync_start_value.unwrap())?; + drop(_progress_handler); + + let mut offset_pre_checked = false; + if let Some(tmp_offsets) = &offsets { + let formats_with_offset: Vec = self + .formats + .iter() + .enumerate() + .map(|(i, f)| { + len_from_segments(&f.video.0.segments()) + - TimeDelta::milliseconds( + tmp_offsets + .get(&i) + .map(|o| (*o as f64 / f.video.0.fps().unwrap() * 1000.0) as i64) + .unwrap_or_default(), + ) + }) + .collect(); + let min = formats_with_offset.iter().min().unwrap(); + let max = formats_with_offset.iter().max().unwrap(); + + if max.num_seconds() - min.num_seconds() > 15 { + warn!("Found difference of >15 seconds after sync, skipping applying it"); + offsets = None; + offset_pre_checked = true + } + } + + if let Some(offsets) = offsets { + let mut root_format_idx = 0; + let mut root_format_length = 0; + let mut audio_count: usize = 0; + let mut subtitle_count: usize = 0; + for (i, format) in self.formats.iter().enumerate() { + let format_fps = format.video.0.fps().unwrap(); + let format_len = format + .video + .0 + .segments() + .iter() + .map(|s| s.length.as_millis()) + .sum::() as u64 + - offsets.get(&i).map_or(0, |o| *o); + if format_len > root_format_length { + root_format_idx = i; + root_format_length = format_len; + } + + for _ in &format.audios { + if let Some(offset) = &offsets.get(&i) { + audio_offsets.insert( + audio_count, + TimeDelta::milliseconds( + (**offset as f64 / format_fps * 1000.0) as i64, + ), + ); + } + audio_count += 1 + } + for _ in &format.subtitles { + if let Some(offset) = &offsets.get(&i) { + subtitle_offsets.insert( + subtitle_count, + TimeDelta::milliseconds( + (**offset as f64 / format_fps * 1000.0) as i64, + ), + ); + } + subtitle_count += 1 + } + } + + let mut root_format = self.formats.remove(root_format_idx); + + let mut audio_prepend = vec![]; + let mut subtitle_prepend = vec![]; + let mut audio_append = vec![]; + let mut subtitle_append = vec![]; + for (i, format) in self.formats.into_iter().enumerate() { + if i < root_format_idx { + audio_prepend.extend(format.audios); + subtitle_prepend.extend(format.subtitles); + } else { + audio_append.extend(format.audios); + subtitle_append.extend(format.subtitles); + } + } + root_format.audios.splice(0..0, audio_prepend); + root_format.subtitles.splice(0..0, subtitle_prepend); + root_format.audios.extend(audio_append); + root_format.subtitles.extend(subtitle_append); + + self.formats = vec![root_format]; + video_offset = offsets.get(&root_format_idx).map(|o| { + TimeDelta::milliseconds( + (*o as f64 / self.formats[0].video.0.fps().unwrap() * 1000.0) as i64, + ) + }) + } else if !offset_pre_checked { + warn!("Couldn't find reliable sync positions") + } + } + + // downloads all videos + for (i, format) in self.formats.iter().enumerate() { + let path = self + .download_video( + &format.video.0, + format!("{:<1$}", format!("Downloading video #{}", i + 1), fmt_space), + None, + ) + .await?; + + let (len, fps) = get_video_stats(&path)?; if max_len < len { max_len = len } - let frames = len.signed_duration_since(NaiveTime::MIN).num_seconds() as f64 * fps; - if frames > max_frames { - max_frames = frames; + let frames = ((len.num_milliseconds() as f64 + - video_offset.unwrap_or_default().num_milliseconds() as f64) + / 1000.0 + * fps) as u64; + if max_frames < frames { + max_frames = frames } - if !format.subtitles.is_empty() { - let progress_spinner = if log::max_level() == LevelFilter::Info { - let progress_spinner = ProgressBar::new_spinner() - .with_style( - ProgressStyle::with_template( - format!( - ":: {:<1$} {{msg}} {{spinner}}", - "Downloading subtitles", fmt_space - ) - .as_str(), + videos.push(FFmpegVideoMeta { + path, + length: len, + start_time: video_offset, + }) + } + + // downloads all audios + for format in &self.formats { + for (j, (stream_data, locale)) in format.audios.iter().enumerate() { + let path = self + .download_audio( + stream_data, + format!("{:<1$}", format!("Downloading {} audio", locale), fmt_space), + ) + .await?; + audios.push(FFmpegAudioMeta { + path, + locale: locale.clone(), + start_time: audio_offsets.get(&j).cloned(), + }) + } + } + + for (i, format) in self.formats.iter().enumerate() { + if format.subtitles.is_empty() { + continue; + } + + let progress_spinner = if log::max_level() == LevelFilter::Info { + let progress_spinner = ProgressBar::new_spinner() + .with_style( + ProgressStyle::with_template( + format!( + ":: {:<1$} {{msg}} {{spinner}}", + "Downloading subtitles", fmt_space ) - .unwrap() - .tick_strings(&["—", "\\", "|", "/", ""]), + .as_str(), ) - .with_finish(ProgressFinish::Abandon); - progress_spinner.enable_steady_tick(Duration::from_millis(100)); - Some(progress_spinner) - } else { - None - }; + .unwrap() + .tick_strings(&["—", "\\", "|", "/", ""]), + ) + .with_finish(ProgressFinish::Abandon); + progress_spinner.enable_steady_tick(Duration::from_millis(100)); + Some(progress_spinner) + } else { + None + }; - for (subtitle, not_cc) in format.subtitles.iter() { - if !not_cc && self.no_closed_caption { - continue; - } - - if let Some(pb) = &progress_spinner { - let mut progress_message = pb.message(); - if !progress_message.is_empty() { - progress_message += ", " - } - progress_message += &subtitle.locale.to_string(); - if !not_cc { - progress_message += " (CC)"; - } - if i != 0 { - progress_message += &format!(" [Video: #{}]", i + 1); - } - pb.set_message(progress_message) - } - - let mut subtitle_title = subtitle.locale.to_human_readable(); - if !not_cc { - subtitle_title += " (CC)" - } - if i != 0 { - subtitle_title += &format!(" [Video: #{}]", i + 1) - } - - let subtitle_path = self.download_subtitle(subtitle.clone(), len).await?; - debug!( - "Downloaded {} subtitles{}{}", - subtitle.locale, - (!not_cc).then_some(" (cc)").unwrap_or_default(), - (i != 0) - .then_some(format!(" for video {}", i)) - .unwrap_or_default() - ); - subtitles.push(FFmpegMeta { - path: subtitle_path, - language: subtitle.locale.clone(), - title: subtitle_title, - }) + for (j, (subtitle, not_cc)) in format.subtitles.iter().enumerate() { + if !not_cc && self.no_closed_caption { + continue; } - } - videos.push(FFmpegMeta { - path: video_path, - language: format.video.1.clone(), - title: if self.formats.len() == 1 { - "Default".to_string() - } else { - format!("#{}", i + 1) - }, - }); + if let Some(pb) = &progress_spinner { + let mut progress_message = pb.message(); + if !progress_message.is_empty() { + progress_message += ", " + } + progress_message += &subtitle.locale.to_string(); + if !not_cc { + progress_message += " (CC)"; + } + if i.min(videos.len() - 1) != 0 { + progress_message += &format!(" [Video: #{}]", i + 1); + } + pb.set_message(progress_message) + } + + let path = self + .download_subtitle(subtitle.clone(), videos[i.min(videos.len() - 1)].length) + .await?; + debug!( + "Downloaded {} subtitles{}", + subtitle.locale, + (!not_cc).then_some(" (cc)").unwrap_or_default(), + ); + subtitles.push(FFmpegSubtitleMeta { + path, + locale: subtitle.locale.clone(), + cc: !not_cc, + start_time: subtitle_offsets.get(&j).cloned(), + }) + } + } + + for format in self.formats.iter() { if let Some(skip_events) = &format.metadata.skip_events { let (file, path) = tempfile(".chapter")?.into_parts(); chapters = Some(( @@ -421,17 +584,30 @@ impl Downloader { let mut metadata = vec![]; for (i, meta) in videos.iter().enumerate() { + if let Some(start_time) = meta.start_time { + input.extend(["-ss".to_string(), format_time_delta(start_time)]) + } input.extend(["-i".to_string(), meta.path.to_string_lossy().to_string()]); maps.extend(["-map".to_string(), i.to_string()]); metadata.extend([ format!("-metadata:s:v:{}", i), - format!("title={}", meta.title), + format!( + "title={}", + if videos.len() == 1 { + "Default".to_string() + } else { + format!("#{}", i + 1) + } + ), ]); // the empty language metadata is created to avoid that metadata from the original track // is copied metadata.extend([format!("-metadata:s:v:{}", i), "language=".to_string()]) } for (i, meta) in audios.iter().enumerate() { + if let Some(start_time) = meta.start_time { + input.extend(["-ss".to_string(), format_time_delta(start_time)]) + } input.extend(["-i".to_string(), meta.path.to_string_lossy().to_string()]); maps.extend(["-map".to_string(), (i + videos.len()).to_string()]); metadata.extend([ @@ -439,13 +615,20 @@ impl Downloader { format!( "language={}", self.audio_locale_output_map - .get(&meta.language) - .unwrap_or(&meta.language.to_string()) + .get(&meta.locale) + .unwrap_or(&meta.locale.to_string()) ), ]); metadata.extend([ format!("-metadata:s:a:{}", i), - format!("title={}", meta.title), + format!( + "title={}", + if videos.len() == 1 { + meta.locale.to_human_readable() + } else { + format!("{} [Video: #{}]", meta.locale.to_human_readable(), i + 1,) + } + ), ]); } @@ -465,6 +648,9 @@ impl Downloader { if container_supports_softsubs { for (i, meta) in subtitles.iter().enumerate() { + if let Some(start_time) = meta.start_time { + input.extend(["-ss".to_string(), format_time_delta(start_time)]) + } input.extend(["-i".to_string(), meta.path.to_string_lossy().to_string()]); maps.extend([ "-map".to_string(), @@ -475,13 +661,22 @@ impl Downloader { format!( "language={}", self.subtitle_locale_output_map - .get(&meta.language) - .unwrap_or(&meta.language.to_string()) + .get(&meta.locale) + .unwrap_or(&meta.locale.to_string()) ), ]); metadata.extend([ format!("-metadata:s:s:{}", i), - format!("title={}", meta.title), + format!("title={}", { + let mut title = meta.locale.to_string(); + if meta.cc { + title += " (CC)" + } + if videos.len() > 1 { + title += &format!(" [Video: #{}]", i + 1) + } + title + }), ]); } } @@ -523,10 +718,7 @@ impl Downloader { // set default subtitle if let Some(default_subtitle) = self.default_subtitle { - if let Some(position) = subtitles - .iter() - .position(|m| m.language == default_subtitle) - { + if let Some(position) = subtitles.iter().position(|m| m.locale == default_subtitle) { if container_supports_softsubs { match dst.extension().unwrap_or_default().to_str().unwrap() { "mov" | "mp4" => output_presets.extend([ @@ -585,7 +777,7 @@ impl Downloader { if container_supports_softsubs { if let Some(position) = subtitles .iter() - .position(|meta| meta.language == default_subtitle) + .position(|meta| meta.locale == default_subtitle) { command_args.extend([ format!("-disposition:s:s:{}", position), @@ -597,9 +789,7 @@ impl Downloader { // set the 'forced' flag to CC subtitles for (i, subtitle) in subtitles.iter().enumerate() { - // well, checking if the title contains '(CC)' might not be the best solutions from a - // performance perspective but easier than adjusting the `FFmpegMeta` struct - if !subtitle.title.contains("(CC)") { + if !subtitle.cc { continue; } @@ -632,7 +822,7 @@ impl Downloader { // create parent directory if it does not exist if let Some(parent) = dst.parent() { if !parent.exists() { - std::fs::create_dir_all(parent)? + fs::create_dir_all(parent)? } } @@ -650,7 +840,7 @@ impl Downloader { let ffmpeg_progress_cancellation_token = ffmpeg_progress_cancel.clone(); let ffmpeg_progress = tokio::spawn(async move { ffmpeg_progress( - max_frames as u64, + max_frames, fifo, format!("{:<1$}", "Generating output file", fmt_space + 1), ffmpeg_progress_cancellation_token, @@ -681,7 +871,7 @@ impl Downloader { let segments = stream_data.segments(); // sum the length of all streams up - estimated_required_space += estimate_variant_file_size(stream_data, &segments); + estimated_required_space += estimate_stream_data_file_size(stream_data, &segments); } let tmp_stat = fs2::statvfs(temp_directory()).unwrap(); @@ -727,11 +917,16 @@ impl Downloader { Ok((tmp_required, dst_required)) } - async fn download_video(&self, stream_data: &StreamData, message: String) -> Result { + async fn download_video( + &self, + stream_data: &StreamData, + message: String, + max_segments: Option, + ) -> Result { let tempfile = tempfile(".mp4")?; let (mut file, path) = tempfile.into_parts(); - self.download_segments(&mut file, message, stream_data) + self.download_segments(&mut file, message, stream_data, max_segments) .await?; Ok(path) @@ -741,7 +936,7 @@ impl Downloader { let tempfile = tempfile(".m4a")?; let (mut file, path) = tempfile.into_parts(); - self.download_segments(&mut file, message, stream_data) + self.download_segments(&mut file, message, stream_data, None) .await?; Ok(path) @@ -750,7 +945,7 @@ impl Downloader { async fn download_subtitle( &self, subtitle: Subtitle, - max_length: NaiveTime, + max_length: TimeDelta, ) -> Result { let tempfile = tempfile(".ass")?; let (mut file, path) = tempfile.into_parts(); @@ -796,14 +991,20 @@ impl Downloader { writer: &mut impl Write, message: String, stream_data: &StreamData, + max_segments: Option, ) -> Result<()> { - let segments = stream_data.segments(); + let mut segments = stream_data.segments(); + if let Some(max_segments) = max_segments { + segments = segments + .drain(0..max_segments.min(segments.len() - 1)) + .collect(); + } let total_segments = segments.len(); let count = Arc::new(Mutex::new(0)); let progress = if log::max_level() == LevelFilter::Info { - let estimated_file_size = estimate_variant_file_size(stream_data, &segments); + let estimated_file_size = estimate_stream_data_file_size(stream_data, &segments); let progress = ProgressBar::new(estimated_file_size) .with_style( @@ -820,7 +1021,7 @@ impl Downloader { None }; - let cpus = self.download_threads; + let cpus = self.download_threads.min(segments.len()); let mut segs: Vec> = Vec::with_capacity(cpus); for _ in 0..cpus { segs.push(vec![]) @@ -964,12 +1165,12 @@ impl Downloader { } } -fn estimate_variant_file_size(stream_data: &StreamData, segments: &[StreamSegment]) -> u64 { +fn estimate_stream_data_file_size(stream_data: &StreamData, segments: &[StreamSegment]) -> u64 { (stream_data.bandwidth / 8) * segments.iter().map(|s| s.length.as_secs()).sum::() } /// Get the length and fps of a video. -fn get_video_stats(path: &Path) -> Result<(NaiveTime, f64)> { +fn get_video_stats(path: &Path) -> Result<(TimeDelta, f64)> { let video_length = Regex::new(r"Duration:\s(?P