From 4bfc6f22e111ff53b6ad73811bbab692b938331c Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 18 Dec 2022 13:50:20 +0100 Subject: [PATCH 001/363] Fix discord link --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 21f84a6..8008de2 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ A [Rust](https://www.rust-lang.org/) written cli client for [Crunchyroll](https: Release - - Discord + + Discord CI From 67bbc00d87c3fd5c0f64f6ae537a331bc5b54702 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 18 Dec 2022 15:03:16 +0100 Subject: [PATCH 002/363] Add pre-release notice --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 8008de2..bf57145 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,10 @@ A [Rust](https://www.rust-lang.org/) written cli client for [Crunchyroll](https: > We are in no way affiliated with, maintained, authorized, sponsored, or officially associated with Crunchyroll LLC or any of its subsidiaries or affiliates. > The official Crunchyroll website can be found at https://crunchyroll.com/. +> This README belongs to the _master_ branch which is currently under heavy development towards the next major version (3.0). +> It is mostly stable but some issues may still occur. +> If you do not want to use an under-development / pre-release version, head over to the _[golang](https://github.com/crunchy-labs/crunchy-cli/tree/golang)_ branch which contains the EOL but last stable version (and documentation for it). + ## ✨ Features - Download single videos and entire series from [Crunchyroll](https://www.crunchyroll.com). From 8bb2c9c7501cf4382d3b0117d55e4f6c7b42d93a Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 19 Dec 2022 15:22:45 +0100 Subject: [PATCH 003/363] Fix file name sanitizing --- crunchy-cli-core/src/cli/archive.rs | 1 + crunchy-cli-core/src/cli/download.rs | 1 + crunchy-cli-core/src/utils/format.rs | 31 ++++++++++++++++++---------- crunchy-cli-core/src/utils/os.rs | 9 +------- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 5aabbd8..a551bd1 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -237,6 +237,7 @@ impl Execute for Archive { } .to_string(), primary, + true, )), ); diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index 31f8cda..e13d3f0 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -209,6 +209,7 @@ impl Execute for Download { } .to_string(), &format, + true, )), ); diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index 3dd7f74..b111dce 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -63,15 +63,24 @@ impl Format { } } -pub fn format_string(s: String, format: &Format) -> String { - s.replace("{title}", &format.title) - .replace("{series_name}", &format.series_name) - .replace("{season_name}", &format.season_title) - .replace("{audio}", &format.audio.to_string()) - .replace("{resolution}", &format.stream.resolution.to_string()) - .replace("{season_number}", &format.season_number.to_string()) - .replace("{episode_number}", &format.number.to_string()) - .replace("{series_id}", &format.series_id) - .replace("{season_id}", &format.season_id) - .replace("{episode_id}", &format.id) +/// Formats the given string if it has specific pattern in it. It's possible to sanitize it which +/// removes characters which can cause failures if the output string is used as a file name. +pub fn format_string(s: String, format: &Format, sanitize: bool) -> String { + let sanitize_func = if sanitize { + |s: &str| sanitize_filename::sanitize(s) + } else { + // converting this to a string is actually unnecessary + |s: &str| s.to_string() + }; + + s.replace("{title}", &sanitize_func(&format.title)) + .replace("{series_name}", &sanitize_func(&format.series_name)) + .replace("{season_name}", &sanitize_func(&format.season_title)) + .replace("{audio}", &sanitize_func(&format.audio.to_string())) + .replace("{resolution}", &sanitize_func(&format.stream.resolution.to_string())) + .replace("{season_number}", &sanitize_func(&format.season_number.to_string())) + .replace("{episode_number}", &sanitize_func(&format.number.to_string())) + .replace("{series_id}", &sanitize_func(&format.series_id)) + .replace("{season_id}", &sanitize_func(&format.season_id)) + .replace("{episode_id}", &sanitize_func(&format.id)) } diff --git a/crunchy-cli-core/src/utils/os.rs b/crunchy-cli-core/src/utils/os.rs index abdf151..01d0f89 100644 --- a/crunchy-cli-core/src/utils/os.rs +++ b/crunchy-cli-core/src/utils/os.rs @@ -50,12 +50,5 @@ pub fn free_file(mut path: PathBuf) -> PathBuf { path.set_file_name(format!("{} ({}).{}", filename, i, ext)) } - sanitize_file(path) -} - -/// Sanitizes the given path to not contain any invalid file character. -pub fn sanitize_file(path: PathBuf) -> PathBuf { - path.with_file_name(sanitize_filename::sanitize( - path.file_name().unwrap().to_string_lossy(), - )) + path } From af9aca4d0cfc3608369507766d63ca3e40fe449a Mon Sep 17 00:00:00 2001 From: Alexandru Dracea Date: Mon, 19 Dec 2022 18:35:37 +0200 Subject: [PATCH 004/363] Add padding --- crunchy-cli-core/src/cli/archive.rs | 22 ++++++++++++---------- crunchy-cli-core/src/cli/download.rs | 22 ++++++++++++---------- crunchy-cli-core/src/utils/format.rs | 23 ++++++++++++++++++++--- 3 files changed, 44 insertions(+), 23 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index a551bd1..6e56e81 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -58,16 +58,18 @@ pub struct Archive { #[arg(help = "Name of the output file")] #[arg(long_help = "Name of the output file.\ If you use one of the following pattern they will get replaced:\n \ - {title} → Title of the video\n \ - {series_name} → Name of the series\n \ - {season_name} → Name of the season\n \ - {audio} → Audio language of the video\n \ - {resolution} → Resolution of the video\n \ - {season_number} → Number of the season\n \ - {episode_number} → Number of the episode\n \ - {series_id} → ID of the series\n \ - {season_id} → ID of the season\n \ - {episode_id} → ID of the episode")] + {title} → Title of the video\n \ + {series_name} → Name of the series\n \ + {season_name} → Name of the season\n \ + {audio} → Audio language of the video\n \ + {resolution} → Resolution of the video\n \ + {padded_season_number} → Number of the season padded to double digits\n \ + {season_number} → Number of the season\n \ + {padded_episode_number} → Number of the episode padded to double digits\n \ + {episode_number} → Number of the episode\n \ + {series_id} → ID of the series\n \ + {season_id} → ID of the season\n \ + {episode_id} → ID of the episode")] #[arg(short, long, default_value = "{title}.mkv")] output: String, diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index e13d3f0..4e4eb3c 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -36,16 +36,18 @@ pub struct Download { #[arg(help = "Name of the output file")] #[arg(long_help = "Name of the output file.\ If you use one of the following pattern they will get replaced:\n \ - {title} → Title of the video\n \ - {series_name} → Name of the series\n \ - {season_name} → Name of the season\n \ - {audio} → Audio language of the video\n \ - {resolution} → Resolution of the video\n \ - {season_number} → Number of the season\n \ - {episode_number} → Number of the episode\n \ - {series_id} → ID of the series\n \ - {season_id} → ID of the season\n \ - {episode_id} → ID of the episode")] + {title} → Title of the video\n \ + {series_name} → Name of the series\n \ + {season_name} → Name of the season\n \ + {audio} → Audio language of the video\n \ + {resolution} → Resolution of the video\n \ + {padded_season_number} → Number of the season padded to double digits\n \ + {season_number} → Number of the season\n \ + {padded_episode_number} → Number of the episode padded to double digits\n \ + {episode_number} → Number of the episode\n \ + {series_id} → ID of the series\n \ + {season_id} → ID of the season\n \ + {episode_id} → ID of the episode")] #[arg(short, long, default_value = "{title}.ts")] output: String, diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index b111dce..c463ea8 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -77,9 +77,26 @@ pub fn format_string(s: String, format: &Format, sanitize: bool) -> String { .replace("{series_name}", &sanitize_func(&format.series_name)) .replace("{season_name}", &sanitize_func(&format.season_title)) .replace("{audio}", &sanitize_func(&format.audio.to_string())) - .replace("{resolution}", &sanitize_func(&format.stream.resolution.to_string())) - .replace("{season_number}", &sanitize_func(&format.season_number.to_string())) - .replace("{episode_number}", &sanitize_func(&format.number.to_string())) + .replace( + "{resolution}", + &sanitize_func(&format.stream.resolution.to_string()), + ) + .replace( + "{padded_season_number}", + &sanitize_func(&format!("{:0>2}", format.season_number.to_string())), + ) + .replace( + "{season_number}", + &sanitize_func(&format.season_number.to_string()), + ) + .replace( + "{padded_episode_number}", + &sanitize_func(&format!("{:0>2}", format.number.to_string())), + ) + .replace( + "{episode_number}", + &sanitize_func(&format.number.to_string()), + ) .replace("{series_id}", &sanitize_func(&format.series_id)) .replace("{season_id}", &sanitize_func(&format.season_id)) .replace("{episode_id}", &sanitize_func(&format.id)) From 17fa045c32e103a13535a819dad4da6b6731347a Mon Sep 17 00:00:00 2001 From: ByteDream Date: Thu, 22 Dec 2022 14:45:56 +0100 Subject: [PATCH 005/363] Use library for progress --- Cargo.lock | 130 ++++++++++++++++++--------- crunchy-cli-core/Cargo.lock | 130 ++++++++++++++++++--------- crunchy-cli-core/Cargo.toml | 1 + crunchy-cli-core/src/cli/archive.rs | 33 ++++--- crunchy-cli-core/src/cli/download.rs | 28 +++--- crunchy-cli-core/src/cli/log.rs | 130 +++++++-------------------- crunchy-cli-core/src/cli/utils.rs | 127 +++++++++----------------- crunchy-cli-core/src/lib.rs | 6 +- crunchy-cli-core/src/utils/log.rs | 17 +++- 9 files changed, 307 insertions(+), 295 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f205883..a7432c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -33,9 +33,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.67" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7724808837b77f4b4de9d283820f9d98bcf496d5692934b857a2399d31ff22e6" +checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" [[package]] name = "async-trait" @@ -148,9 +148,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.29" +version = "4.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d63b9e9c07271b9957ad22c173bae2a4d9a81127680962039296abcd2f8251d" +checksum = "656ad1e55e23d287773f7d8192c300dc715c3eeded93b3da651d11c42cfd74d2" dependencies = [ "bitflags", "clap_derive", @@ -212,6 +212,20 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "console" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c050367d967ced717c04b65d8c619d863ef9292ce0c5760028655a2fb298718c" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "terminal_size 0.1.17", + "unicode-width", + "winapi", +] + [[package]] name = "cookie" version = "0.16.2" @@ -288,6 +302,7 @@ dependencies = [ "csv", "ctrlc", "dirs", + "indicatif", "log", "num_cpus", "regex", @@ -297,7 +312,7 @@ dependencies = [ "signal-hook", "sys-locale", "tempfile", - "terminal_size", + "terminal_size 0.2.3", "tokio", ] @@ -379,9 +394,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.84" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27874566aca772cb515af4c6e997b5fe2119820bca447689145e39bb734d19a0" +checksum = "5add3fc1717409d029b20c5b6903fc0c0b02fa6741d820054f4a2efa5e5816fd" dependencies = [ "cc", "cxxbridge-flags", @@ -391,9 +406,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.84" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7bb951f2523a49533003656a72121306b225ec16a49a09dc6b0ba0d6f3ec3c0" +checksum = "b4c87959ba14bc6fbc61df77c3fcfe180fc32b93538c4f1031dd802ccb5f2ff0" dependencies = [ "cc", "codespan-reporting", @@ -406,15 +421,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.84" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be778b6327031c1c7b61dd2e48124eee5361e6aa76b8de93692f011b08870ab4" +checksum = "69a3e162fde4e594ed2b07d0f83c6c67b745e7f28ce58c6df5e6b6bef99dfb59" [[package]] name = "cxxbridge-macro" -version = "1.0.84" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b8a2b87662fe5a0a0b38507756ab66aff32638876a0866e5a5fc82ceb07ee49" +checksum = "3e7e2adeb6a0d4a282e581096b06e1791532b7d576dcde5ccd9382acf55db8e6" dependencies = [ "proc-macro2", "quote", @@ -476,6 +491,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" version = "0.8.31" @@ -645,15 +666,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.2.6" @@ -808,6 +820,18 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "indicatif" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4295cbb7573c16d310e99e713cf9e75101eb190ab31fccd35f2d2691b4352b19" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "unicode-width", +] + [[package]] name = "inout" version = "0.1.3" @@ -845,11 +869,11 @@ checksum = "11b0d96e660696543b251e58030cf9787df56da39dab19ad60eae7353040917e" [[package]] name = "is-terminal" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927609f78c2913a6f6ac3c27a4fe87f43e2a35367c0c4b0f8265e8f49a104330" +checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi", "io-lifetimes", "rustix", "windows-sys 0.42.0", @@ -1019,14 +1043,20 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi 0.1.19", + "hermit-abi", "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "once_cell" version = "1.16.0" @@ -1035,9 +1065,9 @@ checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" [[package]] name = "openssl" -version = "0.10.44" +version = "0.10.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29d971fd5722fec23977260f6e81aa67d2f22cadbdc2aa049f1022d9a3be1566" +checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" dependencies = [ "bitflags", "cfg-if", @@ -1067,9 +1097,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.79" +version = "0.9.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5454462c0eced1e97f2ec09036abc8da362e66802f66fd20f86854d9d8cbcbc4" +checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" dependencies = [ "autocfg", "cc", @@ -1108,6 +1138,12 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +[[package]] +name = "portable-atomic" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bdd679d533107e090c2704a35982fc06302e30898e63ffa26a81155c012e92" + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -1134,15 +1170,15 @@ dependencies = [ [[package]] name = "proc-macro-hack" -version = "0.5.19" +version = "0.5.20+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" +checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.48" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9d89e5dba24725ae5678020bf8f1357a9aa7ff10736b551adbcd3f8d17d766f" +checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" dependencies = [ "unicode-ident", ] @@ -1165,9 +1201,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556d0f47a940e895261e77dc200d5eadfc6ef644c179c6f5edfc105e3a2292c8" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" dependencies = [ "proc-macro2", ] @@ -1412,9 +1448,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.90" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8778cc0b528968fe72abec38b5db5a20a70d148116cd9325d2bc5f5180ca3faf" +checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" dependencies = [ "itoa 1.0.5", "ryu", @@ -1502,9 +1538,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ee3a69cd2c7e06684677e5629b3878b253af05e4714964204279c6bc02cf0b" +checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" dependencies = [ "proc-macro2", "quote", @@ -1547,6 +1583,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "terminal_size" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "terminal_size" version = "0.2.3" diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index a85defe..8347aa7 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -33,9 +33,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.67" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7724808837b77f4b4de9d283820f9d98bcf496d5692934b857a2399d31ff22e6" +checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" [[package]] name = "async-trait" @@ -148,9 +148,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.29" +version = "4.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d63b9e9c07271b9957ad22c173bae2a4d9a81127680962039296abcd2f8251d" +checksum = "656ad1e55e23d287773f7d8192c300dc715c3eeded93b3da651d11c42cfd74d2" dependencies = [ "bitflags", "clap_derive", @@ -193,6 +193,20 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "console" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c050367d967ced717c04b65d8c619d863ef9292ce0c5760028655a2fb298718c" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "terminal_size 0.1.17", + "unicode-width", + "winapi", +] + [[package]] name = "cookie" version = "0.16.2" @@ -257,6 +271,7 @@ dependencies = [ "csv", "ctrlc", "dirs", + "indicatif", "log", "num_cpus", "regex", @@ -266,7 +281,7 @@ dependencies = [ "signal-hook", "sys-locale", "tempfile", - "terminal_size", + "terminal_size 0.2.3", "tokio", ] @@ -348,9 +363,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.84" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27874566aca772cb515af4c6e997b5fe2119820bca447689145e39bb734d19a0" +checksum = "5add3fc1717409d029b20c5b6903fc0c0b02fa6741d820054f4a2efa5e5816fd" dependencies = [ "cc", "cxxbridge-flags", @@ -360,9 +375,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.84" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7bb951f2523a49533003656a72121306b225ec16a49a09dc6b0ba0d6f3ec3c0" +checksum = "b4c87959ba14bc6fbc61df77c3fcfe180fc32b93538c4f1031dd802ccb5f2ff0" dependencies = [ "cc", "codespan-reporting", @@ -375,15 +390,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.84" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be778b6327031c1c7b61dd2e48124eee5361e6aa76b8de93692f011b08870ab4" +checksum = "69a3e162fde4e594ed2b07d0f83c6c67b745e7f28ce58c6df5e6b6bef99dfb59" [[package]] name = "cxxbridge-macro" -version = "1.0.84" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b8a2b87662fe5a0a0b38507756ab66aff32638876a0866e5a5fc82ceb07ee49" +checksum = "3e7e2adeb6a0d4a282e581096b06e1791532b7d576dcde5ccd9382acf55db8e6" dependencies = [ "proc-macro2", "quote", @@ -445,6 +460,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" version = "0.8.31" @@ -614,15 +635,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.2.6" @@ -777,6 +789,18 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "indicatif" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4295cbb7573c16d310e99e713cf9e75101eb190ab31fccd35f2d2691b4352b19" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "unicode-width", +] + [[package]] name = "inout" version = "0.1.3" @@ -814,11 +838,11 @@ checksum = "11b0d96e660696543b251e58030cf9787df56da39dab19ad60eae7353040917e" [[package]] name = "is-terminal" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927609f78c2913a6f6ac3c27a4fe87f43e2a35367c0c4b0f8265e8f49a104330" +checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi", "io-lifetimes", "rustix", "windows-sys 0.42.0", @@ -988,14 +1012,20 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi 0.1.19", + "hermit-abi", "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "once_cell" version = "1.16.0" @@ -1004,9 +1034,9 @@ checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" [[package]] name = "openssl" -version = "0.10.44" +version = "0.10.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29d971fd5722fec23977260f6e81aa67d2f22cadbdc2aa049f1022d9a3be1566" +checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" dependencies = [ "bitflags", "cfg-if", @@ -1036,9 +1066,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.79" +version = "0.9.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5454462c0eced1e97f2ec09036abc8da362e66802f66fd20f86854d9d8cbcbc4" +checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" dependencies = [ "autocfg", "cc", @@ -1077,6 +1107,12 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +[[package]] +name = "portable-atomic" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bdd679d533107e090c2704a35982fc06302e30898e63ffa26a81155c012e92" + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -1103,15 +1139,15 @@ dependencies = [ [[package]] name = "proc-macro-hack" -version = "0.5.19" +version = "0.5.20+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" +checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.48" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9d89e5dba24725ae5678020bf8f1357a9aa7ff10736b551adbcd3f8d17d766f" +checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" dependencies = [ "unicode-ident", ] @@ -1134,9 +1170,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556d0f47a940e895261e77dc200d5eadfc6ef644c179c6f5edfc105e3a2292c8" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" dependencies = [ "proc-macro2", ] @@ -1375,9 +1411,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.90" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8778cc0b528968fe72abec38b5db5a20a70d148116cd9325d2bc5f5180ca3faf" +checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" dependencies = [ "itoa 1.0.5", "ryu", @@ -1465,9 +1501,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ee3a69cd2c7e06684677e5629b3878b253af05e4714964204279c6bc02cf0b" +checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" dependencies = [ "proc-macro2", "quote", @@ -1510,6 +1546,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "terminal_size" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "terminal_size" version = "0.2.3" diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 6dccd4d..780716f 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -13,6 +13,7 @@ crunchyroll-rs = "0.2" csv = "1.1" ctrlc = "3.2" dirs = "4.0" +indicatif = "0.17" log = { version = "0.4", features = ["std"] } num_cpus = "1.14" regex = "1.7" diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 6e56e81..b5244f7 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -1,5 +1,5 @@ use crate::cli::log::tab_info; -use crate::cli::utils::{download_segments, FFmpegPreset, find_resolution}; +use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset}; use crate::utils::context::Context; use crate::utils::format::{format_string, Format}; use crate::utils::log::progress; @@ -137,7 +137,9 @@ impl Execute for Archive { bail!("File extension is not '.mkv'. Currently only matroska / '.mkv' files are supported") } let _ = FFmpegPreset::ffmpeg_presets(self.ffmpeg_preset.clone())?; - if self.ffmpeg_preset.len() == 1 && self.ffmpeg_preset.get(0).unwrap() == &FFmpegPreset::Nvidia { + if self.ffmpeg_preset.len() == 1 + && self.ffmpeg_preset.get(0).unwrap() == &FFmpegPreset::Nvidia + { warn!("Skipping 'nvidia' hardware acceleration preset since no other codec preset was specified") } @@ -148,20 +150,20 @@ impl Execute for Archive { let mut parsed_urls = vec![]; for (i, url) in self.urls.iter().enumerate() { - let _progress_handler = progress!("Parsing url {}", i + 1); + let progress_handler = progress!("Parsing url {}", i + 1); match parse_url(&ctx.crunchy, url.clone(), true).await { Ok((media_collection, url_filter)) => { parsed_urls.push((media_collection, url_filter)); - info!("Parsed url {}", i + 1) + progress_handler.stop(format!("Parsed url {}", i + 1)) } Err(e) => bail!("url {} could not be parsed: {}", url, e), } } for (i, (media_collection, url_filter)) in parsed_urls.into_iter().enumerate() { + let progress_handler = progress!("Fetching series details"); let archive_formats = match media_collection { MediaCollection::Series(series) => { - let _progress_handler = progress!("Fetching series details"); formats_from_series(&self, series, &url_filter).await? } MediaCollection::Season(_) => bail!("Archiving a season is not supported"), @@ -171,10 +173,13 @@ impl Execute for Archive { }; if archive_formats.is_empty() { - info!("Skipping url {} (no matching episodes found)", i + 1); + progress_handler.stop(format!( + "Skipping url {} (no matching episodes found)", + i + 1 + )); continue; } - info!("Loaded series information for url {}", i + 1); + progress_handler.stop(format!("Loaded series information for url {}", i + 1)); if log::max_level() == log::Level::Debug { let seasons = sort_formats_after_seasons( @@ -310,9 +315,9 @@ impl Execute for Archive { )) } - let _progess_handler = progress!("Generating mkv"); + let progess_handler = progress!("Generating mkv"); generate_mkv(&self, path, video_paths, audio_paths, subtitle_paths)?; - info!("Mkv generated") + progess_handler.stop("Mkv generated") } } @@ -349,7 +354,10 @@ async fn formats_from_series( // remove all seasons with the wrong audio for the current iterated season number seasons.retain(|s| { s.metadata.season_number != season.first().unwrap().metadata.season_number - || archive.locale.iter().any(|l| s.metadata.audio_locales.contains(l)) + || archive + .locale + .iter() + .any(|l| s.metadata.audio_locales.contains(l)) }) } @@ -358,7 +366,10 @@ async fn formats_from_series( BTreeMap::new(); for season in series.seasons().await? { if !url_filter.is_season_valid(season.metadata.season_number) - || !archive.locale.iter().any(|l| season.metadata.audio_locales.contains(l)) + || !archive + .locale + .iter() + .any(|l| season.metadata.audio_locales.contains(l)) { continue; } diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index 4e4eb3c..d8a4ada 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -1,5 +1,5 @@ use crate::cli::log::tab_info; -use crate::cli::utils::{download_segments, FFmpegPreset, find_resolution}; +use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset}; use crate::utils::context::Context; use crate::utils::format::{format_string, Format}; use crate::utils::log::progress; @@ -92,7 +92,9 @@ impl Execute for Download { } let _ = FFmpegPreset::ffmpeg_presets(self.ffmpeg_preset.clone())?; - if self.ffmpeg_preset.len() == 1 && self.ffmpeg_preset.get(0).unwrap() == &FFmpegPreset::Nvidia { + if self.ffmpeg_preset.len() == 1 + && self.ffmpeg_preset.get(0).unwrap() == &FFmpegPreset::Nvidia + { warn!("Skipping 'nvidia' hardware acceleration preset since no other codec preset was specified") } @@ -103,18 +105,18 @@ impl Execute for Download { let mut parsed_urls = vec![]; for (i, url) in self.urls.iter().enumerate() { - let _progress_handler = progress!("Parsing url {}", i + 1); + let progress_handler = progress!("Parsing url {}", i + 1); match parse_url(&ctx.crunchy, url.clone(), true).await { Ok((media_collection, url_filter)) => { parsed_urls.push((media_collection, url_filter)); - info!("Parsed url {}", i + 1) + progress_handler.stop(format!("Parsed url {}", i + 1)) } Err(e) => bail!("url {} could not be parsed: {}", url, e), } } for (i, (media_collection, url_filter)) in parsed_urls.into_iter().enumerate() { - let _progress_handler = progress!("Fetching series details"); + let progress_handler = progress!("Fetching series details"); let formats = match media_collection { MediaCollection::Series(series) => { debug!("Url {} is series ({})", i + 1, series.title); @@ -156,11 +158,10 @@ impl Execute for Download { }; let Some(formats) = formats else { - info!("Skipping url {} (no matching episodes found)", i + 1); + progress_handler.stop(format!("Skipping url {} (no matching episodes found)", i + 1)); continue; }; - info!("Loaded series information for url {}", i + 1); - drop(_progress_handler); + progress_handler.stop(format!("Loaded series information for url {}", i + 1)); if log::max_level() == log::Level::Debug { let seasons = sort_formats_after_seasons(formats.clone()); @@ -231,7 +232,9 @@ impl Execute for Download { tab_info!("Resolution: {}", format.stream.resolution); tab_info!("FPS: {:.2}", format.stream.fps); - if path.extension().unwrap_or_default().to_string_lossy() != "ts" || !self.ffmpeg_preset.is_empty() { + if path.extension().unwrap_or_default().to_string_lossy() != "ts" + || !self.ffmpeg_preset.is_empty() + { download_ffmpeg(&ctx, &self, format.stream, path.as_path()).await?; } else if path.to_str().unwrap() == "-" { let mut stdout = std::io::stdout().lock(); @@ -247,7 +250,12 @@ impl Execute for Download { } } -async fn download_ffmpeg(ctx: &Context, download: &Download, variant_data: VariantData, target: &Path) -> Result<()> { +async fn download_ffmpeg( + ctx: &Context, + download: &Download, + variant_data: VariantData, + target: &Path, +) -> Result<()> { let (input_presets, output_presets) = FFmpegPreset::ffmpeg_presets(download.ffmpeg_preset.clone())?; diff --git a/crunchy-cli-core/src/cli/log.rs b/crunchy-cli-core/src/cli/log.rs index d1f0225..377685b 100644 --- a/crunchy-cli-core/src/cli/log.rs +++ b/crunchy-cli-core/src/cli/log.rs @@ -1,106 +1,17 @@ +use indicatif::{ProgressBar, ProgressStyle}; use log::{ set_boxed_logger, set_max_level, Level, LevelFilter, Log, Metadata, Record, SetLoggerError, }; use std::io::{stdout, Write}; -use std::sync::{mpsc, Mutex}; +use std::sync::Mutex; use std::thread; -use std::thread::JoinHandle; use std::time::Duration; -struct CliProgress { - handler: JoinHandle<()>, - sender: mpsc::SyncSender<(String, Level)>, -} - -impl CliProgress { - fn new(record: &Record) -> Self { - let (tx, rx) = mpsc::sync_channel(1); - - let init_message = format!("{}", record.args()); - let init_level = record.level(); - let handler = thread::spawn(move || { - #[cfg(not(windows))] - let ok = '✔'; - #[cfg(windows)] - // windows does not support all unicode characters by default in their consoles, so - // we're using this (square root?) symbol instead. microsoft. - let ok = '√'; - let states = ["-", "\\", "|", "/"]; - - let mut old_message = init_message.clone(); - let mut latest_info_message = init_message; - let mut old_level = init_level; - for i in 0.. { - let (msg, level) = match rx.try_recv() { - Ok(payload) => payload, - Err(e) => match e { - mpsc::TryRecvError::Empty => (old_message.clone(), old_level), - mpsc::TryRecvError::Disconnected => break, - }, - }; - - // clear last line - // prefix (2), space (1), state (1), space (1), message(n) - let _ = write!(stdout(), "\r {}", " ".repeat(old_message.len())); - - if old_level != level || old_message != msg { - if old_level <= Level::Warn { - let _ = writeln!(stdout(), "\r:: • {}", old_message); - } else if old_level == Level::Info && level <= Level::Warn { - let _ = writeln!(stdout(), "\r:: → {}", old_message); - } else if level == Level::Info { - latest_info_message = msg.clone(); - } - } - - let _ = write!( - stdout(), - "\r:: {} {}", - states[i / 2 % states.len()], - if level == Level::Info { - &msg - } else { - &latest_info_message - } - ); - let _ = stdout().flush(); - - old_message = msg; - old_level = level; - - thread::sleep(Duration::from_millis(100)); - } - - // clear last line - // prefix (2), space (1), state (1), space (1), message(n) - let _ = write!(stdout(), "\r {}", " ".repeat(old_message.len())); - let _ = writeln!(stdout(), "\r:: {} {}", ok, old_message); - let _ = stdout().flush(); - }); - - Self { - handler, - sender: tx, - } - } - - fn send(&self, record: &Record) { - let _ = self - .sender - .send((format!("{}", record.args()), record.level())); - } - - fn stop(self) { - drop(self.sender); - let _ = self.handler.join(); - } -} - #[allow(clippy::type_complexity)] pub struct CliLogger { all: bool, level: LevelFilter, - progress: Mutex>, + progress: Mutex>, } impl Log for CliLogger { @@ -127,7 +38,7 @@ impl Log for CliLogger { "progress_end" => self.progress(record, true), _ => { if self.progress.lock().unwrap().is_some() { - self.progress(record, false); + self.progress(record, false) } else if record.level() > Level::Warn { self.normal(record) } else { @@ -182,13 +93,34 @@ impl CliLogger { } fn progress(&self, record: &Record, stop: bool) { - let mut progress_option = self.progress.lock().unwrap(); - if stop && progress_option.is_some() { - progress_option.take().unwrap().stop() - } else if let Some(p) = &*progress_option { - p.send(record); + let mut progress = self.progress.lock().unwrap(); + + let msg = format!("{}", record.args()); + if stop && progress.is_some() { + if msg.is_empty() { + progress.take().unwrap().finish() + } else { + progress.take().unwrap().finish_with_message(msg) + } + } else if let Some(p) = &*progress { + p.println(format!(":: → {}", msg)) } else { - *progress_option = Some(CliProgress::new(record)) + #[cfg(not(windows))] + let finish_str = "✔"; + #[cfg(windows)] + // windows does not support all unicode characters by default in their consoles, so + // we're using this (square root?) symbol instead. microsoft. + let finish_str = "√"; + + let pb = ProgressBar::new_spinner(); + pb.set_style( + ProgressStyle::with_template(":: {spinner} {msg}") + .unwrap() + .tick_strings(&["-", "\\", "|", "/", finish_str]), + ); + pb.enable_steady_tick(Duration::from_millis(200)); + pb.set_message(msg); + *progress = Some(pb) } } } diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index a64057a..6045ce4 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -1,13 +1,12 @@ use crate::utils::context::Context; use anyhow::{bail, Result}; use crunchyroll_rs::media::{Resolution, VariantData, VariantSegment}; +use indicatif::{ProgressBar, ProgressFinish, ProgressStyle}; use log::{debug, LevelFilter}; use std::borrow::{Borrow, BorrowMut}; use std::collections::BTreeMap; -use std::io; use std::io::Write; use std::sync::{mpsc, Arc, Mutex}; -use std::time::Duration; use tokio::task::JoinSet; pub fn find_resolution( @@ -35,78 +34,25 @@ pub async fn download_segments( let client = Arc::new(ctx.crunchy.client()); let count = Arc::new(Mutex::new(0)); - let amount = Arc::new(Mutex::new(0)); - // only print progress when log level is info - let output_handler = if log::max_level() == LevelFilter::Info { - let output_count = count.clone(); - let output_amount = amount.clone(); - Some(tokio::spawn(async move { - let sleep_time_ms = 100; - let iter_per_sec = 1000f64 / sleep_time_ms as f64; + let progress = if log::max_level() == LevelFilter::Info { + let estimated_file_size = (variant_data.bandwidth / 8) + * segments + .iter() + .map(|s| s.length.unwrap_or_default().as_secs()) + .sum::(); - let mut bytes_start = 0f64; - let mut speed = 0f64; - let mut percentage = 0f64; - - while *output_count.lock().unwrap() < total_segments || percentage < 100f64 { - let tmp_amount = *output_amount.lock().unwrap() as f64; - - let tmp_speed = (tmp_amount - bytes_start) / 1024f64 / 1024f64; - if *output_count.lock().unwrap() < 3 { - speed = tmp_speed; - } else { - let (old_speed_ratio, new_speed_ratio) = if iter_per_sec <= 1f64 { - (0f64, 1f64) - } else { - (1f64 - (1f64 / iter_per_sec), (1f64 / iter_per_sec)) - }; - - // calculate the average download speed "smoother" - speed = (speed * old_speed_ratio) + (tmp_speed * new_speed_ratio); - } - - percentage = - (*output_count.lock().unwrap() as f64 / total_segments as f64) * 100f64; - - let size = terminal_size::terminal_size() - .unwrap_or((terminal_size::Width(60), terminal_size::Height(0))) - .0 - .0 as usize; - - // there is a offset of 1 "length" (idk how to describe it), so removing 1 from - // `progress_available` would fill the terminal width completely. on multiple - // systems there is a bug that printing until the end of the line causes a newline - // even though technically there shouldn't be one. on my tests, this only happens on - // windows and mac machines and (at the addressed environments) only with release - // builds. so maybe an unwanted optimization? - let progress_available = size - - if let Some(msg) = &message { - 35 + msg.len() - } else { - 34 - }; - let progress_done_count = - (progress_available as f64 * (percentage / 100f64)).ceil() as usize; - let progress_to_do_count = progress_available - progress_done_count; - - let _ = write!( - io::stdout(), - "\r:: {}{:>5.1} MiB {:>5.2} MiB/s [{}{}] {:>3}%", - message.clone().map_or("".to_string(), |msg| msg + " "), - tmp_amount / 1024f64 / 1024f64, - speed * iter_per_sec, - "#".repeat(progress_done_count), - "-".repeat(progress_to_do_count), - percentage as usize - ); - - bytes_start = tmp_amount; - - tokio::time::sleep(Duration::from_millis(sleep_time_ms)).await; - } - println!() - })) + let progress = ProgressBar::new(estimated_file_size) + .with_style( + ProgressStyle::with_template( + ":: {msg}{bytes:>10} {bytes_per_sec:>12} [{wide_bar}] {percent:>3}%", + ) + .unwrap() + .progress_chars("##-"), + ) + .with_message(message.map(|m| m + " ").unwrap_or_default()) + .with_finish(ProgressFinish::Abandon); + Some(progress) } else { None }; @@ -116,7 +62,7 @@ pub async fn download_segments( for _ in 0..cpus { segs.push(vec![]) } - for (i, segment) in segments.into_iter().enumerate() { + for (i, segment) in segments.clone().into_iter().enumerate() { segs[i - ((i / cpus) * cpus)].push(segment); } @@ -127,15 +73,12 @@ pub async fn download_segments( let thread_client = client.clone(); let thread_sender = sender.clone(); let thread_segments = segs.remove(0); - let thread_amount = amount.clone(); let thread_count = count.clone(); join_set.spawn(async move { for (i, segment) in thread_segments.into_iter().enumerate() { let response = thread_client.get(&segment.url).send().await?; let mut buf = response.bytes().await?.to_vec(); - *thread_amount.lock().unwrap() += buf.len(); - buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec(); debug!( "Downloaded and decrypted segment {} ({})", @@ -155,13 +98,28 @@ pub async fn download_segments( let mut buf: BTreeMap> = BTreeMap::new(); loop { // is always `Some` because `sender` does not get dropped when all threads are finished - let data = receiver.recv().unwrap(); + let (pos, bytes) = receiver.recv().unwrap(); - if data_pos == data.0 { - writer.write_all(data.1.borrow())?; + if let Some(p) = &progress { + let progress_len = p.length().unwrap(); + let estimated_segment_len = (variant_data.bandwidth / 8) + * segments + .get(pos) + .unwrap() + .length + .unwrap_or_default() + .as_secs(); + let bytes_len = bytes.len() as u64; + + p.set_length(progress_len - estimated_segment_len + bytes_len); + p.inc(bytes_len) + } + + if data_pos == pos { + writer.write_all(bytes.borrow())?; data_pos += 1; } else { - buf.insert(data.0, data.1); + buf.insert(pos, bytes); } while let Some(b) = buf.remove(&data_pos) { writer.write_all(b.borrow())?; @@ -176,9 +134,6 @@ pub async fn download_segments( while let Some(joined) = join_set.join_next().await { joined?? } - if let Some(handler) = output_handler { - handler.await? - } Ok(()) } @@ -200,7 +155,7 @@ impl ToString for FFmpegPreset { &FFmpegPreset::H265 => "h265", &FFmpegPreset::H264 => "h264", } - .to_string() + .to_string() } } @@ -233,7 +188,9 @@ impl FFmpegPreset { }) } - pub(crate) fn ffmpeg_presets(mut presets: Vec) -> Result<(Vec, Vec)> { + pub(crate) fn ffmpeg_presets( + mut presets: Vec, + ) -> Result<(Vec, Vec)> { fn preset_check_remove(presets: &mut Vec, preset: FFmpegPreset) -> bool { if let Some(i) = presets.iter().position(|p| p == &preset) { presets.remove(i); diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index d669b64..00699e4 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -6,7 +6,7 @@ use anyhow::bail; use anyhow::Result; use clap::{Parser, Subcommand}; use crunchyroll_rs::{Crunchyroll, Locale}; -use log::{debug, error, info, LevelFilter}; +use log::{debug, error, LevelFilter}; use std::{env, fs}; mod cli; @@ -196,7 +196,7 @@ async fn crunchyroll_session(cli: &Cli) -> Result { + cli.login_method.etp_rt.is_some() as u8 + cli.login_method.anonymous as u8; - let _progress_handler = progress!("Logging in"); + let progress_handler = progress!("Logging in"); if login_methods_count == 0 { if let Some(login_file_path) = cli::login::login_file_path() { if login_file_path.exists() { @@ -232,7 +232,7 @@ async fn crunchyroll_session(cli: &Cli) -> Result { bail!("should never happen") }; - info!("Logged in"); + progress_handler.stop("Logged in"); Ok(crunchy) } diff --git a/crunchy-cli-core/src/utils/log.rs b/crunchy-cli-core/src/utils/log.rs index b9fa939..24fed5d 100644 --- a/crunchy-cli-core/src/utils/log.rs +++ b/crunchy-cli-core/src/utils/log.rs @@ -1,10 +1,21 @@ use log::info; -pub struct ProgressHandler; +pub struct ProgressHandler { + pub(crate) stopped: bool, +} impl Drop for ProgressHandler { fn drop(&mut self) { - info!(target: "progress_end", "") + if !self.stopped { + info!(target: "progress_end", "") + } + } +} + +impl ProgressHandler { + pub(crate) fn stop>(mut self, msg: S) { + self.stopped = true; + info!(target: "progress_end", "{}", msg.as_ref()) } } @@ -12,7 +23,7 @@ macro_rules! progress { ($($arg:tt)+) => { { log::info!(target: "progress", $($arg)+); - $crate::utils::log::ProgressHandler{} + $crate::utils::log::ProgressHandler{stopped: false} } } } From 2c3bd78fc1a03450d792733820dddf0d8da92368 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Tue, 27 Dec 2022 20:37:45 +0100 Subject: [PATCH 006/363] Leave special files untouched from renaming --- crunchy-cli-core/src/utils/os.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crunchy-cli-core/src/utils/os.rs b/crunchy-cli-core/src/utils/os.rs index 01d0f89..e3320fb 100644 --- a/crunchy-cli-core/src/utils/os.rs +++ b/crunchy-cli-core/src/utils/os.rs @@ -37,6 +37,12 @@ pub fn tempfile>(suffix: S) -> io::Result { /// Check if the given path exists and rename it until the new (renamed) file does not exist. pub fn free_file(mut path: PathBuf) -> PathBuf { + // if path is not a file and not a dir it's probably a pipe on linux which reguarly is intended + // and thus does not need to be renamed. what it is on windows ¯\_(ツ)_/¯ + if !path.is_file() && !path.is_dir() { + return path; + } + let mut i = 0; while path.exists() { i += 1; From c37e2495e1e66d9ee7a5d17d3570dcb5f081e137 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Tue, 27 Dec 2022 20:49:53 +0100 Subject: [PATCH 007/363] Create output parent directory if it doesn't exists (#91) --- crunchy-cli-core/src/cli/archive.rs | 7 +++++++ crunchy-cli-core/src/cli/download.rs | 13 +++++++++++++ 2 files changed, 20 insertions(+) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index b5244f7..70240cf 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -647,6 +647,13 @@ fn generate_mkv( debug!("ffmpeg {}", command_args.join(" ")); + // create parent directory if it does not exist + if let Some(parent) = target.parent() { + if !parent.exists() { + std::fs::create_dir_all(parent)? + } + } + let ffmpeg = Command::new("ffmpeg") .stdout(Stdio::null()) .stderr(Stdio::piped()) diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index d8a4ada..99da758 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -240,6 +240,12 @@ impl Execute for Download { let mut stdout = std::io::stdout().lock(); download_segments(&ctx, &mut stdout, None, format.stream).await?; } else { + // create parent directory if it does not exist + if let Some(parent) = path.parent() { + if !parent.exists() { + std::fs::create_dir_all(parent)? + } + } let mut file = File::options().create(true).write(true).open(&path)?; download_segments(&ctx, &mut file, None, format.stream).await? } @@ -259,6 +265,13 @@ async fn download_ffmpeg( let (input_presets, output_presets) = FFmpegPreset::ffmpeg_presets(download.ffmpeg_preset.clone())?; + // create parent directory if it does not exist + if let Some(parent) = target.parent() { + if !parent.exists() { + std::fs::create_dir_all(parent)? + } + } + let mut ffmpeg = Command::new("ffmpeg") .stdin(Stdio::piped()) .stdout(Stdio::null()) From 14f42833cb5257e0f9ef10f4b243c0ec51cbf28e Mon Sep 17 00:00:00 2001 From: ByteDream Date: Tue, 27 Dec 2022 22:59:35 +0100 Subject: [PATCH 008/363] Fix output to special file (pipes etc.) --- crunchy-cli-core/src/cli/archive.rs | 9 +++++++-- crunchy-cli-core/src/cli/download.rs | 27 ++++++++++++++++++++++----- crunchy-cli-core/src/utils/os.rs | 13 +++++++++---- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 70240cf..e3d5520 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -3,7 +3,7 @@ use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset}; use crate::utils::context::Context; use crate::utils::format::{format_string, Format}; use crate::utils::log::progress; -use crate::utils::os::{free_file, has_ffmpeg, tempfile}; +use crate::utils::os::{free_file, has_ffmpeg, is_special_file, tempfile}; use crate::utils::parse::{parse_url, UrlFilter}; use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number}; use crate::Execute; @@ -133,6 +133,7 @@ impl Execute for Archive { .unwrap_or_default() .to_string_lossy() != "mkv" + && !is_special_file(PathBuf::from(&self.output)) { bail!("File extension is not '.mkv'. Currently only matroska / '.mkv' files are supported") } @@ -251,7 +252,11 @@ impl Execute for Archive { info!( "Downloading {} to '{}'", primary.title, - path.to_str().unwrap() + if is_special_file(&path) { + path.to_str().unwrap() + } else { + path.file_name().unwrap().to_str().unwrap() + } ); tab_info!( "Episode: S{:02}E{:02}", diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index 99da758..0411a5e 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -3,7 +3,7 @@ use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset}; use crate::utils::context::Context; use crate::utils::format::{format_string, Format}; use crate::utils::log::progress; -use crate::utils::os::{free_file, has_ffmpeg}; +use crate::utils::os::{free_file, has_ffmpeg, is_special_file}; use crate::utils::parse::{parse_url, UrlFilter}; use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number}; use crate::Execute; @@ -219,7 +219,11 @@ impl Execute for Download { info!( "Downloading {} to '{}'", format.title, - path.file_name().unwrap().to_str().unwrap() + if is_special_file(&path) { + path.to_str().unwrap() + } else { + path.file_name().unwrap().to_str().unwrap() + } ); tab_info!("Episode: S{:02}E{:02}", format.season_number, format.number); tab_info!("Audio: {}", format.audio); @@ -232,9 +236,9 @@ impl Execute for Download { tab_info!("Resolution: {}", format.stream.resolution); tab_info!("FPS: {:.2}", format.stream.fps); - if path.extension().unwrap_or_default().to_string_lossy() != "ts" - || !self.ffmpeg_preset.is_empty() - { + let extension = path.extension().unwrap_or_default().to_string_lossy(); + + if (!extension.is_empty() && extension != "ts") || !self.ffmpeg_preset.is_empty() { download_ffmpeg(&ctx, &self, format.stream, path.as_path()).await?; } else if path.to_str().unwrap() == "-" { let mut stdout = std::io::stdout().lock(); @@ -279,6 +283,19 @@ async fn download_ffmpeg( .arg("-y") .args(input_presets) .args(["-f", "mpegts", "-i", "pipe:"]) + .args( + if target + .extension() + .unwrap_or_default() + .to_string_lossy() + .is_empty() + { + vec!["-f", "mpegts"] + } else { + vec![] + } + .as_slice(), + ) .args(output_presets) .arg(target.to_str().unwrap()) .spawn()?; diff --git a/crunchy-cli-core/src/utils/os.rs b/crunchy-cli-core/src/utils/os.rs index e3320fb..d2d4c3c 100644 --- a/crunchy-cli-core/src/utils/os.rs +++ b/crunchy-cli-core/src/utils/os.rs @@ -1,6 +1,6 @@ use log::debug; use std::io::ErrorKind; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::{env, io}; use tempfile::{Builder, NamedTempFile}; @@ -37,9 +37,8 @@ pub fn tempfile>(suffix: S) -> io::Result { /// Check if the given path exists and rename it until the new (renamed) file does not exist. pub fn free_file(mut path: PathBuf) -> PathBuf { - // if path is not a file and not a dir it's probably a pipe on linux which reguarly is intended - // and thus does not need to be renamed. what it is on windows ¯\_(ツ)_/¯ - if !path.is_file() && !path.is_dir() { + // if it's a special file does not rename it + if is_special_file(&path) { return path; } @@ -58,3 +57,9 @@ pub fn free_file(mut path: PathBuf) -> PathBuf { } path } + +/// Check if the given path is a special file. On Linux this is probably a pipe and on Windows +/// ¯\_(ツ)_/¯ +pub fn is_special_file>(path: P) -> bool { + path.as_ref().exists() && !path.as_ref().is_file() && !path.as_ref().is_dir() +} From c5940a240c8b2a6c39d6b3033ea6b5f6ee761335 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 28 Dec 2022 02:18:17 +0100 Subject: [PATCH 009/363] Slightly change download process to be more verbose in error situations --- crunchy-cli-core/src/cli/utils.rs | 37 ++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index 6045ce4..1f6bf40 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -93,13 +93,16 @@ pub async fn download_segments( Ok(()) }); } + // drop the sender already here so it does not outlive all (download) threads which are the only + // real consumers of it + drop(sender); + // this is the main loop which writes the data. it uses a BTreeMap as a buffer as the write + // happens synchronized. the download consist of multiple segments. the map keys are representing + // the segment number and the values the corresponding bytes let mut data_pos = 0usize; let mut buf: BTreeMap> = BTreeMap::new(); - loop { - // is always `Some` because `sender` does not get dropped when all threads are finished - let (pos, bytes) = receiver.recv().unwrap(); - + for (pos, bytes) in receiver.iter() { if let Some(p) = &progress { let progress_len = p.length().unwrap(); let estimated_segment_len = (variant_data.bandwidth / 8) @@ -115,26 +118,44 @@ pub async fn download_segments( p.inc(bytes_len) } + // check if the currently sent bytes are the next in the buffer. if so, write them directly + // to the target without first adding them to the buffer. + // if not, add them to the buffer if data_pos == pos { writer.write_all(bytes.borrow())?; data_pos += 1; } else { buf.insert(pos, bytes); } + // check if the buffer contains the next segment(s) while let Some(b) = buf.remove(&data_pos) { writer.write_all(b.borrow())?; data_pos += 1; } - - if *count.lock().unwrap() >= total_segments && buf.is_empty() { - break; - } } + // write the remaining buffer, if existent + while let Some(b) = buf.remove(&data_pos) { + writer.write_all(b.borrow())?; + data_pos += 1; + } + + // if any error has occured while downloading it gets returned here. maybe a little late, if one + // out of, for example 12, threads has the error while let Some(joined) = join_set.join_next().await { joined?? } + if !buf.is_empty() { + bail!( + "Download buffer is not empty. Remaining segments: {}", + buf.into_keys() + .map(|k| k.to_string()) + .collect::>() + .join(", ") + ) + } + Ok(()) } From 240e5563a3db767fccccfe523af74a8d862f5da0 Mon Sep 17 00:00:00 2001 From: Alexandru Dracea Date: Wed, 28 Dec 2022 15:44:45 +0200 Subject: [PATCH 010/363] Add error handling and retry attempts Handles cases where the segments fail to download and sometimes get stuck by introducing a timeout and retrying on failure. --- crunchy-cli-core/src/cli/utils.rs | 51 +++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index 1f6bf40..3d14318 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -76,8 +76,55 @@ pub async fn download_segments( let thread_count = count.clone(); join_set.spawn(async move { for (i, segment) in thread_segments.into_iter().enumerate() { - let response = thread_client.get(&segment.url).send().await?; - let mut buf = response.bytes().await?.to_vec(); + let response_res = thread_client + .get(&segment.url) + .timeout(Duration::from_secs(60u64)) + .send() + .await; + let verfified_response = match response_res { + Ok(x) => x, + Err(y) => panic!("This is likely a netowrking error: {}", y), + }; + let possible_error_in_response = verfified_response.bytes().await; + let mut buf = if let Ok(r) = possible_error_in_response { + r.to_vec() + } else { + debug!( + "Segment Failed to download: {}, retrying.", + num + (i * cpus) + ); + let mut resp = thread_client + .get(&segment.url) + .timeout(Duration::from_secs(60u64)) + .send() + .await + .unwrap() + .bytes() + .await; + if resp.is_err() { + let mut retry_ctr = 1; + loop { + debug!( + "Segment Failed to download: {}, retry {}.", + num + (i * cpus), + retry_ctr + ); + resp = thread_client + .get(&segment.url) + .timeout(Duration::from_secs(60u64)) + .send() + .await + .unwrap() + .bytes() + .await; + if resp.is_ok() { + break; + } + retry_ctr += 1; + } + } + resp.unwrap().to_vec() + }; buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec(); debug!( From 8a3c0132e7e0ec4206d001670fd062bcf9673052 Mon Sep 17 00:00:00 2001 From: Alexandru Dracea Date: Wed, 28 Dec 2022 15:59:55 +0200 Subject: [PATCH 011/363] Update utils.rs --- crunchy-cli-core/src/cli/utils.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index 3d14318..afc431f 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -3,7 +3,8 @@ use anyhow::{bail, Result}; use crunchyroll_rs::media::{Resolution, VariantData, VariantSegment}; use indicatif::{ProgressBar, ProgressFinish, ProgressStyle}; use log::{debug, LevelFilter}; -use std::borrow::{Borrow, BorrowMut}; +use std::borrow::Borrow; +use std::time::Duration; use std::collections::BTreeMap; use std::io::Write; use std::sync::{mpsc, Arc, Mutex}; From c2ae622d01c1913ff51264858a4c6b0f28353690 Mon Sep 17 00:00:00 2001 From: Alexandru Dracea Date: Wed, 28 Dec 2022 16:01:55 +0200 Subject: [PATCH 012/363] Update utils.rs --- crunchy-cli-core/src/cli/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index afc431f..cd45859 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -3,7 +3,7 @@ use anyhow::{bail, Result}; use crunchyroll_rs::media::{Resolution, VariantData, VariantSegment}; use indicatif::{ProgressBar, ProgressFinish, ProgressStyle}; use log::{debug, LevelFilter}; -use std::borrow::Borrow; +use std::borrow::{Borrow, BorrowMut}; use std::time::Duration; use std::collections::BTreeMap; use std::io::Write; From d0681c7f6cc01caa684b39f30c459a20d5f09c24 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 28 Dec 2022 15:18:12 +0100 Subject: [PATCH 013/363] Simplify retry segment download --- crunchy-cli-core/src/cli/utils.rs | 62 ++++++++++--------------------- 1 file changed, 19 insertions(+), 43 deletions(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index cd45859..7c32e86 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -4,10 +4,10 @@ use crunchyroll_rs::media::{Resolution, VariantData, VariantSegment}; use indicatif::{ProgressBar, ProgressFinish, ProgressStyle}; use log::{debug, LevelFilter}; use std::borrow::{Borrow, BorrowMut}; -use std::time::Duration; use std::collections::BTreeMap; use std::io::Write; use std::sync::{mpsc, Arc, Mutex}; +use std::time::Duration; use tokio::task::JoinSet; pub fn find_resolution( @@ -77,54 +77,30 @@ pub async fn download_segments( let thread_count = count.clone(); join_set.spawn(async move { for (i, segment) in thread_segments.into_iter().enumerate() { - let response_res = thread_client - .get(&segment.url) - .timeout(Duration::from_secs(60u64)) - .send() - .await; - let verfified_response = match response_res { - Ok(x) => x, - Err(y) => panic!("This is likely a netowrking error: {}", y), - }; - let possible_error_in_response = verfified_response.bytes().await; - let mut buf = if let Ok(r) = possible_error_in_response { - r.to_vec() - } else { - debug!( - "Segment Failed to download: {}, retrying.", - num + (i * cpus) - ); - let mut resp = thread_client + let mut retry_count = 0; + let mut buf = loop { + let response = thread_client .get(&segment.url) - .timeout(Duration::from_secs(60u64)) + .timeout(Duration::from_secs(10)) .send() .await - .unwrap() - .bytes() - .await; - if resp.is_err() { - let mut retry_ctr = 1; - loop { - debug!( - "Segment Failed to download: {}, retry {}.", - num + (i * cpus), - retry_ctr - ); - resp = thread_client - .get(&segment.url) - .timeout(Duration::from_secs(60u64)) - .send() - .await - .unwrap() - .bytes() - .await; - if resp.is_ok() { - break; + .unwrap(); + + match response.bytes().await { + Ok(b) => break b.to_vec(), + Err(e) => { + if e.is_body() { + if retry_count == 5 { + panic!("Max retry count reached ({}), multiple errors occured while receiving segment {}: {}", retry_count, num + (i * cpus), e) + } + debug!("Failed to download segment {}. Retrying ({} out of 5 retries left)", num + (i * cpus), 5 - retry_count) + } else { + panic!("{}", e) } - retry_ctr += 1; } } - resp.unwrap().to_vec() + + retry_count += 1; }; buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec(); From 7115c5546d1c71c496c26b82d85b2f1aaf114aca Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 28 Dec 2022 15:25:10 +0100 Subject: [PATCH 014/363] Show error message on segment download retry --- crunchy-cli-core/src/cli/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index 7c32e86..e2bbc63 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -93,7 +93,7 @@ pub async fn download_segments( if retry_count == 5 { panic!("Max retry count reached ({}), multiple errors occured while receiving segment {}: {}", retry_count, num + (i * cpus), e) } - debug!("Failed to download segment {}. Retrying ({} out of 5 retries left)", num + (i * cpus), 5 - retry_count) + debug!("Failed to download segment {} ({}). Retrying, {} out of 5 retries left", num + (i * cpus), e, 5 - retry_count) } else { panic!("{}", e) } From b8e46099f9035c609b376c2cfa1b0d28719564c9 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 28 Dec 2022 15:35:38 +0100 Subject: [PATCH 015/363] Re-increase segment request timeout --- crunchy-cli-core/src/cli/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index e2bbc63..841dbef 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -81,7 +81,7 @@ pub async fn download_segments( let mut buf = loop { let response = thread_client .get(&segment.url) - .timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(60)) .send() .await .unwrap(); From 03db38b31ce85f7264fe90d259c09e7ad3ee8679 Mon Sep 17 00:00:00 2001 From: ByteDream <63594396+ByteDream@users.noreply.github.com> Date: Wed, 28 Dec 2022 15:45:33 +0100 Subject: [PATCH 016/363] Add debug segment percentage (#93) * Fix file extension unwrap panic * Change log output name from crunchy_cli_core to crunchy_cli * Add percentage output --- crunchy-cli-core/src/cli/log.rs | 5 +++-- crunchy-cli-core/src/cli/utils.rs | 8 ++++++-- crunchy-cli-core/src/utils/os.rs | 4 ++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/crunchy-cli-core/src/cli/log.rs b/crunchy-cli-core/src/cli/log.rs index 377685b..3f63eb0 100644 --- a/crunchy-cli-core/src/cli/log.rs +++ b/crunchy-cli-core/src/cli/log.rs @@ -75,8 +75,9 @@ impl CliLogger { // replace the 'progress' prefix if this function is invoked via 'progress!' record .target() - .replacen("progress", "crunchy_cli", 1) - .replacen("progress_end", "crunchy_cli", 1), + .replacen("crunchy_cli_core", "crunchy_cli", 1) + .replacen("progress_end", "crunchy_cli", 1) + .replacen("progress", "crunchy_cli", 1), format!("{:?}", thread::current().id()) .replace("ThreadId(", "") .replace(')', ""), diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index 6045ce4..4c8c10e 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -80,14 +80,18 @@ pub async fn download_segments( let mut buf = response.bytes().await?.to_vec(); buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec(); + + let mut c = thread_count.lock().unwrap(); debug!( - "Downloaded and decrypted segment {} ({})", + "Downloaded and decrypted segment [{}/{} {:.2}%] {}", num + (i * cpus), + total_segments, + ((*c + 1) as f64 / total_segments as f64) * 100f64, segment.url ); thread_sender.send((num + (i * cpus), buf))?; - *thread_count.lock().unwrap() += 1; + *c += 1; } Ok(()) diff --git a/crunchy-cli-core/src/utils/os.rs b/crunchy-cli-core/src/utils/os.rs index d2d4c3c..a7b3fbf 100644 --- a/crunchy-cli-core/src/utils/os.rs +++ b/crunchy-cli-core/src/utils/os.rs @@ -46,8 +46,8 @@ pub fn free_file(mut path: PathBuf) -> PathBuf { while path.exists() { i += 1; - let ext = path.extension().unwrap().to_string_lossy(); - let mut filename = path.file_stem().unwrap().to_str().unwrap(); + let ext = path.extension().unwrap_or_default().to_string_lossy(); + let mut filename = path.file_stem().unwrap_or_default().to_str().unwrap(); if filename.ends_with(&format!(" ({})", i - 1)) { filename = filename.strip_suffix(&format!(" ({})", i - 1)).unwrap(); From 0c139420165e412516313d612124dd62fe2b117f Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 2 Jan 2023 17:53:54 +0100 Subject: [PATCH 017/363] Update dependencies --- Cargo.lock | 69 ++++++++++++++++--------------------- crunchy-cli-core/Cargo.lock | 61 ++++++++++++++------------------ 2 files changed, 56 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a7432c8..8221757 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,9 +148,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.30" +version = "4.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "656ad1e55e23d287773f7d8192c300dc715c3eeded93b3da651d11c42cfd74d2" +checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39" dependencies = [ "bitflags", "clap_derive", @@ -163,9 +163,9 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.0.6" +version = "4.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7b3c9eae0de7bf8e3f904a5e40612b21fb2e2e566456d177809a48b892d24da" +checksum = "10861370d2ba66b0f5989f83ebf35db6421713fd92351790e7fdd6c36774c56b" dependencies = [ "clap", ] @@ -194,9 +194,9 @@ dependencies = [ [[package]] name = "clap_mangen" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e503c3058af0a0854668ea01db55c622482a080092fede9dd2e00a00a9436504" +checksum = "904eb24d05ad587557e0f484ddce5c737c30cf81372badb16d13e41c4b8340b1" dependencies = [ "clap", "roff", @@ -214,16 +214,15 @@ dependencies = [ [[package]] name = "console" -version = "0.15.2" +version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c050367d967ced717c04b65d8c619d863ef9292ce0c5760028655a2fb298718c" +checksum = "5556015fe3aad8b968e5d4124980fbe2f6aaee7aeec6b749de1faaa2ca5d0a4c" dependencies = [ "encode_unicode", "lazy_static", "libc", - "terminal_size 0.1.17", "unicode-width", - "winapi", + "windows-sys 0.42.0", ] [[package]] @@ -312,21 +311,23 @@ dependencies = [ "signal-hook", "sys-locale", "tempfile", - "terminal_size 0.2.3", + "terminal_size", "tokio", ] [[package]] name = "crunchyroll-rs" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5032be36dc6c4757af4f28152659e7a772b48f6925f0e0ac81b5c384e291314" +checksum = "5ea0bc79ecd9fafc95c6049d0b38e0e00c8d4665314ae3ee675db67b622ce3b7" dependencies = [ "aes", + "async-trait", "cbc", "chrono", "crunchyroll-rs-internal", "http", + "lazy_static", "m3u8-rs", "regex", "reqwest", @@ -341,9 +342,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e4aa1f09fd76f44329455abfac35e22c654ac782e93bc8a7d3ee1be27509f4" +checksum = "66be957a34f7498e4bc324af67c58bb7ace299c214c9353281545f92dbc45d78" dependencies = [ "darling", "quote", @@ -908,9 +909,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.138" +version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" [[package]] name = "link-cplusplus" @@ -1014,9 +1015,9 @@ dependencies = [ [[package]] name = "nom" -version = "7.1.1" +version = "7.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +checksum = "e5507769c4919c998e69e49c839d9dc6e693ede4cc4290d6ad8b41d4f09c548c" dependencies = [ "memchr", "minimal-lexical", @@ -1059,9 +1060,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "once_cell" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" [[package]] name = "openssl" @@ -1140,9 +1141,9 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "portable-atomic" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bdd679d533107e090c2704a35982fc06302e30898e63ffa26a81155c012e92" +checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b" [[package]] name = "proc-macro-error" @@ -1328,9 +1329,9 @@ checksum = "b833d8d034ea094b1ea68aa6d5c740e0d04bad9d16568d08ba6f76823a114316" [[package]] name = "rustix" -version = "0.36.5" +version = "0.36.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3807b5d10909833d3e9acd1eb5fb988f79376ff10fce42937de71a449c4c588" +checksum = "4feacf7db682c6c329c4ede12649cd36ecab0f3be5b7d74e6a20304725db4549" dependencies = [ "bitflags", "errno", @@ -1428,18 +1429,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.151" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fed41fc1a24994d044e6db6935e69511a1153b52c15eb42493b26fa87feba0" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.151" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "255abe9a125a985c05190d687b320c12f9b1f0b99445e608c21ba0782c719ad8" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2", "quote", @@ -1583,16 +1584,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "terminal_size" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "terminal_size" version = "0.2.3" diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index 8347aa7..6a805d3 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -148,9 +148,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.30" +version = "4.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "656ad1e55e23d287773f7d8192c300dc715c3eeded93b3da651d11c42cfd74d2" +checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39" dependencies = [ "bitflags", "clap_derive", @@ -195,16 +195,15 @@ dependencies = [ [[package]] name = "console" -version = "0.15.2" +version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c050367d967ced717c04b65d8c619d863ef9292ce0c5760028655a2fb298718c" +checksum = "5556015fe3aad8b968e5d4124980fbe2f6aaee7aeec6b749de1faaa2ca5d0a4c" dependencies = [ "encode_unicode", "lazy_static", "libc", - "terminal_size 0.1.17", "unicode-width", - "winapi", + "windows-sys 0.42.0", ] [[package]] @@ -281,21 +280,23 @@ dependencies = [ "signal-hook", "sys-locale", "tempfile", - "terminal_size 0.2.3", + "terminal_size", "tokio", ] [[package]] name = "crunchyroll-rs" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5032be36dc6c4757af4f28152659e7a772b48f6925f0e0ac81b5c384e291314" +checksum = "5ea0bc79ecd9fafc95c6049d0b38e0e00c8d4665314ae3ee675db67b622ce3b7" dependencies = [ "aes", + "async-trait", "cbc", "chrono", "crunchyroll-rs-internal", "http", + "lazy_static", "m3u8-rs", "regex", "reqwest", @@ -310,9 +311,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e4aa1f09fd76f44329455abfac35e22c654ac782e93bc8a7d3ee1be27509f4" +checksum = "66be957a34f7498e4bc324af67c58bb7ace299c214c9353281545f92dbc45d78" dependencies = [ "darling", "quote", @@ -877,9 +878,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.138" +version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" [[package]] name = "link-cplusplus" @@ -983,9 +984,9 @@ dependencies = [ [[package]] name = "nom" -version = "7.1.1" +version = "7.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +checksum = "e5507769c4919c998e69e49c839d9dc6e693ede4cc4290d6ad8b41d4f09c548c" dependencies = [ "memchr", "minimal-lexical", @@ -1028,9 +1029,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "once_cell" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" [[package]] name = "openssl" @@ -1109,9 +1110,9 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "portable-atomic" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bdd679d533107e090c2704a35982fc06302e30898e63ffa26a81155c012e92" +checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b" [[package]] name = "proc-macro-error" @@ -1291,9 +1292,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.5" +version = "0.36.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3807b5d10909833d3e9acd1eb5fb988f79376ff10fce42937de71a449c4c588" +checksum = "4feacf7db682c6c329c4ede12649cd36ecab0f3be5b7d74e6a20304725db4549" dependencies = [ "bitflags", "errno", @@ -1391,18 +1392,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.151" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fed41fc1a24994d044e6db6935e69511a1153b52c15eb42493b26fa87feba0" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.151" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "255abe9a125a985c05190d687b320c12f9b1f0b99445e608c21ba0782c719ad8" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2", "quote", @@ -1546,16 +1547,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "terminal_size" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "terminal_size" version = "0.2.3" From fae5d699331fcb9ced5856f0ab521982edc0fb76 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 2 Jan 2023 17:56:50 +0100 Subject: [PATCH 018/363] Apply stabilizations fixes (#89) --- crunchy-cli-core/src/lib.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index 00699e4..4f2ce32 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -189,8 +189,9 @@ async fn create_ctx(cli: &Cli) -> Result { } async fn crunchyroll_session(cli: &Cli) -> Result { - let mut builder = Crunchyroll::builder(); - builder.locale(cli.lang.clone().unwrap_or_else(system_locale)); + let mut builder = Crunchyroll::builder() + .locale(cli.lang.clone().unwrap_or_else(system_locale)) + .stabilization_locales(true); let login_methods_count = cli.login_method.credentials.is_some() as u8 + cli.login_method.etp_rt.is_some() as u8 From 3c3b7b65662e4f315ff34634d2c7b92a2445a212 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Tue, 3 Jan 2023 01:24:17 +0100 Subject: [PATCH 019/363] Fix panic on specific filenames --- crunchy-cli-core/src/utils/os.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crunchy-cli-core/src/utils/os.rs b/crunchy-cli-core/src/utils/os.rs index d2d4c3c..a7b3fbf 100644 --- a/crunchy-cli-core/src/utils/os.rs +++ b/crunchy-cli-core/src/utils/os.rs @@ -46,8 +46,8 @@ pub fn free_file(mut path: PathBuf) -> PathBuf { while path.exists() { i += 1; - let ext = path.extension().unwrap().to_string_lossy(); - let mut filename = path.file_stem().unwrap().to_str().unwrap(); + let ext = path.extension().unwrap_or_default().to_string_lossy(); + let mut filename = path.file_stem().unwrap_or_default().to_str().unwrap(); if filename.ends_with(&format!(" ({})", i - 1)) { filename = filename.strip_suffix(&format!(" ({})", i - 1)).unwrap(); From b365bda5dca57754a383bdb3de0484f600c9697c Mon Sep 17 00:00:00 2001 From: ByteDream Date: Tue, 3 Jan 2023 01:28:42 +0100 Subject: [PATCH 020/363] Fix download threads to properly return errors --- crunchy-cli-core/src/cli/utils.rs | 93 ++++++++++++++++++------------- 1 file changed, 53 insertions(+), 40 deletions(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index 841dbef..99d4f67 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -74,47 +74,56 @@ pub async fn download_segments( let thread_client = client.clone(); let thread_sender = sender.clone(); let thread_segments = segs.remove(0); - let thread_count = count.clone(); join_set.spawn(async move { - for (i, segment) in thread_segments.into_iter().enumerate() { - let mut retry_count = 0; - let mut buf = loop { - let response = thread_client - .get(&segment.url) - .timeout(Duration::from_secs(60)) - .send() - .await - .unwrap(); + let after_download_sender = thread_sender.clone(); - match response.bytes().await { - Ok(b) => break b.to_vec(), - Err(e) => { - if e.is_body() { - if retry_count == 5 { - panic!("Max retry count reached ({}), multiple errors occured while receiving segment {}: {}", retry_count, num + (i * cpus), e) + // the download process is encapsulated in its own function. this is done to easily + // catch errors which get returned with `...?` and `bail!(...)` and that the thread + // itself can report that an error has occured + let download = || async move { + for (i, segment) in thread_segments.into_iter().enumerate() { + let mut retry_count = 0; + let mut buf = loop { + let response = thread_client + .get(&segment.url) + .timeout(Duration::from_secs(60)) + .send() + .await?; + + match response.bytes().await { + Ok(b) => break b.to_vec(), + Err(e) => { + if e.is_body() { + if retry_count == 5 { + bail!("Max retry count reached ({}), multiple errors occured while receiving segment {}: {}", retry_count, num + (i * cpus), e) + } + debug!("Failed to download segment {} ({}). Retrying, {} out of 5 retries left", num + (i * cpus), e, 5 - retry_count) + } else { + bail!("{}", e) } - debug!("Failed to download segment {} ({}). Retrying, {} out of 5 retries left", num + (i * cpus), e, 5 - retry_count) - } else { - panic!("{}", e) } } - } - retry_count += 1; - }; + retry_count += 1; + }; - buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec(); - debug!( - "Downloaded and decrypted segment {} ({})", - num + (i * cpus), - segment.url - ); - thread_sender.send((num + (i * cpus), buf))?; + buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec(); + debug!( + "Downloaded and decrypted segment {} ({})", + num + (i * cpus), + segment.url + ); + thread_sender.send((num as i32 + (i * cpus) as i32, buf))?; + } + Ok(()) + }; - *thread_count.lock().unwrap() += 1; + let result = download().await; + if result.is_err() { + after_download_sender.send((-1 as i32, vec![]))?; } - Ok(()) + result }); } // drop the sender already here so it does not outlive all (download) threads which are the only @@ -124,14 +133,19 @@ pub async fn download_segments( // this is the main loop which writes the data. it uses a BTreeMap as a buffer as the write // happens synchronized. the download consist of multiple segments. the map keys are representing // the segment number and the values the corresponding bytes - let mut data_pos = 0usize; - let mut buf: BTreeMap> = BTreeMap::new(); + let mut data_pos = 0; + let mut buf: BTreeMap> = BTreeMap::new(); for (pos, bytes) in receiver.iter() { + // if the position is lower than 0, an error occured in the sending download thread + if pos < 0 { + break + } + if let Some(p) = &progress { let progress_len = p.length().unwrap(); let estimated_segment_len = (variant_data.bandwidth / 8) * segments - .get(pos) + .get(pos as usize) .unwrap() .length .unwrap_or_default() @@ -158,18 +172,17 @@ pub async fn download_segments( } } + // if any error has occured while downloading it gets returned here + while let Some(joined) = join_set.join_next().await { + joined?? + } + // write the remaining buffer, if existent while let Some(b) = buf.remove(&data_pos) { writer.write_all(b.borrow())?; data_pos += 1; } - // if any error has occured while downloading it gets returned here. maybe a little late, if one - // out of, for example 12, threads has the error - while let Some(joined) = join_set.join_next().await { - joined?? - } - if !buf.is_empty() { bail!( "Download buffer is not empty. Remaining segments: {}", From 29c6129e6efbb18a1ac1ae7cbbc46893e4870603 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Tue, 3 Jan 2023 14:50:12 +0100 Subject: [PATCH 021/363] Update dependencies & version --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 2 +- crunchy-cli-core/Cargo.lock | 14 +++++++------- crunchy-cli-core/Cargo.toml | 2 +- crunchy-cli-core/src/lib.rs | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8221757..a564687 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -214,9 +214,9 @@ dependencies = [ [[package]] name = "console" -version = "0.15.3" +version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5556015fe3aad8b968e5d4124980fbe2f6aaee7aeec6b749de1faaa2ca5d0a4c" +checksum = "c9b6515d269224923b26b5febea2ed42b2d5f2ce37284a4dd670fedd6cb8347a" dependencies = [ "encode_unicode", "lazy_static", @@ -279,7 +279,7 @@ dependencies = [ [[package]] name = "crunchy-cli" -version = "3.0.0-dev.4" +version = "3.0.0-dev.5" dependencies = [ "chrono", "clap", @@ -291,7 +291,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-dev.4" +version = "3.0.0-dev.5" dependencies = [ "anyhow", "async-trait", @@ -317,9 +317,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea0bc79ecd9fafc95c6049d0b38e0e00c8d4665314ae3ee675db67b622ce3b7" +checksum = "f8ec300b509afbd8977f71cd7feb7d0f20d38d8e38976b7fcd51f6128cbdefe6" dependencies = [ "aes", "async-trait", @@ -342,9 +342,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66be957a34f7498e4bc324af67c58bb7ace299c214c9353281545f92dbc45d78" +checksum = "10d4b870818d5ce0993d70271bf803dbfbcc8a3a0fa7398b182f5f4b4e88509d" dependencies = [ "darling", "quote", diff --git a/Cargo.toml b/Cargo.toml index f5f988b..c67d599 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-dev.4" +version = "3.0.0-dev.5" edition = "2021" [dependencies] diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index 6a805d3..37ee2c3 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "console" -version = "0.15.3" +version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5556015fe3aad8b968e5d4124980fbe2f6aaee7aeec6b749de1faaa2ca5d0a4c" +checksum = "c9b6515d269224923b26b5febea2ed42b2d5f2ce37284a4dd670fedd6cb8347a" dependencies = [ "encode_unicode", "lazy_static", @@ -260,7 +260,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-dev.4" +version = "3.0.0-dev.5" dependencies = [ "anyhow", "async-trait", @@ -286,9 +286,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea0bc79ecd9fafc95c6049d0b38e0e00c8d4665314ae3ee675db67b622ce3b7" +checksum = "f8ec300b509afbd8977f71cd7feb7d0f20d38d8e38976b7fcd51f6128cbdefe6" dependencies = [ "aes", "async-trait", @@ -311,9 +311,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66be957a34f7498e4bc324af67c58bb7ace299c214c9353281545f92dbc45d78" +checksum = "10d4b870818d5ce0993d70271bf803dbfbcc8a3a0fa7398b182f5f4b4e88509d" dependencies = [ "darling", "quote", diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 780716f..e61031b 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-dev.4" +version = "3.0.0-dev.5" edition = "2021" [dependencies] diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index 4f2ce32..52a4da1 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -189,7 +189,7 @@ async fn create_ctx(cli: &Cli) -> Result { } async fn crunchyroll_session(cli: &Cli) -> Result { - let mut builder = Crunchyroll::builder() + let builder = Crunchyroll::builder() .locale(cli.lang.clone().unwrap_or_else(system_locale)) .stabilization_locales(true); From d0a8103e3def5a45c84bf7eafe2c8b43f8c7b98d Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 4 Jan 2023 00:12:13 +0100 Subject: [PATCH 022/363] Update dependencies & version --- Cargo.lock | 12 ++++++------ Cargo.toml | 2 +- crunchy-cli-core/Cargo.lock | 10 +++++----- crunchy-cli-core/Cargo.toml | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a564687..8da4e30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -279,7 +279,7 @@ dependencies = [ [[package]] name = "crunchy-cli" -version = "3.0.0-dev.5" +version = "3.0.0-dev.6" dependencies = [ "chrono", "clap", @@ -291,7 +291,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-dev.5" +version = "3.0.0-dev.6" dependencies = [ "anyhow", "async-trait", @@ -317,9 +317,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ec300b509afbd8977f71cd7feb7d0f20d38d8e38976b7fcd51f6128cbdefe6" +checksum = "dd742baacdfbf9caca8656262ac2397c334928ccd65fb332c95ee8e8b8d9223d" dependencies = [ "aes", "async-trait", @@ -342,9 +342,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d4b870818d5ce0993d70271bf803dbfbcc8a3a0fa7398b182f5f4b4e88509d" +checksum = "aadb66f2b5cdcc8b82f095aec6ca92ca4abb30016a13916f2358cf7bfd8a1997" dependencies = [ "darling", "quote", diff --git a/Cargo.toml b/Cargo.toml index c67d599..76ed950 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-dev.5" +version = "3.0.0-dev.6" edition = "2021" [dependencies] diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index 37ee2c3..af1086a 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -260,7 +260,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-dev.5" +version = "3.0.0-dev.6" dependencies = [ "anyhow", "async-trait", @@ -286,9 +286,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ec300b509afbd8977f71cd7feb7d0f20d38d8e38976b7fcd51f6128cbdefe6" +checksum = "dd742baacdfbf9caca8656262ac2397c334928ccd65fb332c95ee8e8b8d9223d" dependencies = [ "aes", "async-trait", @@ -311,9 +311,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d4b870818d5ce0993d70271bf803dbfbcc8a3a0fa7398b182f5f4b4e88509d" +checksum = "aadb66f2b5cdcc8b82f095aec6ca92ca4abb30016a13916f2358cf7bfd8a1997" dependencies = [ "darling", "quote", diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index e61031b..566b8c4 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-dev.5" +version = "3.0.0-dev.6" edition = "2021" [dependencies] @@ -15,7 +15,7 @@ ctrlc = "3.2" dirs = "4.0" indicatif = "0.17" log = { version = "0.4", features = ["std"] } -num_cpus = "1.14" +num_cpus = "1.15" regex = "1.7" sanitize-filename = "0.4" serde = "1.0" From 7726287859482f508dc584be405db7022ee31b22 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 4 Jan 2023 01:28:14 +0100 Subject: [PATCH 023/363] :) --- Cargo.lock | 8 ++++---- crunchy-cli-core/Cargo.lock | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8da4e30..eb7c6a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -317,9 +317,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd742baacdfbf9caca8656262ac2397c334928ccd65fb332c95ee8e8b8d9223d" +checksum = "184d0c725a09aec815316cbf41a2f362008ecb0e8c8e3b6b9930d01a89b5df21" dependencies = [ "aes", "async-trait", @@ -342,9 +342,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aadb66f2b5cdcc8b82f095aec6ca92ca4abb30016a13916f2358cf7bfd8a1997" +checksum = "5f3c82e1766339727fc2c10d66d0c4f001b1cf42e2993f9d93997b610f408776" dependencies = [ "darling", "quote", diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index af1086a..b5dd8a1 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -286,9 +286,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd742baacdfbf9caca8656262ac2397c334928ccd65fb332c95ee8e8b8d9223d" +checksum = "184d0c725a09aec815316cbf41a2f362008ecb0e8c8e3b6b9930d01a89b5df21" dependencies = [ "aes", "async-trait", @@ -311,9 +311,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aadb66f2b5cdcc8b82f095aec6ca92ca4abb30016a13916f2358cf7bfd8a1997" +checksum = "5f3c82e1766339727fc2c10d66d0c4f001b1cf42e2993f9d93997b610f408776" dependencies = [ "darling", "quote", From 404aa496e157162c35c3b5bf983cbc43c8689676 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Thu, 5 Jan 2023 22:28:23 +0100 Subject: [PATCH 024/363] Fix subtitle look and feel typo --- crunchy-cli-core/src/cli/archive.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index e3d5520..3910d24 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -481,7 +481,7 @@ fn fix_subtitle_look_and_feel(raw: Vec) -> Vec { for line in String::from_utf8_lossy(raw.as_slice()).split('\n') { if line.trim().starts_with('[') && script_info { - new.push_str("ScaledBorderAndShadows: yes\n"); + new.push_str("ScaledBorderAndShadow: yes\n"); script_info = false } else if line.trim() == "[Script Info]" { script_info = true From 892407d1f069b1dfcaa71c4eeca1c942ff40872c Mon Sep 17 00:00:00 2001 From: ByteDream Date: Fri, 6 Jan 2023 00:03:57 +0100 Subject: [PATCH 025/363] Fix --default-subtitle causing no such file error (#98) --- crunchy-cli-core/src/cli/archive.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 3910d24..3eb92a7 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -632,10 +632,10 @@ fn generate_mkv( if let Some(default_subtitle) = &archive.default_subtitle { // if `--default_subtitle ` is given set the default subtitle to the given locale if let Some(position) = subtitle_paths - .into_iter() - .position(|s| &s.1.locale == default_subtitle) + .iter() + .position(|(_, subtitle)| &subtitle.locale == default_subtitle) { - command_args.push(format!("-disposition:s:{}", position)) + command_args.extend([format!("-disposition:s:{}", position), "default".to_string()]) } else { command_args.extend(["-disposition:s:0".to_string(), "0".to_string()]) } From 06fd9a7a98989aa4a2c84c120b96b0477f9cf150 Mon Sep 17 00:00:00 2001 From: Hannes Braun Date: Wed, 4 Jan 2023 22:09:31 +0100 Subject: [PATCH 026/363] Archive subtitles of all versions of an episode --- crunchy-cli-core/src/cli/archive.rs | 95 +++++++++++++++++++------- crunchy-cli-core/src/utils/mod.rs | 1 + crunchy-cli-core/src/utils/subtitle.rs | 11 +++ 3 files changed, 82 insertions(+), 25 deletions(-) create mode 100644 crunchy-cli-core/src/utils/subtitle.rs diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 3eb92a7..8259ab3 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -6,6 +6,7 @@ use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file, tempfile}; use crate::utils::parse::{parse_url, UrlFilter}; use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number}; +use crate::utils::subtitle::Subtitle; use crate::Execute; use anyhow::{bail, Result}; use chrono::NaiveTime; @@ -232,7 +233,7 @@ impl Execute for Archive { } } - for (formats, subtitles) in archive_formats { + for (formats, mut subtitles) in archive_formats { let (primary, additionally) = formats.split_first().unwrap(); let mut path = PathBuf::from(&self.output); @@ -276,13 +277,14 @@ impl Execute for Archive { "Subtitle: {}", subtitles .iter() + .filter(|s| s.primary) // Don't print subtitles of non-primary streams. They might get removed depending on the merge behavior. .map(|s| { if let Some(default) = &self.default_subtitle { - if default == &s.locale { + if default == &s.stream_subtitle.locale { return format!("{} (primary)", default); } } - s.locale.to_string() + s.stream_subtitle.locale.to_string() }) .collect::>() .join(", ") @@ -309,13 +311,23 @@ impl Execute for Archive { } else { video_paths.push((path, additional)) } + + // Remove subtitles of deleted video + if only_audio { + subtitles.retain(|s| s.episode_id != additional.id); + } } let (primary_video, _) = video_paths.get(0).unwrap(); let primary_video_length = get_video_length(primary_video.to_path_buf()).unwrap(); for subtitle in subtitles { subtitle_paths.push(( - download_subtitle(&self, subtitle.clone(), primary_video_length).await?, + download_subtitle( + &self, + subtitle.stream_subtitle.clone(), + primary_video_length, + ) + .await?, subtitle, )) } @@ -334,7 +346,7 @@ async fn formats_from_series( archive: &Archive, series: Media, url_filter: &UrlFilter, -) -> Result, Vec)>> { +) -> Result, Vec)>> { let mut seasons = series.seasons().await?; // filter any season out which does not contain the specified audio languages @@ -367,8 +379,8 @@ async fn formats_from_series( } #[allow(clippy::type_complexity)] - let mut result: BTreeMap, Vec)>> = - BTreeMap::new(); + let mut result: BTreeMap, Vec)>> = BTreeMap::new(); + let mut primary_season = true; for season in series.seasons().await? { if !url_filter.is_season_valid(season.metadata.season_number) || !archive @@ -402,20 +414,26 @@ async fn formats_from_series( ) }; - let (ref mut formats, _) = result + let (ref mut formats, subtitles) = result .entry(season.metadata.season_number) .or_insert_with(BTreeMap::new) .entry(episode.metadata.episode_number) - .or_insert_with(|| { - let subtitles: Vec = archive - .subtitle - .iter() - .filter_map(|l| streams.subtitles.get(l).cloned()) - .collect(); - (vec![], subtitles) - }); + .or_insert_with(|| (vec![], vec![])); + subtitles.extend(archive.subtitle.iter().filter_map(|l| { + let stream_subtitle = streams.subtitles.get(l).cloned()?; + let subtitle = Subtitle { + stream_subtitle, + audio_locale: episode.metadata.audio_locale.clone(), + episode_id: episode.id.clone(), + forced: !episode.metadata.is_subbed, + primary: primary_season, + }; + Some(subtitle) + })); formats.push(Format::new_from_episode(episode, stream)); } + + primary_season = false; } Ok(result.into_values().flat_map(|v| v.into_values()).collect()) @@ -562,11 +580,12 @@ fn generate_mkv( target: PathBuf, video_paths: Vec<(TempPath, &Format)>, audio_paths: Vec<(TempPath, &Format)>, - subtitle_paths: Vec<(TempPath, StreamSubtitle)>, + subtitle_paths: Vec<(TempPath, Subtitle)>, ) -> Result<()> { let mut input = vec![]; let mut maps = vec![]; let mut metadata = vec![]; + let mut dispositions = vec![vec![]; subtitle_paths.len()]; for (i, (video_path, format)) in video_paths.iter().enumerate() { input.extend(["-i".to_string(), video_path.to_string_lossy().to_string()]); @@ -611,12 +630,26 @@ fn generate_mkv( ]); metadata.extend([ format!("-metadata:s:s:{}", i), - format!("language={}", subtitle.locale), + format!("language={}", subtitle.stream_subtitle.locale), ]); metadata.extend([ format!("-metadata:s:s:{}", i), - format!("title={}", subtitle.locale.to_human_readable()), + format!( + "title={}", + subtitle.stream_subtitle.locale.to_human_readable() + + if !subtitle.primary { + format!(" [Video: {}]", subtitle.audio_locale.to_human_readable()) + } else { + "".to_string() + } + .as_str() + ), ]); + + // mark forced subtitles + if subtitle.forced { + dispositions[i].push("forced"); + } } let (input_presets, output_presets) = @@ -633,16 +666,28 @@ fn generate_mkv( // if `--default_subtitle ` is given set the default subtitle to the given locale if let Some(position) = subtitle_paths .iter() - .position(|(_, subtitle)| &subtitle.locale == default_subtitle) + .position(|(_, subtitle)| &subtitle.stream_subtitle.locale == default_subtitle) { - command_args.extend([format!("-disposition:s:{}", position), "default".to_string()]) - } else { - command_args.extend(["-disposition:s:0".to_string(), "0".to_string()]) + dispositions[position].push("default"); } - } else { - command_args.extend(["-disposition:s:0".to_string(), "0".to_string()]) } + let disposition_args: Vec = dispositions + .iter() + .enumerate() + .flat_map(|(i, d)| { + vec![ + format!("-disposition:s:{}", i), + if !d.is_empty() { + d.join("+") + } else { + "0".to_string() + }, + ] + }) + .collect(); + command_args.extend(disposition_args); + command_args.extend(output_presets); command_args.extend([ "-f".to_string(), diff --git a/crunchy-cli-core/src/utils/mod.rs b/crunchy-cli-core/src/utils/mod.rs index 5f7a5d2..3b15a89 100644 --- a/crunchy-cli-core/src/utils/mod.rs +++ b/crunchy-cli-core/src/utils/mod.rs @@ -6,3 +6,4 @@ pub mod log; pub mod os; pub mod parse; pub mod sort; +pub mod subtitle; diff --git a/crunchy-cli-core/src/utils/subtitle.rs b/crunchy-cli-core/src/utils/subtitle.rs new file mode 100644 index 0000000..86b9359 --- /dev/null +++ b/crunchy-cli-core/src/utils/subtitle.rs @@ -0,0 +1,11 @@ +use crunchyroll_rs::media::StreamSubtitle; +use crunchyroll_rs::Locale; + +#[derive(Clone)] +pub struct Subtitle { + pub stream_subtitle: StreamSubtitle, + pub audio_locale: Locale, + pub episode_id: String, + pub forced: bool, + pub primary: bool, +} From 7588621f345e5b699985bda9a84cba2226151341 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sat, 7 Jan 2023 16:02:51 +0100 Subject: [PATCH 027/363] Add interactive input to choose season on duplicated season numbers (#55, #82) --- crunchy-cli-core/src/cli/archive.rs | 14 +++- crunchy-cli-core/src/cli/download.rs | 12 ++- crunchy-cli-core/src/cli/utils.rs | 106 ++++++++++++++++++++++++++- 3 files changed, 128 insertions(+), 4 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 8259ab3..2c952d6 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -1,5 +1,5 @@ use crate::cli::log::tab_info; -use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset}; +use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset, find_multiple_seasons_with_same_number, interactive_season_choosing}; use crate::utils::context::Context; use crate::utils::format::{format_string, Format}; use crate::utils::log::progress; @@ -120,6 +120,10 @@ pub struct Archive { #[arg(long)] no_subtitle_optimizations: bool, + #[arg(help = "Ignore interactive input")] + #[arg(short, long, default_value_t = false)] + yes: bool, + #[arg(help = "Crunchyroll series url(s)")] urls: Vec, } @@ -378,10 +382,16 @@ async fn formats_from_series( }) } + if !archive.yes && !find_multiple_seasons_with_same_number(&seasons).is_empty() { + info!(target: "progress_end", "Fetched seasons"); + seasons = interactive_season_choosing(seasons); + info!(target: "progress", "Fetching series details") + } + #[allow(clippy::type_complexity)] let mut result: BTreeMap, Vec)>> = BTreeMap::new(); let mut primary_season = true; - for season in series.seasons().await? { + for season in seasons { if !url_filter.is_season_valid(season.metadata.season_number) || !archive .locale diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index 0411a5e..8da833a 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -1,5 +1,5 @@ use crate::cli::log::tab_info; -use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset}; +use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset, interactive_season_choosing, find_multiple_seasons_with_same_number}; use crate::utils::context::Context; use crate::utils::format::{format_string, Format}; use crate::utils::log::progress; @@ -71,6 +71,10 @@ pub struct Download { #[arg(value_parser = FFmpegPreset::parse)] ffmpeg_preset: Vec, + #[arg(help = "Ignore interactive input")] + #[arg(short, long, default_value_t = false)] + yes: bool, + #[arg(help = "Url(s) to Crunchyroll episodes or series")] urls: Vec, } @@ -348,6 +352,12 @@ async fn formats_from_series( }) } + if !download.yes && !find_multiple_seasons_with_same_number(&seasons).is_empty() { + info!(target: "progress_end", "Fetched seasons"); + seasons = interactive_season_choosing(seasons); + info!(target: "progress", "Fetching series details") + } + let mut formats = vec![]; for season in seasons { if let Some(fmts) = formats_from_season(download, season, url_filter).await? { diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index 26d939c..fde1d08 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -5,9 +5,11 @@ use indicatif::{ProgressBar, ProgressFinish, ProgressStyle}; use log::{debug, LevelFilter}; use std::borrow::{Borrow, BorrowMut}; use std::collections::BTreeMap; -use std::io::Write; +use std::io::{BufRead, Write}; use std::sync::{mpsc, Arc, Mutex}; use std::time::Duration; +use crunchyroll_rs::{Media, Season}; +use regex::Regex; use tokio::task::JoinSet; pub fn find_resolution( @@ -327,3 +329,105 @@ impl FFmpegPreset { )) } } + +pub(crate) fn find_multiple_seasons_with_same_number(seasons: &Vec>) -> Vec { + let mut seasons_map: BTreeMap = BTreeMap::new(); + for season in seasons { + if let Some(s) = seasons_map.get_mut(&season.metadata.season_number) { + *s += 1; + } else { + seasons_map.insert(season.metadata.season_number, 1); + } + } + + seasons_map + .into_iter() + .filter_map(|(k, v)| if v > 1 { Some(k) } else { None }) + .collect() +} + +pub(crate) fn interactive_season_choosing(seasons: Vec>) -> Vec> { + let input_regex = + Regex::new(r"((?P\d+)|(?P\d+)-(?P\d+)?)(\s|$)").unwrap(); + + let mut seasons_map: BTreeMap>> = BTreeMap::new(); + for season in seasons { + if let Some(s) = seasons_map.get_mut(&season.metadata.season_number) { + s.push(season); + } else { + seasons_map.insert(season.metadata.season_number, vec![season]); + } + } + + for (num, season_vec) in seasons_map.iter_mut() { + if season_vec.len() == 1 { + continue; + } + println!(":: Found multiple seasons for season number {}", num); + println!(":: Select the number of the seasons you want to download (eg \"1 2 4\", \"1-3\", \"1-3 5\"):"); + for (i, season) in season_vec.iter().enumerate() { + println!(":: \t{}. {}", i + 1, season.title) + } + let mut stdout = std::io::stdout(); + let _ = write!(stdout, ":: => "); + let _ = stdout.flush(); + let mut user_input = String::new(); + std::io::stdin().lock() + .read_line(&mut user_input) + .expect("cannot open stdin"); + + let mut nums = vec![]; + for capture in input_regex.captures_iter(&user_input) { + if let Some(single) = capture.name("single") { + nums.push(single.as_str().parse().unwrap()); + } else { + let range_from = capture.name("range_from"); + let range_to = capture.name("range_to"); + + // input is '-' which means use all seasons + if range_from.is_none() && range_to.is_none() { + nums = vec![]; + break; + } + let from = range_from + .map(|f| f.as_str().parse::().unwrap() - 1) + .unwrap_or(usize::MIN); + let to = range_from + .map(|f| f.as_str().parse::().unwrap() - 1) + .unwrap_or(usize::MAX); + + nums.extend( + season_vec + .iter() + .enumerate() + .filter_map(|(i, _)| { + if i >= from && i <= to { + Some(i) + } else { + None + } + }) + .collect::>(), + ) + } + } + nums.dedup(); + + if !nums.is_empty() { + let mut remove_count = 0; + for i in 0..season_vec.len() - 1 { + if !nums.contains(&i) { + season_vec.remove(i - remove_count); + remove_count += 1 + } + } + } + } + + seasons_map + .into_values() + .into_iter() + .flatten() + .collect::>>() +} + From b991614dc38e8d36ffb2026a91429e0c686fc855 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sat, 7 Jan 2023 17:14:47 +0100 Subject: [PATCH 028/363] Fix output and download order on duplicated seasons --- crunchy-cli-core/src/utils/sort.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crunchy-cli-core/src/utils/sort.rs b/crunchy-cli-core/src/utils/sort.rs index 089fe18..21df74e 100644 --- a/crunchy-cli-core/src/utils/sort.rs +++ b/crunchy-cli-core/src/utils/sort.rs @@ -30,8 +30,11 @@ pub fn sort_formats_after_seasons(formats: Vec) -> Vec> { let mut as_map = BTreeMap::new(); for format in formats { - as_map.entry(format.season_number).or_insert_with(Vec::new); - as_map.get_mut(&format.season_number).unwrap().push(format); + // the season title is used as key instead of season number to distinguish duplicated season + // numbers which are actually two different seasons; season id is not used as this somehow + // messes up ordering when duplicated seasons exist + as_map.entry(format.season_title.clone()).or_insert_with(Vec::new); + as_map.get_mut(&format.season_title).unwrap().push(format); } let mut sorted = as_map @@ -41,7 +44,7 @@ pub fn sort_formats_after_seasons(formats: Vec) -> Vec> { values }) .collect::>>(); - sorted.sort_by(|a, b| a[0].series_id.cmp(&b[0].series_id)); + sorted.sort_by(|a, b| a[0].season_number.cmp(&b[0].season_number)); sorted } From b65c0e9dfdefb49add0294e136912729ae308ee4 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 8 Jan 2023 17:44:31 +0100 Subject: [PATCH 029/363] Update dependencies & version --- Cargo.lock | 32 ++++++++++++++++---------------- Cargo.toml | 4 ++-- crunchy-cli-core/Cargo.lock | 30 +++++++++++++++--------------- crunchy-cli-core/Cargo.toml | 4 ++-- 4 files changed, 35 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eb7c6a3..7177ac1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,9 +39,9 @@ checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" [[package]] name = "async-trait" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d1d8ab452a3936018a687b20e6f7cf5363d713b732b8884001317b0e48aa3" +checksum = "705339e0e4a9690e2908d2b3d049d85682cf19fbd5782494498fbf7003a6a282" dependencies = [ "proc-macro2", "quote", @@ -279,7 +279,7 @@ dependencies = [ [[package]] name = "crunchy-cli" -version = "3.0.0-dev.6" +version = "3.0.0-dev.7" dependencies = [ "chrono", "clap", @@ -291,7 +291,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-dev.6" +version = "3.0.0-dev.7" dependencies = [ "anyhow", "async-trait", @@ -395,9 +395,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add3fc1717409d029b20c5b6903fc0c0b02fa6741d820054f4a2efa5e5816fd" +checksum = "51d1075c37807dcf850c379432f0df05ba52cc30f279c5cfc43cc221ce7f8579" dependencies = [ "cc", "cxxbridge-flags", @@ -407,9 +407,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c87959ba14bc6fbc61df77c3fcfe180fc32b93538c4f1031dd802ccb5f2ff0" +checksum = "5044281f61b27bc598f2f6647d480aed48d2bf52d6eb0b627d84c0361b17aa70" dependencies = [ "cc", "codespan-reporting", @@ -422,15 +422,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69a3e162fde4e594ed2b07d0f83c6c67b745e7f28ce58c6df5e6b6bef99dfb59" +checksum = "61b50bc93ba22c27b0d31128d2d130a0a6b3d267ae27ef7e4fae2167dfe8781c" [[package]] name = "cxxbridge-macro" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e7e2adeb6a0d4a282e581096b06e1791532b7d576dcde5ccd9382acf55db8e6" +checksum = "39e61fda7e62115119469c7b3591fd913ecca96fb766cfd3f2e2502ab7bc87a5" dependencies = [ "proc-macro2", "quote", @@ -1669,9 +1669,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.23.0" +version = "1.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46" +checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae" dependencies = [ "autocfg", "bytes", @@ -1759,9 +1759,9 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "typenum" diff --git a/Cargo.toml b/Cargo.toml index 76ed950..ec8e4ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,11 +1,11 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-dev.6" +version = "3.0.0-dev.7" edition = "2021" [dependencies] -tokio = { version = "1.23", features = ["macros", "rt-multi-thread", "time"], default-features = false } +tokio = { version = "1.24", features = ["macros", "rt-multi-thread", "time"], default-features = false } crunchy-cli-core = { path = "./crunchy-cli-core" } diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index b5dd8a1..4b1a723 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -39,9 +39,9 @@ checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" [[package]] name = "async-trait" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d1d8ab452a3936018a687b20e6f7cf5363d713b732b8884001317b0e48aa3" +checksum = "705339e0e4a9690e2908d2b3d049d85682cf19fbd5782494498fbf7003a6a282" dependencies = [ "proc-macro2", "quote", @@ -260,7 +260,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-dev.6" +version = "3.0.0-dev.7" dependencies = [ "anyhow", "async-trait", @@ -364,9 +364,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add3fc1717409d029b20c5b6903fc0c0b02fa6741d820054f4a2efa5e5816fd" +checksum = "51d1075c37807dcf850c379432f0df05ba52cc30f279c5cfc43cc221ce7f8579" dependencies = [ "cc", "cxxbridge-flags", @@ -376,9 +376,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c87959ba14bc6fbc61df77c3fcfe180fc32b93538c4f1031dd802ccb5f2ff0" +checksum = "5044281f61b27bc598f2f6647d480aed48d2bf52d6eb0b627d84c0361b17aa70" dependencies = [ "cc", "codespan-reporting", @@ -391,15 +391,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69a3e162fde4e594ed2b07d0f83c6c67b745e7f28ce58c6df5e6b6bef99dfb59" +checksum = "61b50bc93ba22c27b0d31128d2d130a0a6b3d267ae27ef7e4fae2167dfe8781c" [[package]] name = "cxxbridge-macro" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e7e2adeb6a0d4a282e581096b06e1791532b7d576dcde5ccd9382acf55db8e6" +checksum = "39e61fda7e62115119469c7b3591fd913ecca96fb766cfd3f2e2502ab7bc87a5" dependencies = [ "proc-macro2", "quote", @@ -1632,9 +1632,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.23.0" +version = "1.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46" +checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae" dependencies = [ "autocfg", "bytes", @@ -1722,9 +1722,9 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "typenum" diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 566b8c4..a3ae301 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-dev.6" +version = "3.0.0-dev.7" edition = "2021" [dependencies] @@ -23,7 +23,7 @@ serde_json = "1.0" signal-hook = "0.3" tempfile = "3.3" terminal_size = "0.2" -tokio = { version = "1.23", features = ["macros", "rt-multi-thread", "time"] } +tokio = { version = "1.24", features = ["macros", "rt-multi-thread", "time"] } sys-locale = "0.2" [build-dependencies] From 13f54c0da636e7093d347bba5c126f8e895ebfe1 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 8 Jan 2023 18:06:37 +0100 Subject: [PATCH 030/363] Fix interactive season choosing activation on url filter excluded seasons --- crunchy-cli-core/src/cli/archive.rs | 5 ++++- crunchy-cli-core/src/cli/download.rs | 11 +++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 2c952d6..b9b7de0 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -379,7 +379,10 @@ async fn formats_from_series( .locale .iter() .any(|l| s.metadata.audio_locales.contains(l)) - }) + }); + // remove seasons which match the url filter. this is mostly done to not trigger the + // interactive season choosing when dupilcated seasons are excluded by the filter + seasons.retain(|s| url_filter.is_season_valid(s.metadata.season_number)) } if !archive.yes && !find_multiple_seasons_with_same_number(&seasons).is_empty() { diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index 8da833a..25c1262 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -349,7 +349,10 @@ async fn formats_from_series( seasons.retain(|s| { s.metadata.season_number != season.first().unwrap().metadata.season_number || s.metadata.audio_locales.contains(&download.audio) - }) + }); + // remove seasons which match the url filter. this is mostly done to not trigger the + // interactive season choosing when dupilcated seasons are excluded by the filter + seasons.retain(|s| url_filter.is_season_valid(s.metadata.season_number)) } if !download.yes && !find_multiple_seasons_with_same_number(&seasons).is_empty() { @@ -373,14 +376,14 @@ async fn formats_from_season( season: Media, url_filter: &UrlFilter, ) -> Result>> { - if !season.metadata.audio_locales.contains(&download.audio) { + if !url_filter.is_season_valid(season.metadata.season_number) { + return Ok(None); + } else if !season.metadata.audio_locales.contains(&download.audio) { error!( "Season {} ({}) is not available with {} audio", season.metadata.season_number, season.title, download.audio ); return Ok(None); - } else if !url_filter.is_season_valid(season.metadata.season_number) { - return Ok(None); } let mut formats = vec![]; From 4b33ef02c61d1ce8b60900785868a3445f779e90 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 9 Jan 2023 10:27:28 +0100 Subject: [PATCH 031/363] Fix output formatting for full path (#101) --- crunchy-cli-core/src/cli/archive.rs | 20 ++++++-------------- crunchy-cli-core/src/cli/download.rs | 20 ++++++-------------- crunchy-cli-core/src/utils/format.rs | 9 ++++++--- 3 files changed, 18 insertions(+), 31 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index b9b7de0..f9a6a04 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -1,7 +1,7 @@ use crate::cli::log::tab_info; use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset, find_multiple_seasons_with_same_number, interactive_season_choosing}; use crate::utils::context::Context; -use crate::utils::format::{format_string, Format}; +use crate::utils::format::{Format, format_path}; use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file, tempfile}; use crate::utils::parse::{parse_url, UrlFilter}; @@ -240,19 +240,11 @@ impl Execute for Archive { for (formats, mut subtitles) in archive_formats { let (primary, additionally) = formats.split_first().unwrap(); - let mut path = PathBuf::from(&self.output); - path = free_file( - path.with_file_name(format_string( - if let Some(fname) = path.file_name() { - fname.to_str().unwrap() - } else { - "{title}.mkv" - } - .to_string(), - primary, - true, - )), - ); + let path = free_file(format_path(if self.output.is_empty() { + "{title}.mkv" + } else { + &self.output + }.into(), &primary, true)); info!( "Downloading {} to '{}'", diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index 25c1262..a7ed773 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -1,7 +1,7 @@ use crate::cli::log::tab_info; use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset, interactive_season_choosing, find_multiple_seasons_with_same_number}; use crate::utils::context::Context; -use crate::utils::format::{format_string, Format}; +use crate::utils::format::{Format, format_path}; use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file}; use crate::utils::parse::{parse_url, UrlFilter}; @@ -206,19 +206,11 @@ impl Execute for Download { } for format in formats { - let mut path = PathBuf::from(&self.output); - path = free_file( - path.with_file_name(format_string( - if let Some(fname) = path.file_name() { - fname.to_str().unwrap() - } else { - "{title}.ts" - } - .to_string(), - &format, - true, - )), - ); + let path = free_file(format_path(if self.output.is_empty() { + "{title}.mkv" + } else { + &self.output + }.into(), &format, true)); info!( "Downloading {} to '{}'", diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index c463ea8..ee48e2a 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -1,3 +1,4 @@ +use std::path::PathBuf; use crunchyroll_rs::media::VariantData; use crunchyroll_rs::{Episode, Locale, Media, Movie}; use std::time::Duration; @@ -65,7 +66,7 @@ impl Format { /// Formats the given string if it has specific pattern in it. It's possible to sanitize it which /// removes characters which can cause failures if the output string is used as a file name. -pub fn format_string(s: String, format: &Format, sanitize: bool) -> String { +pub fn format_path(path: PathBuf, format: &Format, sanitize: bool) -> PathBuf { let sanitize_func = if sanitize { |s: &str| sanitize_filename::sanitize(s) } else { @@ -73,7 +74,9 @@ pub fn format_string(s: String, format: &Format, sanitize: bool) -> String { |s: &str| s.to_string() }; - s.replace("{title}", &sanitize_func(&format.title)) + let as_string = path.to_string_lossy().to_string(); + + PathBuf::from(as_string.replace("{title}", &sanitize_func(&format.title)) .replace("{series_name}", &sanitize_func(&format.series_name)) .replace("{season_name}", &sanitize_func(&format.season_title)) .replace("{audio}", &sanitize_func(&format.audio.to_string())) @@ -99,5 +102,5 @@ pub fn format_string(s: String, format: &Format, sanitize: bool) -> String { ) .replace("{series_id}", &sanitize_func(&format.series_id)) .replace("{season_id}", &sanitize_func(&format.season_id)) - .replace("{episode_id}", &sanitize_func(&format.id)) + .replace("{episode_id}", &sanitize_func(&format.id))) } From 12be16417ffdd0bda87b9c06aaf7f50d5454b178 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 9 Jan 2023 16:55:10 +0100 Subject: [PATCH 032/363] Fix interactive season choosing false-positive triggering --- Cargo.lock | 85 +++++++--------------------- crunchy-cli-core/Cargo.lock | 85 +++++++--------------------- crunchy-cli-core/Cargo.toml | 1 + crunchy-cli-core/src/cli/archive.rs | 31 +++++----- crunchy-cli-core/src/cli/download.rs | 22 ++++--- crunchy-cli-core/src/cli/utils.rs | 67 ++++++++++++++++------ crunchy-cli-core/src/utils/format.rs | 59 ++++++++++--------- crunchy-cli-core/src/utils/sort.rs | 4 +- 8 files changed, 157 insertions(+), 197 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7177ac1..77e1342 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -222,7 +222,7 @@ dependencies = [ "lazy_static", "libc", "unicode-width", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -302,6 +302,7 @@ dependencies = [ "ctrlc", "dirs", "indicatif", + "lazy_static", "log", "num_cpus", "regex", @@ -390,7 +391,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1631ca6e3c59112501a9d87fd86f21591ff77acd31331e8a73f8d80a65bbdd71" dependencies = [ "nix", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -859,14 +860,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" dependencies = [ "libc", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] name = "ipnet" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11b0d96e660696543b251e58030cf9787df56da39dab19ad60eae7353040917e" +checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" [[package]] name = "is-terminal" @@ -877,7 +878,7 @@ dependencies = [ "hermit-abi", "io-lifetimes", "rustix", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -980,7 +981,7 @@ dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1231,9 +1232,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" +checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" dependencies = [ "aho-corasick", "memchr", @@ -1338,7 +1339,7 @@ dependencies = [ "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1380,12 +1381,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" +checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" dependencies = [ - "lazy_static", - "windows-sys 0.36.1", + "windows-sys", ] [[package]] @@ -1591,7 +1591,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb20089a8ba2b69debd491f8d2d023761cbf196e999218c591fa1e7e15a21907" dependencies = [ "rustix", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1682,7 +1682,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio-macros", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1973,19 +1973,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" -dependencies = [ - "windows_aarch64_msvc 0.36.1", - "windows_i686_gnu 0.36.1", - "windows_i686_msvc 0.36.1", - "windows_x86_64_gnu 0.36.1", - "windows_x86_64_msvc 0.36.1", -] - [[package]] name = "windows-sys" version = "0.42.0" @@ -1993,12 +1980,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc 0.42.0", - "windows_i686_gnu 0.42.0", - "windows_i686_msvc 0.42.0", - "windows_x86_64_gnu 0.42.0", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", "windows_x86_64_gnullvm", - "windows_x86_64_msvc 0.42.0", + "windows_x86_64_msvc", ] [[package]] @@ -2007,48 +1994,24 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" -[[package]] -name = "windows_aarch64_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" - [[package]] name = "windows_aarch64_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" -[[package]] -name = "windows_i686_gnu" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" - [[package]] name = "windows_i686_gnu" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" -[[package]] -name = "windows_i686_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" - [[package]] name = "windows_i686_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" -[[package]] -name = "windows_x86_64_gnu" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" - [[package]] name = "windows_x86_64_gnu" version = "0.42.0" @@ -2061,12 +2024,6 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" -[[package]] -name = "windows_x86_64_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" - [[package]] name = "windows_x86_64_msvc" version = "0.42.0" diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index 4b1a723..5bdabf0 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -203,7 +203,7 @@ dependencies = [ "lazy_static", "libc", "unicode-width", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -271,6 +271,7 @@ dependencies = [ "ctrlc", "dirs", "indicatif", + "lazy_static", "log", "num_cpus", "regex", @@ -359,7 +360,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1631ca6e3c59112501a9d87fd86f21591ff77acd31331e8a73f8d80a65bbdd71" dependencies = [ "nix", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -828,14 +829,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" dependencies = [ "libc", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] name = "ipnet" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11b0d96e660696543b251e58030cf9787df56da39dab19ad60eae7353040917e" +checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" [[package]] name = "is-terminal" @@ -846,7 +847,7 @@ dependencies = [ "hermit-abi", "io-lifetimes", "rustix", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -949,7 +950,7 @@ dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1200,9 +1201,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" +checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" dependencies = [ "aho-corasick", "memchr", @@ -1301,7 +1302,7 @@ dependencies = [ "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1343,12 +1344,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" +checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" dependencies = [ - "lazy_static", - "windows-sys 0.36.1", + "windows-sys", ] [[package]] @@ -1554,7 +1554,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb20089a8ba2b69debd491f8d2d023761cbf196e999218c591fa1e7e15a21907" dependencies = [ "rustix", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1645,7 +1645,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio-macros", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1936,19 +1936,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" -dependencies = [ - "windows_aarch64_msvc 0.36.1", - "windows_i686_gnu 0.36.1", - "windows_i686_msvc 0.36.1", - "windows_x86_64_gnu 0.36.1", - "windows_x86_64_msvc 0.36.1", -] - [[package]] name = "windows-sys" version = "0.42.0" @@ -1956,12 +1943,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc 0.42.0", - "windows_i686_gnu 0.42.0", - "windows_i686_msvc 0.42.0", - "windows_x86_64_gnu 0.42.0", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", "windows_x86_64_gnullvm", - "windows_x86_64_msvc 0.42.0", + "windows_x86_64_msvc", ] [[package]] @@ -1970,48 +1957,24 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" -[[package]] -name = "windows_aarch64_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" - [[package]] name = "windows_aarch64_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" -[[package]] -name = "windows_i686_gnu" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" - [[package]] name = "windows_i686_gnu" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" -[[package]] -name = "windows_i686_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" - [[package]] name = "windows_i686_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" -[[package]] -name = "windows_x86_64_gnu" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" - [[package]] name = "windows_x86_64_gnu" version = "0.42.0" @@ -2024,12 +1987,6 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" -[[package]] -name = "windows_x86_64_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" - [[package]] name = "windows_x86_64_msvc" version = "0.42.0" diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index a3ae301..5bda1db 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -14,6 +14,7 @@ csv = "1.1" ctrlc = "3.2" dirs = "4.0" indicatif = "0.17" +lazy_static = "1.4" log = { version = "0.4", features = ["std"] } num_cpus = "1.15" regex = "1.7" diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index f9a6a04..7a96c05 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -1,7 +1,10 @@ use crate::cli::log::tab_info; -use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset, find_multiple_seasons_with_same_number, interactive_season_choosing}; +use crate::cli::utils::{ + download_segments, find_multiple_seasons_with_same_number, find_resolution, + interactive_season_choosing, FFmpegPreset, +}; use crate::utils::context::Context; -use crate::utils::format::{Format, format_path}; +use crate::utils::format::{format_path, Format}; use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file, tempfile}; use crate::utils::parse::{parse_url, UrlFilter}; @@ -240,11 +243,16 @@ impl Execute for Archive { for (formats, mut subtitles) in archive_formats { let (primary, additionally) = formats.split_first().unwrap(); - let path = free_file(format_path(if self.output.is_empty() { - "{title}.mkv" - } else { - &self.output - }.into(), &primary, true)); + let path = free_file(format_path( + if self.output.is_empty() { + "{title}.mkv" + } else { + &self.output + } + .into(), + &primary, + true, + )); info!( "Downloading {} to '{}'", @@ -387,15 +395,6 @@ async fn formats_from_series( let mut result: BTreeMap, Vec)>> = BTreeMap::new(); let mut primary_season = true; for season in seasons { - if !url_filter.is_season_valid(season.metadata.season_number) - || !archive - .locale - .iter() - .any(|l| season.metadata.audio_locales.contains(l)) - { - continue; - } - for episode in season.episodes().await? { if !url_filter.is_episode_valid( episode.metadata.episode_number, diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index a7ed773..bdca291 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -1,7 +1,10 @@ use crate::cli::log::tab_info; -use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset, interactive_season_choosing, find_multiple_seasons_with_same_number}; +use crate::cli::utils::{ + download_segments, find_multiple_seasons_with_same_number, find_resolution, + interactive_season_choosing, FFmpegPreset, +}; use crate::utils::context::Context; -use crate::utils::format::{Format, format_path}; +use crate::utils::format::{format_path, Format}; use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file}; use crate::utils::parse::{parse_url, UrlFilter}; @@ -206,11 +209,16 @@ impl Execute for Download { } for format in formats { - let path = free_file(format_path(if self.output.is_empty() { - "{title}.mkv" - } else { - &self.output - }.into(), &format, true)); + let path = free_file(format_path( + if self.output.is_empty() { + "{title}.mkv" + } else { + &self.output + } + .into(), + &format, + true, + )); info!( "Downloading {} to '{}'", diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index fde1d08..bc19b07 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -1,15 +1,16 @@ use crate::utils::context::Context; use anyhow::{bail, Result}; use crunchyroll_rs::media::{Resolution, VariantData, VariantSegment}; +use crunchyroll_rs::{Media, Season}; use indicatif::{ProgressBar, ProgressFinish, ProgressStyle}; +use lazy_static::lazy_static; use log::{debug, LevelFilter}; +use regex::Regex; use std::borrow::{Borrow, BorrowMut}; use std::collections::BTreeMap; use std::io::{BufRead, Write}; use std::sync::{mpsc, Arc, Mutex}; use std::time::Duration; -use crunchyroll_rs::{Media, Season}; -use regex::Regex; use tokio::task::JoinSet; pub fn find_resolution( @@ -111,7 +112,7 @@ pub async fn download_segments( }; buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec(); - + let mut c = thread_count.lock().unwrap(); debug!( "Downloaded and decrypted segment [{}/{} {:.2}%] {}", @@ -120,14 +121,14 @@ pub async fn download_segments( ((*c + 1) as f64 / total_segments as f64) * 100f64, segment.url ); - + thread_sender.send((num as i32 + (i * cpus) as i32, buf))?; - + *c += 1; } Ok(()) }; - + let result = download().await; if result.is_err() { @@ -149,7 +150,7 @@ pub async fn download_segments( for (pos, bytes) in receiver.iter() { // if the position is lower than 0, an error occured in the sending download thread if pos < 0 { - break + break; } if let Some(p) = &progress { @@ -330,6 +331,10 @@ impl FFmpegPreset { } } +lazy_static! { + static ref DUPLICATED_SEASONS_MULTILANG_REGEX: Regex = Regex::new(r"(-castilian|-english|-english-in|-french|-german|-hindi|-italian|-portuguese|-russian|-spanish)$").unwrap(); +} + pub(crate) fn find_multiple_seasons_with_same_number(seasons: &Vec>) -> Vec { let mut seasons_map: BTreeMap = BTreeMap::new(); for season in seasons { @@ -342,7 +347,25 @@ pub(crate) fn find_multiple_seasons_with_same_number(seasons: &Vec seasons_map .into_iter() - .filter_map(|(k, v)| if v > 1 { Some(k) } else { None }) + .filter_map(|(k, v)| { + if v > 1 { + // check if the different seasons are actual the same but with different dub languages + let mut multilang_season_vec: Vec = seasons + .iter() + .map(|s| { + DUPLICATED_SEASONS_MULTILANG_REGEX + .replace(s.slug_title.trim_end_matches("-dub"), "") + .to_string() + }) + .collect(); + multilang_season_vec.dedup(); + + if multilang_season_vec.len() > 1 { + return Some(k); + } + } + None + }) .collect() } @@ -363,6 +386,22 @@ pub(crate) fn interactive_season_choosing(seasons: Vec>) -> Vec = season_vec + .iter() + .map(|s| { + DUPLICATED_SEASONS_MULTILANG_REGEX + .replace(s.slug_title.trim_end_matches("-dub"), "") + .to_string() + }) + .collect(); + multilang_season_vec.dedup(); + + if multilang_season_vec.len() == 1 { + continue; + } + println!(":: Found multiple seasons for season number {}", num); println!(":: Select the number of the seasons you want to download (eg \"1 2 4\", \"1-3\", \"1-3 5\"):"); for (i, season) in season_vec.iter().enumerate() { @@ -372,7 +411,8 @@ pub(crate) fn interactive_season_choosing(seasons: Vec>) -> Vec "); let _ = stdout.flush(); let mut user_input = String::new(); - std::io::stdin().lock() + std::io::stdin() + .lock() .read_line(&mut user_input) .expect("cannot open stdin"); @@ -400,13 +440,7 @@ pub(crate) fn interactive_season_choosing(seasons: Vec>) -> Vec= from && i <= to { - Some(i) - } else { - None - } - }) + .filter_map(|(i, _)| if i >= from && i <= to { Some(i) } else { None }) .collect::>(), ) } @@ -430,4 +464,3 @@ pub(crate) fn interactive_season_choosing(seasons: Vec>) -> Vec>>() } - diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index ee48e2a..035a61c 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -1,6 +1,6 @@ -use std::path::PathBuf; use crunchyroll_rs::media::VariantData; use crunchyroll_rs::{Episode, Locale, Media, Movie}; +use std::path::PathBuf; use std::time::Duration; #[derive(Clone)] @@ -76,31 +76,34 @@ pub fn format_path(path: PathBuf, format: &Format, sanitize: bool) -> PathBuf { let as_string = path.to_string_lossy().to_string(); - PathBuf::from(as_string.replace("{title}", &sanitize_func(&format.title)) - .replace("{series_name}", &sanitize_func(&format.series_name)) - .replace("{season_name}", &sanitize_func(&format.season_title)) - .replace("{audio}", &sanitize_func(&format.audio.to_string())) - .replace( - "{resolution}", - &sanitize_func(&format.stream.resolution.to_string()), - ) - .replace( - "{padded_season_number}", - &sanitize_func(&format!("{:0>2}", format.season_number.to_string())), - ) - .replace( - "{season_number}", - &sanitize_func(&format.season_number.to_string()), - ) - .replace( - "{padded_episode_number}", - &sanitize_func(&format!("{:0>2}", format.number.to_string())), - ) - .replace( - "{episode_number}", - &sanitize_func(&format.number.to_string()), - ) - .replace("{series_id}", &sanitize_func(&format.series_id)) - .replace("{season_id}", &sanitize_func(&format.season_id)) - .replace("{episode_id}", &sanitize_func(&format.id))) + PathBuf::from( + as_string + .replace("{title}", &sanitize_func(&format.title)) + .replace("{series_name}", &sanitize_func(&format.series_name)) + .replace("{season_name}", &sanitize_func(&format.season_title)) + .replace("{audio}", &sanitize_func(&format.audio.to_string())) + .replace( + "{resolution}", + &sanitize_func(&format.stream.resolution.to_string()), + ) + .replace( + "{padded_season_number}", + &sanitize_func(&format!("{:0>2}", format.season_number.to_string())), + ) + .replace( + "{season_number}", + &sanitize_func(&format.season_number.to_string()), + ) + .replace( + "{padded_episode_number}", + &sanitize_func(&format!("{:0>2}", format.number.to_string())), + ) + .replace( + "{episode_number}", + &sanitize_func(&format.number.to_string()), + ) + .replace("{series_id}", &sanitize_func(&format.series_id)) + .replace("{season_id}", &sanitize_func(&format.season_id)) + .replace("{episode_id}", &sanitize_func(&format.id)), + ) } diff --git a/crunchy-cli-core/src/utils/sort.rs b/crunchy-cli-core/src/utils/sort.rs index 21df74e..9f8d81c 100644 --- a/crunchy-cli-core/src/utils/sort.rs +++ b/crunchy-cli-core/src/utils/sort.rs @@ -33,7 +33,9 @@ pub fn sort_formats_after_seasons(formats: Vec) -> Vec> { // the season title is used as key instead of season number to distinguish duplicated season // numbers which are actually two different seasons; season id is not used as this somehow // messes up ordering when duplicated seasons exist - as_map.entry(format.season_title.clone()).or_insert_with(Vec::new); + as_map + .entry(format.season_title.clone()) + .or_insert_with(Vec::new); as_map.get_mut(&format.season_title).unwrap().push(format); } From 29845ba6e53ce877d85aca47db840c52835fa61b Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 9 Jan 2023 17:26:04 +0100 Subject: [PATCH 033/363] Re-order instructions --- crunchy-cli-core/src/cli/archive.rs | 8 +++--- crunchy-cli-core/src/cli/download.rs | 10 +++++-- crunchy-cli-core/src/utils/format.rs | 43 +++++++++++++++------------- crunchy-cli-core/src/utils/sort.rs | 2 +- 4 files changed, 35 insertions(+), 28 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 7a96c05..1de0997 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -208,7 +208,7 @@ impl Execute for Archive { format.stream.resolution, format.stream.fps, format.season_number, - format.number, + format.episode_number, ) } } @@ -234,7 +234,7 @@ impl Execute for Archive { format.stream.resolution, format.stream.fps, format.season_number, - format.number + format.episode_number ) } } @@ -266,7 +266,7 @@ impl Execute for Archive { tab_info!( "Episode: S{:02}E{:02}", primary.season_number, - primary.number + primary.episode_number ); tab_info!( "Audio: {} (primary), {}", @@ -318,7 +318,7 @@ impl Execute for Archive { // Remove subtitles of deleted video if only_audio { - subtitles.retain(|s| s.episode_id != additional.id); + subtitles.retain(|s| s.episode_id != additional.episode_id); } } diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index bdca291..14f9503 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -182,7 +182,7 @@ impl Execute for Download { format.stream.resolution, format.stream.fps, format.season_number, - format.number, + format.episode_number, ) } } @@ -202,7 +202,7 @@ impl Execute for Download { format.stream.resolution, format.stream.fps, format.season_number, - format.number + format.episode_number ) } } @@ -229,7 +229,11 @@ impl Execute for Download { path.file_name().unwrap().to_str().unwrap() } ); - tab_info!("Episode: S{:02}E{:02}", format.season_number, format.number); + tab_info!( + "Episode: S{:02}E{:02}", + format.season_number, + format.episode_number + ); tab_info!("Audio: {}", format.audio); tab_info!( "Subtitles: {}", diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index 035a61c..5570960 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -5,10 +5,9 @@ use std::time::Duration; #[derive(Clone)] pub struct Format { - pub id: String, pub title: String, pub description: String, - pub number: u32, + pub audio: Locale, pub duration: Duration, @@ -20,15 +19,17 @@ pub struct Format { pub season_id: String, pub season_title: String, pub season_number: u32, + + pub episode_id: String, + pub episode_number: f32, } impl Format { pub fn new_from_episode(episode: Media, stream: VariantData) -> Self { Self { - id: episode.id, title: episode.title, description: episode.description, - number: episode.metadata.episode_number, + audio: episode.metadata.audio_locale, duration: episode.metadata.duration.to_std().unwrap(), @@ -40,15 +41,17 @@ impl Format { season_id: episode.metadata.season_id, season_title: episode.metadata.season_title, season_number: episode.metadata.season_number, + + episode_id: episode.id, + episode_number: episode.metadata.episode.parse().unwrap_or(episode.metadata.sequence_number), } } pub fn new_from_movie(movie: Media, stream: VariantData) -> Self { Self { - id: movie.id, title: movie.title, description: movie.description, - number: 1, + audio: Locale::ja_JP, duration: movie.metadata.duration.to_std().unwrap(), @@ -60,6 +63,9 @@ impl Format { season_id: movie.metadata.movie_listing_id, season_title: movie.metadata.movie_listing_title, season_number: 1, + + episode_id: movie.id, + episode_number: 1.0, } } } @@ -79,31 +85,28 @@ pub fn format_path(path: PathBuf, format: &Format, sanitize: bool) -> PathBuf { PathBuf::from( as_string .replace("{title}", &sanitize_func(&format.title)) - .replace("{series_name}", &sanitize_func(&format.series_name)) - .replace("{season_name}", &sanitize_func(&format.season_title)) .replace("{audio}", &sanitize_func(&format.audio.to_string())) .replace( "{resolution}", &sanitize_func(&format.stream.resolution.to_string()), ) - .replace( - "{padded_season_number}", - &sanitize_func(&format!("{:0>2}", format.season_number.to_string())), - ) + .replace("{series_id}", &sanitize_func(&format.series_id)) + .replace("{series_name}", &sanitize_func(&format.series_name)) + .replace("{season_id}", &sanitize_func(&format.season_id)) + .replace("{season_name}", &sanitize_func(&format.season_title)) .replace( "{season_number}", &sanitize_func(&format.season_number.to_string()), ) .replace( - "{padded_episode_number}", - &sanitize_func(&format!("{:0>2}", format.number.to_string())), + "{padded_season_number}", + &sanitize_func(&format!("{:0>2}", format.season_number.to_string())), ) + .replace("{episode_id}", &sanitize_func(&format.episode_id)) + .replace("{episode_number}", &sanitize_func(&format.episode_number.to_string())) .replace( - "{episode_number}", - &sanitize_func(&format.number.to_string()), - ) - .replace("{series_id}", &sanitize_func(&format.series_id)) - .replace("{season_id}", &sanitize_func(&format.season_id)) - .replace("{episode_id}", &sanitize_func(&format.id)), + "{padded_episode_number}", + &sanitize_func(&format!("{:0>2}", format.episode_number.to_string())), + ), ) } diff --git a/crunchy-cli-core/src/utils/sort.rs b/crunchy-cli-core/src/utils/sort.rs index 9f8d81c..1af0194 100644 --- a/crunchy-cli-core/src/utils/sort.rs +++ b/crunchy-cli-core/src/utils/sort.rs @@ -42,7 +42,7 @@ pub fn sort_formats_after_seasons(formats: Vec) -> Vec> { let mut sorted = as_map .into_iter() .map(|(_, mut values)| { - values.sort_by(|a, b| a.number.cmp(&b.number)); + values.sort_by(|a, b| a.episode_number.total_cmp(&b.episode_number)); values }) .collect::>>(); From 7d3a90e8112a4ce27158b217a564ded45c08c576 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 9 Jan 2023 19:12:00 +0100 Subject: [PATCH 034/363] Add relative episode number to format --- crunchy-cli-core/src/cli/archive.rs | 29 +++--- crunchy-cli-core/src/cli/download.rs | 54 +++++++--- crunchy-cli-core/src/utils/format.rs | 150 ++++++++++++++++----------- 3 files changed, 145 insertions(+), 88 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 1de0997..96d0737 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -4,7 +4,7 @@ use crate::cli::utils::{ interactive_season_choosing, FFmpegPreset, }; use crate::utils::context::Context; -use crate::utils::format::{format_path, Format}; +use crate::utils::format::Format; use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file, tempfile}; use crate::utils::parse::{parse_url, UrlFilter}; @@ -243,16 +243,17 @@ impl Execute for Archive { for (formats, mut subtitles) in archive_formats { let (primary, additionally) = formats.split_first().unwrap(); - let path = free_file(format_path( - if self.output.is_empty() { - "{title}.mkv" - } else { - &self.output - } - .into(), - &primary, - true, - )); + let path = free_file( + primary.format_path( + if self.output.is_empty() { + "{title}.mkv" + } else { + &self.output + } + .into(), + true, + ), + ); info!( "Downloading {} to '{}'", @@ -395,7 +396,9 @@ async fn formats_from_series( let mut result: BTreeMap, Vec)>> = BTreeMap::new(); let mut primary_season = true; for season in seasons { - for episode in season.episodes().await? { + let episodes = season.episodes().await?; + + for episode in episodes.iter() { if !url_filter.is_episode_valid( episode.metadata.episode_number, episode.metadata.season_number, @@ -434,7 +437,7 @@ async fn formats_from_series( }; Some(subtitle) })); - formats.push(Format::new_from_episode(episode, stream)); + formats.push(Format::new_from_episode(episode, &episodes, stream)); } primary_season = false; diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index 14f9503..d027df7 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -4,7 +4,7 @@ use crate::cli::utils::{ interactive_season_choosing, FFmpegPreset, }; use crate::utils::context::Context; -use crate::utils::format::{format_path, Format}; +use crate::utils::format::Format; use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file}; use crate::utils::parse::{parse_url, UrlFilter}; @@ -16,6 +16,7 @@ use crunchyroll_rs::{ Episode, Locale, Media, MediaCollection, Movie, MovieListing, Season, Series, }; use log::{debug, error, info, warn}; +use std::borrow::Cow; use std::fs::File; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; @@ -148,7 +149,7 @@ impl Execute for Download { episode.metadata.season_title, episode.metadata.series_title ); - format_from_episode(&self, episode, &url_filter, false) + format_from_episode(&self, &episode, &url_filter, None, false) .await? .map(|fmt| vec![fmt]) } @@ -209,16 +210,17 @@ impl Execute for Download { } for format in formats { - let path = free_file(format_path( - if self.output.is_empty() { - "{title}.mkv" - } else { - &self.output - } - .into(), - &format, - true, - )); + let path = free_file( + format.format_path( + if self.output.is_empty() { + "{title}.mkv" + } else { + &self.output + } + .into(), + true, + ), + ); info!( "Downloading {} to '{}'", @@ -392,8 +394,11 @@ async fn formats_from_season( let mut formats = vec![]; - for episode in season.episodes().await? { - if let Some(fmt) = format_from_episode(download, episode, url_filter, true).await? { + let episodes = season.episodes().await?; + for episode in episodes.iter() { + if let Some(fmt) = + format_from_episode(download, &episode, url_filter, Some(&episodes), true).await? + { formats.push(fmt) } } @@ -403,8 +408,9 @@ async fn formats_from_season( async fn format_from_episode( download: &Download, - episode: Media, + episode: &Media, url_filter: &UrlFilter, + season_episodes: Option<&Vec>>, filter_audio: bool, ) -> Result> { if filter_audio && episode.metadata.audio_locale != download.audio { @@ -457,7 +463,21 @@ async fn format_from_episode( ) }; - Ok(Some(Format::new_from_episode(episode, stream))) + let season_eps = if Format::has_relative_episodes_fmt(&download.output) { + if let Some(eps) = season_episodes { + Cow::from(eps) + } else { + Cow::from(episode.season().await?.episodes().await?) + } + } else { + Cow::from(vec![]) + }; + + Ok(Some(Format::new_from_episode( + episode, + &season_eps.to_vec(), + stream, + ))) } async fn format_from_movie_listing( @@ -515,7 +535,7 @@ async fn format_from_movie( } }; - Ok(Some(Format::new_from_movie(movie, stream))) + Ok(Some(Format::new_from_movie(&movie, stream))) } fn some_vec_or_none(v: Vec) -> Option> { diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index 5570960..3db28c3 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -1,5 +1,6 @@ use crunchyroll_rs::media::VariantData; use crunchyroll_rs::{Episode, Locale, Media, Movie}; +use log::warn; use std::path::PathBuf; use std::time::Duration; @@ -22,35 +23,56 @@ pub struct Format { pub episode_id: String, pub episode_number: f32, + pub relative_episode_number: f32, } impl Format { - pub fn new_from_episode(episode: Media, stream: VariantData) -> Self { + pub fn new_from_episode( + episode: &Media, + season_episodes: &Vec>, + stream: VariantData, + ) -> Self { Self { - title: episode.title, - description: episode.description, + title: episode.title.clone(), + description: episode.description.clone(), - audio: episode.metadata.audio_locale, + audio: episode.metadata.audio_locale.clone(), duration: episode.metadata.duration.to_std().unwrap(), stream, - series_id: episode.metadata.series_id, - series_name: episode.metadata.series_title, + series_id: episode.metadata.series_id.clone(), + series_name: episode.metadata.series_title.clone(), - season_id: episode.metadata.season_id, - season_title: episode.metadata.season_title, - season_number: episode.metadata.season_number, + season_id: episode.metadata.season_id.clone(), + season_title: episode.metadata.season_title.clone(), + season_number: episode.metadata.season_number.clone(), - episode_id: episode.id, - episode_number: episode.metadata.episode.parse().unwrap_or(episode.metadata.sequence_number), + episode_id: episode.id.clone(), + episode_number: episode + .metadata + .episode + .parse() + .unwrap_or(episode.metadata.sequence_number), + relative_episode_number: season_episodes + .iter() + .enumerate() + .find_map(|(i, e)| if e == episode { Some((i + 1) as f32) } else { None }) + .unwrap_or_else(|| { + warn!("Cannot find relative episode number for episode {} ({}) of season {} ({}) of {}, using normal episode number", episode.metadata.episode_number, episode.title, episode.metadata.season_number, episode.metadata.season_title, episode.metadata.series_title); + episode + .metadata + .episode + .parse() + .unwrap_or(episode.metadata.sequence_number) + }), } } - pub fn new_from_movie(movie: Media, stream: VariantData) -> Self { + pub fn new_from_movie(movie: &Media, stream: VariantData) -> Self { Self { - title: movie.title, - description: movie.description, + title: movie.title.clone(), + description: movie.description.clone(), audio: Locale::ja_JP, @@ -60,53 +82,65 @@ impl Format { series_id: movie.metadata.movie_listing_id.clone(), series_name: movie.metadata.movie_listing_title.clone(), - season_id: movie.metadata.movie_listing_id, - season_title: movie.metadata.movie_listing_title, + season_id: movie.metadata.movie_listing_id.clone(), + season_title: movie.metadata.movie_listing_title.clone(), season_number: 1, - episode_id: movie.id, + episode_id: movie.id.clone(), episode_number: 1.0, + relative_episode_number: 1.0, } } -} - -/// Formats the given string if it has specific pattern in it. It's possible to sanitize it which -/// removes characters which can cause failures if the output string is used as a file name. -pub fn format_path(path: PathBuf, format: &Format, sanitize: bool) -> PathBuf { - let sanitize_func = if sanitize { - |s: &str| sanitize_filename::sanitize(s) - } else { - // converting this to a string is actually unnecessary - |s: &str| s.to_string() - }; - - let as_string = path.to_string_lossy().to_string(); - - PathBuf::from( - as_string - .replace("{title}", &sanitize_func(&format.title)) - .replace("{audio}", &sanitize_func(&format.audio.to_string())) - .replace( - "{resolution}", - &sanitize_func(&format.stream.resolution.to_string()), - ) - .replace("{series_id}", &sanitize_func(&format.series_id)) - .replace("{series_name}", &sanitize_func(&format.series_name)) - .replace("{season_id}", &sanitize_func(&format.season_id)) - .replace("{season_name}", &sanitize_func(&format.season_title)) - .replace( - "{season_number}", - &sanitize_func(&format.season_number.to_string()), - ) - .replace( - "{padded_season_number}", - &sanitize_func(&format!("{:0>2}", format.season_number.to_string())), - ) - .replace("{episode_id}", &sanitize_func(&format.episode_id)) - .replace("{episode_number}", &sanitize_func(&format.episode_number.to_string())) - .replace( - "{padded_episode_number}", - &sanitize_func(&format!("{:0>2}", format.episode_number.to_string())), - ), - ) + + /// Formats the given string if it has specific pattern in it. It's possible to sanitize it which + /// removes characters which can cause failures if the output string is used as a file name. + pub fn format_path(&self, path: PathBuf, sanitize: bool) -> PathBuf { + let sanitize_func = if sanitize { + |s: &str| sanitize_filename::sanitize(s) + } else { + // converting this to a string is actually unnecessary + |s: &str| s.to_string() + }; + + let as_string = path.to_string_lossy().to_string(); + + PathBuf::from( + as_string + .replace("{title}", &sanitize_func(&self.title)) + .replace("{audio}", &sanitize_func(&self.audio.to_string())) + .replace( + "{resolution}", + &sanitize_func(&self.stream.resolution.to_string()), + ) + .replace("{series_id}", &sanitize_func(&self.series_id)) + .replace("{series_name}", &sanitize_func(&self.series_name)) + .replace("{season_id}", &sanitize_func(&self.season_id)) + .replace("{season_name}", &sanitize_func(&self.season_title)) + .replace( + "{season_number}", + &sanitize_func(&self.season_number.to_string()), + ) + .replace( + "{padded_season_number}", + &sanitize_func(&format!("{:0>2}", self.season_number.to_string())), + ) + .replace("{episode_id}", &sanitize_func(&self.episode_id)) + .replace( + "{episode_number}", + &sanitize_func(&self.episode_number.to_string()), + ) + .replace( + "{padded_episode_number}", + &sanitize_func(&format!("{:0>2}", self.episode_number.to_string())), + ) + .replace( + "{relative_episode_number}", + &sanitize_func(&format!("{:0>2}", self.relative_episode_number.to_string())), + ), + ) + } + + pub fn has_relative_episodes_fmt>(s: S) -> bool { + return s.as_ref().contains("{relative_episode_number}"); + } } From 2ea036d4c6b7faa5bdc33da1eb9a630126ef1e05 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 9 Jan 2023 19:12:31 +0100 Subject: [PATCH 035/363] Remove padded_*_number and make it default for *_number for output format --- crunchy-cli-core/src/utils/format.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index 3db28c3..60596ad 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -118,19 +118,11 @@ impl Format { .replace("{season_name}", &sanitize_func(&self.season_title)) .replace( "{season_number}", - &sanitize_func(&self.season_number.to_string()), - ) - .replace( - "{padded_season_number}", &sanitize_func(&format!("{:0>2}", self.season_number.to_string())), ) .replace("{episode_id}", &sanitize_func(&self.episode_id)) .replace( "{episode_number}", - &sanitize_func(&self.episode_number.to_string()), - ) - .replace( - "{padded_episode_number}", &sanitize_func(&format!("{:0>2}", self.episode_number.to_string())), ) .replace( From a0aab3bfb964195128abb25ed5c0cfa0f4841707 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 9 Jan 2023 23:25:16 +0100 Subject: [PATCH 036/363] Add arabic locale in duplicated seasons check --- crunchy-cli-core/src/cli/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index bc19b07..da56f5e 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -332,7 +332,7 @@ impl FFmpegPreset { } lazy_static! { - static ref DUPLICATED_SEASONS_MULTILANG_REGEX: Regex = Regex::new(r"(-castilian|-english|-english-in|-french|-german|-hindi|-italian|-portuguese|-russian|-spanish)$").unwrap(); + static ref DUPLICATED_SEASONS_MULTILANG_REGEX: Regex = Regex::new(r"(-arabic|-castilian|-english|-english-in|-french|-german|-hindi|-italian|-portuguese|-russian|-spanish)$").unwrap(); } pub(crate) fn find_multiple_seasons_with_same_number(seasons: &Vec>) -> Vec { From 3029325776de09ef159081df847ae67cea3be93d Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 9 Jan 2023 23:40:53 +0100 Subject: [PATCH 037/363] Add check if request locale is valid (#102) --- crunchy-cli-core/src/lib.rs | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index 52a4da1..6cef42e 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -6,7 +6,7 @@ use anyhow::bail; use anyhow::Result; use clap::{Parser, Subcommand}; use crunchyroll_rs::{Crunchyroll, Locale}; -use log::{debug, error, LevelFilter}; +use log::{debug, error, warn, LevelFilter}; use std::{env, fs}; mod cli; @@ -189,8 +189,41 @@ async fn create_ctx(cli: &Cli) -> Result { } async fn crunchyroll_session(cli: &Cli) -> Result { + let supported_langs = vec![ + Locale::ar_ME, + Locale::de_DE, + Locale::en_US, + Locale::es_ES, + Locale::es_419, + Locale::fr_FR, + Locale::it_IT, + Locale::pt_BR, + Locale::pt_PT, + Locale::ru_RU, + ]; + let locale = if let Some(lang) = &cli.lang { + if !supported_langs.contains(lang) { + bail!( + "Via `--lang` specified language is not supported. Supported languages: {}", + supported_langs + .iter() + .map(|l| format!("`{}` ({})", l.to_string(), l.to_human_readable())) + .collect::>() + .join(", ") + ) + } + lang.clone() + } else { + let mut lang = system_locale(); + if !supported_langs.contains(&lang) { + warn!("Recognized system locale is not supported. Using en-US as default. Use `--lang` to overwrite the used language"); + lang = Locale::en_US + } + lang + }; + let builder = Crunchyroll::builder() - .locale(cli.lang.clone().unwrap_or_else(system_locale)) + .locale(locale) .stabilization_locales(true); let login_methods_count = cli.login_method.credentials.is_some() as u8 From 5ce5b249c91fbfa752e4779bb0a9776c740ec88b Mon Sep 17 00:00:00 2001 From: ByteDream Date: Tue, 10 Jan 2023 19:20:08 +0100 Subject: [PATCH 038/363] Add relative episode number to cli help --- crunchy-cli-core/src/cli/archive.rs | 3 +-- crunchy-cli-core/src/cli/download.rs | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 96d0737..768c629 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -67,10 +67,9 @@ pub struct Archive { {season_name} → Name of the season\n \ {audio} → Audio language of the video\n \ {resolution} → Resolution of the video\n \ - {padded_season_number} → Number of the season padded to double digits\n \ {season_number} → Number of the season\n \ - {padded_episode_number} → Number of the episode padded to double digits\n \ {episode_number} → Number of the episode\n \ + {relative_episode_number} → Number of the episode relative to its season\ {series_id} → ID of the series\n \ {season_id} → ID of the season\n \ {episode_id} → ID of the episode")] diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index d027df7..7a49a4c 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -45,10 +45,9 @@ pub struct Download { {season_name} → Name of the season\n \ {audio} → Audio language of the video\n \ {resolution} → Resolution of the video\n \ - {padded_season_number} → Number of the season padded to double digits\n \ {season_number} → Number of the season\n \ - {padded_episode_number} → Number of the episode padded to double digits\n \ {episode_number} → Number of the episode\n \ + {relative_episode_number} → Number of the episode relative to its season\ {series_id} → ID of the series\n \ {season_id} → ID of the season\n \ {episode_id} → ID of the episode")] From 17233f2fd2721dc4f9e1b86957cb404d09f73249 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Tue, 10 Jan 2023 22:15:36 +0100 Subject: [PATCH 039/363] Update dependencies and version --- Cargo.lock | 26 ++++++++++++++++---------- Cargo.toml | 2 +- crunchy-cli-core/Cargo.lock | 24 +++++++++++++++--------- crunchy-cli-core/Cargo.toml | 2 +- 4 files changed, 33 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 77e1342..42fe778 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -60,6 +60,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" + [[package]] name = "bitflags" version = "1.3.2" @@ -279,7 +285,7 @@ dependencies = [ [[package]] name = "crunchy-cli" -version = "3.0.0-dev.7" +version = "3.0.0-dev.8" dependencies = [ "chrono", "clap", @@ -291,7 +297,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-dev.7" +version = "3.0.0-dev.8" dependencies = [ "anyhow", "async-trait", @@ -318,9 +324,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "184d0c725a09aec815316cbf41a2f362008ecb0e8c8e3b6b9930d01a89b5df21" +checksum = "f3770cda4c67e68c689c8e361af46bb9d017caf82263905358fd0751d10657a0" dependencies = [ "aes", "async-trait", @@ -343,9 +349,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f3c82e1766339727fc2c10d66d0c4f001b1cf42e2993f9d93997b610f408776" +checksum = "4a260a73e733bb0ce30343caaed5e968d3c1cc2ea0ab27c601481e9ef22a2fd7" dependencies = [ "darling", "quote", @@ -1268,7 +1274,7 @@ version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" dependencies = [ - "base64", + "base64 0.13.1", "bytes", "cookie", "cookie_store", @@ -1356,11 +1362,11 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0864aeff53f8c05aa08d86e5ef839d3dfcf07aeba2db32f12db0ef716e87bd55" +checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" dependencies = [ - "base64", + "base64 0.21.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index ec8e4ef..db1f129 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-dev.7" +version = "3.0.0-dev.8" edition = "2021" [dependencies] diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index 5bdabf0..b12b3a3 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -60,6 +60,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" + [[package]] name = "bitflags" version = "1.3.2" @@ -260,7 +266,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-dev.7" +version = "3.0.0-dev.8" dependencies = [ "anyhow", "async-trait", @@ -287,9 +293,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "184d0c725a09aec815316cbf41a2f362008ecb0e8c8e3b6b9930d01a89b5df21" +checksum = "f3770cda4c67e68c689c8e361af46bb9d017caf82263905358fd0751d10657a0" dependencies = [ "aes", "async-trait", @@ -312,9 +318,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f3c82e1766339727fc2c10d66d0c4f001b1cf42e2993f9d93997b610f408776" +checksum = "4a260a73e733bb0ce30343caaed5e968d3c1cc2ea0ab27c601481e9ef22a2fd7" dependencies = [ "darling", "quote", @@ -1237,7 +1243,7 @@ version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" dependencies = [ - "base64", + "base64 0.13.1", "bytes", "cookie", "cookie_store", @@ -1319,11 +1325,11 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0864aeff53f8c05aa08d86e5ef839d3dfcf07aeba2db32f12db0ef716e87bd55" +checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" dependencies = [ - "base64", + "base64 0.21.0", ] [[package]] diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 5bda1db..8eea6a8 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-dev.7" +version = "3.0.0-dev.8" edition = "2021" [dependencies] From 6d1f8d49f67aabbc9b1cf577fce38c3209993371 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Fri, 13 Jan 2023 15:21:23 +0100 Subject: [PATCH 040/363] Add hardsubs manually to download videos (#81) --- crunchy-cli-core/src/cli/archive.rs | 151 +------------------------ crunchy-cli-core/src/cli/download.rs | 134 +++++++++++++--------- crunchy-cli-core/src/cli/utils.rs | 20 ++-- crunchy-cli-core/src/utils/format.rs | 6 +- crunchy-cli-core/src/utils/mod.rs | 1 + crunchy-cli-core/src/utils/subtitle.rs | 108 ++++++++++++++++++ crunchy-cli-core/src/utils/video.rs | 25 ++++ 7 files changed, 233 insertions(+), 212 deletions(-) create mode 100644 crunchy-cli-core/src/utils/video.rs diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 768c629..c7030b4 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -9,16 +9,14 @@ use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file, tempfile}; use crate::utils::parse::{parse_url, UrlFilter}; use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number}; -use crate::utils::subtitle::Subtitle; +use crate::utils::subtitle::{download_subtitle, Subtitle}; +use crate::utils::video::get_video_length; use crate::Execute; use anyhow::{bail, Result}; -use chrono::NaiveTime; -use crunchyroll_rs::media::{Resolution, StreamSubtitle}; +use crunchyroll_rs::media::Resolution; use crunchyroll_rs::{Locale, Media, MediaCollection, Series}; use log::{debug, error, info, warn}; -use regex::Regex; use std::collections::BTreeMap; -use std::io::Write; use std::path::PathBuf; use std::process::{Command, Stdio}; use tempfile::TempPath; @@ -113,14 +111,6 @@ pub struct Archive { )] #[arg(long)] default_subtitle: Option, - #[arg(help = "Disable subtitle optimizations")] - #[arg( - long_help = "By default, Crunchyroll delivers subtitles in a format which may cause issues in some video players. \ - These issues are fixed internally by setting a flag which is not part of the official specification of the subtitle format. \ - If you do not want this fixes or they cause more trouble than they solve (for you), it can be disabled with this flag" - )] - #[arg(long)] - no_subtitle_optimizations: bool, #[arg(help = "Ignore interactive input")] #[arg(short, long, default_value_t = false)] @@ -326,12 +316,8 @@ impl Execute for Archive { let primary_video_length = get_video_length(primary_video.to_path_buf()).unwrap(); for subtitle in subtitles { subtitle_paths.push(( - download_subtitle( - &self, - subtitle.stream_subtitle.clone(), - primary_video_length, - ) - .await?, + download_subtitle(subtitle.stream_subtitle.clone(), primary_video_length) + .await?, subtitle, )) } @@ -436,7 +422,7 @@ async fn formats_from_series( }; Some(subtitle) })); - formats.push(Format::new_from_episode(episode, &episodes, stream)); + formats.push(Format::new_from_episode(episode, &episodes, stream, vec![])); } primary_season = false; @@ -476,111 +462,6 @@ async fn download_video(ctx: &Context, format: &Format, only_audio: bool) -> Res Ok(path) } -async fn download_subtitle( - archive: &Archive, - subtitle: StreamSubtitle, - max_length: NaiveTime, -) -> Result { - let tempfile = tempfile(".ass")?; - let (mut file, path) = tempfile.into_parts(); - - let mut buf = vec![]; - subtitle.write_to(&mut buf).await?; - if !archive.no_subtitle_optimizations { - buf = fix_subtitle_look_and_feel(buf) - } - buf = fix_subtitle_length(buf, max_length); - - file.write_all(buf.as_slice())?; - - Ok(path) -} - -/// Add `ScaledBorderAndShadows: yes` to subtitles; without it they look very messy on some video -/// players. See [crunchy-labs/crunchy-cli#66](https://github.com/crunchy-labs/crunchy-cli/issues/66) -/// for more information. -fn fix_subtitle_look_and_feel(raw: Vec) -> Vec { - let mut script_info = false; - let mut new = String::new(); - - for line in String::from_utf8_lossy(raw.as_slice()).split('\n') { - if line.trim().starts_with('[') && script_info { - new.push_str("ScaledBorderAndShadow: yes\n"); - script_info = false - } else if line.trim() == "[Script Info]" { - script_info = true - } - new.push_str(line); - new.push('\n') - } - - new.into_bytes() -} - -/// Fix the length of subtitles to a specified maximum amount. This is required because sometimes -/// subtitles have an unnecessary entry long after the actual video ends with artificially extends -/// the video length on some video players. To prevent this, the video length must be hard set. See -/// [crunchy-labs/crunchy-cli#32](https://github.com/crunchy-labs/crunchy-cli/issues/32) for more -/// information. -fn fix_subtitle_length(raw: Vec, max_length: NaiveTime) -> Vec { - let re = - Regex::new(r#"^Dialogue:\s\d+,(?P\d+:\d+:\d+\.\d+),(?P\d+:\d+:\d+\.\d+),"#) - .unwrap(); - - // chrono panics if we try to format NaiveTime with `%2f` and the nano seconds has more than 2 - // digits so them have to be reduced manually to avoid the panic - fn format_naive_time(native_time: NaiveTime) -> String { - let formatted_time = native_time.format("%f").to_string(); - format!( - "{}.{}", - native_time.format("%T"), - if formatted_time.len() <= 2 { - native_time.format("%2f").to_string() - } else { - formatted_time.split_at(2).0.parse().unwrap() - } - ) - } - - let length_as_string = format_naive_time(max_length); - let mut new = String::new(); - - for line in String::from_utf8_lossy(raw.as_slice()).split('\n') { - if let Some(capture) = re.captures(line) { - let start = capture.name("start").map_or(NaiveTime::default(), |s| { - NaiveTime::parse_from_str(s.as_str(), "%H:%M:%S.%f").unwrap() - }); - let end = capture.name("end").map_or(NaiveTime::default(), |s| { - NaiveTime::parse_from_str(s.as_str(), "%H:%M:%S.%f").unwrap() - }); - - if start > max_length { - continue; - } else if end > max_length { - new.push_str( - re.replace( - line, - format!( - "Dialogue: {},{},", - format_naive_time(start), - &length_as_string - ), - ) - .to_string() - .as_str(), - ) - } else { - new.push_str(line) - } - } else { - new.push_str(line) - } - new.push('\n') - } - - new.into_bytes() -} - fn generate_mkv( archive: &Archive, target: PathBuf, @@ -721,23 +602,3 @@ fn generate_mkv( Ok(()) } - -/// Get the length of a video. This is required because sometimes subtitles have an unnecessary entry -/// long after the actual video ends with artificially extends the video length on some video players. -/// To prevent this, the video length must be hard set. See -/// [crunchy-labs/crunchy-cli#32](https://github.com/crunchy-labs/crunchy-cli/issues/32) for more -/// information. -fn get_video_length(path: PathBuf) -> Result { - let video_length = Regex::new(r"Duration:\s(?P

@@ -26,13 +26,13 @@ A [Rust](https://www.rust-lang.org/) written cli client for [Crunchyroll](https:

Usage 🖥️ • - Disclaimer ☝️ + Disclaimer 📜 • License ⚖

> We are in no way affiliated with, maintained, authorized, sponsored, or officially associated with Crunchyroll LLC or any of its subsidiaries or affiliates. -> The official Crunchyroll website can be found at https://crunchyroll.com/. +> The official Crunchyroll website can be found at [crunchyroll.com](https://crunchyroll.com/). > This README belongs to the _master_ branch which is currently under heavy development towards the next major version (3.0). > It is mostly stable but some issues may still occur. @@ -41,14 +41,14 @@ A [Rust](https://www.rust-lang.org/) written cli client for [Crunchyroll](https: ## ✨ Features - Download single videos and entire series from [Crunchyroll](https://www.crunchyroll.com). -- Archive episode or seasons in an `.mkv` file with multiple subtitles and audios. -- Specify a range which episodes to download from an anime. +- Archive episodes or seasons in an `.mkv` file with multiple subtitles and audios. +- Specify a range of episodes to download from an anime. ## 💾 Get the executable ### 📥 Download the latest binaries -Checkout the [releases](https://github.com/crunchy-labs/crunchy-cli/releases) tab and get the binary from the newest (pre-)release. +Check out the [releases](https://github.com/crunchy-labs/crunchy-cli/releases) tab and get the binary from the latest (pre-)release. ### 🛠 Build it yourself @@ -58,47 +58,45 @@ This requires [git](https://git-scm.com/) and [Cargo](https://doc.rust-lang.org/ $ git clone https://github.com/crunchy-labs/crunchy-cli $ cd crunchy-cli $ cargo build --release +$ cargo install --force --path . ``` -After the binary has built successfully it is available in `target/release`. ## 🖥️ Usage -> All shown command are just examples +> All shown commands are examples 🧑🏼‍🍳 -Every command requires you to be logged in with an account. -It doesn't matter if this account is premium or not, both works (but as free user you do not have access to premium content). -You can pass your account via credentials (username & password) or refresh token. +crunchy-cli requires you to log in. +Though you can use a non-premium account, you will not have access to premium content without a subscription. +You can authenticate with your credentials (username:password) or by using a refresh token. -- Refresh Token - - To get the token you have to log in at [crunchyroll.com](https://www.crunchyroll.com/) and extract the `etp_rt` cookie. - The easiest way to get it is via a browser extension with lets you view your cookies, like [Cookie-Editor](https://cookie-editor.cgagnier.ca/) ([Firefox Store](https://addons.mozilla.org/en-US/firefox/addon/cookie-editor/); [Chrome Store](https://chrome.google.com/webstore/detail/cookie-editor/hlkenndednhfkekhgcdicdfddnkalmdm)). - If installed, search the `etp_rt` entry and extract the value. - - ```shell - $ crunchy --etp-rt "abcd1234-zyxw-9876-98zy-a1b2c3d4e5f6" - ``` - Credentials - - Credentials must be provided as one single expression. - Username and password must be separated by a `:`. - ```shell $ crunchy --credentials "user:password" ``` -- Anonymous - - Login without an account at all is also possible. +- Refresh Token + - To obtain a refresh token, you have to log in at [crunchyroll.com](https://www.crunchyroll.com/) and extract the `etp_rt` cookie. + The easiest way to get it is via a browser extension which lets you export your cookies, like [Cookie-Editor](https://cookie-editor.cgagnier.ca/) ([Firefox](https://addons.mozilla.org/en-US/firefox/addon/cookie-editor/) / [Chrome](https://chrome.google.com/webstore/detail/cookie-editor/hlkenndednhfkekhgcdicdfddnkalmdm)). + When installed, look for the `etp_rt` entry and extract its value. + - ```shell + $ crunchy --etp-rt "4ebf1690-53a4-491a-a2ac-488309120f5d" + ``` +- Stay Anonymous + - Skip the login check: - ```shell $ crunchy --anonymous ``` ### Login -If you do not want to provide your credentials every time you execute a command, they can be stored permanently on disk. -This can be done with the `login` subcommand. +crunchy-cli can store your session, so you don't have to authenticate every time you execute a command. + +Note that the `login` keyword has to be used *last*. ```shell -$ crunchy --etp-rt "abcd1234-zyxw-9876-98zy-a1b2c3d4e5f6" login +$ crunchy --etp-rt "4ebf1690-53a4-491a-a2ac-488309120f5d" login ``` -Once set, you do not need to provide `--etp-rt` / `--credentials` anymore when using the cli. -This does not work if you've using this with `--anonymous`. +With the session stored, you do not need to use `--credentials` / `--etp-rt` anymore. This does not work with `--anonymous`. ### Download @@ -115,25 +113,25 @@ This does not work if you've using this with `--anonymous`. **Options** - Audio language - Which audio the episode(s) should be can be set via the `-a` / `--audio` flag. + Set the audio language with the `-a` / `--audio` flag. This only works if the url points to a series since episode urls are language specific. ```shell $ crunchy download -a de-DE https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` - Default is your system language. If not supported by Crunchyroll, `en-US` (American English) is the default. + Default is your system locale. If not supported by Crunchyroll, `en-US` (American English) is the default. - Subtitle language - Besides the audio, it's also possible to specify which language the subtitles should have with the `-s` / `--subtitle` flag. - The subtitle will be hardsubbed (burned into the video) and thus, can't be turned off or on. + Besides the audio, you can specify the subtitle language by using the `-s` / `--subtitle` flag. + The subtitles will be burned into the video track (cf. [hardsub](https://www.urbandictionary.com/define.php?term=hardsub)) and thus can not be turned off. ```shell $ crunchy download -s de-DE https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` - Default is no subtitle. + Default is none. -- Output filename +- Output template - You can specify the name of the output file with the `-o` / `--output` flag. + Define an output template by using the `-o` / `--output` flag. If you want to use any other file format than [`.ts`](https://en.wikipedia.org/wiki/MPEG_transport_stream) you need [ffmpeg](https://ffmpeg.org/). ```shell $ crunchy download -o "ditf.ts" https://www.crunchyroll.com/watch/GRDQPM1ZY/alone-and-lonesome @@ -153,7 +151,7 @@ This does not work if you've using this with `--anonymous`. **Supported urls** - Series - Only series urls are supported since single episode urls are (audio) language locked. + Only series urls are supported, because episode urls are locked to a single audio language. ```shell $ crunchy archive https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` @@ -161,25 +159,24 @@ This does not work if you've using this with `--anonymous`. **Options** - Audio languages - Which audios the episode(s) should be can be set via the `-a` / `--audio` flag. + Set the audio language with the `-a` / `--audio` flag. Can be used multiple times. ```shell $ crunchy archive -a ja-JP -a de-DE https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` - Can be used multiple times. - Default is your system language (if not supported by Crunchyroll, `en-US` (American English) is the default) + `ja-JP` (Japanese). + Default is your system locale (if not supported by Crunchyroll, `en-US` (American English) and `ja-JP` (Japanese) are used). - Subtitle languages - Besides the audio, it's also possible to specify which languages the subtitles should have with the `-s` / `--subtitle` flag. + Besides the audio, you can specify the subtitle language by using the `-s` / `--subtitle` flag. ```shell $ crunchy archive -s de-DE https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` - Default is all subtitles. + Default is `all` subtitles. -- Output filename +- Output template - You can specify the name of the output file with the `-o` / `--output` flag. - The only supported file / container format is [`.mkv`](https://en.wikipedia.org/wiki/Matroska) since it stores / can store multiple audio, video and subtitle streams. + Define an output template by using the `-o` / `--output` flag. + crunchy-cli uses the [`.mkv`](https://en.wikipedia.org/wiki/Matroska) container format, because of it's ability to store multiple audio, video and subtitle tracks at once. ```shell $ crunchy archive -o "{title}.mkv" https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` @@ -195,12 +192,12 @@ This does not work if you've using this with `--anonymous`. - Merge behavior - Because of local restrictions (or other reasons) some episodes with different languages does not have the same length (e.g. when some scenes were cut out). - The ideal state, when multiple audios & subtitles used, would be if only one _video_ has to be stored and all other languages can be stored as audio-only. + Due to censorship, some episodes have multiple lengths for different languages. + In the best case, when multiple audio & subtitle tracks are used, there is only one *video* track and all other languages can be stored as audio-only. But, as said, this is not always the case. - With the `-m` / `--merge` flag you can set what you want to do if some video lengths differ. - Valid options are `audio` - store one video and all other languages as audio only; `video` - store the video + audio for every language; `auto` - detect if videos differ in length: if so, behave like `video` else like `audio`. - Subtitles will always match to the first / primary audio and video. + With the `-m` / `--merge` flag you can define the behaviour when an episodes' video tracks differ in length. + Valid options are `audio` - store one video and all other languages as audio only; `video` - store the video + audio for every language; `auto` - detect if videos differ in length: if so, behave like `video` - otherwise like `audio`. + Subtitles will always match the primary audio and video. ```shell $ crunchy archive -m audio https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` @@ -208,52 +205,45 @@ This does not work if you've using this with `--anonymous`. - Default subtitle - `--default_subtitle` set which subtitle language should be set as default / auto appear when starting the downloaded video(s). + `--default_subtitle` Set which subtitle language is to be flagged as **default** and **forced**. ```shell $ crunchy archive --default_subtitle en-US https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` Default is none. -- No subtitle optimizations +- Subtitle optimizations - Subtitles, as Crunchyroll delivers them, look weird in some video players (#66). + Crunchyroll's subtitles look weird in some players (#66). This can be fixed by adding a specific entry to the subtitles. - But since this entry is only a de-factor standard and not represented in the official specification of the subtitle format ([`.ass`](https://en.wikipedia.org/wiki/SubStation_Alpha)) it could cause issues with some video players (but no issue got reported so far, so it's relatively safe to use). - `--no_subtitle_optimizations` can disable these optimizations. + Even though this entry is a de facto standard, it is not defined in the official specification for the `.ass` format (cf. [Advanced SubStation Subtitles](https://wiki.videolan.org/SubStation_Alpha)). This could cause compatibility issues, but no issues have been reported yet. + `--no_subtitle_optimizations` disables these optimizations. ```shell $ crunchy archive --no_subtitle_optimizations https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` -### Url Filtering +### Episode filtering -If you want to download only specific episode of a series, you could either pass every single episode url to the downloader (which is fine for 1 - 3 episodes) or use _filtering_. +Filters patterns can be used to download a specific range of episodes from a single series. -It works pretty simple, just put a specific pattern surrounded by square brackets at the end of the url from the anime you want to download. -A season and / or episode as well as a range from where to where episodes should be downloaded can be specified. -Use the list below to get a better overview what is possible +A filter pattern may consist of either a season, an episode, or a combination of the two. +When used in combination, seasons `S` must be defined before episodes `E`. + +There are many possible patterns, for example: - `...[E5]` - Download the fifth episode. -- `...[S1]` - Download the full first season. -- `...[-S2]` - Download all seasons up to and including season 2. -- `...[S3E4-]` - Download all episodes from and including season 3, episode 4. -- `...[S1E4-S3]` - Download all episodes from and including season 1, episode 4, until and including season 3. -- `...[S3,S5]` - Download episode 3 and 5. -- `...[S1-S3,S4E2-S4E6]` - Download season 1 to 3 and episode 2 to episode 6 of season 4. +- `...[S1]` - Download the whole first season. +- `...[-S2]` - Download the first two seasons. +- `...[S3E4-]` - Download everything from season three, episode four, onwards. +- `...[S1E4-S3]` - Download season one, starting at episode four, then download season two and three. +- `...[S3,S5]` - Download season three and five. +- `...[S1-S3,S4E2-S4E6]` - Download season one to three, then episodes two to six from season four. -In practice, it would look like this: `https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx[E1-E5]`. +In practice, it would look like this: `https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx[E1-E5]` -The `S`, followed by the number indicates the _season_ number, `E`, followed by the number indicates an _episode_ number. -It doesn't matter if `S`, `E` or both are missing. -Note that `S` must always stay before `E` when used. +# 📜 Disclaimer -There is also a regex available at [regex101.com](https://regex101.com/r/SDZyZM) where you can test if your pattern is correct. -Just put in your pattern without square brackets into the big empty field and if the full pattern is highlighted this means it is valid. -If none or only some parts are highlighted, it's not valid not. +This tool is **ONLY** meant for private use. You need a subscription to [`💳 Crunchyroll Premium 💳`](https://www.crunchyroll.com/welcome#plans) to download premium content. -# ☝️ Disclaimer - -This tool is **ONLY** meant to be used for private purposes. To use this tool you need crunchyroll premium anyway, so there is no reason why rip and share the episodes. - -**The responsibility for what happens to the downloaded videos lies entirely with the user who downloaded them.** +**You are entirely responsible for what happens to files you downloaded through crunchy-cli.** # ⚖ License From 0a40f3c40f7eaeca06a677debcb1813f69e45b3d Mon Sep 17 00:00:00 2001 From: ByteDream Date: Thu, 23 Mar 2023 01:17:41 +0100 Subject: [PATCH 063/363] Refactor --- Cargo.lock | 775 ++++++++++------- Cargo.toml | 8 +- crunchy-cli-core/Cargo.lock | 776 +++++++++++------- crunchy-cli-core/Cargo.toml | 12 +- crunchy-cli-core/src/archive/command.rs | 184 +++++ crunchy-cli-core/src/archive/filter.rs | 482 +++++++++++ crunchy-cli-core/src/archive/mod.rs | 4 + crunchy-cli-core/src/cli/archive.rs | 618 -------------- crunchy-cli-core/src/cli/download.rs | 603 -------------- crunchy-cli-core/src/cli/log.rs | 138 ---- crunchy-cli-core/src/cli/mod.rs | 5 - crunchy-cli-core/src/cli/utils.rs | 693 ---------------- crunchy-cli-core/src/download/command.rs | 151 ++++ crunchy-cli-core/src/download/filter.rs | 349 ++++++++ crunchy-cli-core/src/download/mod.rs | 4 + crunchy-cli-core/src/lib.rs | 22 +- .../src/{cli/login.rs => login/command.rs} | 2 + crunchy-cli-core/src/login/mod.rs | 4 + crunchy-cli-core/src/utils/download.rs | 657 +++++++++++++++ crunchy-cli-core/src/utils/ffmpeg.rs | 344 ++++++++ crunchy-cli-core/src/utils/filter.rs | 95 +++ crunchy-cli-core/src/utils/format.rs | 309 +++++-- crunchy-cli-core/src/utils/locale.rs | 14 + crunchy-cli-core/src/utils/log.rs | 140 +++- crunchy-cli-core/src/utils/mod.rs | 5 +- crunchy-cli-core/src/utils/os.rs | 2 +- crunchy-cli-core/src/utils/parse.rs | 24 +- crunchy-cli-core/src/utils/sort.rs | 52 -- crunchy-cli-core/src/utils/subtitle.rs | 119 --- crunchy-cli-core/src/utils/video.rs | 42 +- 30 files changed, 3651 insertions(+), 2982 deletions(-) create mode 100644 crunchy-cli-core/src/archive/command.rs create mode 100644 crunchy-cli-core/src/archive/filter.rs create mode 100644 crunchy-cli-core/src/archive/mod.rs delete mode 100644 crunchy-cli-core/src/cli/archive.rs delete mode 100644 crunchy-cli-core/src/cli/download.rs delete mode 100644 crunchy-cli-core/src/cli/log.rs delete mode 100644 crunchy-cli-core/src/cli/mod.rs delete mode 100644 crunchy-cli-core/src/cli/utils.rs create mode 100644 crunchy-cli-core/src/download/command.rs create mode 100644 crunchy-cli-core/src/download/filter.rs create mode 100644 crunchy-cli-core/src/download/mod.rs rename crunchy-cli-core/src/{cli/login.rs => login/command.rs} (94%) create mode 100644 crunchy-cli-core/src/login/mod.rs create mode 100644 crunchy-cli-core/src/utils/download.rs create mode 100644 crunchy-cli-core/src/utils/ffmpeg.rs create mode 100644 crunchy-cli-core/src/utils/filter.rs delete mode 100644 crunchy-cli-core/src/utils/sort.rs delete mode 100644 crunchy-cli-core/src/utils/subtitle.rs diff --git a/Cargo.lock b/Cargo.lock index ae46f99..2651e95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -33,19 +33,19 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.68" +version = "1.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" +checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" [[package]] name = "async-trait" -version = "0.1.61" +version = "0.1.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "705339e0e4a9690e2908d2b3d049d85682cf19fbd5782494498fbf7003a6a282" +checksum = "86ea188f25f0255d8f92797797c97ebf5631fa88178beb1a46fdf5622c9a00e4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.5", ] [[package]] @@ -72,6 +72,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487f1e0fcbe47deb8b0574e646def1c903389d95241dd1bbcc6ce4a715dfc0c1" + [[package]] name = "block-padding" version = "0.3.2" @@ -81,29 +87,17 @@ dependencies = [ "generic-array", ] -[[package]] -name = "bstr" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" -dependencies = [ - "lazy_static", - "memchr", - "regex-automata", - "serde", -] - [[package]] name = "bumpalo" -version = "3.11.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "bytes" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "cbc" @@ -116,9 +110,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" [[package]] name = "cfg-if" @@ -128,9 +122,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.23" +version = "0.4.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" +checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" dependencies = [ "iana-time-zone", "js-sys", @@ -144,9 +138,9 @@ dependencies = [ [[package]] name = "cipher" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1873270f8f7942c191139cb8a40fd228da6c3fd2fc376d7e92d47aa14aeb59e" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ "crypto-common", "inout", @@ -154,55 +148,55 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.32" +version = "4.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39" +checksum = "42dfd32784433290c51d92c438bb72ea5063797fc3cc9a21a8c4346bebbb2098" dependencies = [ - "bitflags", + "bitflags 2.0.2", "clap_derive", "clap_lex", "is-terminal", "once_cell", - "strsim", + "strsim 0.10.0", "termcolor", ] [[package]] name = "clap_complete" -version = "4.0.7" +version = "4.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10861370d2ba66b0f5989f83ebf35db6421713fd92351790e7fdd6c36774c56b" +checksum = "37686beaba5ac9f3ab01ee3172f792fc6ffdd685bfb9e63cfef02c0571a4e8e1" dependencies = [ "clap", ] [[package]] name = "clap_derive" -version = "4.0.21" +version = "4.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" +checksum = "fddf67631444a3a3e3e5ac51c36a5e01335302de677bd78759eaa90ab1f46644" dependencies = [ "heck", "proc-macro-error", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "clap_lex" -version = "0.3.0" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" +checksum = "033f6b7a4acb1f358c742aaca805c939ee73b4c6209ae4318ec7aca81c42e646" dependencies = [ "os_str_bytes", ] [[package]] name = "clap_mangen" -version = "0.2.6" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904eb24d05ad587557e0f484ddce5c737c30cf81372badb16d13e41c4b8340b1" +checksum = "4237e29de9c6949982ba87d51709204504fb8ed2fd38232fcb1e5bf7d4ba48c8" dependencies = [ "clap", "roff", @@ -220,15 +214,15 @@ dependencies = [ [[package]] name = "console" -version = "0.15.4" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9b6515d269224923b26b5febea2ed42b2d5f2ce37284a4dd670fedd6cb8347a" +checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60" dependencies = [ "encode_unicode", "lazy_static", "libc", "unicode-width", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -238,7 +232,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" dependencies = [ "percent-encoding", - "time 0.3.17", + "time 0.3.20", "version_check", ] @@ -254,7 +248,7 @@ dependencies = [ "publicsuffix", "serde", "serde_json", - "time 0.3.17", + "time 0.3.20", "url", ] @@ -304,8 +298,8 @@ dependencies = [ "chrono", "clap", "crunchyroll-rs", - "csv", "ctrlc", + "derive_setters", "dirs", "indicatif", "lazy_static", @@ -325,15 +319,17 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.5" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3770cda4c67e68c689c8e361af46bb9d017caf82263905358fd0751d10657a0" +checksum = "bddaff98c25bdedca0f1ed2b68c28b12a3e52054144b1b11380236d23051c751" dependencies = [ "aes", "async-trait", "cbc", "chrono", "crunchyroll-rs-internal", + "dash-mpd", + "futures-util", "http", "lazy_static", "m3u8-rs", @@ -350,13 +346,13 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.5" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a260a73e733bb0ce30343caaed5e968d3c1cc2ea0ab27c601481e9ef22a2fd7" +checksum = "3f26b71b36db3139ce788545c2bffa7e69d4fd7b689f143086c6283e847a4668" dependencies = [ - "darling", + "darling 0.14.4", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -369,43 +365,21 @@ dependencies = [ "typenum", ] -[[package]] -name = "csv" -version = "1.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" -dependencies = [ - "bstr", - "csv-core", - "itoa 0.4.8", - "ryu", - "serde", -] - -[[package]] -name = "csv-core" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" -dependencies = [ - "memchr", -] - [[package]] name = "ctrlc" -version = "3.2.4" +version = "3.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1631ca6e3c59112501a9d87fd86f21591ff77acd31331e8a73f8d80a65bbdd71" +checksum = "bbcf33c2a618cbe41ee43ae6e9f2e48368cd9f9db2896f10167d8d762679f639" dependencies = [ "nix", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] name = "cxx" -version = "1.0.86" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d1075c37807dcf850c379432f0df05ba52cc30f279c5cfc43cc221ce7f8579" +checksum = "a9c00419335c41018365ddf7e4d5f1c12ee3659ddcf3e01974650ba1de73d038" dependencies = [ "cc", "cxxbridge-flags", @@ -415,9 +389,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.86" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5044281f61b27bc598f2f6647d480aed48d2bf52d6eb0b627d84c0361b17aa70" +checksum = "fb8307ad413a98fff033c8545ecf133e3257747b3bae935e7602aab8aa92d4ca" dependencies = [ "cc", "codespan-reporting", @@ -425,79 +399,145 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn", + "syn 2.0.5", ] [[package]] name = "cxxbridge-flags" -version = "1.0.86" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61b50bc93ba22c27b0d31128d2d130a0a6b3d267ae27ef7e4fae2167dfe8781c" +checksum = "edc52e2eb08915cb12596d29d55f0b5384f00d697a646dbd269b6ecb0fbd9d31" [[package]] name = "cxxbridge-macro" -version = "1.0.86" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e61fda7e62115119469c7b3591fd913ecca96fb766cfd3f2e2502ab7bc87a5" +checksum = "631569015d0d8d54e6c241733f944042623ab6df7bc3be7466874b05fcdb1c5f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.5", ] [[package]] name = "darling" -version = "0.14.2" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0dd3cd20dc6b5a876612a6e5accfe7f3dd883db6d07acfbf14c128f61550dfa" +checksum = "0d706e75d87e35569db781a9b5e2416cff1236a47ed380831f959382ccd5f858" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.10.2", + "darling_macro 0.10.2", +] + +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core 0.14.4", + "darling_macro 0.14.4", ] [[package]] name = "darling_core" -version = "0.14.2" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a784d2ccaf7c98501746bf0be29b2022ba41fd62a2e622af997a03e9f972859f" +checksum = "f0c960ae2da4de88a91b2d920c2a7233b400bc33cb28453a2987822d8392519b" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", - "strsim", - "syn", + "strsim 0.9.3", + "syn 1.0.109", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn 1.0.109", ] [[package]] name = "darling_macro" -version = "0.14.2" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7618812407e9402654622dd402b0a89dff9ba93badd6540781526117b92aab7e" +checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72" dependencies = [ - "darling_core", + "darling_core 0.10.2", "quote", - "syn", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core 0.14.4", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "dash-mpd" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df35f3b3b0fde2747a01530de0a81f7a27620d050f656e518ef0550557f564de" +dependencies = [ + "chrono", + "fs-err", + "iso8601", + "log", + "num-traits", + "quick-xml", + "regex", + "serde", + "serde_with", + "thiserror", + "xattr", +] + +[[package]] +name = "derive_setters" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1cf41b4580a37cca5ef2ada2cc43cf5d6be3983f4522e83010d67ab6925e84b" +dependencies = [ + "darling 0.10.2", + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] name = "dirs" -version = "4.0.0" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +checksum = "dece029acd3353e3a58ac2e3eb3c8d6c35827a892edc6cc4138ef9c33df46ecd" dependencies = [ "dirs-sys", ] [[package]] name = "dirs-sys" -version = "0.3.7" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +checksum = "04414300db88f70d74c5ff54e50f9e1d1737d9a5b90f53fcf2e95ca2a9ab554b" dependencies = [ "libc", "redox_users", - "winapi", + "windows-sys 0.45.0", ] [[package]] @@ -508,9 +548,9 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" [[package]] name = "encoding_rs" -version = "0.8.31" +version = "0.8.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" dependencies = [ "cfg-if", ] @@ -538,9 +578,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" dependencies = [ "instant", ] @@ -576,43 +616,49 @@ dependencies = [ ] [[package]] -name = "futures-channel" -version = "0.3.25" +name = "fs-err" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" +checksum = "0845fa252299212f0389d64ba26f34fa32cfe41588355f21ed507c59a0f64541" + +[[package]] +name = "futures-channel" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "164713a5a0dcc3e7b4b1ed7d3b433cabc18025386f9339346e8daf15963cf7ac" dependencies = [ "futures-core", ] [[package]] name = "futures-core" -version = "0.3.25" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" +checksum = "86d7a0c1aa76363dac491de0ee99faf6941128376f1cf96f07db7603b7de69dd" [[package]] name = "futures-io" -version = "0.3.25" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" +checksum = "89d422fa3cbe3b40dca574ab087abb5bc98258ea57eea3fd6f1fa7162c778b91" [[package]] name = "futures-sink" -version = "0.3.25" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" +checksum = "ec93083a4aecafb2a80a885c9de1f0ccae9dbd32c2bb54b0c3a65690e0b8d2f2" [[package]] name = "futures-task" -version = "0.3.25" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" +checksum = "fd65540d33b37b16542a0438c12e6aeead10d4ac5d05bd3f805b8f35ab592879" [[package]] name = "futures-util" -version = "0.3.25" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" +checksum = "3ef6b17e481503ec85211fed8f39d1970f128935ca1f814cd32ac4a6842e84ab" dependencies = [ "futures-core", "futures-io", @@ -646,9 +692,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" dependencies = [ "bytes", "fnv", @@ -671,9 +717,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "heck" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" @@ -685,14 +731,26 @@ dependencies = [ ] [[package]] -name = "http" -version = "0.2.8" +name = "hermit-abi" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ "bytes", "fnv", - "itoa 1.0.5", + "itoa", ] [[package]] @@ -720,9 +778,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.23" +version = "0.14.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c" +checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" dependencies = [ "bytes", "futures-channel", @@ -733,7 +791,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa 1.0.5", + "itoa", "pin-project-lite", "socket2", "tokio", @@ -770,16 +828,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.53" +version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +checksum = "0c17cc76786e99f8d2f055c11159e7f0091c42474dcc3189fbab96072e873e6d" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "winapi", + "windows", ] [[package]] @@ -827,13 +885,14 @@ checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" dependencies = [ "autocfg", "hashbrown", + "serde", ] [[package]] name = "indicatif" -version = "0.17.2" +version = "0.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4295cbb7573c16d310e99e713cf9e75101eb190ab31fccd35f2d2691b4352b19" +checksum = "cef509aa9bc73864d6756f0d34d35504af3cf0844373afe9b8669a5b8005a729" dependencies = [ "console", "number_prefix", @@ -862,12 +921,13 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.3" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" +checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" dependencies = [ + "hermit-abi 0.3.1", "libc", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -878,33 +938,36 @@ checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" [[package]] name = "is-terminal" -version = "0.4.2" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189" +checksum = "8687c819457e979cc940d09cb16e42a1bf70aa6b60a549de6d3a62a0ee90c69e" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.1", "io-lifetimes", "rustix", - "windows-sys", + "windows-sys 0.45.0", +] + +[[package]] +name = "iso8601" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924e5d73ea28f59011fec52a0d12185d496a9b075d360657aed2a5707f701153" +dependencies = [ + "nom", ] [[package]] name = "itoa" -version = "0.4.8" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" - -[[package]] -name = "itoa" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "js-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" dependencies = [ "wasm-bindgen", ] @@ -917,9 +980,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.139" +version = "0.2.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" [[package]] name = "link-cplusplus" @@ -957,9 +1020,9 @@ dependencies = [ [[package]] name = "matches" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" +checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" [[package]] name = "memchr" @@ -969,9 +1032,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "mime" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "minimal-lexical" @@ -981,14 +1044,14 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mio" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -1011,11 +1074,11 @@ dependencies = [ [[package]] name = "nix" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46a58d1d356c6597d08cde02c2f09d785b09e28711837b1ed667dc652c08a694" +checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if", "libc", "static_assertions", @@ -1023,9 +1086,9 @@ dependencies = [ [[package]] name = "nom" -version = "7.1.2" +version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5507769c4919c998e69e49c839d9dc6e693ede4cc4290d6ad8b41d4f09c548c" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" dependencies = [ "memchr", "minimal-lexical", @@ -1056,7 +1119,7 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi", + "hermit-abi 0.2.6", "libc", ] @@ -1068,17 +1131,17 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "once_cell" -version = "1.17.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "openssl" -version = "0.10.45" +version = "0.10.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" +checksum = "d8b277f87dacc05a6b709965d1cbafac4649d6ce9f3ce9ceb88508b5666dfec9" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if", "foreign-types", "libc", @@ -1095,7 +1158,7 @@ checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1106,9 +1169,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.80" +version = "0.9.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" +checksum = "a95792af3c4e0153c3914df2261bedd30a98476f94dc892b67dfe1d89d433a04" dependencies = [ "autocfg", "cc", @@ -1119,9 +1182,9 @@ dependencies = [ [[package]] name = "os_str_bytes" -version = "6.4.1" +version = "6.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" +checksum = "ceedf44fb00f2d1984b0bc98102627ce622e083e49a5bacdb3e514fa4238e267" [[package]] name = "percent-encoding" @@ -1162,7 +1225,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "version_check", ] @@ -1177,17 +1240,11 @@ dependencies = [ "version_check", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" - [[package]] name = "proc-macro2" -version = "1.0.49" +version = "1.0.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" +checksum = "ba466839c78239c09faf015484e5cc04860f88242cff4d03eb038f04b4699b73" dependencies = [ "unicode-ident", ] @@ -1209,10 +1266,20 @@ dependencies = [ ] [[package]] -name = "quote" -version = "1.0.23" +name = "quick-xml" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "e5c1a97b1bc42b1d550bfb48d4262153fe400a12bab1511821736f7eac76d7e2" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quote" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" dependencies = [ "proc-macro2", ] @@ -1223,7 +1290,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -1239,43 +1306,28 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" +checksum = "cce168fea28d3e05f158bda4576cf0c844d5045bc2cc3620fa0292ed5bb5814c" dependencies = [ "aho-corasick", "memchr", "regex-syntax", ] -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" - [[package]] name = "regex-syntax" -version = "0.6.28" +version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" - -[[package]] -name = "remove_dir_all" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "reqwest" -version = "0.11.13" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" +checksum = "0ba30cc2c0cd02af1222ed216ba659cdb2f879dfe3181852fe7c50b1d0005949" dependencies = [ - "base64 0.13.1", + "base64 0.21.0", "bytes", "cookie", "cookie_store", @@ -1296,7 +1348,6 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "proc-macro-hack", "rustls", "rustls-pemfile", "serde", @@ -1337,23 +1388,23 @@ checksum = "b833d8d034ea094b1ea68aa6d5c740e0d04bad9d16568d08ba6f76823a114316" [[package]] name = "rustix" -version = "0.36.6" +version = "0.36.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4feacf7db682c6c329c4ede12649cd36ecab0f3be5b7d74e6a20304725db4549" +checksum = "db4165c9963ab29e422d6c26fbc1d37f15bace6b2810221f9d925023480fcf0e" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] name = "rustls" -version = "0.20.7" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "539a2bfe908f471bfa933876bd1eb6a19cf2176d375f82ef7f99530a40e48c2c" +checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" dependencies = [ "log", "ring", @@ -1372,9 +1423,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" [[package]] name = "sanitize-filename" @@ -1392,14 +1443,14 @@ version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" dependencies = [ - "windows-sys", + "windows-sys 0.42.0", ] [[package]] name = "scratch" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" +checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" [[package]] name = "sct" @@ -1413,11 +1464,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.7.0" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" +checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "core-foundation-sys", "libc", @@ -1426,9 +1477,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.6.1" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" dependencies = [ "core-foundation-sys", "libc", @@ -1436,31 +1487,31 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.152" +version = "1.0.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "771d4d9c4163ee138805e12c710dd365e4f44be8be0503cb1bb9eb989425d9c9" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "e801c1712f48475582b7696ac71e0ca34ebb30e09338425384269d9717c62cad" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.5", ] [[package]] name = "serde_json" -version = "1.0.91" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" +checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea" dependencies = [ - "itoa 1.0.5", + "itoa", "ryu", "serde", ] @@ -1472,11 +1523,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.5", + "itoa", "ryu", "serde", ] +[[package]] +name = "serde_with" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85456ffac572dc8826334164f2fb6fb40a7c766aebe195a2a21ee69ee2885ecf" +dependencies = [ + "base64 0.13.1", + "chrono", + "hex", + "indexmap", + "serde", + "serde_json", + "serde_with_macros", + "time 0.3.20", +] + +[[package]] +name = "serde_with_macros" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cbcd6104f8a4ab6af7f6be2a0da6be86b9de3c401f6e86bb856ab2af739232f" +dependencies = [ + "darling 0.14.4", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "shlex" version = "1.1.0" @@ -1485,9 +1564,9 @@ checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" [[package]] name = "signal-hook" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d" +checksum = "732768f1176d21d09e076c23a93123d40bba92d50c4058da34d45c8de8e682b9" dependencies = [ "libc", "signal-hook-registry", @@ -1495,18 +1574,18 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" dependencies = [ "libc", ] [[package]] name = "slab" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" dependencies = [ "autocfg", ] @@ -1519,14 +1598,14 @@ checksum = "133659a15339456eeeb07572eb02a91c91e9815e9cbc89566944d2c8d3efdbf6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", @@ -1544,6 +1623,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strsim" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" + [[package]] name = "strsim" version = "0.10.0" @@ -1552,9 +1637,20 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.107" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89c2d1c76a26822187a1fbb5964e3fff108bc208f02e820ab9dac1234f6b388a" dependencies = [ "proc-macro2", "quote", @@ -1563,68 +1659,67 @@ dependencies = [ [[package]] name = "sys-locale" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3358acbb4acd4146138b9bda219e904a6bb5aaaa237f8eed06f4d6bc1580ecee" +checksum = "f8a11bd9c338fdba09f7881ab41551932ad42e405f61d01e8406baea71c07aee" dependencies = [ "js-sys", "libc", "wasm-bindgen", "web-sys", - "winapi", + "windows-sys 0.45.0", ] [[package]] name = "tempfile" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95" dependencies = [ "cfg-if", "fastrand", - "libc", "redox_syscall", - "remove_dir_all", - "winapi", + "rustix", + "windows-sys 0.42.0", ] [[package]] name = "termcolor" -version = "1.1.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" dependencies = [ "winapi-util", ] [[package]] name = "terminal_size" -version = "0.2.3" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb20089a8ba2b69debd491f8d2d023761cbf196e999218c591fa1e7e15a21907" +checksum = "4c9afddd2cec1c0909f06b00ef33f94ab2cc0578c4a610aa208ddfec8aa2b43a" dependencies = [ "rustix", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] name = "thiserror" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.5", ] [[package]] @@ -1640,11 +1735,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.17" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" +checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" dependencies = [ - "itoa 1.0.5", + "itoa", "serde", "time-core", "time-macros", @@ -1658,9 +1753,9 @@ checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" [[package]] name = "time-macros" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" +checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" dependencies = [ "time-core", ] @@ -1676,15 +1771,15 @@ dependencies = [ [[package]] name = "tinyvec_macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.24.1" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae" +checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" dependencies = [ "autocfg", "bytes", @@ -1695,7 +1790,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -1706,14 +1801,14 @@ checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "tokio-native-tls" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", "tokio", @@ -1732,9 +1827,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.4" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" dependencies = [ "bytes", "futures-core", @@ -1784,15 +1879,15 @@ checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "unicode-bidi" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" [[package]] name = "unicode-normalization" @@ -1862,9 +1957,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1872,24 +1967,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" dependencies = [ "cfg-if", "js-sys", @@ -1899,9 +1994,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1909,28 +2004,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" [[package]] name = "web-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" dependencies = [ "js-sys", "wasm-bindgen", @@ -1986,6 +2081,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.42.0" @@ -2002,46 +2106,70 @@ dependencies = [ ] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.0" +name = "windows-sys" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_msvc" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_i686_gnu" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_msvc" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_x86_64_gnu" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_msvc" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "winreg" @@ -2051,3 +2179,12 @@ checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ "winapi", ] + +[[package]] +name = "xattr" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea263437ca03c1522846a4ddafbca2542d0ad5ed9b784909d4b27b76f62bc34a" +dependencies = [ + "libc", +] diff --git a/Cargo.toml b/Cargo.toml index db1f129..62ff7fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,18 +5,16 @@ version = "3.0.0-dev.8" edition = "2021" [dependencies] -tokio = { version = "1.24", features = ["macros", "rt-multi-thread", "time"], default-features = false } +tokio = { version = "1.26", features = ["macros", "rt-multi-thread", "time"], default-features = false } crunchy-cli-core = { path = "./crunchy-cli-core" } [build-dependencies] chrono = "0.4" -clap = { version = "4.0", features = ["string"] } -clap_complete = "4.0" +clap = { version = "4.1", features = ["string"] } +clap_complete = "4.1" clap_mangen = "0.2" -# The static-* features must be used here since build dependency features cannot be manipulated from the features -# specified in this Cargo.toml [features]. crunchy-cli-core = { path = "./crunchy-cli-core" } [profile.release] diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index b12b3a3..c4e2235 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -33,19 +33,19 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.68" +version = "1.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" +checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" [[package]] name = "async-trait" -version = "0.1.61" +version = "0.1.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "705339e0e4a9690e2908d2b3d049d85682cf19fbd5782494498fbf7003a6a282" +checksum = "86ea188f25f0255d8f92797797c97ebf5631fa88178beb1a46fdf5622c9a00e4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.6", ] [[package]] @@ -72,6 +72,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487f1e0fcbe47deb8b0574e646def1c903389d95241dd1bbcc6ce4a715dfc0c1" + [[package]] name = "block-padding" version = "0.3.2" @@ -81,29 +87,17 @@ dependencies = [ "generic-array", ] -[[package]] -name = "bstr" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" -dependencies = [ - "lazy_static", - "memchr", - "regex-automata", - "serde", -] - [[package]] name = "bumpalo" -version = "3.11.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "bytes" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "cbc" @@ -116,9 +110,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" [[package]] name = "cfg-if" @@ -128,9 +122,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.23" +version = "0.4.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" +checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" dependencies = [ "iana-time-zone", "js-sys", @@ -144,9 +138,9 @@ dependencies = [ [[package]] name = "cipher" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1873270f8f7942c191139cb8a40fd228da6c3fd2fc376d7e92d47aa14aeb59e" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ "crypto-common", "inout", @@ -154,37 +148,37 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.32" +version = "4.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39" +checksum = "42dfd32784433290c51d92c438bb72ea5063797fc3cc9a21a8c4346bebbb2098" dependencies = [ - "bitflags", + "bitflags 2.0.2", "clap_derive", "clap_lex", "is-terminal", "once_cell", - "strsim", + "strsim 0.10.0", "termcolor", ] [[package]] name = "clap_derive" -version = "4.0.21" +version = "4.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" +checksum = "fddf67631444a3a3e3e5ac51c36a5e01335302de677bd78759eaa90ab1f46644" dependencies = [ "heck", "proc-macro-error", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "clap_lex" -version = "0.3.0" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" +checksum = "033f6b7a4acb1f358c742aaca805c939ee73b4c6209ae4318ec7aca81c42e646" dependencies = [ "os_str_bytes", ] @@ -201,15 +195,15 @@ dependencies = [ [[package]] name = "console" -version = "0.15.4" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9b6515d269224923b26b5febea2ed42b2d5f2ce37284a4dd670fedd6cb8347a" +checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60" dependencies = [ "encode_unicode", "lazy_static", "libc", "unicode-width", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -219,7 +213,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" dependencies = [ "percent-encoding", - "time 0.3.17", + "time 0.3.20", "version_check", ] @@ -235,7 +229,7 @@ dependencies = [ "publicsuffix", "serde", "serde_json", - "time 0.3.17", + "time 0.3.20", "url", ] @@ -273,8 +267,8 @@ dependencies = [ "chrono", "clap", "crunchyroll-rs", - "csv", "ctrlc", + "derive_setters", "dirs", "indicatif", "lazy_static", @@ -284,6 +278,7 @@ dependencies = [ "sanitize-filename", "serde", "serde_json", + "shlex", "signal-hook", "sys-locale", "tempfile", @@ -293,15 +288,17 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.5" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3770cda4c67e68c689c8e361af46bb9d017caf82263905358fd0751d10657a0" +checksum = "bddaff98c25bdedca0f1ed2b68c28b12a3e52054144b1b11380236d23051c751" dependencies = [ "aes", "async-trait", "cbc", "chrono", "crunchyroll-rs-internal", + "dash-mpd", + "futures-util", "http", "lazy_static", "m3u8-rs", @@ -318,13 +315,13 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.5" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a260a73e733bb0ce30343caaed5e968d3c1cc2ea0ab27c601481e9ef22a2fd7" +checksum = "3f26b71b36db3139ce788545c2bffa7e69d4fd7b689f143086c6283e847a4668" dependencies = [ - "darling", + "darling 0.14.4", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -337,43 +334,21 @@ dependencies = [ "typenum", ] -[[package]] -name = "csv" -version = "1.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" -dependencies = [ - "bstr", - "csv-core", - "itoa 0.4.8", - "ryu", - "serde", -] - -[[package]] -name = "csv-core" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" -dependencies = [ - "memchr", -] - [[package]] name = "ctrlc" -version = "3.2.4" +version = "3.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1631ca6e3c59112501a9d87fd86f21591ff77acd31331e8a73f8d80a65bbdd71" +checksum = "bbcf33c2a618cbe41ee43ae6e9f2e48368cd9f9db2896f10167d8d762679f639" dependencies = [ "nix", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] name = "cxx" -version = "1.0.86" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d1075c37807dcf850c379432f0df05ba52cc30f279c5cfc43cc221ce7f8579" +checksum = "a9c00419335c41018365ddf7e4d5f1c12ee3659ddcf3e01974650ba1de73d038" dependencies = [ "cc", "cxxbridge-flags", @@ -383,9 +358,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.86" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5044281f61b27bc598f2f6647d480aed48d2bf52d6eb0b627d84c0361b17aa70" +checksum = "fb8307ad413a98fff033c8545ecf133e3257747b3bae935e7602aab8aa92d4ca" dependencies = [ "cc", "codespan-reporting", @@ -393,79 +368,145 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn", + "syn 2.0.6", ] [[package]] name = "cxxbridge-flags" -version = "1.0.86" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61b50bc93ba22c27b0d31128d2d130a0a6b3d267ae27ef7e4fae2167dfe8781c" +checksum = "edc52e2eb08915cb12596d29d55f0b5384f00d697a646dbd269b6ecb0fbd9d31" [[package]] name = "cxxbridge-macro" -version = "1.0.86" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e61fda7e62115119469c7b3591fd913ecca96fb766cfd3f2e2502ab7bc87a5" +checksum = "631569015d0d8d54e6c241733f944042623ab6df7bc3be7466874b05fcdb1c5f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.6", ] [[package]] name = "darling" -version = "0.14.2" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0dd3cd20dc6b5a876612a6e5accfe7f3dd883db6d07acfbf14c128f61550dfa" +checksum = "0d706e75d87e35569db781a9b5e2416cff1236a47ed380831f959382ccd5f858" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.10.2", + "darling_macro 0.10.2", +] + +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core 0.14.4", + "darling_macro 0.14.4", ] [[package]] name = "darling_core" -version = "0.14.2" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a784d2ccaf7c98501746bf0be29b2022ba41fd62a2e622af997a03e9f972859f" +checksum = "f0c960ae2da4de88a91b2d920c2a7233b400bc33cb28453a2987822d8392519b" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", - "strsim", - "syn", + "strsim 0.9.3", + "syn 1.0.109", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn 1.0.109", ] [[package]] name = "darling_macro" -version = "0.14.2" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7618812407e9402654622dd402b0a89dff9ba93badd6540781526117b92aab7e" +checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72" dependencies = [ - "darling_core", + "darling_core 0.10.2", "quote", - "syn", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core 0.14.4", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "dash-mpd" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df35f3b3b0fde2747a01530de0a81f7a27620d050f656e518ef0550557f564de" +dependencies = [ + "chrono", + "fs-err", + "iso8601", + "log", + "num-traits", + "quick-xml", + "regex", + "serde", + "serde_with", + "thiserror", + "xattr", +] + +[[package]] +name = "derive_setters" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1cf41b4580a37cca5ef2ada2cc43cf5d6be3983f4522e83010d67ab6925e84b" +dependencies = [ + "darling 0.10.2", + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] name = "dirs" -version = "4.0.0" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +checksum = "dece029acd3353e3a58ac2e3eb3c8d6c35827a892edc6cc4138ef9c33df46ecd" dependencies = [ "dirs-sys", ] [[package]] name = "dirs-sys" -version = "0.3.7" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +checksum = "04414300db88f70d74c5ff54e50f9e1d1737d9a5b90f53fcf2e95ca2a9ab554b" dependencies = [ "libc", "redox_users", - "winapi", + "windows-sys 0.45.0", ] [[package]] @@ -476,9 +517,9 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" [[package]] name = "encoding_rs" -version = "0.8.31" +version = "0.8.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" dependencies = [ "cfg-if", ] @@ -506,9 +547,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" dependencies = [ "instant", ] @@ -544,43 +585,49 @@ dependencies = [ ] [[package]] -name = "futures-channel" -version = "0.3.25" +name = "fs-err" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" +checksum = "0845fa252299212f0389d64ba26f34fa32cfe41588355f21ed507c59a0f64541" + +[[package]] +name = "futures-channel" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "164713a5a0dcc3e7b4b1ed7d3b433cabc18025386f9339346e8daf15963cf7ac" dependencies = [ "futures-core", ] [[package]] name = "futures-core" -version = "0.3.25" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" +checksum = "86d7a0c1aa76363dac491de0ee99faf6941128376f1cf96f07db7603b7de69dd" [[package]] name = "futures-io" -version = "0.3.25" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" +checksum = "89d422fa3cbe3b40dca574ab087abb5bc98258ea57eea3fd6f1fa7162c778b91" [[package]] name = "futures-sink" -version = "0.3.25" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" +checksum = "ec93083a4aecafb2a80a885c9de1f0ccae9dbd32c2bb54b0c3a65690e0b8d2f2" [[package]] name = "futures-task" -version = "0.3.25" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" +checksum = "fd65540d33b37b16542a0438c12e6aeead10d4ac5d05bd3f805b8f35ab592879" [[package]] name = "futures-util" -version = "0.3.25" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" +checksum = "3ef6b17e481503ec85211fed8f39d1970f128935ca1f814cd32ac4a6842e84ab" dependencies = [ "futures-core", "futures-io", @@ -614,9 +661,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" dependencies = [ "bytes", "fnv", @@ -639,9 +686,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "heck" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" @@ -653,14 +700,26 @@ dependencies = [ ] [[package]] -name = "http" -version = "0.2.8" +name = "hermit-abi" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ "bytes", "fnv", - "itoa 1.0.5", + "itoa", ] [[package]] @@ -688,9 +747,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.23" +version = "0.14.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c" +checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" dependencies = [ "bytes", "futures-channel", @@ -701,7 +760,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa 1.0.5", + "itoa", "pin-project-lite", "socket2", "tokio", @@ -738,16 +797,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.53" +version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +checksum = "0c17cc76786e99f8d2f055c11159e7f0091c42474dcc3189fbab96072e873e6d" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "winapi", + "windows", ] [[package]] @@ -795,13 +854,14 @@ checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" dependencies = [ "autocfg", "hashbrown", + "serde", ] [[package]] name = "indicatif" -version = "0.17.2" +version = "0.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4295cbb7573c16d310e99e713cf9e75101eb190ab31fccd35f2d2691b4352b19" +checksum = "cef509aa9bc73864d6756f0d34d35504af3cf0844373afe9b8669a5b8005a729" dependencies = [ "console", "number_prefix", @@ -830,12 +890,13 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.3" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" +checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" dependencies = [ + "hermit-abi 0.3.1", "libc", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -846,33 +907,36 @@ checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" [[package]] name = "is-terminal" -version = "0.4.2" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189" +checksum = "8687c819457e979cc940d09cb16e42a1bf70aa6b60a549de6d3a62a0ee90c69e" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.1", "io-lifetimes", "rustix", - "windows-sys", + "windows-sys 0.45.0", +] + +[[package]] +name = "iso8601" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924e5d73ea28f59011fec52a0d12185d496a9b075d360657aed2a5707f701153" +dependencies = [ + "nom", ] [[package]] name = "itoa" -version = "0.4.8" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" - -[[package]] -name = "itoa" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "js-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" dependencies = [ "wasm-bindgen", ] @@ -885,9 +949,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.139" +version = "0.2.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" [[package]] name = "link-cplusplus" @@ -925,9 +989,9 @@ dependencies = [ [[package]] name = "matches" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" +checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" [[package]] name = "memchr" @@ -937,9 +1001,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "mime" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "minimal-lexical" @@ -949,14 +1013,14 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mio" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -979,11 +1043,11 @@ dependencies = [ [[package]] name = "nix" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46a58d1d356c6597d08cde02c2f09d785b09e28711837b1ed667dc652c08a694" +checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if", "libc", "static_assertions", @@ -991,9 +1055,9 @@ dependencies = [ [[package]] name = "nom" -version = "7.1.2" +version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5507769c4919c998e69e49c839d9dc6e693ede4cc4290d6ad8b41d4f09c548c" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" dependencies = [ "memchr", "minimal-lexical", @@ -1024,7 +1088,7 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi", + "hermit-abi 0.2.6", "libc", ] @@ -1036,17 +1100,17 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "once_cell" -version = "1.17.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "openssl" -version = "0.10.45" +version = "0.10.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" +checksum = "d8b277f87dacc05a6b709965d1cbafac4649d6ce9f3ce9ceb88508b5666dfec9" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if", "foreign-types", "libc", @@ -1063,7 +1127,7 @@ checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1074,9 +1138,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.80" +version = "0.9.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" +checksum = "a95792af3c4e0153c3914df2261bedd30a98476f94dc892b67dfe1d89d433a04" dependencies = [ "autocfg", "cc", @@ -1087,9 +1151,9 @@ dependencies = [ [[package]] name = "os_str_bytes" -version = "6.4.1" +version = "6.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" +checksum = "ceedf44fb00f2d1984b0bc98102627ce622e083e49a5bacdb3e514fa4238e267" [[package]] name = "percent-encoding" @@ -1130,7 +1194,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "version_check", ] @@ -1145,17 +1209,11 @@ dependencies = [ "version_check", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" - [[package]] name = "proc-macro2" -version = "1.0.49" +version = "1.0.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" +checksum = "ba466839c78239c09faf015484e5cc04860f88242cff4d03eb038f04b4699b73" dependencies = [ "unicode-ident", ] @@ -1177,10 +1235,20 @@ dependencies = [ ] [[package]] -name = "quote" -version = "1.0.23" +name = "quick-xml" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "e5c1a97b1bc42b1d550bfb48d4262153fe400a12bab1511821736f7eac76d7e2" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quote" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" dependencies = [ "proc-macro2", ] @@ -1191,7 +1259,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -1207,43 +1275,28 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" +checksum = "cce168fea28d3e05f158bda4576cf0c844d5045bc2cc3620fa0292ed5bb5814c" dependencies = [ "aho-corasick", "memchr", "regex-syntax", ] -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" - [[package]] name = "regex-syntax" -version = "0.6.28" +version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" - -[[package]] -name = "remove_dir_all" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "reqwest" -version = "0.11.13" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" +checksum = "0ba30cc2c0cd02af1222ed216ba659cdb2f879dfe3181852fe7c50b1d0005949" dependencies = [ - "base64 0.13.1", + "base64 0.21.0", "bytes", "cookie", "cookie_store", @@ -1264,7 +1317,6 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "proc-macro-hack", "rustls", "rustls-pemfile", "serde", @@ -1299,23 +1351,23 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.6" +version = "0.36.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4feacf7db682c6c329c4ede12649cd36ecab0f3be5b7d74e6a20304725db4549" +checksum = "db4165c9963ab29e422d6c26fbc1d37f15bace6b2810221f9d925023480fcf0e" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] name = "rustls" -version = "0.20.7" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "539a2bfe908f471bfa933876bd1eb6a19cf2176d375f82ef7f99530a40e48c2c" +checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" dependencies = [ "log", "ring", @@ -1334,9 +1386,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" [[package]] name = "sanitize-filename" @@ -1354,14 +1406,14 @@ version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" dependencies = [ - "windows-sys", + "windows-sys 0.42.0", ] [[package]] name = "scratch" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" +checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" [[package]] name = "sct" @@ -1375,11 +1427,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.7.0" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" +checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "core-foundation-sys", "libc", @@ -1388,9 +1440,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.6.1" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" dependencies = [ "core-foundation-sys", "libc", @@ -1398,31 +1450,31 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.152" +version = "1.0.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "771d4d9c4163ee138805e12c710dd365e4f44be8be0503cb1bb9eb989425d9c9" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "e801c1712f48475582b7696ac71e0ca34ebb30e09338425384269d9717c62cad" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.6", ] [[package]] name = "serde_json" -version = "1.0.91" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" +checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea" dependencies = [ - "itoa 1.0.5", + "itoa", "ryu", "serde", ] @@ -1434,16 +1486,50 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.5", + "itoa", "ryu", "serde", ] [[package]] -name = "signal-hook" -version = "0.3.14" +name = "serde_with" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d" +checksum = "85456ffac572dc8826334164f2fb6fb40a7c766aebe195a2a21ee69ee2885ecf" +dependencies = [ + "base64 0.13.1", + "chrono", + "hex", + "indexmap", + "serde", + "serde_json", + "serde_with_macros", + "time 0.3.20", +] + +[[package]] +name = "serde_with_macros" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cbcd6104f8a4ab6af7f6be2a0da6be86b9de3c401f6e86bb856ab2af739232f" +dependencies = [ + "darling 0.14.4", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "shlex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" + +[[package]] +name = "signal-hook" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "732768f1176d21d09e076c23a93123d40bba92d50c4058da34d45c8de8e682b9" dependencies = [ "libc", "signal-hook-registry", @@ -1451,18 +1537,18 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" dependencies = [ "libc", ] [[package]] name = "slab" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" dependencies = [ "autocfg", ] @@ -1475,14 +1561,14 @@ checksum = "133659a15339456eeeb07572eb02a91c91e9815e9cbc89566944d2c8d3efdbf6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", @@ -1500,6 +1586,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strsim" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" + [[package]] name = "strsim" version = "0.10.0" @@ -1508,9 +1600,20 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.107" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece519cfaf36269ea69d16c363fa1d59ceba8296bbfbfc003c3176d01f2816ee" dependencies = [ "proc-macro2", "quote", @@ -1519,68 +1622,67 @@ dependencies = [ [[package]] name = "sys-locale" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3358acbb4acd4146138b9bda219e904a6bb5aaaa237f8eed06f4d6bc1580ecee" +checksum = "f8a11bd9c338fdba09f7881ab41551932ad42e405f61d01e8406baea71c07aee" dependencies = [ "js-sys", "libc", "wasm-bindgen", "web-sys", - "winapi", + "windows-sys 0.45.0", ] [[package]] name = "tempfile" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95" dependencies = [ "cfg-if", "fastrand", - "libc", "redox_syscall", - "remove_dir_all", - "winapi", + "rustix", + "windows-sys 0.42.0", ] [[package]] name = "termcolor" -version = "1.1.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" dependencies = [ "winapi-util", ] [[package]] name = "terminal_size" -version = "0.2.3" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb20089a8ba2b69debd491f8d2d023761cbf196e999218c591fa1e7e15a21907" +checksum = "4c9afddd2cec1c0909f06b00ef33f94ab2cc0578c4a610aa208ddfec8aa2b43a" dependencies = [ "rustix", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] name = "thiserror" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.6", ] [[package]] @@ -1596,11 +1698,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.17" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" +checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" dependencies = [ - "itoa 1.0.5", + "itoa", "serde", "time-core", "time-macros", @@ -1614,9 +1716,9 @@ checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" [[package]] name = "time-macros" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" +checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" dependencies = [ "time-core", ] @@ -1632,15 +1734,15 @@ dependencies = [ [[package]] name = "tinyvec_macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.24.1" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae" +checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" dependencies = [ "autocfg", "bytes", @@ -1651,7 +1753,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -1662,14 +1764,14 @@ checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "tokio-native-tls" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", "tokio", @@ -1688,9 +1790,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.4" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" dependencies = [ "bytes", "futures-core", @@ -1740,15 +1842,15 @@ checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "unicode-bidi" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" [[package]] name = "unicode-normalization" @@ -1818,9 +1920,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1828,24 +1930,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" dependencies = [ "cfg-if", "js-sys", @@ -1855,9 +1957,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1865,28 +1967,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" [[package]] name = "web-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" dependencies = [ "js-sys", "wasm-bindgen", @@ -1942,6 +2044,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.42.0" @@ -1958,46 +2069,70 @@ dependencies = [ ] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.0" +name = "windows-sys" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_msvc" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_i686_gnu" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_msvc" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_x86_64_gnu" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_msvc" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "winreg" @@ -2007,3 +2142,12 @@ checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ "winapi", ] + +[[package]] +name = "xattr" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea263437ca03c1522846a4ddafbca2542d0ad5ed9b784909d4b27b76f62bc34a" +dependencies = [ + "libc", +] diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index e56cf71..f86a98f 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -7,12 +7,12 @@ edition = "2021" [dependencies] anyhow = "1.0" async-trait = "0.1" -clap = { version = "4.0", features = ["derive", "string"] } +clap = { version = "4.1", features = ["derive", "string"] } chrono = "0.4" -crunchyroll-rs = "0.2" -csv = "1.1" +crunchyroll-rs = { version = "0.3", features = ["dash-stream"] } ctrlc = "3.2" -dirs = "4.0" +dirs = "5.0" +derive_setters = "0.1" indicatif = "0.17" lazy_static = "1.4" log = { version = "0.4", features = ["std"] } @@ -23,9 +23,9 @@ serde = "1.0" serde_json = "1.0" shlex = "1.1" signal-hook = "0.3" -tempfile = "3.3" +tempfile = "3.4" terminal_size = "0.2" -tokio = { version = "1.24", features = ["macros", "rt-multi-thread", "time"] } +tokio = { version = "1.26", features = ["macros", "rt-multi-thread", "time"] } sys-locale = "0.2" [build-dependencies] diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs new file mode 100644 index 0000000..3583996 --- /dev/null +++ b/crunchy-cli-core/src/archive/command.rs @@ -0,0 +1,184 @@ +use crate::archive::filter::ArchiveFilter; +use crate::utils::context::Context; +use crate::utils::download::MergeBehavior; +use crate::utils::ffmpeg::FFmpegPreset; +use crate::utils::filter::Filter; +use crate::utils::format::formats_visual_output; +use crate::utils::locale::all_locale_in_locales; +use crate::utils::log::progress; +use crate::utils::os::{free_file, has_ffmpeg, is_special_file}; +use crate::utils::parse::parse_url; +use crate::Execute; +use anyhow::bail; +use anyhow::Result; +use crunchyroll_rs::media::Resolution; +use crunchyroll_rs::Locale; +use log::debug; +use std::path::PathBuf; + +#[derive(Clone, Debug, clap::Parser)] +#[clap(about = "Archive a video")] +#[command(arg_required_else_help(true))] +#[command()] +pub struct Archive { + #[arg(help = format!("Audio languages. Can be used multiple times. \ + Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] + #[arg(long_help = format!("Audio languages. Can be used multiple times. \ + Available languages are:\n{}", Locale::all().into_iter().map(|l| format!("{:<6} → {}", l.to_string(), l.to_human_readable())).collect::>().join("\n ")))] + #[arg(short, long, default_values_t = vec![Locale::ja_JP, crate::utils::locale::system_locale()])] + pub(crate) locale: Vec, + #[arg(help = format!("Subtitle languages. Can be used multiple times. \ + Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] + #[arg(long_help = format!("Subtitle languages. Can be used multiple times. \ + Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] + #[arg(short, long, default_values_t = Locale::all())] + pub(crate) subtitle: Vec, + + #[arg(help = "Name of the output file")] + #[arg(long_help = "Name of the output file.\ + If you use one of the following pattern they will get replaced:\n \ + {title} → Title of the video\n \ + {series_name} → Name of the series\n \ + {season_name} → Name of the season\n \ + {audio} → Audio language of the video\n \ + {resolution} → Resolution of the video\n \ + {season_number} → Number of the season\n \ + {episode_number} → Number of the episode\n \ + {relative_episode_number} → Number of the episode relative to its season\ + {series_id} → ID of the series\n \ + {season_id} → ID of the season\n \ + {episode_id} → ID of the episode")] + #[arg(short, long, default_value = "{title}.mkv")] + pub(crate) output: String, + + #[arg(help = "Video resolution")] + #[arg(long_help = "The video resolution.\ + Can either be specified via the pixels (e.g. 1920x1080), the abbreviation for pixels (e.g. 1080p) or 'common-use' words (e.g. best). \ + Specifying the exact pixels is not recommended, use one of the other options instead. \ + Crunchyroll let you choose the quality with pixel abbreviation on their clients, so you might be already familiar with the available options. \ + The available common-use words are 'best' (choose the best resolution available) and 'worst' (worst resolution available)")] + #[arg(short, long, default_value = "best")] + #[arg(value_parser = crate::utils::clap::clap_parse_resolution)] + pub(crate) resolution: Resolution, + + #[arg( + help = "Sets the behavior of the stream merging. Valid behaviors are 'auto', 'audio' and 'video'" + )] + #[arg( + long_help = "Because of local restrictions (or other reasons) some episodes with different languages does not have the same length (e.g. when some scenes were cut out). \ + With this flag you can set the behavior when handling multiple language. + Valid options are 'audio' (stores one video and all other languages as audio only), 'video' (stores the video + audio for every language) and 'auto' (detects if videos differ in length: if so, behave like 'video' else like 'audio')" + )] + #[arg(short, long, default_value = "auto")] + #[arg(value_parser = MergeBehavior::parse)] + pub(crate) merge: MergeBehavior, + + #[arg(help = format!("Presets for video converting. Can be used multiple times. \ + Available presets: \n {}", FFmpegPreset::available_matches_human_readable().join("\n ")))] + #[arg(long_help = format!("Presets for video converting. Can be used multiple times. \ + Generally used to minify the file size with keeping (nearly) the same quality. \ + It is recommended to only use this if you archive videos with high resolutions since low resolution videos tend to result in a larger file with any of the provided presets. \ + Available presets: \n {}", FFmpegPreset::available_matches_human_readable().join("\n ")))] + #[arg(long)] + #[arg(value_parser = FFmpegPreset::parse)] + pub(crate) ffmpeg_preset: Option, + + #[arg( + help = "Set which subtitle language should be set as default / auto shown when starting a video" + )] + #[arg(long)] + pub(crate) default_subtitle: Option, + + #[arg(help = "Skip files which are already existing")] + #[arg(long, default_value_t = false)] + pub(crate) skip_existing: bool, + + #[arg(help = "Crunchyroll series url(s)")] + pub(crate) urls: Vec, +} + +#[async_trait::async_trait(?Send)] +impl Execute for Archive { + fn pre_check(&mut self) -> Result<()> { + if !has_ffmpeg() { + bail!("FFmpeg is needed to run this command") + } else if PathBuf::from(&self.output) + .extension() + .unwrap_or_default() + .to_string_lossy() + != "mkv" + && !is_special_file(PathBuf::from(&self.output)) + { + bail!("File extension is not '.mkv'. Currently only matroska / '.mkv' files are supported") + } + + self.locale = all_locale_in_locales(self.locale.clone()); + self.subtitle = all_locale_in_locales(self.subtitle.clone()); + + Ok(()) + } + + async fn execute(self, ctx: Context) -> Result<()> { + let mut parsed_urls = vec![]; + + for (i, url) in self.urls.clone().into_iter().enumerate() { + let progress_handler = progress!("Parsing url {}", i + 1); + match parse_url(&ctx.crunchy, url.clone(), true).await { + Ok((media_collection, url_filter)) => { + progress_handler.stop(format!("Parsed url {}", i + 1)); + parsed_urls.push((media_collection, url_filter)) + } + Err(e) => bail!("url {} could not be parsed: {}", url, e), + }; + } + + for (i, (media_collection, url_filter)) in parsed_urls.into_iter().enumerate() { + let progress_handler = progress!("Fetching series details"); + let archive_formats = ArchiveFilter::new(url_filter, self.clone()) + .visit(media_collection) + .await?; + + if archive_formats.is_empty() { + progress_handler.stop(format!("Skipping url {} (no matching videos found)", i + 1)); + continue; + } + progress_handler.stop(format!("Loaded series information for url {}", i + 1)); + + formats_visual_output(archive_formats.iter().map(|(_, f)| f).collect()); + + for (downloader, mut format) in archive_formats { + let formatted_path = format.format_path((&self.output).into(), true); + let (path, changed) = free_file(formatted_path.clone()); + + if changed && self.skip_existing { + debug!( + "Skipping already existing file '{}'", + formatted_path.to_string_lossy() + ); + continue; + } + + format.locales.sort_by(|(a, _), (b, _)| { + self.locale + .iter() + .position(|l| l == a) + .cmp(&self.locale.iter().position(|l| l == b)) + }); + for (_, subtitles) in format.locales.iter_mut() { + subtitles.sort_by(|a, b| { + self.subtitle + .iter() + .position(|l| l == a) + .cmp(&self.subtitle.iter().position(|l| l == b)) + }) + } + + format.visual_output(&path); + + downloader.download(&ctx, &path).await? + } + } + + Ok(()) + } +} diff --git a/crunchy-cli-core/src/archive/filter.rs b/crunchy-cli-core/src/archive/filter.rs new file mode 100644 index 0000000..d0db3d5 --- /dev/null +++ b/crunchy-cli-core/src/archive/filter.rs @@ -0,0 +1,482 @@ +use crate::archive::command::Archive; +use crate::utils::download::{DownloadBuilder, DownloadFormat, Downloader, MergeBehavior}; +use crate::utils::filter::{real_dedup_vec, Filter}; +use crate::utils::format::{Format, SingleFormat}; +use crate::utils::parse::UrlFilter; +use crate::utils::video::variant_data_from_stream; +use anyhow::{bail, Result}; +use chrono::Duration; +use crunchyroll_rs::media::{Subtitle, VariantData}; +use crunchyroll_rs::{Concert, Episode, Locale, Movie, MovieListing, MusicVideo, Season, Series}; +use log::warn; +use std::collections::HashMap; +use std::hash::Hash; + +pub(crate) struct FilterResult { + format: SingleFormat, + video: VariantData, + audio: VariantData, + duration: Duration, + subtitles: Vec, +} + +enum Visited { + Series, + Season, + None, +} + +pub(crate) struct ArchiveFilter { + url_filter: UrlFilter, + archive: Archive, + season_episode_count: HashMap>, + season_subtitles_missing: Vec, + visited: Visited, +} + +impl ArchiveFilter { + pub(crate) fn new(url_filter: UrlFilter, archive: Archive) -> Self { + Self { + url_filter, + archive, + season_episode_count: HashMap::new(), + season_subtitles_missing: vec![], + visited: Visited::None, + } + } +} + +#[async_trait::async_trait] +impl Filter for ArchiveFilter { + type T = Vec; + type Output = (Downloader, Format); + + async fn visit_series(&mut self, series: Series) -> Result> { + // `series.audio_locales` isn't always populated b/c of crunchyrolls api. so check if the + // audio is matching only if the field is populated + if !series.audio_locales.is_empty() { + let missing_audio = missing_locales(&series.audio_locales, &self.archive.locale); + if !missing_audio.is_empty() { + warn!( + "Series {} is not available with {} audio", + series.title, + missing_audio + .into_iter() + .map(|l| l.to_string()) + .collect::>() + .join(", ") + ) + } + let missing_subtitle = + missing_locales(&series.subtitle_locales, &self.archive.subtitle); + if !missing_subtitle.is_empty() { + warn!( + "Series {} is not available with {} subtitles", + series.title, + missing_subtitle + .into_iter() + .map(|l| l.to_string()) + .collect::>() + .join(", ") + ) + } + self.visited = Visited::Series + } + Ok(series.seasons().await?) + } + + async fn visit_season(&mut self, mut season: Season) -> Result> { + if !self.url_filter.is_season_valid(season.season_number) { + return Ok(vec![]); + } + + let mut seasons = season.version(self.archive.locale.clone()).await?; + if self + .archive + .locale + .iter() + .any(|l| season.audio_locales.contains(l)) + { + seasons.insert(0, season.clone()); + } + + if !matches!(self.visited, Visited::Series) { + let mut audio_locales: Vec = seasons + .iter() + .map(|s| s.audio_locales.clone()) + .flatten() + .collect(); + real_dedup_vec(&mut audio_locales); + let missing_audio = missing_locales(&audio_locales, &self.archive.locale); + if !missing_audio.is_empty() { + warn!( + "Season {} is not available with {} audio", + season.season_number, + missing_audio + .into_iter() + .map(|l| l.to_string()) + .collect::>() + .join(", ") + ) + } + + let subtitle_locales: Vec = seasons + .iter() + .map(|s| s.subtitle_locales.clone()) + .flatten() + .collect(); + let missing_subtitle = missing_locales(&subtitle_locales, &self.archive.subtitle); + if !missing_subtitle.is_empty() { + warn!( + "Season {} is not available with {} subtitles", + season.season_number, + missing_subtitle + .into_iter() + .map(|l| l.to_string()) + .collect::>() + .join(", ") + ) + } + self.visited = Visited::Season + } + + let mut episodes = vec![]; + for season in seasons { + episodes.extend(season.episodes().await?) + } + + if Format::has_relative_episodes_fmt(&self.archive.output) { + for episode in episodes.iter() { + self.season_episode_count + .entry(episode.season_number) + .or_insert(vec![]) + .push(episode.id.clone()) + } + } + + Ok(episodes) + } + + async fn visit_episode(&mut self, mut episode: Episode) -> Result> { + if !self + .url_filter + .is_episode_valid(episode.episode_number, episode.season_number) + { + return Ok(None); + } + + let mut episodes = vec![]; + if !matches!(self.visited, Visited::Series) && !matches!(self.visited, Visited::Season) { + episodes.extend(episode.version(self.archive.locale.clone()).await?); + let audio_locales: Vec = + episodes.iter().map(|e| e.audio_locale.clone()).collect(); + let missing_audio = missing_locales(&audio_locales, &self.archive.locale); + if !missing_audio.is_empty() { + warn!( + "Episode {} is not available with {} audio", + episode.episode_number, + missing_audio + .into_iter() + .map(|l| l.to_string()) + .collect::>() + .join(", ") + ) + } + + let mut subtitle_locales: Vec = episodes + .iter() + .map(|e| e.subtitle_locales.clone()) + .flatten() + .collect(); + real_dedup_vec(&mut subtitle_locales); + let missing_subtitles = missing_locales(&subtitle_locales, &self.archive.subtitle); + if !missing_subtitles.is_empty() + && !self + .season_subtitles_missing + .contains(&episode.season_number) + { + warn!( + "Episode {} is not available with {} subtitles", + episode.episode_number, + missing_subtitles + .into_iter() + .map(|l| l.to_string()) + .collect::>() + .join(", ") + ); + self.season_subtitles_missing.push(episode.season_number) + } + } else { + episodes.push(episode.clone()) + } + + let mut formats = vec![]; + for episode in episodes { + let stream = episode.streams().await?; + let (video, audio) = if let Some((video, audio)) = + variant_data_from_stream(&stream, &self.archive.resolution).await? + { + (video, audio) + } else { + bail!( + "Resolution ({}) is not available for episode {} ({}) of {} season {}", + &self.archive.resolution, + episode.episode_number, + episode.title, + episode.series_title, + episode.season_number, + ); + }; + let subtitles: Vec = self + .archive + .subtitle + .iter() + .filter_map(|s| stream.subtitles.get(s).cloned()) + .collect(); + + let relative_episode_number = if Format::has_relative_episodes_fmt(&self.archive.output) + { + if self + .season_episode_count + .get(&episode.season_number) + .is_none() + { + let season_episodes = episode.season().await?.episodes().await?; + self.season_episode_count.insert( + episode.season_number, + season_episodes.into_iter().map(|e| e.id).collect(), + ); + } + let relative_episode_number = self + .season_episode_count + .get(&episode.season_number) + .unwrap() + .iter() + .position(|id| id == &episode.id); + if relative_episode_number.is_none() { + warn!( + "Failed to get relative episode number for episode {} ({}) of {} season {}", + episode.episode_number, + episode.title, + episode.series_title, + episode.season_number, + ) + } + relative_episode_number + } else { + None + }; + + formats.push(FilterResult { + format: SingleFormat::new_from_episode( + &episode, + &video, + subtitles.iter().map(|s| s.locale.clone()).collect(), + relative_episode_number.map(|n| n as u32), + ), + video, + audio, + duration: episode.duration.clone(), + subtitles, + }) + } + + Ok(Some(formats)) + } + + async fn visit_movie_listing(&mut self, movie_listing: MovieListing) -> Result> { + Ok(movie_listing.movies().await?) + } + + async fn visit_movie(&mut self, movie: Movie) -> Result> { + let stream = movie.streams().await?; + let subtitles: Vec<&Subtitle> = self + .archive + .subtitle + .iter() + .filter_map(|l| stream.subtitles.get(l)) + .collect(); + + let missing_subtitles = missing_locales( + &subtitles.iter().map(|&s| s.locale.clone()).collect(), + &self.archive.subtitle, + ); + if !missing_subtitles.is_empty() { + warn!( + "Movie '{}' is not available with {} subtitles", + movie.title, + missing_subtitles + .into_iter() + .map(|l| l.to_string()) + .collect::>() + .join(", ") + ) + } + + let (video, audio) = if let Some((video, audio)) = + variant_data_from_stream(&stream, &self.archive.resolution).await? + { + (video, audio) + } else { + bail!( + "Resolution ({}) of movie {} is not available", + self.archive.resolution, + movie.title + ) + }; + + Ok(Some(vec![FilterResult { + format: SingleFormat::new_from_movie(&movie, &video, vec![]), + video, + audio, + duration: movie.duration, + subtitles: vec![], + }])) + } + + async fn visit_music_video(&mut self, music_video: MusicVideo) -> Result> { + let stream = music_video.streams().await?; + let (video, audio) = if let Some((video, audio)) = + variant_data_from_stream(&stream, &self.archive.resolution).await? + { + (video, audio) + } else { + bail!( + "Resolution ({}) of music video {} is not available", + self.archive.resolution, + music_video.title + ) + }; + + Ok(Some(vec![FilterResult { + format: SingleFormat::new_from_music_video(&music_video, &video), + video, + audio, + duration: music_video.duration, + subtitles: vec![], + }])) + } + + async fn visit_concert(&mut self, concert: Concert) -> Result> { + let stream = concert.streams().await?; + let (video, audio) = if let Some((video, audio)) = + variant_data_from_stream(&stream, &self.archive.resolution).await? + { + (video, audio) + } else { + bail!( + "Resolution ({}x{}) of music video {} is not available", + self.archive.resolution.width, + self.archive.resolution.height, + concert.title + ) + }; + + Ok(Some(vec![FilterResult { + format: SingleFormat::new_from_concert(&concert, &video), + video, + audio, + duration: concert.duration, + subtitles: vec![], + }])) + } + + async fn finish(self, input: Vec) -> Result> { + let flatten_input: Vec = input.into_iter().flatten().collect(); + + #[derive(Hash, Eq, PartialEq)] + struct SortKey { + season: u32, + episode: String, + } + + let mut sorted: HashMap> = HashMap::new(); + for data in flatten_input { + sorted + .entry(SortKey { + season: data.format.season_number, + episode: data.format.episode_number.to_string(), + }) + .or_insert(vec![]) + .push(data) + } + + let mut values: Vec> = sorted.into_values().collect(); + values.sort_by(|a, b| { + a.first() + .unwrap() + .format + .sequence_number + .total_cmp(&b.first().unwrap().format.sequence_number) + }); + + let mut result = vec![]; + for data in values { + let single_formats: Vec = + data.iter().map(|fr| fr.format.clone()).collect(); + let format = Format::from_single_formats(single_formats); + + let mut downloader = DownloadBuilder::new() + .default_subtitle(self.archive.default_subtitle.clone()) + .ffmpeg_preset(self.archive.ffmpeg_preset.clone().unwrap_or_default()) + .output_format(Some("matroska".to_string())) + .audio_sort(Some(self.archive.locale.clone())) + .subtitle_sort(Some(self.archive.subtitle.clone())) + .build(); + + match self.archive.merge.clone() { + MergeBehavior::Video => { + for d in data { + downloader.add_format(DownloadFormat { + video: (d.video, d.format.audio.clone()), + audios: vec![(d.audio, d.format.audio.clone())], + subtitles: d.subtitles, + }) + } + } + MergeBehavior::Audio => downloader.add_format(DownloadFormat { + video: ( + data.first().unwrap().video.clone(), + data.first().unwrap().format.audio.clone(), + ), + audios: data + .iter() + .map(|d| (d.audio.clone(), d.format.audio.clone())) + .collect(), + subtitles: data.iter().map(|d| d.subtitles.clone()).flatten().collect(), + }), + MergeBehavior::Auto => { + let mut download_formats: HashMap = HashMap::new(); + + for d in data { + if let Some(download_format) = download_formats.get_mut(&d.duration) { + download_format.audios.push((d.audio, d.format.audio)); + download_format.subtitles.extend(d.subtitles) + } else { + download_formats.insert( + d.duration, + DownloadFormat { + video: (d.video, d.format.audio.clone()), + audios: vec![(d.audio, d.format.audio)], + subtitles: d.subtitles, + }, + ); + } + } + + for download_format in download_formats.into_values() { + downloader.add_format(download_format) + } + } + } + + result.push((downloader, format)) + } + + Ok(result) + } +} + +fn missing_locales<'a>(available: &Vec, searched: &'a Vec) -> Vec<&'a Locale> { + searched.iter().filter(|p| !available.contains(p)).collect() +} diff --git a/crunchy-cli-core/src/archive/mod.rs b/crunchy-cli-core/src/archive/mod.rs new file mode 100644 index 0000000..c3544a4 --- /dev/null +++ b/crunchy-cli-core/src/archive/mod.rs @@ -0,0 +1,4 @@ +mod command; +mod filter; + +pub use command::Archive; diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs deleted file mode 100644 index e500f21..0000000 --- a/crunchy-cli-core/src/cli/archive.rs +++ /dev/null @@ -1,618 +0,0 @@ -use crate::cli::log::tab_info; -use crate::cli::utils::{ - all_locale_in_locales, download_segments, find_multiple_seasons_with_same_number, - find_resolution, interactive_season_choosing, FFmpegPreset, -}; -use crate::utils::context::Context; -use crate::utils::format::Format; -use crate::utils::log::progress; -use crate::utils::os::{free_file, has_ffmpeg, is_special_file, tempfile}; -use crate::utils::parse::{parse_url, UrlFilter}; -use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number}; -use crate::utils::subtitle::{download_subtitle, Subtitle}; -use crate::utils::video::get_video_length; -use crate::Execute; -use anyhow::{bail, Result}; -use crunchyroll_rs::media::Resolution; -use crunchyroll_rs::{Locale, Media, MediaCollection, Series}; -use log::{debug, error, info}; -use std::path::PathBuf; -use std::process::{Command, Stdio}; -use tempfile::TempPath; - -#[derive(Clone, Debug)] -pub enum MergeBehavior { - Auto, - Audio, - Video, -} - -impl MergeBehavior { - fn parse(s: &str) -> Result { - Ok(match s.to_lowercase().as_str() { - "auto" => MergeBehavior::Auto, - "audio" => MergeBehavior::Audio, - "video" => MergeBehavior::Video, - _ => return Err(format!("'{}' is not a valid merge behavior", s)), - }) - } -} - -#[derive(Debug, clap::Parser)] -#[clap(about = "Archive a video")] -#[command(arg_required_else_help(true))] -#[command()] -pub struct Archive { - #[arg(help = format!("Audio languages. Can be used multiple times. \ - Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] - #[arg(long_help = format!("Audio languages. Can be used multiple times. \ - Available languages are:\n{}", Locale::all().into_iter().map(|l| format!("{:<6} → {}", l.to_string(), l.to_human_readable())).collect::>().join("\n ")))] - #[arg(short, long, default_values_t = vec![crate::utils::locale::system_locale(), Locale::ja_JP])] - locale: Vec, - #[arg(help = format!("Subtitle languages. Can be used multiple times. \ - Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] - #[arg(long_help = format!("Subtitle languages. Can be used multiple times. \ - Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] - #[arg(short, long, default_values_t = Locale::all())] - subtitle: Vec, - - #[arg(help = "Name of the output file")] - #[arg(long_help = "Name of the output file.\ - If you use one of the following pattern they will get replaced:\n \ - {title} → Title of the video\n \ - {series_name} → Name of the series\n \ - {season_name} → Name of the season\n \ - {audio} → Audio language of the video\n \ - {resolution} → Resolution of the video\n \ - {season_number} → Number of the season\n \ - {episode_number} → Number of the episode\n \ - {relative_episode_number} → Number of the episode relative to its season\ - {series_id} → ID of the series\n \ - {season_id} → ID of the season\n \ - {episode_id} → ID of the episode")] - #[arg(short, long, default_value = "{title}.mkv")] - output: String, - - #[arg(help = "Video resolution")] - #[arg(long_help = "The video resolution.\ - Can either be specified via the pixels (e.g. 1920x1080), the abbreviation for pixels (e.g. 1080p) or 'common-use' words (e.g. best). \ - Specifying the exact pixels is not recommended, use one of the other options instead. \ - Crunchyroll let you choose the quality with pixel abbreviation on their clients, so you might be already familiar with the available options. \ - The available common-use words are 'best' (choose the best resolution available) and 'worst' (worst resolution available)")] - #[arg(short, long, default_value = "best")] - #[arg(value_parser = crate::utils::clap::clap_parse_resolution)] - resolution: Resolution, - - #[arg( - help = "Sets the behavior of the stream merging. Valid behaviors are 'auto', 'audio' and 'video'" - )] - #[arg( - long_help = "Because of local restrictions (or other reasons) some episodes with different languages does not have the same length (e.g. when some scenes were cut out). \ - With this flag you can set the behavior when handling multiple language. - Valid options are 'audio' (stores one video and all other languages as audio only), 'video' (stores the video + audio for every language) and 'auto' (detects if videos differ in length: if so, behave like 'video' else like 'audio')" - )] - #[arg(short, long, default_value = "auto")] - #[arg(value_parser = MergeBehavior::parse)] - merge: MergeBehavior, - - #[arg(help = format!("Presets for video converting. Can be used multiple times. \ - Available presets: \n {}", FFmpegPreset::available_matches_human_readable().join("\n ")))] - #[arg(long_help = format!("Presets for video converting. Can be used multiple times. \ - Generally used to minify the file size with keeping (nearly) the same quality. \ - It is recommended to only use this if you archive videos with high resolutions since low resolution videos tend to result in a larger file with any of the provided presets. \ - Available presets: \n {}", FFmpegPreset::available_matches_human_readable().join("\n ")))] - #[arg(long)] - #[arg(value_parser = FFmpegPreset::parse)] - ffmpeg_preset: Option, - - #[arg( - help = "Set which subtitle language should be set as default / auto shown when starting a video" - )] - #[arg(long)] - default_subtitle: Option, - - #[arg(help = "Skip files which are already existing")] - #[arg(long, default_value_t = false)] - skip_existing: bool, - - #[arg(help = "Ignore interactive input")] - #[arg(short, long, default_value_t = false)] - yes: bool, - - #[arg(help = "Crunchyroll series url(s)")] - urls: Vec, -} - -#[async_trait::async_trait(?Send)] -impl Execute for Archive { - fn pre_check(&mut self) -> Result<()> { - if !has_ffmpeg() { - bail!("FFmpeg is needed to run this command") - } else if PathBuf::from(&self.output) - .extension() - .unwrap_or_default() - .to_string_lossy() - != "mkv" - && !is_special_file(PathBuf::from(&self.output)) - { - bail!("File extension is not '.mkv'. Currently only matroska / '.mkv' files are supported") - } - - self.locale = all_locale_in_locales(self.locale.clone()); - self.subtitle = all_locale_in_locales(self.subtitle.clone()); - - Ok(()) - } - - async fn execute(self, ctx: Context) -> Result<()> { - let mut parsed_urls = vec![]; - - for (i, url) in self.urls.iter().enumerate() { - let progress_handler = progress!("Parsing url {}", i + 1); - match parse_url(&ctx.crunchy, url.clone(), true).await { - Ok((media_collection, url_filter)) => { - parsed_urls.push((media_collection, url_filter)); - progress_handler.stop(format!("Parsed url {}", i + 1)) - } - Err(e) => bail!("url {} could not be parsed: {}", url, e), - } - } - - for (i, (media_collection, url_filter)) in parsed_urls.into_iter().enumerate() { - let progress_handler = progress!("Fetching series details"); - let archive_formats = match media_collection { - MediaCollection::Series(series) => { - formats_from_series(&self, series, &url_filter).await? - } - MediaCollection::Season(_) => bail!("Archiving a season is not supported"), - MediaCollection::Episode(episode) => bail!("Archiving a episode is not supported. Use url filtering instead to specify the episode (https://www.crunchyroll.com/series/{}/{}[S{}E{}])", episode.metadata.series_id, episode.metadata.series_slug_title, episode.metadata.season_number, episode.metadata.episode_number), - MediaCollection::MovieListing(_) => bail!("Archiving a movie listing is not supported"), - MediaCollection::Movie(_) => bail!("Archiving a movie is not supported") - }; - - if archive_formats.is_empty() { - progress_handler.stop(format!( - "Skipping url {} (no matching episodes found)", - i + 1 - )); - continue; - } - progress_handler.stop(format!("Loaded series information for url {}", i + 1)); - - if log::max_level() == log::Level::Debug { - let seasons = sort_formats_after_seasons( - archive_formats - .clone() - .into_iter() - .map(|(a, _)| a.get(0).unwrap().clone()) - .collect(), - ); - debug!("Series has {} seasons", seasons.len()); - for (i, season) in seasons.into_iter().enumerate() { - info!("Season {} ({})", i + 1, season.get(0).unwrap().season_title); - for format in season { - info!( - "{}: {}px, {:.02} FPS (S{:02}E{:02})", - format.title, - format.stream.resolution, - format.stream.fps, - format.season_number, - format.episode_number, - ) - } - } - } else { - for season in sort_formats_after_seasons( - archive_formats - .clone() - .into_iter() - .map(|(a, _)| a.get(0).unwrap().clone()) - .collect(), - ) { - let first = season.get(0).unwrap(); - info!( - "{} Season {} ({})", - first.series_name, first.season_number, first.season_title - ); - - for (i, format) in season.into_iter().enumerate() { - tab_info!( - "{}. {} » {}px, {:.2} FPS (S{:02}E{:02})", - i + 1, - format.title, - format.stream.resolution, - format.stream.fps, - format.season_number, - format.episode_number - ) - } - } - } - - for (formats, mut subtitles) in archive_formats { - let (primary, additionally) = formats.split_first().unwrap(); - - let formatted_path = primary.format_path((&self.output).into(), true); - let (path, changed) = free_file(formatted_path.clone()); - - if changed && self.skip_existing { - debug!( - "Skipping already existing file '{}'", - formatted_path.to_string_lossy() - ); - continue; - } - - info!( - "Downloading {} to '{}'", - primary.title, - if is_special_file(&path) { - path.to_str().unwrap() - } else { - path.file_name().unwrap().to_str().unwrap() - } - ); - tab_info!( - "Episode: S{:02}E{:02}", - primary.season_number, - primary.episode_number - ); - tab_info!( - "Audio: {} (primary), {}", - primary.audio, - additionally - .iter() - .map(|a| a.audio.to_string()) - .collect::>() - .join(", ") - ); - tab_info!( - "Subtitle: {}", - subtitles - .iter() - .filter(|s| s.primary) // Don't print subtitles of non-primary streams. They might get removed depending on the merge behavior. - .map(|s| { - if let Some(default) = &self.default_subtitle { - if default == &s.stream_subtitle.locale { - return format!("{} (primary)", default); - } - } - s.stream_subtitle.locale.to_string() - }) - .collect::>() - .join(", ") - ); - tab_info!("Resolution: {}", primary.stream.resolution); - tab_info!("FPS: {:.2}", primary.stream.fps); - - let mut video_paths = vec![]; - let mut audio_paths = vec![]; - let mut subtitle_paths = vec![]; - - video_paths.push((download_video(&ctx, primary, false).await?, primary)); - for additional in additionally { - let identical_video = additionally - .iter() - .all(|a| a.stream.bandwidth == primary.stream.bandwidth); - let only_audio = match self.merge { - MergeBehavior::Auto => identical_video, - MergeBehavior::Audio => true, - MergeBehavior::Video => false, - }; - let path = download_video(&ctx, additional, only_audio).await?; - if only_audio { - audio_paths.push((path, additional)) - } else { - video_paths.push((path, additional)) - } - - // Remove subtitles of forcibly deleted video - if matches!(self.merge, MergeBehavior::Audio) && !identical_video { - subtitles.retain(|s| s.episode_id != additional.episode_id); - } - } - - let (primary_video, _) = video_paths.get(0).unwrap(); - let primary_video_length = get_video_length(primary_video.to_path_buf()).unwrap(); - for subtitle in subtitles { - subtitle_paths.push(( - download_subtitle(subtitle.stream_subtitle.clone(), primary_video_length) - .await?, - subtitle, - )) - } - - let progess_handler = progress!("Generating mkv"); - generate_mkv(&self, path, video_paths, audio_paths, subtitle_paths)?; - progess_handler.stop("Mkv generated") - } - } - - Ok(()) - } -} - -async fn formats_from_series( - archive: &Archive, - series: Media, - url_filter: &UrlFilter, -) -> Result, Vec)>> { - let mut seasons = series.seasons().await?; - - // filter any season out which does not contain the specified audio languages - for season in sort_seasons_after_number(seasons.clone()) { - // get all locales which are specified but not present in the current iterated season and - // print an error saying this - let not_present_audio = archive - .locale - .clone() - .into_iter() - .filter(|l| !season.iter().any(|s| s.metadata.audio_locales.contains(l))) - .collect::>(); - if !not_present_audio.is_empty() { - error!( - "Season {} of series {} is not available with {} audio", - season.first().unwrap().metadata.season_number, - series.title, - not_present_audio - .into_iter() - .map(|l| l.to_string()) - .collect::>() - .join(", ") - ) - } - - // remove all seasons with the wrong audio for the current iterated season number - seasons.retain(|s| { - s.metadata.season_number != season.first().unwrap().metadata.season_number - || archive - .locale - .iter() - .any(|l| s.metadata.audio_locales.contains(l)) - }); - // remove seasons which match the url filter. this is mostly done to not trigger the - // interactive season choosing when dupilcated seasons are excluded by the filter - seasons.retain(|s| url_filter.is_season_valid(s.metadata.season_number)) - } - - if !archive.yes && !find_multiple_seasons_with_same_number(&seasons).is_empty() { - info!(target: "progress_end", "Fetched seasons"); - seasons = interactive_season_choosing(seasons); - info!(target: "progress", "Fetching series details") - } - - #[allow(clippy::type_complexity)] - let mut result: Vec<(Vec, Vec)> = Vec::new(); - let mut primary_season = true; - for season in seasons { - let episodes = season.episodes().await?; - - for episode in episodes.iter() { - if !url_filter.is_episode_valid( - episode.metadata.episode_number, - episode.metadata.season_number, - ) { - continue; - } - - let streams = episode.streams().await?; - let streaming_data = streams.hls_streaming_data(None).await?; - let Some(stream) = find_resolution(streaming_data, &archive.resolution) else { - bail!( - "Resolution ({}x{}) is not available for episode {} ({}) of season {} ({}) of {}", - archive.resolution.width, - archive.resolution.height, - episode.metadata.episode_number, - episode.title, - episode.metadata.season_number, - episode.metadata.season_title, - episode.metadata.series_title - ) - }; - - let mut formats: Vec = Vec::new(); - let mut subtitles: Vec = Vec::new(); - subtitles.extend(archive.subtitle.iter().filter_map(|l| { - let stream_subtitle = streams.subtitles.get(l).cloned()?; - let subtitle = Subtitle { - stream_subtitle, - audio_locale: episode.metadata.audio_locale.clone(), - episode_id: episode.id.clone(), - forced: !episode.metadata.is_subbed, - primary: primary_season, - }; - Some(subtitle) - })); - formats.push(Format::new_from_episode(episode, &episodes, stream, vec![])); - - result.push((formats, subtitles)); - } - - primary_season = false; - } - - Ok(result) -} - -async fn download_video(ctx: &Context, format: &Format, only_audio: bool) -> Result { - let tempfile = if only_audio { - tempfile(".aac")? - } else { - tempfile(".ts")? - }; - let (_, path) = tempfile.into_parts(); - - let ffmpeg = Command::new("ffmpeg") - .stdin(Stdio::piped()) - .stdout(Stdio::null()) - .stderr(Stdio::piped()) - .arg("-y") - .args(["-f", "mpegts"]) - .args(["-i", "pipe:"]) - .args(["-c", "copy"]) - .args(if only_audio { vec!["-vn"] } else { vec![] }) - .arg(path.to_str().unwrap()) - .spawn()?; - - download_segments( - ctx, - &mut ffmpeg.stdin.unwrap(), - Some(format!("Download {}", format.audio)), - format.stream.clone(), - ) - .await?; - - Ok(path) -} - -fn generate_mkv( - archive: &Archive, - target: PathBuf, - video_paths: Vec<(TempPath, &Format)>, - audio_paths: Vec<(TempPath, &Format)>, - subtitle_paths: Vec<(TempPath, Subtitle)>, -) -> Result<()> { - let mut input = vec![]; - let mut maps = vec![]; - let mut metadata = vec![]; - let mut dispositions = vec![vec![]; subtitle_paths.len()]; - - for (i, (video_path, format)) in video_paths.iter().enumerate() { - input.extend(["-i".to_string(), video_path.to_string_lossy().to_string()]); - maps.extend(["-map".to_string(), i.to_string()]); - metadata.extend([ - format!("-metadata:s:v:{}", i), - format!("language={}", format.audio), - ]); - metadata.extend([ - format!("-metadata:s:v:{}", i), - format!("title={}", format.audio.to_human_readable()), - ]); - metadata.extend([ - format!("-metadata:s:a:{}", i), - format!("language={}", format.audio), - ]); - metadata.extend([ - format!("-metadata:s:a:{}", i), - format!("title={}", format.audio.to_human_readable()), - ]); - } - for (i, (audio_path, format)) in audio_paths.iter().enumerate() { - input.extend(["-i".to_string(), audio_path.to_string_lossy().to_string()]); - maps.extend(["-map".to_string(), (i + video_paths.len()).to_string()]); - metadata.extend([ - format!("-metadata:s:a:{}", i + video_paths.len()), - format!("language={}", format.audio), - ]); - metadata.extend([ - format!("-metadata:s:a:{}", i + video_paths.len()), - format!("title={}", format.audio.to_human_readable()), - ]); - } - for (i, (subtitle_path, subtitle)) in subtitle_paths.iter().enumerate() { - input.extend([ - "-i".to_string(), - subtitle_path.to_string_lossy().to_string(), - ]); - maps.extend([ - "-map".to_string(), - (i + video_paths.len() + audio_paths.len()).to_string(), - ]); - metadata.extend([ - format!("-metadata:s:s:{}", i), - format!("language={}", subtitle.stream_subtitle.locale), - ]); - metadata.extend([ - format!("-metadata:s:s:{}", i), - format!( - "title={}", - subtitle.stream_subtitle.locale.to_human_readable() - + if !subtitle.primary { - format!(" [Video: {}]", subtitle.audio_locale.to_human_readable()) - } else { - "".to_string() - } - .as_str() - ), - ]); - - // mark forced subtitles - if subtitle.forced { - dispositions[i].push("forced"); - } - } - - let (input_presets, output_presets) = if let Some(preset) = archive.ffmpeg_preset.clone() { - preset.to_input_output_args() - } else { - ( - vec![], - vec![ - "-c:v".to_string(), - "copy".to_string(), - "-c:a".to_string(), - "copy".to_string(), - ], - ) - }; - - let mut command_args = vec!["-y".to_string()]; - command_args.extend(input_presets); - command_args.extend(input); - command_args.extend(maps); - command_args.extend(metadata); - - // set default subtitle - if let Some(default_subtitle) = &archive.default_subtitle { - // if `--default_subtitle ` is given set the default subtitle to the given locale - if let Some(position) = subtitle_paths - .iter() - .position(|(_, subtitle)| &subtitle.stream_subtitle.locale == default_subtitle) - { - dispositions[position].push("default"); - } - } - - let disposition_args: Vec = dispositions - .iter() - .enumerate() - .flat_map(|(i, d)| { - vec![ - format!("-disposition:s:{}", i), - if !d.is_empty() { - d.join("+") - } else { - "0".to_string() - }, - ] - }) - .collect(); - command_args.extend(disposition_args); - - command_args.extend(output_presets); - command_args.extend([ - "-f".to_string(), - "matroska".to_string(), - target.to_string_lossy().to_string(), - ]); - - debug!("ffmpeg {}", command_args.join(" ")); - - // create parent directory if it does not exist - if let Some(parent) = target.parent() { - if !parent.exists() { - std::fs::create_dir_all(parent)? - } - } - - let ffmpeg = Command::new("ffmpeg") - .stdout(Stdio::null()) - .stderr(Stdio::piped()) - .args(command_args) - .output()?; - if !ffmpeg.status.success() { - bail!("{}", String::from_utf8_lossy(ffmpeg.stderr.as_slice())) - } - - Ok(()) -} diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs deleted file mode 100644 index b4c037c..0000000 --- a/crunchy-cli-core/src/cli/download.rs +++ /dev/null @@ -1,603 +0,0 @@ -use crate::cli::log::tab_info; -use crate::cli::utils::{ - download_segments, find_multiple_seasons_with_same_number, find_resolution, - interactive_season_choosing, FFmpegPreset, -}; -use crate::utils::context::Context; -use crate::utils::format::Format; -use crate::utils::log::progress; -use crate::utils::os::{free_file, has_ffmpeg, is_special_file, tempfile}; -use crate::utils::parse::{parse_url, UrlFilter}; -use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number}; -use crate::utils::subtitle::download_subtitle; -use crate::utils::video::get_video_length; -use crate::Execute; -use anyhow::{bail, Result}; -use crunchyroll_rs::media::{Resolution, StreamSubtitle, VariantData}; -use crunchyroll_rs::{ - Episode, Locale, Media, MediaCollection, Movie, MovieListing, Season, Series, -}; -use log::{debug, error, info, warn}; -use std::borrow::Cow; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; - -#[derive(Debug, clap::Parser)] -#[clap(about = "Download a video")] -#[command(arg_required_else_help(true))] -pub struct Download { - #[arg(help = format!("Audio language. Can only be used if the provided url(s) point to a series. \ - Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] - #[arg(long_help = format!("Audio language. Can only be used if the provided url(s) point to a series. \ - Available languages are:\n{}", Locale::all().into_iter().map(|l| format!("{:<6} → {}", l.to_string(), l.to_human_readable())).collect::>().join("\n ")))] - #[arg(short, long, default_value_t = crate::utils::locale::system_locale())] - audio: Locale, - #[arg(help = format!("Subtitle language. Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] - #[arg(long_help = format!("Subtitle language. If set, the subtitle will be burned into the video and cannot be disabled. \ - Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] - #[arg(short, long)] - subtitle: Option, - - #[arg(help = "Name of the output file")] - #[arg(long_help = "Name of the output file.\ - If you use one of the following pattern they will get replaced:\n \ - {title} → Title of the video\n \ - {series_name} → Name of the series\n \ - {season_name} → Name of the season\n \ - {audio} → Audio language of the video\n \ - {resolution} → Resolution of the video\n \ - {season_number} → Number of the season\n \ - {episode_number} → Number of the episode\n \ - {relative_episode_number} → Number of the episode relative to its season\ - {series_id} → ID of the series\n \ - {season_id} → ID of the season\n \ - {episode_id} → ID of the episode")] - #[arg(short, long, default_value = "{title}.mp4")] - output: String, - - #[arg(help = "Video resolution")] - #[arg(long_help = "The video resolution.\ - Can either be specified via the pixels (e.g. 1920x1080), the abbreviation for pixels (e.g. 1080p) or 'common-use' words (e.g. best). \ - Specifying the exact pixels is not recommended, use one of the other options instead. \ - Crunchyroll let you choose the quality with pixel abbreviation on their clients, so you might be already familiar with the available options. \ - The available common-use words are 'best' (choose the best resolution available) and 'worst' (worst resolution available)")] - #[arg(short, long, default_value = "best")] - #[arg(value_parser = crate::utils::clap::clap_parse_resolution)] - resolution: Resolution, - - #[arg(help = format!("Presets for video converting. Can be used multiple times. \ - Available presets: \n {}", FFmpegPreset::available_matches_human_readable().join("\n ")))] - #[arg(long_help = format!("Presets for video converting. Can be used multiple times. \ - Generally used to minify the file size with keeping (nearly) the same quality. \ - It is recommended to only use this if you download videos with high resolutions since low resolution videos tend to result in a larger file with any of the provided presets. \ - Available presets: \n {}", FFmpegPreset::available_matches_human_readable().join("\n ")))] - #[arg(long)] - #[arg(value_parser = FFmpegPreset::parse)] - ffmpeg_preset: Option, - - #[arg(help = "Skip files which are already existing")] - #[arg(long, default_value_t = false)] - skip_existing: bool, - - #[arg(help = "Ignore interactive input")] - #[arg(short, long, default_value_t = false)] - yes: bool, - - #[arg(help = "Url(s) to Crunchyroll episodes or series")] - urls: Vec, -} - -#[async_trait::async_trait(?Send)] -impl Execute for Download { - fn pre_check(&mut self) -> Result<()> { - if !has_ffmpeg() { - bail!("FFmpeg is needed to run this command") - } else if Path::new(&self.output) - .extension() - .unwrap_or_default() - .is_empty() - && self.output != "-" - { - bail!("No file extension found. Please specify a file extension (via `-o`) for the output file") - } - - if self.subtitle.is_some() { - if let Some(ext) = Path::new(&self.output).extension() { - if ext.to_string_lossy() != "mp4" { - warn!("Detected a non mp4 output container. Adding subtitles may take a while") - } - } - } - - Ok(()) - } - - async fn execute(self, ctx: Context) -> Result<()> { - let mut parsed_urls = vec![]; - - for (i, url) in self.urls.iter().enumerate() { - let progress_handler = progress!("Parsing url {}", i + 1); - match parse_url(&ctx.crunchy, url.clone(), true).await { - Ok((media_collection, url_filter)) => { - parsed_urls.push((media_collection, url_filter)); - progress_handler.stop(format!("Parsed url {}", i + 1)) - } - Err(e) => bail!("url {} could not be parsed: {}", url, e), - } - } - - for (i, (media_collection, url_filter)) in parsed_urls.into_iter().enumerate() { - let progress_handler = progress!("Fetching series details"); - let formats = match media_collection { - MediaCollection::Series(series) => { - debug!("Url {} is series ({})", i + 1, series.title); - formats_from_series(&self, series, &url_filter).await? - } - MediaCollection::Season(season) => { - debug!( - "Url {} is season {} ({})", - i + 1, - season.metadata.season_number, - season.title - ); - formats_from_season(&self, season, &url_filter).await? - } - MediaCollection::Episode(episode) => { - debug!( - "Url {} is episode {} ({}) of season {} ({}) of {}", - i + 1, - episode.metadata.episode_number, - episode.title, - episode.metadata.season_number, - episode.metadata.season_title, - episode.metadata.series_title - ); - format_from_episode(&self, &episode, &url_filter, None, false) - .await? - .map(|fmt| vec![fmt]) - } - MediaCollection::MovieListing(movie_listing) => { - debug!("Url {} is movie listing ({})", i + 1, movie_listing.title); - format_from_movie_listing(&self, movie_listing, &url_filter).await? - } - MediaCollection::Movie(movie) => { - debug!("Url {} is movie ({})", i + 1, movie.title); - format_from_movie(&self, movie, &url_filter) - .await? - .map(|fmt| vec![fmt]) - } - }; - - let Some(formats) = formats else { - progress_handler.stop(format!("Skipping url {} (no matching episodes found)", i + 1)); - continue; - }; - progress_handler.stop(format!("Loaded series information for url {}", i + 1)); - - if log::max_level() == log::Level::Debug { - let seasons = sort_formats_after_seasons(formats.clone()); - debug!("Series has {} seasons", seasons.len()); - for (i, season) in seasons.into_iter().enumerate() { - info!("Season {} ({})", i + 1, season.get(0).unwrap().season_title); - for format in season { - info!( - "{}: {}px, {:.02} FPS (S{:02}E{:02})", - format.title, - format.stream.resolution, - format.stream.fps, - format.season_number, - format.episode_number, - ) - } - } - } else { - for season in sort_formats_after_seasons(formats.clone()) { - let first = season.get(0).unwrap(); - info!( - "{} Season {} ({})", - first.series_name, first.season_number, first.season_title - ); - - for (i, format) in season.into_iter().enumerate() { - tab_info!( - "{}. {} » {}px, {:.2} FPS (S{:02}E{:02})", - i + 1, - format.title, - format.stream.resolution, - format.stream.fps, - format.season_number, - format.episode_number - ) - } - } - } - - for format in formats { - let formatted_path = format.format_path((&self.output).into(), true); - let (path, changed) = free_file(formatted_path.clone()); - - if changed && self.skip_existing { - debug!( - "Skipping already existing file '{}'", - formatted_path.to_string_lossy() - ); - continue; - } - - info!( - "Downloading {} to '{}'", - format.title, - if is_special_file(&path) { - path.to_str().unwrap() - } else { - path.file_name().unwrap().to_str().unwrap() - } - ); - tab_info!( - "Episode: S{:02}E{:02}", - format.season_number, - format.episode_number - ); - tab_info!("Audio: {}", format.audio); - tab_info!( - "Subtitles: {}", - self.subtitle - .clone() - .map_or("None".to_string(), |l| l.to_string()) - ); - tab_info!("Resolution: {}", format.stream.resolution); - tab_info!("FPS: {:.2}", format.stream.fps); - - download_ffmpeg( - &ctx, - &self, - format.stream, - format.subtitles.get(0).cloned(), - path.to_path_buf(), - ) - .await?; - } - } - - Ok(()) - } -} - -async fn download_ffmpeg( - ctx: &Context, - download: &Download, - variant_data: VariantData, - subtitle: Option, - mut target: PathBuf, -) -> Result<()> { - let (input_presets, mut output_presets) = if let Some(preset) = download.ffmpeg_preset.clone() { - preset.to_input_output_args() - } else { - ( - vec![], - vec![ - "-c:v".to_string(), - "copy".to_string(), - "-c:a".to_string(), - "copy".to_string(), - ], - ) - }; - - // create parent directory if it does not exist - if let Some(parent) = target.parent() { - if !parent.exists() { - std::fs::create_dir_all(parent)? - } - } - - let mut video_file = tempfile(".ts")?; - download_segments(ctx, &mut video_file, None, variant_data).await?; - let subtitle_file = if let Some(ref sub) = subtitle { - let video_len = get_video_length(video_file.path().to_path_buf())?; - Some(download_subtitle(sub.clone(), video_len).await?) - } else { - None - }; - - let stdout_tempfile = if target.to_string_lossy() == "-" { - let file = tempfile(".mp4")?; - target = file.path().to_path_buf(); - - Some(file) - } else { - None - }; - - let subtitle_presets = if let Some(sub_file) = &subtitle_file { - if target.extension().unwrap_or_default().to_string_lossy() == "mp4" { - vec![ - "-i".to_string(), - sub_file.to_string_lossy().to_string(), - "-movflags".to_string(), - "faststart".to_string(), - "-c:s".to_string(), - "mov_text".to_string(), - "-disposition:s:s:0".to_string(), - "forced".to_string(), - ] - } else { - // remove '-c:v copy' and '-c:a copy' from output presets as its causes issues with - // burning subs into the video - let mut last = String::new(); - let mut remove_count = 0; - for (i, s) in output_presets.clone().iter().enumerate() { - if (last == "-c:v" || last == "-c:a") && s == "copy" { - // remove last - output_presets.remove(i - remove_count - 1); - remove_count += 1; - output_presets.remove(i - remove_count); - remove_count += 1; - } - last = s.clone(); - } - - vec![ - "-vf".to_string(), - format!("subtitles={}", sub_file.to_string_lossy()), - ] - } - } else { - vec![] - }; - - let mut ffmpeg = Command::new("ffmpeg") - .stdout(Stdio::null()) - .stderr(Stdio::piped()) - .arg("-y") - .args(input_presets) - .args(["-i", video_file.path().to_string_lossy().as_ref()]) - .args(subtitle_presets) - .args(output_presets) - .arg(target.to_str().unwrap()) - .spawn()?; - - let progress_handler = progress!("Generating output file"); - if !ffmpeg.wait()?.success() { - bail!("{}", std::io::read_to_string(ffmpeg.stderr.unwrap())?) - } - progress_handler.stop("Output file generated"); - - if let Some(mut stdout_file) = stdout_tempfile { - let mut stdout = std::io::stdout(); - - std::io::copy(&mut stdout_file, &mut stdout)?; - } - - Ok(()) -} - -async fn formats_from_series( - download: &Download, - series: Media, - url_filter: &UrlFilter, -) -> Result>> { - if !series.metadata.audio_locales.is_empty() - && !series.metadata.audio_locales.contains(&download.audio) - { - error!( - "Series {} is not available with {} audio", - series.title, download.audio - ); - return Ok(None); - } - - let mut seasons = series.seasons().await?; - - // filter any season out which does not contain the specified audio language - for season in sort_seasons_after_number(seasons.clone()) { - // check if the current iterated season has the specified audio language - if !season - .iter() - .any(|s| s.metadata.audio_locales.contains(&download.audio)) - { - error!( - "Season {} of series {} is not available with {} audio", - season.first().unwrap().metadata.season_number, - series.title, - download.audio - ); - } - - // remove all seasons with the wrong audio for the current iterated season number - seasons.retain(|s| { - s.metadata.season_number != season.first().unwrap().metadata.season_number - || s.metadata.audio_locales.contains(&download.audio) - }); - // remove seasons which match the url filter. this is mostly done to not trigger the - // interactive season choosing when dupilcated seasons are excluded by the filter - seasons.retain(|s| url_filter.is_season_valid(s.metadata.season_number)) - } - - if !download.yes && !find_multiple_seasons_with_same_number(&seasons).is_empty() { - info!(target: "progress_end", "Fetched seasons"); - seasons = interactive_season_choosing(seasons); - info!(target: "progress", "Fetching series details") - } - - let mut formats = vec![]; - for season in seasons { - if let Some(fmts) = formats_from_season(download, season, url_filter).await? { - formats.extend(fmts) - } - } - - Ok(some_vec_or_none(formats)) -} - -async fn formats_from_season( - download: &Download, - season: Media, - url_filter: &UrlFilter, -) -> Result>> { - if !url_filter.is_season_valid(season.metadata.season_number) { - return Ok(None); - } else if !season.metadata.audio_locales.contains(&download.audio) { - error!( - "Season {} ({}) is not available with {} audio", - season.metadata.season_number, season.title, download.audio - ); - return Ok(None); - } - - let mut formats = vec![]; - - let episodes = season.episodes().await?; - for episode in episodes.iter() { - if let Some(fmt) = - format_from_episode(download, &episode, url_filter, Some(&episodes), true).await? - { - formats.push(fmt) - } - } - - Ok(some_vec_or_none(formats)) -} - -async fn format_from_episode( - download: &Download, - episode: &Media, - url_filter: &UrlFilter, - season_episodes: Option<&Vec>>, - filter_audio: bool, -) -> Result> { - if filter_audio && episode.metadata.audio_locale != download.audio { - error!( - "Episode {} ({}) of season {} ({}) of {} has no {} audio", - episode.metadata.episode_number, - episode.title, - episode.metadata.season_number, - episode.metadata.season_title, - episode.metadata.series_title, - download.audio - ); - return Ok(None); - } else if !url_filter.is_episode_valid( - episode.metadata.episode_number, - episode.metadata.season_number, - ) { - return Ok(None); - } - - let streams = episode.streams().await?; - let streaming_data = streams.hls_streaming_data(None).await?; - let subtitle = if let Some(subtitle) = &download.subtitle { - if let Some(sub) = streams.subtitles.get(subtitle) { - Some(sub.clone()) - } else { - error!( - "Episode {} ({}) of season {} ({}) of {} has no {} subtitles", - episode.metadata.episode_number, - episode.title, - episode.metadata.season_number, - episode.metadata.season_title, - episode.metadata.series_title, - subtitle - ); - return Ok(None); - } - } else { - None - }; - - let Some(stream) = find_resolution(streaming_data, &download.resolution) else { - bail!( - "Resolution ({}x{}) is not available for episode {} ({}) of season {} ({}) of {}", - download.resolution.width, - download.resolution.height, - episode.metadata.episode_number, - episode.title, - episode.metadata.season_number, - episode.metadata.season_title, - episode.metadata.series_title - ) - }; - - let season_eps = if Format::has_relative_episodes_fmt(&download.output) { - if let Some(eps) = season_episodes { - Cow::from(eps) - } else { - Cow::from(episode.season().await?.episodes().await?) - } - } else { - Cow::from(vec![]) - }; - - Ok(Some(Format::new_from_episode( - episode, - &season_eps.to_vec(), - stream, - subtitle.map_or_else(|| vec![], |s| vec![s]), - ))) -} - -async fn format_from_movie_listing( - download: &Download, - movie_listing: Media, - url_filter: &UrlFilter, -) -> Result>> { - let mut formats = vec![]; - - for movie in movie_listing.movies().await? { - if let Some(fmt) = format_from_movie(download, movie, url_filter).await? { - formats.push(fmt) - } - } - - Ok(some_vec_or_none(formats)) -} - -async fn format_from_movie( - download: &Download, - movie: Media, - _: &UrlFilter, -) -> Result> { - let streams = movie.streams().await?; - let mut streaming_data = if let Some(subtitle) = &download.subtitle { - if !streams.subtitles.keys().cloned().any(|x| &x == subtitle) { - error!("Movie {} has no {} subtitles", movie.title, subtitle); - return Ok(None); - } - streams.hls_streaming_data(Some(subtitle.clone())).await? - } else { - streams.hls_streaming_data(None).await? - }; - - streaming_data.sort_by(|a, b| a.resolution.width.cmp(&b.resolution.width).reverse()); - let stream = { - match download.resolution.height { - u64::MAX => streaming_data.into_iter().next().unwrap(), - u64::MIN => streaming_data.into_iter().last().unwrap(), - _ => { - if let Some(streaming_data) = streaming_data.into_iter().find(|v| { - download.resolution.height == u64::MAX - || v.resolution.height == download.resolution.height - }) { - streaming_data - } else { - bail!( - "Resolution ({}x{}) is not available for movie {}", - download.resolution.width, - download.resolution.height, - movie.title - ) - } - } - } - }; - - Ok(Some(Format::new_from_movie(&movie, stream))) -} - -fn some_vec_or_none(v: Vec) -> Option> { - if v.is_empty() { - None - } else { - Some(v) - } -} diff --git a/crunchy-cli-core/src/cli/log.rs b/crunchy-cli-core/src/cli/log.rs deleted file mode 100644 index 7147c47..0000000 --- a/crunchy-cli-core/src/cli/log.rs +++ /dev/null @@ -1,138 +0,0 @@ -use indicatif::{ProgressBar, ProgressStyle}; -use log::{ - set_boxed_logger, set_max_level, Level, LevelFilter, Log, Metadata, Record, SetLoggerError, -}; -use std::io::{stdout, Write}; -use std::sync::Mutex; -use std::thread; -use std::time::Duration; - -#[allow(clippy::type_complexity)] -pub struct CliLogger { - all: bool, - level: LevelFilter, - progress: Mutex>, -} - -impl Log for CliLogger { - fn enabled(&self, metadata: &Metadata) -> bool { - metadata.level() <= self.level - } - - fn log(&self, record: &Record) { - if !self.enabled(record.metadata()) - || (record.target() != "progress" - && record.target() != "progress_end" - && (!self.all && !record.target().starts_with("crunchy_cli"))) - { - return; - } - - if self.level >= LevelFilter::Debug { - self.extended(record); - return; - } - - match record.target() { - "progress" => self.progress(record, false), - "progress_end" => self.progress(record, true), - _ => { - if self.progress.lock().unwrap().is_some() { - self.progress(record, false) - } else if record.level() > Level::Warn { - self.normal(record) - } else { - self.error(record) - } - } - } - } - - fn flush(&self) { - let _ = stdout().flush(); - } -} - -impl CliLogger { - pub fn new(all: bool, level: LevelFilter) -> Self { - Self { - all, - level, - progress: Mutex::new(None), - } - } - - pub fn init(all: bool, level: LevelFilter) -> Result<(), SetLoggerError> { - set_max_level(level); - set_boxed_logger(Box::new(CliLogger::new(all, level))) - } - - fn extended(&self, record: &Record) { - println!( - "[{}] {} {} ({}) {}", - chrono::Utc::now().format("%Y-%m-%d %H:%M:%S"), - record.level(), - // replace the 'progress' prefix if this function is invoked via 'progress!' - record - .target() - .replacen("crunchy_cli_core", "crunchy_cli", 1) - .replacen("progress_end", "crunchy_cli", 1) - .replacen("progress", "crunchy_cli", 1), - format!("{:?}", thread::current().id()) - .replace("ThreadId(", "") - .replace(')', ""), - record.args() - ) - } - - fn normal(&self, record: &Record) { - println!(":: {}", record.args()) - } - - fn error(&self, record: &Record) { - eprintln!(":: {}", record.args()) - } - - fn progress(&self, record: &Record, stop: bool) { - let mut progress = self.progress.lock().unwrap(); - - let msg = format!("{}", record.args()); - if stop && progress.is_some() { - if msg.is_empty() { - progress.take().unwrap().finish() - } else { - progress.take().unwrap().finish_with_message(msg) - } - } else if let Some(p) = &*progress { - p.println(format!(":: → {}", msg)) - } else { - #[cfg(not(windows))] - let finish_str = "✔"; - #[cfg(windows)] - // windows does not support all unicode characters by default in their consoles, so - // we're using this (square root?) symbol instead. microsoft. - let finish_str = "√"; - - let pb = ProgressBar::new_spinner(); - pb.set_style( - ProgressStyle::with_template(":: {spinner} {msg}") - .unwrap() - .tick_strings(&["—", "\\", "|", "/", finish_str]), - ); - pb.enable_steady_tick(Duration::from_millis(200)); - pb.set_message(msg); - *progress = Some(pb) - } - } -} - -macro_rules! tab_info { - ($($arg:tt)+) => { - if log::max_level() == log::LevelFilter::Debug { - info!($($arg)+) - } else { - info!("\t{}", format!($($arg)+)) - } - } -} -pub(crate) use tab_info; diff --git a/crunchy-cli-core/src/cli/mod.rs b/crunchy-cli-core/src/cli/mod.rs deleted file mode 100644 index c28f8e0..0000000 --- a/crunchy-cli-core/src/cli/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub mod archive; -pub mod download; -pub mod log; -pub mod login; -mod utils; diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs deleted file mode 100644 index 753cb72..0000000 --- a/crunchy-cli-core/src/cli/utils.rs +++ /dev/null @@ -1,693 +0,0 @@ -use crate::utils::context::Context; -use anyhow::{bail, Result}; -use crunchyroll_rs::media::{Resolution, VariantData, VariantSegment}; -use crunchyroll_rs::{Locale, Media, Season}; -use indicatif::{ProgressBar, ProgressFinish, ProgressStyle}; -use lazy_static::lazy_static; -use log::{debug, LevelFilter}; -use regex::Regex; -use std::borrow::{Borrow, BorrowMut}; -use std::collections::BTreeMap; -use std::env; -use std::io::{BufRead, Write}; -use std::str::FromStr; -use std::sync::{mpsc, Arc, Mutex}; -use std::time::Duration; -use tokio::task::JoinSet; - -pub fn find_resolution( - mut streaming_data: Vec, - resolution: &Resolution, -) -> Option { - streaming_data.sort_by(|a, b| a.resolution.width.cmp(&b.resolution.width).reverse()); - match resolution.height { - u64::MAX => Some(streaming_data.into_iter().next().unwrap()), - u64::MIN => Some(streaming_data.into_iter().last().unwrap()), - _ => streaming_data - .into_iter() - .find(|v| resolution.height == u64::MAX || v.resolution.height == resolution.height), - } -} - -pub async fn download_segments( - ctx: &Context, - writer: &mut impl Write, - message: Option, - variant_data: VariantData, -) -> Result<()> { - let segments = variant_data.segments().await?; - let total_segments = segments.len(); - - let client = Arc::new(ctx.crunchy.client()); - let count = Arc::new(Mutex::new(0)); - - let progress = if log::max_level() == LevelFilter::Info { - let estimated_file_size = (variant_data.bandwidth / 8) - * segments - .iter() - .map(|s| s.length.unwrap_or_default().as_secs()) - .sum::(); - - let progress = ProgressBar::new(estimated_file_size) - .with_style( - ProgressStyle::with_template( - ":: {msg}{bytes:>10} {bytes_per_sec:>12} [{wide_bar}] {percent:>3}%", - ) - .unwrap() - .progress_chars("##-"), - ) - .with_message(message.map(|m| m + " ").unwrap_or_default()) - .with_finish(ProgressFinish::Abandon); - Some(progress) - } else { - None - }; - - let cpus = num_cpus::get(); - let mut segs: Vec> = Vec::with_capacity(cpus); - for _ in 0..cpus { - segs.push(vec![]) - } - for (i, segment) in segments.clone().into_iter().enumerate() { - segs[i - ((i / cpus) * cpus)].push(segment); - } - - let (sender, receiver) = mpsc::channel(); - - let mut join_set: JoinSet> = JoinSet::new(); - for num in 0..cpus { - let thread_client = client.clone(); - let thread_sender = sender.clone(); - let thread_segments = segs.remove(0); - let thread_count = count.clone(); - join_set.spawn(async move { - let after_download_sender = thread_sender.clone(); - - // the download process is encapsulated in its own function. this is done to easily - // catch errors which get returned with `...?` and `bail!(...)` and that the thread - // itself can report that an error has occured - let download = || async move { - for (i, segment) in thread_segments.into_iter().enumerate() { - let mut retry_count = 0; - let mut buf = loop { - let response = thread_client - .get(&segment.url) - .timeout(Duration::from_secs(60)) - .send() - .await?; - - match response.bytes().await { - Ok(b) => break b.to_vec(), - Err(e) => { - if e.is_body() { - if retry_count == 5 { - bail!("Max retry count reached ({}), multiple errors occured while receiving segment {}: {}", retry_count, num + (i * cpus), e) - } - debug!("Failed to download segment {} ({}). Retrying, {} out of 5 retries left", num + (i * cpus), e, 5 - retry_count) - } else { - bail!("{}", e) - } - } - } - - retry_count += 1; - }; - - buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec(); - - let mut c = thread_count.lock().unwrap(); - debug!( - "Downloaded and decrypted segment [{}/{} {:.2}%] {}", - num + (i * cpus), - total_segments, - ((*c + 1) as f64 / total_segments as f64) * 100f64, - segment.url - ); - - thread_sender.send((num as i32 + (i * cpus) as i32, buf))?; - - *c += 1; - } - Ok(()) - }; - - - let result = download().await; - if result.is_err() { - after_download_sender.send((-1 as i32, vec![]))?; - } - - result - }); - } - // drop the sender already here so it does not outlive all (download) threads which are the only - // real consumers of it - drop(sender); - - // this is the main loop which writes the data. it uses a BTreeMap as a buffer as the write - // happens synchronized. the download consist of multiple segments. the map keys are representing - // the segment number and the values the corresponding bytes - let mut data_pos = 0; - let mut buf: BTreeMap> = BTreeMap::new(); - for (pos, bytes) in receiver.iter() { - // if the position is lower than 0, an error occured in the sending download thread - if pos < 0 { - break; - } - - if let Some(p) = &progress { - let progress_len = p.length().unwrap(); - let estimated_segment_len = (variant_data.bandwidth / 8) - * segments - .get(pos as usize) - .unwrap() - .length - .unwrap_or_default() - .as_secs(); - let bytes_len = bytes.len() as u64; - - p.set_length(progress_len - estimated_segment_len + bytes_len); - p.inc(bytes_len) - } - - // check if the currently sent bytes are the next in the buffer. if so, write them directly - // to the target without first adding them to the buffer. - // if not, add them to the buffer - if data_pos == pos { - writer.write_all(bytes.borrow())?; - data_pos += 1; - } else { - buf.insert(pos, bytes); - } - // check if the buffer contains the next segment(s) - while let Some(b) = buf.remove(&data_pos) { - writer.write_all(b.borrow())?; - data_pos += 1; - } - } - - // if any error has occured while downloading it gets returned here - while let Some(joined) = join_set.join_next().await { - joined?? - } - - // write the remaining buffer, if existent - while let Some(b) = buf.remove(&data_pos) { - writer.write_all(b.borrow())?; - data_pos += 1; - } - - if !buf.is_empty() { - bail!( - "Download buffer is not empty. Remaining segments: {}", - buf.into_keys() - .map(|k| k.to_string()) - .collect::>() - .join(", ") - ) - } - - Ok(()) -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum FFmpegPreset { - Predefined(FFmpegCodec, Option, FFmpegQuality), - Custom(Option, Option), -} - -lazy_static! { - static ref PREDEFINED_PRESET: Regex = Regex::new(r"^\w+(-\w+)*?$").unwrap(); -} - -macro_rules! FFmpegEnum { - (enum $name:ident { $($field:ident),* }) => { - #[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] - pub enum $name { - $( - $field - ),*, - } - - impl $name { - fn all() -> Vec<$name> { - vec![ - $( - $name::$field - ),*, - ] - } - } - - impl ToString for $name { - fn to_string(&self) -> String { - match self { - $( - &$name::$field => stringify!($field).to_string().to_lowercase() - ),* - } - } - } - - impl FromStr for $name { - type Err = anyhow::Error; - - fn from_str(s: &str) -> std::result::Result { - match s { - $( - stringify!($field) => Ok($name::$field) - ),*, - _ => bail!("{} is not a valid {}", s, stringify!($name).to_lowercase()) - } - } - } - } -} - -FFmpegEnum! { - enum FFmpegCodec { - H264, - H265, - Av1 - } -} - -FFmpegEnum! { - enum FFmpegHwAccel { - Nvidia - } -} - -FFmpegEnum! { - enum FFmpegQuality { - Lossless, - Normal, - Low - } -} - -impl FFmpegPreset { - pub(crate) fn available_matches( - ) -> Vec<(FFmpegCodec, Option, Option)> { - let codecs = vec![ - ( - FFmpegCodec::H264, - FFmpegHwAccel::all(), - FFmpegQuality::all(), - ), - ( - FFmpegCodec::H265, - FFmpegHwAccel::all(), - FFmpegQuality::all(), - ), - (FFmpegCodec::Av1, vec![], FFmpegQuality::all()), - ]; - - let mut return_values = vec![]; - - for (codec, hwaccels, qualities) in codecs { - return_values.push((codec.clone(), None, None)); - for hwaccel in hwaccels.clone() { - return_values.push((codec.clone(), Some(hwaccel), None)); - } - for quality in qualities.clone() { - return_values.push((codec.clone(), None, Some(quality))) - } - for hwaccel in hwaccels { - for quality in qualities.clone() { - return_values.push((codec.clone(), Some(hwaccel.clone()), Some(quality))) - } - } - } - - return_values - } - - pub(crate) fn available_matches_human_readable() -> Vec { - let mut return_values = vec![]; - - for (codec, hwaccel, quality) in FFmpegPreset::available_matches() { - let mut description_details = vec![]; - if let Some(h) = &hwaccel { - description_details.push(format!("{} hardware acceleration", h.to_string())) - } - if let Some(q) = &quality { - description_details.push(format!("{} video quality/compression", q.to_string())) - } - - let description = if description_details.len() == 0 { - format!( - "{} encoded with default video quality/compression", - codec.to_string() - ) - } else if description_details.len() == 1 { - format!( - "{} encoded with {}", - codec.to_string(), - description_details[0] - ) - } else { - let first = description_details.remove(0); - let last = description_details.remove(description_details.len() - 1); - let mid = if !description_details.is_empty() { - format!(", {} ", description_details.join(", ")) - } else { - "".to_string() - }; - - format!( - "{} encoded with {}{} and {}", - codec.to_string(), - first, - mid, - last - ) - }; - - return_values.push(format!( - "{} ({})", - vec![ - Some(codec.to_string()), - hwaccel.map(|h| h.to_string()), - quality.map(|q| q.to_string()) - ] - .into_iter() - .flatten() - .collect::>() - .join("-"), - description - )) - } - return_values - } - - pub(crate) fn parse(s: &str) -> Result { - let env_ffmpeg_input_args = env::var("FFMPEG_INPUT_ARGS").ok(); - let env_ffmpeg_output_args = env::var("FFMPEG_OUTPUT_ARGS").ok(); - - if env_ffmpeg_input_args.is_some() || env_ffmpeg_output_args.is_some() { - if let Some(input) = &env_ffmpeg_input_args { - if shlex::split(input).is_none() { - return Err(format!("Failed to parse custom ffmpeg input '{}' (`FFMPEG_INPUT_ARGS` env variable)", input)); - } - } - if let Some(output) = &env_ffmpeg_output_args { - if shlex::split(output).is_none() { - return Err(format!("Failed to parse custom ffmpeg output '{}' (`FFMPEG_INPUT_ARGS` env variable)", output)); - } - } - - return Ok(FFmpegPreset::Custom( - env_ffmpeg_input_args, - env_ffmpeg_output_args, - )); - } else if !PREDEFINED_PRESET.is_match(s) { - return Ok(FFmpegPreset::Custom(None, Some(s.to_string()))); - } - - let mut codec: Option = None; - let mut hwaccel: Option = None; - let mut quality: Option = None; - for token in s.split('-') { - if let Some(c) = FFmpegCodec::all() - .into_iter() - .find(|p| p.to_string() == token.to_lowercase()) - { - if let Some(cc) = codec { - return Err(format!( - "cannot use multiple codecs (found {} and {})", - cc.to_string(), - c.to_string() - )); - } - codec = Some(c) - } else if let Some(h) = FFmpegHwAccel::all() - .into_iter() - .find(|p| p.to_string() == token.to_lowercase()) - { - if let Some(hh) = hwaccel { - return Err(format!( - "cannot use multiple hardware accelerations (found {} and {})", - hh.to_string(), - h.to_string() - )); - } - hwaccel = Some(h) - } else if let Some(q) = FFmpegQuality::all() - .into_iter() - .find(|p| p.to_string() == token.to_lowercase()) - { - if let Some(qq) = quality { - return Err(format!( - "cannot use multiple ffmpeg preset qualities (found {} and {})", - qq.to_string(), - q.to_string() - )); - } - quality = Some(q) - } else { - return Err(format!( - "'{}' is not a valid ffmpeg preset (unknown token '{}'", - s, token - )); - } - } - - if let Some(c) = codec { - if !FFmpegPreset::available_matches().contains(&( - c.clone(), - hwaccel.clone(), - quality.clone(), - )) { - return Err(format!("ffmpeg preset is not supported")); - } - Ok(FFmpegPreset::Predefined( - c, - hwaccel, - quality.unwrap_or(FFmpegQuality::Normal), - )) - } else { - Err(format!("cannot use ffmpeg preset with without a codec")) - } - } - - pub(crate) fn to_input_output_args(self) -> (Vec, Vec) { - match self { - FFmpegPreset::Custom(input, output) => ( - input.map_or(vec![], |i| shlex::split(&i).unwrap_or_default()), - output.map_or(vec![], |o| shlex::split(&o).unwrap_or_default()), - ), - FFmpegPreset::Predefined(codec, hwaccel_opt, quality) => { - let mut input = vec![]; - let mut output = vec![]; - - match codec { - FFmpegCodec::H264 => { - if let Some(hwaccel) = hwaccel_opt { - match hwaccel { - FFmpegHwAccel::Nvidia => { - input.extend(["-hwaccel", "cuvid", "-c:v", "h264_cuvid"]); - output.extend(["-c:v", "h264_nvenc", "-c:a", "copy"]) - } - } - } else { - output.extend(["-c:v", "libx264", "-c:a", "copy"]) - } - - match quality { - FFmpegQuality::Lossless => output.extend(["-crf", "18"]), - FFmpegQuality::Normal => (), - FFmpegQuality::Low => output.extend(["-crf", "35"]), - } - } - FFmpegCodec::H265 => { - if let Some(hwaccel) = hwaccel_opt { - match hwaccel { - FFmpegHwAccel::Nvidia => { - input.extend(["-hwaccel", "cuvid", "-c:v", "h264_cuvid"]); - output.extend(["-c:v", "hevc_nvenc", "-c:a", "copy"]) - } - } - } else { - output.extend(["-c:v", "libx265", "-c:a", "copy"]) - } - - match quality { - FFmpegQuality::Lossless => output.extend(["-crf", "20"]), - FFmpegQuality::Normal => (), - FFmpegQuality::Low => output.extend(["-crf", "35"]), - } - } - FFmpegCodec::Av1 => { - output.extend(["-c:v", "libsvtav1", "-c:a", "copy"]); - - match quality { - FFmpegQuality::Lossless => output.extend(["-crf", "22"]), - FFmpegQuality::Normal => (), - FFmpegQuality::Low => output.extend(["-crf", "35"]), - } - } - } - - ( - input - .into_iter() - .map(|s| s.to_string()) - .collect::>(), - output - .into_iter() - .map(|s| s.to_string()) - .collect::>(), - ) - } - } - } -} - -lazy_static! { - static ref DUPLICATED_SEASONS_MULTILANG_REGEX: Regex = Regex::new(r"(-arabic|-castilian|-english|-english-in|-french|-german|-hindi|-italian|-portuguese|-russian|-spanish)$").unwrap(); -} - -pub(crate) fn find_multiple_seasons_with_same_number(seasons: &Vec>) -> Vec { - let mut seasons_map: BTreeMap = BTreeMap::new(); - for season in seasons { - if let Some(s) = seasons_map.get_mut(&season.metadata.season_number) { - *s += 1; - } else { - seasons_map.insert(season.metadata.season_number, 1); - } - } - - seasons_map - .into_iter() - .filter_map(|(k, v)| { - if v > 1 { - // check if the different seasons are actual the same but with different dub languages - let mut multilang_season_vec: Vec = seasons - .iter() - .map(|s| { - DUPLICATED_SEASONS_MULTILANG_REGEX - .replace(s.slug_title.trim_end_matches("-dub"), "") - .to_string() - }) - .collect(); - multilang_season_vec.dedup(); - - if multilang_season_vec.len() > 1 { - return Some(k); - } - } - None - }) - .collect() -} - -/// Check if [`Locale::Custom("all")`] is in the provided locale list and return [`Locale::all`] if -/// so. If not, just return the provided locale list. -pub(crate) fn all_locale_in_locales(locales: Vec) -> Vec { - if locales - .iter() - .find(|l| l.to_string().to_lowercase().trim() == "all") - .is_some() - { - Locale::all() - } else { - locales - } -} - -pub(crate) fn interactive_season_choosing(seasons: Vec>) -> Vec> { - let input_regex = - Regex::new(r"((?P\d+)|(?P\d+)-(?P\d+)?)(\s|$)").unwrap(); - - let mut seasons_map: BTreeMap>> = BTreeMap::new(); - for season in seasons { - if let Some(s) = seasons_map.get_mut(&season.metadata.season_number) { - s.push(season); - } else { - seasons_map.insert(season.metadata.season_number, vec![season]); - } - } - - for (num, season_vec) in seasons_map.iter_mut() { - if season_vec.len() == 1 { - continue; - } - - // check if the different seasons are actual the same but with different dub languages - let mut multilang_season_vec: Vec = season_vec - .iter() - .map(|s| { - DUPLICATED_SEASONS_MULTILANG_REGEX - .replace(s.slug_title.trim_end_matches("-dub"), "") - .to_string() - }) - .collect(); - multilang_season_vec.dedup(); - - if multilang_season_vec.len() == 1 { - continue; - } - - println!(":: Found multiple seasons for season number {}", num); - println!(":: Select the number of the seasons you want to download (eg \"1 2 4\", \"1-3\", \"1-3 5\"):"); - for (i, season) in season_vec.iter().enumerate() { - println!(":: \t{}. {}", i + 1, season.title) - } - let mut stdout = std::io::stdout(); - let _ = write!(stdout, ":: => "); - let _ = stdout.flush(); - let mut user_input = String::new(); - std::io::stdin() - .lock() - .read_line(&mut user_input) - .expect("cannot open stdin"); - - let mut nums = vec![]; - for capture in input_regex.captures_iter(&user_input) { - if let Some(single) = capture.name("single") { - nums.push(single.as_str().parse::().unwrap() - 1); - } else { - let range_from = capture.name("range_from"); - let range_to = capture.name("range_to"); - - // input is '-' which means use all seasons - if range_from.is_none() && range_to.is_none() { - nums = vec![]; - break; - } - let from = range_from - .map(|f| f.as_str().parse::().unwrap() - 1) - .unwrap_or(usize::MIN); - let to = range_from - .map(|f| f.as_str().parse::().unwrap() - 1) - .unwrap_or(usize::MAX); - - nums.extend( - season_vec - .iter() - .enumerate() - .filter_map(|(i, _)| if i >= from && i <= to { Some(i) } else { None }) - .collect::>(), - ) - } - } - nums.dedup(); - - if !nums.is_empty() { - let mut remove_count = 0; - for i in 0..season_vec.len() - 1 { - if !nums.contains(&i) { - season_vec.remove(i - remove_count); - remove_count += 1 - } - } - } - } - - seasons_map - .into_values() - .into_iter() - .flatten() - .collect::>>() -} diff --git a/crunchy-cli-core/src/download/command.rs b/crunchy-cli-core/src/download/command.rs new file mode 100644 index 0000000..25db55b --- /dev/null +++ b/crunchy-cli-core/src/download/command.rs @@ -0,0 +1,151 @@ +use crate::download::filter::DownloadFilter; +use crate::utils::context::Context; +use crate::utils::ffmpeg::FFmpegPreset; +use crate::utils::filter::Filter; +use crate::utils::format::formats_visual_output; +use crate::utils::log::progress; +use crate::utils::os::{free_file, has_ffmpeg}; +use crate::utils::parse::parse_url; +use crate::Execute; +use anyhow::bail; +use anyhow::Result; +use crunchyroll_rs::media::Resolution; +use crunchyroll_rs::Locale; +use log::{debug, warn}; +use std::path::Path; + +#[derive(Clone, Debug, clap::Parser)] +#[clap(about = "Download a video")] +#[command(arg_required_else_help(true))] +pub struct Download { + #[arg(help = format!("Audio language. Can only be used if the provided url(s) point to a series. \ + Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] + #[arg(long_help = format!("Audio language. Can only be used if the provided url(s) point to a series. \ + Available languages are:\n{}", Locale::all().into_iter().map(|l| format!("{:<6} → {}", l.to_string(), l.to_human_readable())).collect::>().join("\n ")))] + #[arg(short, long, default_value_t = crate::utils::locale::system_locale())] + pub(crate) audio: Locale, + #[arg(help = format!("Subtitle language. Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] + #[arg(long_help = format!("Subtitle language. If set, the subtitle will be burned into the video and cannot be disabled. \ + Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] + #[arg(short, long)] + pub(crate) subtitle: Option, + + #[arg(help = "Name of the output file")] + #[arg(long_help = "Name of the output file.\ + If you use one of the following pattern they will get replaced:\n \ + {title} → Title of the video\n \ + {series_name} → Name of the series\n \ + {season_name} → Name of the season\n \ + {audio} → Audio language of the video\n \ + {resolution} → Resolution of the video\n \ + {season_number} → Number of the season\n \ + {episode_number} → Number of the episode\n \ + {relative_episode_number} → Number of the episode relative to its season\ + {series_id} → ID of the series\n \ + {season_id} → ID of the season\n \ + {episode_id} → ID of the episode")] + #[arg(short, long, default_value = "{title}.mp4")] + pub(crate) output: String, + + #[arg(help = "Video resolution")] + #[arg(long_help = "The video resolution.\ + Can either be specified via the pixels (e.g. 1920x1080), the abbreviation for pixels (e.g. 1080p) or 'common-use' words (e.g. best). \ + Specifying the exact pixels is not recommended, use one of the other options instead. \ + Crunchyroll let you choose the quality with pixel abbreviation on their clients, so you might be already familiar with the available options. \ + The available common-use words are 'best' (choose the best resolution available) and 'worst' (worst resolution available)")] + #[arg(short, long, default_value = "best")] + #[arg(value_parser = crate::utils::clap::clap_parse_resolution)] + pub(crate) resolution: Resolution, + + #[arg(help = format!("Presets for video converting. Can be used multiple times. \ + Available presets: \n {}", FFmpegPreset::available_matches_human_readable().join("\n ")))] + #[arg(long_help = format!("Presets for video converting. Can be used multiple times. \ + Generally used to minify the file size with keeping (nearly) the same quality. \ + It is recommended to only use this if you download videos with high resolutions since low resolution videos tend to result in a larger file with any of the provided presets. \ + Available presets: \n {}", FFmpegPreset::available_matches_human_readable().join("\n ")))] + #[arg(long)] + #[arg(value_parser = FFmpegPreset::parse)] + pub(crate) ffmpeg_preset: Option, + + #[arg(help = "Skip files which are already existing")] + #[arg(long, default_value_t = false)] + pub(crate) skip_existing: bool, + + #[arg(help = "Url(s) to Crunchyroll episodes or series")] + pub(crate) urls: Vec, +} + +#[async_trait::async_trait(?Send)] +impl Execute for Download { + fn pre_check(&mut self) -> Result<()> { + if !has_ffmpeg() { + bail!("FFmpeg is needed to run this command") + } else if Path::new(&self.output) + .extension() + .unwrap_or_default() + .is_empty() + && self.output != "-" + { + bail!("No file extension found. Please specify a file extension (via `-o`) for the output file") + } + + if self.subtitle.is_some() { + if let Some(ext) = Path::new(&self.output).extension() { + if ext.to_string_lossy() != "mp4" { + warn!("Detected a non mp4 output container. Adding subtitles may take a while") + } + } + } + + Ok(()) + } + + async fn execute(self, ctx: Context) -> Result<()> { + let mut parsed_urls = vec![]; + + for (i, url) in self.urls.clone().into_iter().enumerate() { + let progress_handler = progress!("Parsing url {}", i + 1); + match parse_url(&ctx.crunchy, url.clone(), true).await { + Ok((media_collection, url_filter)) => { + progress_handler.stop(format!("Parsed url {}", i + 1)); + parsed_urls.push((media_collection, url_filter)) + } + Err(e) => bail!("url {} could not be parsed: {}", url, e), + }; + } + + for (i, (media_collection, url_filter)) in parsed_urls.into_iter().enumerate() { + let progress_handler = progress!("Fetching series details"); + let download_formats = DownloadFilter::new(url_filter, self.clone()) + .visit(media_collection) + .await?; + + if download_formats.is_empty() { + progress_handler.stop(format!("Skipping url {} (no matching videos found)", i + 1)); + continue; + } + progress_handler.stop(format!("Loaded series information for url {}", i + 1)); + + formats_visual_output(download_formats.iter().map(|(_, f)| f).collect()); + + for (downloader, format) in download_formats { + let formatted_path = format.format_path((&self.output).into(), true); + let (path, changed) = free_file(formatted_path.clone()); + + if changed && self.skip_existing { + debug!( + "Skipping already existing file '{}'", + formatted_path.to_string_lossy() + ); + continue; + } + + format.visual_output(&path); + + downloader.download(&ctx, &path).await? + } + } + + Ok(()) + } +} diff --git a/crunchy-cli-core/src/download/filter.rs b/crunchy-cli-core/src/download/filter.rs new file mode 100644 index 0000000..81b8fa9 --- /dev/null +++ b/crunchy-cli-core/src/download/filter.rs @@ -0,0 +1,349 @@ +use crate::download::Download; +use crate::utils::download::{DownloadBuilder, DownloadFormat, Downloader}; +use crate::utils::filter::Filter; +use crate::utils::format::{Format, SingleFormat}; +use crate::utils::parse::UrlFilter; +use crate::utils::video::variant_data_from_stream; +use anyhow::{bail, Result}; +use crunchyroll_rs::media::{Subtitle, VariantData}; +use crunchyroll_rs::{Concert, Episode, Movie, MovieListing, MusicVideo, Season, Series}; +use log::{error, warn}; +use std::collections::HashMap; + +pub(crate) struct FilterResult { + format: SingleFormat, + video: VariantData, + audio: VariantData, + subtitle: Option, +} + +pub(crate) struct DownloadFilter { + url_filter: UrlFilter, + download: Download, + season_episode_count: HashMap>, + season_subtitles_missing: Vec, +} + +impl DownloadFilter { + pub(crate) fn new(url_filter: UrlFilter, download: Download) -> Self { + Self { + url_filter, + download, + season_episode_count: HashMap::new(), + season_subtitles_missing: vec![], + } + } +} + +#[async_trait::async_trait] +impl Filter for DownloadFilter { + type T = FilterResult; + type Output = (Downloader, Format); + + async fn visit_series(&mut self, series: Series) -> Result> { + // `series.audio_locales` isn't always populated b/c of crunchyrolls api. so check if the + // audio is matching only if the field is populated + if !series.audio_locales.is_empty() { + if !series.audio_locales.contains(&self.download.audio) { + error!( + "Series {} is not available with {} audio", + series.title, self.download.audio + ); + return Ok(vec![]); + } + } + + let seasons = series.seasons().await?; + + Ok(seasons) + } + + async fn visit_season(&mut self, mut season: Season) -> Result> { + if !self.url_filter.is_season_valid(season.season_number) { + return Ok(vec![]); + } + + if !season + .audio_locales + .iter() + .any(|l| l == &self.download.audio) + { + if season + .available_versions() + .await? + .iter() + .any(|l| l == &self.download.audio) + { + season = season + .version(vec![self.download.audio.clone()]) + .await? + .remove(0) + } else { + error!( + "Season {} - '{}' is not available with {} audio", + season.season_number, + season.title, + self.download.audio.clone(), + ); + return Ok(vec![]); + } + } + + let mut episodes = season.episodes().await?; + + if Format::has_relative_episodes_fmt(&self.download.output) { + for episode in episodes.iter() { + self.season_episode_count + .entry(episode.season_number) + .or_insert(vec![]) + .push(episode.id.clone()) + } + } + + episodes.retain(|e| { + self.url_filter + .is_episode_valid(e.episode_number, season.season_number) + }); + + Ok(episodes) + } + + async fn visit_episode(&mut self, mut episode: Episode) -> Result> { + if !self + .url_filter + .is_episode_valid(episode.episode_number, episode.season_number) + { + return Ok(None); + } + + // check if the audio locale is correct. + // should only be incorrect if the console input was a episode url. otherwise + // `DownloadFilter::visit_season` returns the correct episodes with matching audio + if episode.audio_locale != self.download.audio { + // check if any other version (same episode, other language) of this episode is available + // with the requested audio. if not, return an error + if !episode + .available_versions() + .await? + .contains(&self.download.audio) + { + bail!( + "Episode {} ({}) of {} season {} is not available with {} audio", + episode.episode_number, + episode.title, + episode.series_title, + episode.season_number, + self.download.audio + ) + } + // overwrite the current episode with the other version episode + episode = episode + .version(vec![self.download.audio.clone()]) + .await? + .remove(0) + } + + // check if the subtitles are supported + if let Some(subtitle_locale) = &self.download.subtitle { + if !episode.subtitle_locales.contains(subtitle_locale) { + // if the episode doesn't have the requested subtitles, print a error. to print this + // error only once per season, it's checked if an error got printed before by looking + // up if the season id is present in `self.season_subtitles_missing`. if not, print + // the error and add the season id to `self.season_subtitles_missing`. if it is + // present, skip the error printing + if !self + .season_subtitles_missing + .contains(&episode.season_number) + { + self.season_subtitles_missing.push(episode.season_number); + error!( + "{} season {} is not available with {} subtitles", + episode.series_title, episode.season_number, subtitle_locale + ); + } + return Ok(None); + } + } + + // get the correct video stream + let stream = episode.streams().await?; + let (video, audio) = if let Some((video, audio)) = + variant_data_from_stream(&stream, &self.download.resolution).await? + { + (video, audio) + } else { + bail!( + "Resolution ({}) is not available for episode {} ({}) of {} season {}", + self.download.resolution, + episode.episode_number, + episode.title, + episode.series_title, + episode.season_number, + ) + }; + + // it is assumed that the subtitle, if requested, exists b/c the subtitle check above must + // be passed to reach this condition. + // the check isn't done in this if block to reduce unnecessary fetching of the stream + let subtitle = if let Some(subtitle_locale) = &self.download.subtitle { + stream.subtitles.get(subtitle_locale).map(|s| s.clone()) + } else { + None + }; + + // get the relative episode number. only done if the output string has the pattern to include + // the relative episode number as this requires some extra fetching + let relative_episode_number = if Format::has_relative_episodes_fmt(&self.download.output) { + if self + .season_episode_count + .get(&episode.season_number) + .is_none() + { + let season_episodes = episode.season().await?.episodes().await?; + self.season_episode_count.insert( + episode.season_number, + season_episodes.into_iter().map(|e| e.id).collect(), + ); + } + let relative_episode_number = self + .season_episode_count + .get(&episode.season_number) + .unwrap() + .iter() + .position(|id| id == &episode.id); + if relative_episode_number.is_none() { + warn!( + "Failed to get relative episode number for episode {} ({}) of {} season {}", + episode.episode_number, + episode.title, + episode.series_title, + episode.season_number, + ) + } + relative_episode_number + } else { + None + }; + + Ok(Some(FilterResult { + format: SingleFormat::new_from_episode( + &episode, + &video, + subtitle.clone().map_or(vec![], |s| vec![s.locale]), + relative_episode_number.map(|n| n as u32), + ), + video, + audio, + subtitle, + })) + } + + async fn visit_movie_listing(&mut self, movie_listing: MovieListing) -> Result> { + Ok(movie_listing.movies().await?) + } + + async fn visit_movie(&mut self, movie: Movie) -> Result> { + let stream = movie.streams().await?; + let (video, audio) = if let Some((video, audio)) = + variant_data_from_stream(&stream, &self.download.resolution).await? + { + (video, audio) + } else { + bail!( + "Resolution ({}) of movie '{}' is not available", + self.download.resolution, + movie.title + ) + }; + let subtitle = if let Some(subtitle_locale) = &self.download.subtitle { + let Some(subtitle) = stream.subtitles.get(subtitle_locale) else { + error!( + "Movie '{}' has no {} subtitles", + movie.title, + subtitle_locale + ); + return Ok(None) + }; + Some(subtitle.clone()) + } else { + None + }; + + Ok(Some(FilterResult { + format: SingleFormat::new_from_movie( + &movie, + &video, + subtitle.clone().map_or(vec![], |s| vec![s.locale]), + ), + video, + audio, + subtitle, + })) + } + + async fn visit_music_video(&mut self, music_video: MusicVideo) -> Result> { + let stream = music_video.streams().await?; + let (video, audio) = if let Some((video, audio)) = + variant_data_from_stream(&stream, &self.download.resolution).await? + { + (video, audio) + } else { + bail!( + "Resolution ({}) of music video {} is not available", + self.download.resolution, + music_video.title + ) + }; + + Ok(Some(FilterResult { + format: SingleFormat::new_from_music_video(&music_video, &video), + video, + audio, + subtitle: None, + })) + } + + async fn visit_concert(&mut self, concert: Concert) -> Result> { + let stream = concert.streams().await?; + let (video, audio) = if let Some((video, audio)) = + variant_data_from_stream(&stream, &self.download.resolution).await? + { + (video, audio) + } else { + bail!( + "Resolution ({}) of music video {} is not available", + self.download.resolution, + concert.title + ) + }; + + Ok(Some(FilterResult { + format: SingleFormat::new_from_concert(&concert, &video), + video, + audio, + subtitle: None, + })) + } + + async fn finish(self, mut input: Vec) -> Result> { + let mut result = vec![]; + input.sort_by(|a, b| { + a.format + .sequence_number + .total_cmp(&b.format.sequence_number) + }); + for data in input { + let mut downloader = DownloadBuilder::new() + .default_subtitle(self.download.subtitle.clone()) + .build(); + downloader.add_format(DownloadFormat { + video: (data.video, data.format.audio.clone()), + audios: vec![(data.audio, data.format.audio.clone())], + subtitles: data.subtitle.map_or(vec![], |s| vec![s]), + }); + result.push((downloader, Format::from_single_formats(vec![data.format]))) + } + + Ok(result) + } +} diff --git a/crunchy-cli-core/src/download/mod.rs b/crunchy-cli-core/src/download/mod.rs new file mode 100644 index 0000000..696872e --- /dev/null +++ b/crunchy-cli-core/src/download/mod.rs @@ -0,0 +1,4 @@ +mod command; +mod filter; + +pub use command::Download; diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index ab3f33c..5e0e113 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -1,7 +1,6 @@ -use crate::cli::log::CliLogger; use crate::utils::context::Context; use crate::utils::locale::system_locale; -use crate::utils::log::progress; +use crate::utils::log::{progress, CliLogger}; use anyhow::bail; use anyhow::Result; use clap::{Parser, Subcommand}; @@ -9,10 +8,14 @@ use crunchyroll_rs::{Crunchyroll, Locale}; use log::{debug, error, warn, LevelFilter}; use std::{env, fs}; -mod cli; +mod archive; +mod download; +mod login; mod utils; -pub use cli::{archive::Archive, download::Download, login::Login}; +pub use archive::Archive; +pub use download::Download; +pub use login::Login; #[async_trait::async_trait(?Send)] trait Execute { @@ -222,9 +225,14 @@ async fn crunchyroll_session(cli: &Cli) -> Result { lang }; - let builder = Crunchyroll::builder() + let mut builder = Crunchyroll::builder() .locale(locale) - .stabilization_locales(true); + .stabilization_locales(true) + .stabilization_season_number(true); + + if let Command::Download(download) = &cli.command { + builder = builder.preferred_audio_locale(download.audio.clone()) + } let login_methods_count = cli.login_method.credentials.is_some() as u8 + cli.login_method.etp_rt.is_some() as u8 @@ -232,7 +240,7 @@ async fn crunchyroll_session(cli: &Cli) -> Result { let progress_handler = progress!("Logging in"); if login_methods_count == 0 { - if let Some(login_file_path) = cli::login::login_file_path() { + if let Some(login_file_path) = login::login_file_path() { if login_file_path.exists() { let session = fs::read_to_string(login_file_path)?; if let Some((token_type, token)) = session.split_once(':') { diff --git a/crunchy-cli-core/src/cli/login.rs b/crunchy-cli-core/src/login/command.rs similarity index 94% rename from crunchy-cli-core/src/cli/login.rs rename to crunchy-cli-core/src/login/command.rs index c41b1b7..f164a0b 100644 --- a/crunchy-cli-core/src/cli/login.rs +++ b/crunchy-cli-core/src/login/command.rs @@ -18,6 +18,8 @@ pub struct Login { impl Execute for Login { async fn execute(self, ctx: Context) -> Result<()> { if let Some(login_file_path) = login_file_path() { + fs::create_dir_all(login_file_path.parent().unwrap())?; + match ctx.crunchy.session_token().await { SessionToken::RefreshToken(refresh_token) => Ok(fs::write( login_file_path, diff --git a/crunchy-cli-core/src/login/mod.rs b/crunchy-cli-core/src/login/mod.rs new file mode 100644 index 0000000..9025e68 --- /dev/null +++ b/crunchy-cli-core/src/login/mod.rs @@ -0,0 +1,4 @@ +mod command; + +pub use command::login_file_path; +pub use command::Login; diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs new file mode 100644 index 0000000..ce0c443 --- /dev/null +++ b/crunchy-cli-core/src/utils/download.rs @@ -0,0 +1,657 @@ +use crate::utils::context::Context; +use crate::utils::ffmpeg::FFmpegPreset; +use crate::utils::log::progress; +use crate::utils::os::tempfile; +use anyhow::{bail, Result}; +use chrono::NaiveTime; +use crunchyroll_rs::media::{Subtitle, VariantData, VariantSegment}; +use crunchyroll_rs::Locale; +use indicatif::{ProgressBar, ProgressFinish, ProgressStyle}; +use log::{debug, LevelFilter}; +use regex::Regex; +use std::borrow::Borrow; +use std::borrow::BorrowMut; +use std::collections::BTreeMap; +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::sync::{mpsc, Arc, Mutex}; +use std::time::Duration; +use tempfile::TempPath; +use tokio::task::JoinSet; + +#[derive(Clone, Debug)] +pub enum MergeBehavior { + Video, + Audio, + Auto, +} + +impl MergeBehavior { + pub fn parse(s: &str) -> Result { + Ok(match s.to_lowercase().as_str() { + "video" => MergeBehavior::Video, + "audio" => MergeBehavior::Audio, + "auto" => MergeBehavior::Auto, + _ => return Err(format!("'{}' is not a valid merge behavior", s)), + }) + } +} + +#[derive(derive_setters::Setters)] +pub struct DownloadBuilder { + ffmpeg_preset: FFmpegPreset, + default_subtitle: Option, + output_format: Option, + audio_sort: Option>, + subtitle_sort: Option>, +} + +impl DownloadBuilder { + pub fn new() -> DownloadBuilder { + Self { + ffmpeg_preset: FFmpegPreset::default(), + default_subtitle: None, + output_format: None, + audio_sort: None, + subtitle_sort: None, + } + } + + pub fn build(self) -> Downloader { + Downloader { + ffmpeg_preset: self.ffmpeg_preset, + default_subtitle: self.default_subtitle, + output_format: self.output_format, + audio_sort: self.audio_sort, + subtitle_sort: self.subtitle_sort, + + formats: vec![], + } + } +} + +struct FFmpegMeta { + path: TempPath, + language: Locale, + title: String, +} + +pub struct DownloadFormat { + pub video: (VariantData, Locale), + pub audios: Vec<(VariantData, Locale)>, + pub subtitles: Vec, +} + +pub struct Downloader { + ffmpeg_preset: FFmpegPreset, + default_subtitle: Option, + output_format: Option, + audio_sort: Option>, + subtitle_sort: Option>, + + formats: Vec, +} + +impl Downloader { + pub fn add_format(&mut self, format: DownloadFormat) { + self.formats.push(format); + } + + pub async fn download(mut self, ctx: &Context, dst: &Path) -> Result<()> { + if let Some(audio_sort_locales) = &self.audio_sort { + self.formats.sort_by(|a, b| { + audio_sort_locales + .iter() + .position(|l| l == &a.video.1) + .cmp(&audio_sort_locales.iter().position(|l| l == &b.video.1)) + }); + } + for format in self.formats.iter_mut() { + if let Some(audio_sort_locales) = &self.audio_sort { + format.audios.sort_by(|(_, a), (_, b)| { + audio_sort_locales + .iter() + .position(|l| l == a) + .cmp(&audio_sort_locales.iter().position(|l| l == b)) + }) + } + if let Some(subtitle_sort) = &self.subtitle_sort { + format.subtitles.sort_by(|a, b| { + subtitle_sort + .iter() + .position(|l| l == &a.locale) + .cmp(&subtitle_sort.iter().position(|l| l == &b.locale)) + }) + } + } + + let mut videos = vec![]; + let mut audios = vec![]; + let mut subtitles = vec![]; + + for (i, format) in self.formats.iter().enumerate() { + let fmt_space = format + .audios + .iter() + .map(|(_, locale)| format!("Downloading {} audio", locale).len()) + .max() + .unwrap(); + + let video_path = self + .download_video( + ctx, + &format.video.0, + format!("{:<1$}", format!("Downloading video #{}", i + 1), fmt_space), + ) + .await?; + for (variant_data, locale) in format.audios.iter() { + let audio_path = self + .download_audio( + ctx, + variant_data, + format!("{:<1$}", format!("Downloading {} audio", locale), fmt_space), + ) + .await?; + audios.push(FFmpegMeta { + path: audio_path, + language: locale.clone(), + title: if i == 0 { + locale.to_human_readable() + } else { + format!("{} [Video: #{}]", locale.to_human_readable(), i + 1) + }, + }) + } + let len = get_video_length(&video_path)?; + for subtitle in format.subtitles.iter() { + let subtitle_path = self.download_subtitle(subtitle.clone(), len).await?; + subtitles.push(FFmpegMeta { + path: subtitle_path, + language: subtitle.locale.clone(), + title: if i == 0 { + subtitle.locale.to_human_readable() + } else { + format!( + "{} [Video: #{}]", + subtitle.locale.to_human_readable(), + i + 1 + ) + }, + }) + } + videos.push(FFmpegMeta { + path: video_path, + language: format.video.1.clone(), + title: if self.formats.len() == 1 { + "Default".to_string() + } else { + format!("#{}", i + 1) + }, + }); + } + + let mut input = vec![]; + let mut maps = vec![]; + let mut metadata = vec![]; + + for (i, meta) in videos.iter().enumerate() { + input.extend(["-i".to_string(), meta.path.to_string_lossy().to_string()]); + maps.extend(["-map".to_string(), i.to_string()]); + metadata.extend([ + format!("-metadata:s:v:{}", i), + format!("title={}", meta.title), + ]); + // the empty language metadata is created to avoid that metadata from the original track + // is copied + metadata.extend([format!("-metadata:s:v:{}", i), format!("language=")]) + } + for (i, meta) in audios.iter().enumerate() { + input.extend(["-i".to_string(), meta.path.to_string_lossy().to_string()]); + maps.extend(["-map".to_string(), (i + videos.len()).to_string()]); + metadata.extend([ + format!("-metadata:s:a:{}", i), + format!("language={}", meta.language), + ]); + metadata.extend([ + format!("-metadata:s:a:{}", i), + format!("title={}", meta.title), + ]); + } + for (i, meta) in subtitles.iter().enumerate() { + input.extend(["-i".to_string(), meta.path.to_string_lossy().to_string()]); + maps.extend([ + "-map".to_string(), + (i + videos.len() + audios.len()).to_string(), + ]); + metadata.extend([ + format!("-metadata:s:s:{}", i), + format!("language={}", meta.language), + ]); + metadata.extend([ + format!("-metadata:s:s:{}", i), + format!("title={}", meta.title), + ]); + } + + let (input_presets, mut output_presets) = self.ffmpeg_preset.into_input_output_args(); + + let mut command_args = vec!["-y".to_string(), "-hide_banner".to_string()]; + command_args.extend(input_presets); + command_args.extend(input); + command_args.extend(maps); + command_args.extend(metadata); + + // set default subtitle + if let Some(default_subtitle) = self.default_subtitle { + if let Some(position) = subtitles + .iter() + .position(|m| m.language == default_subtitle) + { + match dst.extension().unwrap_or_default().to_str().unwrap() { + "mp4" => output_presets.extend([ + "-movflags".to_string(), + "faststart".to_string(), + "-c:s".to_string(), + "mov_text".to_string(), + format!("-disposition:s:s:{}", position), + "forced".to_string(), + ]), + "mkv" => output_presets.extend([ + format!("-disposition:s:s:{}", position), + "forced".to_string(), + ]), + _ => { + // remove '-c:v copy' and '-c:a copy' from output presets as its causes issues with + // burning subs into the video + let mut last = String::new(); + let mut remove_count = 0; + for (i, s) in output_presets.clone().iter().enumerate() { + if (last == "-c:v" || last == "-c:a") && s == "copy" { + // remove last + output_presets.remove(i - remove_count - 1); + remove_count += 1; + output_presets.remove(i - remove_count); + remove_count += 1; + } + last = s.clone(); + } + + output_presets.extend([ + "-vf".to_string(), + format!( + "subtitles={}", + subtitles.get(position).unwrap().path.to_str().unwrap() + ), + ]) + } + } + } + + if let Some(position) = subtitles + .iter() + .position(|meta| meta.language == default_subtitle) + { + command_args.extend([ + format!("-disposition:s:s:{}", position), + "forced".to_string(), + ]) + } + } + + command_args.extend(output_presets); + if let Some(output_format) = self.output_format { + command_args.extend(["-f".to_string(), output_format]); + } + command_args.push(dst.to_str().unwrap().to_string()); + + debug!("ffmpeg {}", command_args.join(" ")); + + // create parent directory if it does not exist + if let Some(parent) = dst.parent() { + if !parent.exists() { + std::fs::create_dir_all(parent)? + } + } + + let progress_handler = progress!("Generating output file"); + + let ffmpeg = Command::new("ffmpeg") + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .args(command_args) + .output()?; + if !ffmpeg.status.success() { + bail!("{}", String::from_utf8_lossy(ffmpeg.stderr.as_slice())) + } + + progress_handler.stop("Output file generated"); + + Ok(()) + } + + async fn download_video( + &self, + ctx: &Context, + variant_data: &VariantData, + message: String, + ) -> Result { + let tempfile = tempfile(".mp4")?; + let (mut file, path) = tempfile.into_parts(); + + download_segments(ctx, &mut file, Some(message), variant_data).await?; + + Ok(path) + } + + async fn download_audio( + &self, + ctx: &Context, + variant_data: &VariantData, + message: String, + ) -> Result { + let tempfile = tempfile(".m4a")?; + let (mut file, path) = tempfile.into_parts(); + + download_segments(ctx, &mut file, Some(message), variant_data).await?; + + Ok(path) + } + + async fn download_subtitle( + &self, + subtitle: Subtitle, + max_length: NaiveTime, + ) -> Result { + let tempfile = tempfile(".ass")?; + let (mut file, path) = tempfile.into_parts(); + + let mut buf = vec![]; + subtitle.write_to(&mut buf).await?; + fix_subtitle_look_and_feel(&mut buf); + fix_subtitle_length(&mut buf, max_length); + + file.write_all(buf.as_slice())?; + + Ok(path) + } +} + +pub async fn download_segments( + ctx: &Context, + writer: &mut impl Write, + message: Option, + variant_data: &VariantData, +) -> Result<()> { + let segments = variant_data.segments().await?; + let total_segments = segments.len(); + + let client = Arc::new(ctx.crunchy.client()); + let count = Arc::new(Mutex::new(0)); + + let progress = if log::max_level() == LevelFilter::Info { + let estimated_file_size = + (variant_data.bandwidth / 8) * segments.iter().map(|s| s.length.as_secs()).sum::(); + + let progress = ProgressBar::new(estimated_file_size) + .with_style( + ProgressStyle::with_template( + ":: {msg}{bytes:>10} {bytes_per_sec:>12} [{wide_bar}] {percent:>3}%", + ) + .unwrap() + .progress_chars("##-"), + ) + .with_message(message.map(|m| m + " ").unwrap_or_default()) + .with_finish(ProgressFinish::Abandon); + Some(progress) + } else { + None + }; + + let cpus = num_cpus::get(); + let mut segs: Vec> = Vec::with_capacity(cpus); + for _ in 0..cpus { + segs.push(vec![]) + } + for (i, segment) in segments.clone().into_iter().enumerate() { + segs[i - ((i / cpus) * cpus)].push(segment); + } + + let (sender, receiver) = mpsc::channel(); + + let mut join_set: JoinSet> = JoinSet::new(); + for num in 0..cpus { + let thread_client = client.clone(); + let thread_sender = sender.clone(); + let thread_segments = segs.remove(0); + let thread_count = count.clone(); + join_set.spawn(async move { + let after_download_sender = thread_sender.clone(); + + // the download process is encapsulated in its own function. this is done to easily + // catch errors which get returned with `...?` and `bail!(...)` and that the thread + // itself can report that an error has occurred + let download = || async move { + for (i, segment) in thread_segments.into_iter().enumerate() { + let mut retry_count = 0; + let mut buf = loop { + let response = thread_client + .get(&segment.url) + .timeout(Duration::from_secs(60)) + .send() + .await?; + + match response.bytes().await { + Ok(b) => break b.to_vec(), + Err(e) => { + if e.is_body() { + if retry_count == 5 { + bail!("Max retry count reached ({}), multiple errors occurred while receiving segment {}: {}", retry_count, num + (i * cpus), e) + } + debug!("Failed to download segment {} ({}). Retrying, {} out of 5 retries left", num + (i * cpus), e, 5 - retry_count) + } else { + bail!("{}", e) + } + } + } + + retry_count += 1; + }; + + buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec(); + + let mut c = thread_count.lock().unwrap(); + debug!( + "Downloaded and decrypted segment [{}/{} {:.2}%] {}", + num + (i * cpus) + 1, + total_segments, + ((*c + 1) as f64 / total_segments as f64) * 100f64, + segment.url + ); + + thread_sender.send((num as i32 + (i * cpus) as i32, buf))?; + + *c += 1; + } + Ok(()) + }; + + + let result = download().await; + if result.is_err() { + after_download_sender.send((-1 as i32, vec![]))?; + } + + result + }); + } + // drop the sender already here so it does not outlive all download threads which are the only + // real consumers of it + drop(sender); + + // this is the main loop which writes the data. it uses a BTreeMap as a buffer as the write + // happens synchronized. the download consist of multiple segments. the map keys are representing + // the segment number and the values the corresponding bytes + let mut data_pos = 0; + let mut buf: BTreeMap> = BTreeMap::new(); + for (pos, bytes) in receiver.iter() { + // if the position is lower than 0, an error occurred in the sending download thread + if pos < 0 { + break; + } + + if let Some(p) = &progress { + let progress_len = p.length().unwrap(); + let estimated_segment_len = + (variant_data.bandwidth / 8) * segments.get(pos as usize).unwrap().length.as_secs(); + let bytes_len = bytes.len() as u64; + + p.set_length(progress_len - estimated_segment_len + bytes_len); + p.inc(bytes_len) + } + + // check if the currently sent bytes are the next in the buffer. if so, write them directly + // to the target without first adding them to the buffer. + // if not, add them to the buffer + if data_pos == pos { + writer.write_all(bytes.borrow())?; + data_pos += 1; + } else { + buf.insert(pos, bytes); + } + // check if the buffer contains the next segment(s) + while let Some(b) = buf.remove(&data_pos) { + writer.write_all(b.borrow())?; + data_pos += 1; + } + } + + // if any error has occurred while downloading it gets returned here + while let Some(joined) = join_set.join_next().await { + joined?? + } + + // write the remaining buffer, if existent + while let Some(b) = buf.remove(&data_pos) { + writer.write_all(b.borrow())?; + data_pos += 1; + } + + if !buf.is_empty() { + bail!( + "Download buffer is not empty. Remaining segments: {}", + buf.into_keys() + .map(|k| k.to_string()) + .collect::>() + .join(", ") + ) + } + + Ok(()) +} + +/// Add `ScaledBorderAndShadows: yes` to subtitles; without it they look very messy on some video +/// players. See [crunchy-labs/crunchy-cli#66](https://github.com/crunchy-labs/crunchy-cli/issues/66) +/// for more information. +fn fix_subtitle_look_and_feel(raw: &mut Vec) { + let mut script_info = false; + let mut new = String::new(); + + for line in String::from_utf8_lossy(raw.as_slice()).split('\n') { + if line.trim().starts_with('[') && script_info { + new.push_str("ScaledBorderAndShadow: yes\n"); + script_info = false + } else if line.trim() == "[Script Info]" { + script_info = true + } + new.push_str(line); + new.push('\n') + } + + *raw = new.into_bytes() +} + +/// Get the length of a video. This is required because sometimes subtitles have an unnecessary entry +/// long after the actual video ends with artificially extends the video length on some video players. +/// To prevent this, the video length must be hard set. See +/// [crunchy-labs/crunchy-cli#32](https://github.com/crunchy-labs/crunchy-cli/issues/32) for more +/// information. +pub fn get_video_length(path: &Path) -> Result { + let video_length = Regex::new(r"Duration:\s(?P