From 99002e606f11aca2f07d079ee60af555c4fc99f4 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 28 Nov 2022 22:14:57 +0100 Subject: [PATCH 001/417] Change windows ci toolchain to gnu --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5d01b6..9aba2cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: - os: ubuntu-latest toolchain: x86_64-unknown-linux-musl - os: windows-latest - toolchain: x86_64-pc-windows-msvc + toolchain: x86_64-pc-windows-gnu - os: macos-latest toolchain: x86_64-apple-darwin steps: @@ -61,7 +61,7 @@ jobs: platform: linux ext: - os: windows-latest - toolchain: x86_64-pc-windows-msvc + toolchain: x86_64-pc-windows-gnu platform: windows ext: .exe - os: macos-latest From 4095b80477f30788aa262a50655e29265adba9d0 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 30 Nov 2022 19:04:01 +0100 Subject: [PATCH 002/417] Update dependencies --- Cargo.lock | 196 +++++++++++++++++++++++++----------- crunchy-cli-core/Cargo.lock | 188 ++++++++++++++++++++++++---------- crunchy-cli-core/Cargo.toml | 1 + 3 files changed, 277 insertions(+), 108 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 26ffd7f..144cff4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,9 +39,9 @@ checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" [[package]] name = "async-channel" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14485364214912d3b19cc3435dde4df66065127f05fa0d75c712f36f12c2f28" +checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833" dependencies = [ "concurrent-queue", "event-listener", @@ -50,32 +50,27 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.58" +version = "0.1.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e805d94e6b5001b651426cf4cd446b1ab5f319d27bab5c644f61de0a804360c" +checksum = "31e6e93155431f3931513b243d371981bb2770112b370c82745a1d19d2f99364" dependencies = [ "proc-macro2", "quote", "syn", ] -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi", - "libc", - "winapi", -] - [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "bitflags" version = "1.3.2" @@ -103,12 +98,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" -[[package]] -name = "cache-padded" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1db59621ec70f09c5e9b597b220c7a2b43611f4710dc03ceb8748637775692c" - [[package]] name = "castaway" version = "0.1.2" @@ -164,14 +153,14 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.26" +version = "4.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2148adefda54e14492fb9bddcc600b4344c5d1a3123bd666dcb939c6f0e0e57e" +checksum = "4d63b9e9c07271b9957ad22c173bae2a4d9a81127680962039296abcd2f8251d" dependencies = [ - "atty", "bitflags", "clap_derive", "clap_lex", + "is-terminal", "once_cell", "strsim", "termcolor", @@ -179,9 +168,9 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.0.5" +version = "4.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96b0fba905b035a30d25c1b585bf1171690712fbb0ad3ac47214963aa4acc36c" +checksum = "b7b3c9eae0de7bf8e3f904a5e40612b21fb2e2e566456d177809a48b892d24da" dependencies = [ "clap", ] @@ -210,9 +199,9 @@ dependencies = [ [[package]] name = "clap_mangen" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa149477df7854a1497db0def32b8a65bf98f72a14d04ac75b01938285d83420" +checksum = "e503c3058af0a0854668ea01db55c622482a080092fede9dd2e00a00a9436504" dependencies = [ "clap", "roff", @@ -230,11 +219,21 @@ dependencies = [ [[package]] name = "concurrent-queue" -version = "1.2.4" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af4780a44ab5696ea9e28294517f1fffb421a83a25af521333c838635509db9c" +checksum = "bd7bef69dc86e3c610e4e7aed41035e2a7ed12e72dd7530f61327a6579a4390b" dependencies = [ - "cache-padded", + "crossbeam-utils", +] + +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", ] [[package]] @@ -298,27 +297,30 @@ dependencies = [ [[package]] name = "crunchyroll-rs" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#7aedfc6c9a91a42ef46639ba9e99adba63cd0dda" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#3e9f7bfaab68649ecdb486aee0b15fe658d51917" dependencies = [ "aes", "cbc", "chrono", "crunchyroll-rs-internal", + "curl-sys", "http", "isahc", "m3u8-rs", "regex", + "rustls-native-certs", "serde", "serde_json", "serde_urlencoded", "smart-default", + "static_vcruntime", "tokio", ] [[package]] name = "crunchyroll-rs-internal" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#7aedfc6c9a91a42ef46639ba9e99adba63cd0dda" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#3e9f7bfaab68649ecdb486aee0b15fe658d51917" dependencies = [ "darling", "quote", @@ -455,6 +457,12 @@ dependencies = [ "syn", ] +[[package]] +name = "data-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" + [[package]] name = "dirs" version = "4.0.0" @@ -598,6 +606,15 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + [[package]] name = "http" version = "0.2.8" @@ -676,9 +693,25 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "0.7.5" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ce5ef949d49ee85593fc4d3f3f95ad61657076395cbbce23e2121fc5542074" +checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" +dependencies = [ + "libc", + "windows-sys 0.42.0", +] + +[[package]] +name = "is-terminal" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "927609f78c2913a6f6ac3c27a4fe87f43e2a35367c0c4b0f8265e8f49a104330" +dependencies = [ + "hermit-abi 0.2.6", + "io-lifetimes", + "rustix", + "windows-sys 0.42.0", +] [[package]] name = "isahc" @@ -690,6 +723,7 @@ dependencies = [ "crossbeam-utils", "curl", "curl-sys", + "data-encoding", "encoding_rs", "event-listener", "futures-lite", @@ -767,9 +801,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.0.46" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d" +checksum = "8f9f08d8963a6c613f4b1a78f4f4a4dbfadf8e6545b2d72861731e4858b8b47f" [[package]] name = "log" @@ -855,7 +889,7 @@ version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", ] @@ -946,16 +980,16 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "polling" -version = "2.4.0" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab4609a838d88b73d8238967b60dd115cc08d38e2bbaf51ee1e4b695f89122e2" +checksum = "166ca89eb77fd403230b9c156612965a81e094ec6ec3aa13663d4c8b113fa748" dependencies = [ "autocfg", "cfg-if", "libc", "log", "wepoll-ffi", - "winapi", + "windows-sys 0.42.0", ] [[package]] @@ -1054,9 +1088,9 @@ checksum = "b833d8d034ea094b1ea68aa6d5c740e0d04bad9d16568d08ba6f76823a114316" [[package]] name = "rustix" -version = "0.35.13" +version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727a1a6d65f786ec22df8a81ca3121107f235970dc1705ed681d3e6e8b9cd5f9" +checksum = "cb93e85278e08bb5788653183213d3a60fc242b10cb9be96586f5a73dcb67c23" dependencies = [ "bitflags", "errno", @@ -1066,6 +1100,27 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0864aeff53f8c05aa08d86e5ef839d3dfcf07aeba2db32f12db0ef716e87bd55" +dependencies = [ + "base64", +] + [[package]] name = "ryu" version = "1.0.11" @@ -1089,19 +1144,42 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" [[package]] -name = "serde" -version = "1.0.147" +name = "security-framework" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" +checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.148" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e53f64bb4ba0191d6d0676e1b141ca55047d83b74f5607e6d8eb88126c52c2dc" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.147" +version = "1.0.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852" +checksum = "a55492425aa53521babf6137309e7d34c20bbfbbfcfe2c7f3a047fd1f6b92c0c" dependencies = [ "proc-macro2", "quote", @@ -1191,6 +1269,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "static_vcruntime" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "954e3e877803def9dc46075bf4060147c55cd70db97873077232eae0269dc89b" + [[package]] name = "strsim" version = "0.10.0" @@ -1199,9 +1283,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.103" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" +checksum = "4ae548ec36cf198c0ef7710d3c230987c2d6d7bd98ad6edc0274462724c585ce" dependencies = [ "proc-macro2", "quote", @@ -1246,9 +1330,9 @@ dependencies = [ [[package]] name = "terminal_size" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ca90c434fd12083d1a6bdcbe9f92a14f96c8a1ba600ba451734ac334521f7a" +checksum = "cb20089a8ba2b69debd491f8d2d023761cbf196e999218c591fa1e7e15a21907" dependencies = [ "rustix", "windows-sys 0.42.0", @@ -1276,9 +1360,9 @@ dependencies = [ [[package]] name = "time" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" dependencies = [ "libc", "wasi 0.10.0+wasi-snapshot-preview1", @@ -1314,9 +1398,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.8.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" +checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ "proc-macro2", "quote", diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index 61bc6f6..3ee8448 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -39,9 +39,9 @@ checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" [[package]] name = "async-channel" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14485364214912d3b19cc3435dde4df66065127f05fa0d75c712f36f12c2f28" +checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833" dependencies = [ "concurrent-queue", "event-listener", @@ -50,32 +50,27 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.58" +version = "0.1.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e805d94e6b5001b651426cf4cd446b1ab5f319d27bab5c644f61de0a804360c" +checksum = "31e6e93155431f3931513b243d371981bb2770112b370c82745a1d19d2f99364" dependencies = [ "proc-macro2", "quote", "syn", ] -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi", - "libc", - "winapi", -] - [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "bitflags" version = "1.3.2" @@ -103,12 +98,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" -[[package]] -name = "cache-padded" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1db59621ec70f09c5e9b597b220c7a2b43611f4710dc03ceb8748637775692c" - [[package]] name = "castaway" version = "0.1.2" @@ -164,14 +153,14 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.26" +version = "4.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2148adefda54e14492fb9bddcc600b4344c5d1a3123bd666dcb939c6f0e0e57e" +checksum = "4d63b9e9c07271b9957ad22c173bae2a4d9a81127680962039296abcd2f8251d" dependencies = [ - "atty", "bitflags", "clap_derive", "clap_lex", + "is-terminal", "once_cell", "strsim", "termcolor", @@ -211,11 +200,21 @@ dependencies = [ [[package]] name = "concurrent-queue" -version = "1.2.4" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af4780a44ab5696ea9e28294517f1fffb421a83a25af521333c838635509db9c" +checksum = "bd7bef69dc86e3c610e4e7aed41035e2a7ed12e72dd7530f61327a6579a4390b" dependencies = [ - "cache-padded", + "crossbeam-utils", +] + +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", ] [[package]] @@ -267,27 +266,30 @@ dependencies = [ [[package]] name = "crunchyroll-rs" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#7aedfc6c9a91a42ef46639ba9e99adba63cd0dda" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#3e9f7bfaab68649ecdb486aee0b15fe658d51917" dependencies = [ "aes", "cbc", "chrono", "crunchyroll-rs-internal", + "curl-sys", "http", "isahc", "m3u8-rs", "regex", + "rustls-native-certs", "serde", "serde_json", "serde_urlencoded", "smart-default", + "static_vcruntime", "tokio", ] [[package]] name = "crunchyroll-rs-internal" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#7aedfc6c9a91a42ef46639ba9e99adba63cd0dda" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#3e9f7bfaab68649ecdb486aee0b15fe658d51917" dependencies = [ "darling", "quote", @@ -424,6 +426,12 @@ dependencies = [ "syn", ] +[[package]] +name = "data-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" + [[package]] name = "dirs" version = "4.0.0" @@ -567,6 +575,15 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + [[package]] name = "http" version = "0.2.8" @@ -645,9 +662,25 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "0.7.5" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ce5ef949d49ee85593fc4d3f3f95ad61657076395cbbce23e2121fc5542074" +checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" +dependencies = [ + "libc", + "windows-sys 0.42.0", +] + +[[package]] +name = "is-terminal" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "927609f78c2913a6f6ac3c27a4fe87f43e2a35367c0c4b0f8265e8f49a104330" +dependencies = [ + "hermit-abi 0.2.6", + "io-lifetimes", + "rustix", + "windows-sys 0.42.0", +] [[package]] name = "isahc" @@ -659,6 +692,7 @@ dependencies = [ "crossbeam-utils", "curl", "curl-sys", + "data-encoding", "encoding_rs", "event-listener", "futures-lite", @@ -736,9 +770,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.0.46" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d" +checksum = "8f9f08d8963a6c613f4b1a78f4f4a4dbfadf8e6545b2d72861731e4858b8b47f" [[package]] name = "log" @@ -824,7 +858,7 @@ version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", ] @@ -915,16 +949,16 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "polling" -version = "2.4.0" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab4609a838d88b73d8238967b60dd115cc08d38e2bbaf51ee1e4b695f89122e2" +checksum = "166ca89eb77fd403230b9c156612965a81e094ec6ec3aa13663d4c8b113fa748" dependencies = [ "autocfg", "cfg-if", "libc", "log", "wepoll-ffi", - "winapi", + "windows-sys 0.42.0", ] [[package]] @@ -1017,9 +1051,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.35.13" +version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727a1a6d65f786ec22df8a81ca3121107f235970dc1705ed681d3e6e8b9cd5f9" +checksum = "cb93e85278e08bb5788653183213d3a60fc242b10cb9be96586f5a73dcb67c23" dependencies = [ "bitflags", "errno", @@ -1029,6 +1063,27 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0864aeff53f8c05aa08d86e5ef839d3dfcf07aeba2db32f12db0ef716e87bd55" +dependencies = [ + "base64", +] + [[package]] name = "ryu" version = "1.0.11" @@ -1052,19 +1107,42 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" [[package]] -name = "serde" -version = "1.0.147" +name = "security-framework" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" +checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.148" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e53f64bb4ba0191d6d0676e1b141ca55047d83b74f5607e6d8eb88126c52c2dc" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.147" +version = "1.0.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852" +checksum = "a55492425aa53521babf6137309e7d34c20bbfbbfcfe2c7f3a047fd1f6b92c0c" dependencies = [ "proc-macro2", "quote", @@ -1154,6 +1232,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "static_vcruntime" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "954e3e877803def9dc46075bf4060147c55cd70db97873077232eae0269dc89b" + [[package]] name = "strsim" version = "0.10.0" @@ -1162,9 +1246,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.103" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" +checksum = "4ae548ec36cf198c0ef7710d3c230987c2d6d7bd98ad6edc0274462724c585ce" dependencies = [ "proc-macro2", "quote", @@ -1209,9 +1293,9 @@ dependencies = [ [[package]] name = "terminal_size" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ca90c434fd12083d1a6bdcbe9f92a14f96c8a1ba600ba451734ac334521f7a" +checksum = "cb20089a8ba2b69debd491f8d2d023761cbf196e999218c591fa1e7e15a21907" dependencies = [ "rustix", "windows-sys 0.42.0", @@ -1239,9 +1323,9 @@ dependencies = [ [[package]] name = "time" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" dependencies = [ "libc", "wasi 0.10.0+wasi-snapshot-preview1", @@ -1277,9 +1361,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.8.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" +checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ "proc-macro2", "quote", diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 00635b0..f8c5007 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" static-curl = ["crunchyroll-rs/static-curl"] # Embed a static openssl library into the binary instead of just linking it. If you want to compile this project against # musl and have openssl issues, this might solve these issues. +# Has no effect on Windows, the library is always statically linked there. static-ssl = ["crunchyroll-rs/static-ssl"] [dependencies] From e200aab8ab47d746659c3c1d4969fd637f6a26c1 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 30 Nov 2022 19:04:11 +0100 Subject: [PATCH 003/417] Update ci --- .github/workflows/ci.yml | 58 ++++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9aba2cd..1fa94a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: command: test args: --all-features - build: + build-nix: if: github.ref == 'refs/heads/master' needs: - test @@ -59,15 +59,9 @@ jobs: - os: ubuntu-latest toolchain: x86_64-unknown-linux-musl platform: linux - ext: - - os: windows-latest - toolchain: x86_64-pc-windows-gnu - platform: windows - ext: .exe - os: macos-latest toolchain: x86_64-apple-darwin platform: darwin - ext: steps: - name: Checkout uses: actions/checkout@v3 @@ -101,7 +95,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: crunchy-cli_${{ matrix.platform }} - path: ./target/release/crunchy-cli${{ matrix.ext }} + path: ./target/release/crunchy-cli if-no-files-found: error - name: Upload manpages artifact @@ -117,3 +111,51 @@ jobs: name: completions path: ./target/release/completions if-no-files-found: error + + build-windows: + if: github.ref == 'refs/heads/master' + needs: + - test + runs-on: windows-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Install OpenSSL + run: vcpkg install openssl:x64-windows-static-md + + - name: Set env variables + shell: bash + run: echo "CFLAGS=-I$(echo $VCPKG_INSTALLATION_ROOT)\packages\openssl_x64-windows-static-md\include" >> $GITHUB_ENV + + - name: Cargo cache # https://github.com/actions/cache/blob/main/examples.md#rust---cargo + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Install toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + target: x86_64-pc-windows-msvc + default: true + + - name: Build + uses: actions-rs/cargo@v1 + with: + command: build + args: --release + + - name: Upload binary artifact + uses: actions/upload-artifact@v3 + with: + name: crunchy-cli_windows.exe + path: ./target/release/crunchy-cli.exe + if-no-files-found: error From 6ecd23bcd0c8b87fd49f2321fe09d08f623af9b4 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 30 Nov 2022 19:25:00 +0100 Subject: [PATCH 004/417] Fix ci --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1fa94a0..277a591 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,6 +121,8 @@ jobs: - name: Checkout uses: actions/checkout@v3 + - run: vcpkg integrate install + - name: Install OpenSSL run: vcpkg install openssl:x64-windows-static-md From 12d49a27e468632976f4c8d269c9e124e1e2a898 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 30 Nov 2022 19:25:21 +0100 Subject: [PATCH 005/417] Add static vc runtime --- Cargo.lock | 6 +++--- Cargo.toml | 3 +++ build.rs | 3 +++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 144cff4..2690925 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -269,6 +269,7 @@ dependencies = [ "clap_complete", "clap_mangen", "crunchy-cli-core", + "static_vcruntime", "tokio", ] @@ -297,7 +298,7 @@ dependencies = [ [[package]] name = "crunchyroll-rs" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#3e9f7bfaab68649ecdb486aee0b15fe658d51917" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#3d455ebca34cfa83df39777f3e1414cee4f84002" dependencies = [ "aes", "cbc", @@ -313,14 +314,13 @@ dependencies = [ "serde_json", "serde_urlencoded", "smart-default", - "static_vcruntime", "tokio", ] [[package]] name = "crunchyroll-rs-internal" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#3e9f7bfaab68649ecdb486aee0b15fe658d51917" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#3d455ebca34cfa83df39777f3e1414cee4f84002" dependencies = [ "darling", "quote", diff --git a/Cargo.toml b/Cargo.toml index 7614202..5e3b153 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,9 @@ clap_mangen = "0.2" # specified in this Cargo.toml [features]. crunchy-cli-core = { path = "./crunchy-cli-core", features = ["static-curl", "static-ssl"] } +[target.'cfg(all(windows, target_env = "msvc"))'.build-dependencies] +static_vcruntime = "2.0" + [profile.release] strip = true opt-level = "z" diff --git a/build.rs b/build.rs index 1e4d71f..927f4dd 100644 --- a/build.rs +++ b/build.rs @@ -3,6 +3,9 @@ use clap_complete::shells; use std::path::{Path, PathBuf}; fn main() -> std::io::Result<()> { + #[cfg(all(windows, target_env = "msvc"))] + static_vcruntime::metabuild(); + // note that we're using an anti-pattern here / violate the rust conventions. build script are // not supposed to write outside of 'OUT_DIR'. to have the generated files in the build "root" // (the same directory where the output binary lives) is much simpler than in 'OUT_DIR' since From a3c717dc1a4e9bd27e2bf508bfac8f02143268e0 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 30 Nov 2022 19:54:39 +0100 Subject: [PATCH 006/417] Split test job --- .github/workflows/ci.yml | 62 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 277a591..9f55b5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,15 +8,13 @@ on: workflow_dispatch: jobs: - test: + test-nix: runs-on: ${{ matrix.os }} strategy: matrix: include: - os: ubuntu-latest toolchain: x86_64-unknown-linux-musl - - os: windows-latest - toolchain: x86_64-pc-windows-gnu - os: macos-latest toolchain: x86_64-apple-darwin steps: @@ -48,10 +46,50 @@ jobs: command: test args: --all-features + test-windows: + runs-on: windows-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - run: vcpkg integrate install + + - name: Install OpenSSL + run: vcpkg install openssl:x64-windows-static-md + + - name: Set env variables + shell: bash + run: echo "CFLAGS=-I$(echo $VCPKG_INSTALLATION_ROOT)\packages\openssl_x64-windows-static-md\include" >> $GITHUB_ENV + + - name: Cargo cache # https://github.com/actions/cache/blob/main/examples.md#rust---cargo + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Install toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + target: x86_64-pc-windows-msvc + default: true + + - name: Test + uses: actions-rs/cargo@v1 + with: + command: test + args: + build-nix: if: github.ref == 'refs/heads/master' needs: - - test + - test-nix runs-on: ${{ matrix.os }} strategy: matrix: @@ -115,7 +153,7 @@ jobs: build-windows: if: github.ref == 'refs/heads/master' needs: - - test + - test-windows runs-on: windows-latest steps: - name: Checkout @@ -161,3 +199,17 @@ jobs: name: crunchy-cli_windows.exe path: ./target/release/crunchy-cli.exe if-no-files-found: error + + - name: Upload manpages artifact + uses: actions/upload-artifact@v3 + with: + name: manpages + path: ./target/release/manpages + if-no-files-found: error + + - name: Upload completions artifact + uses: actions/upload-artifact@v3 + with: + name: completions + path: ./target/release/completions + if-no-files-found: error From 0bb20d24a2bbbfcc882791a5aa25c70d7217ebcc Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 30 Nov 2022 21:34:20 +0100 Subject: [PATCH 007/417] Update dependency versions --- Cargo.lock | 59 +++++++++++++++++++++++++++++-------- Cargo.toml | 5 +--- crunchy-cli-core/Cargo.lock | 59 ++++++++++--------------------------- crunchy-cli-core/Cargo.toml | 10 +++---- 4 files changed, 68 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2690925..290b5d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -104,6 +104,15 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2698f953def977c68f935bb0dfa959375ad4638570e969e2f1e9f433cbf1af6" +[[package]] +name = "castaway" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a17ed5635fc8536268e5d4de1e22e81ac34419e5f052d4d51f4e01dcc263fcc" +dependencies = [ + "rustversion", +] + [[package]] name = "cbc" version = "0.1.2" @@ -284,10 +293,11 @@ dependencies = [ "crunchyroll-rs", "ctrlc", "dirs", - "isahc", + "isahc 1.7.0 (git+https://github.com/sagebind/isahc?rev=c39f6f8)", "log", "num_cpus", "regex", + "rustls-native-certs", "signal-hook", "sys-locale", "tempfile", @@ -306,7 +316,7 @@ dependencies = [ "crunchyroll-rs-internal", "curl-sys", "http", - "isahc", + "isahc 1.7.0 (git+https://github.com/sagebind/isahc?rev=34f158ef)", "m3u8-rs", "regex", "rustls-native-certs", @@ -719,7 +729,7 @@ version = "1.7.0" source = "git+https://github.com/sagebind/isahc?rev=34f158ef#34f158ef9f87b2387bed2c81936916a29c1eaad1" dependencies = [ "async-channel", - "castaway", + "castaway 0.1.2", "crossbeam-utils", "curl", "curl-sys", @@ -741,6 +751,33 @@ dependencies = [ "waker-fn", ] +[[package]] +name = "isahc" +version = "1.7.0" +source = "git+https://github.com/sagebind/isahc?rev=c39f6f8#c39f6f85aaa1f36c5857064c6c3c80f4d307d863" +dependencies = [ + "async-channel", + "castaway 0.2.2", + "crossbeam-utils", + "curl", + "curl-sys", + "data-encoding", + "encoding_rs", + "event-listener", + "futures-io", + "futures-lite", + "http", + "log", + "mime", + "once_cell", + "polling", + "sluice", + "tracing", + "tracing-futures", + "url", + "waker-fn", +] + [[package]] name = "itoa" version = "1.0.4" @@ -905,15 +942,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" -[[package]] -name = "openssl-src" -version = "111.24.0+1.1.1s" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3498f259dab01178c6228c6b00dcef0ed2a2d5e20d648c017861227773ea4abd" -dependencies = [ - "cc", -] - [[package]] name = "openssl-sys" version = "0.9.78" @@ -923,7 +951,6 @@ dependencies = [ "autocfg", "cc", "libc", - "openssl-src", "pkg-config", "vcpkg", ] @@ -1121,6 +1148,12 @@ dependencies = [ "base64", ] +[[package]] +name = "rustversion" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8" + [[package]] name = "ryu" version = "1.0.11" diff --git a/Cargo.toml b/Cargo.toml index 5e3b153..d08c47f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,9 +9,6 @@ default = ["static-curl"] # Embed a static curl library into the binary instead of just linking it. static-curl = ["crunchy-cli-core/static-curl"] -# Embed a static openssl library into the binary instead of just linking it. If you want to compile this project against -# musl and have openssl issues, this might solve these issues. -static-ssl = ["crunchy-cli-core/static-ssl"] [dependencies] tokio = { version = "1.22", features = ["macros", "rt-multi-thread", "time"], default-features = false } @@ -26,7 +23,7 @@ clap_mangen = "0.2" # The static-* features must be used here since build dependency features cannot be manipulated from the features # specified in this Cargo.toml [features]. -crunchy-cli-core = { path = "./crunchy-cli-core", features = ["static-curl", "static-ssl"] } +crunchy-cli-core = { path = "./crunchy-cli-core", features = ["static-curl"] } [target.'cfg(all(windows, target_env = "msvc"))'.build-dependencies] static_vcruntime = "2.0" diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index 3ee8448..cd16c1b 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -100,9 +100,12 @@ checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" [[package]] name = "castaway" -version = "0.1.2" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2698f953def977c68f935bb0dfa959375ad4638570e969e2f1e9f433cbf1af6" +checksum = "8a17ed5635fc8536268e5d4de1e22e81ac34419e5f052d4d51f4e01dcc263fcc" +dependencies = [ + "rustversion", +] [[package]] name = "cbc" @@ -256,6 +259,7 @@ dependencies = [ "log", "num_cpus", "regex", + "rustls-native-certs", "signal-hook", "sys-locale", "tempfile", @@ -266,7 +270,7 @@ dependencies = [ [[package]] name = "crunchyroll-rs" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#3e9f7bfaab68649ecdb486aee0b15fe658d51917" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#ffb054cbae4079bf2357a23e90a353ce875978af" dependencies = [ "aes", "cbc", @@ -282,14 +286,13 @@ dependencies = [ "serde_json", "serde_urlencoded", "smart-default", - "static_vcruntime", "tokio", ] [[package]] name = "crunchyroll-rs-internal" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#3e9f7bfaab68649ecdb486aee0b15fe658d51917" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#ffb054cbae4079bf2357a23e90a353ce875978af" dependencies = [ "darling", "quote", @@ -530,13 +533,8 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" dependencies = [ - "fastrand", "futures-core", - "futures-io", - "memchr", - "parking", "pin-project-lite", - "waker-fn", ] [[package]] @@ -685,7 +683,7 @@ dependencies = [ [[package]] name = "isahc" version = "1.7.0" -source = "git+https://github.com/sagebind/isahc?rev=34f158ef#34f158ef9f87b2387bed2c81936916a29c1eaad1" +source = "git+https://github.com/sagebind/isahc?rev=c39f6f8#c39f6f85aaa1f36c5857064c6c3c80f4d307d863" dependencies = [ "async-channel", "castaway", @@ -695,6 +693,7 @@ dependencies = [ "data-encoding", "encoding_rs", "event-listener", + "futures-io", "futures-lite", "http", "httpdate", @@ -702,7 +701,6 @@ dependencies = [ "mime", "once_cell", "polling", - "slab", "sluice", "tracing", "tracing-futures", @@ -874,15 +872,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" -[[package]] -name = "openssl-src" -version = "111.24.0+1.1.1s" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3498f259dab01178c6228c6b00dcef0ed2a2d5e20d648c017861227773ea4abd" -dependencies = [ - "cc", -] - [[package]] name = "openssl-sys" version = "0.9.78" @@ -892,7 +881,6 @@ dependencies = [ "autocfg", "cc", "libc", - "openssl-src", "pkg-config", "vcpkg", ] @@ -903,12 +891,6 @@ version = "6.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" -[[package]] -name = "parking" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" - [[package]] name = "percent-encoding" version = "2.2.0" @@ -1084,6 +1066,12 @@ dependencies = [ "base64", ] +[[package]] +name = "rustversion" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8" + [[package]] name = "ryu" version = "1.0.11" @@ -1191,15 +1179,6 @@ dependencies = [ "libc", ] -[[package]] -name = "slab" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" -dependencies = [ - "autocfg", -] - [[package]] name = "sluice" version = "0.5.5" @@ -1232,12 +1211,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "static_vcruntime" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "954e3e877803def9dc46075bf4060147c55cd70db97873077232eae0269dc89b" - [[package]] name = "strsim" version = "0.10.0" diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index f8c5007..30e6d44 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -7,10 +7,6 @@ edition = "2021" [features] # Embed a static curl library into the binary instead of just linking it. static-curl = ["crunchyroll-rs/static-curl"] -# Embed a static openssl library into the binary instead of just linking it. If you want to compile this project against -# musl and have openssl issues, this might solve these issues. -# Has no effect on Windows, the library is always statically linked there. -static-ssl = ["crunchyroll-rs/static-ssl"] [dependencies] anyhow = "1.0" @@ -20,7 +16,7 @@ chrono = "0.4" crunchyroll-rs = { git = "https://github.com/crunchy-labs/crunchyroll-rs", default-features = false, features = ["stream", "parse"] } ctrlc = "3.2" dirs = "4.0" -isahc = { git = "https://github.com/sagebind/isahc", rev = "34f158ef" } +isahc = { git = "https://github.com/sagebind/isahc", rev = "c39f6f8" } log = { version = "0.4", features = ["std"] } num_cpus = "1.13" regex = "1.6" @@ -30,5 +26,9 @@ terminal_size = "0.2" tokio = { version = "1.21", features = ["macros", "rt-multi-thread", "time"] } sys-locale = "0.2" +[target.'cfg(all(windows, target_env = "msvc"))'.dependencies] +isahc = { git = "https://github.com/sagebind/isahc", rev = "c39f6f8", features = ["data-encoding"] } +rustls-native-certs = "0.6" + [build-dependencies] chrono = "0.4" From 64bb39362e9f937bb852ded778e7ee315de72483 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 30 Nov 2022 21:34:30 +0100 Subject: [PATCH 008/417] Add Windows certificates --- crunchy-cli-core/src/lib.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index 4571337..c0cfce5 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -167,7 +167,21 @@ async fn create_ctx(cli: &Cli) -> Result { // TODO: Use crunchy.client() when it's possible // currently crunchy.client() has a cloudflare bypass built-in to access crunchyroll. the servers // where crunchy stores their videos can't handle this bypass and simply refuses to connect + #[cfg(not(all(windows, target_env = "msvc")))] let client = isahc::HttpClient::new().unwrap(); + #[cfg(all(windows, target_env = "msvc"))] + let client = isahc::HttpClientBuilder::default() + .proxy_tls_config( + isahc::tls::TlsConfigBuilder::default().root_cert_store( + isahc::tls::RootCertStore::custom( + rustls_native_certs::load_native_certs() + .unwrap() + .into_iter() + .map(|l| isahc::tls::Certificate::from_der(l.0)), + ), + ), + ) + .build(); Ok(Context { crunchy, client }) } From f687969f0456323dbffe2146e02657b2afa45a85 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 30 Nov 2022 21:53:23 +0100 Subject: [PATCH 009/417] Fix isahc tls config --- crunchy-cli-core/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index c0cfce5..d64dae5 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -171,7 +171,7 @@ async fn create_ctx(cli: &Cli) -> Result { let client = isahc::HttpClient::new().unwrap(); #[cfg(all(windows, target_env = "msvc"))] let client = isahc::HttpClientBuilder::default() - .proxy_tls_config( + .tls_config( isahc::tls::TlsConfigBuilder::default().root_cert_store( isahc::tls::RootCertStore::custom( rustls_native_certs::load_native_certs() From f6d6c9435c7b6c91001cfad4f53dd3cb7819be15 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 30 Nov 2022 22:28:51 +0100 Subject: [PATCH 010/417] Fix invalid Windows client builder --- crunchy-cli-core/src/lib.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index d64dae5..a1e8381 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -164,24 +164,27 @@ pub async fn cli_entrypoint() { async fn create_ctx(cli: &Cli) -> Result { let crunchy = crunchyroll_session(cli).await?; - // TODO: Use crunchy.client() when it's possible + // use crunchy.client() when it's possible // currently crunchy.client() has a cloudflare bypass built-in to access crunchyroll. the servers // where crunchy stores their videos can't handle this bypass and simply refuses to connect #[cfg(not(all(windows, target_env = "msvc")))] let client = isahc::HttpClient::new().unwrap(); #[cfg(all(windows, target_env = "msvc"))] + use isahc::config::Configurable; + #[cfg(all(windows, target_env = "msvc"))] let client = isahc::HttpClientBuilder::default() .tls_config( - isahc::tls::TlsConfigBuilder::default().root_cert_store( - isahc::tls::RootCertStore::custom( + isahc::tls::TlsConfigBuilder::default() + .root_cert_store(isahc::tls::RootCertStore::custom( rustls_native_certs::load_native_certs() .unwrap() .into_iter() .map(|l| isahc::tls::Certificate::from_der(l.0)), - ), - ), + )) + .build(), ) - .build(); + .build() + .unwrap(); Ok(Context { crunchy, client }) } From 474e9f5e31c0d07727b0ce39d8dc08373a7bd278 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 30 Nov 2022 23:43:57 +0100 Subject: [PATCH 011/417] Add very verbose output flag --- crunchy-cli-core/src/cli/log.rs | 10 ++++++---- crunchy-cli-core/src/lib.rs | 14 ++++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/crunchy-cli-core/src/cli/log.rs b/crunchy-cli-core/src/cli/log.rs index d4bb1b3..3539733 100644 --- a/crunchy-cli-core/src/cli/log.rs +++ b/crunchy-cli-core/src/cli/log.rs @@ -92,6 +92,7 @@ impl CliProgress { #[allow(clippy::type_complexity)] pub struct CliLogger { + all: bool, level: LevelFilter, progress: Mutex>, } @@ -105,7 +106,7 @@ impl Log for CliLogger { if !self.enabled(record.metadata()) || (record.target() != "progress" && record.target() != "progress_end" - && !record.target().starts_with("crunchy_cli")) + && (!self.all && !record.target().starts_with("crunchy_cli"))) { return; } @@ -136,16 +137,17 @@ impl Log for CliLogger { } impl CliLogger { - pub fn new(level: LevelFilter) -> Self { + pub fn new(all: bool, level: LevelFilter) -> Self { Self { + all, level, progress: Mutex::new(None), } } - pub fn init(level: LevelFilter) -> Result<(), SetLoggerError> { + pub fn init(all: bool, level: LevelFilter) -> Result<(), SetLoggerError> { set_max_level(level); - set_boxed_logger(Box::new(CliLogger::new(level))) + set_boxed_logger(Box::new(CliLogger::new(all, level))) } fn extended(&self, record: &Record) { diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index a1e8381..9b57859 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -64,6 +64,10 @@ struct Verbosity { #[arg(short)] v: bool, + #[arg(help = "Very verbose output. Generally not recommended, use '-v' instead")] + #[arg(long)] + vv: bool, + #[arg(help = "Quiet output. Does not print anything unless it's a error")] #[arg( long_help = "Quiet output. Does not print anything unless it's a error. Can be helpful if you pipe the output to stdout" @@ -92,16 +96,18 @@ pub async fn cli_entrypoint() { let cli: Cli = Cli::parse(); if let Some(verbosity) = &cli.verbosity { - if verbosity.v && verbosity.q { + if verbosity.v as u8 + verbosity.q as u8 + verbosity.vv as u8 > 1 { eprintln!("Output cannot be verbose ('-v') and quiet ('-q') at the same time"); std::process::exit(1) } else if verbosity.v { - CliLogger::init(LevelFilter::Debug).unwrap() + CliLogger::init(false, LevelFilter::Debug).unwrap() } else if verbosity.q { - CliLogger::init(LevelFilter::Error).unwrap() + CliLogger::init(false, LevelFilter::Error).unwrap() + } else if verbosity.vv { + CliLogger::init(true, LevelFilter::Debug).unwrap() } } else { - CliLogger::init(LevelFilter::Info).unwrap() + CliLogger::init(false, LevelFilter::Info).unwrap() } debug!("cli input: {:?}", cli); From 6aa4078be343c99503f62bd5e5179109a9706b2a Mon Sep 17 00:00:00 2001 From: ByteDream Date: Fri, 2 Dec 2022 17:28:54 +0100 Subject: [PATCH 012/417] Update dependencies --- Cargo.lock | 386 ++++++++++++++++++++++---- crunchy-cli-core/src/cli/archive.rs | 2 +- crunchy-cli-core/src/cli/download.rs | 14 +- crunchy-cli-core/src/cli/utils.rs | 7 +- crunchy-cli-core/src/lib.rs | 24 +- crunchy-cli-core/src/utils/context.rs | 1 - 6 files changed, 342 insertions(+), 92 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 290b5d1..fddbc62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -98,12 +98,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" -[[package]] -name = "castaway" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2698f953def977c68f935bb0dfa959375ad4638570e969e2f1e9f433cbf1af6" - [[package]] name = "castaway" version = "0.2.2" @@ -293,7 +287,7 @@ dependencies = [ "crunchyroll-rs", "ctrlc", "dirs", - "isahc 1.7.0 (git+https://github.com/sagebind/isahc?rev=c39f6f8)", + "isahc", "log", "num_cpus", "regex", @@ -308,7 +302,7 @@ dependencies = [ [[package]] name = "crunchyroll-rs" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#3d455ebca34cfa83df39777f3e1414cee4f84002" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#b05ab04af60547f4d5b8d008e3b3f829eb99aa0f" dependencies = [ "aes", "cbc", @@ -316,10 +310,10 @@ dependencies = [ "crunchyroll-rs-internal", "curl-sys", "http", - "isahc 1.7.0 (git+https://github.com/sagebind/isahc?rev=34f158ef)", + "isahc", "m3u8-rs", "regex", - "rustls-native-certs", + "reqwest", "serde", "serde_json", "serde_urlencoded", @@ -330,7 +324,7 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#3d455ebca34cfa83df39777f3e1414cee4f84002" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#b05ab04af60547f4d5b8d008e3b3f829eb99aa0f" dependencies = [ "darling", "quote", @@ -544,6 +538,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.1.0" @@ -553,6 +562,15 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures-channel" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" +dependencies = [ + "futures-core", +] + [[package]] name = "futures-core" version = "0.3.25" @@ -571,13 +589,35 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" dependencies = [ - "fastrand", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "futures-sink" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" + +[[package]] +name = "futures-task" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" + +[[package]] +name = "futures-util" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" +dependencies = [ "futures-core", "futures-io", + "futures-task", "memchr", - "parking", "pin-project-lite", - "waker-fn", + "pin-utils", + "slab", ] [[package]] @@ -601,6 +641,31 @@ dependencies = [ "wasi 0.11.0+wasi-snapshot-preview1", ] +[[package]] +name = "h2" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "heck" version = "0.4.0" @@ -636,12 +701,66 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + [[package]] name = "httpdate" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "hyper" +version = "0.14.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "iana-time-zone" version = "0.1.53" @@ -682,6 +801,16 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "indexmap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +dependencies = [ + "autocfg", + "hashbrown", +] + [[package]] name = "inout" version = "0.1.3" @@ -711,6 +840,12 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "ipnet" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f88c5561171189e69df9d98bcf18fd5f9558300f7ea7b801eb8a0fd748bd8745" + [[package]] name = "is-terminal" version = "0.4.1" @@ -723,41 +858,13 @@ dependencies = [ "windows-sys 0.42.0", ] -[[package]] -name = "isahc" -version = "1.7.0" -source = "git+https://github.com/sagebind/isahc?rev=34f158ef#34f158ef9f87b2387bed2c81936916a29c1eaad1" -dependencies = [ - "async-channel", - "castaway 0.1.2", - "crossbeam-utils", - "curl", - "curl-sys", - "data-encoding", - "encoding_rs", - "event-listener", - "futures-lite", - "http", - "httpdate", - "log", - "mime", - "once_cell", - "polling", - "slab", - "sluice", - "tracing", - "tracing-futures", - "url", - "waker-fn", -] - [[package]] name = "isahc" version = "1.7.0" source = "git+https://github.com/sagebind/isahc?rev=c39f6f8#c39f6f85aaa1f36c5857064c6c3c80f4d307d863" dependencies = [ "async-channel", - "castaway 0.2.2", + "castaway", "crossbeam-utils", "curl", "curl-sys", @@ -767,6 +874,7 @@ dependencies = [ "futures-io", "futures-lite", "http", + "httpdate", "log", "mime", "once_cell", @@ -853,9 +961,9 @@ dependencies = [ [[package]] name = "m3u8-rs" -version = "5.0.2" +version = "5.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15d091887fd4a920417805283b7a838d0dcda68e8d632cd305a4439ee776d1ce" +checksum = "2c3b9f971e885044cb57330d3c89a4f3bd45fe97d01c93b3dcec57096efacffc" dependencies = [ "chrono", "nom", @@ -880,10 +988,40 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] -name = "nix" -version = "0.25.0" +name = "mio" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e322c04a9e3440c327fca7b6c8a63e6890a32fa2ad689db972425f07e0d22abb" +checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +dependencies = [ + "libc", + "log", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.42.0", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nix" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4" dependencies = [ "autocfg", "bitflags", @@ -936,6 +1074,32 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +[[package]] +name = "openssl" +version = "0.10.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020433887e44c27ff16365eaa2d380547a94544ad509aff6eb5b6e3e0b27b376" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.5" @@ -961,12 +1125,6 @@ version = "6.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" -[[package]] -name = "parking" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" - [[package]] name = "percent-encoding" version = "2.2.0" @@ -999,6 +1157,12 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkg-config" version = "0.3.26" @@ -1107,6 +1271,43 @@ dependencies = [ "winapi", ] +[[package]] +name = "reqwest" +version = "0.11.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "roff" version = "0.2.1" @@ -1316,9 +1517,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.104" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae548ec36cf198c0ef7710d3c230987c2d6d7bd98ad6edc0274462724c585ce" +checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908" dependencies = [ "proc-macro2", "quote", @@ -1424,9 +1625,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d76ce4a75fb488c605c54bf610f221cea8b0dafb53333c1a67e8ee199dcd2ae3" dependencies = [ "autocfg", + "bytes", + "libc", + "memchr", + "mio", "num_cpus", "pin-project-lite", + "socket2", "tokio-macros", + "winapi", ] [[package]] @@ -1440,6 +1647,36 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + [[package]] name = "tracing" version = "0.1.37" @@ -1483,6 +1720,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + [[package]] name = "typenum" version = "1.15.0" @@ -1545,6 +1788,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.10.0+wasi-snapshot-preview1" @@ -1582,6 +1835,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.83" @@ -1760,3 +2025,12 @@ name = "windows_x86_64_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 754d6cb..a057aed 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -387,7 +387,7 @@ async fn download_video(ctx: &Context, format: &Format, only_audio: bool) -> Res ctx, &mut ffmpeg.stdin.unwrap(), Some(format!("Download {}", format.audio)), - format.stream.segments().await?, + format.stream.clone() ) .await?; diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index 7f9e3ec..8535869 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -8,7 +8,7 @@ use crate::utils::parse::{parse_url, UrlFilter}; use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number}; use crate::Execute; use anyhow::{bail, Result}; -use crunchyroll_rs::media::{Resolution, VariantSegment}; +use crunchyroll_rs::media::{Resolution, VariantData}; use crunchyroll_rs::{ Episode, Locale, Media, MediaCollection, Movie, MovieListing, Season, Series, }; @@ -212,16 +212,14 @@ impl Execute for Download { tab_info!("Resolution: {}", format.stream.resolution); tab_info!("FPS: {:.2}", format.stream.fps); - let segments = format.stream.segments().await?; - if use_ffmpeg { - download_ffmpeg(&ctx, segments, path.as_path()).await?; + download_ffmpeg(&ctx, format.stream, path.as_path()).await?; } else if path.to_str().unwrap() == "-" { let mut stdout = std::io::stdout().lock(); - download_segments(&ctx, &mut stdout, None, segments).await?; + download_segments(&ctx, &mut stdout, None, format.stream).await?; } else { let mut file = File::options().create(true).write(true).open(&path)?; - download_segments(&ctx, &mut file, None, segments).await? + download_segments(&ctx, &mut file, None, format.stream).await? } } } @@ -232,7 +230,7 @@ impl Execute for Download { async fn download_ffmpeg( ctx: &Context, - segments: Vec, + variant_data: VariantData, target: &Path, ) -> Result<()> { let ffmpeg = Command::new("ffmpeg") @@ -246,7 +244,7 @@ async fn download_ffmpeg( .arg(target.to_str().unwrap()) .spawn()?; - download_segments(ctx, &mut ffmpeg.stdin.unwrap(), None, segments).await?; + download_segments(ctx, &mut ffmpeg.stdin.unwrap(), None, variant_data).await?; Ok(()) } diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index 59816d1..e513101 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -26,14 +26,15 @@ pub fn find_resolution( } pub async fn download_segments( - ctx: &Context, + _ctx: &Context, writer: &mut impl Write, message: Option, - segments: Vec, + variant_data: VariantData, ) -> Result<()> { + let segments = variant_data.segments().await?; let total_segments = segments.len(); - let client = Arc::new(ctx.client.clone()); + let client = Arc::new(variant_data.download_client()); let count = Arc::new(Mutex::new(0)); let amount = Arc::new(Mutex::new(0)); diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index 9b57859..083b401 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -170,29 +170,7 @@ pub async fn cli_entrypoint() { async fn create_ctx(cli: &Cli) -> Result { let crunchy = crunchyroll_session(cli).await?; - // use crunchy.client() when it's possible - // currently crunchy.client() has a cloudflare bypass built-in to access crunchyroll. the servers - // where crunchy stores their videos can't handle this bypass and simply refuses to connect - #[cfg(not(all(windows, target_env = "msvc")))] - let client = isahc::HttpClient::new().unwrap(); - #[cfg(all(windows, target_env = "msvc"))] - use isahc::config::Configurable; - #[cfg(all(windows, target_env = "msvc"))] - let client = isahc::HttpClientBuilder::default() - .tls_config( - isahc::tls::TlsConfigBuilder::default() - .root_cert_store(isahc::tls::RootCertStore::custom( - rustls_native_certs::load_native_certs() - .unwrap() - .into_iter() - .map(|l| isahc::tls::Certificate::from_der(l.0)), - )) - .build(), - ) - .build() - .unwrap(); - - Ok(Context { crunchy, client }) + Ok(Context { crunchy }) } async fn crunchyroll_session(cli: &Cli) -> Result { diff --git a/crunchy-cli-core/src/utils/context.rs b/crunchy-cli-core/src/utils/context.rs index 6a2fd21..f8df024 100644 --- a/crunchy-cli-core/src/utils/context.rs +++ b/crunchy-cli-core/src/utils/context.rs @@ -2,5 +2,4 @@ use crunchyroll_rs::Crunchyroll; pub struct Context { pub crunchy: Crunchyroll, - pub client: isahc::HttpClient, } From 3fcb512c187e968df987b4c954925e514c223d56 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Fri, 2 Dec 2022 18:19:13 +0100 Subject: [PATCH 013/417] Merge test and build ci --- .github/workflows/ci.yml | 96 ++++++---------------------------------- 1 file changed, 13 insertions(+), 83 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f55b5d..d0e5960 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,88 +8,8 @@ on: workflow_dispatch: jobs: - test-nix: - runs-on: ${{ matrix.os }} - strategy: - matrix: - include: - - os: ubuntu-latest - toolchain: x86_64-unknown-linux-musl - - os: macos-latest - toolchain: x86_64-apple-darwin - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Cargo cache # https://github.com/actions/cache/blob/main/examples.md#rust---cargo - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - name: Install toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - target: ${{ matrix.toolchain }} - default: true - - - name: Test - uses: actions-rs/cargo@v1 - with: - command: test - args: --all-features - - test-windows: - runs-on: windows-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - - run: vcpkg integrate install - - - name: Install OpenSSL - run: vcpkg install openssl:x64-windows-static-md - - - name: Set env variables - shell: bash - run: echo "CFLAGS=-I$(echo $VCPKG_INSTALLATION_ROOT)\packages\openssl_x64-windows-static-md\include" >> $GITHUB_ENV - - - name: Cargo cache # https://github.com/actions/cache/blob/main/examples.md#rust---cargo - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - name: Install toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - target: x86_64-pc-windows-msvc - default: true - - - name: Test - uses: actions-rs/cargo@v1 - with: - command: test - args: - build-nix: if: github.ref == 'refs/heads/master' - needs: - - test-nix runs-on: ${{ matrix.os }} strategy: matrix: @@ -123,6 +43,12 @@ jobs: target: ${{ matrix.toolchain }} default: true + - name: Test + uses: actions-rs/cargo@v1 + with: + command: test + args: --release --all-features + - name: Build uses: actions-rs/cargo@v1 with: @@ -152,8 +78,6 @@ jobs: build-windows: if: github.ref == 'refs/heads/master' - needs: - - test-windows runs-on: windows-latest steps: - name: Checkout @@ -187,11 +111,17 @@ jobs: target: x86_64-pc-windows-msvc default: true + - name: Test + uses: actions-rs/cargo@v1 + with: + command: test + args: --release --all-features + - name: Build uses: actions-rs/cargo@v1 with: command: build - args: --release + args: --release --all-features - name: Upload binary artifact uses: actions/upload-artifact@v3 From 33e27504f2ba2b981caa90103f2c29badfd1b0ed Mon Sep 17 00:00:00 2001 From: ByteDream Date: Fri, 2 Dec 2022 19:41:50 +0100 Subject: [PATCH 014/417] Update dependencies --- Cargo.lock | 4 +- crunchy-cli-core/Cargo.lock | 347 ++++++++++++++++++++++++++++++++++-- 2 files changed, 339 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fddbc62..ffdd71b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -302,7 +302,7 @@ dependencies = [ [[package]] name = "crunchyroll-rs" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#b05ab04af60547f4d5b8d008e3b3f829eb99aa0f" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#66678d49bc071f49c83a7fa73ab280d1c1de5b2f" dependencies = [ "aes", "cbc", @@ -324,7 +324,7 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#b05ab04af60547f4d5b8d008e3b3f829eb99aa0f" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#66678d49bc071f49c83a7fa73ab280d1c1de5b2f" dependencies = [ "darling", "quote", diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index cd16c1b..4fa983a 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -270,7 +270,7 @@ dependencies = [ [[package]] name = "crunchyroll-rs" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#ffb054cbae4079bf2357a23e90a353ce875978af" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#66678d49bc071f49c83a7fa73ab280d1c1de5b2f" dependencies = [ "aes", "cbc", @@ -281,7 +281,7 @@ dependencies = [ "isahc", "m3u8-rs", "regex", - "rustls-native-certs", + "reqwest", "serde", "serde_json", "serde_urlencoded", @@ -292,7 +292,7 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#ffb054cbae4079bf2357a23e90a353ce875978af" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#66678d49bc071f49c83a7fa73ab280d1c1de5b2f" dependencies = [ "darling", "quote", @@ -506,6 +506,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.1.0" @@ -515,6 +530,15 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures-channel" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" +dependencies = [ + "futures-core", +] + [[package]] name = "futures-core" version = "0.3.25" @@ -537,6 +561,33 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "futures-sink" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" + +[[package]] +name = "futures-task" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" + +[[package]] +name = "futures-util" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" +dependencies = [ + "futures-core", + "futures-io", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "generic-array" version = "0.14.6" @@ -558,6 +609,31 @@ dependencies = [ "wasi 0.11.0+wasi-snapshot-preview1", ] +[[package]] +name = "h2" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "heck" version = "0.4.0" @@ -593,12 +669,66 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + [[package]] name = "httpdate" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "hyper" +version = "0.14.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "iana-time-zone" version = "0.1.53" @@ -639,6 +769,16 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "indexmap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +dependencies = [ + "autocfg", + "hashbrown", +] + [[package]] name = "inout" version = "0.1.3" @@ -668,6 +808,12 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "ipnet" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f88c5561171189e69df9d98bcf18fd5f9558300f7ea7b801eb8a0fd748bd8745" + [[package]] name = "is-terminal" version = "0.4.1" @@ -783,9 +929,9 @@ dependencies = [ [[package]] name = "m3u8-rs" -version = "5.0.2" +version = "5.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15d091887fd4a920417805283b7a838d0dcda68e8d632cd305a4439ee776d1ce" +checksum = "2c3b9f971e885044cb57330d3c89a4f3bd45fe97d01c93b3dcec57096efacffc" dependencies = [ "chrono", "nom", @@ -810,10 +956,40 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] -name = "nix" -version = "0.25.0" +name = "mio" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e322c04a9e3440c327fca7b6c8a63e6890a32fa2ad689db972425f07e0d22abb" +checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +dependencies = [ + "libc", + "log", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.42.0", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nix" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4" dependencies = [ "autocfg", "bitflags", @@ -866,6 +1042,32 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +[[package]] +name = "openssl" +version = "0.10.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020433887e44c27ff16365eaa2d380547a94544ad509aff6eb5b6e3e0b27b376" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.5" @@ -923,6 +1125,12 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkg-config" version = "0.3.26" @@ -1031,6 +1239,43 @@ dependencies = [ "winapi", ] +[[package]] +name = "reqwest" +version = "0.11.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "rustix" version = "0.36.4" @@ -1179,6 +1424,15 @@ dependencies = [ "libc", ] +[[package]] +name = "slab" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg", +] + [[package]] name = "sluice" version = "0.5.5" @@ -1219,9 +1473,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.104" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae548ec36cf198c0ef7710d3c230987c2d6d7bd98ad6edc0274462724c585ce" +checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908" dependencies = [ "proc-macro2", "quote", @@ -1327,9 +1581,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d76ce4a75fb488c605c54bf610f221cea8b0dafb53333c1a67e8ee199dcd2ae3" dependencies = [ "autocfg", + "bytes", + "libc", + "memchr", + "mio", "num_cpus", "pin-project-lite", + "socket2", "tokio-macros", + "winapi", ] [[package]] @@ -1343,6 +1603,36 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + [[package]] name = "tracing" version = "0.1.37" @@ -1386,6 +1676,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + [[package]] name = "typenum" version = "1.15.0" @@ -1448,6 +1744,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.10.0+wasi-snapshot-preview1" @@ -1485,6 +1791,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.83" @@ -1663,3 +1981,12 @@ name = "windows_x86_64_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] From cd9c69baf1ecb47865da2db8902b580155c9278e Mon Sep 17 00:00:00 2001 From: ByteDream Date: Fri, 2 Dec 2022 20:23:30 +0100 Subject: [PATCH 015/417] Fix windows artifact name --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0e5960..3900976 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -126,7 +126,7 @@ jobs: - name: Upload binary artifact uses: actions/upload-artifact@v3 with: - name: crunchy-cli_windows.exe + name: crunchy-cli_windows path: ./target/release/crunchy-cli.exe if-no-files-found: error From e9b3088cde86aa8468a47602e2694b5799fdc081 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Fri, 2 Dec 2022 21:42:24 +0100 Subject: [PATCH 016/417] Fix windows ok output character --- crunchy-cli-core/src/cli/log.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/cli/log.rs b/crunchy-cli-core/src/cli/log.rs index 3539733..d1f0225 100644 --- a/crunchy-cli-core/src/cli/log.rs +++ b/crunchy-cli-core/src/cli/log.rs @@ -19,6 +19,12 @@ impl CliProgress { let init_message = format!("{}", record.args()); let init_level = record.level(); let handler = thread::spawn(move || { + #[cfg(not(windows))] + let ok = '✔'; + #[cfg(windows)] + // windows does not support all unicode characters by default in their consoles, so + // we're using this (square root?) symbol instead. microsoft. + let ok = '√'; let states = ["-", "\\", "|", "/"]; let mut old_message = init_message.clone(); @@ -68,7 +74,7 @@ impl CliProgress { // clear last line // prefix (2), space (1), state (1), space (1), message(n) let _ = write!(stdout(), "\r {}", " ".repeat(old_message.len())); - let _ = writeln!(stdout(), "\r:: ✓ {}", old_message); + let _ = writeln!(stdout(), "\r:: {} {}", ok, old_message); let _ = stdout().flush(); }); From afab3826c945d5906aaaf07e0d6b43749e7a338d Mon Sep 17 00:00:00 2001 From: ByteDream Date: Fri, 2 Dec 2022 22:06:08 +0100 Subject: [PATCH 017/417] Extend function to get free file --- Cargo.lock | 11 +++++++++++ crunchy-cli-core/Cargo.lock | 11 +++++++++++ crunchy-cli-core/Cargo.toml | 1 + crunchy-cli-core/src/cli/archive.rs | 5 ++--- crunchy-cli-core/src/cli/download.rs | 9 ++------- crunchy-cli-core/src/utils/os.rs | 17 ++++++++++------- 6 files changed, 37 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ffdd71b..ff1f3dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -292,6 +292,7 @@ dependencies = [ "num_cpus", "regex", "rustls-native-certs", + "sanitize-filename", "signal-hook", "sys-locale", "tempfile", @@ -1361,6 +1362,16 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +[[package]] +name = "sanitize-filename" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c502bdb638f1396509467cb0580ef3b29aa2a45c5d43e5d84928241280296c" +dependencies = [ + "lazy_static", + "regex", +] + [[package]] name = "schannel" version = "0.1.20" diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index 4fa983a..b842f38 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -260,6 +260,7 @@ dependencies = [ "num_cpus", "regex", "rustls-native-certs", + "sanitize-filename", "signal-hook", "sys-locale", "tempfile", @@ -1323,6 +1324,16 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +[[package]] +name = "sanitize-filename" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c502bdb638f1396509467cb0580ef3b29aa2a45c5d43e5d84928241280296c" +dependencies = [ + "lazy_static", + "regex", +] + [[package]] name = "schannel" version = "0.1.20" diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 30e6d44..d797098 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -20,6 +20,7 @@ isahc = { git = "https://github.com/sagebind/isahc", rev = "c39f6f8" } log = { version = "0.4", features = ["std"] } num_cpus = "1.13" regex = "1.6" +sanitize-filename = "0.4" signal-hook = "0.3" tempfile = "3.3" terminal_size = "0.2" diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index a057aed..e798e75 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -206,8 +206,7 @@ impl Execute for Archive { .to_string(), primary, )), - ) - .0; + ); info!( "Downloading {} to '{}'", @@ -387,7 +386,7 @@ async fn download_video(ctx: &Context, format: &Format, only_audio: bool) -> Res ctx, &mut ffmpeg.stdin.unwrap(), Some(format!("Download {}", format.audio)), - format.stream.clone() + format.stream.clone(), ) .await?; diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index 8535869..acb086b 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -178,8 +178,7 @@ impl Execute for Download { .to_string(), &format, )), - ) - .0; + ); let use_ffmpeg = if let Some(extension) = path.extension() { if extension != "ts" { @@ -228,11 +227,7 @@ impl Execute for Download { } } -async fn download_ffmpeg( - ctx: &Context, - variant_data: VariantData, - target: &Path, -) -> Result<()> { +async fn download_ffmpeg(ctx: &Context, variant_data: VariantData, target: &Path) -> Result<()> { let ffmpeg = Command::new("ffmpeg") .stdin(Stdio::piped()) .stdout(Stdio::null()) diff --git a/crunchy-cli-core/src/utils/os.rs b/crunchy-cli-core/src/utils/os.rs index 317381d..92e4e9f 100644 --- a/crunchy-cli-core/src/utils/os.rs +++ b/crunchy-cli-core/src/utils/os.rs @@ -19,7 +19,7 @@ pub fn has_ffmpeg() -> bool { } } -/// Any tempfiles should be created with this function. The prefix and directory of every file +/// Any tempfile should be created with this function. The prefix and directory of every file /// created with this method stays the same which is helpful to query all existing tempfiles and /// e.g. remove them in a case of ctrl-c. Having one function also good to prevent mistakes like /// setting the wrong prefix if done manually. @@ -36,17 +36,20 @@ pub fn tempfile>(suffix: S) -> io::Result { } /// Check if the given path exists and rename it until the new (renamed) file does not exist. -pub fn free_file(mut path: PathBuf) -> (PathBuf, bool) { +pub fn free_file(mut path: PathBuf) -> PathBuf { let mut i = 0; while path.exists() { i += 1; - let ext = path.extension().unwrap().to_str().unwrap(); - let mut filename = path.file_name().unwrap().to_str().unwrap(); - - filename = &filename[0..filename.len() - ext.len() - 1]; + let ext = path.extension().unwrap().to_string_lossy(); + let filename = path.file_stem().unwrap().to_string_lossy(); path.set_file_name(format!("{} ({}).{}", filename, i, ext)) } - (path, i != 0) + sanitize_file(path) +} + +/// Sanitizes the given path to not contain any invalid file character. +pub fn sanitize_file(path: PathBuf) -> PathBuf { + path.with_file_name(sanitize_filename::sanitize(path.file_name().unwrap().to_string_lossy())) } From 9d45995e86c4c2b1c8979fa71e6e7abba14937fd Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sat, 3 Dec 2022 00:26:52 +0100 Subject: [PATCH 018/417] Set progress width with message to use complete space --- crunchy-cli-core/src/cli/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index e513101..6197684 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -77,7 +77,7 @@ pub async fn download_segments( let progress_available = size - if let Some(msg) = &message { - 35 + msg.len() + 34 + msg.len() } else { 33 }; From 135d59ce8b3ac8230c385c3a9469564bd460ce5b Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sat, 3 Dec 2022 00:39:52 +0100 Subject: [PATCH 019/417] Add pre-check function --- crunchy-cli-core/src/cli/archive.rs | 12 +++++++++++- crunchy-cli-core/src/cli/download.rs | 27 +++++++++++---------------- crunchy-cli-core/src/lib.rs | 3 ++- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index e798e75..d4b0a5d 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -3,7 +3,7 @@ use crate::cli::utils::{download_segments, find_resolution}; use crate::utils::context::Context; use crate::utils::format::{format_string, Format}; use crate::utils::log::progress; -use crate::utils::os::{free_file, tempfile}; +use crate::utils::os::{free_file, has_ffmpeg, tempfile}; use crate::utils::parse::{parse_url, UrlFilter}; use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number}; use crate::Execute; @@ -110,6 +110,16 @@ pub struct Archive { #[async_trait::async_trait(?Send)] impl Execute for Archive { + fn pre_check(&self) -> Result<()> { + if !has_ffmpeg() { + bail!("FFmpeg is needed to run this command") + } else if PathBuf::from(&self.output).extension().unwrap_or_default().to_string_lossy() != "mkv" { + bail!("File extension is not '.mkv'. Currently only matroska / '.mkv' files are supported") + } + + Ok(()) + } + async fn execute(self, ctx: Context) -> Result<()> { let mut parsed_urls = vec![]; diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index acb086b..b0d0ffa 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -65,6 +65,16 @@ pub struct Download { #[async_trait::async_trait(?Send)] impl Execute for Download { + fn pre_check(&self) -> Result<()> { + if has_ffmpeg() { + debug!("FFmpeg detected") + } else if PathBuf::from(&self.output).extension().unwrap_or_default().to_string_lossy() != "ts" { + bail!("File extension is not '.ts'. If you want to use a custom file format, please install ffmpeg") + } + + Ok(()) + } + async fn execute(self, ctx: Context) -> Result<()> { let mut parsed_urls = vec![]; @@ -180,21 +190,6 @@ impl Execute for Download { )), ); - let use_ffmpeg = if let Some(extension) = path.extension() { - if extension != "ts" { - if !has_ffmpeg() { - bail!( - "File ending is not `.ts`, ffmpeg is required to convert the video" - ) - } - true - } else { - false - } - } else { - false - }; - info!( "Downloading {} to '{}'", format.title, @@ -211,7 +206,7 @@ impl Execute for Download { tab_info!("Resolution: {}", format.stream.resolution); tab_info!("FPS: {:.2}", format.stream.fps); - if use_ffmpeg { + if path.extension().unwrap_or_default().to_string_lossy() != "ts" { download_ffmpeg(&ctx, format.stream, path.as_path()).await?; } else if path.to_str().unwrap() == "-" { let mut stdout = std::io::stdout().lock(); diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index 083b401..c895d51 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -16,6 +16,7 @@ pub use cli::{archive::Archive, download::Download, login::Login}; #[async_trait::async_trait(?Send)] trait Execute { + fn pre_check(&self) -> Result<()> { Ok(()) } async fn execute(self, ctx: Context) -> Result<()>; } @@ -163,7 +164,7 @@ pub async fn cli_entrypoint() { } }; if let Err(err) = result { - error!("{}", err); + error!("a unexpected error occurred: {}", err); std::process::exit(1) } } From 5826d95e6aa2964dfabf189a3b785bab89972c14 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sat, 3 Dec 2022 01:04:29 +0100 Subject: [PATCH 020/417] Add progress width offset --- crunchy-cli-core/src/cli/utils.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index 6197684..0bf6b45 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -75,11 +75,17 @@ pub async fn download_segments( .0 .0 as usize; + // there is a offset of 1 "length" (idk how to describe it), so removing 1 from + // `progress_available` would fill the terminal width completely. on multiple + // systems there is a bug that printing until the end of the line causes a newline + // even though technically there shouldn't be one. on my tests, this only happens on + // windows and mac machines and (at the addressed environments) only with release + // builds. so maybe an unwanted optimization? let progress_available = size - if let Some(msg) = &message { - 34 + msg.len() + 35 + msg.len() } else { - 33 + 34 }; let progress_done_count = (progress_available as f64 * (percentage / 100f64)).ceil() as usize; From cd1308426ee6611292fcdb6bfcf2d24da1a18ea0 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sat, 3 Dec 2022 12:26:36 +0100 Subject: [PATCH 021/417] Re-add static-ssl feature --- Cargo.lock | 30 ++++++++++++++++++++---------- Cargo.toml | 3 +++ crunchy-cli-core/Cargo.lock | 30 ++++++++++++++++++++---------- crunchy-cli-core/Cargo.toml | 2 +- 4 files changed, 44 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ff1f3dd..720a1c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -385,9 +385,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4a41a86530d0fe7f5d9ea779916b7cadd2d4f9add748b99c2c029cbbdfaf453" +checksum = "bdf07d07d6531bfcdbe9b8b739b104610c6508dcc4d63b410585faf338241daf" dependencies = [ "cc", "cxxbridge-flags", @@ -397,9 +397,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06416d667ff3e3ad2df1cd8cd8afae5da26cf9cec4d0825040f88b5ca659a2f0" +checksum = "d2eb5b96ecdc99f72657332953d4d9c50135af1bac34277801cc3937906ebd39" dependencies = [ "cc", "codespan-reporting", @@ -412,15 +412,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "820a9a2af1669deeef27cb271f476ffd196a2c4b6731336011e0ba63e2c7cf71" +checksum = "ac040a39517fd1674e0f32177648334b0f4074625b5588a64519804ba0553b12" [[package]] name = "cxxbridge-macro" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08a6e2fcc370a089ad3b4aaf54db3b1b4cee38ddabce5896b33eb693275f470" +checksum = "1362b0ddcfc4eb0a1f57b68bd77dd99f0e826958a96abd0ae9bd092e114ffed6" dependencies = [ "proc-macro2", "quote", @@ -910,9 +910,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.137" +version = "0.2.138" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" +checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8" [[package]] name = "libnghttp2-sys" @@ -1107,6 +1107,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-src" +version = "111.24.0+1.1.1s" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3498f259dab01178c6228c6b00dcef0ed2a2d5e20d648c017861227773ea4abd" +dependencies = [ + "cc", +] + [[package]] name = "openssl-sys" version = "0.9.78" @@ -1116,6 +1125,7 @@ dependencies = [ "autocfg", "cc", "libc", + "openssl-src", "pkg-config", "vcpkg", ] diff --git a/Cargo.toml b/Cargo.toml index d08c47f..1b815c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,9 @@ default = ["static-curl"] # Embed a static curl library into the binary instead of just linking it. static-curl = ["crunchy-cli-core/static-curl"] +# Embed a static tls library into the binary instead of just linking it. +# has no effect on Windows, always activated there. +static-ssl = ["crunchy-cli-core/static-ssl"] [dependencies] tokio = { version = "1.22", features = ["macros", "rt-multi-thread", "time"], default-features = false } diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index b842f38..2612b36 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -353,9 +353,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4a41a86530d0fe7f5d9ea779916b7cadd2d4f9add748b99c2c029cbbdfaf453" +checksum = "bdf07d07d6531bfcdbe9b8b739b104610c6508dcc4d63b410585faf338241daf" dependencies = [ "cc", "cxxbridge-flags", @@ -365,9 +365,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06416d667ff3e3ad2df1cd8cd8afae5da26cf9cec4d0825040f88b5ca659a2f0" +checksum = "d2eb5b96ecdc99f72657332953d4d9c50135af1bac34277801cc3937906ebd39" dependencies = [ "cc", "codespan-reporting", @@ -380,15 +380,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "820a9a2af1669deeef27cb271f476ffd196a2c4b6731336011e0ba63e2c7cf71" +checksum = "ac040a39517fd1674e0f32177648334b0f4074625b5588a64519804ba0553b12" [[package]] name = "cxxbridge-macro" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08a6e2fcc370a089ad3b4aaf54db3b1b4cee38ddabce5896b33eb693275f470" +checksum = "1362b0ddcfc4eb0a1f57b68bd77dd99f0e826958a96abd0ae9bd092e114ffed6" dependencies = [ "proc-macro2", "quote", @@ -878,9 +878,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.137" +version = "0.2.138" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" +checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8" [[package]] name = "libnghttp2-sys" @@ -1075,6 +1075,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-src" +version = "111.24.0+1.1.1s" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3498f259dab01178c6228c6b00dcef0ed2a2d5e20d648c017861227773ea4abd" +dependencies = [ + "cc", +] + [[package]] name = "openssl-sys" version = "0.9.78" @@ -1084,6 +1093,7 @@ dependencies = [ "autocfg", "cc", "libc", + "openssl-src", "pkg-config", "vcpkg", ] diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index d797098..395030a 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -5,8 +5,8 @@ version = "3.0.0-dev.1" edition = "2021" [features] -# Embed a static curl library into the binary instead of just linking it. static-curl = ["crunchyroll-rs/static-curl"] +static-ssl = ["isahc/native-tls-static"] [dependencies] anyhow = "1.0" From 64717fd405256daa3a764107da92c8bfad309138 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sat, 3 Dec 2022 12:32:45 +0100 Subject: [PATCH 022/417] Update dependencies --- Cargo.lock | 4 ++-- crunchy-cli-core/Cargo.lock | 4 ++-- crunchy-cli-core/Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 720a1c4..108a431 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -303,7 +303,7 @@ dependencies = [ [[package]] name = "crunchyroll-rs" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#66678d49bc071f49c83a7fa73ab280d1c1de5b2f" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#7644b24864608c059d702db8b373be81ffcec643" dependencies = [ "aes", "cbc", @@ -325,7 +325,7 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#66678d49bc071f49c83a7fa73ab280d1c1de5b2f" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#7644b24864608c059d702db8b373be81ffcec643" dependencies = [ "darling", "quote", diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index 2612b36..d0f951a 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -271,7 +271,7 @@ dependencies = [ [[package]] name = "crunchyroll-rs" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#66678d49bc071f49c83a7fa73ab280d1c1de5b2f" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#7644b24864608c059d702db8b373be81ffcec643" dependencies = [ "aes", "cbc", @@ -293,7 +293,7 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#66678d49bc071f49c83a7fa73ab280d1c1de5b2f" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#7644b24864608c059d702db8b373be81ffcec643" dependencies = [ "darling", "quote", diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 395030a..a1d898e 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" [features] static-curl = ["crunchyroll-rs/static-curl"] -static-ssl = ["isahc/native-tls-static"] +static-ssl = ["crunchyroll-rs/static-ssl"] [dependencies] anyhow = "1.0" From 7c3bbfc1737ecdaf291d25ca817f1d35eccd21e3 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 4 Dec 2022 18:54:19 +0100 Subject: [PATCH 023/417] Wait until buffer is empty when downloading --- crunchy-cli-core/src/cli/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index 0bf6b45..758636d 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -169,7 +169,7 @@ pub async fn download_segments( data_pos += 1; } - if *count.lock().unwrap() >= total_segments { + if *count.lock().unwrap() >= total_segments && buf.is_empty() { break; } } From 285d27772c10b288022bda19f4b027505f96c284 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 4 Dec 2022 18:54:22 +0100 Subject: [PATCH 024/417] Add manually .ass file editing to fix #32 --- crunchy-cli-core/src/cli/archive.rs | 108 ++++++++++++++++++++++------ 1 file changed, 86 insertions(+), 22 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index d4b0a5d..d586725 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -8,6 +8,7 @@ use crate::utils::parse::{parse_url, UrlFilter}; use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number}; use crate::Execute; use anyhow::{bail, Result}; +use chrono::{NaiveTime, Timelike}; use crunchyroll_rs::media::{Resolution, StreamSubtitle}; use crunchyroll_rs::{Locale, Media, MediaCollection, Series}; use log::{debug, error, info}; @@ -113,7 +114,12 @@ impl Execute for Archive { fn pre_check(&self) -> Result<()> { if !has_ffmpeg() { bail!("FFmpeg is needed to run this command") - } else if PathBuf::from(&self.output).extension().unwrap_or_default().to_string_lossy() != "mkv" { + } else if PathBuf::from(&self.output) + .extension() + .unwrap_or_default() + .to_string_lossy() + != "mkv" + { bail!("File extension is not '.mkv'. Currently only matroska / '.mkv' files are supported") } @@ -276,9 +282,18 @@ impl Execute for Archive { } } + let (primary_video, _) = video_paths.get(0).unwrap(); + let primary_video_length = get_video_length(primary_video.to_path_buf()).unwrap(); for subtitle in subtitles { - subtitle_paths - .push((download_subtitle(&self, subtitle.clone()).await?, subtitle)) + subtitle_paths.push(( + download_subtitle( + &self, + subtitle.clone(), + primary_video_length + ) + .await?, + subtitle, + )) } generate_mkv(&self, path, video_paths, audio_paths, subtitle_paths)? @@ -403,15 +418,20 @@ async fn download_video(ctx: &Context, format: &Format, only_audio: bool) -> Res Ok(path) } -async fn download_subtitle(archive: &Archive, subtitle: StreamSubtitle) -> Result { +async fn download_subtitle( + archive: &Archive, + subtitle: StreamSubtitle, + max_length: NaiveTime, +) -> Result { let tempfile = tempfile(".ass")?; let (mut file, path) = tempfile.into_parts(); let mut buf = vec![]; subtitle.write_to(&mut buf).await?; if !archive.no_subtitle_optimizations { - buf = fix_subtitle(buf) + buf = fix_subtitle_look_and_feel(buf) } + buf = fix_subtitle_length(buf, max_length); file.write_all(buf.as_slice())?; @@ -421,7 +441,7 @@ async fn download_subtitle(archive: &Archive, subtitle: StreamSubtitle) -> Resul /// Add `ScaledBorderAndShadows: yes` to subtitles; without it they look very messy on some video /// players. See [crunchy-labs/crunchy-cli#66](https://github.com/crunchy-labs/crunchy-cli/issues/66) /// for more information. -fn fix_subtitle(raw: Vec) -> Vec { +fn fix_subtitle_look_and_feel(raw: Vec) -> Vec { let mut script_info = false; let mut new = String::new(); @@ -439,6 +459,62 @@ fn fix_subtitle(raw: Vec) -> Vec { new.into_bytes() } +/// Fix the length of subtitles to a specified maximum amount. This is required because sometimes +/// subtitles have an unnecessary entry long after the actual video ends with artificially extends +/// the video length on some video players. To prevent this, the video length must be hard set. See +/// [crunchy-labs/crunchy-cli#32](https://github.com/crunchy-labs/crunchy-cli/issues/32) for more +/// information. +fn fix_subtitle_length(raw: Vec, max_length: NaiveTime) -> Vec { + let re = + Regex::new(r#"^Dialogue:\s\d+,(?P\d+:\d+:\d+\.\d+),(?P\d+:\d+:\d+\.\d+),"#) + .unwrap(); + + // chrono panics if we try to format NaiveTime with `%2f` and the nano seconds has more than 2 + // digits so them have to be reduced manually to avoid the panic + fn format_naive_time(native_time: NaiveTime) -> String { + let formatted_time = native_time.format("%f").to_string(); + format!("{}.{}", native_time.format("%T"), if formatted_time.len() <= 2 { + native_time.format("%2f").to_string() + } else { + formatted_time.split_at(2).0.parse().unwrap() + }) + } + + let length_as_string = format_naive_time(max_length); + let mut new = String::new(); + + for line in String::from_utf8_lossy(raw.as_slice()).split('\n') { + if let Some(capture) = re.captures(line) { + let start = capture.name("start").map_or(NaiveTime::default(), |s| { + NaiveTime::parse_from_str(s.as_str(), "%H:%M:%S.%f").unwrap() + }); + let end = capture.name("end").map_or(NaiveTime::default(), |s| { + NaiveTime::parse_from_str(s.as_str(), "%H:%M:%S.%f").unwrap() + }); + + if start > max_length { + continue + } else if end > max_length { + new.push_str(re.replace( + line, + format!( + "Dialogue: {},{},", + format_naive_time(start), + &length_as_string + ), + ).to_string().as_str()) + } else { + new.push_str(line) + } + } else { + new.push_str(line) + } + new.push('\n') + } + + new.into_bytes() +} + fn generate_mkv( archive: &Archive, target: PathBuf, @@ -450,8 +526,6 @@ fn generate_mkv( let mut maps = vec![]; let mut metadata = vec![]; - let mut video_length = (0, 0, 0, 0); - for (i, (video_path, format)) in video_paths.iter().enumerate() { input.extend(["-i".to_string(), video_path.to_string_lossy().to_string()]); maps.extend(["-map".to_string(), i.to_string()]); @@ -471,11 +545,6 @@ fn generate_mkv( format!("-metadata:s:a:{}", i), format!("title={}", format.audio.to_human_readable()), ]); - - let vid_len = get_video_length(video_path.to_path_buf())?; - if vid_len > video_length { - video_length = vid_len - } } for (i, (audio_path, format)) in audio_paths.iter().enumerate() { input.extend(["-i".to_string(), audio_path.to_string_lossy().to_string()]); @@ -552,11 +621,11 @@ fn generate_mkv( /// Get the length of a video. This is required because sometimes subtitles have an unnecessary entry /// long after the actual video ends with artificially extends the video length on some video players. -/// To prevent this, the video length must be hard set with ffmpeg. See +/// To prevent this, the video length must be hard set. See /// [crunchy-labs/crunchy-cli#32](https://github.com/crunchy-labs/crunchy-cli/issues/32) for more /// information. -fn get_video_length(path: PathBuf) -> Result<(u32, u32, u32, u32)> { - let video_length = Regex::new(r"Duration:\s?(\d+):(\d+):(\d+).(\d+),")?; +fn get_video_length(path: PathBuf) -> Result { + let video_length = Regex::new(r"Duration:\s(?P