NyxErinys/crunchy-cli
1
0
Fork 0
mirror of https://github.com/crunchy-labs/crunchy-cli.git synced 2026-01-21 12:12:00 -06:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #95 from crunchy-labs/fix/rework-download

Browse source 
Fix & rework download
This commit is contained in:
ByteDream 2023-01-03 01:51:19 +01:00 • committed by GitHub
commit c11851adc9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 147 additions and 101 deletions
Download patch file Download diff file Expand all files Collapse all files

69
Cargo.lock generated
View file

@ -148,9 +148,9 @@ dependencies = [
[[package]] [[package]]
name = "clap" name = "clap"
version = "4.0.30" version = "4.0.32"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "656ad1e55e23d287773f7d8192c300dc715c3eeded93b3da651d11c42cfd74d2" checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39"
dependencies = [ dependencies = [
"bitflags", "bitflags",
"clap_derive", "clap_derive",
@ -163,9 +163,9 @@ dependencies = [
[[package]] [[package]]
name = "clap_complete" name = "clap_complete"
version = "4.0.6" version = "4.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7b3c9eae0de7bf8e3f904a5e40612b21fb2e2e566456d177809a48b892d24da" checksum = "10861370d2ba66b0f5989f83ebf35db6421713fd92351790e7fdd6c36774c56b"
dependencies = [ dependencies = [
"clap", "clap",
] ]
@ -194,9 +194,9 @@ dependencies = [
[[package]] [[package]]
name = "clap_mangen" name = "clap_mangen"
version = "0.2.5" version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e503c3058af0a0854668ea01db55c622482a080092fede9dd2e00a00a9436504" checksum = "904eb24d05ad587557e0f484ddce5c737c30cf81372badb16d13e41c4b8340b1"
dependencies = [ dependencies = [
"clap", "clap",
"roff", "roff",
@ -214,16 +214,15 @@ dependencies = [
[[package]] [[package]]
name = "console" name = "console"
version = "0.15.2" version = "0.15.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c050367d967ced717c04b65d8c619d863ef9292ce0c5760028655a2fb298718c" checksum = "5556015fe3aad8b968e5d4124980fbe2f6aaee7aeec6b749de1faaa2ca5d0a4c"
dependencies = [ dependencies = [
"encode_unicode", "encode_unicode",
"lazy_static", "lazy_static",
"libc", "libc",
"terminal_size 0.1.17",
"unicode-width", "unicode-width",
"winapi", "windows-sys 0.42.0",
] ]
[[package]] [[package]]
@ -312,21 +311,23 @@ dependencies = [
"signal-hook", "signal-hook",
"sys-locale", "sys-locale",
"tempfile", "tempfile",
"terminal_size 0.2.3", "terminal_size",
"tokio", "tokio",
] ]
[[package]] [[package]]
name = "crunchyroll-rs" name = "crunchyroll-rs"
version = "0.2.0" version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5032be36dc6c4757af4f28152659e7a772b48f6925f0e0ac81b5c384e291314" checksum = "5ea0bc79ecd9fafc95c6049d0b38e0e00c8d4665314ae3ee675db67b622ce3b7"
dependencies = [ dependencies = [
"aes", "aes",
"async-trait",
"cbc", "cbc",
"chrono", "chrono",
"crunchyroll-rs-internal", "crunchyroll-rs-internal",
"http", "http",
"lazy_static",
"m3u8-rs", "m3u8-rs",
"regex", "regex",
"reqwest", "reqwest",
@ -341,9 +342,9 @@ dependencies = [
[[package]] [[package]]
name = "crunchyroll-rs-internal" name = "crunchyroll-rs-internal"
version = "0.2.0" version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1e4aa1f09fd76f44329455abfac35e22c654ac782e93bc8a7d3ee1be27509f4" checksum = "66be957a34f7498e4bc324af67c58bb7ace299c214c9353281545f92dbc45d78"
dependencies = [ dependencies = [
"darling", "darling",
"quote", "quote",
@ -908,9 +909,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.138" version = "0.2.139"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8" checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
[[package]] [[package]]
name = "link-cplusplus" name = "link-cplusplus"
@ -1014,9 +1015,9 @@ dependencies = [
[[package]] [[package]]
name = "nom" name = "nom"
version = "7.1.1" version = "7.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" checksum = "e5507769c4919c998e69e49c839d9dc6e693ede4cc4290d6ad8b41d4f09c548c"
dependencies = [ dependencies = [
"memchr", "memchr",
"minimal-lexical", "minimal-lexical",
@ -1059,9 +1060,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
[[package]] [[package]]
name = "once_cell" name = "once_cell"
version = "1.16.0" version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"
[[package]] [[package]]
name = "openssl" name = "openssl"
@ -1140,9 +1141,9 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
[[package]] [[package]]
name = "portable-atomic" name = "portable-atomic"
version = "0.3.18" version = "0.3.19"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81bdd679d533107e090c2704a35982fc06302e30898e63ffa26a81155c012e92" checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b"
[[package]] [[package]]
name = "proc-macro-error" name = "proc-macro-error"
@ -1328,9 +1329,9 @@ checksum = "b833d8d034ea094b1ea68aa6d5c740e0d04bad9d16568d08ba6f76823a114316"
[[package]] [[package]]
name = "rustix" name = "rustix"
version = "0.36.5" version = "0.36.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3807b5d10909833d3e9acd1eb5fb988f79376ff10fce42937de71a449c4c588" checksum = "4feacf7db682c6c329c4ede12649cd36ecab0f3be5b7d74e6a20304725db4549"
dependencies = [ dependencies = [
"bitflags", "bitflags",
"errno", "errno",
@ -1428,18 +1429,18 @@ dependencies = [
[[package]] [[package]]
name = "serde" name = "serde"
version = "1.0.151" version = "1.0.152"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97fed41fc1a24994d044e6db6935e69511a1153b52c15eb42493b26fa87feba0" checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
dependencies = [ dependencies = [
"serde_derive", "serde_derive",
] ]
[[package]] [[package]]
name = "serde_derive" name = "serde_derive"
version = "1.0.151" version = "1.0.152"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "255abe9a125a985c05190d687b320c12f9b1f0b99445e608c21ba0782c719ad8" checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -1583,16 +1584,6 @@ dependencies = [
"winapi-util", "winapi-util",
] ]
[[package]]
name = "terminal_size"
version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df"
dependencies = [
"libc",
"winapi",
]
[[package]] [[package]]
name = "terminal_size" name = "terminal_size"
version = "0.2.3" version = "0.2.3"

61
crunchy-cli-core/Cargo.lock generated
View file

@ -148,9 +148,9 @@ dependencies = [
[[package]] [[package]]
name = "clap" name = "clap"
version = "4.0.30" version = "4.0.32"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "656ad1e55e23d287773f7d8192c300dc715c3eeded93b3da651d11c42cfd74d2" checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39"
dependencies = [ dependencies = [
"bitflags", "bitflags",
"clap_derive", "clap_derive",
@ -195,16 +195,15 @@ dependencies = [
[[package]] [[package]]
name = "console" name = "console"
version = "0.15.2" version = "0.15.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c050367d967ced717c04b65d8c619d863ef9292ce0c5760028655a2fb298718c" checksum = "5556015fe3aad8b968e5d4124980fbe2f6aaee7aeec6b749de1faaa2ca5d0a4c"
dependencies = [ dependencies = [
"encode_unicode", "encode_unicode",
"lazy_static", "lazy_static",
"libc", "libc",
"terminal_size 0.1.17",
"unicode-width", "unicode-width",
"winapi", "windows-sys 0.42.0",
] ]
[[package]] [[package]]
@ -281,21 +280,23 @@ dependencies = [
"signal-hook", "signal-hook",
"sys-locale", "sys-locale",
"tempfile", "tempfile",
"terminal_size 0.2.3", "terminal_size",
"tokio", "tokio",
] ]
[[package]] [[package]]
name = "crunchyroll-rs" name = "crunchyroll-rs"
version = "0.2.0" version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5032be36dc6c4757af4f28152659e7a772b48f6925f0e0ac81b5c384e291314" checksum = "5ea0bc79ecd9fafc95c6049d0b38e0e00c8d4665314ae3ee675db67b622ce3b7"
dependencies = [ dependencies = [
"aes", "aes",
"async-trait",
"cbc", "cbc",
"chrono", "chrono",
"crunchyroll-rs-internal", "crunchyroll-rs-internal",
"http", "http",
"lazy_static",
"m3u8-rs", "m3u8-rs",
"regex", "regex",
"reqwest", "reqwest",
@ -310,9 +311,9 @@ dependencies = [
[[package]] [[package]]
name = "crunchyroll-rs-internal" name = "crunchyroll-rs-internal"
version = "0.2.0" version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1e4aa1f09fd76f44329455abfac35e22c654ac782e93bc8a7d3ee1be27509f4" checksum = "66be957a34f7498e4bc324af67c58bb7ace299c214c9353281545f92dbc45d78"
dependencies = [ dependencies = [
"darling", "darling",
"quote", "quote",
@ -877,9 +878,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.138" version = "0.2.139"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8" checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
[[package]] [[package]]
name = "link-cplusplus" name = "link-cplusplus"
@ -983,9 +984,9 @@ dependencies = [
[[package]] [[package]]
name = "nom" name = "nom"
version = "7.1.1" version = "7.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" checksum = "e5507769c4919c998e69e49c839d9dc6e693ede4cc4290d6ad8b41d4f09c548c"
dependencies = [ dependencies = [
"memchr", "memchr",
"minimal-lexical", "minimal-lexical",
@ -1028,9 +1029,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
[[package]] [[package]]
name = "once_cell" name = "once_cell"
version = "1.16.0" version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"
[[package]] [[package]]
name = "openssl" name = "openssl"
@ -1109,9 +1110,9 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
[[package]] [[package]]
name = "portable-atomic" name = "portable-atomic"
version = "0.3.18" version = "0.3.19"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81bdd679d533107e090c2704a35982fc06302e30898e63ffa26a81155c012e92" checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b"
[[package]] [[package]]
name = "proc-macro-error" name = "proc-macro-error"
@ -1291,9 +1292,9 @@ dependencies = [
[[package]] [[package]]
name = "rustix" name = "rustix"
version = "0.36.5" version = "0.36.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3807b5d10909833d3e9acd1eb5fb988f79376ff10fce42937de71a449c4c588" checksum = "4feacf7db682c6c329c4ede12649cd36ecab0f3be5b7d74e6a20304725db4549"
dependencies = [ dependencies = [
"bitflags", "bitflags",
"errno", "errno",
@ -1391,18 +1392,18 @@ dependencies = [
[[package]] [[package]]
name = "serde" name = "serde"
version = "1.0.151" version = "1.0.152"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97fed41fc1a24994d044e6db6935e69511a1153b52c15eb42493b26fa87feba0" checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
dependencies = [ dependencies = [
"serde_derive", "serde_derive",
] ]
[[package]] [[package]]
name = "serde_derive" name = "serde_derive"
version = "1.0.151" version = "1.0.152"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "255abe9a125a985c05190d687b320c12f9b1f0b99445e608c21ba0782c719ad8" checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -1546,16 +1547,6 @@ dependencies = [
"winapi-util", "winapi-util",
] ]
[[package]]
name = "terminal_size"
version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df"
dependencies = [
"libc",
"winapi",
]
[[package]] [[package]]
name = "terminal_size" name = "terminal_size"
version = "0.2.3" version = "0.2.3"

99
crunchy-cli-core/src/cli/utils.rs
View file

@ -7,6 +7,7 @@ use std::borrow::{Borrow, BorrowMut};
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::io::Write; use std::io::Write;
use std::sync::{mpsc, Arc, Mutex}; use std::sync::{mpsc, Arc, Mutex};
use std::time::Duration;
use tokio::task::JoinSet; use tokio::task::JoinSet;
pub fn find_resolution( pub fn find_resolution(
@ -75,9 +76,37 @@ pub async fn download_segments(
let thread_segments = segs.remove(0); let thread_segments = segs.remove(0);
let thread_count = count.clone(); let thread_count = count.clone();
join_set.spawn(async move { join_set.spawn(async move {
let after_download_sender = thread_sender.clone();
// the download process is encapsulated in its own function. this is done to easily
// catch errors which get returned with `...?` and `bail!(...)` and that the thread
// itself can report that an error has occured
let download = || async move {
for (i, segment) in thread_segments.into_iter().enumerate() { for (i, segment) in thread_segments.into_iter().enumerate() {
let response = thread_client.get(&segment.url).send().await?; let mut retry_count = 0;
let mut buf = response.bytes().await?.to_vec(); let mut buf = loop {
let response = thread_client
.get(&segment.url)
.timeout(Duration::from_secs(60))
.send()
.await?;
match response.bytes().await {
Ok(b) => break b.to_vec(),
Err(e) => {
if e.is_body() {
if retry_count == 5 {
bail!("Max retry count reached ({}), multiple errors occured while receiving segment {}: {}", retry_count, num + (i * cpus), e)
}
debug!("Failed to download segment {} ({}). Retrying, {} out of 5 retries left", num + (i * cpus), e, 5 - retry_count)
} else {
bail!("{}", e)
}
}
}
retry_count += 1;
};
buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec(); buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec();
@ -89,26 +118,43 @@ pub async fn download_segments(
((*c + 1) as f64 / total_segments as f64) * 100f64, ((*c + 1) as f64 / total_segments as f64) * 100f64,
segment.url segment.url
); );
thread_sender.send((num + (i * cpus), buf))?;
thread_sender.send((num as i32 + (i * cpus) as i32, buf))?;
*c += 1; *c += 1;
} }
Ok(()) Ok(())
}); };
let result = download().await;
if result.is_err() {
after_download_sender.send((-1 as i32, vec![]))?;
} }
let mut data_pos = 0usize; result
let mut buf: BTreeMap<usize, Vec<u8>> = BTreeMap::new(); });
loop { }
// is always `Some` because `sender` does not get dropped when all threads are finished // drop the sender already here so it does not outlive all (download) threads which are the only
let (pos, bytes) = receiver.recv().unwrap(); // real consumers of it
drop(sender);
// this is the main loop which writes the data. it uses a BTreeMap as a buffer as the write
// happens synchronized. the download consist of multiple segments. the map keys are representing
// the segment number and the values the corresponding bytes
let mut data_pos = 0;
let mut buf: BTreeMap<i32, Vec<u8>> = BTreeMap::new();
for (pos, bytes) in receiver.iter() {
// if the position is lower than 0, an error occured in the sending download thread
if pos < 0 {
break
}
if let Some(p) = &progress { if let Some(p) = &progress {
let progress_len = p.length().unwrap(); let progress_len = p.length().unwrap();
let estimated_segment_len = (variant_data.bandwidth / 8) let estimated_segment_len = (variant_data.bandwidth / 8)
* segments * segments
.get(pos) .get(pos as usize)
.unwrap() .unwrap()
.length .length
.unwrap_or_default() .unwrap_or_default()
@ -119,24 +165,41 @@ pub async fn download_segments(
p.inc(bytes_len) p.inc(bytes_len)
} }
// check if the currently sent bytes are the next in the buffer. if so, write them directly
// to the target without first adding them to the buffer.
// if not, add them to the buffer
if data_pos == pos { if data_pos == pos {
writer.write_all(bytes.borrow())?; writer.write_all(bytes.borrow())?;
data_pos += 1; data_pos += 1;
} else { } else {
buf.insert(pos, bytes); buf.insert(pos, bytes);
} }
// check if the buffer contains the next segment(s)
while let Some(b) = buf.remove(&data_pos) {
writer.write_all(b.borrow())?;
data_pos += 1;
}
}
// if any error has occured while downloading it gets returned here
while let Some(joined) = join_set.join_next().await {
joined??
}
// write the remaining buffer, if existent
while let Some(b) = buf.remove(&data_pos) { while let Some(b) = buf.remove(&data_pos) {
writer.write_all(b.borrow())?; writer.write_all(b.borrow())?;
data_pos += 1; data_pos += 1;
} }
if *count.lock().unwrap() >= total_segments && buf.is_empty() { if !buf.is_empty() {
break; bail!(
} "Download buffer is not empty. Remaining segments: {}",
} buf.into_keys()
.map(|k| k.to_string())
while let Some(joined) = join_set.join_next().await { .collect::<Vec<String>>()
joined?? .join(", ")
)
} }
Ok(()) Ok(())

5
crunchy-cli-core/src/lib.rs
View file

@ -189,8 +189,9 @@ async fn create_ctx(cli: &Cli) -> Result<Context> {
} }
async fn crunchyroll_session(cli: &Cli) -> Result<Crunchyroll> { async fn crunchyroll_session(cli: &Cli) -> Result<Crunchyroll> {
let mut builder = Crunchyroll::builder(); let mut builder = Crunchyroll::builder()
builder.locale(cli.lang.clone().unwrap_or_else(system_locale)); .locale(cli.lang.clone().unwrap_or_else(system_locale))
.stabilization_locales(true);
let login_methods_count = cli.login_method.credentials.is_some() as u8 let login_methods_count = cli.login_method.credentials.is_some() as u8
+ cli.login_method.etp_rt.is_some() as u8 + cli.login_method.etp_rt.is_some() as u8