From 14e71c05b84ff7458478443b2de4ee71273c61a3 Mon Sep 17 00:00:00 2001 From: bytedream Date: Thu, 16 Nov 2023 13:51:30 +0100 Subject: [PATCH 001/151] Fix aur binary checksums (#266) --- .github/workflows/publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f777f9e..43ea795 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -43,8 +43,8 @@ jobs: curl -LO https://github.com/crunchy-labs/crunchy-cli/releases/download/${{ github.ref_name }}/crunchy-cli-${{ github.ref_name }}-completions.zip curl -LO https://github.com/crunchy-labs/crunchy-cli/releases/download/${{ github.ref_name }}/crunchy-cli-${{ github.ref_name }}-manpages.zip curl -LO https://raw.githubusercontent.com/crunchy-labs/crunchy-cli/${{ github.ref_name }}/LICENSE - echo "CRUNCHY_CLI_BIN_x86_64_SHA256=$(sha256sum crunchy-cli-${{ github.ref_name }}-x86_64-linux | cut -f 1 -d ' ')" >> $GITHUB_ENV - echo "CRUNCHY_CLI_BIN_aarch64_SHA256=$(sha256sum crunchy-cli-${{ github.ref_name }}-aarch64-linux | cut -f 1 -d ' ')" >> $GITHUB_ENV + echo "CRUNCHY_CLI_BIN_x86_64_SHA256=$(sha256sum crunchy-cli-${{ github.ref_name }}-linux-x86_64 | cut -f 1 -d ' ')" >> $GITHUB_ENV + echo "CRUNCHY_CLI_BIN_aarch64_SHA256=$(sha256sum crunchy-cli-${{ github.ref_name }}-linux-aarch64 | cut -f 1 -d ' ')" >> $GITHUB_ENV echo "CRUNCHY_CLI_BIN_COMPLETIONS_SHA256=$(sha256sum crunchy-cli-${{ github.ref_name }}-completions.zip | cut -f 1 -d ' ')" >> $GITHUB_ENV echo "CRUNCHY_CLI_BIN_MANPAGES_SHA256=$(sha256sum crunchy-cli-${{ github.ref_name }}-manpages.zip | cut -f 1 -d ' ')" >> $GITHUB_ENV echo "CRUNCHY_CLI_BIN_LICENSE_SHA256=$(sha256sum LICENSE | cut -f 1 -d ' ')" >> $GITHUB_ENV From 2c370939595149c20ed6ba2fbde3274134be0aad Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 19 Nov 2023 19:24:15 +0100 Subject: [PATCH 002/151] Manually burn-in subtitles only if no pre-burned video is available (#268) --- crunchy-cli-core/src/archive/command.rs | 3 +- crunchy-cli-core/src/download/command.rs | 56 +++++++++++++++++++++--- crunchy-cli-core/src/utils/ffmpeg.rs | 2 + crunchy-cli-core/src/utils/format.rs | 4 ++ crunchy-cli-core/src/utils/video.rs | 37 ++++++++++++---- 5 files changed, 88 insertions(+), 14 deletions(-) diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index 3585e9c..006bbfa 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -248,7 +248,8 @@ async fn get_format( for single_format in single_formats { let stream = single_format.stream().await?; - let Some((video, audio)) = variant_data_from_stream(&stream, &archive.resolution).await? + let Some((video, audio, _)) = + variant_data_from_stream(&stream, &archive.resolution, None).await? else { if single_format.is_episode() { bail!( diff --git a/crunchy-cli-core/src/download/command.rs b/crunchy-cli-core/src/download/command.rs index 82cb8f8..cf1a049 100644 --- a/crunchy-cli-core/src/download/command.rs +++ b/crunchy-cli-core/src/download/command.rs @@ -1,7 +1,7 @@ use crate::download::filter::DownloadFilter; use crate::utils::context::Context; use crate::utils::download::{DownloadBuilder, DownloadFormat}; -use crate::utils::ffmpeg::FFmpegPreset; +use crate::utils::ffmpeg::{FFmpegPreset, SOFTSUB_CONTAINERS}; use crate::utils::filter::Filter; use crate::utils::format::{Format, SingleFormat}; use crate::utils::log::progress; @@ -149,6 +149,25 @@ impl Execute for Download { async fn execute(self, ctx: Context) -> Result<()> { let mut parsed_urls = vec![]; + let output_supports_softsubs = SOFTSUB_CONTAINERS.contains( + &Path::new(&self.output) + .extension() + .unwrap_or_default() + .to_string_lossy() + .as_ref(), + ); + let special_output_supports_softsubs = if let Some(so) = &self.output_specials { + SOFTSUB_CONTAINERS.contains( + &Path::new(so) + .extension() + .unwrap_or_default() + .to_string_lossy() + .as_ref(), + ) + } else { + output_supports_softsubs + }; + for (i, url) in self.urls.clone().into_iter().enumerate() { let progress_handler = progress!("Parsing url {}", i + 1); match parse_url(&ctx.crunchy, url.clone(), true).await { @@ -190,7 +209,18 @@ impl Execute for Download { // the vec contains always only one item let single_format = single_formats.remove(0); - let (download_format, format) = get_format(&self, &single_format).await?; + let (download_format, format) = get_format( + &self, + &single_format, + if self.force_hardsub { + true + } else if single_format.is_special() { + !special_output_supports_softsubs + } else { + !output_supports_softsubs + }, + ) + .await?; let mut downloader = download_builder.clone().build(); downloader.add_format(download_format); @@ -227,9 +257,19 @@ impl Execute for Download { async fn get_format( download: &Download, single_format: &SingleFormat, + try_peer_hardsubs: bool, ) -> Result<(DownloadFormat, Format)> { let stream = single_format.stream().await?; - let Some((video, audio)) = variant_data_from_stream(&stream, &download.resolution).await? + let Some((video, audio, contains_hardsub)) = variant_data_from_stream( + &stream, + &download.resolution, + if try_peer_hardsubs { + download.subtitle.clone() + } else { + None + }, + ) + .await? else { if single_format.is_episode() { bail!( @@ -250,7 +290,9 @@ async fn get_format( } }; - let subtitle = if let Some(subtitle_locale) = &download.subtitle { + let subtitle = if contains_hardsub { + None + } else if let Some(subtitle_locale) = &download.subtitle { stream.subtitles.get(subtitle_locale).cloned() } else { None @@ -266,7 +308,7 @@ async fn get_format( )] }), }; - let format = Format::from_single_formats(vec![( + let mut format = Format::from_single_formats(vec![( single_format.clone(), video, subtitle.map_or(vec![], |s| { @@ -276,6 +318,10 @@ async fn get_format( )] }), )]); + if contains_hardsub { + let (_, subs) = format.locales.get_mut(0).unwrap(); + subs.push(download.subtitle.clone().unwrap()) + } Ok((download_format, format)) } diff --git a/crunchy-cli-core/src/utils/ffmpeg.rs b/crunchy-cli-core/src/utils/ffmpeg.rs index 787c79a..7d77833 100644 --- a/crunchy-cli-core/src/utils/ffmpeg.rs +++ b/crunchy-cli-core/src/utils/ffmpeg.rs @@ -2,6 +2,8 @@ use lazy_static::lazy_static; use regex::Regex; use std::str::FromStr; +pub const SOFTSUB_CONTAINERS: [&str; 3] = ["mkv", "mov", "mp4"]; + #[derive(Clone, Debug, Eq, PartialEq)] pub enum FFmpegPreset { Predefined(FFmpegCodec, Option, FFmpegQuality), diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index 4c8d3c8..05ac232 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -172,6 +172,10 @@ impl SingleFormat { pub fn is_episode(&self) -> bool { matches!(self.source, MediaCollection::Episode(_)) } + + pub fn is_special(&self) -> bool { + self.sequence_number == 0.0 || self.sequence_number.fract() != 0.0 + } } struct SingleFormatCollectionEpisodeKey(f32); diff --git a/crunchy-cli-core/src/utils/video.rs b/crunchy-cli-core/src/utils/video.rs index 5b3eaeb..0ae4ba4 100644 --- a/crunchy-cli-core/src/utils/video.rs +++ b/crunchy-cli-core/src/utils/video.rs @@ -1,32 +1,47 @@ -use anyhow::Result; +use anyhow::{bail, Result}; use crunchyroll_rs::media::{Resolution, Stream, VariantData}; use crunchyroll_rs::Locale; pub async fn variant_data_from_stream( stream: &Stream, resolution: &Resolution, -) -> Result> { + subtitle: Option, +) -> Result> { // sometimes Crunchyroll marks episodes without real subtitles that they have subtitles and // reports that only hardsub episode are existing. the following lines are trying to prevent // potential errors which might get caused by this incorrect reporting // (https://github.com/crunchy-labs/crunchy-cli/issues/231) let mut hardsub_locales = stream.streaming_hardsub_locales(); - let hardsub_locale = if !hardsub_locales.contains(&Locale::Custom("".to_string())) + let (hardsub_locale, mut contains_hardsub) = if !hardsub_locales + .contains(&Locale::Custom("".to_string())) && !hardsub_locales.contains(&Locale::Custom(":".to_string())) { // if only one hardsub locale exists, assume that this stream doesn't really contains hardsubs if hardsub_locales.len() == 1 { - Some(hardsub_locales.remove(0)) + (Some(hardsub_locales.remove(0)), false) } else { // fallback to `None`. this should trigger an error message in `stream.dash_streaming_data` // that the requested stream is not available - None + (None, false) } } else { - None + let hardsubs_requested = subtitle.is_some(); + (subtitle, hardsubs_requested) }; - let mut streaming_data = stream.dash_streaming_data(hardsub_locale).await?; + let mut streaming_data = match stream.dash_streaming_data(hardsub_locale).await { + Ok(data) => data, + Err(e) => { + // the error variant is only `crunchyroll_rs::error::Error::Input` when the requested + // hardsub is not available + if let crunchyroll_rs::error::Error::Input { .. } = e { + contains_hardsub = false; + stream.dash_streaming_data(None).await? + } else { + bail!(e) + } + } + }; streaming_data .0 .sort_by(|a, b| a.bandwidth.cmp(&b.bandwidth).reverse()); @@ -42,5 +57,11 @@ pub async fn variant_data_from_stream( .into_iter() .find(|v| resolution.height == v.resolution.height), }; - Ok(video_variant.map(|v| (v, streaming_data.1.first().unwrap().clone()))) + Ok(video_variant.map(|v| { + ( + v, + streaming_data.1.first().unwrap().clone(), + contains_hardsub, + ) + })) } From 440ccd99b5482af43b0cec222d49a45ddd5b43e1 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 20 Nov 2023 22:05:06 +0100 Subject: [PATCH 003/151] Update dependencies and version --- Cargo.lock | 82 +++++++++++++------------ Cargo.toml | 4 +- crunchy-cli-core/Cargo.toml | 6 +- crunchy-cli-core/src/archive/filter.rs | 8 +-- crunchy-cli-core/src/download/filter.rs | 4 +- crunchy-cli-core/src/search/format.rs | 2 +- 6 files changed, 54 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8608f0e..897731f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -238,9 +238,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.7" +version = "4.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b" +checksum = "2275f18819641850fa26c89acc84d465c1bf91ce57bc2748b28c420473352f64" dependencies = [ "clap_builder", "clap_derive", @@ -248,9 +248,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.7" +version = "4.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663" +checksum = "07cdf1b148b25c1e1f7a42225e30a0d99a615cd4637eae7365548dd4529b95bc" dependencies = [ "anstream", "anstyle", @@ -369,7 +369,7 @@ dependencies = [ [[package]] name = "crunchy-cli" -version = "3.1.0" +version = "3.1.1" dependencies = [ "chrono", "clap", @@ -382,7 +382,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.1.0" +version = "3.1.1" dependencies = [ "anyhow", "async-trait", @@ -412,9 +412,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.6.3" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19bdd19b3f1b26e1a45ed32f7b2db68981111a9f5e9551b2d225bf470dca1e11" +checksum = "cc4ce434784eee7892ad8c3d1ecaea0c0858db51bbb295b474db38c256e8d2fb" dependencies = [ "aes", "async-trait", @@ -423,7 +423,6 @@ dependencies = [ "crunchyroll-rs-internal", "dash-mpd", "futures-util", - "http", "lazy_static", "m3u8-rs", "regex", @@ -439,9 +438,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.6.3" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0a12fb14fd65dede6da7dad3e4c434f6aee9c18cf2bae0d2cc5021cd5a29fec" +checksum = "be840f8cf2ce6afc9a9eae268d41423093141ec88f664a515d5ed2a85a66fb60" dependencies = [ "darling", "quote", @@ -505,9 +504,9 @@ dependencies = [ [[package]] name = "dash-mpd" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad33027a1ac2f37def4c33a5987a8e047fd34f77ff7cabc14bd437aa6d8d4dd2" +checksum = "5471fc46c0b229c8f2308d83be857c745c9f06cc83a433d7047909722e0453b4" dependencies = [ "base64", "base64-serde", @@ -609,9 +608,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.5" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" +checksum = "f258a7194e7f7c2a7837a8913aeab7fd8c383457034fa20ce4dd3dcb813e8eb8" dependencies = [ "libc", "windows-sys 0.48.0", @@ -655,9 +654,12 @@ dependencies = [ [[package]] name = "fs-err" -version = "2.9.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0845fa252299212f0389d64ba26f34fa32cfe41588355f21ed507c59a0f64541" +checksum = "88a41f105fe1d5b6b34b2055e3dc59bb79b46b48b2040b9e6c7b4b5de097aa41" +dependencies = [ + "autocfg", +] [[package]] name = "fs2" @@ -721,9 +723,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" dependencies = [ "cfg-if", "libc", @@ -738,9 +740,9 @@ checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" [[package]] name = "h2" -version = "0.3.21" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" dependencies = [ "bytes", "fnv", @@ -748,7 +750,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 1.9.3", + "indexmap 2.1.0", "slab", "tokio", "tokio-util", @@ -787,9 +789,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" dependencies = [ "bytes", "fnv", @@ -1039,9 +1041,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" +checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" [[package]] name = "log" @@ -1443,9 +1445,9 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.38.21" +version = "0.38.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3" +checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" dependencies = [ "bitflags 2.4.1", "errno", @@ -1456,9 +1458,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.8" +version = "0.21.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446e14c5cda4f3f30fe71863c34ec70f5ac79d6087097ad0bb433e1be5edf04c" +checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" dependencies = [ "log", "ring", @@ -1480,9 +1482,9 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ "base64", ] @@ -1547,18 +1549,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.191" +version = "1.0.192" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a834c4821019838224821468552240d4d95d14e751986442c816572d39a080c9" +checksum = "bca2a08484b285dcb282d0f67b26cadc0df8b19f8c12502c13d966bf9482f001" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.191" +version = "1.0.192" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46fa52d5646bce91b680189fe5b1c049d2ea38dabb4e2e7c8d00ca12cfbfbcfd" +checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1" dependencies = [ "proc-macro2", "quote", @@ -1810,9 +1812,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.33.0" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" +checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" dependencies = [ "backtrace", "bytes", @@ -1827,9 +1829,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index a0c1871..098c048 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.1.0" +version = "3.1.1" edition = "2021" license = "MIT" @@ -18,7 +18,7 @@ openssl = ["openssl-tls"] openssl-static = ["openssl-tls-static"] [dependencies] -tokio = { version = "1.33", features = ["macros", "rt-multi-thread", "time"], default-features = false } +tokio = { version = "1.34", features = ["macros", "rt-multi-thread", "time"], default-features = false } native-tls-crate = { package = "native-tls", version = "0.2.11", optional = true } diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 9103831..5771888 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.1.0" +version = "3.1.1" edition = "2021" license = "MIT" @@ -16,7 +16,7 @@ anyhow = "1.0" async-trait = "0.1" clap = { version = "4.4", features = ["derive", "string"] } chrono = "0.4" -crunchyroll-rs = { version = "0.6.3", features = ["dash-stream"] } +crunchyroll-rs = { version = "0.7.0", features = ["dash-stream"] } ctrlc = "3.4" dialoguer = { version = "0.11", default-features = false } dirs = "5.0" @@ -34,7 +34,7 @@ serde_plain = "1.0" shlex = "1.2" sys-locale = "0.3" tempfile = "3.8" -tokio = { version = "1.33", features = ["macros", "rt-multi-thread", "time"] } +tokio = { version = "1.34", features = ["macros", "rt-multi-thread", "time"] } rustls-native-certs = { version = "0.6", optional = true } [build-dependencies] diff --git a/crunchy-cli-core/src/archive/filter.rs b/crunchy-cli-core/src/archive/filter.rs index 01612b9..1777072 100644 --- a/crunchy-cli-core/src/archive/filter.rs +++ b/crunchy-cli-core/src/archive/filter.rs @@ -223,7 +223,7 @@ impl Filter for ArchiveFilter { "Season {} is only available with {} audio until episode {} ({})", season.season_number, season_locale.unwrap_or(Locale::ja_JP), - last_episode.episode_number, + last_episode.sequence_number, last_episode.title ) } @@ -278,7 +278,7 @@ impl Filter for ArchiveFilter { if !missing_audio.is_empty() { warn!( "Episode {} is not available with {} audio", - episode.episode_number, + episode.sequence_number, missing_audio .into_iter() .map(|l| l.to_string()) @@ -298,7 +298,7 @@ impl Filter for ArchiveFilter { { warn!( "Episode {} is not available with {} subtitles", - episode.episode_number, + episode.sequence_number, missing_subtitles .into_iter() .map(|l| l.to_string()) @@ -343,7 +343,7 @@ impl Filter for ArchiveFilter { if relative_episode_number.is_none() || relative_sequence_number.is_none() { warn!( "Failed to get relative episode number for episode {} ({}) of {} season {}", - episode.episode_number, + episode.sequence_number, episode.title, episode.series_title, episode.season_number, diff --git a/crunchy-cli-core/src/download/filter.rs b/crunchy-cli-core/src/download/filter.rs index fb2e563..51f1ad8 100644 --- a/crunchy-cli-core/src/download/filter.rs +++ b/crunchy-cli-core/src/download/filter.rs @@ -158,7 +158,7 @@ impl Filter for DownloadFilter { { let error_message = format!( "Episode {} ({}) of {} season {} is not available with {} audio", - episode.episode_number, + episode.sequence_number, episode.title, episode.series_title, episode.season_number, @@ -234,7 +234,7 @@ impl Filter for DownloadFilter { if relative_episode_number.is_none() || relative_sequence_number.is_none() { warn!( "Failed to get relative episode number for episode {} ({}) of {} season {}", - episode.episode_number, + episode.sequence_number, episode.title, episode.series_title, episode.season_number, diff --git a/crunchy-cli-core/src/search/format.rs b/crunchy-cli-core/src/search/format.rs index f9746b1..156bd95 100644 --- a/crunchy-cli-core/src/search/format.rs +++ b/crunchy-cli-core/src/search/format.rs @@ -70,7 +70,7 @@ impl From<&Episode> for FormatEpisode { title: value.title.clone(), description: value.description.clone(), locale: value.audio_locale.clone(), - number: value.episode_number, + number: value.episode_number.unwrap_or_default(), sequence_number: value.sequence_number, duration: value.duration.num_milliseconds(), air_date: value.episode_air_date.timestamp(), From d5df3df95f5972495c8c317f0fb243c32ede0b37 Mon Sep 17 00:00:00 2001 From: bytedream Date: Fri, 1 Dec 2023 01:02:53 +0100 Subject: [PATCH 004/151] Fix fixed subtitle formatting and sorting (#272) --- crunchy-cli-core/src/utils/download.rs | 33 +++++++++++++++++++------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index 945cc14..c65b4d2 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -17,6 +17,7 @@ use std::env; use std::io::Write; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; +use std::str::FromStr; use std::sync::Arc; use std::time::Duration; use tempfile::TempPath; @@ -788,8 +789,10 @@ pub fn get_video_length(path: &Path) -> Result { /// [crunchy-labs/crunchy-cli#208](https://github.com/crunchy-labs/crunchy-cli/issues/208) for more /// information. fn fix_subtitles(raw: &mut Vec, max_length: NaiveTime) { - let re = Regex::new(r"^Dialogue:\s\d+,(?P\d+:\d+:\d+\.\d+),(?P\d+:\d+:\d+\.\d+),") - .unwrap(); + let re = Regex::new( + r"^Dialogue:\s(?P\d+),(?P\d+:\d+:\d+\.\d+),(?P\d+:\d+:\d+\.\d+),", + ) + .unwrap(); // chrono panics if we try to format NaiveTime with `%2f` and the nano seconds has more than 2 // digits so them have to be reduced manually to avoid the panic @@ -804,9 +807,9 @@ fn fix_subtitles(raw: &mut Vec, max_length: NaiveTime) { formatted_time.split_at(2).0.parse().unwrap() } ) + .split_off(1) // <- in the ASS spec, the hour has only one digit } - let length_as_string = format_naive_time(max_length); let mut entries = (vec![], vec![]); let mut as_lines: Vec = String::from_utf8_lossy(raw.as_slice()) @@ -818,21 +821,33 @@ fn fix_subtitles(raw: &mut Vec, max_length: NaiveTime) { if line.trim() == "[Script Info]" { line.push_str("\nScaledBorderAndShadow: yes") } else if let Some(capture) = re.captures(line) { - let start = capture.name("start").map_or(NaiveTime::default(), |s| { + let mut start = capture.name("start").map_or(NaiveTime::default(), |s| { NaiveTime::parse_from_str(s.as_str(), "%H:%M:%S.%f").unwrap() }); - let end = capture.name("end").map_or(NaiveTime::default(), |s| { - NaiveTime::parse_from_str(s.as_str(), "%H:%M:%S.%f").unwrap() + let mut end = capture.name("end").map_or(NaiveTime::default(), |e| { + NaiveTime::parse_from_str(e.as_str(), "%H:%M:%S.%f").unwrap() }); - if end > max_length { + if start > max_length || end > max_length { + let layer = capture + .name("layer") + .map_or(0, |l| i32::from_str(l.as_str()).unwrap()); + + if start > max_length { + start = max_length; + } + if start > max_length || end > max_length { + end = max_length; + } + *line = re .replace( line, format!( - "Dialogue: {},{},", + "Dialogue: {},{},{},", + layer, format_naive_time(start), - &length_as_string + format_naive_time(end) ), ) .to_string() From 8f77028fcb933137d66c4f4fb2e0d3b7f4454843 Mon Sep 17 00:00:00 2001 From: bytedream Date: Fri, 1 Dec 2023 01:17:49 +0100 Subject: [PATCH 005/151] Show error message instead of panicking when capturing video length of invalid file (#258) --- crunchy-cli-core/src/utils/download.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index c65b4d2..de1aed0 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -768,7 +768,13 @@ pub fn get_video_length(path: &Path) -> Result { .args(["-i", path.to_str().unwrap()]) .output()?; let ffmpeg_output = String::from_utf8(ffmpeg.stderr)?; - let caps = video_length.captures(ffmpeg_output.as_str()).unwrap(); + let caps = video_length.captures(ffmpeg_output.as_str()).map_or( + Err(anyhow::anyhow!( + "failed to get video length: {}", + ffmpeg_output + )), + Ok, + )?; Ok(NaiveTime::parse_from_str(caps.name("time").unwrap().as_str(), "%H:%M:%S%.f").unwrap()) } From 9ca3b79291da1d4d4f3f6abb4c0728f641f75ae7 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 3 Dec 2023 00:15:57 +0100 Subject: [PATCH 006/151] Fix spelling --- crunchy-cli-core/src/utils/ffmpeg.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/crunchy-cli-core/src/utils/ffmpeg.rs b/crunchy-cli-core/src/utils/ffmpeg.rs index 7d77833..39678dc 100644 --- a/crunchy-cli-core/src/utils/ffmpeg.rs +++ b/crunchy-cli-core/src/utils/ffmpeg.rs @@ -229,7 +229,7 @@ impl FFmpegPreset { quality = Some(q) } else { return Err(format!( - "'{}' is not a valid ffmpeg preset (unknown token '{}'", + "'{}' is not a valid ffmpeg preset (unknown token '{}')", s, token )); } @@ -286,12 +286,10 @@ impl FFmpegPreset { output.extend(["-c:v", "h264_nvenc", "-c:a", "copy"]) } FFmpegHwAccel::Apple => { - // Apple's Video Toolbox encoders ignore `-crf`, - // use `-q:v` instead. It's on a scale of 1-100, - // 100 being lossless. Just did some math - // ((-a/51+1)*99+1 where `a` is the old crf value) - // so these settings very likely need some more - // tweeking. + // Apple's Video Toolbox encoders ignore `-crf`, use `-q:v` + // instead. It's on a scale of 1-100, 100 being lossless. Just + // did some math ((-a/51+1)*99+1 where `a` is the old crf value) + // so these settings very likely need some more tweaking match quality { FFmpegQuality::Lossless => output.extend(["-q:v", "65"]), FFmpegQuality::Normal => (), From 9487dd3dbffd4646713ce4e0ec1d6210fc1a1f14 Mon Sep 17 00:00:00 2001 From: bytedream Date: Fri, 8 Dec 2023 22:27:12 +0100 Subject: [PATCH 007/151] Show ffmpeg progress (#270) --- Cargo.lock | 1 + crunchy-cli-core/Cargo.toml | 5 +- crunchy-cli-core/src/utils/download.rs | 142 +++++++++++++++++++------ crunchy-cli-core/src/utils/os.rs | 60 +++++++++++ 4 files changed, 176 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 897731f..fa5723f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -397,6 +397,7 @@ dependencies = [ "indicatif", "lazy_static", "log", + "nix", "num_cpus", "regex", "reqwest", diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 5771888..4f0912f 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -34,8 +34,11 @@ serde_plain = "1.0" shlex = "1.2" sys-locale = "0.3" tempfile = "3.8" -tokio = { version = "1.34", features = ["macros", "rt-multi-thread", "time"] } +tokio = { version = "1.34", features = ["io-util", "macros", "net", "rt-multi-thread", "time"] } rustls-native-certs = { version = "0.6", optional = true } +[target.'cfg(not(target_os = "windows"))'.dependencies] +nix = { version = "0.27", features = ["fs"] } + [build-dependencies] chrono = "0.4" diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index de1aed0..c154103 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -1,16 +1,14 @@ use crate::utils::context::Context; use crate::utils::ffmpeg::FFmpegPreset; -use crate::utils::log::progress; -use crate::utils::os::{is_special_file, temp_directory, tempfile}; +use crate::utils::os::{is_special_file, temp_directory, temp_named_pipe, tempfile}; use anyhow::{bail, Result}; use chrono::NaiveTime; use crunchyroll_rs::media::{Subtitle, VariantData, VariantSegment}; use crunchyroll_rs::Locale; -use indicatif::{ProgressBar, ProgressFinish, ProgressStyle}; +use indicatif::{ProgressBar, ProgressDrawTarget, ProgressFinish, ProgressStyle}; use log::{debug, warn, LevelFilter}; use regex::Regex; -use std::borrow::Borrow; -use std::borrow::BorrowMut; +use std::borrow::{Borrow, BorrowMut}; use std::cmp::Ordering; use std::collections::BTreeMap; use std::env; @@ -21,6 +19,7 @@ use std::str::FromStr; use std::sync::Arc; use std::time::Duration; use tempfile::TempPath; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader}; use tokio::sync::mpsc::unbounded_channel; use tokio::sync::Mutex; use tokio::task::JoinSet; @@ -177,15 +176,19 @@ impl Downloader { let mut videos = vec![]; let mut audios = vec![]; let mut subtitles = vec![]; + let mut max_frames = 0f64; + let fmt_space = self + .formats + .iter() + .flat_map(|f| { + f.audios + .iter() + .map(|(_, locale)| format!("Downloading {} audio", locale).len()) + }) + .max() + .unwrap(); for (i, format) in self.formats.iter().enumerate() { - let fmt_space = format - .audios - .iter() - .map(|(_, locale)| format!("Downloading {} audio", locale).len()) - .max() - .unwrap(); - let video_path = self .download_video( ctx, @@ -232,7 +235,11 @@ impl Downloader { None }; - let len = get_video_length(&video_path)?; + let (len, fps) = get_video_stats(&video_path)?; + let frames = len.signed_duration_since(NaiveTime::MIN).num_seconds() as f64 * fps; + if frames > max_frames { + max_frames = frames; + } for (subtitle, not_cc) in format.subtitles.iter() { if let Some(pb) = &progress_spinner { let mut progress_message = pb.message(); @@ -337,8 +344,14 @@ impl Downloader { } let (input_presets, mut output_presets) = self.ffmpeg_preset.into_input_output_args(); + let fifo = temp_named_pipe()?; - let mut command_args = vec!["-y".to_string(), "-hide_banner".to_string()]; + let mut command_args = vec![ + "-y".to_string(), + "-hide_banner".to_string(), + "-vstats_file".to_string(), + fifo.name(), + ]; command_args.extend(input_presets); command_args.extend(input); command_args.extend(maps); @@ -433,8 +446,6 @@ impl Downloader { } } - let progress_handler = progress!("Generating output file"); - let ffmpeg = Command::new("ffmpeg") // pass ffmpeg stdout to real stdout only if output file is stdout .stdout(if dst.to_str().unwrap() == "-" { @@ -444,14 +455,26 @@ impl Downloader { }) .stderr(Stdio::piped()) .args(command_args) - .output()?; - if !ffmpeg.status.success() { - bail!("{}", String::from_utf8_lossy(ffmpeg.stderr.as_slice())) + .spawn()?; + let ffmpeg_progress = tokio::spawn(async move { + ffmpeg_progress( + max_frames as u64, + fifo, + format!("{:<1$}", "Generating output file", fmt_space + 1), + ) + .await + }); + + let result = ffmpeg.wait_with_output()?; + if !result.status.success() { + bail!("{}", String::from_utf8_lossy(result.stderr.as_slice())) + } + ffmpeg_progress.abort(); + match ffmpeg_progress.await { + Ok(r) => Ok(r?), + Err(e) if e.is_cancelled() => Ok(()), + Err(e) => Err(anyhow::Error::from(e)), } - - progress_handler.stop("Output file generated"); - - Ok(()) } async fn check_free_space( @@ -752,13 +775,10 @@ fn estimate_variant_file_size(variant_data: &VariantData, segments: &[VariantSeg (variant_data.bandwidth / 8) * segments.iter().map(|s| s.length.as_secs()).sum::() } -/// Get the length of a video. This is required because sometimes subtitles have an unnecessary entry -/// long after the actual video ends with artificially extends the video length on some video players. -/// To prevent this, the video length must be hard set. See -/// [crunchy-labs/crunchy-cli#32](https://github.com/crunchy-labs/crunchy-cli/issues/32) for more -/// information. -pub fn get_video_length(path: &Path) -> Result { +/// Get the length and fps of a video. +pub fn get_video_stats(path: &Path) -> Result<(NaiveTime, f64)> { let video_length = Regex::new(r"Duration:\s(?P