From 7c3bbfc1737ecdaf291d25ca817f1d35eccd21e3 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 4 Dec 2022 18:54:19 +0100 Subject: [PATCH 001/395] Wait until buffer is empty when downloading --- crunchy-cli-core/src/cli/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index 0bf6b45..758636d 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -169,7 +169,7 @@ pub async fn download_segments( data_pos += 1; } - if *count.lock().unwrap() >= total_segments { + if *count.lock().unwrap() >= total_segments && buf.is_empty() { break; } } From 285d27772c10b288022bda19f4b027505f96c284 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 4 Dec 2022 18:54:22 +0100 Subject: [PATCH 002/395] Add manually .ass file editing to fix #32 --- crunchy-cli-core/src/cli/archive.rs | 108 ++++++++++++++++++++++------ 1 file changed, 86 insertions(+), 22 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index d4b0a5d..d586725 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -8,6 +8,7 @@ use crate::utils::parse::{parse_url, UrlFilter}; use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number}; use crate::Execute; use anyhow::{bail, Result}; +use chrono::{NaiveTime, Timelike}; use crunchyroll_rs::media::{Resolution, StreamSubtitle}; use crunchyroll_rs::{Locale, Media, MediaCollection, Series}; use log::{debug, error, info}; @@ -113,7 +114,12 @@ impl Execute for Archive { fn pre_check(&self) -> Result<()> { if !has_ffmpeg() { bail!("FFmpeg is needed to run this command") - } else if PathBuf::from(&self.output).extension().unwrap_or_default().to_string_lossy() != "mkv" { + } else if PathBuf::from(&self.output) + .extension() + .unwrap_or_default() + .to_string_lossy() + != "mkv" + { bail!("File extension is not '.mkv'. Currently only matroska / '.mkv' files are supported") } @@ -276,9 +282,18 @@ impl Execute for Archive { } } + let (primary_video, _) = video_paths.get(0).unwrap(); + let primary_video_length = get_video_length(primary_video.to_path_buf()).unwrap(); for subtitle in subtitles { - subtitle_paths - .push((download_subtitle(&self, subtitle.clone()).await?, subtitle)) + subtitle_paths.push(( + download_subtitle( + &self, + subtitle.clone(), + primary_video_length + ) + .await?, + subtitle, + )) } generate_mkv(&self, path, video_paths, audio_paths, subtitle_paths)? @@ -403,15 +418,20 @@ async fn download_video(ctx: &Context, format: &Format, only_audio: bool) -> Res Ok(path) } -async fn download_subtitle(archive: &Archive, subtitle: StreamSubtitle) -> Result { +async fn download_subtitle( + archive: &Archive, + subtitle: StreamSubtitle, + max_length: NaiveTime, +) -> Result { let tempfile = tempfile(".ass")?; let (mut file, path) = tempfile.into_parts(); let mut buf = vec![]; subtitle.write_to(&mut buf).await?; if !archive.no_subtitle_optimizations { - buf = fix_subtitle(buf) + buf = fix_subtitle_look_and_feel(buf) } + buf = fix_subtitle_length(buf, max_length); file.write_all(buf.as_slice())?; @@ -421,7 +441,7 @@ async fn download_subtitle(archive: &Archive, subtitle: StreamSubtitle) -> Resul /// Add `ScaledBorderAndShadows: yes` to subtitles; without it they look very messy on some video /// players. See [crunchy-labs/crunchy-cli#66](https://github.com/crunchy-labs/crunchy-cli/issues/66) /// for more information. -fn fix_subtitle(raw: Vec) -> Vec { +fn fix_subtitle_look_and_feel(raw: Vec) -> Vec { let mut script_info = false; let mut new = String::new(); @@ -439,6 +459,62 @@ fn fix_subtitle(raw: Vec) -> Vec { new.into_bytes() } +/// Fix the length of subtitles to a specified maximum amount. This is required because sometimes +/// subtitles have an unnecessary entry long after the actual video ends with artificially extends +/// the video length on some video players. To prevent this, the video length must be hard set. See +/// [crunchy-labs/crunchy-cli#32](https://github.com/crunchy-labs/crunchy-cli/issues/32) for more +/// information. +fn fix_subtitle_length(raw: Vec, max_length: NaiveTime) -> Vec { + let re = + Regex::new(r#"^Dialogue:\s\d+,(?P\d+:\d+:\d+\.\d+),(?P\d+:\d+:\d+\.\d+),"#) + .unwrap(); + + // chrono panics if we try to format NaiveTime with `%2f` and the nano seconds has more than 2 + // digits so them have to be reduced manually to avoid the panic + fn format_naive_time(native_time: NaiveTime) -> String { + let formatted_time = native_time.format("%f").to_string(); + format!("{}.{}", native_time.format("%T"), if formatted_time.len() <= 2 { + native_time.format("%2f").to_string() + } else { + formatted_time.split_at(2).0.parse().unwrap() + }) + } + + let length_as_string = format_naive_time(max_length); + let mut new = String::new(); + + for line in String::from_utf8_lossy(raw.as_slice()).split('\n') { + if let Some(capture) = re.captures(line) { + let start = capture.name("start").map_or(NaiveTime::default(), |s| { + NaiveTime::parse_from_str(s.as_str(), "%H:%M:%S.%f").unwrap() + }); + let end = capture.name("end").map_or(NaiveTime::default(), |s| { + NaiveTime::parse_from_str(s.as_str(), "%H:%M:%S.%f").unwrap() + }); + + if start > max_length { + continue + } else if end > max_length { + new.push_str(re.replace( + line, + format!( + "Dialogue: {},{},", + format_naive_time(start), + &length_as_string + ), + ).to_string().as_str()) + } else { + new.push_str(line) + } + } else { + new.push_str(line) + } + new.push('\n') + } + + new.into_bytes() +} + fn generate_mkv( archive: &Archive, target: PathBuf, @@ -450,8 +526,6 @@ fn generate_mkv( let mut maps = vec![]; let mut metadata = vec![]; - let mut video_length = (0, 0, 0, 0); - for (i, (video_path, format)) in video_paths.iter().enumerate() { input.extend(["-i".to_string(), video_path.to_string_lossy().to_string()]); maps.extend(["-map".to_string(), i.to_string()]); @@ -471,11 +545,6 @@ fn generate_mkv( format!("-metadata:s:a:{}", i), format!("title={}", format.audio.to_human_readable()), ]); - - let vid_len = get_video_length(video_path.to_path_buf())?; - if vid_len > video_length { - video_length = vid_len - } } for (i, (audio_path, format)) in audio_paths.iter().enumerate() { input.extend(["-i".to_string(), audio_path.to_string_lossy().to_string()]); @@ -552,11 +621,11 @@ fn generate_mkv( /// Get the length of a video. This is required because sometimes subtitles have an unnecessary entry /// long after the actual video ends with artificially extends the video length on some video players. -/// To prevent this, the video length must be hard set with ffmpeg. See +/// To prevent this, the video length must be hard set. See /// [crunchy-labs/crunchy-cli#32](https://github.com/crunchy-labs/crunchy-cli/issues/32) for more /// information. -fn get_video_length(path: PathBuf) -> Result<(u32, u32, u32, u32)> { - let video_length = Regex::new(r"Duration:\s?(\d+):(\d+):(\d+).(\d+),")?; +fn get_video_length(path: PathBuf) -> Result { + let video_length = Regex::new(r"Duration:\s(?P