From 0da81a481484112a588b33dc20a2e20355462f16 Mon Sep 17 00:00:00 2001 From: bytedream Date: Tue, 19 Dec 2023 22:37:16 +0100 Subject: [PATCH] Add `--include-fonts` flag for `archive` (#277) --- crunchy-cli-core/src/archive/command.rs | 6 +- crunchy-cli-core/src/download/command.rs | 2 +- crunchy-cli-core/src/utils/download.rs | 219 ++++++++++++++++++++++- crunchy-cli-core/src/utils/os.rs | 8 +- 4 files changed, 228 insertions(+), 7 deletions(-) diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index ea81d15..9ccc0ff 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -106,6 +106,9 @@ pub struct Archive { )] #[arg(long)] pub(crate) default_subtitle: Option, + #[arg(help = "Include fonts in the downloaded file")] + #[arg(long)] + pub(crate) include_fonts: bool, #[arg(help = "Skip files which are already existing")] #[arg(long, default_value_t = false)] @@ -189,8 +192,9 @@ impl Execute for Archive { single_format_collection.full_visual_output(); - let download_builder = DownloadBuilder::new() + let download_builder = DownloadBuilder::new(ctx.crunchy.client()) .default_subtitle(self.default_subtitle.clone()) + .download_fonts(self.include_fonts) .ffmpeg_preset(self.ffmpeg_preset.clone().unwrap_or_default()) .ffmpeg_threads(self.ffmpeg_threads) .output_format(Some("matroska".to_string())) diff --git a/crunchy-cli-core/src/download/command.rs b/crunchy-cli-core/src/download/command.rs index ade6adf..faa265c 100644 --- a/crunchy-cli-core/src/download/command.rs +++ b/crunchy-cli-core/src/download/command.rs @@ -204,7 +204,7 @@ impl Execute for Download { single_format_collection.full_visual_output(); - let download_builder = DownloadBuilder::new() + let download_builder = DownloadBuilder::new(ctx.crunchy.client()) .default_subtitle(self.subtitle.clone()) .force_hardsub(self.force_hardsub) .output_format(if is_special_file(&self.output) || self.output == "-" { diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index 706b539..64a6d1a 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -1,5 +1,6 @@ use crate::utils::ffmpeg::FFmpegPreset; -use crate::utils::os::{is_special_file, temp_directory, temp_named_pipe, tempfile}; +use crate::utils::filter::real_dedup_vec; +use crate::utils::os::{cache_dir, is_special_file, temp_directory, temp_named_pipe, tempfile}; use anyhow::{bail, Result}; use chrono::NaiveTime; use crunchyroll_rs::media::{Subtitle, VariantData, VariantSegment}; @@ -7,16 +8,17 @@ use crunchyroll_rs::Locale; use indicatif::{ProgressBar, ProgressDrawTarget, ProgressFinish, ProgressStyle}; use log::{debug, warn, LevelFilter}; use regex::Regex; +use reqwest::Client; use std::borrow::Borrow; use std::cmp::Ordering; use std::collections::BTreeMap; -use std::env; use std::io::Write; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::str::FromStr; use std::sync::Arc; use std::time::Duration; +use std::{env, fs}; use tempfile::TempPath; use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader}; use tokio::select; @@ -45,25 +47,29 @@ impl MergeBehavior { #[derive(Clone, derive_setters::Setters)] pub struct DownloadBuilder { + client: Client, ffmpeg_preset: FFmpegPreset, default_subtitle: Option, output_format: Option, audio_sort: Option>, subtitle_sort: Option>, force_hardsub: bool, + download_fonts: bool, threads: usize, ffmpeg_threads: Option, } impl DownloadBuilder { - pub fn new() -> DownloadBuilder { + pub fn new(client: Client) -> DownloadBuilder { Self { + client, ffmpeg_preset: FFmpegPreset::default(), default_subtitle: None, output_format: None, audio_sort: None, subtitle_sort: None, force_hardsub: false, + download_fonts: false, threads: num_cpus::get(), ffmpeg_threads: None, } @@ -71,6 +77,7 @@ impl DownloadBuilder { pub fn build(self) -> Downloader { Downloader { + client: self.client, ffmpeg_preset: self.ffmpeg_preset, default_subtitle: self.default_subtitle, output_format: self.output_format, @@ -78,6 +85,7 @@ impl DownloadBuilder { subtitle_sort: self.subtitle_sort, force_hardsub: self.force_hardsub, + download_fonts: self.download_fonts, download_threads: self.threads, ffmpeg_threads: self.ffmpeg_threads, @@ -100,6 +108,8 @@ pub struct DownloadFormat { } pub struct Downloader { + client: Client, + ffmpeg_preset: FFmpegPreset, default_subtitle: Option, output_format: Option, @@ -107,6 +117,7 @@ pub struct Downloader { subtitle_sort: Option>, force_hardsub: bool, + download_fonts: bool, download_threads: usize, ffmpeg_threads: Option, @@ -183,6 +194,7 @@ impl Downloader { let mut videos = vec![]; let mut audios = vec![]; let mut subtitles = vec![]; + let mut fonts = vec![]; let mut max_frames = 0f64; let fmt_space = self .formats @@ -296,8 +308,64 @@ impl Downloader { }); } + if self.download_fonts + && !self.force_hardsub + && dst.extension().unwrap_or_default().to_str().unwrap() == "mkv" + { + let mut font_names = vec![]; + for subtitle in subtitles.iter() { + font_names.extend(get_subtitle_stats(&subtitle.path)?) + } + real_dedup_vec(&mut font_names); + + let progress_spinner = if log::max_level() == LevelFilter::Info { + let progress_spinner = ProgressBar::new_spinner() + .with_style( + ProgressStyle::with_template( + format!( + ":: {:<1$} {{msg}} {{spinner}}", + "Downloading fonts", fmt_space + ) + .as_str(), + ) + .unwrap() + .tick_strings(&["—", "\\", "|", "/", ""]), + ) + .with_finish(ProgressFinish::Abandon); + progress_spinner.enable_steady_tick(Duration::from_millis(100)); + Some(progress_spinner) + } else { + None + }; + for font_name in font_names { + if let Some(pb) = &progress_spinner { + let mut progress_message = pb.message(); + if !progress_message.is_empty() { + progress_message += ", " + } + progress_message += &font_name; + pb.set_message(progress_message) + } + if let Some((font, cached)) = self.download_font(&font_name).await? { + if cached { + if let Some(pb) = &progress_spinner { + let mut progress_message = pb.message(); + progress_message += " (cached)"; + pb.set_message(progress_message) + } + debug!("Downloaded font {} (cached)", font_name); + } else { + debug!("Downloaded font {}", font_name); + } + + fonts.push(font) + } + } + } + let mut input = vec![]; let mut maps = vec![]; + let mut attachments = vec![]; let mut metadata = vec![]; for (i, meta) in videos.iter().enumerate() { @@ -324,6 +392,14 @@ impl Downloader { ]); } + for (i, font) in fonts.iter().enumerate() { + attachments.extend(["-attach".to_string(), font.to_string_lossy().to_string()]); + metadata.extend([ + format!("-metadata:s:t:{}", i), + "mimetype=font/woff2".to_string(), + ]) + } + // this formats are supporting embedding subtitles into the video container instead of // burning it into the video stream directly let container_supports_softsubs = !self.force_hardsub @@ -361,6 +437,7 @@ impl Downloader { command_args.extend(input_presets); command_args.extend(input); command_args.extend(maps); + command_args.extend(attachments); command_args.extend(metadata); if !preset_custom { if let Some(ffmpeg_threads) = self.ffmpeg_threads { @@ -607,6 +684,33 @@ impl Downloader { Ok(path) } + async fn download_font(&self, name: &str) -> Result> { + let Some((_, font_file)) = FONTS.iter().find(|(f, _)| f == &name) else { + return Ok(None); + }; + + let cache_dir = cache_dir("fonts")?; + let file = cache_dir.join(font_file); + if file.exists() { + return Ok(Some((file, true))); + } + + // the speed limiter does not apply to this + let font = self + .client + .get(format!( + "https://static.crunchyroll.com/vilos-v2/web/vilos/assets/libass-fonts/{}", + font_file + )) + .send() + .await? + .bytes() + .await?; + fs::write(&file, font.to_vec())?; + + Ok(Some((file, false))) + } + async fn download_segments( &self, writer: &mut impl Write, @@ -772,7 +876,7 @@ fn estimate_variant_file_size(variant_data: &VariantData, segments: &[VariantSeg } /// Get the length and fps of a video. -pub fn get_video_stats(path: &Path) -> Result<(NaiveTime, f64)> { +fn get_video_stats(path: &Path) -> Result<(NaiveTime, f64)> { let video_length = Regex::new(r"Duration:\s(?P