From 658bb86800cd4447a28859fbc9a59e1c673ac3c4 Mon Sep 17 00:00:00 2001 From: bytedream Date: Fri, 26 Jan 2024 00:07:15 +0100 Subject: [PATCH 001/113] Run ci on every branch --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa70330..3d3c984 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,8 +2,6 @@ name: ci on: push: - branches: - - master pull_request: workflow_dispatch: From 444dc65a298dd34d5beb33c9e3bef11b6c7c1fd2 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 28 Jan 2024 00:45:37 +0100 Subject: [PATCH 002/113] Clarify risks of using the `--experimental-fixes` flag --- crunchy-cli-core/src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index bc04ae3..d362a2c 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -44,9 +44,12 @@ pub struct Cli { #[arg(global = true, long)] lang: Option, - #[arg(help = "Enable experimental fixes which may resolve some unexpected errors")] + #[arg( + help = "Enable experimental fixes which may resolve some unexpected errors. Generally not recommended as this flag may crash the program completely" + )] #[arg( long_help = "Enable experimental fixes which may resolve some unexpected errors. \ + It is not recommended to use this this flag regularly, it might cause unexpected errors which may crash the program completely. \ If everything works as intended this option isn't needed, but sometimes Crunchyroll mislabels \ the audio of a series/season or episode or returns a wrong season number. This is when using this option might help to solve the issue" )] From 3b9fc5289066b696565c52e1f14209c583ea8664 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 28 Jan 2024 01:03:39 +0100 Subject: [PATCH 003/113] Add notice & warning that an anonymous or non-premium account may result to incomplete results with `search` (#288) --- README.md | 4 ++++ crunchy-cli-core/src/search/command.rs | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/README.md b/README.md index be97bec..f088fd7 100644 --- a/README.md +++ b/README.md @@ -517,6 +517,10 @@ The `archive` command lets you download episodes with multiple audios and subtit ### Search +The `search` command is a powerful tool to query the Crunchyroll library. +It behaves like the regular search on the website but is able to further process the results and return everything it can find, from the series title down to the raw stream url. +_Using this command with the `--anonymous` flag or a non-premium account may return incomplete results._ + **Supported urls/input** - Single episode (with [episode filtering](#episode-filtering)) diff --git a/crunchy-cli-core/src/search/command.rs b/crunchy-cli-core/src/search/command.rs index c357ab4..3d03d28 100644 --- a/crunchy-cli-core/src/search/command.rs +++ b/crunchy-cli-core/src/search/command.rs @@ -7,6 +7,7 @@ use anyhow::{bail, Result}; use crunchyroll_rs::common::StreamExt; use crunchyroll_rs::search::QueryResults; use crunchyroll_rs::{Episode, Locale, MediaCollection, MovieListing, MusicVideo, Series}; +use log::warn; #[derive(Debug, clap::Parser)] #[clap(about = "Search in videos")] @@ -102,6 +103,10 @@ pub struct Search { impl Execute for Search { async fn execute(self, ctx: Context) -> Result<()> { + if !ctx.crunchy.premium() { + warn!("Using `search` anonymously or with a non-premium account may return incomplete results") + } + let input = if crunchyroll_rs::parse::parse_url(&self.input).is_some() { match parse_url(&ctx.crunchy, self.input.clone(), true).await { Ok(ok) => vec![ok], From 7cf7a8e71c476c56806da00a8f41a92db0e44350 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 28 Jan 2024 02:04:42 +0100 Subject: [PATCH 004/113] Take closed_captions api field for subtitles into account (#297) --- crunchy-cli-core/src/archive/command.rs | 12 +++++++++--- crunchy-cli-core/src/download/command.rs | 7 ++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index 331374a..065b3da 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -301,8 +301,8 @@ async fn get_format( let subtitles: Vec<(Subtitle, bool)> = archive .subtitle .iter() - .filter_map(|s| { - stream + .flat_map(|s| { + let subtitles = stream .subtitles .get(s) .cloned() @@ -313,7 +313,13 @@ async fn get_format( l, single_format.audio == Locale::ja_JP || stream.subtitles.len() > 1, ) - }) + }); + let cc = stream.closed_captions.get(s).cloned().map(|l| (l, false)); + + subtitles + .into_iter() + .chain(cc.into_iter()) + .collect::>() }) .collect(); diff --git a/crunchy-cli-core/src/download/command.rs b/crunchy-cli-core/src/download/command.rs index f72923f..d1565c7 100644 --- a/crunchy-cli-core/src/download/command.rs +++ b/crunchy-cli-core/src/download/command.rs @@ -317,7 +317,12 @@ async fn get_format( let subtitle = if contains_hardsub { None } else if let Some(subtitle_locale) = &download.subtitle { - stream.subtitles.get(subtitle_locale).cloned() + stream + .subtitles + .get(subtitle_locale) + .cloned() + // use closed captions as fallback if no actual subtitles are found + .or_else(|| stream.closed_captions.get(subtitle_locale).cloned()) } else { None }; From a4abb14ae357657132cf8e4e1d0edb7e92d5379b Mon Sep 17 00:00:00 2001 From: kralverde <80051564+kralverde@users.noreply.github.com> Date: Mon, 29 Jan 2024 02:18:42 -0500 Subject: [PATCH 005/113] use a 'close enough' method to audio auto merge (#286) (#320) * use a 'close enough' method to audio merge * change default, rename flag, and use more gooder words --- crunchy-cli-core/src/archive/command.rs | 62 ++++++++++++++++++------- 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index 065b3da..b045cce 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -16,7 +16,7 @@ use chrono::Duration; use crunchyroll_rs::media::{Resolution, Subtitle}; use crunchyroll_rs::Locale; use log::{debug, warn}; -use std::collections::HashMap; +use std::ops::Sub; use std::path::PathBuf; #[derive(Clone, Debug, clap::Parser)] @@ -86,6 +86,12 @@ pub struct Archive { #[arg(value_parser = MergeBehavior::parse)] pub(crate) merge: MergeBehavior, + #[arg( + help = "If the merge behavior is 'auto', only download multiple video tracks if their length difference is higher than the given milliseconds" + )] + #[arg(long, default_value_t = 200)] + pub(crate) merge_auto_tolerance: u32, + #[arg(help = format!("Presets for converting the video to a specific coding format. \ Available presets: \n {}", FFmpegPreset::available_matches_human_readable().join("\n ")))] #[arg(long_help = format!("Presets for converting the video to a specific coding format. \ @@ -356,26 +362,46 @@ async fn get_format( .collect(), }), MergeBehavior::Auto => { - let mut d_formats: HashMap = HashMap::new(); - + let mut d_formats: Vec<(Duration, DownloadFormat)> = vec![]; + for (single_format, video, audio, subtitles) in format_pairs { - if let Some(d_format) = d_formats.get_mut(&single_format.duration) { - d_format.audios.push((audio, single_format.audio.clone())); - d_format.subtitles.extend(subtitles) - } else { - d_formats.insert( - single_format.duration, - DownloadFormat { - video: (video, single_format.audio.clone()), - audios: vec![(audio, single_format.audio.clone())], - subtitles, - }, - ); - } + let closest_format = d_formats.iter_mut().min_by(|(x, _), (y, _)| { + x.sub(single_format.duration) + .abs() + .cmp(&y.sub(single_format.duration).abs()) + }); + + match closest_format { + Some(closest_format) + if closest_format + .0 + .sub(single_format.duration) + .abs() + .num_milliseconds() + < archive.merge_auto_tolerance.into() => + { + // If less than `audio_error` apart, use same audio. + closest_format + .1 + .audios + .push((audio, single_format.audio.clone())); + closest_format.1.subtitles.extend(subtitles); + } + _ => { + d_formats.push(( + single_format.duration, + DownloadFormat { + video: (video, single_format.audio.clone()), + audios: vec![(audio, single_format.audio.clone())], + subtitles, + }, + )); + } + }; } - for d_format in d_formats.into_values() { - download_formats.push(d_format) + for (_, d_format) in d_formats.into_iter() { + download_formats.push(d_format); } } } From 982e521e0b9cbd98d0443005a68b5a72a4bdcfa6 Mon Sep 17 00:00:00 2001 From: kralverde <80051564+kralverde@users.noreply.github.com> Date: Mon, 29 Jan 2024 02:24:56 -0500 Subject: [PATCH 006/113] add universal output flag (#319) * add universal filenames setting * rename flag and help --- crunchy-cli-core/src/archive/command.rs | 8 ++++- crunchy-cli-core/src/download/command.rs | 8 ++++- crunchy-cli-core/src/utils/format.rs | 38 ++++++++++++------------ crunchy-cli-core/src/utils/os.rs | 4 +-- 4 files changed, 35 insertions(+), 23 deletions(-) diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index b045cce..7ed45d7 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -64,6 +64,11 @@ pub struct Archive { #[arg(long)] pub(crate) output_specials: Option, + #[arg(help = "Sanitize the output file for use with all operating systems. \ + This option only affects template options and not static characters.")] + #[arg(long, default_value_t = false)] + pub(crate) universal_output: bool, + #[arg(help = "Video resolution")] #[arg(long_help = "The video resolution. \ Can either be specified via the pixels (e.g. 1920x1080), the abbreviation for pixels (e.g. 1080p) or 'common-use' words (e.g. best). \ @@ -234,9 +239,10 @@ impl Execute for Archive { self.output_specials .as_ref() .map_or((&self.output).into(), |so| so.into()), + self.universal_output, ) } else { - format.format_path((&self.output).into()) + format.format_path((&self.output).into(), self.universal_output) }; let (path, changed) = free_file(formatted_path.clone()); diff --git a/crunchy-cli-core/src/download/command.rs b/crunchy-cli-core/src/download/command.rs index d1565c7..7f3f305 100644 --- a/crunchy-cli-core/src/download/command.rs +++ b/crunchy-cli-core/src/download/command.rs @@ -60,6 +60,11 @@ pub struct Download { #[arg(long)] pub(crate) output_specials: Option, + #[arg(help = "Sanitize the output file for use with all operating systems. \ + This option only affects template options and not static characters.")] + #[arg(long, default_value_t = false)] + pub(crate) universal_output: bool, + #[arg(help = "Video resolution")] #[arg(long_help = "The video resolution. \ Can either be specified via the pixels (e.g. 1920x1080), the abbreviation for pixels (e.g. 1080p) or 'common-use' words (e.g. best). \ @@ -254,9 +259,10 @@ impl Execute for Download { self.output_specials .as_ref() .map_or((&self.output).into(), |so| so.into()), + self.universal_output, ) } else { - format.format_path((&self.output).into()) + format.format_path((&self.output).into(), self.universal_output) }; let (path, changed) = free_file(formatted_path.clone()); diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index 716f124..b3ec050 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -408,11 +408,11 @@ impl Format { } /// Formats the given string if it has specific pattern in it. It also sanitizes the filename. - pub fn format_path(&self, path: PathBuf) -> PathBuf { + pub fn format_path(&self, path: PathBuf, universal: bool) -> PathBuf { let path = path .to_string_lossy() .to_string() - .replace("{title}", &sanitize(&self.title, true)) + .replace("{title}", &sanitize(&self.title, true, universal)) .replace( "{audio}", &sanitize( @@ -421,30 +421,30 @@ impl Format { .map(|(a, _)| a.to_string()) .collect::>() .join("|"), - true, + true, universal, ), ) - .replace("{resolution}", &sanitize(self.resolution.to_string(), true)) + .replace("{resolution}", &sanitize(self.resolution.to_string(), true, universal)) .replace( "{width}", - &sanitize(self.resolution.width.to_string(), true), + &sanitize(self.resolution.width.to_string(), true, universal), ) .replace( "{height}", - &sanitize(self.resolution.height.to_string(), true), + &sanitize(self.resolution.height.to_string(), true, universal), ) - .replace("{series_id}", &sanitize(&self.series_id, true)) - .replace("{series_name}", &sanitize(&self.series_name, true)) - .replace("{season_id}", &sanitize(&self.season_id, true)) - .replace("{season_name}", &sanitize(&self.season_title, true)) + .replace("{series_id}", &sanitize(&self.series_id, true, universal)) + .replace("{series_name}", &sanitize(&self.series_name, true, universal)) + .replace("{season_id}", &sanitize(&self.season_id, true, universal)) + .replace("{season_name}", &sanitize(&self.season_title, true, universal)) .replace( "{season_number}", - &format!("{:0>2}", sanitize(self.season_number.to_string(), true)), + &format!("{:0>2}", sanitize(self.season_number.to_string(), true, universal)), ) - .replace("{episode_id}", &sanitize(&self.episode_id, true)) + .replace("{episode_id}", &sanitize(&self.episode_id, true, universal)) .replace( "{episode_number}", - &format!("{:0>2}", sanitize(&self.episode_number, true)), + &format!("{:0>2}", sanitize(&self.episode_number, true, universal)), ) .replace( "{relative_episode_number}", @@ -452,13 +452,13 @@ impl Format { "{:0>2}", sanitize( self.relative_episode_number.unwrap_or_default().to_string(), - true, + true, universal, ) ), ) .replace( "{sequence_number}", - &format!("{:0>2}", sanitize(self.sequence_number.to_string(), true)), + &format!("{:0>2}", sanitize(self.sequence_number.to_string(), true, universal)), ) .replace( "{relative_sequence_number}", @@ -468,21 +468,21 @@ impl Format { self.relative_sequence_number .unwrap_or_default() .to_string(), - true, + true, universal, ) ), ) .replace( "{release_year}", - &sanitize(self.release_year.to_string(), true), + &sanitize(self.release_year.to_string(), true, universal), ) .replace( "{release_month}", - &format!("{:0>2}", sanitize(self.release_month.to_string(), true)), + &format!("{:0>2}", sanitize(self.release_month.to_string(), true, universal)), ) .replace( "{release_day}", - &format!("{:0>2}", sanitize(self.release_day.to_string(), true)), + &format!("{:0>2}", sanitize(self.release_day.to_string(), true, universal)), ); PathBuf::from(path) diff --git a/crunchy-cli-core/src/utils/os.rs b/crunchy-cli-core/src/utils/os.rs index a9d3ede..e57c4d0 100644 --- a/crunchy-cli-core/src/utils/os.rs +++ b/crunchy-cli-core/src/utils/os.rs @@ -191,7 +191,7 @@ lazy_static::lazy_static! { } /// Sanitizes a filename with the option to include/exclude the path separator from sanitizing. -pub fn sanitize>(path: S, include_path_separator: bool) -> String { +pub fn sanitize>(path: S, include_path_separator: bool, universal: bool) -> String { let path = Cow::from(path.as_ref().trim()); let path = RESERVED_RE.replace(&path, ""); @@ -204,7 +204,7 @@ pub fn sanitize>(path: S, include_path_separator: bool) -> String } }; - if cfg!(windows) { + if universal || cfg!(windows) { let path = WINDOWS_NON_PRINTABLE_RE.replace_all(&path, ""); let path = WINDOWS_ILLEGAL_RE.replace_all(&path, ""); let path = WINDOWS_RESERVED_RE.replace_all(&path, ""); From f8309f2e803fed7f450e219779253643e72f60b3 Mon Sep 17 00:00:00 2001 From: kralverde <80051564+kralverde@users.noreply.github.com> Date: Mon, 29 Jan 2024 02:26:40 -0500 Subject: [PATCH 007/113] add archive no-closed-captions flag (#323) --- crunchy-cli-core/src/archive/command.rs | 5 +++++ crunchy-cli-core/src/utils/download.rs | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index 7ed45d7..c860b0a 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -125,6 +125,10 @@ pub struct Archive { #[arg(long)] pub(crate) include_fonts: bool, + #[arg(help = "Omit closed caption subtitles in the downloaded file")] + #[arg(long, default_value_t = false)] + pub(crate) no_closed_caption: bool, + #[arg(help = "Skip files which are already existing")] #[arg(long, default_value_t = false)] pub(crate) skip_existing: bool, @@ -224,6 +228,7 @@ impl Execute for Archive { .output_format(Some("matroska".to_string())) .audio_sort(Some(self.audio.clone())) .subtitle_sort(Some(self.subtitle.clone())) + .no_closed_caption(self.no_closed_caption) .threads(self.threads); for single_formats in single_format_collection.into_iter() { diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index df32ad7..317a709 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -58,6 +58,7 @@ pub struct DownloadBuilder { subtitle_sort: Option>, force_hardsub: bool, download_fonts: bool, + no_closed_caption: bool, threads: usize, ffmpeg_threads: Option, } @@ -74,6 +75,7 @@ impl DownloadBuilder { subtitle_sort: None, force_hardsub: false, download_fonts: false, + no_closed_caption: false, threads: num_cpus::get(), ffmpeg_threads: None, } @@ -91,6 +93,7 @@ impl DownloadBuilder { force_hardsub: self.force_hardsub, download_fonts: self.download_fonts, + no_closed_caption: self.no_closed_caption, download_threads: self.threads, ffmpeg_threads: self.ffmpeg_threads, @@ -124,6 +127,7 @@ pub struct Downloader { force_hardsub: bool, download_fonts: bool, + no_closed_caption: bool, download_threads: usize, ffmpeg_threads: Option, @@ -266,6 +270,10 @@ impl Downloader { }; for (subtitle, not_cc) in format.subtitles.iter() { + if !not_cc && self.no_closed_caption { + continue; + } + if let Some(pb) = &progress_spinner { let mut progress_message = pb.message(); if !progress_message.is_empty() { From 0f06c7ac71e6eeff81e468b1f3a9b95829378c96 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 29 Jan 2024 11:52:24 +0100 Subject: [PATCH 008/113] Change delimiter of audio template option to `_` and make it configurable via the `CRUNCHY_CLI_FORMAT_DELIMITER` env variable (#311) --- crunchy-cli-core/src/utils/format.rs | 50 ++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index b3ec050..e60ef54 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -8,6 +8,7 @@ use crunchyroll_rs::{Concert, Episode, Locale, MediaCollection, Movie, MusicVide use log::{debug, info}; use std::cmp::Ordering; use std::collections::BTreeMap; +use std::env; use std::path::{Path, PathBuf}; #[derive(Clone)] @@ -420,11 +421,18 @@ impl Format { .iter() .map(|(a, _)| a.to_string()) .collect::>() - .join("|"), - true, universal, + .join( + &env::var("CRUNCHY_CLI_FORMAT_DELIMITER") + .map_or("_".to_string(), |e| e), + ), + true, + universal, ), ) - .replace("{resolution}", &sanitize(self.resolution.to_string(), true, universal)) + .replace( + "{resolution}", + &sanitize(self.resolution.to_string(), true, universal), + ) .replace( "{width}", &sanitize(self.resolution.width.to_string(), true, universal), @@ -434,12 +442,21 @@ impl Format { &sanitize(self.resolution.height.to_string(), true, universal), ) .replace("{series_id}", &sanitize(&self.series_id, true, universal)) - .replace("{series_name}", &sanitize(&self.series_name, true, universal)) + .replace( + "{series_name}", + &sanitize(&self.series_name, true, universal), + ) .replace("{season_id}", &sanitize(&self.season_id, true, universal)) - .replace("{season_name}", &sanitize(&self.season_title, true, universal)) + .replace( + "{season_name}", + &sanitize(&self.season_title, true, universal), + ) .replace( "{season_number}", - &format!("{:0>2}", sanitize(self.season_number.to_string(), true, universal)), + &format!( + "{:0>2}", + sanitize(self.season_number.to_string(), true, universal) + ), ) .replace("{episode_id}", &sanitize(&self.episode_id, true, universal)) .replace( @@ -452,13 +469,17 @@ impl Format { "{:0>2}", sanitize( self.relative_episode_number.unwrap_or_default().to_string(), - true, universal, + true, + universal, ) ), ) .replace( "{sequence_number}", - &format!("{:0>2}", sanitize(self.sequence_number.to_string(), true, universal)), + &format!( + "{:0>2}", + sanitize(self.sequence_number.to_string(), true, universal) + ), ) .replace( "{relative_sequence_number}", @@ -468,7 +489,8 @@ impl Format { self.relative_sequence_number .unwrap_or_default() .to_string(), - true, universal, + true, + universal, ) ), ) @@ -478,11 +500,17 @@ impl Format { ) .replace( "{release_month}", - &format!("{:0>2}", sanitize(self.release_month.to_string(), true, universal)), + &format!( + "{:0>2}", + sanitize(self.release_month.to_string(), true, universal) + ), ) .replace( "{release_day}", - &format!("{:0>2}", sanitize(self.release_day.to_string(), true, universal)), + &format!( + "{:0>2}", + sanitize(self.release_day.to_string(), true, universal) + ), ); PathBuf::from(path) From a2464bad4ed7528e67ac6d0579cfb16b30982919 Mon Sep 17 00:00:00 2001 From: bytedream Date: Tue, 30 Jan 2024 23:49:20 +0100 Subject: [PATCH 009/113] Add M1 runner to mac build ci --- .github/workflows/ci.yml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d3c984..ef9e921 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,7 +63,18 @@ jobs: if-no-files-found: error build-mac: - runs-on: macos-latest + runs-on: ${{ matrix.os }} + strategy: + matrix: + # macos-13 uses x86_64, macos-14 aarch64 + # see https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources + include: + - os: macos-13 + arch: x86_64 + toolchain: x86_64-apple-darwin + - os: macos-14 + arch: aarch64 + toolchain: aarch64-apple-darwin steps: - name: Checkout uses: actions/checkout@v4 @@ -85,13 +96,13 @@ jobs: toolchain: stable - name: Build - run: cargo build --release --target x86_64-apple-darwin + run: cargo build --release --target ${{ matrix.toolchain }} - name: Upload binary artifact uses: actions/upload-artifact@v3 with: - name: crunchy-cli-darwin-x86_64 - path: ./target/x86_64-apple-darwin/release/crunchy-cli + name: crunchy-cli-darwin-${{ matrix.arch }} + path: ./target/${{ matrix.toolchain }}/release/crunchy-cli if-no-files-found: error build-windows: From 5d68f0334abf602ad5ea6f14c83af14912fae783 Mon Sep 17 00:00:00 2001 From: bytedream Date: Tue, 30 Jan 2024 23:55:52 +0100 Subject: [PATCH 010/113] Update actions used in ci --- .github/workflows/ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef9e921..5727dbd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: uses: actions/checkout@v4 - name: Cargo cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/.cargo/bin/ @@ -42,21 +42,21 @@ jobs: run: cross build --release --no-default-features --features openssl-tls-static --target ${{ matrix.toolchain }} - name: Upload binary artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: crunchy-cli-linux-${{ matrix.arch }} path: ./target/${{ matrix.toolchain }}/release/crunchy-cli if-no-files-found: error - name: Upload manpages artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: manpages path: ./target/${{ matrix.toolchain }}/release/manpages if-no-files-found: error - name: Upload completions artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: completions path: ./target/${{ matrix.toolchain }}/release/completions @@ -80,7 +80,7 @@ jobs: uses: actions/checkout@v4 - name: Cargo cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/.cargo/bin/ @@ -99,7 +99,7 @@ jobs: run: cargo build --release --target ${{ matrix.toolchain }} - name: Upload binary artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: crunchy-cli-darwin-${{ matrix.arch }} path: ./target/${{ matrix.toolchain }}/release/crunchy-cli @@ -112,7 +112,7 @@ jobs: uses: actions/checkout@v4 - name: Cargo cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/.cargo/bin/ @@ -133,7 +133,7 @@ jobs: run: cargo build --release --target x86_64-pc-windows-gnu - name: Upload binary artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: crunchy-cli-windows-x86_64 path: ./target/x86_64-pc-windows-gnu/release/crunchy-cli.exe From 8187269128a88047a6c601ab763e22f3dd9f2490 Mon Sep 17 00:00:00 2001 From: bytedream Date: Thu, 1 Feb 2024 14:45:12 +0100 Subject: [PATCH 011/113] Upload manpages and completions only once in ci --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5727dbd..ae289c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,7 @@ jobs: if-no-files-found: error - name: Upload manpages artifact + if: ${{ matrix.arch == 'x86_64' }} # only upload the manpages once uses: actions/upload-artifact@v4 with: name: manpages @@ -56,6 +57,7 @@ jobs: if-no-files-found: error - name: Upload completions artifact + if: ${{ matrix.arch == 'x86_64' }} # only upload the completions once uses: actions/upload-artifact@v4 with: name: completions From c31b1f4db94295e8b903799280189a3c52e0b5e8 Mon Sep 17 00:00:00 2001 From: bytedream Date: Wed, 14 Feb 2024 20:27:00 +0100 Subject: [PATCH 012/113] Update nix flake.lock (#333) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index b9c63db..e92c958 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1692128808, - "narHash": "sha256-Di1Zm/P042NuwThMiZNrtmaAjd4Tm2qBOKHX7xUOfMk=", + "lastModified": 1707877513, + "narHash": "sha256-sp0w2apswd3wv0sAEF7StOGHkns3XUQaO5erhWFZWXk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "4ed9856be002a730234a1a1ed9dcd9dd10cbdb40", + "rev": "89653a03e0915e4a872788d10680e7eec92f8600", "type": "github" }, "original": { @@ -41,11 +41,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1689068808, - "narHash": "sha256-6ixXo3wt24N/melDWjq70UuHQLxGV8jZvooRanIHXw0=", + "lastModified": 1705309234, + "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", "owner": "numtide", "repo": "flake-utils", - "rev": "919d646de7be200f3bf08cb76ae1f09402b6f9b4", + "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", "type": "github" }, "original": { From d3ab2245a84d549f1e31ece5c6f208fd484ec139 Mon Sep 17 00:00:00 2001 From: Kevin <16837578+KevinStaude@users.noreply.github.com> Date: Thu, 15 Feb 2024 23:52:47 +0100 Subject: [PATCH 013/113] Update README.md minor fix --output-specials -o "something" isn't working --output-specials "something" is correct --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f088fd7..a4adc17 100644 --- a/README.md +++ b/README.md @@ -278,7 +278,7 @@ The `download` command lets you download episodes with a specific audio language Define an output template which only gets used when the episode is a special (episode number is 0 or has non-zero decimal places) by using the `--output-special` flag. ```shell - $ crunchy-cli download --output-specials -o "Special EP - {title}" https://www.crunchyroll.com/watch/GY8D975JY/veldoras-journal + $ crunchy-cli download --output-specials "Special EP - {title}" https://www.crunchyroll.com/watch/GY8D975JY/veldoras-journal ``` Default is the template, set by the `-o` / `--output` flag. See the [Template Options section](#output-template-options) below for more options. @@ -411,7 +411,7 @@ The `archive` command lets you download episodes with multiple audios and subtit _crunchy-cli_ exclusively uses the [`.mkv`](https://en.wikipedia.org/wiki/Matroska) container format, because of its ability to store multiple audio, video and subtitle tracks at once. ```shell - $ crunchy-cli archive --output-specials -o "Special EP - {title}" https://www.crunchyroll.com/watch/GY8D975JY/veldoras-journal + $ crunchy-cli archive --output-specials "Special EP - {title}" https://www.crunchyroll.com/watch/GY8D975JY/veldoras-journal ``` Default is the template, set by the `-o` / `--output` flag. See the [Template Options section](#output-template-options) below for more options. From 2084328069dcd253dd8c60e1151d63c176660d38 Mon Sep 17 00:00:00 2001 From: bytedream Date: Fri, 23 Feb 2024 17:35:12 +0100 Subject: [PATCH 014/113] Fix ffmpeg progress panic (#337) --- crunchy-cli-core/src/utils/download.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index 317a709..9a67f7f 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -1202,13 +1202,16 @@ async fn ffmpeg_progress( let Some(line) = line? else { break }; - let frame: u64 = current_frame - .captures(line.as_str()) - .unwrap() - .name("frame") - .unwrap() - .as_str() - .parse()?; + + // we're manually unpack the regex here as `.unwrap()` may fail in some cases, e.g. + // https://github.com/crunchy-labs/crunchy-cli/issues/337 + let Some(frame_cap) = current_frame.captures(line.as_str()) else { + break + }; + let Some(frame_str) = frame_cap.name("frame") else { + break + }; + let frame: u64 = frame_str.as_str().parse()?; if let Some(p) = &progress { p.set_position(frame) From 6a7aa25e1a6f58392daed84400656b8d7744bf9c Mon Sep 17 00:00:00 2001 From: bytedream <63594396+bytedream@users.noreply.github.com> Date: Sun, 25 Feb 2024 18:46:48 +0100 Subject: [PATCH 015/113] Add ffmpeg amd hardware acceleration presets (#324) --- crunchy-cli-core/src/utils/ffmpeg.rs | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/crunchy-cli-core/src/utils/ffmpeg.rs b/crunchy-cli-core/src/utils/ffmpeg.rs index 39678dc..a61ed93 100644 --- a/crunchy-cli-core/src/utils/ffmpeg.rs +++ b/crunchy-cli-core/src/utils/ffmpeg.rs @@ -69,6 +69,7 @@ ffmpeg_enum! { ffmpeg_enum! { enum FFmpegHwAccel { Nvidia, + Amd, Apple } } @@ -101,7 +102,11 @@ impl FFmpegPreset { FFmpegHwAccel::all(), FFmpegQuality::all(), ), - (FFmpegCodec::Av1, vec![], FFmpegQuality::all()), + ( + FFmpegCodec::Av1, + vec![FFmpegHwAccel::Amd], + FFmpegQuality::all(), + ), ]; let mut return_values = vec![]; @@ -285,6 +290,10 @@ impl FFmpegPreset { crf_quality(); output.extend(["-c:v", "h264_nvenc", "-c:a", "copy"]) } + FFmpegHwAccel::Amd => { + crf_quality(); + output.extend(["-c:v", "h264_amf", "-c:a", "copy"]) + } FFmpegHwAccel::Apple => { // Apple's Video Toolbox encoders ignore `-crf`, use `-q:v` // instead. It's on a scale of 1-100, 100 being lossless. Just @@ -332,8 +341,12 @@ impl FFmpegPreset { "hvc1", ]) } + FFmpegHwAccel::Amd => { + crf_quality(); + output.extend(["-c:v", "hevc_amf", "-c:a", "copy"]) + } FFmpegHwAccel::Apple => { - // See the comment that starts on line 287. + // See the comment for apple h264 hwaccel match quality { FFmpegQuality::Lossless => output.extend(["-q:v", "61"]), FFmpegQuality::Normal => (), @@ -356,12 +369,17 @@ impl FFmpegPreset { } } FFmpegCodec::Av1 => { - output.extend(["-c:v", "libsvtav1", "-c:a", "copy"]); - - match quality { + let mut crf_quality = || match quality { FFmpegQuality::Lossless => output.extend(["-crf", "22"]), FFmpegQuality::Normal => (), FFmpegQuality::Low => output.extend(["-crf", "35"]), + }; + + crf_quality(); + if let Some(FFmpegHwAccel::Amd) = hwaccel_opt { + output.extend(["-c:v", "av1_amf", "-c:a", "copy"]); + } else { + output.extend(["-c:v", "libsvtav1", "-c:a", "copy"]); } } } From 5634ce3277f6e570a1ea9b5ca20ed0391a9f9d68 Mon Sep 17 00:00:00 2001 From: bytedream <63594396+bytedream@users.noreply.github.com> Date: Sun, 25 Feb 2024 18:48:18 +0100 Subject: [PATCH 016/113] Add archive `--skip-existing-method` flag (#292) (#325) * Add archive `--skip-existing-method` flag (#292) * Fix re-download only issued when local file has more audios/subtitles & respect `--no-closed-captions` flag --- crunchy-cli-core/src/archive/command.rs | 148 +++++++++++++++++++++-- crunchy-cli-core/src/download/command.rs | 4 +- 2 files changed, 139 insertions(+), 13 deletions(-) diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index c860b0a..741dd5b 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -16,8 +16,11 @@ use chrono::Duration; use crunchyroll_rs::media::{Resolution, Subtitle}; use crunchyroll_rs::Locale; use log::{debug, warn}; +use regex::Regex; +use std::fmt::{Display, Formatter}; use std::ops::Sub; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; #[derive(Clone, Debug, clap::Parser)] #[clap(about = "Archive a video")] @@ -68,7 +71,7 @@ pub struct Archive { This option only affects template options and not static characters.")] #[arg(long, default_value_t = false)] pub(crate) universal_output: bool, - + #[arg(help = "Video resolution")] #[arg(long_help = "The video resolution. \ Can either be specified via the pixels (e.g. 1920x1080), the abbreviation for pixels (e.g. 1080p) or 'common-use' words (e.g. best). \ @@ -129,9 +132,19 @@ pub struct Archive { #[arg(long, default_value_t = false)] pub(crate) no_closed_caption: bool, - #[arg(help = "Skip files which are already existing")] + #[arg(help = "Skip files which are already existing by their name")] #[arg(long, default_value_t = false)] pub(crate) skip_existing: bool, + #[arg( + help = "Only works in combination with `--skip-existing`. Sets the method how already existing files should be skipped. Valid methods are 'audio' and 'subtitle'" + )] + #[arg(long_help = "Only works in combination with `--skip-existing`. \ + By default, already existing files are determined by their name and the download of the corresponding episode is skipped. \ + With this flag you can modify this behavior. \ + Valid options are 'audio' and 'subtitle' (if the file already exists but the audio/subtitle are less from what should be downloaded, the episode gets downloaded and the file overwritten).")] + #[arg(long, default_values_t = SkipExistingMethod::default())] + #[arg(value_parser = SkipExistingMethod::parse)] + pub(crate) skip_existing_method: Vec, #[arg(help = "Skip special episodes")] #[arg(long, default_value_t = false)] pub(crate) skip_specials: bool, @@ -244,19 +257,69 @@ impl Execute for Archive { self.output_specials .as_ref() .map_or((&self.output).into(), |so| so.into()), - self.universal_output, + self.universal_output, ) } else { format.format_path((&self.output).into(), self.universal_output) }; - let (path, changed) = free_file(formatted_path.clone()); + let (mut path, changed) = free_file(formatted_path.clone()); if changed && self.skip_existing { - debug!( - "Skipping already existing file '{}'", - formatted_path.to_string_lossy() - ); - continue; + let mut skip = true; + + if !self.skip_existing_method.is_empty() { + if let Some((mut audio_locales, mut subtitle_locales)) = + get_video_streams(&formatted_path)? + { + let method_audio = self + .skip_existing_method + .contains(&SkipExistingMethod::Audio); + let method_subtitle = self + .skip_existing_method + .contains(&SkipExistingMethod::Subtitle); + + let audio_differ = if method_audio { + format + .locales + .iter() + .any(|(a, _)| !audio_locales.contains(a)) + } else { + false + }; + let subtitle_differ = if method_subtitle { + format + .locales + .clone() + .into_iter() + .flat_map(|(a, mut s)| { + // remove the closed caption if the flag is given to omit + // closed captions + if self.no_closed_caption && a != Locale::ja_JP { + s.retain(|l| l != &a) + } + s + }) + .any(|l| !subtitle_locales.contains(&l)) + } else { + false + }; + + if (method_audio && audio_differ) + || (method_subtitle && subtitle_differ) + { + skip = false; + path = formatted_path.clone() + } + } + } + + if skip { + debug!( + "Skipping already existing file '{}'", + formatted_path.to_string_lossy() + ); + continue; + } } format.locales.sort_by(|(a, _), (b, _)| { @@ -284,6 +347,36 @@ impl Execute for Archive { } } +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum SkipExistingMethod { + Audio, + Subtitle, +} + +impl Display for SkipExistingMethod { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let value = match self { + SkipExistingMethod::Audio => "audio", + SkipExistingMethod::Subtitle => "subtitle", + }; + write!(f, "{}", value) + } +} + +impl SkipExistingMethod { + fn parse(s: &str) -> Result { + match s.to_lowercase().as_str() { + "audio" => Ok(Self::Audio), + "subtitle" => Ok(Self::Subtitle), + _ => Err(format!("invalid skip existing method '{}'", s)), + } + } + + fn default<'a>() -> &'a [Self] { + &[] + } +} + async fn get_format( archive: &Archive, single_formats: &Vec, @@ -374,7 +467,7 @@ async fn get_format( }), MergeBehavior::Auto => { let mut d_formats: Vec<(Duration, DownloadFormat)> = vec![]; - + for (single_format, video, audio, subtitles) in format_pairs { let closest_format = d_formats.iter_mut().min_by(|(x, _), (y, _)| { x.sub(single_format.duration) @@ -422,3 +515,36 @@ async fn get_format( Format::from_single_formats(single_format_to_format_pairs), )) } + +fn get_video_streams(path: &Path) -> Result, Vec)>> { + let video_streams = + Regex::new(r"(?m)Stream\s#\d+:\d+\((?P.+)\):\s(?P(Audio|Subtitle))") + .unwrap(); + + let ffmpeg = Command::new("ffmpeg") + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .arg("-hide_banner") + .args(["-i", &path.to_string_lossy().to_string()]) + .output()?; + let ffmpeg_output = String::from_utf8(ffmpeg.stderr)?; + + let mut audio = vec![]; + let mut subtitle = vec![]; + for cap in video_streams.captures_iter(&ffmpeg_output) { + let locale = cap.name("language").unwrap().as_str(); + let type_ = cap.name("type").unwrap().as_str(); + + match type_ { + "Audio" => audio.push(Locale::from(locale.to_string())), + "Subtitle" => subtitle.push(Locale::from(locale.to_string())), + _ => unreachable!(), + } + } + + if audio.is_empty() && subtitle.is_empty() { + Ok(None) + } else { + Ok(Some((audio, subtitle))) + } +} diff --git a/crunchy-cli-core/src/download/command.rs b/crunchy-cli-core/src/download/command.rs index 7f3f305..1f8cd8e 100644 --- a/crunchy-cli-core/src/download/command.rs +++ b/crunchy-cli-core/src/download/command.rs @@ -94,7 +94,7 @@ pub struct Download { #[arg(long)] pub(crate) ffmpeg_threads: Option, - #[arg(help = "Skip files which are already existing")] + #[arg(help = "Skip files which are already existing by their name")] #[arg(long, default_value_t = false)] pub(crate) skip_existing: bool, #[arg(help = "Skip special episodes")] @@ -259,7 +259,7 @@ impl Execute for Download { self.output_specials .as_ref() .map_or((&self.output).into(), |so| so.into()), - self.universal_output, + self.universal_output, ) } else { format.format_path((&self.output).into(), self.universal_output) From 52da6eacc9165a2681667c68b0878823834b02b4 Mon Sep 17 00:00:00 2001 From: bytedream Date: Fri, 23 Feb 2024 21:41:47 +0100 Subject: [PATCH 017/113] Fix search command always showing non-premium account warning message --- Cargo.lock | 72 +++++++++++++++++++++++--- crunchy-cli-core/Cargo.toml | 2 +- crunchy-cli-core/src/search/command.rs | 2 +- 3 files changed, 68 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 58e768c..1ca5441 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -429,9 +429,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "828ff3c0f11de8f8afda7dc3bd24e206e1b13cee6abfd87856123305864681d2" +checksum = "0fcce6c297f8f78d4a56511a2c4321d4eb72f09132469e07955116f8e23582a6" dependencies = [ "aes", "async-trait", @@ -440,6 +440,7 @@ dependencies = [ "crunchyroll-rs-internal", "dash-mpd", "futures-util", + "jsonwebtoken", "lazy_static", "m3u8-rs", "regex", @@ -456,9 +457,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7051a39e25a19ef0aa753e7da179787a3db0fb8a01977a7e22cd288f7ff0e27" +checksum = "d1f3c9595b79a54c90aa96ba5e15ebbcc944690a9e0cb24989dc677872370b39" dependencies = [ "darling", "quote", @@ -775,8 +776,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -1063,6 +1066,21 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonwebtoken" +version = "9.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7ea04a7c5c055c175f189b6dc6ba036fd62306b58c66c9f6389036c503a3f4" +dependencies = [ + "base64", + "js-sys", + "pem", + "ring", + "serde", + "serde_json", + "simple_asn1", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -1100,9 +1118,9 @@ checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "m3u8-rs" -version = "5.0.5" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1d7ba86f7ea62f17f4310c55e93244619ddc7dadfc7e565de1967e4e41e6e7" +checksum = "f03cd3335fb5f2447755d45cda9c70f76013626a9db44374973791b0926a86c3" dependencies = [ "chrono", "nom", @@ -1189,6 +1207,26 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num-bigint" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.17" @@ -1289,6 +1327,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "pem" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8fcc794035347fb64beda2d3b462595dd2753e3f268d89c5aae77e8cf2c310" +dependencies = [ + "base64", + "serde", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -1705,6 +1753,18 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" +[[package]] +name = "simple_asn1" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" +dependencies = [ + "num-bigint", + "num-traits", + "thiserror", + "time", +] + [[package]] name = "slab" version = "0.4.9" diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index a937e01..a0d4fb3 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -16,7 +16,7 @@ anyhow = "1.0" async-speed-limit = "0.4" clap = { version = "4.4", features = ["derive", "string"] } chrono = "0.4" -crunchyroll-rs = { version = "0.8.2", features = ["dash-stream", "experimental-stabilizations", "tower"] } +crunchyroll-rs = { version = "0.8.3", features = ["dash-stream", "experimental-stabilizations", "tower"] } ctrlc = "3.4" dialoguer = { version = "0.11", default-features = false } dirs = "5.0" diff --git a/crunchy-cli-core/src/search/command.rs b/crunchy-cli-core/src/search/command.rs index 3d03d28..226e242 100644 --- a/crunchy-cli-core/src/search/command.rs +++ b/crunchy-cli-core/src/search/command.rs @@ -103,7 +103,7 @@ pub struct Search { impl Execute for Search { async fn execute(self, ctx: Context) -> Result<()> { - if !ctx.crunchy.premium() { + if !ctx.crunchy.premium().await { warn!("Using `search` anonymously or with a non-premium account may return incomplete results") } From d2589a3a6f8579c4f16ed70ba66a4e11bc25be85 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 25 Feb 2024 19:01:27 +0100 Subject: [PATCH 018/113] Use macos 12 instead of 13 for ci --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae289c1..82b604d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,10 +68,10 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - # macos-13 uses x86_64, macos-14 aarch64 + # macos-12 uses x86_64, macos-14 aarch64 # see https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources include: - - os: macos-13 + - os: macos-12 arch: x86_64 toolchain: x86_64-apple-darwin - os: macos-14 From 9a6959970ab9b03da91029096fdf387a7b5f44a8 Mon Sep 17 00:00:00 2001 From: Hannes Braun Date: Mon, 26 Feb 2024 20:09:54 +0100 Subject: [PATCH 019/113] Remove superfluous mut keywords (#341) --- crunchy-cli-core/src/archive/command.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index 741dd5b..8e70c2e 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -268,7 +268,7 @@ impl Execute for Archive { let mut skip = true; if !self.skip_existing_method.is_empty() { - if let Some((mut audio_locales, mut subtitle_locales)) = + if let Some((audio_locales, subtitle_locales)) = get_video_streams(&formatted_path)? { let method_audio = self From 3099aac0e7260c32d65cff5528dd0e942b838fdf Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 26 Feb 2024 20:42:33 +0100 Subject: [PATCH 020/113] Revert macos action downgrade and disable caching instead --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82b604d..6991cad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,10 +68,10 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - # macos-12 uses x86_64, macos-14 aarch64 + # macos-13 uses x86_64, macos-14 aarch64 # see https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources include: - - os: macos-12 + - os: macos-13 arch: x86_64 toolchain: x86_64-apple-darwin - os: macos-14 @@ -82,6 +82,7 @@ jobs: uses: actions/checkout@v4 - name: Cargo cache + if: ${{ matrix.os != 'macos-13' }} # when using cache, the 'Setup Rust' step fails for macos 13 uses: actions/cache@v4 with: path: | From 9c44fa7dae8ca9f2edb01cf08675dc2d62d46ea7 Mon Sep 17 00:00:00 2001 From: Username404-59 <53659497+Username404-59@users.noreply.github.com> Date: Sun, 3 Mar 2024 22:40:41 +0100 Subject: [PATCH 021/113] README.md: Fix a typo (#344) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a4adc17..57a8b7c 100644 --- a/README.md +++ b/README.md @@ -300,7 +300,7 @@ The `download` command lets you download episodes with a specific audio language If you need more specific ffmpeg customizations you could either convert the output file manually or use ffmpeg output arguments as value for this flag. ```shell - $ crunchy-cli downlaod --ffmpeg-preset av1-lossless https://www.crunchyroll.com/watch/GRDQPM1ZY/alone-and-lonesome + $ crunchy-cli download --ffmpeg-preset av1-lossless https://www.crunchyroll.com/watch/GRDQPM1ZY/alone-and-lonesome ``` - FFmpeg threads From 56f0ed1795d971fb4342b9f0a66216e8af18fd6f Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 10 Mar 2024 01:58:35 +0100 Subject: [PATCH 022/113] Add `--include-chapters` flag to `archive` and `download` (#301) --- Cargo.lock | 320 ++++++++++------------- crunchy-cli-core/Cargo.toml | 2 +- crunchy-cli-core/src/archive/command.rs | 24 +- crunchy-cli-core/src/download/command.rs | 17 +- crunchy-cli-core/src/utils/download.rs | 90 ++++++- crunchy-cli-core/src/utils/format.rs | 10 +- 6 files changed, 272 insertions(+), 191 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1ca5441..7c2a6f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -54,9 +54,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.7" +version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd2405b3ac1faab2990b74d728624cd9fd115651fcecc7c2d8daf01376275ba" +checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" dependencies = [ "anstyle", "anstyle-parse", @@ -68,9 +68,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.4" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" [[package]] name = "anstyle-parse" @@ -174,9 +174,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" [[package]] name = "block-padding" @@ -210,12 +210,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.83" +version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" -dependencies = [ - "libc", -] +checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" [[package]] name = "cfg-if" @@ -225,9 +222,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "8eaf5903dcbc0a39312feb77df2ff4c76387d591b9fc7b04a238dcf8bb62639a" dependencies = [ "android-tzdata", "iana-time-zone", @@ -235,7 +232,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.48.5", + "windows-targets 0.52.4", ] [[package]] @@ -250,9 +247,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.16" +version = "4.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58e54881c004cec7895b0068a0a954cd5d62da01aef83fa35b1e594497bf5445" +checksum = "b230ab84b0ffdf890d5a10abdbc8b83ae1c4918275daea1ab8801f71536b2651" dependencies = [ "clap_builder", "clap_derive", @@ -260,30 +257,30 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.16" +version = "4.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59cb82d7f531603d2fd1f507441cdd35184fa81beff7bd489570de7f773460bb" +checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim", + "strsim 0.11.0", ] [[package]] name = "clap_complete" -version = "4.4.6" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97aeaa95557bd02f23fbb662f981670c3d20c5a26e69f7354b28f57092437fcd" +checksum = "885e4d7d5af40bfb99ae6f9433e292feac98d452dcb3ec3d25dfe7552b77da8c" dependencies = [ "clap", ] [[package]] name = "clap_derive" -version = "4.4.7" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +checksum = "307bc0538d5f0f83b8248db3087aa92fe504e4691294d0c96c0eabc33f47ba47" dependencies = [ "heck", "proc-macro2", @@ -293,9 +290,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" [[package]] name = "clap_mangen" @@ -328,9 +325,9 @@ dependencies = [ [[package]] name = "cookie" -version = "0.16.2" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" +checksum = "7efb37c3e1ccb1ff97164ad95ac1606e8ccd35b3fa0a7d99a304c7f4a428cc24" dependencies = [ "percent-encoding", "time", @@ -339,12 +336,12 @@ dependencies = [ [[package]] name = "cookie_store" -version = "0.16.2" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d606d0fba62e13cf04db20536c05cb7f13673c161cb47a47a82b9b9e7d3f1daa" +checksum = "387461abbc748185c3a6e1673d826918b450b87ff22639429c694619a83b6cf6" dependencies = [ "cookie", - "idna 0.2.3", + "idna 0.3.0", "log", "publicsuffix", "serde", @@ -429,9 +426,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcce6c297f8f78d4a56511a2c4321d4eb72f09132469e07955116f8e23582a6" +checksum = "467fc4159e38121aa5efb3807de957eefff02d14ba3439494f89f351e3539b73" dependencies = [ "aes", "async-trait", @@ -457,9 +454,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1f3c9595b79a54c90aa96ba5e15ebbcc944690a9e0cb24989dc677872370b39" +checksum = "62db42661f84dc2e2f7c5fef1e8906fff29ff316624da54039aec748a49e7a3b" dependencies = [ "darling", "quote", @@ -506,7 +503,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", + "strsim 0.10.0", "syn", ] @@ -523,9 +520,9 @@ dependencies = [ [[package]] name = "dash-mpd" -version = "0.14.7" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cf94350e05e27c941b8cfc06bffeec3afcac11f42df289378ddf43e192d2e15" +checksum = "18c18f28b58beade78e0f61a846a63a122cb92c5f5ed6bad29d7ad13287c7526" dependencies = [ "base64", "base64-serde", @@ -540,7 +537,6 @@ dependencies = [ "serde_path_to_error", "serde_with", "thiserror", - "tokio", "tracing", "url", "xattr", @@ -827,9 +823,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "hex" @@ -951,17 +947,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "idna" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" -dependencies = [ - "matches", - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "idna" version = "0.3.0" @@ -1074,11 +1059,9 @@ checksum = "5c7ea04a7c5c055c175f189b6dc6ba036fd62306b58c66c9f6389036c503a3f4" dependencies = [ "base64", "js-sys", - "pem", "ring", "serde", "serde_json", - "simple_asn1", ] [[package]] @@ -1089,9 +1072,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.152" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libredox" @@ -1099,16 +1082,16 @@ version = "0.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "libc", "redox_syscall", ] [[package]] name = "linux-raw-sys" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "log" @@ -1126,12 +1109,6 @@ dependencies = [ "nom", ] -[[package]] -name = "matches" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" - [[package]] name = "memchr" version = "2.7.1" @@ -1152,18 +1129,18 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", ] [[package]] name = "mio" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "wasi", @@ -1192,7 +1169,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cfg-if", "libc", ] @@ -1207,31 +1184,11 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "num-bigint" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg", ] @@ -1273,7 +1230,7 @@ version = "0.10.62" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8cde4d2d9200ad5909f8dac647e29482e07c3a35de8a13fce7c9c7747ad9f671" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cfg-if", "foreign-types", "libc", @@ -1327,16 +1284,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" -[[package]] -name = "pem" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8fcc794035347fb64beda2d3b462595dd2753e3f268d89c5aae77e8cf2c310" -dependencies = [ - "base64", - "serde", -] - [[package]] name = "percent-encoding" version = "2.3.1" @@ -1375,9 +1322,9 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "proc-macro2" -version = "1.0.76" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" dependencies = [ "unicode-ident", ] @@ -1439,9 +1386,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.2" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", @@ -1451,9 +1398,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" dependencies = [ "aho-corasick", "memchr", @@ -1468,9 +1415,9 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "reqwest" -version = "0.11.23" +version = "0.11.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" +checksum = "0eea5a9eb898d3783f17c6407670e3592fd174cb81a10e51d4c37f49450b9946" dependencies = [ "base64", "bytes", @@ -1498,6 +1445,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", "system-configuration", "tokio", "tokio-native-tls", @@ -1542,11 +1490,11 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.38.30" +version = "0.38.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "errno", "libc", "linux-raw-sys", @@ -1588,9 +1536,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.1.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e9d979b3ce68192e42760c7810125eb6cf2ea10efae545a156063e61f314e2a" +checksum = "5ede67b28608b4c60685c7d54122d4400d90f62b40caee7700e700380a390fa8" [[package]] name = "rustls-webpki" @@ -1652,18 +1600,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.195" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.195" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", @@ -1714,9 +1662,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.4.0" +version = "3.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64cd236ccc1b7a29e7e2739f27c0b2dd199804abc4290e32f59f3b68d6405c23" +checksum = "15d167997bd841ec232f5b2b8e0e26606df2e7caa4c31b95ea9ca52b200bd270" dependencies = [ "base64", "chrono", @@ -1724,6 +1672,7 @@ dependencies = [ "indexmap 1.9.3", "indexmap 2.1.0", "serde", + "serde_derive", "serde_json", "serde_with_macros", "time", @@ -1731,9 +1680,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.4.0" +version = "3.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93634eb5f75a2323b16de4748022ac4297f9e76b6dced2be287a099f41b5e788" +checksum = "865f9743393e638991566a8b7a479043c2c8da94a33e0a31f18214c9cae0a64d" dependencies = [ "darling", "proc-macro2", @@ -1749,21 +1698,9 @@ checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" [[package]] name = "shlex" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" - -[[package]] -name = "simple_asn1" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" -dependencies = [ - "num-bigint", - "num-traits", - "thiserror", - "time", -] +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "slab" @@ -1787,12 +1724,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -1808,16 +1745,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] -name = "syn" -version = "2.0.48" +name = "strsim" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" +checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" + +[[package]] +name = "syn" +version = "2.0.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "sys-locale" version = "0.3.1" @@ -1829,20 +1778,20 @@ dependencies = [ [[package]] name = "system-configuration" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +checksum = "658bc6ee10a9b4fcf576e9b0819d95ec16f4d2c02d39fd83ac1c8789785c4a42" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.2", "core-foundation", "system-configuration-sys", ] [[package]] name = "system-configuration-sys" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" dependencies = [ "core-foundation-sys", "libc", @@ -1850,31 +1799,30 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.9.0" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if", "fastrand", - "redox_syscall", "rustix", "windows-sys 0.52.0", ] [[package]] name = "thiserror" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" +checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" +checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" dependencies = [ "proc-macro2", "quote", @@ -1927,9 +1875,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.35.1" +version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" dependencies = [ "backtrace", "bytes", @@ -2193,9 +2141,9 @@ checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" [[package]] name = "wasm-streams" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" dependencies = [ "futures-util", "js-sys", @@ -2257,7 +2205,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.0", + "windows-targets 0.52.4", ] [[package]] @@ -2275,7 +2223,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.0", + "windows-targets 0.52.4", ] [[package]] @@ -2295,17 +2243,17 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" dependencies = [ - "windows_aarch64_gnullvm 0.52.0", - "windows_aarch64_msvc 0.52.0", - "windows_i686_gnu 0.52.0", - "windows_i686_msvc 0.52.0", - "windows_x86_64_gnu 0.52.0", - "windows_x86_64_gnullvm 0.52.0", - "windows_x86_64_msvc 0.52.0", + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", ] [[package]] @@ -2316,9 +2264,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" [[package]] name = "windows_aarch64_msvc" @@ -2328,9 +2276,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" [[package]] name = "windows_i686_gnu" @@ -2340,9 +2288,9 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" [[package]] name = "windows_i686_msvc" @@ -2352,9 +2300,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" [[package]] name = "windows_x86_64_gnu" @@ -2364,9 +2312,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" [[package]] name = "windows_x86_64_gnullvm" @@ -2376,9 +2324,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" [[package]] name = "windows_x86_64_msvc" @@ -2388,9 +2336,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" [[package]] name = "winreg" @@ -2404,9 +2352,9 @@ dependencies = [ [[package]] name = "xattr" -version = "1.2.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "914566e6413e7fa959cc394fb30e563ba80f3541fbd40816d4c05a0fc3f2a0f1" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" dependencies = [ "libc", "linux-raw-sys", diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index a0d4fb3..2d3ac61 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -16,7 +16,7 @@ anyhow = "1.0" async-speed-limit = "0.4" clap = { version = "4.4", features = ["derive", "string"] } chrono = "0.4" -crunchyroll-rs = { version = "0.8.3", features = ["dash-stream", "experimental-stabilizations", "tower"] } +crunchyroll-rs = { version = "0.8.5", features = ["dash-stream", "experimental-stabilizations", "tower"] } ctrlc = "3.4" dialoguer = { version = "0.11", default-features = false } dirs = "5.0" diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index 8e70c2e..bf61fdf 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -1,6 +1,8 @@ use crate::archive::filter::ArchiveFilter; use crate::utils::context::Context; -use crate::utils::download::{DownloadBuilder, DownloadFormat, MergeBehavior}; +use crate::utils::download::{ + DownloadBuilder, DownloadFormat, DownloadFormatMetadata, MergeBehavior, +}; use crate::utils::ffmpeg::FFmpegPreset; use crate::utils::filter::Filter; use crate::utils::format::{Format, SingleFormat}; @@ -127,6 +129,17 @@ pub struct Archive { #[arg(help = "Include fonts in the downloaded file")] #[arg(long)] pub(crate) include_fonts: bool, + #[arg( + help = "Includes chapters (e.g. intro, credits, ...). Only works if `--merge` is set to 'audio'" + )] + #[arg( + long_help = "Includes chapters (e.g. intro, credits, ...). . Only works if `--merge` is set to 'audio'. \ + Because chapters are essentially only special timeframes in episodes like the intro, most of the video timeline isn't covered by a chapter. + These \"gaps\" are filled with an 'Episode' chapter because many video players are ignore those gaps and just assume that a chapter ends when the next chapter start is reached, even if a specific end-time is set. + Also chapters aren't always available, so in this case, just a big 'Episode' chapter from start to end will be created" + )] + #[arg(long, default_value_t = false)] + pub(crate) include_chapters: bool, #[arg(help = "Omit closed caption subtitles in the downloaded file")] #[arg(long, default_value_t = false)] @@ -188,6 +201,10 @@ impl Execute for Archive { } } + if self.include_chapters && !matches!(self.merge, MergeBehavior::Audio) { + bail!("`--include-chapters` can only be used if `--merge` is set to 'audio'") + } + if self.output.contains("{resolution}") || self .output_specials @@ -446,6 +463,7 @@ async fn get_format( video: (video, single_format.audio.clone()), audios: vec![(audio, single_format.audio.clone())], subtitles, + metadata: DownloadFormatMetadata { skip_events: None }, }) } } @@ -464,6 +482,9 @@ async fn get_format( .iter() .flat_map(|(_, _, _, subtitles)| subtitles.clone()) .collect(), + metadata: DownloadFormatMetadata { + skip_events: format_pairs.first().unwrap().0.skip_events().await?, + }, }), MergeBehavior::Auto => { let mut d_formats: Vec<(Duration, DownloadFormat)> = vec![]; @@ -498,6 +519,7 @@ async fn get_format( video: (video, single_format.audio.clone()), audios: vec![(audio, single_format.audio.clone())], subtitles, + metadata: DownloadFormatMetadata { skip_events: None }, }, )); } diff --git a/crunchy-cli-core/src/download/command.rs b/crunchy-cli-core/src/download/command.rs index 1f8cd8e..08756e0 100644 --- a/crunchy-cli-core/src/download/command.rs +++ b/crunchy-cli-core/src/download/command.rs @@ -1,6 +1,6 @@ use crate::download::filter::DownloadFilter; use crate::utils::context::Context; -use crate::utils::download::{DownloadBuilder, DownloadFormat}; +use crate::utils::download::{DownloadBuilder, DownloadFormat, DownloadFormatMetadata}; use crate::utils::ffmpeg::{FFmpegPreset, SOFTSUB_CONTAINERS}; use crate::utils::filter::Filter; use crate::utils::format::{Format, SingleFormat}; @@ -101,6 +101,14 @@ pub struct Download { #[arg(long, default_value_t = false)] pub(crate) skip_specials: bool, + #[arg(help = "Includes chapters (e.g. intro, credits, ...)")] + #[arg(long_help = "Includes chapters (e.g. intro, credits, ...). \ + Because chapters are essentially only special timeframes in episodes like the intro, most of the video timeline isn't covered by a chapter. + These \"gaps\" are filled with an 'Episode' chapter because many video players are ignore those gaps and just assume that a chapter ends when the next chapter start is reached, even if a specific end-time is set. + Also chapters aren't always available, so in this case, just a big 'Episode' chapter from start to end will be created")] + #[arg(long, default_value_t = false)] + pub(crate) include_chapters: bool, + #[arg(help = "Skip any interactive input")] #[arg(short, long, default_value_t = false)] pub(crate) yes: bool, @@ -342,6 +350,13 @@ async fn get_format( single_format.audio == Locale::ja_JP || stream.subtitles.len() > 1, )] }), + metadata: DownloadFormatMetadata { + skip_events: if download.include_chapters { + single_format.skip_events().await? + } else { + None + }, + }, }; let mut format = Format::from_single_formats(vec![( single_format.clone(), diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index 9a67f7f..c768195 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -4,7 +4,7 @@ use crate::utils::os::{cache_dir, is_special_file, temp_directory, temp_named_pi use crate::utils::rate_limit::RateLimiterService; use anyhow::{bail, Result}; use chrono::NaiveTime; -use crunchyroll_rs::media::{Subtitle, VariantData, VariantSegment}; +use crunchyroll_rs::media::{SkipEvents, SkipEventsEvent, Subtitle, VariantData, VariantSegment}; use crunchyroll_rs::Locale; use indicatif::{ProgressBar, ProgressDrawTarget, ProgressFinish, ProgressStyle}; use log::{debug, warn, LevelFilter}; @@ -113,6 +113,11 @@ pub struct DownloadFormat { pub video: (VariantData, Locale), pub audios: Vec<(VariantData, Locale)>, pub subtitles: Vec<(Subtitle, bool)>, + pub metadata: DownloadFormatMetadata, +} + +pub struct DownloadFormatMetadata { + pub skip_events: Option, } pub struct Downloader { @@ -205,6 +210,8 @@ impl Downloader { let mut audios = vec![]; let mut subtitles = vec![]; let mut fonts = vec![]; + let mut chapters = None; + let mut max_len = NaiveTime::MIN; let mut max_frames = 0f64; let fmt_space = self .formats @@ -243,6 +250,9 @@ impl Downloader { } let (len, fps) = get_video_stats(&video_path)?; + if max_len < len { + max_len = len + } let frames = len.signed_duration_since(NaiveTime::MIN).num_seconds() as f64 * fps; if frames > max_frames { max_frames = frames; @@ -322,6 +332,22 @@ impl Downloader { format!("#{}", i + 1) }, }); + + if let Some(skip_events) = &format.metadata.skip_events { + let (file, path) = tempfile(".chapter")?.into_parts(); + chapters = Some(( + (file, path), + [ + skip_events.recap.as_ref().map(|e| ("Recap", e)), + skip_events.intro.as_ref().map(|e| ("Intro", e)), + skip_events.credits.as_ref().map(|e| ("Credits", e)), + skip_events.preview.as_ref().map(|e| ("Preview", e)), + ] + .into_iter() + .flatten() + .collect::>(), + )); + } } if self.download_fonts @@ -440,6 +466,20 @@ impl Downloader { } } + if let Some(((file, path), chapters)) = chapters.as_mut() { + write_ffmpeg_chapters(file, max_len, chapters)?; + input.extend(["-i".to_string(), path.to_string_lossy().to_string()]); + maps.extend([ + "-map_metadata".to_string(), + (videos.len() + + audios.len() + + container_supports_softsubs + .then_some(subtitles.len()) + .unwrap_or_default()) + .to_string(), + ]) + } + let preset_custom = matches!(self.ffmpeg_preset, FFmpegPreset::Custom(_)); let (input_presets, mut output_presets) = self.ffmpeg_preset.into_input_output_args(); let fifo = temp_named_pipe()?; @@ -1156,6 +1196,54 @@ fn fix_subtitles(raw: &mut Vec, max_length: NaiveTime) { *raw = as_lines.join("\n").into_bytes() } +fn write_ffmpeg_chapters( + file: &mut fs::File, + video_len: NaiveTime, + events: &mut Vec<(&str, &SkipEventsEvent)>, +) -> Result<()> { + let video_len = video_len + .signed_duration_since(NaiveTime::MIN) + .num_seconds() as u32; + events.sort_by(|(_, event_a), (_, event_b)| event_a.start.cmp(&event_b.start)); + + writeln!(file, ";FFMETADATA1")?; + + let mut last_end_time = 0; + for (name, event) in events { + // include an extra 'Episode' chapter if the start of the current chapter is more than 10 + // seconds later than the end of the last chapter. + // this is done before writing the actual chapter of this loop to keep the chapter + // chronologically in order + if event.start as i32 - last_end_time as i32 > 10 { + writeln!(file, "[CHAPTER]")?; + writeln!(file, "TIMEBASE=1/1")?; + writeln!(file, "START={}", last_end_time)?; + writeln!(file, "END={}", event.start)?; + writeln!(file, "title=Episode")?; + } + + writeln!(file, "[CHAPTER]")?; + writeln!(file, "TIMEBASE=1/1")?; + writeln!(file, "START={}", event.start)?; + writeln!(file, "END={}", event.end)?; + writeln!(file, "title={}", name)?; + + last_end_time = event.end; + } + + // only add a traling chapter if the gab between the end of the last chapter and the total video + // length is greater than 10 seconds + if video_len as i32 - last_end_time as i32 > 10 { + writeln!(file, "[CHAPTER]")?; + writeln!(file, "TIMEBASE=1/1")?; + writeln!(file, "START={}", last_end_time)?; + writeln!(file, "END={}", video_len)?; + writeln!(file, "title=Episode")?; + } + + Ok(()) +} + async fn ffmpeg_progress( total_frames: u64, stats: R, diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index e60ef54..8956a04 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -3,7 +3,7 @@ use crate::utils::log::tab_info; use crate::utils::os::{is_special_file, sanitize}; use anyhow::Result; use chrono::{Datelike, Duration}; -use crunchyroll_rs::media::{Resolution, Stream, Subtitle, VariantData}; +use crunchyroll_rs::media::{Resolution, SkipEvents, Stream, Subtitle, VariantData}; use crunchyroll_rs::{Concert, Episode, Locale, MediaCollection, Movie, MusicVideo}; use log::{debug, info}; use std::cmp::Ordering; @@ -175,6 +175,14 @@ impl SingleFormat { Ok(stream) } + pub async fn skip_events(&self) -> Result> { + match &self.source { + MediaCollection::Episode(e) => Ok(Some(e.skip_events().await?)), + MediaCollection::Movie(m) => Ok(Some(m.skip_events().await?)), + _ => Ok(None), + } + } + pub fn source_type(&self) -> String { match &self.source { MediaCollection::Episode(_) => "episode", From 3f33db6728daab51b48ed6a150068071aeeff030 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 10 Mar 2024 02:07:05 +0100 Subject: [PATCH 023/113] Remove deprecated `openssl` and `openssl-static` features --- Cargo.toml | 4 ---- build.rs | 7 ------- 2 files changed, 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 425e078..01c5be1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,10 +13,6 @@ native-tls = ["crunchy-cli-core/native-tls"] openssl-tls = ["dep:native-tls-crate", "native-tls-crate/openssl", "crunchy-cli-core/openssl-tls"] openssl-tls-static = ["dep:native-tls-crate", "native-tls-crate/openssl", "crunchy-cli-core/openssl-tls-static"] -# deprecated -openssl = ["openssl-tls"] -openssl-static = ["openssl-tls-static"] - [dependencies] tokio = { version = "1.35", features = ["macros", "rt-multi-thread", "time"], default-features = false } diff --git a/build.rs b/build.rs index 5b464c4..313cb6d 100644 --- a/build.rs +++ b/build.rs @@ -19,13 +19,6 @@ fn main() -> std::io::Result<()> { println!("cargo:warning=Multiple tls backends are activated (through the '*-tls' features). Consider to activate only one as it is not possible to change the backend during runtime. The active backend for this build will be '{}'.", active_tls_backend) } - if cfg!(feature = "openssl") { - println!("cargo:warning=The 'openssl' feature is deprecated and will be removed in a future version. Use the 'openssl-tls' feature instead.") - } - if cfg!(feature = "openssl-static") { - println!("cargo:warning=The 'openssl-static' feature is deprecated and will be removed in a future version. Use the 'openssl-tls-static' feature instead.") - } - // note that we're using an anti-pattern here / violate the rust conventions. build script are // not supposed to write outside of 'OUT_DIR'. to have the generated files in the build "root" // (the same directory where the output binary lives) is much simpler than in 'OUT_DIR' since From f1d266c940f508a00b6a4408b70c73b263f785b7 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 10 Mar 2024 04:04:58 +0100 Subject: [PATCH 024/113] Add options to specify audio & subtitle locales as IETF language tag and add `--language_tagging` flag for `archive` and `download` to modify the output file language tagging (#330) --- crunchy-cli-core/src/archive/command.rs | 58 +++++++++-- crunchy-cli-core/src/download/command.rs | 61 +++++++++++- crunchy-cli-core/src/utils/download.rs | 26 ++++- crunchy-cli-core/src/utils/format.rs | 10 +- crunchy-cli-core/src/utils/locale.rs | 120 +++++++++++++++++++++++ 5 files changed, 260 insertions(+), 15 deletions(-) diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index bf61fdf..600065a 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -6,7 +6,7 @@ use crate::utils::download::{ use crate::utils::ffmpeg::FFmpegPreset; use crate::utils::filter::Filter; use crate::utils::format::{Format, SingleFormat}; -use crate::utils::locale::all_locale_in_locales; +use crate::utils::locale::{all_locale_in_locales, resolve_locales, LanguageTagging}; use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file}; use crate::utils::parse::parse_url; @@ -20,6 +20,7 @@ use crunchyroll_rs::Locale; use log::{debug, warn}; use regex::Regex; use std::fmt::{Display, Formatter}; +use std::iter::zip; use std::ops::Sub; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; @@ -31,15 +32,19 @@ pub struct Archive { #[arg(help = format!("Audio languages. Can be used multiple times. \ Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] #[arg(long_help = format!("Audio languages. Can be used multiple times. \ - Available languages are:\n {}", Locale::all().into_iter().map(|l| format!("{:<6} → {}", l.to_string(), l.to_human_readable())).collect::>().join("\n ")))] + Available languages are:\n {}\nIETF tagged language codes for the shown available locales can be used too", Locale::all().into_iter().map(|l| format!("{:<6} → {}", l.to_string(), l.to_human_readable())).collect::>().join("\n ")))] #[arg(short, long, default_values_t = vec![Locale::ja_JP, crate::utils::locale::system_locale()])] pub(crate) audio: Vec, + #[arg(skip)] + output_audio_locales: Vec, #[arg(help = format!("Subtitle languages. Can be used multiple times. \ Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] #[arg(long_help = format!("Subtitle languages. Can be used multiple times. \ - Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] + Available languages are: {}\nIETF tagged language codes for the shown available locales can be used too", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] #[arg(short, long, default_values_t = Locale::all())] pub(crate) subtitle: Vec, + #[arg(skip)] + output_subtitle_locales: Vec, #[arg(help = "Name of the output file")] #[arg(long_help = "Name of the output file. \ @@ -95,12 +100,22 @@ pub struct Archive { #[arg(short, long, default_value = "auto")] #[arg(value_parser = MergeBehavior::parse)] pub(crate) merge: MergeBehavior, - #[arg( help = "If the merge behavior is 'auto', only download multiple video tracks if their length difference is higher than the given milliseconds" )] #[arg(long, default_value_t = 200)] pub(crate) merge_auto_tolerance: u32, + #[arg( + long, + help = "Specified which language tagging the audio and subtitle tracks and language specific format options should have. \ + Valid options are: 'default' (how Crunchyroll uses it internally), 'ietf' (according to the IETF standard)" + )] + #[arg( + long_help = "Specified which language tagging the audio and subtitle tracks and language specific format options should have. \ + Valid options are: 'default' (how Crunchyroll uses it internally), 'ietf' (according to the IETF standard; you might run in issues as there are multiple locales which resolve to the same IETF language code, e.g. 'es-LA' and 'es-ES' are both resolving to 'es')" + )] + #[arg(value_parser = LanguageTagging::parse)] + pub(crate) language_tagging: Option, #[arg(help = format!("Presets for converting the video to a specific coding format. \ Available presets: \n {}", FFmpegPreset::available_matches_human_readable().join("\n ")))] @@ -217,6 +232,26 @@ impl Execute for Archive { self.audio = all_locale_in_locales(self.audio.clone()); self.subtitle = all_locale_in_locales(self.subtitle.clone()); + if let Some(language_tagging) = &self.language_tagging { + self.audio = resolve_locales(&self.audio); + self.subtitle = resolve_locales(&self.subtitle); + self.output_audio_locales = language_tagging.convert_locales(&self.audio); + self.output_subtitle_locales = language_tagging.convert_locales(&self.subtitle); + } else { + self.output_audio_locales = self + .audio + .clone() + .into_iter() + .map(|l| l.to_string()) + .collect(); + self.output_subtitle_locales = self + .subtitle + .clone() + .into_iter() + .map(|l| l.to_string()) + .collect(); + } + Ok(()) } @@ -259,7 +294,13 @@ impl Execute for Archive { .audio_sort(Some(self.audio.clone())) .subtitle_sort(Some(self.subtitle.clone())) .no_closed_caption(self.no_closed_caption) - .threads(self.threads); + .threads(self.threads) + .audio_locale_output_map( + zip(self.audio.clone(), self.output_audio_locales.clone()).collect(), + ) + .subtitle_locale_output_map( + zip(self.subtitle.clone(), self.output_subtitle_locales.clone()).collect(), + ); for single_formats in single_format_collection.into_iter() { let (download_formats, mut format) = get_format(&self, &single_formats).await?; @@ -275,9 +316,14 @@ impl Execute for Archive { .as_ref() .map_or((&self.output).into(), |so| so.into()), self.universal_output, + self.language_tagging.as_ref(), ) } else { - format.format_path((&self.output).into(), self.universal_output) + format.format_path( + (&self.output).into(), + self.universal_output, + self.language_tagging.as_ref(), + ) }; let (mut path, changed) = free_file(formatted_path.clone()); diff --git a/crunchy-cli-core/src/download/command.rs b/crunchy-cli-core/src/download/command.rs index 08756e0..843f5cd 100644 --- a/crunchy-cli-core/src/download/command.rs +++ b/crunchy-cli-core/src/download/command.rs @@ -4,6 +4,7 @@ use crate::utils::download::{DownloadBuilder, DownloadFormat, DownloadFormatMeta use crate::utils::ffmpeg::{FFmpegPreset, SOFTSUB_CONTAINERS}; use crate::utils::filter::Filter; use crate::utils::format::{Format, SingleFormat}; +use crate::utils::locale::{resolve_locales, LanguageTagging}; use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file}; use crate::utils::parse::parse_url; @@ -14,6 +15,7 @@ use anyhow::Result; use crunchyroll_rs::media::Resolution; use crunchyroll_rs::Locale; use log::{debug, warn}; +use std::collections::HashMap; use std::path::Path; #[derive(Clone, Debug, clap::Parser)] @@ -23,14 +25,18 @@ pub struct Download { #[arg(help = format!("Audio language. Can only be used if the provided url(s) point to a series. \ Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] #[arg(long_help = format!("Audio language. Can only be used if the provided url(s) point to a series. \ - Available languages are:\n {}", Locale::all().into_iter().map(|l| format!("{:<6} → {}", l.to_string(), l.to_human_readable())).collect::>().join("\n ")))] + Available languages are:\n {}\nIETF tagged language codes for the shown available locales can be used too", Locale::all().into_iter().map(|l| format!("{:<6} → {}", l.to_string(), l.to_human_readable())).collect::>().join("\n ")))] #[arg(short, long, default_value_t = crate::utils::locale::system_locale())] pub(crate) audio: Locale, + #[arg(skip)] + output_audio_locale: String, #[arg(help = format!("Subtitle language. Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] #[arg(long_help = format!("Subtitle language. If set, the subtitle will be burned into the video and cannot be disabled. \ - Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] + Available languages are: {}\nIETF tagged language codes for the shown available locales can be used too", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] #[arg(short, long)] pub(crate) subtitle: Option, + #[arg(skip)] + output_subtitle_locale: String, #[arg(help = "Name of the output file")] #[arg(long_help = "Name of the output file. \ @@ -75,6 +81,18 @@ pub struct Download { #[arg(value_parser = crate::utils::clap::clap_parse_resolution)] pub(crate) resolution: Resolution, + #[arg( + long, + help = "Specified which language tagging the audio and subtitle tracks and language specific format options should have. \ + Valid options are: 'default' (how Crunchyroll uses it internally), 'ietf' (according to the IETF standard)" + )] + #[arg( + long_help = "Specified which language tagging the audio and subtitle tracks and language specific format options should have. \ + Valid options are: 'default' (how Crunchyroll uses it internally), 'ietf' (according to the IETF standard; you might run in issues as there are multiple locales which resolve to the same IETF language code, e.g. 'es-LA' and 'es-ES' are both resolving to 'es')" + )] + #[arg(value_parser = LanguageTagging::parse)] + pub(crate) language_tagging: Option, + #[arg(help = format!("Presets for converting the video to a specific coding format. \ Available presets: \n {}", FFmpegPreset::available_matches_human_readable().join("\n ")))] #[arg(long_help = format!("Presets for converting the video to a specific coding format. \ @@ -178,6 +196,27 @@ impl Execute for Download { warn!("The '{{resolution}}' format option is deprecated and will be removed in a future version. Please use '{{width}}' and '{{height}}' instead") } + if let Some(language_tagging) = &self.language_tagging { + self.audio = resolve_locales(&[self.audio.clone()]).remove(0); + self.subtitle = self + .subtitle + .as_ref() + .map(|s| resolve_locales(&[s.clone()]).remove(0)); + self.output_audio_locale = language_tagging.for_locale(&self.audio); + self.output_subtitle_locale = self + .subtitle + .as_ref() + .map(|s| language_tagging.for_locale(s)) + .unwrap_or_default() + } else { + self.output_audio_locale = self.audio.to_string(); + self.output_subtitle_locale = self + .subtitle + .as_ref() + .map(|s| s.to_string()) + .unwrap_or_default(); + } + Ok(()) } @@ -240,7 +279,16 @@ impl Execute for Download { }) .ffmpeg_preset(self.ffmpeg_preset.clone().unwrap_or_default()) .ffmpeg_threads(self.ffmpeg_threads) - .threads(self.threads); + .threads(self.threads) + .audio_locale_output_map(HashMap::from([( + self.audio.clone(), + self.output_audio_locale.clone(), + )])) + .subtitle_locale_output_map( + self.subtitle.as_ref().map_or(HashMap::new(), |s| { + HashMap::from([(s.clone(), self.output_subtitle_locale.clone())]) + }), + ); for mut single_formats in single_format_collection.into_iter() { // the vec contains always only one item @@ -268,9 +316,14 @@ impl Execute for Download { .as_ref() .map_or((&self.output).into(), |so| so.into()), self.universal_output, + self.language_tagging.as_ref(), ) } else { - format.format_path((&self.output).into(), self.universal_output) + format.format_path( + (&self.output).into(), + self.universal_output, + self.language_tagging.as_ref(), + ) }; let (path, changed) = free_file(formatted_path.clone()); diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index c768195..f3dc631 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -12,7 +12,7 @@ use regex::Regex; use reqwest::Client; use std::borrow::Borrow; use std::cmp::Ordering; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use std::io::Write; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; @@ -61,6 +61,8 @@ pub struct DownloadBuilder { no_closed_caption: bool, threads: usize, ffmpeg_threads: Option, + audio_locale_output_map: HashMap, + subtitle_locale_output_map: HashMap, } impl DownloadBuilder { @@ -78,6 +80,8 @@ impl DownloadBuilder { no_closed_caption: false, threads: num_cpus::get(), ffmpeg_threads: None, + audio_locale_output_map: HashMap::new(), + subtitle_locale_output_map: HashMap::new(), } } @@ -99,6 +103,9 @@ impl DownloadBuilder { ffmpeg_threads: self.ffmpeg_threads, formats: vec![], + + audio_locale_output_map: self.audio_locale_output_map, + subtitle_locale_output_map: self.subtitle_locale_output_map, } } } @@ -138,6 +145,9 @@ pub struct Downloader { ffmpeg_threads: Option, formats: Vec, + + audio_locale_output_map: HashMap, + subtitle_locale_output_map: HashMap, } impl Downloader { @@ -426,7 +436,12 @@ impl Downloader { maps.extend(["-map".to_string(), (i + videos.len()).to_string()]); metadata.extend([ format!("-metadata:s:a:{}", i), - format!("language={}", meta.language), + format!( + "language={}", + self.audio_locale_output_map + .get(&meta.language) + .unwrap_or(&meta.language.to_string()) + ), ]); metadata.extend([ format!("-metadata:s:a:{}", i), @@ -457,7 +472,12 @@ impl Downloader { ]); metadata.extend([ format!("-metadata:s:s:{}", i), - format!("language={}", meta.language), + format!( + "language={}", + self.subtitle_locale_output_map + .get(&meta.language) + .unwrap_or(&meta.language.to_string()) + ), ]); metadata.extend([ format!("-metadata:s:s:{}", i), diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index 8956a04..7146a55 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -1,4 +1,5 @@ use crate::utils::filter::real_dedup_vec; +use crate::utils::locale::LanguageTagging; use crate::utils::log::tab_info; use crate::utils::os::{is_special_file, sanitize}; use anyhow::Result; @@ -417,7 +418,12 @@ impl Format { } /// Formats the given string if it has specific pattern in it. It also sanitizes the filename. - pub fn format_path(&self, path: PathBuf, universal: bool) -> PathBuf { + pub fn format_path( + &self, + path: PathBuf, + universal: bool, + language_tagging: Option<&LanguageTagging>, + ) -> PathBuf { let path = path .to_string_lossy() .to_string() @@ -427,7 +433,7 @@ impl Format { &sanitize( self.locales .iter() - .map(|(a, _)| a.to_string()) + .map(|(a, _)| language_tagging.map_or(a.to_string(), |t| t.for_locale(a))) .collect::>() .join( &env::var("CRUNCHY_CLI_FORMAT_DELIMITER") diff --git a/crunchy-cli-core/src/utils/locale.rs b/crunchy-cli-core/src/utils/locale.rs index 8651078..0827299 100644 --- a/crunchy-cli-core/src/utils/locale.rs +++ b/crunchy-cli-core/src/utils/locale.rs @@ -1,4 +1,124 @@ use crunchyroll_rs::Locale; +use log::warn; + +#[derive(Clone, Debug)] +#[allow(clippy::upper_case_acronyms)] +pub enum LanguageTagging { + Default, + IETF, +} + +impl LanguageTagging { + pub fn parse(s: &str) -> Result { + Ok(match s.to_lowercase().as_str() { + "default" => Self::Default, + "ietf" => Self::IETF, + _ => return Err(format!("'{}' is not a valid language tagging", s)), + }) + } + + pub fn convert_locales(&self, locales: &[Locale]) -> Vec { + let ietf_language_codes = ietf_language_codes(); + let mut converted = vec![]; + + match &self { + LanguageTagging::Default => { + for locale in locales { + let Some((_, available)) = + ietf_language_codes.iter().find(|(_, l)| l.contains(locale)) + else { + // if no matching IETF language code was found, just pass it as it is + converted.push(locale.to_string()); + continue; + }; + converted.push(available.first().unwrap().to_string()) + } + } + LanguageTagging::IETF => { + for locale in locales { + let Some((tag, _)) = + ietf_language_codes.iter().find(|(_, l)| l.contains(locale)) + else { + // if no matching IETF language code was found, just pass it as it is + converted.push(locale.to_string()); + continue; + }; + converted.push(tag.to_string()) + } + } + } + + converted + } + + pub fn for_locale(&self, locale: &Locale) -> String { + match &self { + LanguageTagging::Default => ietf_language_codes() + .iter() + .find(|(_, l)| l.contains(locale)) + .map_or(locale.to_string(), |(_, l)| l[0].to_string()), + LanguageTagging::IETF => ietf_language_codes() + .iter() + .find(|(_, l)| l.contains(locale)) + .map_or(locale.to_string(), |(tag, _)| tag.to_string()), + } + } +} + +pub fn resolve_locales(locales: &[Locale]) -> Vec { + let ietf_language_codes = ietf_language_codes(); + let all_locales = Locale::all(); + + let mut resolved = vec![]; + for locale in locales { + if all_locales.contains(locale) { + resolved.push(locale.clone()) + } else if let Some((_, resolved_locales)) = ietf_language_codes + .iter() + .find(|(tag, _)| tag == &locale.to_string().as_str()) + { + let (first, alternatives) = resolved_locales.split_first().unwrap(); + + resolved.push(first.clone()); + // ignoring `Locale::en_IN` because I think the majority of users which want english + // audio / subs want the "actual" english version and not the hindi accent dub + if !alternatives.is_empty() && resolved_locales.first().unwrap() != &Locale::en_IN { + warn!("Resolving locale '{}' to '{}', but there are some alternatives: {}. If you an alternative instead, please write it completely out instead of '{}'", locale, first, alternatives.iter().map(|l| format!("'{l}'")).collect::>().join(", "), locale) + } + } else { + resolved.push(locale.clone()); + warn!("Unknown locale '{}'", locale) + } + } + + resolved +} + +fn ietf_language_codes<'a>() -> Vec<(&'a str, Vec)> { + vec![ + ("ar", vec![Locale::ar_ME, Locale::ar_SA]), + ("ca", vec![Locale::ca_ES]), + ("de", vec![Locale::de_DE]), + ("en", vec![Locale::en_US, Locale::hi_IN]), + ("es", vec![Locale::es_ES, Locale::es_419, Locale::es_LA]), + ("fr", vec![Locale::fr_FR]), + ("hi", vec![Locale::hi_IN]), + ("id", vec![Locale::id_ID]), + ("it", vec![Locale::it_IT]), + ("ja", vec![Locale::ja_JP]), + ("ko", vec![Locale::ko_KR]), + ("ms", vec![Locale::ms_MY]), + ("pl", vec![Locale::pl_PL]), + ("pt", vec![Locale::pt_PT, Locale::pt_BR]), + ("ru", vec![Locale::ru_RU]), + ("ta", vec![Locale::ta_IN]), + ("te", vec![Locale::te_IN]), + ("th", vec![Locale::th_TH]), + ("tr", vec![Locale::tr_TR]), + ("vi", vec![Locale::vi_VN]), + ("zh", vec![Locale::zh_CN, Locale::zh_HK, Locale::zh_TW]), + ] +} /// Return the locale of the system. pub fn system_locale() -> Locale { From e3a7fd92468a9cc1b61556f4eefff4d9c08e95b8 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 10 Mar 2024 12:49:47 +0100 Subject: [PATCH 025/113] Add option so specify different proxies for api and download requests (#282) --- crunchy-cli-core/src/lib.rs | 86 +++++++++++++++++------------- crunchy-cli-core/src/utils/clap.rs | 37 ++++++++++++- 2 files changed, 85 insertions(+), 38 deletions(-) diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index d362a2c..9dd3360 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -61,9 +61,10 @@ pub struct Cli { #[arg(help = "Use a proxy to route all traffic through")] #[arg(long_help = "Use a proxy to route all traffic through. \ - Make sure that the proxy can either forward TLS requests, which is needed to bypass the (cloudflare) bot protection, or that it is configured so that the proxy can bypass the protection itself")] - #[arg(global = true, long, value_parser = crate::utils::clap::clap_parse_proxy)] - proxy: Option, + Make sure that the proxy can either forward TLS requests, which is needed to bypass the (cloudflare) bot protection, or that it is configured so that the proxy can bypass the protection itself. \ + Besides specifying a simple url, you also can partially control where a proxy should be used: ':' only proxies api requests, ':' only proxies download traffic, ':' proxies api requests through the first url and download traffic through the second url")] + #[arg(global = true, long, value_parser = crate::utils::clap::clap_parse_proxies)] + proxy: Option<(Option, Option)>, #[arg(help = "Use custom user agent")] #[arg(global = true, long)] @@ -238,43 +239,29 @@ async fn execute_executor(executor: impl Execute, ctx: Context) { } async fn create_ctx(cli: &mut Cli) -> Result { - let client = { - let mut builder = CrunchyrollBuilder::predefined_client_builder(); - if let Some(p) = &cli.proxy { - builder = builder.proxy(p.clone()) - } - if let Some(ua) = &cli.user_agent { - builder = builder.user_agent(ua) - } + let crunchy_client = reqwest_client( + cli.proxy.as_ref().and_then(|p| p.0.clone()), + cli.user_agent.clone(), + ); + let internal_client = reqwest_client( + cli.proxy.as_ref().and_then(|p| p.1.clone()), + cli.user_agent.clone(), + ); - #[cfg(any(feature = "openssl-tls", feature = "openssl-tls-static"))] - let client = { - let mut builder = builder.use_native_tls().tls_built_in_root_certs(false); - - for certificate in rustls_native_certs::load_native_certs().unwrap() { - builder = builder.add_root_certificate( - reqwest::Certificate::from_der(certificate.0.as_slice()).unwrap(), - ) - } - - builder.build().unwrap() - }; - #[cfg(not(any(feature = "openssl-tls", feature = "openssl-tls-static")))] - let client = builder.build().unwrap(); - - client - }; - - let rate_limiter = cli - .speed_limit - .map(|l| RateLimiterService::new(l, client.clone())); - - let crunchy = crunchyroll_session(cli, client.clone(), rate_limiter.clone()).await?; + let crunchy = crunchyroll_session( + cli, + crunchy_client.clone(), + cli.speed_limit + .map(|l| RateLimiterService::new(l, crunchy_client)), + ) + .await?; Ok(Context { crunchy, - client, - rate_limiter, + client: internal_client.clone(), + rate_limiter: cli + .speed_limit + .map(|l| RateLimiterService::new(l, internal_client)), }) } @@ -372,3 +359,30 @@ async fn crunchyroll_session( Ok(crunchy) } + +fn reqwest_client(proxy: Option, user_agent: Option) -> Client { + let mut builder = CrunchyrollBuilder::predefined_client_builder(); + if let Some(p) = proxy { + builder = builder.proxy(p) + } + if let Some(ua) = user_agent { + builder = builder.user_agent(ua) + } + + #[cfg(any(feature = "openssl-tls", feature = "openssl-tls-static"))] + let client = { + let mut builder = builder.use_native_tls().tls_built_in_root_certs(false); + + for certificate in rustls_native_certs::load_native_certs().unwrap() { + builder = builder.add_root_certificate( + reqwest::Certificate::from_der(certificate.0.as_slice()).unwrap(), + ) + } + + builder.build().unwrap() + }; + #[cfg(not(any(feature = "openssl-tls", feature = "openssl-tls-static")))] + let client = builder.build().unwrap(); + + client +} diff --git a/crunchy-cli-core/src/utils/clap.rs b/crunchy-cli-core/src/utils/clap.rs index 37a34d3..35de71f 100644 --- a/crunchy-cli-core/src/utils/clap.rs +++ b/crunchy-cli-core/src/utils/clap.rs @@ -1,13 +1,46 @@ use crate::utils::parse::parse_resolution; use crunchyroll_rs::media::Resolution; +use regex::Regex; use reqwest::Proxy; pub fn clap_parse_resolution(s: &str) -> Result { parse_resolution(s.to_string()).map_err(|e| e.to_string()) } -pub fn clap_parse_proxy(s: &str) -> Result { - Proxy::all(s).map_err(|e| e.to_string()) +pub fn clap_parse_proxies(s: &str) -> Result<(Option, Option), String> { + let double_proxy_regex = + Regex::new(r"^(?P(https?|socks5h?)://.+):(?P(https?|socks5h?)://.+)$") + .unwrap(); + + if let Some(capture) = double_proxy_regex.captures(s) { + // checks if the input is formatted like 'https://example.com:socks5://examples.com' and + // splits the string into 2 separate proxies at the middle colon + + let first = capture.name("first").unwrap().as_str(); + let second = capture.name("second").unwrap().as_str(); + Ok(( + Some(Proxy::all(first).map_err(|e| format!("first proxy: {e}"))?), + Some(Proxy::all(second).map_err(|e| format!("second proxy: {e}"))?), + )) + } else if s.starts_with(':') { + // checks if the input is formatted like ':https://example.com' and returns a proxy on the + // second tuple position + Ok(( + None, + Some(Proxy::all(s.trim_start_matches(':')).map_err(|e| e.to_string())?), + )) + } else if s.ends_with(':') { + // checks if the input is formatted like 'https://example.com:' and returns a proxy on the + // first tuple position + Ok(( + Some(Proxy::all(s.trim_end_matches(':')).map_err(|e| e.to_string())?), + None, + )) + } else { + // returns the same proxy for both tuple positions + let proxy = Proxy::all(s).map_err(|e| e.to_string())?; + Ok((Some(proxy.clone()), Some(proxy))) + } } pub fn clap_parse_speed_limit(s: &str) -> Result { From 3bf24587745b99d963826cb293cadf9603c186f2 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 10 Mar 2024 13:28:18 +0100 Subject: [PATCH 026/113] Pass command args manually to cli entrypoint instead of parsing from environment --- crunchy-cli-core/src/lib.rs | 4 ++-- src/main.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index 9dd3360..38f22f4 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -117,8 +117,8 @@ struct Verbosity { quiet: bool, } -pub async fn cli_entrypoint() { - let mut cli: Cli = Cli::parse(); +pub async fn main(args: &[String]) { + let mut cli: Cli = Cli::parse_from(args); if cli.verbosity.verbose || cli.verbosity.quiet { if cli.verbosity.verbose && cli.verbosity.quiet { diff --git a/src/main.rs b/src/main.rs index da3c699..9d44bef 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,5 +8,5 @@ compile_error!("At least one tls feature must be activated"); #[tokio::main] async fn main() { - crunchy_cli_core::cli_entrypoint().await + crunchy_cli_core::main(&std::env::args().collect::>()).await } From 013273b832387aa9e3818b3abca97956342e73a9 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 10 Mar 2024 13:40:16 +0100 Subject: [PATCH 027/113] Format code --- crunchy-cli-core/src/archive/command.rs | 2 +- crunchy-cli-core/src/archive/filter.rs | 2 +- crunchy-cli-core/src/utils/download.rs | 20 +++++++++----------- crunchy-cli-core/src/utils/parse.rs | 2 +- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index 600065a..8258b36 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -593,7 +593,7 @@ fn get_video_streams(path: &Path) -> Result, Vec)>> .stdout(Stdio::null()) .stderr(Stdio::piped()) .arg("-hide_banner") - .args(["-i", &path.to_string_lossy().to_string()]) + .args(["-i", &path.to_string_lossy()]) .output()?; let ffmpeg_output = String::from_utf8(ffmpeg.stderr)?; diff --git a/crunchy-cli-core/src/archive/filter.rs b/crunchy-cli-core/src/archive/filter.rs index 2a47738..90ab373 100644 --- a/crunchy-cli-core/src/archive/filter.rs +++ b/crunchy-cli-core/src/archive/filter.rs @@ -180,7 +180,7 @@ impl Filter for ArchiveFilter { Some( season .audio_locales - .get(0) + .first() .cloned() .unwrap_or(Locale::ja_JP), ) diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index f3dc631..7f81bc9 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -793,7 +793,7 @@ impl Downloader { .await? .bytes() .await?; - fs::write(&file, font.to_vec())?; + fs::write(&file, font)?; Ok(Some((file, false))) } @@ -990,20 +990,18 @@ fn get_video_stats(path: &Path) -> Result<(NaiveTime, f64)> { .args(["-i", path.to_str().unwrap()]) .output()?; let ffmpeg_output = String::from_utf8(ffmpeg.stderr)?; - let length_caps = video_length.captures(ffmpeg_output.as_str()).map_or( - Err(anyhow::anyhow!( + let length_caps = video_length + .captures(ffmpeg_output.as_str()) + .ok_or(anyhow::anyhow!( "failed to get video length: {}", ffmpeg_output - )), - Ok, - )?; - let fps_caps = video_fps.captures(ffmpeg_output.as_str()).map_or( - Err(anyhow::anyhow!( + ))?; + let fps_caps = video_fps + .captures(ffmpeg_output.as_str()) + .ok_or(anyhow::anyhow!( "failed to get video fps: {}", ffmpeg_output - )), - Ok, - )?; + ))?; Ok(( NaiveTime::parse_from_str(length_caps.name("time").unwrap().as_str(), "%H:%M:%S%.f") diff --git a/crunchy-cli-core/src/utils/parse.rs b/crunchy-cli-core/src/utils/parse.rs index 70cbfbb..2c4a53b 100644 --- a/crunchy-cli-core/src/utils/parse.rs +++ b/crunchy-cli-core/src/utils/parse.rs @@ -147,7 +147,7 @@ pub async fn parse_url( url = crunchy.client().get(&url).send().await?.url().to_string() } - let parsed_url = crunchyroll_rs::parse_url(url).map_or(Err(anyhow!("Invalid url")), Ok)?; + let parsed_url = crunchyroll_rs::parse_url(url).ok_or(anyhow!("Invalid url"))?; debug!("Url type: {:?}", parsed_url); let media_collection = match parsed_url { UrlType::Series(id) From a0fa2bfd8a131a213457a977394ad4d078ac4317 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 10 Mar 2024 13:40:26 +0100 Subject: [PATCH 028/113] Update dependencies and version --- Cargo.lock | 45 ++++++++++++++++++---- Cargo.toml | 8 ++-- README.md | 76 +++++++++++++++++++++++++++++++++++++ crunchy-cli-core/Cargo.toml | 14 +++---- crunchy-cli-core/src/lib.rs | 5 +-- 5 files changed, 126 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7c2a6f4..99b223b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -220,6 +220,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + [[package]] name = "chrono" version = "0.4.35" @@ -378,7 +384,7 @@ dependencies = [ [[package]] name = "crunchy-cli" -version = "3.2.5" +version = "3.3.0" dependencies = [ "chrono", "clap", @@ -391,7 +397,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.2.5" +version = "3.3.0" dependencies = [ "anyhow", "async-speed-limit", @@ -408,7 +414,7 @@ dependencies = [ "indicatif", "lazy_static", "log", - "nix", + "nix 0.28.0", "num_cpus", "regex", "reqwest", @@ -479,7 +485,7 @@ version = "3.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b467862cc8610ca6fc9a1532d7777cee0804e678ab45410897b9396495994a0b" dependencies = [ - "nix", + "nix 0.27.1", "windows-sys 0.52.0", ] @@ -1174,6 +1180,18 @@ dependencies = [ "libc", ] +[[package]] +name = "nix" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +dependencies = [ + "bitflags 2.4.2", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -1441,7 +1459,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rustls", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", @@ -1515,12 +1533,13 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.6.3" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 2.1.1", + "rustls-pki-types", "schannel", "security-framework", ] @@ -1534,6 +1553,16 @@ dependencies = [ "base64", ] +[[package]] +name = "rustls-pemfile" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f48172685e6ff52a556baa527774f61fcaa884f59daf3375c62a3f1cd2549dab" +dependencies = [ + "base64", + "rustls-pki-types", +] + [[package]] name = "rustls-pki-types" version = "1.3.1" diff --git a/Cargo.toml b/Cargo.toml index 01c5be1..cf4f676 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.2.5" +version = "3.3.0" edition = "2021" license = "MIT" @@ -14,7 +14,7 @@ openssl-tls = ["dep:native-tls-crate", "native-tls-crate/openssl", "crunchy-cli- openssl-tls-static = ["dep:native-tls-crate", "native-tls-crate/openssl", "crunchy-cli-core/openssl-tls-static"] [dependencies] -tokio = { version = "1.35", features = ["macros", "rt-multi-thread", "time"], default-features = false } +tokio = { version = "1.36", features = ["macros", "rt-multi-thread", "time"], default-features = false } native-tls-crate = { package = "native-tls", version = "0.2.11", optional = true } @@ -22,8 +22,8 @@ crunchy-cli-core = { path = "./crunchy-cli-core" } [build-dependencies] chrono = "0.4" -clap = { version = "4.4", features = ["string"] } -clap_complete = "4.4" +clap = { version = "4.5", features = ["string"] } +clap_complete = "4.5" clap_mangen = "0.2" crunchy-cli-core = { path = "./crunchy-cli-core" } diff --git a/README.md b/README.md index 57a8b7c..46bee72 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,8 @@ You can set specific settings which will be The `--proxy` flag supports https and socks5 proxies to route all your traffic through. This may be helpful to bypass the geo-restrictions Crunchyroll has on certain series. + You are also able to set in which part of the cli a proxy should be used. + Instead of a normal url you can also use: `:` (only proxies api requests), `:` (only proxies download traffic), `:` (proxies api requests through the first url and download traffic through the second url). ```shell $ crunchy-cli --proxy socks5://127.0.0.1:8080 @@ -283,6 +285,14 @@ The `download` command lets you download episodes with a specific audio language Default is the template, set by the `-o` / `--output` flag. See the [Template Options section](#output-template-options) below for more options. +- Universal output + + The output template options can be forced to get sanitized via the `--universal-output` flag to be valid across all supported operating systems (Windows has a lot of characters which aren't allowed in filenames...). + + ```shell + $ crunchy-cli download --universal-output -o https://www.crunchyroll.com/watch/G7PU4XD48/tales-veldoras-journal-2 + ``` + - Resolution The resolution for videos can be set via the `-r` / `--resolution` flag. @@ -293,6 +303,15 @@ The `download` command lets you download episodes with a specific audio language Default is `best`. +- Language tagging + + You can force the usage of a specific language tagging in the output file with the `--language-tagging` flag. + This might be useful as some video players doesn't recognize the language tagging Crunchyroll uses internally. + + ```shell + $ crunchy-cli download --language-tagging ietf https://www.crunchyroll.com/watch/GRDQPM1ZY/alone-and-lonesome + ``` + - FFmpeg Preset You can specify specific built-in presets with the `--ffmpeg-preset` flag to convert videos to a specific coding while downloading. @@ -327,6 +346,15 @@ The `download` command lets you download episodes with a specific audio language $ crunchy-cli download --skip-specials https://www.crunchyroll.com/series/GYZJ43JMR/that-time-i-got-reincarnated-as-a-slime[S2] ``` +- Include chapters + + Crunchyroll sometimes provide information about skippable events like the intro or credits. + These information can be stored as chapters in the resulting video file via the `--include-chapters` flag. + + ```shell + $ crunchy-cli download --include-chapters https://www.crunchyroll.com/watch/G0DUND0K2/the-journeys-end + ``` + - Yes Sometimes different seasons have the same season number (e.g. Sword Art Online Alicization and Alicization War of Underworld are both marked as season 3), in such cases an interactive prompt is shown which needs user further user input to decide which season to download. @@ -416,6 +444,14 @@ The `archive` command lets you download episodes with multiple audios and subtit Default is the template, set by the `-o` / `--output` flag. See the [Template Options section](#output-template-options) below for more options. +- Universal output + + The output template options can be forced to get sanitized via the `--universal-output` flag to be valid across all supported operating systems (Windows has a lot of characters which aren't allowed in filenames...). + + ```shell + $ crunchy-cli archive --universal-output -o https://www.crunchyroll.com/watch/G7PU4XD48/tales-veldoras-journal-2 + ``` + - Resolution The resolution for videos can be set via the `-r` / `--resolution` flag. @@ -441,6 +477,26 @@ The `archive` command lets you download episodes with multiple audios and subtit Default is `auto`. +- Merge auto tolerance + + Sometimes two video tracks are downloaded with `--merge` set to `auto` even if they only differ some milliseconds in length which shouldn't be noticeable to the viewer. + To prevent this, you can specify a range in milliseconds with the `--merge-auto-tolerance` flag that only downloads one video if the length difference is in the given range. + + ```shell + $ crunchy-cli archive -m auto --merge-auto-tolerance 100 https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx + ``` + + Default are `200` milliseconds. + +- Language tagging + + You can force the usage of a specific language tagging in the output file with the `--language-tagging` flag. + This might be useful as some video players doesn't recognize the language tagging Crunchyroll uses internally. + + ```shell + $ crunchy-cli archive --language-tagging ietf https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx + ``` + - FFmpeg Preset You can specify specific built-in presets with the `--ffmpeg-preset` flag to convert videos to a specific coding while downloading. @@ -477,6 +533,16 @@ The `archive` command lets you download episodes with multiple audios and subtit $ crunchy-cli archive --include-fonts https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` +- Include chapters + + Crunchyroll sometimes provide information about skippable events like the intro or credits. + These information can be stored as chapters in the resulting video file via the `--include-chapters` flag. + This flag only works if `--merge` is set to `audio` because chapters cannot be mapped to a specific video steam. + + ```shell + $ crunchy-cli archive --include-chapters https://www.crunchyroll.com/watch/G0DUND0K2/the-journeys-end + ``` + - Skip existing If you re-download a series but want to skip episodes you've already downloaded, the `--skip-existing` flag skips the already existing/downloaded files. @@ -485,6 +551,16 @@ The `archive` command lets you download episodes with multiple audios and subtit $ crunchy-cli archive --skip-existing https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` +- Skip existing method + + By default, already existing files are determined by their name and the download of the corresponding episode is skipped. + But sometimes Crunchyroll adds dubs or subs to an already existing episode and these changes aren't recognized and `--skip-existing` just skips it. + This behavior can be changed by the `--skip-existing-method` flag. Valid options are `audio` and `subtitle` (if the file already exists but the audio/subtitle are less from what should be downloaded, the episode gets downloaded and the file overwritten). + + ```shell + $ crunchy-cli archive --skip-existing-method audio --skip-existing-method video https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx + ``` + - Skip specials If you doesn't want to download special episodes, use the `--skip-specials` flag to skip the download of them. diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 2d3ac61..fcb178a 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.2.5" +version = "3.3.0" edition = "2021" license = "MIT" @@ -14,7 +14,7 @@ openssl-tls-static = ["reqwest/native-tls", "reqwest/native-tls-alpn", "reqwest/ [dependencies] anyhow = "1.0" async-speed-limit = "0.4" -clap = { version = "4.4", features = ["derive", "string"] } +clap = { version = "4.5", features = ["derive", "string"] } chrono = "0.4" crunchyroll-rs = { version = "0.8.5", features = ["dash-stream", "experimental-stabilizations", "tower"] } ctrlc = "3.4" @@ -33,16 +33,16 @@ reqwest = { version = "0.11", default-features = false, features = ["socks", "st serde = "1.0" serde_json = "1.0" serde_plain = "1.0" -shlex = "1.2" +shlex = "1.3" sys-locale = "0.3" -tempfile = "3.9" -tokio = { version = "1.35", features = ["io-util", "macros", "net", "rt-multi-thread", "time"] } +tempfile = "3.10" +tokio = { version = "1.36", features = ["io-util", "macros", "net", "rt-multi-thread", "time"] } tokio-util = "0.7" tower-service = "0.3" -rustls-native-certs = { version = "0.6", optional = true } +rustls-native-certs = { version = "0.7", optional = true } [target.'cfg(not(target_os = "windows"))'.dependencies] -nix = { version = "0.27", features = ["fs"] } +nix = { version = "0.28", features = ["fs"] } [build-dependencies] chrono = "0.4" diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index 38f22f4..8067c42 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -374,9 +374,8 @@ fn reqwest_client(proxy: Option, user_agent: Option) -> Client { let mut builder = builder.use_native_tls().tls_built_in_root_certs(false); for certificate in rustls_native_certs::load_native_certs().unwrap() { - builder = builder.add_root_certificate( - reqwest::Certificate::from_der(certificate.0.as_slice()).unwrap(), - ) + builder = + builder.add_root_certificate(reqwest::Certificate::from_der(&certificate).unwrap()) } builder.build().unwrap() From 88a28e843f53be5e3f8919e64e8fa11eaf97d8bf Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 10 Mar 2024 19:40:36 +0100 Subject: [PATCH 029/113] Manually specify ffmpeg output color format --- crunchy-cli-core/src/utils/download.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index 7f81bc9..7f87583 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -606,6 +606,10 @@ impl Downloader { command_args.extend([format!("-disposition:s:s:{}", i), "forced".to_string()]) } + // manually specifying the color model for the output file. this must be done manually + // because some Crunchyroll episodes are encoded in a way that ffmpeg cannot re-encode + command_args.extend(["-pix_fmt".to_string(), "yuv420p".to_string()]); + command_args.extend(output_presets); if let Some(output_format) = self.output_format { command_args.extend(["-f".to_string(), output_format]); From d3696c783cef5e8f368978e4e014d07b8a83b913 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 10 Mar 2024 21:57:20 +0100 Subject: [PATCH 030/113] Include archive chapters only if flag is set --- crunchy-cli-core/src/archive/command.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index 8258b36..f7bea0e 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -529,7 +529,11 @@ async fn get_format( .flat_map(|(_, _, _, subtitles)| subtitles.clone()) .collect(), metadata: DownloadFormatMetadata { - skip_events: format_pairs.first().unwrap().0.skip_events().await?, + skip_events: if archive.include_chapters { + format_pairs.first().unwrap().0.skip_events().await? + } else { + None + }, }, }), MergeBehavior::Auto => { From 26a858c1a13d66f303d7256f4c6d168d04a25a69 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 10 Mar 2024 22:04:58 +0100 Subject: [PATCH 031/113] Update dependencies and version --- Cargo.lock | 198 ++++++++++++++++++------------------ Cargo.toml | 2 +- crunchy-cli-core/Cargo.toml | 4 +- 3 files changed, 101 insertions(+), 103 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 99b223b..113b430 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,9 +19,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aes" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", @@ -102,9 +102,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.79" +version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" +checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1" [[package]] name = "async-speed-limit" @@ -189,9 +189,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.14.0" +version = "3.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" [[package]] name = "bytes" @@ -302,9 +302,9 @@ checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" [[package]] name = "clap_mangen" -version = "0.2.17" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a7c2b01e5e779c19f46a94bbd398f33ae63b0f78c07108351fb4536845bb7fd" +checksum = "e1dd95b5ebb5c1c54581dd6346f3ed6a79a3eef95dd372fc2ac13d535535300e" dependencies = [ "clap", "roff", @@ -384,7 +384,7 @@ dependencies = [ [[package]] name = "crunchy-cli" -version = "3.3.0" +version = "3.3.1" dependencies = [ "chrono", "clap", @@ -397,7 +397,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.3.0" +version = "3.3.1" dependencies = [ "anyhow", "async-speed-limit", @@ -414,7 +414,7 @@ dependencies = [ "indicatif", "lazy_static", "log", - "nix 0.28.0", + "nix", "num_cpus", "regex", "reqwest", @@ -432,9 +432,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "467fc4159e38121aa5efb3807de957eefff02d14ba3439494f89f351e3539b73" +checksum = "0f99fcd7627d214fd57cd1d030e8c859a773e19aa29fb0d15017aa84efaba353" dependencies = [ "aes", "async-trait", @@ -455,14 +455,14 @@ dependencies = [ "smart-default", "tokio", "tower-service", - "webpki-roots 0.26.0", + "webpki-roots 0.26.1", ] [[package]] name = "crunchyroll-rs-internal" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62db42661f84dc2e2f7c5fef1e8906fff29ff316624da54039aec748a49e7a3b" +checksum = "d2dd269b2df82ebbec9e8164e9950c6ad14a01cfcbb85eceeb3f3ef26c7da90c" dependencies = [ "darling", "quote", @@ -481,19 +481,19 @@ dependencies = [ [[package]] name = "ctrlc" -version = "3.4.2" +version = "3.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b467862cc8610ca6fc9a1532d7777cee0804e678ab45410897b9396495994a0b" +checksum = "672465ae37dc1bc6380a6547a8883d5dd397b0f1faaad4f265726cc7042a5345" dependencies = [ - "nix 0.27.1", + "nix", "windows-sys 0.52.0", ] [[package]] name = "darling" -version = "0.20.3" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" +checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391" dependencies = [ "darling_core", "darling_macro", @@ -501,9 +501,9 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.3" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" +checksum = "9c2cf1c23a687a1feeb728783b993c4e1ad83d99f351801977dd809b48d0a70f" dependencies = [ "fnv", "ident_case", @@ -515,9 +515,9 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.20.3" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" +checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" dependencies = [ "darling_core", "quote", @@ -604,9 +604,9 @@ dependencies = [ [[package]] name = "either" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" [[package]] name = "encode_unicode" @@ -740,9 +740,9 @@ checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-timer" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" [[package]] name = "futures-util" @@ -792,9 +792,9 @@ checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "h2" -version = "0.3.23" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b553656127a00601c8ae5590fcfdc118e4083a7924b6cf4ffc1ea4b99dc429d7" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" dependencies = [ "bytes", "fnv", @@ -802,7 +802,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 2.1.0", + "indexmap 2.2.5", "slab", "tokio", "tokio-util", @@ -841,9 +841,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes", "fnv", @@ -926,9 +926,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.59" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -986,9 +986,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.1.0" +version = "2.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -997,9 +997,9 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.17.7" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" +checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3" dependencies = [ "console", "instant", @@ -1050,9 +1050,9 @@ checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "js-sys" -version = "0.3.67" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] @@ -1101,9 +1101,9 @@ checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "log" -version = "0.4.20" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "m3u8-rs" @@ -1169,17 +1169,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "nix" -version = "0.27.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" -dependencies = [ - "bitflags 2.4.2", - "cfg-if", - "libc", -] - [[package]] name = "nix" version = "0.28.0" @@ -1202,6 +1191,12 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-traits" version = "0.2.18" @@ -1244,9 +1239,9 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "openssl" -version = "0.10.62" +version = "0.10.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cde4d2d9200ad5909f8dac647e29482e07c3a35de8a13fce7c9c7747ad9f671" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" dependencies = [ "bitflags 2.4.2", "cfg-if", @@ -1276,18 +1271,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "300.2.1+3.2.0" +version = "300.2.3+3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fe476c29791a5ca0d1273c697e96085bbabbbea2ef7afd5617e78a4b40332d3" +checksum = "5cff92b6f71555b61bb9315f7c64da3ca43d87531622120fea0195fc761b4843" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.98" +version = "0.9.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1665caf8ab2dc9aef43d1c0023bd904633a6a05cb30b0ad59bec2ae986e57a7" +checksum = "dda2b0f344e78efc2facf7d195d098df0dd72151b26ab98da807afc26c198dff" dependencies = [ "cc", "libc", @@ -1322,9 +1317,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "portable-atomic" @@ -1476,22 +1471,23 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 0.25.3", + "webpki-roots 0.25.4", "winreg", ] [[package]] name = "ring" -version = "0.17.7" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", + "cfg-if", "getrandom", "libc", "spin", "untrusted", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -1581,9 +1577,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" [[package]] name = "schannel" @@ -1649,9 +1645,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.111" +version = "1.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" +checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" dependencies = [ "itoa", "ryu", @@ -1660,9 +1656,9 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebd154a240de39fdebcf5775d2675c204d7c13cf39a4c697be6493c8e734337c" +checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" dependencies = [ "itoa", "serde", @@ -1699,7 +1695,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.1.0", + "indexmap 2.2.5", "serde", "serde_derive", "serde_json", @@ -1860,12 +1856,13 @@ dependencies = [ [[package]] name = "time" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" +checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" dependencies = [ "deranged", "itoa", + "num-conv", "powerfmt", "serde", "time-core", @@ -1880,10 +1877,11 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" +checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" dependencies = [ + "num-conv", "time-core", ] @@ -2027,9 +2025,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-bidi" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" @@ -2039,9 +2037,9 @@ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ "tinyvec", ] @@ -2104,9 +2102,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.90" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2114,9 +2112,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.90" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", @@ -2129,9 +2127,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.40" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" dependencies = [ "cfg-if", "js-sys", @@ -2141,9 +2139,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.90" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2151,9 +2149,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.90" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", @@ -2164,9 +2162,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.90" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "wasm-streams" @@ -2183,9 +2181,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.67" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" dependencies = [ "js-sys", "wasm-bindgen", @@ -2193,15 +2191,15 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.25.3" +version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "webpki-roots" -version = "0.26.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de2cfda980f21be5a7ed2eadb3e6fe074d56022bea2cdeb1a62eb220fc04188" +checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009" dependencies = [ "rustls-pki-types", ] diff --git a/Cargo.toml b/Cargo.toml index cf4f676..681fe73 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.3.0" +version = "3.3.1" edition = "2021" license = "MIT" diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index fcb178a..0242606 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.3.0" +version = "3.3.1" edition = "2021" license = "MIT" @@ -16,7 +16,7 @@ anyhow = "1.0" async-speed-limit = "0.4" clap = { version = "4.5", features = ["derive", "string"] } chrono = "0.4" -crunchyroll-rs = { version = "0.8.5", features = ["dash-stream", "experimental-stabilizations", "tower"] } +crunchyroll-rs = { version = "0.8.6", features = ["dash-stream", "experimental-stabilizations", "tower"] } ctrlc = "3.4" dialoguer = { version = "0.11", default-features = false } dirs = "5.0" From 89be8ac4296a72df4f877348892e7a6b27f1850f Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 25 Mar 2024 13:30:40 +0100 Subject: [PATCH 032/113] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 46bee72..7636dc1 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +# This project has been sunset as Crunchyroll moved to a DRM-only system. See [#362](https://github.com/crunchy-labs/crunchy-cli/issues/362). + # crunchy-cli 👇 A Command-line downloader for [Crunchyroll](https://www.crunchyroll.com). From ba8028737dbd5e99f7cc7d40d432e0f34505d9fa Mon Sep 17 00:00:00 2001 From: Amelia Date: Wed, 3 Apr 2024 06:49:51 -0700 Subject: [PATCH 033/113] Update missing fonts (#360) * Update missing fonts * Compile fix --- crunchy-cli-core/src/utils/download.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index 7f87583..736f9e6 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -1015,7 +1015,7 @@ fn get_video_stats(path: &Path) -> Result<(NaiveTime, f64)> { } // all subtitle fonts (extracted from javascript) -const FONTS: [(&str, &str); 66] = [ +const FONTS: [(&str, &str); 68] = [ ("Adobe Arabic", "AdobeArabic-Bold.woff2"), ("Andale Mono", "andalemo.woff2"), ("Arial", "arial.woff2"), @@ -1073,6 +1073,8 @@ const FONTS: [(&str, &str); 66] = [ ("Impact", "impact.woff2"), ("Mangal", "MANGAL.woff2"), ("Meera Inimai", "MeeraInimai-Regular.woff2"), + ("Noto Sans Tamil", "NotoSansTamil.woff2"), + ("Noto Sans Telugu", "NotoSansTelegu.woff2"), ("Noto Sans Thai", "NotoSansThai.woff2"), ("Rubik", "Rubik-Regular.woff2"), ("Rubik Black", "Rubik-Black.woff2"), From e694046b07fbcabf40714e62d1ac7a98f511fcdd Mon Sep 17 00:00:00 2001 From: bytedream Date: Wed, 3 Apr 2024 15:48:15 +0200 Subject: [PATCH 034/113] Move to new, DRM-free, endpoint --- Cargo.lock | 404 ++++++++++------------- crunchy-cli-core/Cargo.toml | 6 +- crunchy-cli-core/src/archive/command.rs | 2 +- crunchy-cli-core/src/download/command.rs | 2 +- crunchy-cli-core/src/search/format.rs | 36 +- crunchy-cli-core/src/utils/download.rs | 57 ++-- crunchy-cli-core/src/utils/format.rs | 31 +- crunchy-cli-core/src/utils/video.rs | 38 +-- 8 files changed, 245 insertions(+), 331 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 113b430..b7617b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,17 +17,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher", - "cpufeatures", -] - [[package]] name = "aho-corasick" version = "1.1.2" @@ -156,13 +145,19 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" + [[package]] name = "base64-serde" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba368df5de76a5bea49aaf0cf1b39ccfbbef176924d1ba5db3e4135216cbe3c7" dependencies = [ - "base64", + "base64 0.21.7", "serde", ] @@ -178,15 +173,6 @@ version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" -[[package]] -name = "block-padding" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" -dependencies = [ - "generic-array", -] - [[package]] name = "bumpalo" version = "3.15.4" @@ -199,15 +185,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" -[[package]] -name = "cbc" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" -dependencies = [ - "cipher", -] - [[package]] name = "cc" version = "1.0.90" @@ -228,9 +205,9 @@ checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" [[package]] name = "chrono" -version = "0.4.35" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eaf5903dcbc0a39312feb77df2ff4c76387d591b9fc7b04a238dcf8bb62639a" +checksum = "8a0d04d43504c61aa6c7531f1871dd0d418d91130162063b789da00fd7057a5e" dependencies = [ "android-tzdata", "iana-time-zone", @@ -241,16 +218,6 @@ dependencies = [ "windows-targets 0.52.4", ] -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common", - "inout", -] - [[package]] name = "clap" version = "4.5.2" @@ -373,15 +340,6 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" -[[package]] -name = "cpufeatures" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" -dependencies = [ - "libc", -] - [[package]] name = "crunchy-cli" version = "3.3.1" @@ -432,20 +390,17 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.8.6" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f99fcd7627d214fd57cd1d030e8c859a773e19aa29fb0d15017aa84efaba353" +checksum = "05fcd99a09d001333ab482412473aaa03c84f980d451845b4c43d58986b6eb64" dependencies = [ - "aes", "async-trait", - "cbc", "chrono", "crunchyroll-rs-internal", "dash-mpd", "futures-util", "jsonwebtoken", "lazy_static", - "m3u8-rs", "regex", "reqwest", "rustls", @@ -455,30 +410,20 @@ dependencies = [ "smart-default", "tokio", "tower-service", - "webpki-roots 0.26.1", + "webpki-roots", ] [[package]] name = "crunchyroll-rs-internal" -version = "0.8.6" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2dd269b2df82ebbec9e8164e9950c6ad14a01cfcbb85eceeb3f3ef26c7da90c" +checksum = "d406a27eca9ceab379b601101cd96b0f7bfff34e93487ca58d54118a8b4fbf91" dependencies = [ "darling", "quote", "syn", ] -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - [[package]] name = "ctrlc" version = "3.4.4" @@ -526,11 +471,11 @@ dependencies = [ [[package]] name = "dash-mpd" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18c18f28b58beade78e0f61a846a63a122cb92c5f5ed6bad29d7ad13287c7526" +checksum = "6cafa2c33eff2857e1a14c38aa9a432aa565a01e77804a541fce7aec3affb8f8" dependencies = [ - "base64", + "base64 0.22.0", "base64-serde", "chrono", "fs-err", @@ -614,15 +559,6 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" -[[package]] -name = "encoding_rs" -version = "0.8.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" -dependencies = [ - "cfg-if", -] - [[package]] name = "equivalent" version = "1.0.1" @@ -761,16 +697,6 @@ dependencies = [ "slab", ] -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getrandom" version = "0.2.12" @@ -790,25 +716,6 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" -[[package]] -name = "h2" -version = "0.3.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap 2.2.5", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "hashbrown" version = "0.12.3" @@ -841,9 +748,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "0.2.12" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ "bytes", "fnv", @@ -852,12 +759,24 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.6" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" dependencies = [ "bytes", "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", "pin-project-lite", ] @@ -867,61 +786,76 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - [[package]] name = "hyper" -version = "0.14.28" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +checksum = "186548d73ac615b32a73aafe38fb4f56c0d340e110e5a200bcadbaf2e199263a" dependencies = [ "bytes", "futures-channel", - "futures-core", "futures-util", - "h2", "http", "http-body", "httparse", - "httpdate", "itoa", "pin-project-lite", - "socket2", + "smallvec", "tokio", - "tower-service", - "tracing", "want", ] [[package]] name = "hyper-rustls" -version = "0.24.2" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" dependencies = [ "futures-util", "http", "hyper", + "hyper-util", "rustls", + "rustls-pki-types", "tokio", "tokio-rustls", + "tower-service", ] [[package]] name = "hyper-tls" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", + "http-body-util", "hyper", + "hyper-util", "native-tls", "tokio", "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "pin-project-lite", + "socket2", + "tokio", + "tower", + "tower-service", + "tracing", ] [[package]] @@ -1008,16 +942,6 @@ dependencies = [ "unicode-width", ] -[[package]] -name = "inout" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" -dependencies = [ - "block-padding", - "generic-array", -] - [[package]] name = "instant" version = "0.1.12" @@ -1059,11 +983,11 @@ dependencies = [ [[package]] name = "jsonwebtoken" -version = "9.2.0" +version = "9.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7ea04a7c5c055c175f189b6dc6ba036fd62306b58c66c9f6389036c503a3f4" +checksum = "b9ae10193d25051e74945f1ea2d0b42e03cc3b890f7e4cc5faa44997d808193f" dependencies = [ - "base64", + "base64 0.21.7", "js-sys", "ring", "serde", @@ -1105,16 +1029,6 @@ version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" -[[package]] -name = "m3u8-rs" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f03cd3335fb5f2447755d45cda9c70f76013626a9db44374973791b0926a86c3" -dependencies = [ - "chrono", - "nom", -] - [[package]] name = "memchr" version = "2.7.1" @@ -1127,6 +1041,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1303,6 +1227,26 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.13" @@ -1399,9 +1343,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.3" +version = "1.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ "aho-corasick", "memchr", @@ -1428,38 +1372,39 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "reqwest" -version = "0.11.25" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eea5a9eb898d3783f17c6407670e3592fd174cb81a10e51d4c37f49450b9946" +checksum = "2d66674f2b6fb864665eea7a3c1ac4e3dfacd2fda83cf6f935a612e01b0e3338" dependencies = [ - "base64", + "base64 0.21.7", "bytes", "cookie", "cookie_store", - "encoding_rs", "futures-core", "futures-util", - "h2", "http", "http-body", + "http-body-util", "hyper", "hyper-rustls", "hyper-tls", + "hyper-util", "ipnet", "js-sys", "log", "mime", + "mime_guess", "native-tls", "once_cell", "percent-encoding", "pin-project-lite", "rustls", "rustls-pemfile 1.0.4", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", - "system-configuration", "tokio", "tokio-native-tls", "tokio-rustls", @@ -1471,7 +1416,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 0.25.4", + "webpki-roots", "winreg", ] @@ -1517,14 +1462,16 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.10" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" +checksum = "99008d7ad0bbbea527ec27bddbc0e432c5b87d8175178cee68d2eec9c4a1813c" dependencies = [ "log", "ring", + "rustls-pki-types", "rustls-webpki", - "sct", + "subtle", + "zeroize", ] [[package]] @@ -1546,7 +1493,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64", + "base64 0.21.7", ] [[package]] @@ -1555,7 +1502,7 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f48172685e6ff52a556baa527774f61fcaa884f59daf3375c62a3f1cd2549dab" dependencies = [ - "base64", + "base64 0.21.7", "rustls-pki-types", ] @@ -1567,11 +1514,12 @@ checksum = "5ede67b28608b4c60685c7d54122d4400d90f62b40caee7700e700380a390fa8" [[package]] name = "rustls-webpki" -version = "0.101.7" +version = "0.102.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" dependencies = [ "ring", + "rustls-pki-types", "untrusted", ] @@ -1590,16 +1538,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "security-framework" version = "2.9.2" @@ -1687,11 +1625,11 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.6.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15d167997bd841ec232f5b2b8e0e26606df2e7caa4c31b95ea9ca52b200bd270" +checksum = "ee80b0e361bbf88fd2f6e242ccd19cfda072cb0faa6ae694ecee08199938569a" dependencies = [ - "base64", + "base64 0.21.7", "chrono", "hex", "indexmap 1.9.3", @@ -1705,9 +1643,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.6.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "865f9743393e638991566a8b7a479043c2c8da94a33e0a31f18214c9cae0a64d" +checksum = "6561dc161a9224638a31d876ccdfefbc1df91d3f3a8342eddb35f055d48c7655" dependencies = [ "darling", "proc-macro2", @@ -1736,6 +1674,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + [[package]] name = "smart-default" version = "0.7.1" @@ -1775,6 +1719,12 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + [[package]] name = "syn" version = "2.0.52" @@ -1801,27 +1751,6 @@ dependencies = [ "libc", ] -[[package]] -name = "system-configuration" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658bc6ee10a9b4fcf576e9b0819d95ec16f4d2c02d39fd83ac1c8789785c4a42" -dependencies = [ - "bitflags 2.4.2", - "core-foundation", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "tempfile" version = "3.10.1" @@ -1836,18 +1765,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.57" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" +checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.57" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" +checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ "proc-macro2", "quote", @@ -1902,9 +1831,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.36.0" +version = "1.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" +checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" dependencies = [ "backtrace", "bytes", @@ -1940,11 +1869,12 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.24.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" dependencies = [ "rustls", + "rustls-pki-types", "tokio", ] @@ -1974,6 +1904,28 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" @@ -1986,6 +1938,7 @@ version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -2018,10 +1971,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] -name = "typenum" -version = "1.17.0" +name = "unicase" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" +dependencies = [ + "version_check", +] [[package]] name = "unicode-bidi" @@ -2189,12 +2145,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki-roots" -version = "0.25.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" - [[package]] name = "webpki-roots" version = "0.26.1" @@ -2387,3 +2337,9 @@ dependencies = [ "linux-raw-sys", "rustix", ] + +[[package]] +name = "zeroize" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 0242606..7617c58 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -16,20 +16,20 @@ anyhow = "1.0" async-speed-limit = "0.4" clap = { version = "4.5", features = ["derive", "string"] } chrono = "0.4" -crunchyroll-rs = { version = "0.8.6", features = ["dash-stream", "experimental-stabilizations", "tower"] } +crunchyroll-rs = { version = "0.10.0", features = ["experimental-stabilizations", "tower"] } ctrlc = "3.4" dialoguer = { version = "0.11", default-features = false } dirs = "5.0" derive_setters = "0.1" futures-util = { version = "0.3", features = ["io"] } fs2 = "0.4" -http = "0.2" +http = "1.1" indicatif = "0.17" lazy_static = "1.4" log = { version = "0.4", features = ["std"] } num_cpus = "1.16" regex = "1.10" -reqwest = { version = "0.11", default-features = false, features = ["socks", "stream"] } +reqwest = { version = "0.12", default-features = false, features = ["socks", "stream"] } serde = "1.0" serde_json = "1.0" serde_plain = "1.0" diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index f7bea0e..0dc0b86 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -487,7 +487,7 @@ async fn get_format( single_format.audio == Locale::ja_JP || stream.subtitles.len() > 1, ) }); - let cc = stream.closed_captions.get(s).cloned().map(|l| (l, false)); + let cc = stream.captions.get(s).cloned().map(|l| (l, false)); subtitles .into_iter() diff --git a/crunchy-cli-core/src/download/command.rs b/crunchy-cli-core/src/download/command.rs index 843f5cd..47b29c9 100644 --- a/crunchy-cli-core/src/download/command.rs +++ b/crunchy-cli-core/src/download/command.rs @@ -389,7 +389,7 @@ async fn get_format( .get(subtitle_locale) .cloned() // use closed captions as fallback if no actual subtitles are found - .or_else(|| stream.closed_captions.get(subtitle_locale).cloned()) + .or_else(|| stream.captions.get(subtitle_locale).cloned()) } else { None }; diff --git a/crunchy-cli-core/src/search/format.rs b/crunchy-cli-core/src/search/format.rs index 156bd95..10eefd8 100644 --- a/crunchy-cli-core/src/search/format.rs +++ b/crunchy-cli-core/src/search/format.rs @@ -163,37 +163,15 @@ impl From<&Concert> for FormatConcert { struct FormatStream { pub locale: Locale, pub dash_url: String, - pub drm_dash_url: String, - pub hls_url: String, - pub drm_hls_url: String, + pub is_drm: bool, } impl From<&Stream> for FormatStream { fn from(value: &Stream) -> Self { - let (dash_url, drm_dash_url, hls_url, drm_hls_url) = - value.variants.get(&Locale::Custom("".to_string())).map_or( - ( - "".to_string(), - "".to_string(), - "".to_string(), - "".to_string(), - ), - |v| { - ( - v.adaptive_dash.clone().unwrap_or_default().url, - v.drm_adaptive_dash.clone().unwrap_or_default().url, - v.adaptive_hls.clone().unwrap_or_default().url, - v.drm_adaptive_hls.clone().unwrap_or_default().url, - ) - }, - ); - Self { locale: value.audio_locale.clone(), - dash_url, - drm_dash_url, - hls_url, - drm_hls_url, + dash_url: value.url.clone(), + is_drm: value.session.uses_stream_limits, } } } @@ -441,7 +419,7 @@ impl Format { if !stream_empty { for (_, episodes) in tree.iter_mut() { for (episode, streams) in episodes { - streams.push(episode.stream().await?) + streams.push(episode.stream_maybe_without_drm().await?) } } } else { @@ -510,7 +488,7 @@ impl Format { } if !stream_empty { for (movie, streams) in tree.iter_mut() { - streams.push(movie.stream().await?) + streams.push(movie.stream_maybe_without_drm().await?) } } else { for (_, streams) in tree.iter_mut() { @@ -548,7 +526,7 @@ impl Format { let stream_empty = self.check_pattern_count_empty(Scope::Stream); let music_video = must_match_if_true!(!music_video_empty => media_collection|MediaCollection::MusicVideo(music_video) => music_video.clone()).unwrap_or_default(); - let stream = must_match_if_true!(!stream_empty => media_collection|MediaCollection::MusicVideo(music_video) => music_video.stream().await?).unwrap_or_default(); + let stream = must_match_if_true!(!stream_empty => media_collection|MediaCollection::MusicVideo(music_video) => music_video.stream_maybe_without_drm().await?).unwrap_or_default(); let music_video_map = self.serializable_to_json_map(FormatMusicVideo::from(&music_video)); let stream_map = self.serializable_to_json_map(FormatStream::from(&stream)); @@ -570,7 +548,7 @@ impl Format { let stream_empty = self.check_pattern_count_empty(Scope::Stream); let concert = must_match_if_true!(!concert_empty => media_collection|MediaCollection::Concert(concert) => concert.clone()).unwrap_or_default(); - let stream = must_match_if_true!(!stream_empty => media_collection|MediaCollection::Concert(concert) => concert.stream().await?).unwrap_or_default(); + let stream = must_match_if_true!(!stream_empty => media_collection|MediaCollection::Concert(concert) => concert.stream_maybe_without_drm().await?).unwrap_or_default(); let concert_map = self.serializable_to_json_map(FormatConcert::from(&concert)); let stream_map = self.serializable_to_json_map(FormatStream::from(&stream)); diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index 736f9e6..8a1fe66 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -4,7 +4,7 @@ use crate::utils::os::{cache_dir, is_special_file, temp_directory, temp_named_pi use crate::utils::rate_limit::RateLimiterService; use anyhow::{bail, Result}; use chrono::NaiveTime; -use crunchyroll_rs::media::{SkipEvents, SkipEventsEvent, Subtitle, VariantData, VariantSegment}; +use crunchyroll_rs::media::{SkipEvents, SkipEventsEvent, StreamData, StreamSegment, Subtitle}; use crunchyroll_rs::Locale; use indicatif::{ProgressBar, ProgressDrawTarget, ProgressFinish, ProgressStyle}; use log::{debug, warn, LevelFilter}; @@ -117,8 +117,8 @@ struct FFmpegMeta { } pub struct DownloadFormat { - pub video: (VariantData, Locale), - pub audios: Vec<(VariantData, Locale)>, + pub video: (StreamData, Locale), + pub audios: Vec<(StreamData, Locale)>, pub subtitles: Vec<(Subtitle, bool)>, pub metadata: DownloadFormatMetadata, } @@ -671,20 +671,17 @@ impl Downloader { &self, dst: &Path, ) -> Result<(Option<(PathBuf, u64)>, Option<(PathBuf, u64)>)> { - let mut all_variant_data = vec![]; + let mut all_stream_data = vec![]; for format in &self.formats { - all_variant_data.push(&format.video.0); - all_variant_data.extend(format.audios.iter().map(|(a, _)| a)) + all_stream_data.push(&format.video.0); + all_stream_data.extend(format.audios.iter().map(|(a, _)| a)) } let mut estimated_required_space: u64 = 0; - for variant_data in all_variant_data { - // nearly no overhead should be generated with this call(s) as we're using dash as - // stream provider and generating the dash segments does not need any fetching of - // additional (http) resources as hls segments would - let segments = variant_data.segments().await?; + for stream_data in all_stream_data { + let segments = stream_data.segments(); // sum the length of all streams up - estimated_required_space += estimate_variant_file_size(variant_data, &segments); + estimated_required_space += estimate_variant_file_size(stream_data, &segments); } let tmp_stat = fs2::statvfs(temp_directory()).unwrap(); @@ -730,29 +727,21 @@ impl Downloader { Ok((tmp_required, dst_required)) } - async fn download_video( - &self, - variant_data: &VariantData, - message: String, - ) -> Result { + async fn download_video(&self, stream_data: &StreamData, message: String) -> Result { let tempfile = tempfile(".mp4")?; let (mut file, path) = tempfile.into_parts(); - self.download_segments(&mut file, message, variant_data) + self.download_segments(&mut file, message, stream_data) .await?; Ok(path) } - async fn download_audio( - &self, - variant_data: &VariantData, - message: String, - ) -> Result { + async fn download_audio(&self, stream_data: &StreamData, message: String) -> Result { let tempfile = tempfile(".m4a")?; let (mut file, path) = tempfile.into_parts(); - self.download_segments(&mut file, message, variant_data) + self.download_segments(&mut file, message, stream_data) .await?; Ok(path) @@ -806,15 +795,15 @@ impl Downloader { &self, writer: &mut impl Write, message: String, - variant_data: &VariantData, + stream_data: &StreamData, ) -> Result<()> { - let segments = variant_data.segments().await?; + let segments = stream_data.segments(); let total_segments = segments.len(); let count = Arc::new(Mutex::new(0)); let progress = if log::max_level() == LevelFilter::Info { - let estimated_file_size = estimate_variant_file_size(variant_data, &segments); + let estimated_file_size = estimate_variant_file_size(stream_data, &segments); let progress = ProgressBar::new(estimated_file_size) .with_style( @@ -832,7 +821,7 @@ impl Downloader { }; let cpus = self.download_threads; - let mut segs: Vec> = Vec::with_capacity(cpus); + let mut segs: Vec> = Vec::with_capacity(cpus); for _ in 0..cpus { segs.push(vec![]) } @@ -858,7 +847,7 @@ impl Downloader { let download = || async move { for (i, segment) in thread_segments.into_iter().enumerate() { let mut retry_count = 0; - let mut buf = loop { + let buf = loop { let request = thread_client .get(&segment.url) .timeout(Duration::from_secs(60)); @@ -884,11 +873,9 @@ impl Downloader { retry_count += 1; }; - buf = VariantSegment::decrypt(&mut buf, segment.key)?.to_vec(); - let mut c = thread_count.lock().await; debug!( - "Downloaded and decrypted segment [{}/{} {:.2}%] {}", + "Downloaded segment [{}/{} {:.2}%] {}", num + (i * cpus) + 1, total_segments, ((*c + 1) as f64 / total_segments as f64) * 100f64, @@ -928,7 +915,7 @@ impl Downloader { if let Some(p) = &progress { let progress_len = p.length().unwrap(); - let estimated_segment_len = (variant_data.bandwidth / 8) + let estimated_segment_len = (stream_data.bandwidth / 8) * segments.get(pos as usize).unwrap().length.as_secs(); let bytes_len = bytes.len() as u64; @@ -977,8 +964,8 @@ impl Downloader { } } -fn estimate_variant_file_size(variant_data: &VariantData, segments: &[VariantSegment]) -> u64 { - (variant_data.bandwidth / 8) * segments.iter().map(|s| s.length.as_secs()).sum::() +fn estimate_variant_file_size(stream_data: &StreamData, segments: &[StreamSegment]) -> u64 { + (stream_data.bandwidth / 8) * segments.iter().map(|s| s.length.as_secs()).sum::() } /// Get the length and fps of a video. diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index 7146a55..df79d64 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -2,9 +2,9 @@ use crate::utils::filter::real_dedup_vec; use crate::utils::locale::LanguageTagging; use crate::utils::log::tab_info; use crate::utils::os::{is_special_file, sanitize}; -use anyhow::Result; +use anyhow::{bail, Result}; use chrono::{Datelike, Duration}; -use crunchyroll_rs::media::{Resolution, SkipEvents, Stream, Subtitle, VariantData}; +use crunchyroll_rs::media::{Resolution, SkipEvents, Stream, StreamData, Subtitle}; use crunchyroll_rs::{Concert, Episode, Locale, MediaCollection, Movie, MusicVideo}; use log::{debug, info}; use std::cmp::Ordering; @@ -167,12 +167,17 @@ impl SingleFormat { pub async fn stream(&self) -> Result { let stream = match &self.source { - MediaCollection::Episode(e) => e.stream().await?, - MediaCollection::Movie(m) => m.stream().await?, - MediaCollection::MusicVideo(mv) => mv.stream().await?, - MediaCollection::Concert(c) => c.stream().await?, + MediaCollection::Episode(e) => e.stream_maybe_without_drm().await?, + MediaCollection::Movie(m) => m.stream_maybe_without_drm().await?, + MediaCollection::MusicVideo(mv) => mv.stream_maybe_without_drm().await?, + MediaCollection::Concert(c) => c.stream_maybe_without_drm().await?, _ => unreachable!(), }; + + if stream.session.uses_stream_limits { + bail!("Found a stream which probably uses DRM. DRM downloads aren't supported") + } + Ok(stream) } @@ -331,9 +336,7 @@ impl Iterator for SingleFormatCollectionIterator { type Item = Vec; fn next(&mut self) -> Option { - let Some((_, episodes)) = self.0 .0.iter_mut().next() else { - return None; - }; + let (_, episodes) = self.0 .0.iter_mut().next()?; let value = episodes.pop_first().unwrap().1; if episodes.is_empty() { @@ -377,7 +380,7 @@ pub struct Format { impl Format { #[allow(clippy::type_complexity)] pub fn from_single_formats( - mut single_formats: Vec<(SingleFormat, VariantData, Vec<(Subtitle, bool)>)>, + mut single_formats: Vec<(SingleFormat, StreamData, Vec<(Subtitle, bool)>)>, ) -> Self { let locales: Vec<(Locale, Vec)> = single_formats .iter() @@ -397,10 +400,10 @@ impl Format { title: first_format.title, description: first_format.description, locales, - resolution: first_stream.resolution.clone(), - width: first_stream.resolution.width, - height: first_stream.resolution.height, - fps: first_stream.fps, + resolution: first_stream.resolution().unwrap(), + width: first_stream.resolution().unwrap().width, + height: first_stream.resolution().unwrap().height, + fps: first_stream.fps().unwrap(), release_year: first_format.release_year, release_month: first_format.release_month, release_day: first_format.release_day, diff --git a/crunchy-cli-core/src/utils/video.rs b/crunchy-cli-core/src/utils/video.rs index 0ae4ba4..7f7d73e 100644 --- a/crunchy-cli-core/src/utils/video.rs +++ b/crunchy-cli-core/src/utils/video.rs @@ -1,17 +1,17 @@ use anyhow::{bail, Result}; -use crunchyroll_rs::media::{Resolution, Stream, VariantData}; +use crunchyroll_rs::media::{Resolution, Stream, StreamData}; use crunchyroll_rs::Locale; pub async fn variant_data_from_stream( stream: &Stream, resolution: &Resolution, subtitle: Option, -) -> Result> { +) -> Result> { // sometimes Crunchyroll marks episodes without real subtitles that they have subtitles and // reports that only hardsub episode are existing. the following lines are trying to prevent // potential errors which might get caused by this incorrect reporting // (https://github.com/crunchy-labs/crunchy-cli/issues/231) - let mut hardsub_locales = stream.streaming_hardsub_locales(); + let mut hardsub_locales: Vec = stream.hard_subs.keys().cloned().collect(); let (hardsub_locale, mut contains_hardsub) = if !hardsub_locales .contains(&Locale::Custom("".to_string())) && !hardsub_locales.contains(&Locale::Custom(":".to_string())) @@ -29,39 +29,29 @@ pub async fn variant_data_from_stream( (subtitle, hardsubs_requested) }; - let mut streaming_data = match stream.dash_streaming_data(hardsub_locale).await { + let (mut videos, mut audios) = match stream.stream_data(hardsub_locale).await { Ok(data) => data, Err(e) => { // the error variant is only `crunchyroll_rs::error::Error::Input` when the requested // hardsub is not available if let crunchyroll_rs::error::Error::Input { .. } = e { contains_hardsub = false; - stream.dash_streaming_data(None).await? + stream.stream_data(None).await? } else { bail!(e) } } - }; - streaming_data - .0 - .sort_by(|a, b| a.bandwidth.cmp(&b.bandwidth).reverse()); - streaming_data - .1 - .sort_by(|a, b| a.bandwidth.cmp(&b.bandwidth).reverse()); + } + .unwrap(); + videos.sort_by(|a, b| a.bandwidth.cmp(&b.bandwidth).reverse()); + audios.sort_by(|a, b| a.bandwidth.cmp(&b.bandwidth).reverse()); let video_variant = match resolution.height { - u64::MAX => Some(streaming_data.0.into_iter().next().unwrap()), - u64::MIN => Some(streaming_data.0.into_iter().last().unwrap()), - _ => streaming_data - .0 + u64::MAX => Some(videos.into_iter().next().unwrap()), + u64::MIN => Some(videos.into_iter().last().unwrap()), + _ => videos .into_iter() - .find(|v| resolution.height == v.resolution.height), + .find(|v| resolution.height == v.resolution().unwrap().height), }; - Ok(video_variant.map(|v| { - ( - v, - streaming_data.1.first().unwrap().clone(), - contains_hardsub, - ) - })) + Ok(video_variant.map(|v| (v, audios.first().unwrap().clone(), contains_hardsub))) } From f16cd25ea4a40f48b1744c67de09549756ec8505 Mon Sep 17 00:00:00 2001 From: Amelia Frost Date: Wed, 3 Apr 2024 16:09:33 +0200 Subject: [PATCH 035/113] Fix for some chapters being sent by CR as floats (#351) * Fix for some chapters being sent by CR as floats. See: https://github.com/crunchy-labs/crunchyroll-rs/commit/3f3a80f7f7205e8ecb67e15fcf82b988eb88d9b2 * Compile fix for error[E0277]: cannot multiply `f32` by `u32` * Format Co-authored-by: bytedream --- crunchy-cli-core/src/utils/download.rs | 42 ++++++++++++++------------ 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index 8a1fe66..d7904f3 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -1214,41 +1214,45 @@ fn write_ffmpeg_chapters( ) -> Result<()> { let video_len = video_len .signed_duration_since(NaiveTime::MIN) - .num_seconds() as u32; - events.sort_by(|(_, event_a), (_, event_b)| event_a.start.cmp(&event_b.start)); + .num_milliseconds() as f32 + / 1000.0; + events.sort_by(|(_, event_a), (_, event_b)| event_a.start.total_cmp(&event_b.start)); writeln!(file, ";FFMETADATA1")?; - let mut last_end_time = 0; + let mut last_end_time = 0.0; for (name, event) in events { - // include an extra 'Episode' chapter if the start of the current chapter is more than 10 - // seconds later than the end of the last chapter. - // this is done before writing the actual chapter of this loop to keep the chapter - // chronologically in order - if event.start as i32 - last_end_time as i32 > 10 { + /* + - Convert from seconds to milliseconds for the correct timescale + - Include an extra 'Episode' chapter if the start of the current chapter is more than 10 + seconds later than the end of the last chapter. + This is done before writing the actual chapter of this loop to keep the chapter + chronologically in order + */ + if event.start - last_end_time > 10.0 { writeln!(file, "[CHAPTER]")?; - writeln!(file, "TIMEBASE=1/1")?; - writeln!(file, "START={}", last_end_time)?; - writeln!(file, "END={}", event.start)?; + writeln!(file, "TIMEBASE=1/1000")?; + writeln!(file, "START={}", (last_end_time * 1000.0) as u32)?; + writeln!(file, "END={}", (event.start * 1000.0) as u32)?; writeln!(file, "title=Episode")?; } writeln!(file, "[CHAPTER]")?; - writeln!(file, "TIMEBASE=1/1")?; - writeln!(file, "START={}", event.start)?; - writeln!(file, "END={}", event.end)?; + writeln!(file, "TIMEBASE=1/1000")?; + writeln!(file, "START={}", (event.start * 1000.0) as u32)?; + writeln!(file, "END={}", (event.end * 1000.0) as u32)?; writeln!(file, "title={}", name)?; last_end_time = event.end; } - // only add a traling chapter if the gab between the end of the last chapter and the total video + // only add a trailing chapter if the gap between the end of the last chapter and the total video // length is greater than 10 seconds - if video_len as i32 - last_end_time as i32 > 10 { + if video_len - last_end_time > 10.0 { writeln!(file, "[CHAPTER]")?; - writeln!(file, "TIMEBASE=1/1")?; - writeln!(file, "START={}", last_end_time)?; - writeln!(file, "END={}", video_len)?; + writeln!(file, "TIMEBASE=1/1000")?; + writeln!(file, "START={}", (last_end_time * 1000.0) as u32)?; + writeln!(file, "END={}", (video_len * 1000.0) as u32)?; writeln!(file, "title=Episode")?; } From 111e461b302544f027f31d8b2453a268a7ab0013 Mon Sep 17 00:00:00 2001 From: bytedream Date: Wed, 3 Apr 2024 16:20:24 +0200 Subject: [PATCH 036/113] Update dependencies and version --- Cargo.lock | 134 +++++++++++++++++------------------- Cargo.toml | 4 +- crunchy-cli-core/Cargo.toml | 4 +- 3 files changed, 66 insertions(+), 76 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b7617b1..52bba41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,9 +19,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aho-corasick" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] @@ -91,9 +91,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.80" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1" +checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" [[package]] name = "async-speed-limit" @@ -109,9 +109,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.77" +version = "0.1.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" +checksum = "a507401cad91ec6a857ed5513a2073c82a9b9048762b885bb98655b306964681" dependencies = [ "proc-macro2", "quote", @@ -120,15 +120,15 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" [[package]] name = "backtrace" -version = "0.3.69" +version = "0.3.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" dependencies = [ "addr2line", "cc", @@ -169,9 +169,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" [[package]] name = "bumpalo" @@ -181,9 +181,9 @@ checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" [[package]] name = "bytes" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "cc" @@ -220,9 +220,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.2" +version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b230ab84b0ffdf890d5a10abdbc8b83ae1c4918275daea1ab8801f71536b2651" +checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" dependencies = [ "clap_builder", "clap_derive", @@ -237,7 +237,7 @@ dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim 0.11.0", + "strsim 0.11.1", ] [[package]] @@ -251,9 +251,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.0" +version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "307bc0538d5f0f83b8248db3087aa92fe504e4691294d0c96c0eabc33f47ba47" +checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" dependencies = [ "heck", "proc-macro2", @@ -342,7 +342,7 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "crunchy-cli" -version = "3.3.1" +version = "3.3.2" dependencies = [ "chrono", "clap", @@ -355,7 +355,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.3.1" +version = "3.3.2" dependencies = [ "anyhow", "async-speed-limit", @@ -577,9 +577,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" [[package]] name = "fnv" @@ -730,9 +730,9 @@ checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" [[package]] name = "heck" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" @@ -920,9 +920,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.5" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -968,9 +968,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "js-sys" @@ -1008,13 +1008,12 @@ checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libredox" -version = "0.0.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "libc", - "redox_syscall", ] [[package]] @@ -1031,9 +1030,9 @@ checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "memchr" -version = "2.7.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "mime" @@ -1099,7 +1098,7 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "cfg-if", "cfg_aliases", "libc", @@ -1167,7 +1166,7 @@ version = "0.10.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "cfg-if", "foreign-types", "libc", @@ -1204,9 +1203,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.101" +version = "0.9.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dda2b0f344e78efc2facf7d195d098df0dd72151b26ab98da807afc26c198dff" +checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" dependencies = [ "cc", "libc", @@ -1249,9 +1248,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" [[package]] name = "pin-utils" @@ -1279,9 +1278,9 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "proc-macro2" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" dependencies = [ "unicode-ident", ] @@ -1321,20 +1320,11 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "redox_syscall" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "redox_users" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" +checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" dependencies = [ "getrandom", "libredox", @@ -1366,9 +1356,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" [[package]] name = "reqwest" @@ -1449,11 +1439,11 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.38.31" +version = "0.38.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "errno", "libc", "linux-raw-sys", @@ -1508,9 +1498,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.3.1" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ede67b28608b4c60685c7d54122d4400d90f62b40caee7700e700380a390fa8" +checksum = "ecd36cc4259e3e4514335c4a138c6b43171a8d61d8f5c9348f9fc7529416f247" [[package]] name = "rustls-webpki" @@ -1540,9 +1530,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.9.2" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6" dependencies = [ "bitflags 1.3.2", "core-foundation", @@ -1553,9 +1543,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.9.1" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +checksum = "41f3cc463c0ef97e11c3461a9d3787412d30e8e7eb907c79180c4a57bf7c04ef" dependencies = [ "core-foundation-sys", "libc", @@ -1583,9 +1573,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.114" +version = "1.0.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" +checksum = "12dc5c46daa8e9fdf4f5e71b6cf9a53f2487da0e86e55808e2d35539666497dd" dependencies = [ "itoa", "ryu", @@ -1633,7 +1623,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.2.5", + "indexmap 2.2.6", "serde", "serde_derive", "serde_json", @@ -1715,9 +1705,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "strsim" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "subtle" @@ -1727,9 +1717,9 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" -version = "2.0.52" +version = "2.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07" +checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 681fe73..159b3cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.3.1" +version = "3.3.2" edition = "2021" license = "MIT" @@ -14,7 +14,7 @@ openssl-tls = ["dep:native-tls-crate", "native-tls-crate/openssl", "crunchy-cli- openssl-tls-static = ["dep:native-tls-crate", "native-tls-crate/openssl", "crunchy-cli-core/openssl-tls-static"] [dependencies] -tokio = { version = "1.36", features = ["macros", "rt-multi-thread", "time"], default-features = false } +tokio = { version = "1.37", features = ["macros", "rt-multi-thread", "time"], default-features = false } native-tls-crate = { package = "native-tls", version = "0.2.11", optional = true } diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 7617c58..f3e8b5f 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.3.1" +version = "3.3.2" edition = "2021" license = "MIT" @@ -36,7 +36,7 @@ serde_plain = "1.0" shlex = "1.3" sys-locale = "0.3" tempfile = "3.10" -tokio = { version = "1.36", features = ["io-util", "macros", "net", "rt-multi-thread", "time"] } +tokio = { version = "1.37", features = ["io-util", "macros", "net", "rt-multi-thread", "time"] } tokio-util = "0.7" tower-service = "0.3" rustls-native-certs = { version = "0.7", optional = true } From c0f334684679665a06cb6dc247017f63d52e2090 Mon Sep 17 00:00:00 2001 From: bytedream Date: Wed, 3 Apr 2024 16:46:49 +0200 Subject: [PATCH 037/113] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7636dc1..5463e30 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -# This project has been sunset as Crunchyroll moved to a DRM-only system. See [#362](https://github.com/crunchy-labs/crunchy-cli/issues/362). +> ~~This project has been sunset as Crunchyroll moved to a DRM-only system. See [#362](https://github.com/crunchy-labs/crunchy-cli/issues/362).~~ +> +> Well there is one endpoint which still has DRM-free streams, I guess I still have a bit time until (finally) everything is DRM-only. # crunchy-cli From af8ab248261c7d3b56930363fa6956ed01a8be9d Mon Sep 17 00:00:00 2001 From: bytedream Date: Wed, 3 Apr 2024 17:13:44 +0200 Subject: [PATCH 038/113] Update search command url help --- crunchy-cli-core/src/search/command.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crunchy-cli-core/src/search/command.rs b/crunchy-cli-core/src/search/command.rs index 226e242..f683e24 100644 --- a/crunchy-cli-core/src/search/command.rs +++ b/crunchy-cli-core/src/search/command.rs @@ -88,9 +88,7 @@ pub struct Search { /// /// stream.locale → Stream locale/language /// stream.dash_url → Stream url in DASH format - /// stream.drm_dash_url → Stream url in DRM protected DASH format - /// stream.hls_url → Stream url in HLS format - /// stream.drm_hls_url → Stream url in DRM protected HLS format + /// stream.is_drm → If `stream.is_drm` is DRM encrypted /// /// subtitle.locale → Subtitle locale/language /// subtitle.url → Url to the subtitle From 8c1868f2fd9fb6348a09313f8cdf6f41239d58b3 Mon Sep 17 00:00:00 2001 From: bytedream Date: Wed, 3 Apr 2024 17:14:07 +0200 Subject: [PATCH 039/113] Update dependencies and version --- Cargo.lock | 65 +++++++++++++++++++++++++++++++++---- Cargo.toml | 2 +- crunchy-cli-core/Cargo.toml | 6 ++-- 3 files changed, 63 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 52bba41..54d364c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -342,7 +342,7 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "crunchy-cli" -version = "3.3.2" +version = "3.3.3" dependencies = [ "chrono", "clap", @@ -355,7 +355,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.3.2" +version = "3.3.3" dependencies = [ "anyhow", "async-speed-limit", @@ -390,9 +390,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05fcd99a09d001333ab482412473aaa03c84f980d451845b4c43d58986b6eb64" +checksum = "b9dba87354272cbe34eea8c27cab75b5104d7334aa6374db4807bd145f77a5ac" dependencies = [ "async-trait", "chrono", @@ -415,9 +415,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d406a27eca9ceab379b601101cd96b0f7bfff34e93487ca58d54118a8b4fbf91" +checksum = "7f7727afdfa43dcc8981a83c299a2e416262053402dc0586b15bfe0488f05e23" dependencies = [ "darling", "quote", @@ -559,6 +559,15 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -716,6 +725,25 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +[[package]] +name = "h2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51ee2dd2e4f378392eeff5d51618cd9a63166a2513846bbc55f21cfacd9199d4" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap 2.2.6", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -795,6 +823,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", + "h2", "http", "http-body", "httparse", @@ -1370,8 +1399,10 @@ dependencies = [ "bytes", "cookie", "cookie_store", + "encoding_rs", "futures-core", "futures-util", + "h2", "http", "http-body", "http-body-util", @@ -1395,6 +1426,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper", + "system-configuration", "tokio", "tokio-native-tls", "tokio-rustls", @@ -1741,6 +1773,27 @@ dependencies = [ "libc", ] +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tempfile" version = "3.10.1" diff --git a/Cargo.toml b/Cargo.toml index 159b3cd..3a86a3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.3.2" +version = "3.3.3" edition = "2021" license = "MIT" diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index f3e8b5f..9d175d8 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.3.2" +version = "3.3.3" edition = "2021" license = "MIT" @@ -16,7 +16,7 @@ anyhow = "1.0" async-speed-limit = "0.4" clap = { version = "4.5", features = ["derive", "string"] } chrono = "0.4" -crunchyroll-rs = { version = "0.10.0", features = ["experimental-stabilizations", "tower"] } +crunchyroll-rs = { version = "0.10.1", features = ["experimental-stabilizations", "tower"] } ctrlc = "3.4" dialoguer = { version = "0.11", default-features = false } dirs = "5.0" @@ -29,7 +29,7 @@ lazy_static = "1.4" log = { version = "0.4", features = ["std"] } num_cpus = "1.16" regex = "1.10" -reqwest = { version = "0.12", default-features = false, features = ["socks", "stream"] } +reqwest = { version = "0.12", features = ["socks", "stream"] } serde = "1.0" serde_json = "1.0" serde_plain = "1.0" From 6b6d24a575ab4a4e873b93d11127d201f0efebb8 Mon Sep 17 00:00:00 2001 From: bytedream Date: Thu, 4 Apr 2024 21:01:34 +0200 Subject: [PATCH 040/113] Update dependencies and version --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 2 +- crunchy-cli-core/Cargo.toml | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 54d364c..f584cee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -342,7 +342,7 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "crunchy-cli" -version = "3.3.3" +version = "3.3.4" dependencies = [ "chrono", "clap", @@ -355,7 +355,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.3.3" +version = "3.3.4" dependencies = [ "anyhow", "async-speed-limit", @@ -390,9 +390,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9dba87354272cbe34eea8c27cab75b5104d7334aa6374db4807bd145f77a5ac" +checksum = "0010e5dded0388e3a1e69105c2e65637d092eff049ba10f132f216c8e26a2473" dependencies = [ "async-trait", "chrono", @@ -415,9 +415,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7727afdfa43dcc8981a83c299a2e416262053402dc0586b15bfe0488f05e23" +checksum = "7e5226275711b3d1dc6afdc5e2241a45bb5d4dc1a813902265d628ccfe1ab67d" dependencies = [ "darling", "quote", @@ -727,9 +727,9 @@ checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "h2" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51ee2dd2e4f378392eeff5d51618cd9a63166a2513846bbc55f21cfacd9199d4" +checksum = "816ec7294445779408f36fe57bc5b7fc1cf59664059096c65f905c1c61f58069" dependencies = [ "bytes", "fnv", diff --git a/Cargo.toml b/Cargo.toml index 3a86a3d..221db3f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.3.3" +version = "3.3.4" edition = "2021" license = "MIT" diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 9d175d8..0bbf493 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.3.3" +version = "3.3.4" edition = "2021" license = "MIT" @@ -16,7 +16,7 @@ anyhow = "1.0" async-speed-limit = "0.4" clap = { version = "4.5", features = ["derive", "string"] } chrono = "0.4" -crunchyroll-rs = { version = "0.10.1", features = ["experimental-stabilizations", "tower"] } +crunchyroll-rs = { version = "0.10.2", features = ["experimental-stabilizations", "tower"] } ctrlc = "3.4" dialoguer = { version = "0.11", default-features = false } dirs = "5.0" From c40ea8b1320eda6b2243147ec345161d7b81b53a Mon Sep 17 00:00:00 2001 From: bytedream Date: Fri, 5 Apr 2024 22:31:39 +0200 Subject: [PATCH 041/113] Update dependencies and version --- Cargo.lock | 35 +++++++++++++---------------------- Cargo.toml | 2 +- crunchy-cli-core/Cargo.toml | 4 ++-- 3 files changed, 16 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f584cee..7d0b7e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -342,7 +342,7 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "crunchy-cli" -version = "3.3.4" +version = "3.3.5" dependencies = [ "chrono", "clap", @@ -355,7 +355,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.3.4" +version = "3.3.5" dependencies = [ "anyhow", "async-speed-limit", @@ -390,9 +390,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0010e5dded0388e3a1e69105c2e65637d092eff049ba10f132f216c8e26a2473" +checksum = "2eae6c95ec38118d02ef2ca738e245d8afc404f05e502051013dc37e0295bb32" dependencies = [ "async-trait", "chrono", @@ -415,9 +415,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e5226275711b3d1dc6afdc5e2241a45bb5d4dc1a813902265d628ccfe1ab67d" +checksum = "2f589713700c948db9a976d3f83816ab12efebdf759044a7bb963dad62000c12" dependencies = [ "darling", "quote", @@ -1391,11 +1391,11 @@ checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" [[package]] name = "reqwest" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d66674f2b6fb864665eea7a3c1ac4e3dfacd2fda83cf6f935a612e01b0e3338" +checksum = "3e6cc1e89e689536eb5aeede61520e874df5a4707df811cd5da4aa5fbb2aae19" dependencies = [ - "base64 0.21.7", + "base64 0.22.0", "bytes", "cookie", "cookie_store", @@ -1420,7 +1420,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rustls", - "rustls-pemfile 1.0.4", + "rustls-pemfile", "rustls-pki-types", "serde", "serde_json", @@ -1503,21 +1503,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" dependencies = [ "openssl-probe", - "rustls-pemfile 2.1.1", + "rustls-pemfile", "rustls-pki-types", "schannel", "security-framework", ] -[[package]] -name = "rustls-pemfile" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" -dependencies = [ - "base64 0.21.7", -] - [[package]] name = "rustls-pemfile" version = "2.1.1" @@ -2362,9 +2353,9 @@ checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" [[package]] name = "winreg" -version = "0.50.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" dependencies = [ "cfg-if", "windows-sys 0.48.0", diff --git a/Cargo.toml b/Cargo.toml index 221db3f..c49244b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.3.4" +version = "3.3.5" edition = "2021" license = "MIT" diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 0bbf493..07973e1 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.3.4" +version = "3.3.5" edition = "2021" license = "MIT" @@ -16,7 +16,7 @@ anyhow = "1.0" async-speed-limit = "0.4" clap = { version = "4.5", features = ["derive", "string"] } chrono = "0.4" -crunchyroll-rs = { version = "0.10.2", features = ["experimental-stabilizations", "tower"] } +crunchyroll-rs = { version = "0.10.3", features = ["experimental-stabilizations", "tower"] } ctrlc = "3.4" dialoguer = { version = "0.11", default-features = false } dirs = "5.0" From 4b74299733732ee07f94dcc503d1c64920c888f2 Mon Sep 17 00:00:00 2001 From: bytedream Date: Fri, 5 Apr 2024 22:53:53 +0200 Subject: [PATCH 042/113] Only run ci action on branch push --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6991cad..70f4400 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,8 @@ name: ci on: push: + branches: + - '*' pull_request: workflow_dispatch: From 25cde6163c8c0a5b13459274f4239b1b7f99181a Mon Sep 17 00:00:00 2001 From: bytedream Date: Sat, 6 Apr 2024 21:23:21 +0200 Subject: [PATCH 043/113] Add account scope for search command --- crunchy-cli-core/src/lib.rs | 2 - crunchy-cli-core/src/search/command.rs | 13 ++++-- crunchy-cli-core/src/search/format.rs | 59 ++++++++++++++++++++++++-- 3 files changed, 65 insertions(+), 9 deletions(-) diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index 8067c42..483cb63 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -225,8 +225,6 @@ async fn execute_executor(executor: impl Execute, ctx: Context) { if let Some(crunchy_error) = err.downcast_mut::() { if let Error::Block { message, .. } = crunchy_error { *message = "Triggered Cloudflare bot protection. Try again later or use a VPN or proxy to spoof your location".to_string() - } else if let Error::Request { message, .. } = crunchy_error { - *message = "You've probably hit a rate limit. Try again later, generally after 10-20 minutes the rate limit is over and you can continue to use the cli".to_string() } error!("An error occurred: {}", crunchy_error) diff --git a/crunchy-cli-core/src/search/command.rs b/crunchy-cli-core/src/search/command.rs index f683e24..c29ce34 100644 --- a/crunchy-cli-core/src/search/command.rs +++ b/crunchy-cli-core/src/search/command.rs @@ -8,6 +8,7 @@ use crunchyroll_rs::common::StreamExt; use crunchyroll_rs::search::QueryResults; use crunchyroll_rs::{Episode, Locale, MediaCollection, MovieListing, MusicVideo, Series}; use log::warn; +use std::sync::Arc; #[derive(Debug, clap::Parser)] #[clap(about = "Search in videos")] @@ -87,11 +88,16 @@ pub struct Search { /// concert.premium_only → If the concert is only available with Crunchyroll premium /// /// stream.locale → Stream locale/language - /// stream.dash_url → Stream url in DASH format - /// stream.is_drm → If `stream.is_drm` is DRM encrypted + /// stream.dash_url → Stream url in DASH format. You need to set the `Authorization` header to `Bearer ` when requesting this url + /// stream.is_drm → If `stream.dash_url` is DRM encrypted /// /// subtitle.locale → Subtitle locale/language /// subtitle.url → Url to the subtitle + /// + /// account.token → Access token to make request to restricted endpoints. This token is only valid for a max. of 5 minutes + /// account.id → Internal ID of the user account + /// account.profile_name → Profile name of the account + /// account.email → Email address of the account #[arg(short, long, verbatim_doc_comment)] #[arg(default_value = "S{{season.number}}E{{episode.number}} - {{episode.title}}")] output: String, @@ -143,13 +149,14 @@ impl Execute for Search { output }; + let crunchy_arc = Arc::new(ctx.crunchy); for (media_collection, url_filter) in input { let filter_options = FilterOptions { audio: self.audio.clone(), url_filter, }; - let format = Format::new(self.output.clone(), filter_options)?; + let format = Format::new(self.output.clone(), filter_options, crunchy_arc.clone())?; println!("{}", format.parse(media_collection).await?); } diff --git a/crunchy-cli-core/src/search/format.rs b/crunchy-cli-core/src/search/format.rs index 10eefd8..7ea84d8 100644 --- a/crunchy-cli-core/src/search/format.rs +++ b/crunchy-cli-core/src/search/format.rs @@ -2,13 +2,15 @@ use crate::search::filter::FilterOptions; use anyhow::{bail, Result}; use crunchyroll_rs::media::{Stream, Subtitle}; use crunchyroll_rs::{ - Concert, Episode, Locale, MediaCollection, Movie, MovieListing, MusicVideo, Season, Series, + Concert, Crunchyroll, Episode, Locale, MediaCollection, Movie, MovieListing, MusicVideo, + Season, Series, }; use regex::Regex; use serde::Serialize; use serde_json::{Map, Value}; use std::collections::HashMap; use std::ops::Range; +use std::sync::Arc; #[derive(Default, Serialize)] struct FormatSeries { @@ -191,6 +193,27 @@ impl From<&Subtitle> for FormatSubtitle { } } +#[derive(Default, Serialize)] +struct FormatAccount { + pub token: String, + pub id: String, + pub profile_name: String, + pub email: String, +} + +impl FormatAccount { + pub async fn async_from(value: &Crunchyroll) -> Result { + let account = value.account().await?; + + Ok(Self { + token: value.access_token().await, + id: account.account_id, + profile_name: account.profile_name, + email: account.email, + }) + } +} + #[derive(Clone, Debug, Eq, PartialEq, Hash)] enum Scope { Series, @@ -202,6 +225,7 @@ enum Scope { Concert, Stream, Subtitle, + Account, } macro_rules! must_match_if_true { @@ -230,10 +254,15 @@ pub struct Format { pattern_count: HashMap, input: String, filter_options: FilterOptions, + crunchyroll: Arc, } impl Format { - pub fn new(input: String, filter_options: FilterOptions) -> Result { + pub fn new( + input: String, + filter_options: FilterOptions, + crunchyroll: Arc, + ) -> Result { let scope_regex = Regex::new(r"(?m)\{\{\s*(?P\w+)\.(?P\w+)\s*}}").unwrap(); let mut pattern = vec![]; let mut pattern_count = HashMap::new(); @@ -260,6 +289,7 @@ impl Format { Scope::Concert => FormatConcert Scope::Stream => FormatStream Scope::Subtitle => FormatSubtitle + Scope::Account => FormatAccount ); for capture in scope_regex.captures_iter(&input) { @@ -277,6 +307,7 @@ impl Format { "concert" => Scope::Concert, "stream" => Scope::Stream, "subtitle" => Scope::Subtitle, + "account" => Scope::Account, _ => bail!("'{}.{}' is not a valid keyword", scope, field), }; @@ -302,6 +333,7 @@ impl Format { pattern_count, input, filter_options, + crunchyroll, }) } @@ -316,6 +348,7 @@ impl Format { Scope::Episode, Scope::Stream, Scope::Subtitle, + Scope::Account, ])?; self.parse_series(media_collection).await @@ -326,17 +359,28 @@ impl Format { Scope::Movie, Scope::Stream, Scope::Subtitle, + Scope::Account, ])?; self.parse_movie_listing(media_collection).await } MediaCollection::MusicVideo(_) => { - self.check_scopes(vec![Scope::MusicVideo, Scope::Stream, Scope::Subtitle])?; + self.check_scopes(vec![ + Scope::MusicVideo, + Scope::Stream, + Scope::Subtitle, + Scope::Account, + ])?; self.parse_music_video(media_collection).await } MediaCollection::Concert(_) => { - self.check_scopes(vec![Scope::Concert, Scope::Stream, Scope::Subtitle])?; + self.check_scopes(vec![ + Scope::Concert, + Scope::Stream, + Scope::Subtitle, + Scope::Account, + ])?; self.parse_concert(media_collection).await } @@ -349,6 +393,7 @@ impl Format { let episode_empty = self.check_pattern_count_empty(Scope::Episode); let stream_empty = self.check_pattern_count_empty(Scope::Stream) && self.check_pattern_count_empty(Scope::Subtitle); + let account_empty = self.check_pattern_count_empty(Scope::Account); #[allow(clippy::type_complexity)] let mut tree: Vec<(Season, Vec<(Episode, Vec)>)> = vec![]; @@ -431,6 +476,11 @@ impl Format { } let mut output = vec![]; + let account_map = if !account_empty { + self.serializable_to_json_map(FormatAccount::async_from(&self.crunchyroll).await?) + } else { + Map::default() + }; let series_map = self.serializable_to_json_map(FormatSeries::from(&series)); for (season, episodes) in tree { let season_map = self.serializable_to_json_map(FormatSeason::from(&season)); @@ -442,6 +492,7 @@ impl Format { output.push( self.replace_all( HashMap::from([ + (Scope::Account, &account_map), (Scope::Series, &series_map), (Scope::Season, &season_map), (Scope::Episode, &episode_map), From fe49161e933e739784393a5ba92e29291025b803 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 8 Apr 2024 00:37:19 +0200 Subject: [PATCH 044/113] End ffmpeg progress always with at least 100% --- crunchy-cli-core/src/utils/download.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index d7904f3..cee89e2 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -1285,21 +1285,11 @@ async fn ffmpeg_progress( let reader = BufReader::new(stats); let mut lines = reader.lines(); + let mut frame = 0; loop { select! { - // when gracefully canceling this future, set the progress to 100% (finished). sometimes - // ffmpeg is too fast or already finished when the reading process of 'stats' starts - // which causes the progress to be stuck at 0% _ = cancellation_token.cancelled() => { - if let Some(p) = &progress { - p.set_position(total_frames) - } - debug!( - "Processed frame [{}/{} 100%]", - total_frames, - total_frames - ); - return Ok(()) + break } line = lines.next_line() => { let Some(line) = line? else { @@ -1314,7 +1304,7 @@ async fn ffmpeg_progress( let Some(frame_str) = frame_cap.name("frame") else { break }; - let frame: u64 = frame_str.as_str().parse()?; + frame = frame_str.as_str().parse()?; if let Some(p) = &progress { p.set_position(frame) @@ -1330,5 +1320,15 @@ async fn ffmpeg_progress( } } + // when this future is gracefully cancelled or if ffmpeg is too fast or already finished when + // reading process of 'stats' starts (which causes the progress to be stuck at 0%), the progress + // is manually set to 100% here + if frame < total_frames { + if let Some(p) = &progress { + p.set_position(frame) + } + debug!("Processed frame [{}/{} 100%]", total_frames, total_frames); + } + Ok(()) } From 1a511e12f95e7e4e76d97a4624306387909c6590 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 8 Apr 2024 13:57:06 +0200 Subject: [PATCH 045/113] Add archive start sync flag --- Cargo.lock | 122 +++++ crunchy-cli-core/Cargo.toml | 2 + crunchy-cli-core/src/archive/command.rs | 35 +- crunchy-cli-core/src/download/command.rs | 4 +- crunchy-cli-core/src/lib.rs | 33 +- crunchy-cli-core/src/utils/download.rs | 653 +++++++++++++++++------ crunchy-cli-core/src/utils/os.rs | 20 +- crunchy-cli-core/src/utils/video.rs | 2 +- 8 files changed, 692 insertions(+), 179 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7d0b7e1..c1f24f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -179,6 +179,18 @@ version = "3.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" +[[package]] +name = "bytemuck" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d6d68c57235a3a081186990eca2867354726650f42f7516ca50c28d6281fd15" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.6.0" @@ -369,6 +381,8 @@ dependencies = [ "fs2", "futures-util", "http", + "image", + "image_hasher", "indicatif", "lazy_static", "log", @@ -936,6 +950,32 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "image" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd54d660e773627692c524beaad361aca785a4f9f5730ce91f42aabe5bce3d11" +dependencies = [ + "bytemuck", + "byteorder", + "num-traits", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image_hasher" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481465fe767d92494987319b0b447a5829edf57f09c52bf8639396abaaeaf78" +dependencies = [ + "base64 0.22.0", + "image", + "rustdct", + "serde", + "transpose", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -1143,12 +1183,30 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num-complex" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23c6602fda94a57c990fe0df199a035d83576b496aa29f4e634a8ac6004e68a6" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.18" @@ -1305,6 +1363,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "primal-check" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9df7f93fd637f083201473dab4fee2db4c429d32e55e3299980ab3957ab916a0" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro2" version = "1.0.79" @@ -1469,6 +1536,30 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustdct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b61555105d6a9bf98797c063c362a1d24ed8ab0431655e38f1cf51e52089551" +dependencies = [ + "rustfft", +] + +[[package]] +name = "rustfft" +version = "6.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43806561bc506d0c5d160643ad742e3161049ac01027b5e6d7524091fd401d86" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", + "version_check", +] + [[package]] name = "rustix" version = "0.38.32" @@ -1720,6 +1811,12 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "strsim" version = "0.10.0" @@ -1998,6 +2095,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -2377,3 +2484,18 @@ name = "zeroize" version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" + +[[package]] +name = "zune-core" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" + +[[package]] +name = "zune-jpeg" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec866b44a2a1fd6133d363f073ca1b179f438f99e7e5bfb1e33f7181facfe448" +dependencies = [ + "zune-core", +] diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 07973e1..bd47aba 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -24,6 +24,8 @@ derive_setters = "0.1" futures-util = { version = "0.3", features = ["io"] } fs2 = "0.4" http = "1.1" +image = { version = "0.25", features = ["jpeg"], default-features = false } +image_hasher = "2.0" indicatif = "0.17" lazy_static = "1.4" log = { version = "0.4", features = ["std"] } diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index 0dc0b86..64ad66a 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -10,7 +10,7 @@ use crate::utils::locale::{all_locale_in_locales, resolve_locales, LanguageTaggi use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file}; use crate::utils::parse::parse_url; -use crate::utils::video::variant_data_from_stream; +use crate::utils::video::stream_data_from_stream; use crate::Execute; use anyhow::bail; use anyhow::Result; @@ -89,6 +89,17 @@ pub struct Archive { #[arg(value_parser = crate::utils::clap::clap_parse_resolution)] pub(crate) resolution: Resolution, + #[arg(help = "Tries to sync the timing of all downloaded audios to match one video")] + #[arg( + long_help = "Tries to sync the timing of all downloaded audios to match one video. \ + This is done by downloading the first few segments/frames of all video tracks that differ in length and comparing them frame by frame. \ + The value of this flag determines how accurate the syncing is, generally speaking everything over 15 begins to be more inaccurate and everything below 6 is too accurate (and won't succeed). \ + If you want to provide a custom value to this flag, you have to set it with an equals (e.g. `--sync-start=10` instead of `--sync-start 10`). \ + When the syncing fails, the command is continued as if `--sync-start` wasn't provided for this episode + " + )] + #[arg(long, require_equals = true, num_args = 0..=1, default_missing_value = "7.5")] + pub(crate) sync_start: Option, #[arg( help = "Sets the behavior of the stream merging. Valid behaviors are 'auto', 'audio' and 'video'" )] @@ -216,8 +227,19 @@ impl Execute for Archive { } } - if self.include_chapters && !matches!(self.merge, MergeBehavior::Audio) { - bail!("`--include-chapters` can only be used if `--merge` is set to 'audio'") + if self.include_chapters + && !matches!(self.merge, MergeBehavior::Audio) + && self.sync_start.is_none() + { + bail!("`--include-chapters` can only be used if `--merge` is set to 'audio' or `--sync-start` is set") + } + + if !matches!(self.merge, MergeBehavior::Auto) && self.sync_start.is_some() { + bail!("`--sync-start` can only be used if `--merge` is set to `auto`") + } + + if self.sync_start.is_some() && self.ffmpeg_preset.is_none() { + warn!("Using `--sync-start` without `--ffmpeg-preset` might produce worse sync results than with `--ffmpeg-preset` set") } if self.output.contains("{resolution}") @@ -294,6 +316,7 @@ impl Execute for Archive { .audio_sort(Some(self.audio.clone())) .subtitle_sort(Some(self.subtitle.clone())) .no_closed_caption(self.no_closed_caption) + .sync_start_value(self.sync_start) .threads(self.threads) .audio_locale_output_map( zip(self.audio.clone(), self.output_audio_locales.clone()).collect(), @@ -450,7 +473,7 @@ async fn get_format( for single_format in single_formats { let stream = single_format.stream().await?; let Some((video, audio, _)) = - variant_data_from_stream(&stream, &archive.resolution, None).await? + stream_data_from_stream(&stream, &archive.resolution, None).await? else { if single_format.is_episode() { bail!( @@ -569,7 +592,9 @@ async fn get_format( video: (video, single_format.audio.clone()), audios: vec![(audio, single_format.audio.clone())], subtitles, - metadata: DownloadFormatMetadata { skip_events: None }, + metadata: DownloadFormatMetadata { + skip_events: single_format.skip_events().await?, + }, }, )); } diff --git a/crunchy-cli-core/src/download/command.rs b/crunchy-cli-core/src/download/command.rs index 47b29c9..fd29030 100644 --- a/crunchy-cli-core/src/download/command.rs +++ b/crunchy-cli-core/src/download/command.rs @@ -8,7 +8,7 @@ use crate::utils::locale::{resolve_locales, LanguageTagging}; use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file}; use crate::utils::parse::parse_url; -use crate::utils::video::variant_data_from_stream; +use crate::utils::video::stream_data_from_stream; use crate::Execute; use anyhow::bail; use anyhow::Result; @@ -351,7 +351,7 @@ async fn get_format( try_peer_hardsubs: bool, ) -> Result<(DownloadFormat, Format)> { let stream = single_format.stream().await?; - let Some((video, audio, contains_hardsub)) = variant_data_from_stream( + let Some((video, audio, contains_hardsub)) = stream_data_from_stream( &stream, &download.resolution, if try_peer_hardsubs { diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index 483cb63..d8d5ec5 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -184,16 +184,29 @@ pub async fn main(args: &[String]) { .unwrap_or_default() .starts_with(".crunchy-cli_") { - let result = fs::remove_file(file.path()); - debug!( - "Ctrl-c removed temporary file {} {}", - file.path().to_string_lossy(), - if result.is_ok() { - "successfully" - } else { - "not successfully" - } - ) + if file.file_type().map_or(true, |ft| ft.is_file()) { + let result = fs::remove_file(file.path()); + debug!( + "Ctrl-c removed temporary file {} {}", + file.path().to_string_lossy(), + if result.is_ok() { + "successfully" + } else { + "not successfully" + } + ) + } else { + let result = fs::remove_dir_all(file.path()); + debug!( + "Ctrl-c removed temporary directory {} {}", + file.path().to_string_lossy(), + if result.is_ok() { + "successfully" + } else { + "not successfully" + } + ) + } } } } diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index cee89e2..fe0e84a 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -1,11 +1,15 @@ use crate::utils::ffmpeg::FFmpegPreset; use crate::utils::filter::real_dedup_vec; -use crate::utils::os::{cache_dir, is_special_file, temp_directory, temp_named_pipe, tempfile}; +use crate::utils::log::progress; +use crate::utils::os::{ + cache_dir, is_special_file, temp_directory, temp_named_pipe, tempdir, tempfile, +}; use crate::utils::rate_limit::RateLimiterService; use anyhow::{bail, Result}; -use chrono::NaiveTime; +use chrono::{NaiveTime, TimeDelta}; use crunchyroll_rs::media::{SkipEvents, SkipEventsEvent, StreamData, StreamSegment, Subtitle}; use crunchyroll_rs::Locale; +use image_hasher::{Hasher, HasherConfig, ImageHash}; use indicatif::{ProgressBar, ProgressDrawTarget, ProgressFinish, ProgressStyle}; use log::{debug, warn, LevelFilter}; use regex::Regex; @@ -59,6 +63,7 @@ pub struct DownloadBuilder { force_hardsub: bool, download_fonts: bool, no_closed_caption: bool, + sync_start_value: Option, threads: usize, ffmpeg_threads: Option, audio_locale_output_map: HashMap, @@ -78,6 +83,7 @@ impl DownloadBuilder { force_hardsub: false, download_fonts: false, no_closed_caption: false, + sync_start_value: None, threads: num_cpus::get(), ffmpeg_threads: None, audio_locale_output_map: HashMap::new(), @@ -99,6 +105,8 @@ impl DownloadBuilder { download_fonts: self.download_fonts, no_closed_caption: self.no_closed_caption, + sync_start_value: self.sync_start_value, + download_threads: self.threads, ffmpeg_threads: self.ffmpeg_threads, @@ -110,10 +118,23 @@ impl DownloadBuilder { } } -struct FFmpegMeta { +struct FFmpegVideoMeta { path: TempPath, - language: Locale, - title: String, + length: TimeDelta, + start_time: Option, +} + +struct FFmpegAudioMeta { + path: TempPath, + locale: Locale, + start_time: Option, +} + +struct FFmpegSubtitleMeta { + path: TempPath, + locale: Locale, + cc: bool, + start_time: Option, } pub struct DownloadFormat { @@ -141,6 +162,8 @@ pub struct Downloader { download_fonts: bool, no_closed_caption: bool, + sync_start_value: Option, + download_threads: usize, ffmpeg_threads: Option, @@ -216,13 +239,16 @@ impl Downloader { } } + let mut video_offset = None; + let mut audio_offsets = HashMap::new(); + let mut subtitle_offsets = HashMap::new(); let mut videos = vec![]; let mut audios = vec![]; let mut subtitles = vec![]; let mut fonts = vec![]; let mut chapters = None; - let mut max_len = NaiveTime::MIN; - let mut max_frames = 0f64; + let mut max_len = TimeDelta::min_value(); + let mut max_frames = 0; let fmt_space = self .formats .iter() @@ -234,115 +260,252 @@ impl Downloader { .max() .unwrap(); - for (i, format) in self.formats.iter().enumerate() { - let video_path = self - .download_video( - &format.video.0, - format!("{:<1$}", format!("Downloading video #{}", i + 1), fmt_space), - ) - .await?; - for (variant_data, locale) in format.audios.iter() { - let audio_path = self - .download_audio( - variant_data, - format!("{:<1$}", format!("Downloading {} audio", locale), fmt_space), + if self.formats.len() > 1 && self.sync_start_value.is_some() { + let all_segments_count: Vec = self + .formats + .iter() + .map(|f| f.video.0.segments().len()) + .collect(); + let sync_segments = 11.max( + all_segments_count.iter().max().unwrap() - all_segments_count.iter().min().unwrap(), + ); + let mut sync_vids = vec![]; + for (i, format) in self.formats.iter().enumerate() { + let path = self + .download_video( + &format.video.0, + format!("Downloading video #{} sync segments", i + 1), + Some(sync_segments), ) .await?; - audios.push(FFmpegMeta { - path: audio_path, - language: locale.clone(), - title: if i == 0 { - locale.to_human_readable() - } else { - format!("{} [Video: #{}]", locale.to_human_readable(), i + 1) - }, + sync_vids.push(SyncVideo { + path, + length: len_from_segments(&format.video.0.segments()), + available_frames: (len_from_segments( + &format.video.0.segments()[0..sync_segments], + ) + .num_milliseconds() as f64 + * format.video.0.fps().unwrap() + / 1000.0) as u64, + idx: i, }) } - let (len, fps) = get_video_stats(&video_path)?; + let _progress_handler = + progress!("Syncing video start times (this might take some time)"); + let mut offsets = sync_videos(sync_vids, self.sync_start_value.unwrap())?; + drop(_progress_handler); + + let mut offset_pre_checked = false; + if let Some(tmp_offsets) = &offsets { + let formats_with_offset: Vec = self + .formats + .iter() + .enumerate() + .map(|(i, f)| { + len_from_segments(&f.video.0.segments()) + - TimeDelta::milliseconds( + tmp_offsets + .get(&i) + .map(|o| (*o as f64 / f.video.0.fps().unwrap() * 1000.0) as i64) + .unwrap_or_default(), + ) + }) + .collect(); + let min = formats_with_offset.iter().min().unwrap(); + let max = formats_with_offset.iter().max().unwrap(); + + if max.num_seconds() - min.num_seconds() > 15 { + warn!("Found difference of >15 seconds after sync, skipping applying it"); + offsets = None; + offset_pre_checked = true + } + } + + if let Some(offsets) = offsets { + let mut root_format_idx = 0; + let mut root_format_length = 0; + let mut audio_count: usize = 0; + let mut subtitle_count: usize = 0; + for (i, format) in self.formats.iter().enumerate() { + let format_fps = format.video.0.fps().unwrap(); + let format_len = format + .video + .0 + .segments() + .iter() + .map(|s| s.length.as_millis()) + .sum::() as u64 + - offsets.get(&i).map_or(0, |o| *o); + if format_len > root_format_length { + root_format_idx = i; + root_format_length = format_len; + } + + for _ in &format.audios { + if let Some(offset) = &offsets.get(&i) { + audio_offsets.insert( + audio_count, + TimeDelta::milliseconds( + (**offset as f64 / format_fps * 1000.0) as i64, + ), + ); + } + audio_count += 1 + } + for _ in &format.subtitles { + if let Some(offset) = &offsets.get(&i) { + subtitle_offsets.insert( + subtitle_count, + TimeDelta::milliseconds( + (**offset as f64 / format_fps * 1000.0) as i64, + ), + ); + } + subtitle_count += 1 + } + } + + let mut root_format = self.formats.remove(root_format_idx); + + let mut audio_prepend = vec![]; + let mut subtitle_prepend = vec![]; + let mut audio_append = vec![]; + let mut subtitle_append = vec![]; + for (i, format) in self.formats.into_iter().enumerate() { + if i < root_format_idx { + audio_prepend.extend(format.audios); + subtitle_prepend.extend(format.subtitles); + } else { + audio_append.extend(format.audios); + subtitle_append.extend(format.subtitles); + } + } + root_format.audios.splice(0..0, audio_prepend); + root_format.subtitles.splice(0..0, subtitle_prepend); + root_format.audios.extend(audio_append); + root_format.subtitles.extend(subtitle_append); + + self.formats = vec![root_format]; + video_offset = offsets.get(&root_format_idx).map(|o| { + TimeDelta::milliseconds( + (*o as f64 / self.formats[0].video.0.fps().unwrap() * 1000.0) as i64, + ) + }) + } else if !offset_pre_checked { + warn!("Couldn't find reliable sync positions") + } + } + + // downloads all videos + for (i, format) in self.formats.iter().enumerate() { + let path = self + .download_video( + &format.video.0, + format!("{:<1$}", format!("Downloading video #{}", i + 1), fmt_space), + None, + ) + .await?; + + let (len, fps) = get_video_stats(&path)?; if max_len < len { max_len = len } - let frames = len.signed_duration_since(NaiveTime::MIN).num_seconds() as f64 * fps; - if frames > max_frames { - max_frames = frames; + let frames = ((len.num_milliseconds() as f64 + - video_offset.unwrap_or_default().num_milliseconds() as f64) + / 1000.0 + * fps) as u64; + if max_frames < frames { + max_frames = frames } - if !format.subtitles.is_empty() { - let progress_spinner = if log::max_level() == LevelFilter::Info { - let progress_spinner = ProgressBar::new_spinner() - .with_style( - ProgressStyle::with_template( - format!( - ":: {:<1$} {{msg}} {{spinner}}", - "Downloading subtitles", fmt_space - ) - .as_str(), + videos.push(FFmpegVideoMeta { + path, + length: len, + start_time: video_offset, + }) + } + + // downloads all audios + for format in &self.formats { + for (j, (stream_data, locale)) in format.audios.iter().enumerate() { + let path = self + .download_audio( + stream_data, + format!("{:<1$}", format!("Downloading {} audio", locale), fmt_space), + ) + .await?; + audios.push(FFmpegAudioMeta { + path, + locale: locale.clone(), + start_time: audio_offsets.get(&j).cloned(), + }) + } + } + + for (i, format) in self.formats.iter().enumerate() { + if format.subtitles.is_empty() { + continue; + } + + let progress_spinner = if log::max_level() == LevelFilter::Info { + let progress_spinner = ProgressBar::new_spinner() + .with_style( + ProgressStyle::with_template( + format!( + ":: {:<1$} {{msg}} {{spinner}}", + "Downloading subtitles", fmt_space ) - .unwrap() - .tick_strings(&["—", "\\", "|", "/", ""]), + .as_str(), ) - .with_finish(ProgressFinish::Abandon); - progress_spinner.enable_steady_tick(Duration::from_millis(100)); - Some(progress_spinner) - } else { - None - }; + .unwrap() + .tick_strings(&["—", "\\", "|", "/", ""]), + ) + .with_finish(ProgressFinish::Abandon); + progress_spinner.enable_steady_tick(Duration::from_millis(100)); + Some(progress_spinner) + } else { + None + }; - for (subtitle, not_cc) in format.subtitles.iter() { - if !not_cc && self.no_closed_caption { - continue; - } - - if let Some(pb) = &progress_spinner { - let mut progress_message = pb.message(); - if !progress_message.is_empty() { - progress_message += ", " - } - progress_message += &subtitle.locale.to_string(); - if !not_cc { - progress_message += " (CC)"; - } - if i != 0 { - progress_message += &format!(" [Video: #{}]", i + 1); - } - pb.set_message(progress_message) - } - - let mut subtitle_title = subtitle.locale.to_human_readable(); - if !not_cc { - subtitle_title += " (CC)" - } - if i != 0 { - subtitle_title += &format!(" [Video: #{}]", i + 1) - } - - let subtitle_path = self.download_subtitle(subtitle.clone(), len).await?; - debug!( - "Downloaded {} subtitles{}{}", - subtitle.locale, - (!not_cc).then_some(" (cc)").unwrap_or_default(), - (i != 0) - .then_some(format!(" for video {}", i)) - .unwrap_or_default() - ); - subtitles.push(FFmpegMeta { - path: subtitle_path, - language: subtitle.locale.clone(), - title: subtitle_title, - }) + for (j, (subtitle, not_cc)) in format.subtitles.iter().enumerate() { + if !not_cc && self.no_closed_caption { + continue; } - } - videos.push(FFmpegMeta { - path: video_path, - language: format.video.1.clone(), - title: if self.formats.len() == 1 { - "Default".to_string() - } else { - format!("#{}", i + 1) - }, - }); + if let Some(pb) = &progress_spinner { + let mut progress_message = pb.message(); + if !progress_message.is_empty() { + progress_message += ", " + } + progress_message += &subtitle.locale.to_string(); + if !not_cc { + progress_message += " (CC)"; + } + if i.min(videos.len() - 1) != 0 { + progress_message += &format!(" [Video: #{}]", i + 1); + } + pb.set_message(progress_message) + } + + let path = self + .download_subtitle(subtitle.clone(), videos[i.min(videos.len() - 1)].length) + .await?; + debug!( + "Downloaded {} subtitles{}", + subtitle.locale, + (!not_cc).then_some(" (cc)").unwrap_or_default(), + ); + subtitles.push(FFmpegSubtitleMeta { + path, + locale: subtitle.locale.clone(), + cc: !not_cc, + start_time: subtitle_offsets.get(&j).cloned(), + }) + } + } + + for format in self.formats.iter() { if let Some(skip_events) = &format.metadata.skip_events { let (file, path) = tempfile(".chapter")?.into_parts(); chapters = Some(( @@ -421,17 +584,30 @@ impl Downloader { let mut metadata = vec![]; for (i, meta) in videos.iter().enumerate() { + if let Some(start_time) = meta.start_time { + input.extend(["-ss".to_string(), format_time_delta(start_time)]) + } input.extend(["-i".to_string(), meta.path.to_string_lossy().to_string()]); maps.extend(["-map".to_string(), i.to_string()]); metadata.extend([ format!("-metadata:s:v:{}", i), - format!("title={}", meta.title), + format!( + "title={}", + if videos.len() == 1 { + "Default".to_string() + } else { + format!("#{}", i + 1) + } + ), ]); // the empty language metadata is created to avoid that metadata from the original track // is copied metadata.extend([format!("-metadata:s:v:{}", i), "language=".to_string()]) } for (i, meta) in audios.iter().enumerate() { + if let Some(start_time) = meta.start_time { + input.extend(["-ss".to_string(), format_time_delta(start_time)]) + } input.extend(["-i".to_string(), meta.path.to_string_lossy().to_string()]); maps.extend(["-map".to_string(), (i + videos.len()).to_string()]); metadata.extend([ @@ -439,13 +615,20 @@ impl Downloader { format!( "language={}", self.audio_locale_output_map - .get(&meta.language) - .unwrap_or(&meta.language.to_string()) + .get(&meta.locale) + .unwrap_or(&meta.locale.to_string()) ), ]); metadata.extend([ format!("-metadata:s:a:{}", i), - format!("title={}", meta.title), + format!( + "title={}", + if videos.len() == 1 { + meta.locale.to_human_readable() + } else { + format!("{} [Video: #{}]", meta.locale.to_human_readable(), i + 1,) + } + ), ]); } @@ -465,6 +648,9 @@ impl Downloader { if container_supports_softsubs { for (i, meta) in subtitles.iter().enumerate() { + if let Some(start_time) = meta.start_time { + input.extend(["-ss".to_string(), format_time_delta(start_time)]) + } input.extend(["-i".to_string(), meta.path.to_string_lossy().to_string()]); maps.extend([ "-map".to_string(), @@ -475,13 +661,22 @@ impl Downloader { format!( "language={}", self.subtitle_locale_output_map - .get(&meta.language) - .unwrap_or(&meta.language.to_string()) + .get(&meta.locale) + .unwrap_or(&meta.locale.to_string()) ), ]); metadata.extend([ format!("-metadata:s:s:{}", i), - format!("title={}", meta.title), + format!("title={}", { + let mut title = meta.locale.to_string(); + if meta.cc { + title += " (CC)" + } + if videos.len() > 1 { + title += &format!(" [Video: #{}]", i + 1) + } + title + }), ]); } } @@ -523,10 +718,7 @@ impl Downloader { // set default subtitle if let Some(default_subtitle) = self.default_subtitle { - if let Some(position) = subtitles - .iter() - .position(|m| m.language == default_subtitle) - { + if let Some(position) = subtitles.iter().position(|m| m.locale == default_subtitle) { if container_supports_softsubs { match dst.extension().unwrap_or_default().to_str().unwrap() { "mov" | "mp4" => output_presets.extend([ @@ -585,7 +777,7 @@ impl Downloader { if container_supports_softsubs { if let Some(position) = subtitles .iter() - .position(|meta| meta.language == default_subtitle) + .position(|meta| meta.locale == default_subtitle) { command_args.extend([ format!("-disposition:s:s:{}", position), @@ -597,9 +789,7 @@ impl Downloader { // set the 'forced' flag to CC subtitles for (i, subtitle) in subtitles.iter().enumerate() { - // well, checking if the title contains '(CC)' might not be the best solutions from a - // performance perspective but easier than adjusting the `FFmpegMeta` struct - if !subtitle.title.contains("(CC)") { + if !subtitle.cc { continue; } @@ -632,7 +822,7 @@ impl Downloader { // create parent directory if it does not exist if let Some(parent) = dst.parent() { if !parent.exists() { - std::fs::create_dir_all(parent)? + fs::create_dir_all(parent)? } } @@ -650,7 +840,7 @@ impl Downloader { let ffmpeg_progress_cancellation_token = ffmpeg_progress_cancel.clone(); let ffmpeg_progress = tokio::spawn(async move { ffmpeg_progress( - max_frames as u64, + max_frames, fifo, format!("{:<1$}", "Generating output file", fmt_space + 1), ffmpeg_progress_cancellation_token, @@ -681,7 +871,7 @@ impl Downloader { let segments = stream_data.segments(); // sum the length of all streams up - estimated_required_space += estimate_variant_file_size(stream_data, &segments); + estimated_required_space += estimate_stream_data_file_size(stream_data, &segments); } let tmp_stat = fs2::statvfs(temp_directory()).unwrap(); @@ -727,11 +917,16 @@ impl Downloader { Ok((tmp_required, dst_required)) } - async fn download_video(&self, stream_data: &StreamData, message: String) -> Result { + async fn download_video( + &self, + stream_data: &StreamData, + message: String, + max_segments: Option, + ) -> Result { let tempfile = tempfile(".mp4")?; let (mut file, path) = tempfile.into_parts(); - self.download_segments(&mut file, message, stream_data) + self.download_segments(&mut file, message, stream_data, max_segments) .await?; Ok(path) @@ -741,7 +936,7 @@ impl Downloader { let tempfile = tempfile(".m4a")?; let (mut file, path) = tempfile.into_parts(); - self.download_segments(&mut file, message, stream_data) + self.download_segments(&mut file, message, stream_data, None) .await?; Ok(path) @@ -750,7 +945,7 @@ impl Downloader { async fn download_subtitle( &self, subtitle: Subtitle, - max_length: NaiveTime, + max_length: TimeDelta, ) -> Result { let tempfile = tempfile(".ass")?; let (mut file, path) = tempfile.into_parts(); @@ -796,14 +991,20 @@ impl Downloader { writer: &mut impl Write, message: String, stream_data: &StreamData, + max_segments: Option, ) -> Result<()> { - let segments = stream_data.segments(); + let mut segments = stream_data.segments(); + if let Some(max_segments) = max_segments { + segments = segments + .drain(0..max_segments.min(segments.len() - 1)) + .collect(); + } let total_segments = segments.len(); let count = Arc::new(Mutex::new(0)); let progress = if log::max_level() == LevelFilter::Info { - let estimated_file_size = estimate_variant_file_size(stream_data, &segments); + let estimated_file_size = estimate_stream_data_file_size(stream_data, &segments); let progress = ProgressBar::new(estimated_file_size) .with_style( @@ -820,7 +1021,7 @@ impl Downloader { None }; - let cpus = self.download_threads; + let cpus = self.download_threads.min(segments.len()); let mut segs: Vec> = Vec::with_capacity(cpus); for _ in 0..cpus { segs.push(vec![]) @@ -964,12 +1165,12 @@ impl Downloader { } } -fn estimate_variant_file_size(stream_data: &StreamData, segments: &[StreamSegment]) -> u64 { +fn estimate_stream_data_file_size(stream_data: &StreamData, segments: &[StreamSegment]) -> u64 { (stream_data.bandwidth / 8) * segments.iter().map(|s| s.length.as_secs()).sum::() } /// Get the length and fps of a video. -fn get_video_stats(path: &Path) -> Result<(NaiveTime, f64)> { +fn get_video_stats(path: &Path) -> Result<(TimeDelta, f64)> { let video_length = Regex::new(r"Duration:\s(?P