From a93a1fa807758cf41829233281ca5832a52c6151 Mon Sep 17 00:00:00 2001 From: bytedream Date: Fri, 22 Sep 2023 12:11:00 +0200 Subject: [PATCH 001/176] Fix env variable resolving in publish pipeline --- .github/workflows/publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 80cb5bb..1599111 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -34,7 +34,7 @@ jobs: commit_username: release-action commit_email: ${{ secrets.AUR_EMAIL }} ssh_private_key: ${{ secrets.AUR_SSH_PRIVATE_KEY }} - commit_message: Update to version {{ env.RELEASE_VERSION }} + commit_message: Update to version ${{ env.RELEASE_VERSION }} - name: Generate crunchy-cli-bin sha sums run: | @@ -67,4 +67,4 @@ jobs: commit_username: release-action commit_email: ${{ secrets.AUR_EMAIL }} ssh_private_key: ${{ secrets.AUR_SSH_PRIVATE_KEY }} - commit_message: Update to version {{ env.RELEASE_VERSION }} + commit_message: Update to version ${{ env.RELEASE_VERSION }} From d79197edc6fb9bf2bac699d2f6b55bfae4d7dea7 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sat, 23 Sep 2023 16:56:38 +0200 Subject: [PATCH 002/176] Use async mutex and channel instead of the std equivalents --- crunchy-cli-core/src/utils/download.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index 4531c69..29f160a 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -17,9 +17,11 @@ use std::env; use std::io::Write; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -use std::sync::{mpsc, Arc, Mutex}; +use std::sync::Arc; use std::time::Duration; use tempfile::TempPath; +use tokio::sync::mpsc::unbounded_channel; +use tokio::sync::Mutex; use tokio::task::JoinSet; #[derive(Clone, Debug)] @@ -576,7 +578,7 @@ pub async fn download_segments( segs[i - ((i / cpus) * cpus)].push(segment); } - let (sender, receiver) = mpsc::channel(); + let (sender, mut receiver) = unbounded_channel(); let mut join_set: JoinSet> = JoinSet::new(); for num in 0..cpus { @@ -629,7 +631,7 @@ pub async fn download_segments( buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec(); - let mut c = thread_count.lock().unwrap(); + let mut c = thread_count.lock().await; debug!( "Downloaded and decrypted segment [{}/{} {:.2}%] {}", num + (i * cpus) + 1, @@ -663,7 +665,7 @@ pub async fn download_segments( // the segment number and the values the corresponding bytes let mut data_pos = 0; let mut buf: BTreeMap> = BTreeMap::new(); - for (pos, bytes) in receiver.iter() { + while let Some((pos, bytes)) = receiver.recv().await { // if the position is lower than 0, an error occurred in the sending download thread if pos < 0 { break; From f48474ba776bce8a6d00e6f0c97f38cf047b8eb4 Mon Sep 17 00:00:00 2001 From: bytedream Date: Wed, 27 Sep 2023 00:03:26 +0200 Subject: [PATCH 003/176] Remove numbers from binary PKGBUILD env variables --- .github/scripts/PKGBUILD.binary | 4 ++-- .github/workflows/publish.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/scripts/PKGBUILD.binary b/.github/scripts/PKGBUILD.binary index 3493ee6..00fee47 100644 --- a/.github/scripts/PKGBUILD.binary +++ b/.github/scripts/PKGBUILD.binary @@ -24,8 +24,8 @@ source_aarch64=( "LICENSE::https://raw.githubusercontent.com/crunchy-labs/crunchy-cli/v${pkgver}/LICENSE" ) noextract=("manpages.zip" "completions.zip") -sha256sums_x86_64=('$CI_x86_64_SHA_SUM' '$CI_MANPAGES_SHA_SUM' '$CI_COMPLETIONS_SHA_SUM' '$CI_LICENSE_SHA_SUM') -sha256sums_aarch64=('$CI_aarch64_SHA_SUM' '$CI_MANPAGES_SHA_SUM' '$CI_COMPLETIONS_SHA_SUM' '$CI_LICENSE_SHA_SUM') +sha256sums_x86_64=('$CI_AMD_BINARY_SHA_SUM' '$CI_MANPAGES_SHA_SUM' '$CI_COMPLETIONS_SHA_SUM' '$CI_LICENSE_SHA_SUM') +sha256sums_aarch64=('$CI_ARM_BINARY_SHA_SUM' '$CI_MANPAGES_SHA_SUM' '$CI_COMPLETIONS_SHA_SUM' '$CI_LICENSE_SHA_SUM') package() { cd "$srcdir" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1599111..f777f9e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -52,12 +52,12 @@ jobs: - name: Generate crunchy-cli-bin PKGBUILD env: CI_PKG_VERSION: ${{ env.RELEASE_VERSION }} - CI_x86_64_SHA_SUM: ${{ env.CRUNCHY_CLI_BIN_x86_64_SHA256 }} - CI_aarch64_SHA_SUM: ${{ env.CRUNCHY_CLI_BIN_aarch64_SHA256 }} + CI_AMD_BINARY_SHA_SUM: ${{ env.CRUNCHY_CLI_BIN_x86_64_SHA256 }} + CI_ARM_BINARY_SHA_SUM: ${{ env.CRUNCHY_CLI_BIN_aarch64_SHA256 }} CI_MANPAGES_SHA_SUM: ${{ env.CRUNCHY_CLI_BIN_MANPAGES_SHA256 }} CI_COMPLETIONS_SHA_SUM: ${{ env.CRUNCHY_CLI_BIN_COMPLETIONS_SHA256 }} CI_LICENSE_SHA_SUM: ${{ env.CRUNCHY_CLI_BIN_LICENSE_SHA256 }} - run: envsubst '$CI_PKG_VERSION,$CI_x86_64_SHA_SUM,$CI_aarch64_SHA_SUM,$CI_COMPLETIONS_SHA_SUM,$CI_MANPAGES_SHA_SUM,$CI_LICENSE_SHA_SUM' < .github/scripts/PKGBUILD.binary > PKGBUILD + run: envsubst '$CI_PKG_VERSION,$CI_AMD_BINARY_SHA_SUM,$CI_ARM_BINARY_SHA_SUM,$CI_COMPLETIONS_SHA_SUM,$CI_MANPAGES_SHA_SUM,$CI_LICENSE_SHA_SUM' < .github/scripts/PKGBUILD.binary > PKGBUILD - name: Publish crunchy-cli-bin to AUR uses: KSXGitHub/github-actions-deploy-aur@v2.7.0 From 9596175f7fe2a40fe2f0c20772d373aa13150a0f Mon Sep 17 00:00:00 2001 From: Valentine Briese Date: Wed, 11 Oct 2023 18:24:45 -0700 Subject: [PATCH 004/176] Add FFmpeg Apple hardware acceleration --- crunchy-cli-core/src/utils/ffmpeg.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/utils/ffmpeg.rs b/crunchy-cli-core/src/utils/ffmpeg.rs index 210b0f7..3407d91 100644 --- a/crunchy-cli-core/src/utils/ffmpeg.rs +++ b/crunchy-cli-core/src/utils/ffmpeg.rs @@ -66,7 +66,8 @@ ffmpeg_enum! { ffmpeg_enum! { enum FFmpegHwAccel { - Nvidia + Nvidia, + Apple } } @@ -275,6 +276,9 @@ impl FFmpegPreset { ]); output.extend(["-c:v", "h264_nvenc", "-c:a", "copy"]) } + FFmpegHwAccel::Apple => { + output.extend(["-c:v", "h264_videotoolbox", "-c:a", "copy"]) + } } } else { output.extend(["-c:v", "libx264", "-c:a", "copy"]) @@ -300,6 +304,9 @@ impl FFmpegPreset { ]); output.extend(["-c:v", "hevc_nvenc", "-c:a", "copy"]) } + FFmpegHwAccel::Apple => { + output.extend(["-c:v", "hevc_videotoolbox", "-c:a", "copy"]) + } } } else { output.extend(["-c:v", "libx265", "-c:a", "copy"]) From 610593a79547b577b46707f996adefa7cf27db15 Mon Sep 17 00:00:00 2001 From: Valentine Briese Date: Wed, 11 Oct 2023 18:26:51 -0700 Subject: [PATCH 005/176] Make H265 codec compatible with Apple HEVC standards --- crunchy-cli-core/src/utils/ffmpeg.rs | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/crunchy-cli-core/src/utils/ffmpeg.rs b/crunchy-cli-core/src/utils/ffmpeg.rs index 3407d91..6145fcf 100644 --- a/crunchy-cli-core/src/utils/ffmpeg.rs +++ b/crunchy-cli-core/src/utils/ffmpeg.rs @@ -302,14 +302,26 @@ impl FFmpegPreset { "-c:v", "h264_cuvid", ]); - output.extend(["-c:v", "hevc_nvenc", "-c:a", "copy"]) - } - FFmpegHwAccel::Apple => { - output.extend(["-c:v", "hevc_videotoolbox", "-c:a", "copy"]) + output.extend([ + "-c:v", + "hevc_nvenc", + "-c:a", + "copy", + "-tag:v", + "hvc1", + ]) } + FFmpegHwAccel::Apple => output.extend([ + "-c:v", + "hevc_videotoolbox", + "-c:a", + "copy", + "-tag:v", + "hvc1", + ]), } } else { - output.extend(["-c:v", "libx265", "-c:a", "copy"]) + output.extend(["-c:v", "libx265", "-c:a", "copy", "-tag:v", "hvc1"]) } match quality { From 7095e2b8b6464edeb9d79a21540365d02133a93b Mon Sep 17 00:00:00 2001 From: Valentine Briese Date: Wed, 11 Oct 2023 18:54:47 -0700 Subject: [PATCH 006/176] Use `-q:v` FFmpeg option for Apple hardware acceleration --- crunchy-cli-core/src/utils/ffmpeg.rs | 65 +++++++++++++++++++--------- 1 file changed, 45 insertions(+), 20 deletions(-) diff --git a/crunchy-cli-core/src/utils/ffmpeg.rs b/crunchy-cli-core/src/utils/ffmpeg.rs index 6145fcf..af29bfd 100644 --- a/crunchy-cli-core/src/utils/ffmpeg.rs +++ b/crunchy-cli-core/src/utils/ffmpeg.rs @@ -263,6 +263,12 @@ impl FFmpegPreset { match codec { FFmpegCodec::H264 => { + let mut crf_quality = || match quality { + FFmpegQuality::Lossless => output.extend(["-crf", "18"]), + FFmpegQuality::Normal => (), + FFmpegQuality::Low => output.extend(["-crf", "35"]), + }; + if let Some(hwaccel) = hwaccel_opt { match hwaccel { FFmpegHwAccel::Nvidia => { @@ -274,23 +280,37 @@ impl FFmpegPreset { "-c:v", "h264_cuvid", ]); + crf_quality(); output.extend(["-c:v", "h264_nvenc", "-c:a", "copy"]) } FFmpegHwAccel::Apple => { + // Apple's Video Toolbox encoders ignore `-crf`, + // use `-q:v` instead. It's on a scale of 1-100, + // 100 being lossless. Just did some math + // ((-a/51+1)*99+1 where `a` is the old crf value) + // so these settings very likely need some more + // tweeking. + match quality { + FFmpegQuality::Lossless => output.extend(["-q:v", "65"]), + FFmpegQuality::Normal => (), + FFmpegQuality::Low => output.extend(["-q:v", "32"]), + } + output.extend(["-c:v", "h264_videotoolbox", "-c:a", "copy"]) } } } else { + crf_quality(); output.extend(["-c:v", "libx264", "-c:a", "copy"]) } - - match quality { - FFmpegQuality::Lossless => output.extend(["-crf", "18"]), - FFmpegQuality::Normal => (), - FFmpegQuality::Low => output.extend(["-crf", "35"]), - } } FFmpegCodec::H265 => { + let mut crf_quality = || match quality { + FFmpegQuality::Lossless => output.extend(["-crf", "20"]), + FFmpegQuality::Normal => (), + FFmpegQuality::Low => output.extend(["-crf", "35"]), + }; + if let Some(hwaccel) = hwaccel_opt { match hwaccel { FFmpegHwAccel::Nvidia => { @@ -302,6 +322,7 @@ impl FFmpegPreset { "-c:v", "h264_cuvid", ]); + crf_quality(); output.extend([ "-c:v", "hevc_nvenc", @@ -311,24 +332,28 @@ impl FFmpegPreset { "hvc1", ]) } - FFmpegHwAccel::Apple => output.extend([ - "-c:v", - "hevc_videotoolbox", - "-c:a", - "copy", - "-tag:v", - "hvc1", - ]), + FFmpegHwAccel::Apple => { + // See the comment that starts on line 287. + match quality { + FFmpegQuality::Lossless => output.extend(["-q:v", "61"]), + FFmpegQuality::Normal => (), + FFmpegQuality::Low => output.extend(["-q:v", "32"]), + } + + output.extend([ + "-c:v", + "hevc_videotoolbox", + "-c:a", + "copy", + "-tag:v", + "hvc1", + ]) + } } } else { + crf_quality(); output.extend(["-c:v", "libx265", "-c:a", "copy", "-tag:v", "hvc1"]) } - - match quality { - FFmpegQuality::Lossless => output.extend(["-crf", "20"]), - FFmpegQuality::Normal => (), - FFmpegQuality::Low => output.extend(["-crf", "35"]), - } } FFmpegCodec::Av1 => { output.extend(["-c:v", "libsvtav1", "-c:a", "copy"]); From e5db8e95043c7d9dabeb6e05c0f2f6311563a58d Mon Sep 17 00:00:00 2001 From: Valentine Briese Date: Thu, 12 Oct 2023 12:20:06 -0700 Subject: [PATCH 007/176] Fix `ffmpeg-preset` option in `download` command (#254) --- crunchy-cli-core/src/download/command.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/download/command.rs b/crunchy-cli-core/src/download/command.rs index 031ed04..baf27bf 100644 --- a/crunchy-cli-core/src/download/command.rs +++ b/crunchy-cli-core/src/download/command.rs @@ -148,7 +148,8 @@ impl Execute for Download { Some("mpegts".to_string()) } else { None - }); + }) + .ffmpeg_preset(self.ffmpeg_preset.clone().unwrap_or_default()); for mut single_formats in single_format_collection.into_iter() { // the vec contains always only one item From 13335c020b19f80cde7e56455d4380c9921a681f Mon Sep 17 00:00:00 2001 From: bytedream Date: Fri, 13 Oct 2023 11:41:56 +0200 Subject: [PATCH 008/176] Sanitize the full output filename (#253) --- crunchy-cli-core/src/utils/format.rs | 80 +++++++++++++--------------- 1 file changed, 36 insertions(+), 44 deletions(-) diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index b6dcf35..618f7d5 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -371,51 +371,43 @@ impl Format { /// Formats the given string if it has specific pattern in it. It's possible to sanitize it which /// removes characters which can cause failures if the output string is used as a file name. pub fn format_path(&self, path: PathBuf, sanitize: bool) -> PathBuf { - let sanitize_func = if sanitize { - |s: &str| sanitize_filename::sanitize(s) + let path = path + .to_string_lossy() + .to_string() + .replace("{title}", &self.title) + .replace( + "{audio}", + &self + .locales + .iter() + .map(|(a, _)| a.to_string()) + .collect::>() + .join("|"), + ) + .replace("{resolution}", &self.resolution.to_string()) + .replace("{series_id}", &self.series_id) + .replace("{series_name}", &self.series_name) + .replace("{season_id}", &self.season_id) + .replace("{season_name}", &self.season_title) + .replace( + "{season_number}", + &format!("{:0>2}", self.season_number.to_string()), + ) + .replace("{episode_id}", &self.episode_id) + .replace( + "{episode_number}", + &format!("{:0>2}", self.episode_number.to_string()), + ) + .replace( + "{relative_episode_number}", + &self.relative_episode_number.unwrap_or_default().to_string(), + ); + + if sanitize { + PathBuf::from(sanitize_filename::sanitize(path)) } else { - // converting this to a string is actually unnecessary - |s: &str| s.to_string() - }; - - let as_string = path.to_string_lossy().to_string(); - - PathBuf::from( - as_string - .replace("{title}", &sanitize_func(&self.title)) - .replace( - "{audio}", - &sanitize_func( - &self - .locales - .iter() - .map(|(a, _)| a.to_string()) - .collect::>() - .join("|"), - ), - ) - .replace("{resolution}", &sanitize_func(&self.resolution.to_string())) - .replace("{series_id}", &sanitize_func(&self.series_id)) - .replace("{series_name}", &sanitize_func(&self.series_name)) - .replace("{season_id}", &sanitize_func(&self.season_id)) - .replace("{season_name}", &sanitize_func(&self.season_title)) - .replace( - "{season_number}", - &sanitize_func(&format!("{:0>2}", self.season_number.to_string())), - ) - .replace("{episode_id}", &sanitize_func(&self.episode_id)) - .replace( - "{episode_number}", - &sanitize_func(&format!("{:0>2}", self.episode_number.to_string())), - ) - .replace( - "{relative_episode_number}", - &sanitize_func(&format!( - "{:0>2}", - self.relative_episode_number.unwrap_or_default().to_string() - )), - ), - ) + PathBuf::from(path) + } } pub fn visual_output(&self, dst: &Path) { From 81385ef6ce257522dc197d2ce8dd9a252d2f99c7 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 15 Oct 2023 20:49:03 +0200 Subject: [PATCH 009/176] Add relative_sequence_number format option (#206, #241, #246) --- crunchy-cli-core/src/archive/command.rs | 24 +++++---- crunchy-cli-core/src/archive/filter.rs | 64 +++++++++++++++--------- crunchy-cli-core/src/download/command.rs | 24 +++++---- crunchy-cli-core/src/download/filter.rs | 60 +++++++++++----------- crunchy-cli-core/src/utils/format.rs | 33 +++++++++--- 5 files changed, 122 insertions(+), 83 deletions(-) diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index 7ce1658..ffdc00c 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -39,17 +39,19 @@ pub struct Archive { #[arg(help = "Name of the output file")] #[arg(long_help = "Name of the output file.\ If you use one of the following pattern they will get replaced:\n \ - {title} → Title of the video\n \ - {series_name} → Name of the series\n \ - {season_name} → Name of the season\n \ - {audio} → Audio language of the video\n \ - {resolution} → Resolution of the video\n \ - {season_number} → Number of the season\n \ - {episode_number} → Number of the episode\n \ - {relative_episode_number} → Number of the episode relative to its season\n \ - {series_id} → ID of the series\n \ - {season_id} → ID of the season\n \ - {episode_id} → ID of the episode")] + {title} → Title of the video\n \ + {series_name} → Name of the series\n \ + {season_name} → Name of the season\n \ + {audio} → Audio language of the video\n \ + {resolution} → Resolution of the video\n \ + {season_number} → Number of the season\n \ + {episode_number} → Number of the episode\n \ + {relative_episode_number} → Number of the episode relative to its season\n \ + {sequence_number} → Like '{episode_number}' but without possible non-number characters\n \ + {relative_sequence_number} → Like '{relative_episode_number}' but with support for episode 0's and .5's\n \ + {series_id} → ID of the series\n \ + {season_id} → ID of the season\n \ + {episode_id} → ID of the episode")] #[arg(short, long, default_value = "{title}.mkv")] pub(crate) output: String, diff --git a/crunchy-cli-core/src/archive/filter.rs b/crunchy-cli-core/src/archive/filter.rs index 851b4b5..c2cc206 100644 --- a/crunchy-cli-core/src/archive/filter.rs +++ b/crunchy-cli-core/src/archive/filter.rs @@ -18,7 +18,7 @@ pub(crate) struct ArchiveFilter { url_filter: UrlFilter, archive: Archive, interactive_input: bool, - season_episode_count: HashMap>, + season_episodes: HashMap>, season_subtitles_missing: Vec, season_sorting: Vec, visited: Visited, @@ -30,7 +30,7 @@ impl ArchiveFilter { url_filter, archive, interactive_input, - season_episode_count: HashMap::new(), + season_episodes: HashMap::new(), season_subtitles_missing: vec![], season_sorting: vec![], visited: Visited::None, @@ -226,12 +226,12 @@ impl Filter for ArchiveFilter { episodes.extend(eps) } - if Format::has_relative_episodes_fmt(&self.archive.output) { + if Format::has_relative_fmt(&self.archive.output) { for episode in episodes.iter() { - self.season_episode_count + self.season_episodes .entry(episode.season_id.clone()) .or_insert(vec![]) - .push(episode.id.clone()) + .push(episode.clone()) } } @@ -299,22 +299,34 @@ impl Filter for ArchiveFilter { episodes.push((episode.clone(), episode.subtitle_locales.clone())) } - let relative_episode_number = if Format::has_relative_episodes_fmt(&self.archive.output) { - if self.season_episode_count.get(&episode.season_id).is_none() { - let season_episodes = episode.season().await?.episodes().await?; - self.season_episode_count.insert( - episode.season_id.clone(), - season_episodes.into_iter().map(|e| e.id).collect(), - ); + let mut relative_episode_number = None; + let mut relative_sequence_number = None; + // get the relative episode number. only done if the output string has the pattern to include + // the relative episode number as this requires some extra fetching + if Format::has_relative_fmt(&self.archive.output) { + let season_eps = match self.season_episodes.get(&episode.season_id) { + Some(eps) => eps, + None => { + self.season_episodes.insert( + episode.season_id.clone(), + episode.season().await?.episodes().await?, + ); + self.season_episodes.get(&episode.season_id).unwrap() + } + }; + let mut non_integer_sequence_number_count = 0; + for (i, ep) in season_eps.iter().enumerate() { + if ep.sequence_number.fract() != 0.0 || ep.sequence_number == 0.0 { + non_integer_sequence_number_count += 1; + } + if ep.id == episode.id { + relative_episode_number = Some(i + 1); + relative_sequence_number = + Some((i + 1 - non_integer_sequence_number_count) as f32); + break; + } } - let relative_episode_number = self - .season_episode_count - .get(&episode.season_id) - .unwrap() - .iter() - .position(|id| id == &episode.id) - .map(|index| index + 1); - if relative_episode_number.is_none() { + if relative_episode_number.is_none() || relative_sequence_number.is_none() { warn!( "Failed to get relative episode number for episode {} ({}) of {} season {}", episode.episode_number, @@ -323,16 +335,18 @@ impl Filter for ArchiveFilter { episode.season_number, ) } - relative_episode_number - } else { - None - }; + } Ok(Some( episodes .into_iter() .map(|(e, s)| { - SingleFormat::new_from_episode(e, s, relative_episode_number.map(|n| n as u32)) + SingleFormat::new_from_episode( + e, + s, + relative_episode_number.map(|n| n as u32), + relative_sequence_number, + ) }) .collect(), )) diff --git a/crunchy-cli-core/src/download/command.rs b/crunchy-cli-core/src/download/command.rs index baf27bf..6d059ef 100644 --- a/crunchy-cli-core/src/download/command.rs +++ b/crunchy-cli-core/src/download/command.rs @@ -35,17 +35,19 @@ pub struct Download { #[arg(help = "Name of the output file")] #[arg(long_help = "Name of the output file.\ If you use one of the following pattern they will get replaced:\n \ - {title} → Title of the video\n \ - {series_name} → Name of the series\n \ - {season_name} → Name of the season\n \ - {audio} → Audio language of the video\n \ - {resolution} → Resolution of the video\n \ - {season_number} → Number of the season\n \ - {episode_number} → Number of the episode\n \ - {relative_episode_number} → Number of the episode relative to its season\n \ - {series_id} → ID of the series\n \ - {season_id} → ID of the season\n \ - {episode_id} → ID of the episode")] + {title} → Title of the video\n \ + {series_name} → Name of the series\n \ + {season_name} → Name of the season\n \ + {audio} → Audio language of the video\n \ + {resolution} → Resolution of the video\n \ + {season_number} → Number of the season\n \ + {episode_number} → Number of the episode\n \ + {relative_episode_number} → Number of the episode relative to its season\n \ + {sequence_number} → Like '{episode_number}' but without possible non-number characters\n \ + {relative_sequence_number} → Like '{relative_episode_number}' but with support for episode 0's and .5's\n \ + {series_id} → ID of the series\n \ + {season_id} → ID of the season\n \ + {episode_id} → ID of the episode")] #[arg(short, long, default_value = "{title}.mp4")] pub(crate) output: String, diff --git a/crunchy-cli-core/src/download/filter.rs b/crunchy-cli-core/src/download/filter.rs index 31c0db6..c5aef7e 100644 --- a/crunchy-cli-core/src/download/filter.rs +++ b/crunchy-cli-core/src/download/filter.rs @@ -12,7 +12,7 @@ pub(crate) struct DownloadFilter { url_filter: UrlFilter, download: Download, interactive_input: bool, - season_episode_count: HashMap>, + season_episodes: HashMap>, season_subtitles_missing: Vec, season_visited: bool, } @@ -23,7 +23,7 @@ impl DownloadFilter { url_filter, download, interactive_input, - season_episode_count: HashMap::new(), + season_episodes: HashMap::new(), season_subtitles_missing: vec![], season_visited: false, } @@ -107,12 +107,12 @@ impl Filter for DownloadFilter { let mut episodes = season.episodes().await?; - if Format::has_relative_episodes_fmt(&self.download.output) { + if Format::has_relative_fmt(&self.download.output) { for episode in episodes.iter() { - self.season_episode_count + self.season_episodes .entry(episode.season_number) .or_insert(vec![]) - .push(episode.id.clone()) + .push(episode.clone()) } } @@ -189,28 +189,34 @@ impl Filter for DownloadFilter { } } + let mut relative_episode_number = None; + let mut relative_sequence_number = None; // get the relative episode number. only done if the output string has the pattern to include // the relative episode number as this requires some extra fetching - let relative_episode_number = if Format::has_relative_episodes_fmt(&self.download.output) { - if self - .season_episode_count - .get(&episode.season_number) - .is_none() - { - let season_episodes = episode.season().await?.episodes().await?; - self.season_episode_count.insert( - episode.season_number, - season_episodes.into_iter().map(|e| e.id).collect(), - ); + if Format::has_relative_fmt(&self.download.output) { + let season_eps = match self.season_episodes.get(&episode.season_number) { + Some(eps) => eps, + None => { + self.season_episodes.insert( + episode.season_number, + episode.season().await?.episodes().await?, + ); + self.season_episodes.get(&episode.season_number).unwrap() + } + }; + let mut non_integer_sequence_number_count = 0; + for (i, ep) in season_eps.iter().enumerate() { + if ep.sequence_number.fract() != 0.0 || ep.sequence_number == 0.0 { + non_integer_sequence_number_count += 1; + } + if ep.id == episode.id { + relative_episode_number = Some(i + 1); + relative_sequence_number = + Some((i + 1 - non_integer_sequence_number_count) as f32); + break; + } } - let relative_episode_number = self - .season_episode_count - .get(&episode.season_number) - .unwrap() - .iter() - .position(|id| id == &episode.id) - .map(|index| index + 1); - if relative_episode_number.is_none() { + if relative_episode_number.is_none() || relative_sequence_number.is_none() { warn!( "Failed to get relative episode number for episode {} ({}) of {} season {}", episode.episode_number, @@ -219,10 +225,7 @@ impl Filter for DownloadFilter { episode.season_number, ) } - relative_episode_number - } else { - None - }; + } Ok(Some(SingleFormat::new_from_episode( episode.clone(), @@ -234,6 +237,7 @@ impl Filter for DownloadFilter { } }), relative_episode_number.map(|n| n as u32), + relative_sequence_number, ))) } diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index 618f7d5..f32d82a 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -29,8 +29,9 @@ pub struct SingleFormat { pub episode_id: String, pub episode_number: String, - pub sequence_number: f32, pub relative_episode_number: Option, + pub sequence_number: f32, + pub relative_sequence_number: Option, pub duration: Duration, @@ -42,6 +43,7 @@ impl SingleFormat { episode: Episode, subtitles: Vec, relative_episode_number: Option, + relative_sequence_number: Option, ) -> Self { Self { identifier: if episode.identifier.is_empty() { @@ -73,6 +75,7 @@ impl SingleFormat { }, sequence_number: episode.sequence_number, relative_episode_number, + relative_sequence_number, duration: episode.duration, source: episode.into(), } @@ -92,8 +95,9 @@ impl SingleFormat { season_number: 1, episode_id: movie.id.clone(), episode_number: "1".to_string(), - sequence_number: 1.0, relative_episode_number: Some(1), + sequence_number: 1.0, + relative_sequence_number: Some(1.0), duration: movie.duration, source: movie.into(), } @@ -113,8 +117,9 @@ impl SingleFormat { season_number: 1, episode_id: music_video.id.clone(), episode_number: "1".to_string(), - sequence_number: 1.0, relative_episode_number: Some(1), + sequence_number: 1.0, + relative_sequence_number: Some(1.0), duration: music_video.duration, source: music_video.into(), } @@ -134,8 +139,9 @@ impl SingleFormat { season_number: 1, episode_id: concert.id.clone(), episode_number: "1".to_string(), - sequence_number: 1.0, relative_episode_number: Some(1), + sequence_number: 1.0, + relative_sequence_number: Some(1.0), duration: concert.duration, source: concert.into(), } @@ -328,8 +334,9 @@ pub struct Format { pub episode_id: String, pub episode_number: String, - pub sequence_number: f32, pub relative_episode_number: Option, + pub sequence_number: f32, + pub relative_sequence_number: Option, } impl Format { @@ -363,8 +370,9 @@ impl Format { season_number: first_format.season_number, episode_id: first_format.episode_id, episode_number: first_format.episode_number, - sequence_number: first_format.sequence_number, relative_episode_number: first_format.relative_episode_number, + sequence_number: first_format.sequence_number, + relative_sequence_number: first_format.relative_sequence_number, } } @@ -401,6 +409,14 @@ impl Format { .replace( "{relative_episode_number}", &self.relative_episode_number.unwrap_or_default().to_string(), + ) + .replace("{sequence_number}", &self.sequence_number.to_string()) + .replace( + "{relative_sequence_number}", + &self + .relative_sequence_number + .unwrap_or_default() + .to_string(), ); if sanitize { @@ -447,7 +463,8 @@ impl Format { tab_info!("FPS: {:.2}", self.fps) } - pub fn has_relative_episodes_fmt>(s: S) -> bool { - return s.as_ref().contains("{relative_episode_number}"); + pub fn has_relative_fmt>(s: S) -> bool { + return s.as_ref().contains("{relative_episode_number}") + || s.as_ref().contains("{relative_sequence_number}"); } } From bbb5a7876520384e2e7725d1adb6bb08faa9caa5 Mon Sep 17 00:00:00 2001 From: Valentine Briese Date: Sun, 15 Oct 2023 11:52:53 -0700 Subject: [PATCH 010/176] Add `--threads` (`-t`) option to downloading commands (#256) * Add `single-threaded` option to downloading commands * Replace `--single-threaded` boolean option with `--threads` optional `usize` option * Simplify `threads` field unwrapping * Make `--threads` `usize` with a default value --- crunchy-cli-core/src/archive/command.rs | 7 +- crunchy-cli-core/src/download/command.rs | 7 +- crunchy-cli-core/src/utils/download.rs | 301 ++++++++++++----------- 3 files changed, 166 insertions(+), 149 deletions(-) diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index 7ce1658..bb4794f 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -98,6 +98,10 @@ pub struct Archive { #[arg(short, long, default_value_t = false)] pub(crate) yes: bool, + #[arg(help = "The number of threads used to download")] + #[arg(short, long, default_value_t = num_cpus::get())] + pub(crate) threads: usize, + #[arg(help = "Crunchyroll series url(s)")] #[arg(required = true)] pub(crate) urls: Vec, @@ -158,7 +162,8 @@ impl Execute for Archive { .ffmpeg_preset(self.ffmpeg_preset.clone().unwrap_or_default()) .output_format(Some("matroska".to_string())) .audio_sort(Some(self.audio.clone())) - .subtitle_sort(Some(self.subtitle.clone())); + .subtitle_sort(Some(self.subtitle.clone())) + .threads(self.threads); for single_formats in single_format_collection.into_iter() { let (download_formats, mut format) = get_format(&self, &single_formats).await?; diff --git a/crunchy-cli-core/src/download/command.rs b/crunchy-cli-core/src/download/command.rs index baf27bf..760bf38 100644 --- a/crunchy-cli-core/src/download/command.rs +++ b/crunchy-cli-core/src/download/command.rs @@ -80,6 +80,10 @@ pub struct Download { #[arg(long, default_value_t = false)] pub(crate) force_hardsub: bool, + #[arg(help = "The number of threads used to download")] + #[arg(short, long, default_value_t = num_cpus::get())] + pub(crate) threads: usize, + #[arg(help = "Url(s) to Crunchyroll episodes or series")] #[arg(required = true)] pub(crate) urls: Vec, @@ -149,7 +153,8 @@ impl Execute for Download { } else { None }) - .ffmpeg_preset(self.ffmpeg_preset.clone().unwrap_or_default()); + .ffmpeg_preset(self.ffmpeg_preset.clone().unwrap_or_default()) + .threads(self.threads); for mut single_formats in single_format_collection.into_iter() { // the vec contains always only one item diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index 29f160a..ff9f240 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -50,6 +50,7 @@ pub struct DownloadBuilder { audio_sort: Option>, subtitle_sort: Option>, force_hardsub: bool, + threads: usize, } impl DownloadBuilder { @@ -61,6 +62,7 @@ impl DownloadBuilder { audio_sort: None, subtitle_sort: None, force_hardsub: false, + threads: num_cpus::get(), } } @@ -73,6 +75,7 @@ impl DownloadBuilder { subtitle_sort: self.subtitle_sort, force_hardsub: self.force_hardsub, + threads: self.threads, formats: vec![], } @@ -99,6 +102,7 @@ pub struct Downloader { subtitle_sort: Option>, force_hardsub: bool, + threads: usize, formats: Vec, } @@ -502,7 +506,8 @@ impl Downloader { let tempfile = tempfile(".mp4")?; let (mut file, path) = tempfile.into_parts(); - download_segments(ctx, &mut file, message, variant_data).await?; + self.download_segments(ctx, &mut file, message, variant_data) + .await?; Ok(path) } @@ -516,7 +521,8 @@ impl Downloader { let tempfile = tempfile(".m4a")?; let (mut file, path) = tempfile.into_parts(); - download_segments(ctx, &mut file, message, variant_data).await?; + self.download_segments(ctx, &mut file, message, variant_data) + .await?; Ok(path) } @@ -537,188 +543,189 @@ impl Downloader { Ok(path) } -} -pub async fn download_segments( - ctx: &Context, - writer: &mut impl Write, - message: String, - variant_data: &VariantData, -) -> Result<()> { - let segments = variant_data.segments().await?; - let total_segments = segments.len(); + async fn download_segments( + &self, + ctx: &Context, + writer: &mut impl Write, + message: String, + variant_data: &VariantData, + ) -> Result<()> { + let segments = variant_data.segments().await?; + let total_segments = segments.len(); - let client = Arc::new(ctx.crunchy.client()); - let count = Arc::new(Mutex::new(0)); + let client = Arc::new(ctx.crunchy.client()); + let count = Arc::new(Mutex::new(0)); - let progress = if log::max_level() == LevelFilter::Info { - let estimated_file_size = estimate_variant_file_size(variant_data, &segments); + let progress = if log::max_level() == LevelFilter::Info { + let estimated_file_size = estimate_variant_file_size(variant_data, &segments); - let progress = ProgressBar::new(estimated_file_size) - .with_style( - ProgressStyle::with_template( - ":: {msg} {bytes:>10} {bytes_per_sec:>12} [{wide_bar}] {percent:>3}%", + let progress = ProgressBar::new(estimated_file_size) + .with_style( + ProgressStyle::with_template( + ":: {msg} {bytes:>10} {bytes_per_sec:>12} [{wide_bar}] {percent:>3}%", + ) + .unwrap() + .progress_chars("##-"), ) - .unwrap() - .progress_chars("##-"), - ) - .with_message(message) - .with_finish(ProgressFinish::Abandon); - Some(progress) - } else { - None - }; + .with_message(message) + .with_finish(ProgressFinish::Abandon); + Some(progress) + } else { + None + }; - let cpus = num_cpus::get(); - let mut segs: Vec> = Vec::with_capacity(cpus); - for _ in 0..cpus { - segs.push(vec![]) - } - for (i, segment) in segments.clone().into_iter().enumerate() { - segs[i - ((i / cpus) * cpus)].push(segment); - } + let cpus = self.threads; + let mut segs: Vec> = Vec::with_capacity(cpus); + for _ in 0..cpus { + segs.push(vec![]) + } + for (i, segment) in segments.clone().into_iter().enumerate() { + segs[i - ((i / cpus) * cpus)].push(segment); + } - let (sender, mut receiver) = unbounded_channel(); + let (sender, mut receiver) = unbounded_channel(); - let mut join_set: JoinSet> = JoinSet::new(); - for num in 0..cpus { - let thread_client = client.clone(); - let thread_sender = sender.clone(); - let thread_segments = segs.remove(0); - let thread_count = count.clone(); - join_set.spawn(async move { - let after_download_sender = thread_sender.clone(); + let mut join_set: JoinSet> = JoinSet::new(); + for num in 0..cpus { + let thread_client = client.clone(); + let thread_sender = sender.clone(); + let thread_segments = segs.remove(0); + let thread_count = count.clone(); + join_set.spawn(async move { + let after_download_sender = thread_sender.clone(); - // the download process is encapsulated in its own function. this is done to easily - // catch errors which get returned with `...?` and `bail!(...)` and that the thread - // itself can report that an error has occurred - let download = || async move { - for (i, segment) in thread_segments.into_iter().enumerate() { - let mut retry_count = 0; - let mut buf = loop { - let request = thread_client - .get(&segment.url) - .timeout(Duration::from_secs(60)) - .send(); + // the download process is encapsulated in its own function. this is done to easily + // catch errors which get returned with `...?` and `bail!(...)` and that the thread + // itself can report that an error has occurred + let download = || async move { + for (i, segment) in thread_segments.into_iter().enumerate() { + let mut retry_count = 0; + let mut buf = loop { + let request = thread_client + .get(&segment.url) + .timeout(Duration::from_secs(60)) + .send(); - let response = match request.await { - Ok(r) => r, - Err(e) => { - if retry_count == 5 { - bail!("Max retry count reached ({}), multiple errors occurred while receiving segment {}: {}", retry_count, num + (i * cpus), e) - } - debug!("Failed to download segment {} ({}). Retrying, {} out of 5 retries left", num + (i * cpus), e, 5 - retry_count); - continue - } - }; - - match response.bytes().await { - Ok(b) => break b.to_vec(), - Err(e) => { - if e.is_body() { + let response = match request.await { + Ok(r) => r, + Err(e) => { if retry_count == 5 { bail!("Max retry count reached ({}), multiple errors occurred while receiving segment {}: {}", retry_count, num + (i * cpus), e) } - debug!("Failed to download segment {} ({}). Retrying, {} out of 5 retries left", num + (i * cpus), e, 5 - retry_count) - } else { - bail!("{}", e) + debug!("Failed to download segment {} ({}). Retrying, {} out of 5 retries left", num + (i * cpus), e, 5 - retry_count); + continue + } + }; + + match response.bytes().await { + Ok(b) => break b.to_vec(), + Err(e) => { + if e.is_body() { + if retry_count == 5 { + bail!("Max retry count reached ({}), multiple errors occurred while receiving segment {}: {}", retry_count, num + (i * cpus), e) + } + debug!("Failed to download segment {} ({}). Retrying, {} out of 5 retries left", num + (i * cpus), e, 5 - retry_count) + } else { + bail!("{}", e) + } } } - } - retry_count += 1; - }; + retry_count += 1; + }; - buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec(); + buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec(); - let mut c = thread_count.lock().await; - debug!( - "Downloaded and decrypted segment [{}/{} {:.2}%] {}", - num + (i * cpus) + 1, - total_segments, - ((*c + 1) as f64 / total_segments as f64) * 100f64, - segment.url - ); + let mut c = thread_count.lock().await; + debug!( + "Downloaded and decrypted segment [{}/{} {:.2}%] {}", + num + (i * cpus) + 1, + total_segments, + ((*c + 1) as f64 / total_segments as f64) * 100f64, + segment.url + ); - thread_sender.send((num as i32 + (i * cpus) as i32, buf))?; + thread_sender.send((num as i32 + (i * cpus) as i32, buf))?; - *c += 1; + *c += 1; + } + Ok(()) + }; + + + let result = download().await; + if result.is_err() { + after_download_sender.send((-1 as i32, vec![]))?; } - Ok(()) - }; + result + }); + } + // drop the sender already here so it does not outlive all download threads which are the only + // real consumers of it + drop(sender); - let result = download().await; - if result.is_err() { - after_download_sender.send((-1 as i32, vec![]))?; + // this is the main loop which writes the data. it uses a BTreeMap as a buffer as the write + // happens synchronized. the download consist of multiple segments. the map keys are representing + // the segment number and the values the corresponding bytes + let mut data_pos = 0; + let mut buf: BTreeMap> = BTreeMap::new(); + while let Some((pos, bytes)) = receiver.recv().await { + // if the position is lower than 0, an error occurred in the sending download thread + if pos < 0 { + break; } - result - }); - } - // drop the sender already here so it does not outlive all download threads which are the only - // real consumers of it - drop(sender); + if let Some(p) = &progress { + let progress_len = p.length().unwrap(); + let estimated_segment_len = (variant_data.bandwidth / 8) + * segments.get(pos as usize).unwrap().length.as_secs(); + let bytes_len = bytes.len() as u64; - // this is the main loop which writes the data. it uses a BTreeMap as a buffer as the write - // happens synchronized. the download consist of multiple segments. the map keys are representing - // the segment number and the values the corresponding bytes - let mut data_pos = 0; - let mut buf: BTreeMap> = BTreeMap::new(); - while let Some((pos, bytes)) = receiver.recv().await { - // if the position is lower than 0, an error occurred in the sending download thread - if pos < 0 { - break; + p.set_length(progress_len - estimated_segment_len + bytes_len); + p.inc(bytes_len) + } + + // check if the currently sent bytes are the next in the buffer. if so, write them directly + // to the target without first adding them to the buffer. + // if not, add them to the buffer + if data_pos == pos { + writer.write_all(bytes.borrow())?; + data_pos += 1; + } else { + buf.insert(pos, bytes); + } + // check if the buffer contains the next segment(s) + while let Some(b) = buf.remove(&data_pos) { + writer.write_all(b.borrow())?; + data_pos += 1; + } } - if let Some(p) = &progress { - let progress_len = p.length().unwrap(); - let estimated_segment_len = - (variant_data.bandwidth / 8) * segments.get(pos as usize).unwrap().length.as_secs(); - let bytes_len = bytes.len() as u64; - - p.set_length(progress_len - estimated_segment_len + bytes_len); - p.inc(bytes_len) + // if any error has occurred while downloading it gets returned here + while let Some(joined) = join_set.join_next().await { + joined?? } - // check if the currently sent bytes are the next in the buffer. if so, write them directly - // to the target without first adding them to the buffer. - // if not, add them to the buffer - if data_pos == pos { - writer.write_all(bytes.borrow())?; - data_pos += 1; - } else { - buf.insert(pos, bytes); - } - // check if the buffer contains the next segment(s) + // write the remaining buffer, if existent while let Some(b) = buf.remove(&data_pos) { writer.write_all(b.borrow())?; data_pos += 1; } - } - // if any error has occurred while downloading it gets returned here - while let Some(joined) = join_set.join_next().await { - joined?? - } + if !buf.is_empty() { + bail!( + "Download buffer is not empty. Remaining segments: {}", + buf.into_keys() + .map(|k| k.to_string()) + .collect::>() + .join(", ") + ) + } - // write the remaining buffer, if existent - while let Some(b) = buf.remove(&data_pos) { - writer.write_all(b.borrow())?; - data_pos += 1; + Ok(()) } - - if !buf.is_empty() { - bail!( - "Download buffer is not empty. Remaining segments: {}", - buf.into_keys() - .map(|k| k.to_string()) - .collect::>() - .join(", ") - ) - } - - Ok(()) } fn estimate_variant_file_size(variant_data: &VariantData, segments: &Vec) -> u64 { From 568bce00085cc301029d0b39c39cf15b97aa1d3d Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 15 Oct 2023 22:39:53 +0200 Subject: [PATCH 011/176] Manually implement filename sanitizing to allow the usage of file separators --- Cargo.lock | 11 ----- crunchy-cli-core/Cargo.toml | 1 - crunchy-cli-core/src/archive/command.rs | 2 +- crunchy-cli-core/src/download/command.rs | 2 +- crunchy-cli-core/src/utils/format.rs | 55 ++++++++++++------------ crunchy-cli-core/src/utils/os.rs | 48 +++++++++++++++++++++ 6 files changed, 77 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 09cad3e..409a005 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -401,7 +401,6 @@ dependencies = [ "regex", "reqwest", "rustls-native-certs", - "sanitize-filename", "serde", "serde_json", "serde_plain", @@ -1517,16 +1516,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" -[[package]] -name = "sanitize-filename" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ed72fbaf78e6f2d41744923916966c4fbe3d7c74e3037a8ee482f1115572603" -dependencies = [ - "lazy_static", - "regex", -] - [[package]] name = "schannel" version = "0.1.22" diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index fc298f3..e38cb1a 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -28,7 +28,6 @@ log = { version = "0.4", features = ["std"] } num_cpus = "1.16" regex = "1.9" reqwest = { version = "0.11", default-features = false, features = ["socks"] } -sanitize-filename = "0.5" serde = "1.0" serde_json = "1.0" serde_plain = "1.0" diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index bb4794f..3f455ed 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -173,7 +173,7 @@ impl Execute for Archive { downloader.add_format(download_format) } - let formatted_path = format.format_path((&self.output).into(), true); + let formatted_path = format.format_path((&self.output).into()); let (path, changed) = free_file(formatted_path.clone()); if changed && self.skip_existing { diff --git a/crunchy-cli-core/src/download/command.rs b/crunchy-cli-core/src/download/command.rs index 760bf38..9c9ecb4 100644 --- a/crunchy-cli-core/src/download/command.rs +++ b/crunchy-cli-core/src/download/command.rs @@ -165,7 +165,7 @@ impl Execute for Download { let mut downloader = download_builder.clone().build(); downloader.add_format(download_format); - let formatted_path = format.format_path((&self.output).into(), true); + let formatted_path = format.format_path((&self.output).into()); let (path, changed) = free_file(formatted_path.clone()); if changed && self.skip_existing { diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index 618f7d5..f462263 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -1,6 +1,6 @@ use crate::utils::filter::real_dedup_vec; use crate::utils::log::tab_info; -use crate::utils::os::is_special_file; +use crate::utils::os::{is_special_file, sanitize}; use anyhow::Result; use chrono::Duration; use crunchyroll_rs::media::{Resolution, Stream, Subtitle, VariantData}; @@ -368,46 +368,45 @@ impl Format { } } - /// Formats the given string if it has specific pattern in it. It's possible to sanitize it which - /// removes characters which can cause failures if the output string is used as a file name. - pub fn format_path(&self, path: PathBuf, sanitize: bool) -> PathBuf { - let path = path - .to_string_lossy() - .to_string() - .replace("{title}", &self.title) + /// Formats the given string if it has specific pattern in it. It also sanitizes the filename. + pub fn format_path(&self, path: PathBuf) -> PathBuf { + let mut path = sanitize(path.to_string_lossy(), false); + path = path + .replace("{title}", &sanitize(&self.title, true)) .replace( "{audio}", - &self - .locales - .iter() - .map(|(a, _)| a.to_string()) - .collect::>() - .join("|"), + &sanitize( + self.locales + .iter() + .map(|(a, _)| a.to_string()) + .collect::>() + .join("|"), + true, + ), ) - .replace("{resolution}", &self.resolution.to_string()) - .replace("{series_id}", &self.series_id) - .replace("{series_name}", &self.series_name) - .replace("{season_id}", &self.season_id) - .replace("{season_name}", &self.season_title) + .replace("{resolution}", &sanitize(self.resolution.to_string(), true)) + .replace("{series_id}", &sanitize(&self.series_id, true)) + .replace("{series_name}", &sanitize(&self.series_name, true)) + .replace("{season_id}", &sanitize(&self.season_id, true)) + .replace("{season_name}", &sanitize(&self.season_title, true)) .replace( "{season_number}", - &format!("{:0>2}", self.season_number.to_string()), + &format!("{:0>2}", sanitize(self.season_number.to_string(), true)), ) - .replace("{episode_id}", &self.episode_id) + .replace("{episode_id}", &sanitize(&self.episode_id, true)) .replace( "{episode_number}", - &format!("{:0>2}", self.episode_number.to_string()), + &format!("{:0>2}", sanitize(&self.episode_number, true)), ) .replace( "{relative_episode_number}", - &self.relative_episode_number.unwrap_or_default().to_string(), + &sanitize( + self.relative_episode_number.unwrap_or_default().to_string(), + true, + ), ); - if sanitize { - PathBuf::from(sanitize_filename::sanitize(path)) - } else { - PathBuf::from(path) - } + PathBuf::from(path) } pub fn visual_output(&self, dst: &Path) { diff --git a/crunchy-cli-core/src/utils/os.rs b/crunchy-cli-core/src/utils/os.rs index f6d9ad0..1f76b90 100644 --- a/crunchy-cli-core/src/utils/os.rs +++ b/crunchy-cli-core/src/utils/os.rs @@ -1,4 +1,6 @@ use log::debug; +use regex::{Regex, RegexBuilder}; +use std::borrow::Cow; use std::io::ErrorKind; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; @@ -78,3 +80,49 @@ pub fn free_file(mut path: PathBuf) -> (PathBuf, bool) { pub fn is_special_file>(path: P) -> bool { path.as_ref().exists() && !path.as_ref().is_file() && !path.as_ref().is_dir() } + +lazy_static::lazy_static! { + static ref ILLEGAL_RE: Regex = Regex::new(r#"[\?<>:\*\|":]"#).unwrap(); + static ref CONTROL_RE: Regex = Regex::new(r"[\x00-\x1f\x80-\x9f]").unwrap(); + static ref RESERVED_RE: Regex = Regex::new(r"^\.+$").unwrap(); + static ref WINDOWS_RESERVED_RE: Regex = RegexBuilder::new(r"(?i)^(con|prn|aux|nul|com[0-9]|lpt[0-9])(\..*)?$") + .case_insensitive(true) + .build() + .unwrap(); + static ref WINDOWS_TRAILING_RE: Regex = Regex::new(r"[\. ]+$").unwrap(); +} + +/// Sanitizes a filename with the option to include/exclude the path separator from sanitizing. This +/// is based of the implementation of the +/// [`sanitize-filename`](https://crates.io/crates/sanitize-filename) crate. +pub fn sanitize>(path: S, include_path_separator: bool) -> String { + let path = Cow::from(path.as_ref()); + + let path = ILLEGAL_RE.replace_all(&path, ""); + let path = CONTROL_RE.replace_all(&path, ""); + let path = RESERVED_RE.replace(&path, ""); + + let collect = |name: String| { + if name.len() > 255 { + name[..255].to_string() + } else { + name + } + }; + + if cfg!(windows) { + let path = WINDOWS_RESERVED_RE.replace(&path, ""); + let path = WINDOWS_TRAILING_RE.replace(&path, ""); + let mut path = path.to_string(); + if include_path_separator { + path = path.replace(['\\', '/'], ""); + } + collect(path) + } else { + let mut path = path.to_string(); + if include_path_separator { + path = path.replace('/', ""); + } + collect(path) + } +} From 685c79d673538aa981515806595cbba2da4decbf Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 15 Oct 2023 22:55:44 +0200 Subject: [PATCH 012/176] Add 2-digit padding to relative_episode_number, sequence_number and relative_sequence_number format option --- crunchy-cli-core/src/utils/format.rs | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index 1ba8136..4679878 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -408,22 +408,28 @@ impl Format { ) .replace( "{relative_episode_number}", - &sanitize( - self.relative_episode_number.unwrap_or_default().to_string(), - true, + &format!( + "{:0>2}", + sanitize( + self.relative_episode_number.unwrap_or_default().to_string(), + true, + ) ), ) .replace( "{sequence_number}", - &sanitize(self.sequence_number.to_string(), true), + &format!("{:0>2}", sanitize(self.sequence_number.to_string(), true)), ) .replace( "{relative_sequence_number}", - &sanitize( - self.relative_sequence_number - .unwrap_or_default() - .to_string(), - true, + &format!( + "{:0>2}", + sanitize( + self.relative_sequence_number + .unwrap_or_default() + .to_string(), + true, + ) ), ); From d0fe7f54f6c0e74f87d13a6d072599c12aad8367 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 15 Oct 2023 23:34:22 +0200 Subject: [PATCH 013/176] Show fractal in relative_sequence_number if present --- crunchy-cli-core/src/archive/filter.rs | 8 +++++--- crunchy-cli-core/src/download/filter.rs | 8 +++++--- crunchy-cli-core/src/utils/parse.rs | 10 ++++++++++ 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/crunchy-cli-core/src/archive/filter.rs b/crunchy-cli-core/src/archive/filter.rs index c2cc206..024ad17 100644 --- a/crunchy-cli-core/src/archive/filter.rs +++ b/crunchy-cli-core/src/archive/filter.rs @@ -2,7 +2,7 @@ use crate::archive::command::Archive; use crate::utils::filter::{real_dedup_vec, Filter}; use crate::utils::format::{Format, SingleFormat, SingleFormatCollection}; use crate::utils::interactive_select::{check_for_duplicated_seasons, get_duplicated_seasons}; -use crate::utils::parse::UrlFilter; +use crate::utils::parse::{fract, UrlFilter}; use anyhow::Result; use crunchyroll_rs::{Concert, Episode, Locale, Movie, MovieListing, MusicVideo, Season, Series}; use log::{info, warn}; @@ -321,8 +321,10 @@ impl Filter for ArchiveFilter { } if ep.id == episode.id { relative_episode_number = Some(i + 1); - relative_sequence_number = - Some((i + 1 - non_integer_sequence_number_count) as f32); + relative_sequence_number = Some( + (i + 1 - non_integer_sequence_number_count) as f32 + + fract(ep.sequence_number), + ); break; } } diff --git a/crunchy-cli-core/src/download/filter.rs b/crunchy-cli-core/src/download/filter.rs index c5aef7e..ff3c2ff 100644 --- a/crunchy-cli-core/src/download/filter.rs +++ b/crunchy-cli-core/src/download/filter.rs @@ -2,7 +2,7 @@ use crate::download::Download; use crate::utils::filter::Filter; use crate::utils::format::{Format, SingleFormat, SingleFormatCollection}; use crate::utils::interactive_select::{check_for_duplicated_seasons, get_duplicated_seasons}; -use crate::utils::parse::UrlFilter; +use crate::utils::parse::{fract, UrlFilter}; use anyhow::{bail, Result}; use crunchyroll_rs::{Concert, Episode, Movie, MovieListing, MusicVideo, Season, Series}; use log::{error, info, warn}; @@ -211,8 +211,10 @@ impl Filter for DownloadFilter { } if ep.id == episode.id { relative_episode_number = Some(i + 1); - relative_sequence_number = - Some((i + 1 - non_integer_sequence_number_count) as f32); + relative_sequence_number = Some( + (i + 1 - non_integer_sequence_number_count) as f32 + + fract(ep.sequence_number), + ); break; } } diff --git a/crunchy-cli-core/src/utils/parse.rs b/crunchy-cli-core/src/utils/parse.rs index f85d8c2..c0ac2b0 100644 --- a/crunchy-cli-core/src/utils/parse.rs +++ b/crunchy-cli-core/src/utils/parse.rs @@ -192,3 +192,13 @@ pub fn parse_resolution(mut resolution: String) -> Result { bail!("Could not find resolution") } } + +/// Dirty implementation of [`f32::fract`] with more accuracy. +pub fn fract(input: f32) -> f32 { + if input.fract() == 0.0 { + return 0.0; + } + format!("0.{}", input.to_string().split('.').last().unwrap()) + .parse::() + .unwrap() +} From 5a3a30444329548134ca998eebaa20e96eb88634 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 15 Oct 2023 23:52:44 +0200 Subject: [PATCH 014/176] Use episode sequence number as filter number for url episode filtering --- crunchy-cli-core/src/archive/filter.rs | 2 +- crunchy-cli-core/src/download/filter.rs | 4 ++-- crunchy-cli-core/src/search/filter.rs | 2 +- crunchy-cli-core/src/utils/parse.rs | 10 +++++----- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/crunchy-cli-core/src/archive/filter.rs b/crunchy-cli-core/src/archive/filter.rs index 024ad17..91023a3 100644 --- a/crunchy-cli-core/src/archive/filter.rs +++ b/crunchy-cli-core/src/archive/filter.rs @@ -241,7 +241,7 @@ impl Filter for ArchiveFilter { async fn visit_episode(&mut self, mut episode: Episode) -> Result> { if !self .url_filter - .is_episode_valid(episode.episode_number, episode.season_number) + .is_episode_valid(episode.sequence_number, episode.season_number) { return Ok(None); } diff --git a/crunchy-cli-core/src/download/filter.rs b/crunchy-cli-core/src/download/filter.rs index ff3c2ff..55b1e8b 100644 --- a/crunchy-cli-core/src/download/filter.rs +++ b/crunchy-cli-core/src/download/filter.rs @@ -118,7 +118,7 @@ impl Filter for DownloadFilter { episodes.retain(|e| { self.url_filter - .is_episode_valid(e.episode_number, season.season_number) + .is_episode_valid(e.sequence_number, season.season_number) }); Ok(episodes) @@ -127,7 +127,7 @@ impl Filter for DownloadFilter { async fn visit_episode(&mut self, mut episode: Episode) -> Result> { if !self .url_filter - .is_episode_valid(episode.episode_number, episode.season_number) + .is_episode_valid(episode.sequence_number, episode.season_number) { return Ok(None); } diff --git a/crunchy-cli-core/src/search/filter.rs b/crunchy-cli-core/src/search/filter.rs index 0b31823..264b31d 100644 --- a/crunchy-cli-core/src/search/filter.rs +++ b/crunchy-cli-core/src/search/filter.rs @@ -24,7 +24,7 @@ impl FilterOptions { self.check_audio_language(&vec![e.audio_locale.clone()]) && self .url_filter - .is_episode_valid(e.episode_number, e.season_number) + .is_episode_valid(e.sequence_number, e.season_number) }); episodes } diff --git a/crunchy-cli-core/src/utils/parse.rs b/crunchy-cli-core/src/utils/parse.rs index c0ac2b0..3fc7e7c 100644 --- a/crunchy-cli-core/src/utils/parse.rs +++ b/crunchy-cli-core/src/utils/parse.rs @@ -10,8 +10,8 @@ use regex::Regex; /// If `to_*` is [`None`] they're set to [`u32::MAX`]. #[derive(Debug, Default)] pub struct InnerUrlFilter { - from_episode: Option, - to_episode: Option, + from_episode: Option, + to_episode: Option, from_season: Option, to_season: Option, } @@ -39,10 +39,10 @@ impl UrlFilter { }) } - pub fn is_episode_valid(&self, episode: u32, season: u32) -> bool { + pub fn is_episode_valid(&self, episode: f32, season: u32) -> bool { self.inner.iter().any(|f| { - let from_episode = f.from_episode.unwrap_or(u32::MIN); - let to_episode = f.to_episode.unwrap_or(u32::MAX); + let from_episode = f.from_episode.unwrap_or(f32::MIN); + let to_episode = f.to_episode.unwrap_or(f32::MAX); let from_season = f.from_season.unwrap_or(u32::MIN); let to_season = f.to_season.unwrap_or(u32::MAX); From f56d9ecabf356ab9870c94f8ca5f3b46a2a39484 Mon Sep 17 00:00:00 2001 From: Catd <34575742+Nannk@users.noreply.github.com> Date: Mon, 16 Oct 2023 15:04:45 +0000 Subject: [PATCH 015/176] Changes in Readme regarding subtitles and flag usage (#255) * Update README.md updated Flags and subtitles sections * Update README.md * Update README.md Comma in a better place --- README.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 7a620cb..df285cc 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,8 @@ $ cargo install --force --path . > All shown commands are examples 🧑🏼‍🍳 +### Global Flags + crunchy-cli requires you to log in. Though you can use a non-premium account, you will not have access to premium content without a subscription. You can authenticate with your credentials (user:password) or by using a refresh token. @@ -99,7 +101,7 @@ You can authenticate with your credentials (user:password) or by using a refresh - Credentials ```shell - $ crunchy-cli --credentials "user:password" + $ crunchy-cli --credentials "user:password" ``` - Refresh Token @@ -107,13 +109,13 @@ You can authenticate with your credentials (user:password) or by using a refresh The easiest way to get it is via a browser extension which lets you export your cookies, like [Cookie-Editor](https://cookie-editor.cgagnier.ca/) ([Firefox](https://addons.mozilla.org/en-US/firefox/addon/cookie-editor/) / [Chrome](https://chrome.google.com/webstore/detail/cookie-editor/hlkenndednhfkekhgcdicdfddnkalmdm)). When installed, look for the `etp_rt` entry and extract its value. ```shell - $ crunchy-cli --etp-rt "4ebf1690-53a4-491a-a2ac-488309120f5d" + $ crunchy-cli --etp-rt "4ebf1690-53a4-491a-a2ac-488309120f5d" ``` - Stay Anonymous Login without an account (you won't be able to access premium content): ```shell - $ crunchy-cli --anonymous + $ crunchy-cli --anonymous ``` ### Global settings @@ -124,7 +126,7 @@ You can set specific settings which will be If you want to include debug information in the output, use the `-v` / `--verbose` flag to show it. ```shell - $ crunchy-cli -v + $ crunchy-cli -v ``` This flag can't be used with `-q` / `--quiet`. @@ -133,7 +135,7 @@ You can set specific settings which will be If you want to hide all output, use the `-q` / `--quiet` flag to do so. This is especially useful if you want to pipe the output video to an external program (like a video player). ```shell - $ crunchy-cli -q + $ crunchy-cli -q ``` - Language @@ -141,7 +143,7 @@ You can set specific settings which will be By default, the resulting metadata like title or description are shown in your system language (if Crunchyroll supports it, else in English). If you want to show the results in another language, use the `--lang` flag to set it. ```shell - $ crunchy-cli --lang de-DE + $ crunchy-cli --lang de-DE ``` - Experimental fixes @@ -150,7 +152,7 @@ You can set specific settings which will be The `--experimental-fixes` flag tries to fix some of those issues. As the *experimental* in `--experimental-fixes` states, these fixes may or may not break other functionality. ```shell - $ crunchy-cli --experimental-fixes + $ crunchy-cli --experimental-fixes ``` For an overview which parts this flag affects, see the [documentation](https://docs.rs/crunchyroll-rs/latest/crunchyroll_rs/crunchyroll/struct.CrunchyrollBuilder.html) of the underlying Crunchyroll library, all functions beginning with `stabilization_` are applied. @@ -159,7 +161,7 @@ You can set specific settings which will be The `--proxy` flag supports https and socks5 proxies to route all your traffic through. This may be helpful to bypass the geo-restrictions Crunchyroll has on certain series. ```shell - $ crunchy-cli --proxy socks5://127.0.0.1:8080 + $ crunchy-cli --proxy socks5://127.0.0.1:8080 ``` Make sure that proxy can either forward TLS requests, which is needed to bypass the (cloudflare) bot protection, or that it is configured so that the proxy can bypass the protection itself. @@ -204,7 +206,7 @@ The `download` command lets you download episodes with a specific audio language - Subtitle language Besides the audio, you can specify the subtitle language by using the `-s` / `--subtitle` flag. - The subtitles will be burned into the video track (cf. [hardsub](https://www.urbandictionary.com/define.php?term=hardsub)) and thus can not be turned off. + In formats that support it (.mp4, .mov and .mkv ), subtitles are stored as soft-subs. All other formats are hardsubbed: the subtitles will be burned into the video track (cf. [hardsub](https://www.urbandictionary.com/define.php?term=hardsub)) and thus can not be turned off. ```shell $ crunchy-cli download -s de-DE https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` From d8b76f8cc7985d5c838d6081985e686042051bd5 Mon Sep 17 00:00:00 2001 From: kennedy <854543+kennedy@users.noreply.github.com> Date: Sun, 29 Oct 2023 05:12:25 +0000 Subject: [PATCH 016/176] Add homebrew instructions (#261) Added details about homebrew and what archs are supported. made minor style linting: add space surrounding shell code blocks, and headers. --- README.md | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index df285cc..aa99c45 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ Check out the [releases](https://github.com/crunchy-labs/crunchy-cli/releases) t If you're using Arch or an Arch based Linux distribution you are able to install our [AUR](https://aur.archlinux.org/) package. You need an [AUR helper](https://wiki.archlinux.org/title/AUR_helpers) like [yay](https://github.com/Jguer/yay) to install it. + ```shell # this package builds crunchy-cli manually (recommended) $ yay -S crunchy-cli @@ -63,6 +64,7 @@ Check out the [releases](https://github.com/crunchy-labs/crunchy-cli/releases) t - [Nix](https://nixos.org/) This requires [nix](https://nixos.org) and you'll probably need `--extra-experimental-features "nix-command flakes"`, depending on your configurations. + ```shell $ nix github:crunchy-labs/crunchy-cli ``` @@ -70,15 +72,27 @@ Check out the [releases](https://github.com/crunchy-labs/crunchy-cli/releases) t - [Scoop](https://scoop.sh/) For Windows users, we support the [scoop](https://scoop.sh/#/) command-line installer. + ```shell $ scoop bucket add extras $ scoop install extras/crunchy-cli ``` +- [Homebrew](https://brew.sh/) + + For macOS/linux users, we support the [brew](https://brew.sh/#/) command-line installer. Packages are compile by the [homebrew project](https://github.com/Homebrew/homebrew-core/pkgs/container/core%2Fcrunchy-cli), and will also install the `openssl@3` and `ffmpeg` dependencies. + + ```shell + $ brew install crunchy-cli + ``` + + Supported archs: `x86_64_linux`, `arm64_monterey`, `sonoma`, `ventura` + ### 🛠 Build it yourself Since we do not support every platform and architecture you may have to build the project yourself. This requires [git](https://git-scm.com/) and [Cargo](https://doc.rust-lang.org/cargo). + ```shell $ git clone https://github.com/crunchy-labs/crunchy-cli $ cd crunchy-cli @@ -103,18 +117,22 @@ You can authenticate with your credentials (user:password) or by using a refresh ```shell $ crunchy-cli --credentials "user:password" ``` + - Refresh Token To obtain a refresh token, you have to log in at [crunchyroll.com](https://www.crunchyroll.com/) and extract the `etp_rt` cookie. The easiest way to get it is via a browser extension which lets you export your cookies, like [Cookie-Editor](https://cookie-editor.cgagnier.ca/) ([Firefox](https://addons.mozilla.org/en-US/firefox/addon/cookie-editor/) / [Chrome](https://chrome.google.com/webstore/detail/cookie-editor/hlkenndednhfkekhgcdicdfddnkalmdm)). When installed, look for the `etp_rt` entry and extract its value. + ```shell $ crunchy-cli --etp-rt "4ebf1690-53a4-491a-a2ac-488309120f5d" ``` + - Stay Anonymous Login without an account (you won't be able to access premium content): - ```shell + + ```shell $ crunchy-cli --anonymous ``` @@ -125,15 +143,18 @@ You can set specific settings which will be - Verbose output If you want to include debug information in the output, use the `-v` / `--verbose` flag to show it. + ```shell $ crunchy-cli -v ``` + This flag can't be used with `-q` / `--quiet`. - Quiet output If you want to hide all output, use the `-q` / `--quiet` flag to do so. This is especially useful if you want to pipe the output video to an external program (like a video player). + ```shell $ crunchy-cli -q ``` @@ -142,6 +163,7 @@ You can set specific settings which will be By default, the resulting metadata like title or description are shown in your system language (if Crunchyroll supports it, else in English). If you want to show the results in another language, use the `--lang` flag to set it. + ```shell $ crunchy-cli --lang de-DE ``` @@ -151,18 +173,22 @@ You can set specific settings which will be Crunchyroll constantly changes and breaks its services or just delivers incorrect answers. The `--experimental-fixes` flag tries to fix some of those issues. As the *experimental* in `--experimental-fixes` states, these fixes may or may not break other functionality. + ```shell $ crunchy-cli --experimental-fixes ``` + For an overview which parts this flag affects, see the [documentation](https://docs.rs/crunchyroll-rs/latest/crunchyroll_rs/crunchyroll/struct.CrunchyrollBuilder.html) of the underlying Crunchyroll library, all functions beginning with `stabilization_` are applied. - Proxy The `--proxy` flag supports https and socks5 proxies to route all your traffic through. This may be helpful to bypass the geo-restrictions Crunchyroll has on certain series. + ```shell $ crunchy-cli --proxy socks5://127.0.0.1:8080 ``` + Make sure that proxy can either forward TLS requests, which is needed to bypass the (cloudflare) bot protection, or that it is configured so that the proxy can bypass the protection itself. ### Login @@ -184,6 +210,7 @@ With the session stored, you do not need to pass `--credentials` / `--etp-rt` / The `download` command lets you download episodes with a specific audio language and optional subtitles. **Supported urls** + - Single episode (with [episode filtering](#episode-filtering)) ```shell $ crunchy-cli download https://www.crunchyroll.com/watch/GRDQPM1ZY/alone-and-lonesome @@ -194,38 +221,47 @@ The `download` command lets you download episodes with a specific audio language ``` **Options** + - Audio language Set the audio language with the `-a` / `--audio` flag. This only works if the url points to a series since episode urls are language specific. + ```shell $ crunchy-cli download -a de-DE https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` + Default is your system locale. If not supported by Crunchyroll, `en-US` (American English) is the default. - Subtitle language Besides the audio, you can specify the subtitle language by using the `-s` / `--subtitle` flag. In formats that support it (.mp4, .mov and .mkv ), subtitles are stored as soft-subs. All other formats are hardsubbed: the subtitles will be burned into the video track (cf. [hardsub](https://www.urbandictionary.com/define.php?term=hardsub)) and thus can not be turned off. + ```shell $ crunchy-cli download -s de-DE https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` + Default is none. - Output template Define an output template by using the `-o` / `--output` flag. + ```shell $ crunchy-cli download -o "ditf.mp4" https://www.crunchyroll.com/watch/GRDQPM1ZY/alone-and-lonesome ``` + Default is `{title}.mp4`. See the [Template Options section](#output-template-options) below for more options. - Resolution The resolution for videos can be set via the `-r` / `--resolution` flag. + ```shell $ crunchy-cli download -r worst https://www.crunchyroll.com/watch/GRDQPM1ZY/alone-and-lonesome ``` + Default is `best`. - FFmpeg Preset @@ -233,6 +269,7 @@ The `download` command lets you download episodes with a specific audio language You can specify specific built-in presets with the `--ffmpeg-preset` flag to convert videos to a specific coding while downloading. Multiple predefined presets how videos should be encoded (h264, h265, av1, ...) are available, you can see them with `crunchy-cli download --help`. If you need more specific ffmpeg customizations you could either convert the output file manually or use ffmpeg output arguments as value for this flag. + ```shell $ crunchy-cli downlaod --ffmpeg-preset av1-lossless https://www.crunchyroll.com/watch/GRDQPM1ZY/alone-and-lonesome ``` @@ -240,6 +277,7 @@ The `download` command lets you download episodes with a specific audio language - Skip existing If you re-download a series but want to skip episodes you've already downloaded, the `--skip-existing` flag skips the already existing/downloaded files. + ```shell $ crunchy-cli download --skip-existing https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` @@ -248,14 +286,17 @@ The `download` command lets you download episodes with a specific audio language Sometimes different seasons have the same season number (e.g. Sword Art Online Alicization and Alicization War of Underworld are both marked as season 3), in such cases an interactive prompt is shown which needs user further user input to decide which season to download. The `--yes` flag suppresses this interactive prompt and just downloads all seasons. + ```shell $ crunchy-cli download --yes https://www.crunchyroll.com/series/GR49G9VP6/sword-art-online ``` + If you've passed the `-q` / `--quiet` [global flag](#global-settings), this flag is automatically set. - Force hardsub If you want to burn-in the subtitles, even if the output format/container supports soft-subs (e.g. `.mp4`), use the `--force-hardsub` flag to do so. + ```shell $ crunchy-cli download --force-hardsub -s en-US https://www.crunchyroll.com/watch/GRDQPM1ZY/alone-and-lonesome ``` @@ -265,6 +306,7 @@ The `download` command lets you download episodes with a specific audio language The `archive` command lets you download episodes with multiple audios and subtitles and merges it into a `.mkv` file. **Supported urls** + - Single episode (with [episode filtering](#episode-filtering)) ```shell $ crunchy-cli archive https://www.crunchyroll.com/watch/GRDQPM1ZY/alone-and-lonesome @@ -275,37 +317,46 @@ The `archive` command lets you download episodes with multiple audios and subtit ``` **Options** + - Audio languages Set the audio language with the `-a` / `--audio` flag. Can be used multiple times. + ```shell $ crunchy-cli archive -a ja-JP -a de-DE https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` + Default is your system locale (if not supported by Crunchyroll, `en-US` (American English) and `ja-JP` (Japanese) are used). - Subtitle languages Besides the audio, you can specify the subtitle language by using the `-s` / `--subtitle` flag. + ```shell $ crunchy-cli archive -s de-DE https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` + Default is `all` subtitles. - Output template Define an output template by using the `-o` / `--output` flag. crunchy-cli uses the [`.mkv`](https://en.wikipedia.org/wiki/Matroska) container format, because of it's ability to store multiple audio, video and subtitle tracks at once. + ```shell $ crunchy-cli archive -o "{title}.mkv" https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` + Default is `{title}.mkv`. See the [Template Options section](#output-template-options) below for more options. - Resolution The resolution for videos can be set via the `-r` / `--resolution` flag. + ```shell $ crunchy-cli archive -r worst https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` + Default is `best`. - Merge behavior @@ -316,9 +367,11 @@ The `archive` command lets you download episodes with multiple audios and subtit With the `-m` / `--merge` flag you can define the behaviour when an episodes' video tracks differ in length. Valid options are `audio` - store one video and all other languages as audio only; `video` - store the video + audio for every language; `auto` - detect if videos differ in length: if so, behave like `video` - otherwise like `audio`. Subtitles will always match the primary audio and video. + ```shell $ crunchy-cli archive -m audio https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` + Default is `auto`. - FFmpeg Preset @@ -326,6 +379,7 @@ The `archive` command lets you download episodes with multiple audios and subtit You can specify specific built-in presets with the `--ffmpeg-preset` flag to convert videos to a specific coding while downloading. Multiple predefined presets how videos should be encoded (h264, h265, av1, ...) are available, you can see them with `crunchy-cli archive --help`. If you need more specific ffmpeg customizations you could either convert the output file manually or use ffmpeg output arguments as value for this flag. + ```shell $ crunchy-cli archive --ffmpeg-preset av1-lossless https://www.crunchyroll.com/watch/GRDQPM1ZY/alone-and-lonesome ``` @@ -333,14 +387,17 @@ The `archive` command lets you download episodes with multiple audios and subtit - Default subtitle `--default-subtitle` Set which subtitle language is to be flagged as **default** and **forced**. + ```shell $ crunchy-cli archive --default-subtitle en-US https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` + Default is none. - Skip existing If you re-download a series but want to skip episodes you've already downloaded, the `--skip-existing` flag skips the already existing/downloaded files. + ```shell $ crunchy-cli archive --skip-existing https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` @@ -349,14 +406,17 @@ The `archive` command lets you download episodes with multiple audios and subtit Sometimes different seasons have the same season number (e.g. Sword Art Online Alicization and Alicization War of Underworld are both marked as season 3), in such cases an interactive prompt is shown which needs user further user input to decide which season to download. The `--yes` flag suppresses this interactive prompt and just downloads all seasons. + ```shell $ crunchy-cli archive --yes https://www.crunchyroll.com/series/GR49G9VP6/sword-art-online ``` + If you've passed the `-q` / `--quiet` [global flag](#global-settings), this flag is automatically set. ### Search **Supported urls/input** + - Single episode (with [episode filtering](#episode-filtering)) ```shell $ crunchy-cli search https://www.crunchyroll.com/watch/GRDQPM1ZY/alone-and-lonesome @@ -371,12 +431,15 @@ The `archive` command lets you download episodes with multiple audios and subtit ``` **Options** + - Audio Set the audio language to search via the `--audio` flag. Can be used multiple times. + ```shell $ crunchy-cli search --audio en-US https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` + Default is your system locale. - Result limit @@ -384,6 +447,7 @@ The `archive` command lets you download episodes with multiple audios and subtit If your input is a search term instead of an url, you have multiple options to control which results to process. The `--search-top-results-limit` flag sets the limit of top search results to process. `--search-series-limit` sets the limit of only series, `--search-movie-listing-limit` of only movie listings, `--search-episode-limit` of only episodes and `--search-music-limit` of only concerts and music videos. + ```shell $ crunchy-cli search --search-top-results-limit 10 "darling in the franxx" # only return series which have 'darling' in it. do not return top results which might also be non-series items @@ -391,6 +455,7 @@ The `archive` command lets you download episodes with multiple audios and subtit # this returns 2 top results, 3 movie listings, 5 episodes and 1 music item as result $ crunchy-cli search --search-top-results-limit 2 --search-movie-listing-limit 3 --search-episode-limit 5 --search-music-limit 1 "test" ``` + Default is `5` for `--search-top-results-limit`, `0` for all others. - Output template @@ -401,9 +466,11 @@ The `archive` command lets you download episodes with multiple audios and subtit The required pattern for this begins with `{{`, then the keyword, and closes with `}}` (e.g. `{{episode.title}}`). For example, if you want to get the title of an episode, you can use `Title: {{episode.title}}` and `{{episode.title}}` will be replaced with the episode title. You can see all supported keywords with `crunchy-cli search --help`. + ```shell $ crunchy-cli search -o "{{series.title}}" https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` + Default is `S{{season.number}}E{{episode.number}} - {{episode.title}}`. --- @@ -425,6 +492,7 @@ You can use various template options to change how the filename is processed. Th - `{episode_id}` → ID of the episode Example: + ```shell $ crunchy-cli archive -o "[S{season_number}E{episode_number}] {title}.mkv" https://www.crunchyroll.com/series/G8DHV7W21/dragon-ball # Output file: '[S01E01] Secret of the Dragon Ball.mkv' @@ -438,6 +506,7 @@ A filter pattern may consist of either a season, an episode, or a combination of When used in combination, seasons `S` must be defined before episodes `E`. There are many possible patterns, for example: + - `...[E5]` - Download the fifth episode. - `...[S1]` - Download the whole first season. - `...[-S2]` - Download the first two seasons. @@ -447,6 +516,7 @@ There are many possible patterns, for example: - `...[S1-S3,S4E2-S4E6]` - Download season one to three, then episodes two to six from season four. In practice, it would look like this: + ``` https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx[E1-E5] ``` From 7594412f5845d0746df755b9536413432ed3a36a Mon Sep 17 00:00:00 2001 From: kennedy <854543+kennedy@users.noreply.github.com> Date: Thu, 2 Nov 2023 08:37:40 -0400 Subject: [PATCH 017/176] updated brew url (#263) * updated brew url Its most appropriate to forward users to the brew's information page generated for crunchy-cli. There are stats on amount of downloads, see where the manifest is location, and what architectures are built for it. * Update README.md Co-authored-by: ByteDream <63594396+ByteDream@users.noreply.github.com> --------- Co-authored-by: ByteDream <63594396+ByteDream@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aa99c45..3cbb8c9 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ Check out the [releases](https://github.com/crunchy-labs/crunchy-cli/releases) t - [Homebrew](https://brew.sh/) - For macOS/linux users, we support the [brew](https://brew.sh/#/) command-line installer. Packages are compile by the [homebrew project](https://github.com/Homebrew/homebrew-core/pkgs/container/core%2Fcrunchy-cli), and will also install the `openssl@3` and `ffmpeg` dependencies. + For macOS/linux users, we support the [brew](https://brew.sh/#/) command-line installer. Packages are compiled by the [homebrew project](https://formulae.brew.sh/formula/crunchy-cli), and will also install the `openssl@3` and `ffmpeg` dependencies. ```shell $ brew install crunchy-cli From 787d8ab02c206a763b2e5a81b413aee11bfe11b5 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sat, 4 Nov 2023 15:24:14 +0100 Subject: [PATCH 018/176] Add --special-output and --skip-specials flag --- crunchy-cli-core/src/archive/command.rs | 36 ++++++++++++++++--- crunchy-cli-core/src/archive/filter.rs | 16 ++++++++- crunchy-cli-core/src/download/command.rs | 44 +++++++++++++++++++++--- crunchy-cli-core/src/download/filter.rs | 16 ++++++++- crunchy-cli-core/src/utils/format.rs | 4 +++ 5 files changed, 106 insertions(+), 10 deletions(-) diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index 7d28e2e..f6da256 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -54,6 +54,11 @@ pub struct Archive { {episode_id} → ID of the episode")] #[arg(short, long, default_value = "{title}.mkv")] pub(crate) output: String, + #[arg(help = "Name of the output file if the episode is a special")] + #[arg(long_help = "Name of the output file if the episode is a special. \ + If not set, the '-o'/'--output' flag will be used as name template")] + #[arg(long)] + pub(crate) special_output: Option, #[arg(help = "Video resolution")] #[arg(long_help = "The video resolution.\ @@ -95,6 +100,9 @@ pub struct Archive { #[arg(help = "Skip files which are already existing")] #[arg(long, default_value_t = false)] pub(crate) skip_existing: bool, + #[arg(help = "Skip special episodes")] + #[arg(long, default_value_t = false)] + pub(crate) skip_specials: bool, #[arg(help = "Skip any interactive input")] #[arg(short, long, default_value_t = false)] @@ -123,6 +131,17 @@ impl Execute for Archive { && self.output != "-" { bail!("File extension is not '.mkv'. Currently only matroska / '.mkv' files are supported") + } else if let Some(special_output) = &self.special_output { + if PathBuf::from(special_output) + .extension() + .unwrap_or_default() + .to_string_lossy() + != "mkv" + && !is_special_file(special_output) + && special_output != "-" + { + bail!("File extension for special episodes is not '.mkv'. Currently only matroska / '.mkv' files are supported") + } } self.audio = all_locale_in_locales(self.audio.clone()); @@ -147,9 +166,10 @@ impl Execute for Archive { for (i, (media_collection, url_filter)) in parsed_urls.into_iter().enumerate() { let progress_handler = progress!("Fetching series details"); - let single_format_collection = ArchiveFilter::new(url_filter, self.clone(), !self.yes) - .visit(media_collection) - .await?; + let single_format_collection = + ArchiveFilter::new(url_filter, self.clone(), !self.yes, self.skip_specials) + .visit(media_collection) + .await?; if single_format_collection.is_empty() { progress_handler.stop(format!("Skipping url {} (no matching videos found)", i + 1)); @@ -175,7 +195,15 @@ impl Execute for Archive { downloader.add_format(download_format) } - let formatted_path = format.format_path((&self.output).into()); + let formatted_path = if format.is_special() { + format.format_path( + self.special_output + .as_ref() + .map_or((&self.output).into(), |so| so.into()), + ) + } else { + format.format_path((&self.output).into()) + }; let (path, changed) = free_file(formatted_path.clone()); if changed && self.skip_existing { diff --git a/crunchy-cli-core/src/archive/filter.rs b/crunchy-cli-core/src/archive/filter.rs index 91023a3..a4e3188 100644 --- a/crunchy-cli-core/src/archive/filter.rs +++ b/crunchy-cli-core/src/archive/filter.rs @@ -18,6 +18,7 @@ pub(crate) struct ArchiveFilter { url_filter: UrlFilter, archive: Archive, interactive_input: bool, + skip_special: bool, season_episodes: HashMap>, season_subtitles_missing: Vec, season_sorting: Vec, @@ -25,11 +26,17 @@ pub(crate) struct ArchiveFilter { } impl ArchiveFilter { - pub(crate) fn new(url_filter: UrlFilter, archive: Archive, interactive_input: bool) -> Self { + pub(crate) fn new( + url_filter: UrlFilter, + archive: Archive, + interactive_input: bool, + skip_special: bool, + ) -> Self { Self { url_filter, archive, interactive_input, + skip_special, season_episodes: HashMap::new(), season_subtitles_missing: vec![], season_sorting: vec![], @@ -246,6 +253,13 @@ impl Filter for ArchiveFilter { return Ok(None); } + // skip the episode if it's a special + if self.skip_special + && (episode.sequence_number == 0.0 || episode.sequence_number.fract() != 0.0) + { + return Ok(None); + } + let mut episodes = vec![]; if !matches!(self.visited, Visited::Series) && !matches!(self.visited, Visited::Season) { if self.archive.audio.contains(&episode.audio_locale) { diff --git a/crunchy-cli-core/src/download/command.rs b/crunchy-cli-core/src/download/command.rs index 376cc2d..da885c3 100644 --- a/crunchy-cli-core/src/download/command.rs +++ b/crunchy-cli-core/src/download/command.rs @@ -50,6 +50,11 @@ pub struct Download { {episode_id} → ID of the episode")] #[arg(short, long, default_value = "{title}.mp4")] pub(crate) output: String, + #[arg(help = "Name of the output file if the episode is a special")] + #[arg(long_help = "Name of the output file if the episode is a special. \ + If not set, the '-o'/'--output' flag will be used as name template")] + #[arg(long)] + pub(crate) special_output: Option, #[arg(help = "Video resolution")] #[arg(long_help = "The video resolution.\ @@ -73,6 +78,9 @@ pub struct Download { #[arg(help = "Skip files which are already existing")] #[arg(long, default_value_t = false)] pub(crate) skip_existing: bool, + #[arg(help = "Skip special episodes")] + #[arg(long, default_value_t = false)] + pub(crate) skip_specials: bool, #[arg(help = "Skip any interactive input")] #[arg(short, long, default_value_t = false)] @@ -116,6 +124,25 @@ impl Execute for Download { } } + if let Some(special_output) = &self.special_output { + if Path::new(special_output) + .extension() + .unwrap_or_default() + .is_empty() + && !is_special_file(special_output) + && special_output != "-" + { + bail!("No file extension found. Please specify a file extension (via `--special-output`) for the output file") + } + if let Some(ext) = Path::new(special_output).extension() { + if self.force_hardsub { + warn!("Hardsubs are forced for special episodes. Adding subtitles may take a while") + } else if !["mkv", "mov", "mp4"].contains(&ext.to_string_lossy().as_ref()) { + warn!("Detected a container which does not support softsubs. Adding subtitles for special episodes may take a while") + } + } + } + Ok(()) } @@ -135,9 +162,10 @@ impl Execute for Download { for (i, (media_collection, url_filter)) in parsed_urls.into_iter().enumerate() { let progress_handler = progress!("Fetching series details"); - let single_format_collection = DownloadFilter::new(url_filter, self.clone(), !self.yes) - .visit(media_collection) - .await?; + let single_format_collection = + DownloadFilter::new(url_filter, self.clone(), !self.yes, self.skip_specials) + .visit(media_collection) + .await?; if single_format_collection.is_empty() { progress_handler.stop(format!("Skipping url {} (no matching videos found)", i + 1)); @@ -167,7 +195,15 @@ impl Execute for Download { let mut downloader = download_builder.clone().build(); downloader.add_format(download_format); - let formatted_path = format.format_path((&self.output).into()); + let formatted_path = if format.is_special() { + format.format_path( + self.special_output + .as_ref() + .map_or((&self.output).into(), |so| so.into()), + ) + } else { + format.format_path((&self.output).into()) + }; let (path, changed) = free_file(formatted_path.clone()); if changed && self.skip_existing { diff --git a/crunchy-cli-core/src/download/filter.rs b/crunchy-cli-core/src/download/filter.rs index 55b1e8b..626896c 100644 --- a/crunchy-cli-core/src/download/filter.rs +++ b/crunchy-cli-core/src/download/filter.rs @@ -12,17 +12,24 @@ pub(crate) struct DownloadFilter { url_filter: UrlFilter, download: Download, interactive_input: bool, + skip_special: bool, season_episodes: HashMap>, season_subtitles_missing: Vec, season_visited: bool, } impl DownloadFilter { - pub(crate) fn new(url_filter: UrlFilter, download: Download, interactive_input: bool) -> Self { + pub(crate) fn new( + url_filter: UrlFilter, + download: Download, + interactive_input: bool, + skip_special: bool, + ) -> Self { Self { url_filter, download, interactive_input, + skip_special, season_episodes: HashMap::new(), season_subtitles_missing: vec![], season_visited: false, @@ -132,6 +139,13 @@ impl Filter for DownloadFilter { return Ok(None); } + // skip the episode if it's a special + if self.skip_special + && (episode.sequence_number == 0.0 || episode.sequence_number.fract() != 0.0) + { + return Ok(None); + } + // check if the audio locale is correct. // should only be incorrect if the console input was a episode url. otherwise // `DownloadFilter::visit_season` returns the correct episodes with matching audio diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index 4679878..4afaa07 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -473,6 +473,10 @@ impl Format { tab_info!("FPS: {:.2}", self.fps) } + pub fn is_special(&self) -> bool { + self.sequence_number == 0.0 || self.sequence_number.fract() != 0.0 + } + pub fn has_relative_fmt>(s: S) -> bool { return s.as_ref().contains("{relative_episode_number}") || s.as_ref().contains("{relative_sequence_number}"); From e5d9c27af7bf2056dfd96ca87bbe34506674a73f Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 6 Nov 2023 21:10:15 +0100 Subject: [PATCH 019/176] Fix ass filter path escape on windows (#262) --- crunchy-cli-core/src/utils/download.rs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index ff9f240..23d5863 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -378,8 +378,27 @@ impl Downloader { output_presets.extend([ "-vf".to_string(), format!( - "ass={}", - subtitles.get(position).unwrap().path.to_str().unwrap() + "ass='{}'", + // ffmpeg doesn't removes all ':' and '\' from the filename when using + // the ass filter. well, on windows these characters are used in + // absolute paths, so they have to be correctly escaped here + if cfg!(windows) { + subtitles + .get(position) + .unwrap() + .path + .to_str() + .unwrap() + .replace('\\', "\\\\") + .replace(':', "\\:") + } else { + subtitles + .get(position) + .unwrap() + .path + .to_string_lossy() + .to_string() + } ), ]) } From f31437fba2bc675fbfeb431f30b01dac5463708f Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 6 Nov 2023 21:20:43 +0100 Subject: [PATCH 020/176] Remove leading and trailing whitespaces from output file --- crunchy-cli-core/src/utils/os.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/utils/os.rs b/crunchy-cli-core/src/utils/os.rs index 1f76b90..0596789 100644 --- a/crunchy-cli-core/src/utils/os.rs +++ b/crunchy-cli-core/src/utils/os.rs @@ -96,7 +96,7 @@ lazy_static::lazy_static! { /// is based of the implementation of the /// [`sanitize-filename`](https://crates.io/crates/sanitize-filename) crate. pub fn sanitize>(path: S, include_path_separator: bool) -> String { - let path = Cow::from(path.as_ref()); + let path = Cow::from(path.as_ref().trim()); let path = ILLEGAL_RE.replace_all(&path, ""); let path = CONTROL_RE.replace_all(&path, ""); From cd35dfe2766f6448ea43747dcfe1c24c2f599c51 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 6 Nov 2023 21:48:49 +0100 Subject: [PATCH 021/176] Rename --special-output to --output-specials --- crunchy-cli-core/src/archive/command.rs | 6 +++--- crunchy-cli-core/src/download/command.rs | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index f6da256..0ee723e 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -58,7 +58,7 @@ pub struct Archive { #[arg(long_help = "Name of the output file if the episode is a special. \ If not set, the '-o'/'--output' flag will be used as name template")] #[arg(long)] - pub(crate) special_output: Option, + pub(crate) output_specials: Option, #[arg(help = "Video resolution")] #[arg(long_help = "The video resolution.\ @@ -131,7 +131,7 @@ impl Execute for Archive { && self.output != "-" { bail!("File extension is not '.mkv'. Currently only matroska / '.mkv' files are supported") - } else if let Some(special_output) = &self.special_output { + } else if let Some(special_output) = &self.output_specials { if PathBuf::from(special_output) .extension() .unwrap_or_default() @@ -197,7 +197,7 @@ impl Execute for Archive { let formatted_path = if format.is_special() { format.format_path( - self.special_output + self.output_specials .as_ref() .map_or((&self.output).into(), |so| so.into()), ) diff --git a/crunchy-cli-core/src/download/command.rs b/crunchy-cli-core/src/download/command.rs index da885c3..18355cd 100644 --- a/crunchy-cli-core/src/download/command.rs +++ b/crunchy-cli-core/src/download/command.rs @@ -54,7 +54,7 @@ pub struct Download { #[arg(long_help = "Name of the output file if the episode is a special. \ If not set, the '-o'/'--output' flag will be used as name template")] #[arg(long)] - pub(crate) special_output: Option, + pub(crate) output_specials: Option, #[arg(help = "Video resolution")] #[arg(long_help = "The video resolution.\ @@ -124,7 +124,7 @@ impl Execute for Download { } } - if let Some(special_output) = &self.special_output { + if let Some(special_output) = &self.output_specials { if Path::new(special_output) .extension() .unwrap_or_default() @@ -132,7 +132,7 @@ impl Execute for Download { && !is_special_file(special_output) && special_output != "-" { - bail!("No file extension found. Please specify a file extension (via `--special-output`) for the output file") + bail!("No file extension found. Please specify a file extension (via `--output-specials`) for the output file") } if let Some(ext) = Path::new(special_output).extension() { if self.force_hardsub { @@ -197,7 +197,7 @@ impl Execute for Download { let formatted_path = if format.is_special() { format.format_path( - self.special_output + self.output_specials .as_ref() .map_or((&self.output).into(), |so| so.into()), ) From 56411c6547223e59c8cb4e8c61d6a29627889537 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 6 Nov 2023 22:01:44 +0100 Subject: [PATCH 022/176] Add missing whitespaces in command help --- crunchy-cli-core/src/archive/command.rs | 4 ++-- crunchy-cli-core/src/download/command.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index 0ee723e..7add620 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -37,7 +37,7 @@ pub struct Archive { pub(crate) subtitle: Vec, #[arg(help = "Name of the output file")] - #[arg(long_help = "Name of the output file.\ + #[arg(long_help = "Name of the output file. \ If you use one of the following pattern they will get replaced:\n \ {title} → Title of the video\n \ {series_name} → Name of the series\n \ @@ -61,7 +61,7 @@ pub struct Archive { pub(crate) output_specials: Option, #[arg(help = "Video resolution")] - #[arg(long_help = "The video resolution.\ + #[arg(long_help = "The video resolution. \ Can either be specified via the pixels (e.g. 1920x1080), the abbreviation for pixels (e.g. 1080p) or 'common-use' words (e.g. best). \ Specifying the exact pixels is not recommended, use one of the other options instead. \ Crunchyroll let you choose the quality with pixel abbreviation on their clients, so you might be already familiar with the available options. \ diff --git a/crunchy-cli-core/src/download/command.rs b/crunchy-cli-core/src/download/command.rs index 18355cd..4f189c5 100644 --- a/crunchy-cli-core/src/download/command.rs +++ b/crunchy-cli-core/src/download/command.rs @@ -33,7 +33,7 @@ pub struct Download { pub(crate) subtitle: Option, #[arg(help = "Name of the output file")] - #[arg(long_help = "Name of the output file.\ + #[arg(long_help = "Name of the output file. \ If you use one of the following pattern they will get replaced:\n \ {title} → Title of the video\n \ {series_name} → Name of the series\n \ From fc6511a361b7ee98830385bf273cd2ce464b1058 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 6 Nov 2023 22:09:21 +0100 Subject: [PATCH 023/176] Format code --- crunchy-cli-core/src/archive/command.rs | 4 ++-- crunchy-cli-core/src/archive/filter.rs | 16 +++++++--------- crunchy-cli-core/src/download/command.rs | 2 +- crunchy-cli-core/src/download/filter.rs | 19 +++++++++---------- crunchy-cli-core/src/lib.rs | 4 ++-- crunchy-cli-core/src/search/filter.rs | 4 ++-- crunchy-cli-core/src/search/format.rs | 1 + crunchy-cli-core/src/utils/download.rs | 13 ++++++------- crunchy-cli-core/src/utils/ffmpeg.rs | 6 +++--- crunchy-cli-core/src/utils/format.rs | 15 +++++++-------- crunchy-cli-core/src/utils/locale.rs | 3 +-- crunchy-cli-core/src/utils/log.rs | 1 - crunchy-cli-core/src/utils/os.rs | 2 +- 13 files changed, 42 insertions(+), 48 deletions(-) diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index 7add620..3585e9c 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -306,8 +306,8 @@ async fn get_format( } MergeBehavior::Audio => download_formats.push(DownloadFormat { video: ( - (*format_pairs.first().unwrap()).1.clone(), - (*format_pairs.first().unwrap()).0.audio.clone(), + format_pairs.first().unwrap().1.clone(), + format_pairs.first().unwrap().0.audio.clone(), ), audios: format_pairs .iter() diff --git a/crunchy-cli-core/src/archive/filter.rs b/crunchy-cli-core/src/archive/filter.rs index a4e3188..01612b9 100644 --- a/crunchy-cli-core/src/archive/filter.rs +++ b/crunchy-cli-core/src/archive/filter.rs @@ -103,7 +103,7 @@ impl Filter for ArchiveFilter { seasons.retain(|s| !remove_ids.contains(&s.id)); let duplicated_seasons = get_duplicated_seasons(&seasons); - if duplicated_seasons.len() > 0 { + if !duplicated_seasons.is_empty() { if self.interactive_input { check_for_duplicated_seasons(&mut seasons); } else { @@ -139,8 +139,7 @@ impl Filter for ArchiveFilter { if !matches!(self.visited, Visited::Series) { let mut audio_locales: Vec = seasons .iter() - .map(|s| s.audio_locales.clone()) - .flatten() + .flat_map(|s| s.audio_locales.clone()) .collect(); real_dedup_vec(&mut audio_locales); let missing_audio = missing_locales(&audio_locales, &self.archive.audio); @@ -158,8 +157,7 @@ impl Filter for ArchiveFilter { let subtitle_locales: Vec = seasons .iter() - .map(|s| s.subtitle_locales.clone()) - .flatten() + .flat_map(|s| s.subtitle_locales.clone()) .collect(); let missing_subtitle = missing_locales(&subtitle_locales, &self.archive.subtitle); if !missing_subtitle.is_empty() { @@ -211,7 +209,7 @@ impl Filter for ArchiveFilter { } } if eps.len() < before_len { - if eps.len() == 0 { + if eps.is_empty() { if matches!(self.visited, Visited::Series) { warn!( "Season {} is not available with {} audio", @@ -237,7 +235,7 @@ impl Filter for ArchiveFilter { for episode in episodes.iter() { self.season_episodes .entry(episode.season_id.clone()) - .or_insert(vec![]) + .or_default() .push(episode.clone()) } } @@ -290,7 +288,7 @@ impl Filter for ArchiveFilter { } let mut subtitle_locales: Vec = - episodes.iter().map(|(_, s)| s.clone()).flatten().collect(); + episodes.iter().flat_map(|(_, s)| s.clone()).collect(); real_dedup_vec(&mut subtitle_locales); let missing_subtitles = missing_locales(&subtitle_locales, &self.archive.subtitle); if !missing_subtitles.is_empty() @@ -435,6 +433,6 @@ impl Filter for ArchiveFilter { } } -fn missing_locales<'a>(available: &Vec, searched: &'a Vec) -> Vec<&'a Locale> { +fn missing_locales<'a>(available: &[Locale], searched: &'a [Locale]) -> Vec<&'a Locale> { searched.iter().filter(|p| !available.contains(p)).collect() } diff --git a/crunchy-cli-core/src/download/command.rs b/crunchy-cli-core/src/download/command.rs index 4f189c5..82cb8f8 100644 --- a/crunchy-cli-core/src/download/command.rs +++ b/crunchy-cli-core/src/download/command.rs @@ -251,7 +251,7 @@ async fn get_format( }; let subtitle = if let Some(subtitle_locale) = &download.subtitle { - stream.subtitles.get(subtitle_locale).map(|s| s.clone()) + stream.subtitles.get(subtitle_locale).cloned() } else { None }; diff --git a/crunchy-cli-core/src/download/filter.rs b/crunchy-cli-core/src/download/filter.rs index 626896c..fb2e563 100644 --- a/crunchy-cli-core/src/download/filter.rs +++ b/crunchy-cli-core/src/download/filter.rs @@ -45,14 +45,13 @@ impl Filter for DownloadFilter { async fn visit_series(&mut self, series: Series) -> Result> { // `series.audio_locales` isn't always populated b/c of crunchyrolls api. so check if the // audio is matching only if the field is populated - if !series.audio_locales.is_empty() { - if !series.audio_locales.contains(&self.download.audio) { - error!( - "Series {} is not available with {} audio", - series.title, self.download.audio - ); - return Ok(vec![]); - } + if !series.audio_locales.is_empty() && !series.audio_locales.contains(&self.download.audio) + { + error!( + "Series {} is not available with {} audio", + series.title, self.download.audio + ); + return Ok(vec![]); } let mut seasons = vec![]; @@ -91,7 +90,7 @@ impl Filter for DownloadFilter { } let duplicated_seasons = get_duplicated_seasons(&seasons); - if duplicated_seasons.len() > 0 { + if !duplicated_seasons.is_empty() { if self.interactive_input { check_for_duplicated_seasons(&mut seasons); } else { @@ -118,7 +117,7 @@ impl Filter for DownloadFilter { for episode in episodes.iter() { self.season_episodes .entry(episode.season_number) - .or_insert(vec![]) + .or_default() .push(episode.clone()) } } diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index 02bef6f..d6c2220 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -77,7 +77,7 @@ fn version() -> String { let build_date = env!("BUILD_DATE"); if git_commit_hash.is_empty() { - format!("{}", package_version) + package_version.to_string() } else { format!("{} ({} {})", package_version, git_commit_hash, build_date) } @@ -250,7 +250,7 @@ async fn crunchyroll_session(cli: &mut Cli) -> Result { "Via `--lang` specified language is not supported. Supported languages: {}", supported_langs .iter() - .map(|l| format!("`{}` ({})", l.to_string(), l.to_human_readable())) + .map(|l| format!("`{}` ({})", l, l.to_human_readable())) .collect::>() .join(", ") ) diff --git a/crunchy-cli-core/src/search/filter.rs b/crunchy-cli-core/src/search/filter.rs index 264b31d..3bb6d9f 100644 --- a/crunchy-cli-core/src/search/filter.rs +++ b/crunchy-cli-core/src/search/filter.rs @@ -21,7 +21,7 @@ impl FilterOptions { pub fn filter_episodes(&self, mut episodes: Vec) -> Vec { episodes.retain(|e| { - self.check_audio_language(&vec![e.audio_locale.clone()]) + self.check_audio_language(&[e.audio_locale.clone()]) && self .url_filter .is_episode_valid(e.sequence_number, e.season_number) @@ -38,7 +38,7 @@ impl FilterOptions { ) } - fn check_audio_language(&self, audio: &Vec) -> bool { + fn check_audio_language(&self, audio: &[Locale]) -> bool { if !self.audio.is_empty() { return self.audio.iter().any(|a| audio.contains(a)); } diff --git a/crunchy-cli-core/src/search/format.rs b/crunchy-cli-core/src/search/format.rs index 55cba7c..f9746b1 100644 --- a/crunchy-cli-core/src/search/format.rs +++ b/crunchy-cli-core/src/search/format.rs @@ -372,6 +372,7 @@ impl Format { let stream_empty = self.check_pattern_count_empty(Scope::Stream) && self.check_pattern_count_empty(Scope::Subtitle); + #[allow(clippy::type_complexity)] let mut tree: Vec<(Season, Vec<(Episode, Vec)>)> = vec![]; let series = if !series_empty { diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index 23d5863..945cc14 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -296,7 +296,7 @@ impl Downloader { ]); // the empty language metadata is created to avoid that metadata from the original track // is copied - metadata.extend([format!("-metadata:s:v:{}", i), format!("language=")]) + metadata.extend([format!("-metadata:s:v:{}", i), "language=".to_string()]) } for (i, meta) in audios.iter().enumerate() { input.extend(["-i".to_string(), meta.path.to_string_lossy().to_string()]); @@ -675,7 +675,7 @@ impl Downloader { let result = download().await; if result.is_err() { - after_download_sender.send((-1 as i32, vec![]))?; + after_download_sender.send((-1, vec![]))?; } result @@ -747,7 +747,7 @@ impl Downloader { } } -fn estimate_variant_file_size(variant_data: &VariantData, segments: &Vec) -> u64 { +fn estimate_variant_file_size(variant_data: &VariantData, segments: &[VariantSegment]) -> u64 { (variant_data.bandwidth / 8) * segments.iter().map(|s| s.length.as_secs()).sum::() } @@ -788,9 +788,8 @@ pub fn get_video_length(path: &Path) -> Result { /// [crunchy-labs/crunchy-cli#208](https://github.com/crunchy-labs/crunchy-cli/issues/208) for more /// information. fn fix_subtitles(raw: &mut Vec, max_length: NaiveTime) { - let re = - Regex::new(r#"^Dialogue:\s\d+,(?P\d+:\d+:\d+\.\d+),(?P\d+:\d+:\d+\.\d+),"#) - .unwrap(); + let re = Regex::new(r"^Dialogue:\s\d+,(?P\d+:\d+:\d+\.\d+),(?P\d+:\d+:\d+\.\d+),") + .unwrap(); // chrono panics if we try to format NaiveTime with `%2f` and the nano seconds has more than 2 // digits so them have to be reduced manually to avoid the panic @@ -832,7 +831,7 @@ fn fix_subtitles(raw: &mut Vec, max_length: NaiveTime) { line, format!( "Dialogue: {},{},", - format_naive_time(start.clone()), + format_naive_time(start), &length_as_string ), ) diff --git a/crunchy-cli-core/src/utils/ffmpeg.rs b/crunchy-cli-core/src/utils/ffmpeg.rs index af29bfd..787c79a 100644 --- a/crunchy-cli-core/src/utils/ffmpeg.rs +++ b/crunchy-cli-core/src/utils/ffmpeg.rs @@ -134,7 +134,7 @@ impl FFmpegPreset { description_details.push(format!("{} video quality/compression", q.to_string())) } - let description = if description_details.len() == 0 { + let description = if description_details.is_empty() { format!( "{} encoded with default video quality/compression", codec.to_string() @@ -239,7 +239,7 @@ impl FFmpegPreset { hwaccel.clone(), quality.clone(), )) { - return Err(format!("ffmpeg preset is not supported")); + return Err("ffmpeg preset is not supported".to_string()); } Ok(FFmpegPreset::Predefined( c, @@ -247,7 +247,7 @@ impl FFmpegPreset { quality.unwrap_or(FFmpegQuality::Normal), )) } else { - Err(format!("cannot use ffmpeg preset with without a codec")) + Err("cannot use ffmpeg preset with without a codec".to_string()) } } diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index 4afaa07..4c8d3c8 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -170,10 +170,7 @@ impl SingleFormat { } pub fn is_episode(&self) -> bool { - match self.source { - MediaCollection::Episode(_) => true, - _ => false, - } + matches!(self.source, MediaCollection::Episode(_)) } } @@ -181,7 +178,7 @@ struct SingleFormatCollectionEpisodeKey(f32); impl PartialOrd for SingleFormatCollectionEpisodeKey { fn partial_cmp(&self, other: &Self) -> Option { - self.0.partial_cmp(&other.0) + Some(self.cmp(other)) } } impl Ord for SingleFormatCollectionEpisodeKey { @@ -198,6 +195,7 @@ impl Eq for SingleFormatCollectionEpisodeKey {} struct SingleFormatCollectionSeasonKey((u32, String)); +#[allow(clippy::incorrect_partial_ord_impl_on_ord_type)] impl PartialOrd for SingleFormatCollectionSeasonKey { fn partial_cmp(&self, other: &Self) -> Option { let mut cmp = self.0 .0.partial_cmp(&other.0 .0); @@ -250,7 +248,7 @@ impl SingleFormatCollection { format.season_number, format.season_id.clone(), ))) - .or_insert(BTreeMap::new()) + .or_default() .insert( SingleFormatCollectionEpisodeKey(format.sequence_number), single_formats, @@ -340,6 +338,7 @@ pub struct Format { } impl Format { + #[allow(clippy::type_complexity)] pub fn from_single_formats( mut single_formats: Vec<(SingleFormat, VariantData, Vec<(Subtitle, bool)>)>, ) -> Self { @@ -349,7 +348,7 @@ impl Format { ( single_format.audio.clone(), subtitles - .into_iter() + .iter() .map(|(s, _)| s.locale.clone()) .collect::>(), ) @@ -440,7 +439,7 @@ impl Format { info!( "Downloading {} to {}", self.title, - if is_special_file(&dst) || dst.to_str().unwrap() == "-" { + if is_special_file(dst) || dst.to_str().unwrap() == "-" { dst.to_string_lossy().to_string() } else { format!("'{}'", dst.to_str().unwrap()) diff --git a/crunchy-cli-core/src/utils/locale.rs b/crunchy-cli-core/src/utils/locale.rs index d749fcb..8651078 100644 --- a/crunchy-cli-core/src/utils/locale.rs +++ b/crunchy-cli-core/src/utils/locale.rs @@ -19,8 +19,7 @@ pub fn system_locale() -> Locale { pub fn all_locale_in_locales(locales: Vec) -> Vec { if locales .iter() - .find(|l| l.to_string().to_lowercase().trim() == "all") - .is_some() + .any(|l| l.to_string().to_lowercase().trim() == "all") { Locale::all() } else { diff --git a/crunchy-cli-core/src/utils/log.rs b/crunchy-cli-core/src/utils/log.rs index 6650e58..942c652 100644 --- a/crunchy-cli-core/src/utils/log.rs +++ b/crunchy-cli-core/src/utils/log.rs @@ -57,7 +57,6 @@ macro_rules! tab_info { } pub(crate) use tab_info; -#[allow(clippy::type_complexity)] pub struct CliLogger { level: LevelFilter, progress: Mutex>, diff --git a/crunchy-cli-core/src/utils/os.rs b/crunchy-cli-core/src/utils/os.rs index 0596789..977e968 100644 --- a/crunchy-cli-core/src/utils/os.rs +++ b/crunchy-cli-core/src/utils/os.rs @@ -24,7 +24,7 @@ pub fn has_ffmpeg() -> bool { /// Get the temp directory either by the specified `CRUNCHY_CLI_TEMP_DIR` env variable or the dir /// provided by the os. pub fn temp_directory() -> PathBuf { - env::var("CRUNCHY_CLI_TEMP_DIR").map_or(env::temp_dir(), |d| PathBuf::from(d)) + env::var("CRUNCHY_CLI_TEMP_DIR").map_or(env::temp_dir(), PathBuf::from) } /// Any tempfile should be created with this function. The prefix and directory of every file From c08931b6105d3a6756862d2548e5a428c3f93272 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 6 Nov 2023 22:47:09 +0100 Subject: [PATCH 024/176] Add new commands and format option to readme --- README.md | 112 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 92 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 3cbb8c9..2491939 100644 --- a/README.md +++ b/README.md @@ -61,14 +61,6 @@ Check out the [releases](https://github.com/crunchy-labs/crunchy-cli/releases) t $ yay -S crunchy-cli-bin ``` -- [Nix](https://nixos.org/) - - This requires [nix](https://nixos.org) and you'll probably need `--extra-experimental-features "nix-command flakes"`, depending on your configurations. - - ```shell - $ nix github:crunchy-labs/crunchy-cli - ``` - - [Scoop](https://scoop.sh/) For Windows users, we support the [scoop](https://scoop.sh/#/) command-line installer. @@ -88,6 +80,14 @@ Check out the [releases](https://github.com/crunchy-labs/crunchy-cli/releases) t Supported archs: `x86_64_linux`, `arm64_monterey`, `sonoma`, `ventura` +- [Nix](https://nixos.org/) + + This requires [nix](https://nixos.org) and you'll probably need `--extra-experimental-features "nix-command flakes"`, depending on your configurations. + + ```shell + $ nix github:crunchy-labs/crunchy-cli + ``` + ### 🛠 Build it yourself Since we do not support every platform and architecture you may have to build the project yourself. @@ -191,6 +191,17 @@ You can set specific settings which will be Make sure that proxy can either forward TLS requests, which is needed to bypass the (cloudflare) bot protection, or that it is configured so that the proxy can bypass the protection itself. +- User Agent + + There might be cases where a custom user agent is necessary, e.g. to bypass the cloudflare bot protection (#104). + In such cases, the `--user-agent` flag can be used to set a custom user agent. + + ```shell + $ crunchy-cli --user-agent "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)" + ``` + + Default is the user agent, defined in the underlying [library](https://github.com/crunchy-labs/crunchyroll-rs). + ### Login The `login` command can store your session, so you don't have to authenticate every time you execute a command. @@ -254,6 +265,16 @@ The `download` command lets you download episodes with a specific audio language Default is `{title}.mp4`. See the [Template Options section](#output-template-options) below for more options. +- Output template for special episodes + + Define an output template which only gets used when the episode is a special (episode number is 0 or has non-zero decimal places) by using the `--output-special` flag. + + ```shell + $ crunchy-cli download --output-specials -o "Special EP: {title}" https://www.crunchyroll.com/watch/GY8D975JY/veldoras-journal + ``` + + Default is the template, set by the `-o` / `--output` flag. See the [Template Options section](#output-template-options) below for more options. + - Resolution The resolution for videos can be set via the `-r` / `--resolution` flag. @@ -282,6 +303,14 @@ The `download` command lets you download episodes with a specific audio language $ crunchy-cli download --skip-existing https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` +- Skip specials + + If you doesn't want to download special episodes, use the `--skip-specials` flag to skip the download of them. + + ```shell + $ crunchy-cli download --skip-specials https://www.crunchyroll.com/series/GYZJ43JMR/that-time-i-got-reincarnated-as-a-slime[S2] + ``` + - Yes Sometimes different seasons have the same season number (e.g. Sword Art Online Alicization and Alicization War of Underworld are both marked as season 3), in such cases an interactive prompt is shown which needs user further user input to decide which season to download. @@ -301,6 +330,17 @@ The `download` command lets you download episodes with a specific audio language $ crunchy-cli download --force-hardsub -s en-US https://www.crunchyroll.com/watch/GRDQPM1ZY/alone-and-lonesome ``` +- Threads + + To increase the download speed, video segments are downloaded simultaneously by creating multiple threads. + If you want to manually specify how many threads to use when downloading, do this with the `-t` / `--threads` flag. + + ```shell + $ crunchy-cli download -t 1 https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx + ``` + + The default thread count is the count of cpu threads your pc has. + ### Archive The `archive` command lets you download episodes with multiple audios and subtitles and merges it into a `.mkv` file. @@ -341,7 +381,7 @@ The `archive` command lets you download episodes with multiple audios and subtit - Output template Define an output template by using the `-o` / `--output` flag. - crunchy-cli uses the [`.mkv`](https://en.wikipedia.org/wiki/Matroska) container format, because of it's ability to store multiple audio, video and subtitle tracks at once. + _crunchy-cli_ exclusively uses the [`.mkv`](https://en.wikipedia.org/wiki/Matroska) container format, because of its ability to store multiple audio, video and subtitle tracks at once. ```shell $ crunchy-cli archive -o "{title}.mkv" https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx @@ -349,6 +389,17 @@ The `archive` command lets you download episodes with multiple audios and subtit Default is `{title}.mkv`. See the [Template Options section](#output-template-options) below for more options. +- Output template for special episodes + + Define an output template which only gets used when the episode is a special (episode number is 0 or has non-zero decimal places) by using the `--output-special` flag. + _crunchy-cli_ exclusively uses the [`.mkv`](https://en.wikipedia.org/wiki/Matroska) container format, because of its ability to store multiple audio, video and subtitle tracks at once. + + ```shell + $ crunchy-cli archive --output-specials -o "Special EP: {title}" https://www.crunchyroll.com/watch/GY8D975JY/veldoras-journal + ``` + + Default is the template, set by the `-o` / `--output` flag. See the [Template Options section](#output-template-options) below for more options. + - Resolution The resolution for videos can be set via the `-r` / `--resolution` flag. @@ -402,6 +453,14 @@ The `archive` command lets you download episodes with multiple audios and subtit $ crunchy-cli archive --skip-existing https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` +- Skip specials + + If you doesn't want to download special episodes, use the `--skip-specials` flag to skip the download of them. + + ```shell + $ crunchy-cli archive --skip-specials https://www.crunchyroll.com/series/GYZJ43JMR/that-time-i-got-reincarnated-as-a-slime[S2] + ``` + - Yes Sometimes different seasons have the same season number (e.g. Sword Art Online Alicization and Alicization War of Underworld are both marked as season 3), in such cases an interactive prompt is shown which needs user further user input to decide which season to download. @@ -413,6 +472,17 @@ The `archive` command lets you download episodes with multiple audios and subtit If you've passed the `-q` / `--quiet` [global flag](#global-settings), this flag is automatically set. +- Threads + + To increase the download speed, video segments are downloaded simultaneously by creating multiple threads. + If you want to manually specify how many threads to use when downloading, do this with the `-t` / `--threads` flag. + + ```shell + $ crunchy-cli archive -t 1 https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx + ``` + + The default thread count is the count of cpu threads your pc has. + ### Search **Supported urls/input** @@ -479,17 +549,19 @@ The `archive` command lets you download episodes with multiple audios and subtit You can use various template options to change how the filename is processed. The following tags are available: -- `{title}` → Title of the video -- `{series_name}` → Name of the series -- `{season_name}` → Name of the season -- `{audio}` → Audio language of the video -- `{resolution}` → Resolution of the video -- `{season_number}` → Number of the season -- `{episode_number}` → Number of the episode -- `{relative_episode_number}` → Number of the episode relative to its season -- `{series_id}` → ID of the series -- `{season_id}` → ID of the season -- `{episode_id}` → ID of the episode +- `{title}` → Title of the video +- `{series_name}` → Name of the series +- `{season_name}` → Name of the season +- `{audio}` → Audio language of the video +- `{resolution}` → Resolution of the video +- `{season_number}` → Number of the season +- `{episode_number}` → Number of the episode +- `{relative_episode_number}` → Number of the episode relative to its season +- `{sequence_number}` → Like `{episode_number}` but without possible non-number characters +- `{relative_sequence_number}` → Like `{relative_episode_number}` but with support for episode 0's and .5's +- `{series_id}` → ID of the series +- `{season_id}` → ID of the season +- `{episode_id}` → ID of the episode Example: From d52fe7fb923b7b60c0b54b3cda2fe3bf1087bb09 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 6 Nov 2023 22:56:51 +0100 Subject: [PATCH 025/176] Update dependencies and version --- Cargo.lock | 436 ++++++++++++++++++------------------ Cargo.toml | 4 +- crunchy-cli-core/Cargo.toml | 8 +- 3 files changed, 228 insertions(+), 220 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 409a005..8608f0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -30,9 +30,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea5d730647d4fadd988536d06fecce94b7b4f2a7efdae548f1cf4b63205518ab" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] @@ -54,9 +54,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.5.0" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" +checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" dependencies = [ "anstyle", "anstyle-parse", @@ -68,15 +68,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84bf0a05bbb2a83e5eb6fa36bb6e87baa08193c35ff52bbf6b38d8af2890e46" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] name = "anstyle-parse" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" dependencies = [ "utf8parse", ] @@ -92,9 +92,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "2.1.0" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" +checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" dependencies = [ "anstyle", "windows-sys 0.48.0", @@ -108,9 +108,9 @@ checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", @@ -140,9 +140,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.4" +version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" [[package]] name = "base64-serde" @@ -162,9 +162,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" [[package]] name = "block-padding" @@ -238,9 +238,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.4" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d7b8d5ec32af0fadc644bf1fd509a688c2103b185644bb1e29d164e0703136" +checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b" dependencies = [ "clap_builder", "clap_derive", @@ -248,9 +248,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.4" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5179bb514e4d7c2051749d8fcefa2ed6d06a9f4e6d69faf3805f5d80b8cf8d56" +checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663" dependencies = [ "anstream", "anstyle", @@ -260,18 +260,18 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.4.1" +version = "4.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4110a1e6af615a9e6d0a36f805d5c99099f8bab9b8042f5bc1fa220a4a89e36f" +checksum = "bffe91f06a11b4b9420f62103854e90867812cd5d01557f853c5ee8e791b12ae" dependencies = [ "clap", ] [[package]] name = "clap_derive" -version = "4.4.2" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck", "proc-macro2", @@ -281,15 +281,15 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" [[package]] name = "clap_mangen" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b44f35c514163027542f7147797ff930523eea288e03642727348ef1a9666f6b" +checksum = "d3be86020147691e1d2ef58f75346a3d4d94807bfc473e377d52f09f0f7d77f7" dependencies = [ "clap", "roff", @@ -360,16 +360,16 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" dependencies = [ "libc", ] [[package]] name = "crunchy-cli" -version = "3.0.3" +version = "3.1.0" dependencies = [ "chrono", "clap", @@ -382,7 +382,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.3" +version = "3.1.0" dependencies = [ "anyhow", "async-trait", @@ -412,9 +412,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "771cd92c5a4cc050f301674d77bca6c23f8f260ef346bd06c11b4b05ab9e0166" +checksum = "19bdd19b3f1b26e1a45ed32f7b2db68981111a9f5e9551b2d225bf470dca1e11" dependencies = [ "aes", "async-trait", @@ -439,9 +439,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b260191f1125c7ba31e35071524938de5c6b0c2d248e5a35c62c4605e2da427e" +checksum = "f0a12fb14fd65dede6da7dad3e4c434f6aee9c18cf2bae0d2cc5021cd5a29fec" dependencies = [ "darling", "quote", @@ -505,9 +505,9 @@ dependencies = [ [[package]] name = "dash-mpd" -version = "0.13.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b18de4d072c5be455129422f6a69eb17c032467750d857820cb0c9a92e86a4ed" +checksum = "ad33027a1ac2f37def4c33a5987a8e047fd34f77ff7cabc14bd437aa6d8d4dd2" dependencies = [ "base64", "base64-serde", @@ -522,15 +522,17 @@ dependencies = [ "serde_with", "thiserror", "tokio", + "url", "xattr", ] [[package]] name = "deranged" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" dependencies = [ + "powerfmt", "serde", ] @@ -607,30 +609,19 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" +checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" dependencies = [ - "errno-dragonfly", "libc", "windows-sys 0.48.0", ] -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", - "libc", -] - [[package]] name = "fastrand" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] name = "fnv" @@ -680,50 +671,38 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" dependencies = [ "futures-core", ] [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" - -[[package]] -name = "futures-macro" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" dependencies = [ "futures-core", - "futures-macro", "futures-task", "pin-project-lite", "pin-utils", @@ -784,9 +763,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.0" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" [[package]] name = "heck" @@ -857,7 +836,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2 0.4.10", "tokio", "tower-service", "tracing", @@ -866,9 +845,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http", @@ -893,16 +872,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.57" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows", + "windows-core", ] [[package]] @@ -964,12 +943,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", - "hashbrown 0.14.0", + "hashbrown 0.14.2", "serde", ] @@ -1007,9 +986,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "iso8601" @@ -1028,9 +1007,9 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" dependencies = [ "wasm-bindgen", ] @@ -1043,15 +1022,26 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.148" +version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" + +[[package]] +name = "libredox" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" +dependencies = [ + "bitflags 2.4.1", + "libc", + "redox_syscall", +] [[package]] name = "linux-raw-sys" -version = "0.4.7" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" +checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" [[package]] name = "log" @@ -1077,9 +1067,9 @@ checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" [[package]] name = "memchr" -version = "2.6.3" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "mime" @@ -1104,9 +1094,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" dependencies = [ "libc", "wasi", @@ -1135,7 +1125,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "cfg-if", "libc", ] @@ -1152,9 +1142,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ "autocfg", ] @@ -1192,11 +1182,11 @@ checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "openssl" -version = "0.10.57" +version = "0.10.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" +checksum = "7a257ad03cd8fb16ad4172fedf8094451e1af1c4b70097636ef2eac9a5f0cc33" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "cfg-if", "foreign-types", "libc", @@ -1224,18 +1214,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "300.1.5+3.1.3" +version = "300.1.6+3.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "559068e4c12950d7dcaa1857a61725c0d38d4fc03ff8e070ab31a75d6e316491" +checksum = "439fac53e092cd7442a3660c85dde4643ab3b5bd39040912388dcdabf6b88085" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.93" +version = "0.9.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +checksum = "40a4130519a360279579c2053038317e40eff64d13fd3f004f9e1b72b8a6aaf9" dependencies = [ "cc", "libc", @@ -1276,15 +1266,21 @@ checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "portable-atomic" -version = "1.4.3" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b" +checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "proc-macro2" -version = "1.0.67" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" dependencies = [ "unicode-ident", ] @@ -1307,9 +1303,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" dependencies = [ "memchr", "serde", @@ -1326,38 +1322,29 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.16" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "redox_syscall" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "redox_users" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" dependencies = [ "getrandom", - "redox_syscall 0.2.16", + "libredox", "thiserror", ] [[package]] name = "regex" -version = "1.9.5" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" dependencies = [ "aho-corasick", "memchr", @@ -1367,9 +1354,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.8" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ "aho-corasick", "memchr", @@ -1378,15 +1365,15 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.5" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "reqwest" -version = "0.11.20" +version = "0.11.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" +checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" dependencies = [ "base64", "bytes", @@ -1414,6 +1401,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "system-configuration", "tokio", "tokio-native-tls", "tokio-rustls", @@ -1429,17 +1417,16 @@ dependencies = [ [[package]] name = "ring" -version = "0.16.20" +version = "0.17.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b" dependencies = [ "cc", + "getrandom", "libc", - "once_cell", "spin", "untrusted", - "web-sys", - "winapi", + "windows-sys 0.48.0", ] [[package]] @@ -1456,11 +1443,11 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.38.14" +version = "0.38.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "747c788e9ce8e92b12cd485c49ddf90723550b654b32508f979b71a7b1ecda4f" +checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "errno", "libc", "linux-raw-sys", @@ -1469,9 +1456,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.7" +version = "0.21.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +checksum = "446e14c5cda4f3f30fe71863c34ec70f5ac79d6087097ad0bb433e1be5edf04c" dependencies = [ "log", "ring", @@ -1502,9 +1489,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.6" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ "ring", "untrusted", @@ -1527,9 +1514,9 @@ dependencies = [ [[package]] name = "sct" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ "ring", "untrusted", @@ -1560,18 +1547,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.188" +version = "1.0.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "a834c4821019838224821468552240d4d95d14e751986442c816572d39a080c9" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "46fa52d5646bce91b680189fe5b1c049d2ea38dabb4e2e7c8d00ca12cfbfbcfd" dependencies = [ "proc-macro2", "quote", @@ -1580,9 +1567,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ "itoa", "ryu", @@ -1612,15 +1599,15 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ca3b16a3d82c4088f343b7480a93550b3eabe1a358569c2dfe38bbcead07237" +checksum = "64cd236ccc1b7a29e7e2739f27c0b2dd199804abc4290e32f59f3b68d6405c23" dependencies = [ "base64", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.0.0", + "indexmap 2.1.0", "serde", "serde_json", "serde_with_macros", @@ -1629,9 +1616,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e6be15c453eb305019bfa438b1593c731f36a289a7853f7707ee29e870b3b3c" +checksum = "93634eb5f75a2323b16de4748022ac4297f9e76b6dced2be287a099f41b5e788" dependencies = [ "darling", "proc-macro2", @@ -1673,9 +1660,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" dependencies = [ "libc", "winapi", @@ -1683,9 +1670,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" dependencies = [ "libc", "windows-sys 0.48.0", @@ -1693,9 +1680,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.5.2" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "strsim" @@ -1705,9 +1692,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "2.0.37" +version = "2.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" dependencies = [ "proc-macro2", "quote", @@ -1724,32 +1711,53 @@ dependencies = [ ] [[package]] -name = "tempfile" -version = "3.8.0" +name = "system-configuration" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" dependencies = [ "cfg-if", "fastrand", - "redox_syscall 0.3.5", + "redox_syscall", "rustix", "windows-sys 0.48.0", ] [[package]] name = "thiserror" -version = "1.0.48" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.48" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", @@ -1758,12 +1766,13 @@ dependencies = [ [[package]] name = "time" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" +checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" dependencies = [ "deranged", "itoa", + "powerfmt", "serde", "time-core", "time-macros", @@ -1771,15 +1780,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" +checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" dependencies = [ "time-core", ] @@ -1801,9 +1810,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.32.0" +version = "1.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" dependencies = [ "backtrace", "bytes", @@ -1811,7 +1820,7 @@ dependencies = [ "mio", "num_cpus", "pin-project-lite", - "socket2 0.5.4", + "socket2 0.5.5", "tokio-macros", "windows-sys 0.48.0", ] @@ -1861,9 +1870,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ "bytes", "futures-core", @@ -1881,20 +1890,19 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "pin-project-lite", "tracing-core", ] [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", ] @@ -1940,9 +1948,9 @@ checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "untrusted" -version = "0.7.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" @@ -1990,9 +1998,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2000,9 +2008,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217" dependencies = [ "bumpalo", "log", @@ -2015,9 +2023,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.37" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" +checksum = "9afec9963e3d0994cac82455b2b3502b81a7f40f9a0d32181f7528d9f4b43e02" dependencies = [ "cfg-if", "js-sys", @@ -2027,9 +2035,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2037,9 +2045,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" dependencies = [ "proc-macro2", "quote", @@ -2050,15 +2058,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" [[package]] name = "web-sys" -version = "0.3.64" +version = "0.3.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85" dependencies = [ "js-sys", "wasm-bindgen", @@ -2093,10 +2101,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows" -version = "0.48.0" +name = "windows-core" +version = "0.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" dependencies = [ "windows-targets 0.48.5", ] diff --git a/Cargo.toml b/Cargo.toml index e5882d6..a0c1871 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.0.3" +version = "3.1.0" edition = "2021" license = "MIT" @@ -18,7 +18,7 @@ openssl = ["openssl-tls"] openssl-static = ["openssl-tls-static"] [dependencies] -tokio = { version = "1.32", features = ["macros", "rt-multi-thread", "time"], default-features = false } +tokio = { version = "1.33", features = ["macros", "rt-multi-thread", "time"], default-features = false } native-tls-crate = { package = "native-tls", version = "0.2.11", optional = true } diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index e38cb1a..9103831 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.0.3" +version = "3.1.0" edition = "2021" license = "MIT" @@ -16,7 +16,7 @@ anyhow = "1.0" async-trait = "0.1" clap = { version = "4.4", features = ["derive", "string"] } chrono = "0.4" -crunchyroll-rs = { version = "0.6.2", features = ["dash-stream"] } +crunchyroll-rs = { version = "0.6.3", features = ["dash-stream"] } ctrlc = "3.4" dialoguer = { version = "0.11", default-features = false } dirs = "5.0" @@ -26,7 +26,7 @@ indicatif = "0.17" lazy_static = "1.4" log = { version = "0.4", features = ["std"] } num_cpus = "1.16" -regex = "1.9" +regex = "1.10" reqwest = { version = "0.11", default-features = false, features = ["socks"] } serde = "1.0" serde_json = "1.0" @@ -34,7 +34,7 @@ serde_plain = "1.0" shlex = "1.2" sys-locale = "0.3" tempfile = "3.8" -tokio = { version = "1.32", features = ["macros", "rt-multi-thread", "time"] } +tokio = { version = "1.33", features = ["macros", "rt-multi-thread", "time"] } rustls-native-certs = { version = "0.6", optional = true } [build-dependencies] From 14e71c05b84ff7458478443b2de4ee71273c61a3 Mon Sep 17 00:00:00 2001 From: bytedream Date: Thu, 16 Nov 2023 13:51:30 +0100 Subject: [PATCH 026/176] Fix aur binary checksums (#266) --- .github/workflows/publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f777f9e..43ea795 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -43,8 +43,8 @@ jobs: curl -LO https://github.com/crunchy-labs/crunchy-cli/releases/download/${{ github.ref_name }}/crunchy-cli-${{ github.ref_name }}-completions.zip curl -LO https://github.com/crunchy-labs/crunchy-cli/releases/download/${{ github.ref_name }}/crunchy-cli-${{ github.ref_name }}-manpages.zip curl -LO https://raw.githubusercontent.com/crunchy-labs/crunchy-cli/${{ github.ref_name }}/LICENSE - echo "CRUNCHY_CLI_BIN_x86_64_SHA256=$(sha256sum crunchy-cli-${{ github.ref_name }}-x86_64-linux | cut -f 1 -d ' ')" >> $GITHUB_ENV - echo "CRUNCHY_CLI_BIN_aarch64_SHA256=$(sha256sum crunchy-cli-${{ github.ref_name }}-aarch64-linux | cut -f 1 -d ' ')" >> $GITHUB_ENV + echo "CRUNCHY_CLI_BIN_x86_64_SHA256=$(sha256sum crunchy-cli-${{ github.ref_name }}-linux-x86_64 | cut -f 1 -d ' ')" >> $GITHUB_ENV + echo "CRUNCHY_CLI_BIN_aarch64_SHA256=$(sha256sum crunchy-cli-${{ github.ref_name }}-linux-aarch64 | cut -f 1 -d ' ')" >> $GITHUB_ENV echo "CRUNCHY_CLI_BIN_COMPLETIONS_SHA256=$(sha256sum crunchy-cli-${{ github.ref_name }}-completions.zip | cut -f 1 -d ' ')" >> $GITHUB_ENV echo "CRUNCHY_CLI_BIN_MANPAGES_SHA256=$(sha256sum crunchy-cli-${{ github.ref_name }}-manpages.zip | cut -f 1 -d ' ')" >> $GITHUB_ENV echo "CRUNCHY_CLI_BIN_LICENSE_SHA256=$(sha256sum LICENSE | cut -f 1 -d ' ')" >> $GITHUB_ENV From 2c370939595149c20ed6ba2fbde3274134be0aad Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 19 Nov 2023 19:24:15 +0100 Subject: [PATCH 027/176] Manually burn-in subtitles only if no pre-burned video is available (#268) --- crunchy-cli-core/src/archive/command.rs | 3 +- crunchy-cli-core/src/download/command.rs | 56 +++++++++++++++++++++--- crunchy-cli-core/src/utils/ffmpeg.rs | 2 + crunchy-cli-core/src/utils/format.rs | 4 ++ crunchy-cli-core/src/utils/video.rs | 37 ++++++++++++---- 5 files changed, 88 insertions(+), 14 deletions(-) diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index 3585e9c..006bbfa 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -248,7 +248,8 @@ async fn get_format( for single_format in single_formats { let stream = single_format.stream().await?; - let Some((video, audio)) = variant_data_from_stream(&stream, &archive.resolution).await? + let Some((video, audio, _)) = + variant_data_from_stream(&stream, &archive.resolution, None).await? else { if single_format.is_episode() { bail!( diff --git a/crunchy-cli-core/src/download/command.rs b/crunchy-cli-core/src/download/command.rs index 82cb8f8..cf1a049 100644 --- a/crunchy-cli-core/src/download/command.rs +++ b/crunchy-cli-core/src/download/command.rs @@ -1,7 +1,7 @@ use crate::download::filter::DownloadFilter; use crate::utils::context::Context; use crate::utils::download::{DownloadBuilder, DownloadFormat}; -use crate::utils::ffmpeg::FFmpegPreset; +use crate::utils::ffmpeg::{FFmpegPreset, SOFTSUB_CONTAINERS}; use crate::utils::filter::Filter; use crate::utils::format::{Format, SingleFormat}; use crate::utils::log::progress; @@ -149,6 +149,25 @@ impl Execute for Download { async fn execute(self, ctx: Context) -> Result<()> { let mut parsed_urls = vec![]; + let output_supports_softsubs = SOFTSUB_CONTAINERS.contains( + &Path::new(&self.output) + .extension() + .unwrap_or_default() + .to_string_lossy() + .as_ref(), + ); + let special_output_supports_softsubs = if let Some(so) = &self.output_specials { + SOFTSUB_CONTAINERS.contains( + &Path::new(so) + .extension() + .unwrap_or_default() + .to_string_lossy() + .as_ref(), + ) + } else { + output_supports_softsubs + }; + for (i, url) in self.urls.clone().into_iter().enumerate() { let progress_handler = progress!("Parsing url {}", i + 1); match parse_url(&ctx.crunchy, url.clone(), true).await { @@ -190,7 +209,18 @@ impl Execute for Download { // the vec contains always only one item let single_format = single_formats.remove(0); - let (download_format, format) = get_format(&self, &single_format).await?; + let (download_format, format) = get_format( + &self, + &single_format, + if self.force_hardsub { + true + } else if single_format.is_special() { + !special_output_supports_softsubs + } else { + !output_supports_softsubs + }, + ) + .await?; let mut downloader = download_builder.clone().build(); downloader.add_format(download_format); @@ -227,9 +257,19 @@ impl Execute for Download { async fn get_format( download: &Download, single_format: &SingleFormat, + try_peer_hardsubs: bool, ) -> Result<(DownloadFormat, Format)> { let stream = single_format.stream().await?; - let Some((video, audio)) = variant_data_from_stream(&stream, &download.resolution).await? + let Some((video, audio, contains_hardsub)) = variant_data_from_stream( + &stream, + &download.resolution, + if try_peer_hardsubs { + download.subtitle.clone() + } else { + None + }, + ) + .await? else { if single_format.is_episode() { bail!( @@ -250,7 +290,9 @@ async fn get_format( } }; - let subtitle = if let Some(subtitle_locale) = &download.subtitle { + let subtitle = if contains_hardsub { + None + } else if let Some(subtitle_locale) = &download.subtitle { stream.subtitles.get(subtitle_locale).cloned() } else { None @@ -266,7 +308,7 @@ async fn get_format( )] }), }; - let format = Format::from_single_formats(vec![( + let mut format = Format::from_single_formats(vec![( single_format.clone(), video, subtitle.map_or(vec![], |s| { @@ -276,6 +318,10 @@ async fn get_format( )] }), )]); + if contains_hardsub { + let (_, subs) = format.locales.get_mut(0).unwrap(); + subs.push(download.subtitle.clone().unwrap()) + } Ok((download_format, format)) } diff --git a/crunchy-cli-core/src/utils/ffmpeg.rs b/crunchy-cli-core/src/utils/ffmpeg.rs index 787c79a..7d77833 100644 --- a/crunchy-cli-core/src/utils/ffmpeg.rs +++ b/crunchy-cli-core/src/utils/ffmpeg.rs @@ -2,6 +2,8 @@ use lazy_static::lazy_static; use regex::Regex; use std::str::FromStr; +pub const SOFTSUB_CONTAINERS: [&str; 3] = ["mkv", "mov", "mp4"]; + #[derive(Clone, Debug, Eq, PartialEq)] pub enum FFmpegPreset { Predefined(FFmpegCodec, Option, FFmpegQuality), diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index 4c8d3c8..05ac232 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -172,6 +172,10 @@ impl SingleFormat { pub fn is_episode(&self) -> bool { matches!(self.source, MediaCollection::Episode(_)) } + + pub fn is_special(&self) -> bool { + self.sequence_number == 0.0 || self.sequence_number.fract() != 0.0 + } } struct SingleFormatCollectionEpisodeKey(f32); diff --git a/crunchy-cli-core/src/utils/video.rs b/crunchy-cli-core/src/utils/video.rs index 5b3eaeb..0ae4ba4 100644 --- a/crunchy-cli-core/src/utils/video.rs +++ b/crunchy-cli-core/src/utils/video.rs @@ -1,32 +1,47 @@ -use anyhow::Result; +use anyhow::{bail, Result}; use crunchyroll_rs::media::{Resolution, Stream, VariantData}; use crunchyroll_rs::Locale; pub async fn variant_data_from_stream( stream: &Stream, resolution: &Resolution, -) -> Result> { + subtitle: Option, +) -> Result> { // sometimes Crunchyroll marks episodes without real subtitles that they have subtitles and // reports that only hardsub episode are existing. the following lines are trying to prevent // potential errors which might get caused by this incorrect reporting // (https://github.com/crunchy-labs/crunchy-cli/issues/231) let mut hardsub_locales = stream.streaming_hardsub_locales(); - let hardsub_locale = if !hardsub_locales.contains(&Locale::Custom("".to_string())) + let (hardsub_locale, mut contains_hardsub) = if !hardsub_locales + .contains(&Locale::Custom("".to_string())) && !hardsub_locales.contains(&Locale::Custom(":".to_string())) { // if only one hardsub locale exists, assume that this stream doesn't really contains hardsubs if hardsub_locales.len() == 1 { - Some(hardsub_locales.remove(0)) + (Some(hardsub_locales.remove(0)), false) } else { // fallback to `None`. this should trigger an error message in `stream.dash_streaming_data` // that the requested stream is not available - None + (None, false) } } else { - None + let hardsubs_requested = subtitle.is_some(); + (subtitle, hardsubs_requested) }; - let mut streaming_data = stream.dash_streaming_data(hardsub_locale).await?; + let mut streaming_data = match stream.dash_streaming_data(hardsub_locale).await { + Ok(data) => data, + Err(e) => { + // the error variant is only `crunchyroll_rs::error::Error::Input` when the requested + // hardsub is not available + if let crunchyroll_rs::error::Error::Input { .. } = e { + contains_hardsub = false; + stream.dash_streaming_data(None).await? + } else { + bail!(e) + } + } + }; streaming_data .0 .sort_by(|a, b| a.bandwidth.cmp(&b.bandwidth).reverse()); @@ -42,5 +57,11 @@ pub async fn variant_data_from_stream( .into_iter() .find(|v| resolution.height == v.resolution.height), }; - Ok(video_variant.map(|v| (v, streaming_data.1.first().unwrap().clone()))) + Ok(video_variant.map(|v| { + ( + v, + streaming_data.1.first().unwrap().clone(), + contains_hardsub, + ) + })) } From 440ccd99b5482af43b0cec222d49a45ddd5b43e1 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 20 Nov 2023 22:05:06 +0100 Subject: [PATCH 028/176] Update dependencies and version --- Cargo.lock | 82 +++++++++++++------------ Cargo.toml | 4 +- crunchy-cli-core/Cargo.toml | 6 +- crunchy-cli-core/src/archive/filter.rs | 8 +-- crunchy-cli-core/src/download/filter.rs | 4 +- crunchy-cli-core/src/search/format.rs | 2 +- 6 files changed, 54 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8608f0e..897731f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -238,9 +238,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.7" +version = "4.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b" +checksum = "2275f18819641850fa26c89acc84d465c1bf91ce57bc2748b28c420473352f64" dependencies = [ "clap_builder", "clap_derive", @@ -248,9 +248,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.7" +version = "4.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663" +checksum = "07cdf1b148b25c1e1f7a42225e30a0d99a615cd4637eae7365548dd4529b95bc" dependencies = [ "anstream", "anstyle", @@ -369,7 +369,7 @@ dependencies = [ [[package]] name = "crunchy-cli" -version = "3.1.0" +version = "3.1.1" dependencies = [ "chrono", "clap", @@ -382,7 +382,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.1.0" +version = "3.1.1" dependencies = [ "anyhow", "async-trait", @@ -412,9 +412,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.6.3" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19bdd19b3f1b26e1a45ed32f7b2db68981111a9f5e9551b2d225bf470dca1e11" +checksum = "cc4ce434784eee7892ad8c3d1ecaea0c0858db51bbb295b474db38c256e8d2fb" dependencies = [ "aes", "async-trait", @@ -423,7 +423,6 @@ dependencies = [ "crunchyroll-rs-internal", "dash-mpd", "futures-util", - "http", "lazy_static", "m3u8-rs", "regex", @@ -439,9 +438,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.6.3" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0a12fb14fd65dede6da7dad3e4c434f6aee9c18cf2bae0d2cc5021cd5a29fec" +checksum = "be840f8cf2ce6afc9a9eae268d41423093141ec88f664a515d5ed2a85a66fb60" dependencies = [ "darling", "quote", @@ -505,9 +504,9 @@ dependencies = [ [[package]] name = "dash-mpd" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad33027a1ac2f37def4c33a5987a8e047fd34f77ff7cabc14bd437aa6d8d4dd2" +checksum = "5471fc46c0b229c8f2308d83be857c745c9f06cc83a433d7047909722e0453b4" dependencies = [ "base64", "base64-serde", @@ -609,9 +608,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.5" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" +checksum = "f258a7194e7f7c2a7837a8913aeab7fd8c383457034fa20ce4dd3dcb813e8eb8" dependencies = [ "libc", "windows-sys 0.48.0", @@ -655,9 +654,12 @@ dependencies = [ [[package]] name = "fs-err" -version = "2.9.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0845fa252299212f0389d64ba26f34fa32cfe41588355f21ed507c59a0f64541" +checksum = "88a41f105fe1d5b6b34b2055e3dc59bb79b46b48b2040b9e6c7b4b5de097aa41" +dependencies = [ + "autocfg", +] [[package]] name = "fs2" @@ -721,9 +723,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" dependencies = [ "cfg-if", "libc", @@ -738,9 +740,9 @@ checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" [[package]] name = "h2" -version = "0.3.21" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" dependencies = [ "bytes", "fnv", @@ -748,7 +750,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 1.9.3", + "indexmap 2.1.0", "slab", "tokio", "tokio-util", @@ -787,9 +789,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" dependencies = [ "bytes", "fnv", @@ -1039,9 +1041,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" +checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" [[package]] name = "log" @@ -1443,9 +1445,9 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.38.21" +version = "0.38.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3" +checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" dependencies = [ "bitflags 2.4.1", "errno", @@ -1456,9 +1458,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.8" +version = "0.21.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446e14c5cda4f3f30fe71863c34ec70f5ac79d6087097ad0bb433e1be5edf04c" +checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" dependencies = [ "log", "ring", @@ -1480,9 +1482,9 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ "base64", ] @@ -1547,18 +1549,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.191" +version = "1.0.192" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a834c4821019838224821468552240d4d95d14e751986442c816572d39a080c9" +checksum = "bca2a08484b285dcb282d0f67b26cadc0df8b19f8c12502c13d966bf9482f001" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.191" +version = "1.0.192" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46fa52d5646bce91b680189fe5b1c049d2ea38dabb4e2e7c8d00ca12cfbfbcfd" +checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1" dependencies = [ "proc-macro2", "quote", @@ -1810,9 +1812,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.33.0" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" +checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" dependencies = [ "backtrace", "bytes", @@ -1827,9 +1829,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index a0c1871..098c048 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.1.0" +version = "3.1.1" edition = "2021" license = "MIT" @@ -18,7 +18,7 @@ openssl = ["openssl-tls"] openssl-static = ["openssl-tls-static"] [dependencies] -tokio = { version = "1.33", features = ["macros", "rt-multi-thread", "time"], default-features = false } +tokio = { version = "1.34", features = ["macros", "rt-multi-thread", "time"], default-features = false } native-tls-crate = { package = "native-tls", version = "0.2.11", optional = true } diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 9103831..5771888 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.1.0" +version = "3.1.1" edition = "2021" license = "MIT" @@ -16,7 +16,7 @@ anyhow = "1.0" async-trait = "0.1" clap = { version = "4.4", features = ["derive", "string"] } chrono = "0.4" -crunchyroll-rs = { version = "0.6.3", features = ["dash-stream"] } +crunchyroll-rs = { version = "0.7.0", features = ["dash-stream"] } ctrlc = "3.4" dialoguer = { version = "0.11", default-features = false } dirs = "5.0" @@ -34,7 +34,7 @@ serde_plain = "1.0" shlex = "1.2" sys-locale = "0.3" tempfile = "3.8" -tokio = { version = "1.33", features = ["macros", "rt-multi-thread", "time"] } +tokio = { version = "1.34", features = ["macros", "rt-multi-thread", "time"] } rustls-native-certs = { version = "0.6", optional = true } [build-dependencies] diff --git a/crunchy-cli-core/src/archive/filter.rs b/crunchy-cli-core/src/archive/filter.rs index 01612b9..1777072 100644 --- a/crunchy-cli-core/src/archive/filter.rs +++ b/crunchy-cli-core/src/archive/filter.rs @@ -223,7 +223,7 @@ impl Filter for ArchiveFilter { "Season {} is only available with {} audio until episode {} ({})", season.season_number, season_locale.unwrap_or(Locale::ja_JP), - last_episode.episode_number, + last_episode.sequence_number, last_episode.title ) } @@ -278,7 +278,7 @@ impl Filter for ArchiveFilter { if !missing_audio.is_empty() { warn!( "Episode {} is not available with {} audio", - episode.episode_number, + episode.sequence_number, missing_audio .into_iter() .map(|l| l.to_string()) @@ -298,7 +298,7 @@ impl Filter for ArchiveFilter { { warn!( "Episode {} is not available with {} subtitles", - episode.episode_number, + episode.sequence_number, missing_subtitles .into_iter() .map(|l| l.to_string()) @@ -343,7 +343,7 @@ impl Filter for ArchiveFilter { if relative_episode_number.is_none() || relative_sequence_number.is_none() { warn!( "Failed to get relative episode number for episode {} ({}) of {} season {}", - episode.episode_number, + episode.sequence_number, episode.title, episode.series_title, episode.season_number, diff --git a/crunchy-cli-core/src/download/filter.rs b/crunchy-cli-core/src/download/filter.rs index fb2e563..51f1ad8 100644 --- a/crunchy-cli-core/src/download/filter.rs +++ b/crunchy-cli-core/src/download/filter.rs @@ -158,7 +158,7 @@ impl Filter for DownloadFilter { { let error_message = format!( "Episode {} ({}) of {} season {} is not available with {} audio", - episode.episode_number, + episode.sequence_number, episode.title, episode.series_title, episode.season_number, @@ -234,7 +234,7 @@ impl Filter for DownloadFilter { if relative_episode_number.is_none() || relative_sequence_number.is_none() { warn!( "Failed to get relative episode number for episode {} ({}) of {} season {}", - episode.episode_number, + episode.sequence_number, episode.title, episode.series_title, episode.season_number, diff --git a/crunchy-cli-core/src/search/format.rs b/crunchy-cli-core/src/search/format.rs index f9746b1..156bd95 100644 --- a/crunchy-cli-core/src/search/format.rs +++ b/crunchy-cli-core/src/search/format.rs @@ -70,7 +70,7 @@ impl From<&Episode> for FormatEpisode { title: value.title.clone(), description: value.description.clone(), locale: value.audio_locale.clone(), - number: value.episode_number, + number: value.episode_number.unwrap_or_default(), sequence_number: value.sequence_number, duration: value.duration.num_milliseconds(), air_date: value.episode_air_date.timestamp(), From d5df3df95f5972495c8c317f0fb243c32ede0b37 Mon Sep 17 00:00:00 2001 From: bytedream Date: Fri, 1 Dec 2023 01:02:53 +0100 Subject: [PATCH 029/176] Fix fixed subtitle formatting and sorting (#272) --- crunchy-cli-core/src/utils/download.rs | 33 +++++++++++++++++++------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index 945cc14..c65b4d2 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -17,6 +17,7 @@ use std::env; use std::io::Write; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; +use std::str::FromStr; use std::sync::Arc; use std::time::Duration; use tempfile::TempPath; @@ -788,8 +789,10 @@ pub fn get_video_length(path: &Path) -> Result { /// [crunchy-labs/crunchy-cli#208](https://github.com/crunchy-labs/crunchy-cli/issues/208) for more /// information. fn fix_subtitles(raw: &mut Vec, max_length: NaiveTime) { - let re = Regex::new(r"^Dialogue:\s\d+,(?P\d+:\d+:\d+\.\d+),(?P\d+:\d+:\d+\.\d+),") - .unwrap(); + let re = Regex::new( + r"^Dialogue:\s(?P\d+),(?P\d+:\d+:\d+\.\d+),(?P\d+:\d+:\d+\.\d+),", + ) + .unwrap(); // chrono panics if we try to format NaiveTime with `%2f` and the nano seconds has more than 2 // digits so them have to be reduced manually to avoid the panic @@ -804,9 +807,9 @@ fn fix_subtitles(raw: &mut Vec, max_length: NaiveTime) { formatted_time.split_at(2).0.parse().unwrap() } ) + .split_off(1) // <- in the ASS spec, the hour has only one digit } - let length_as_string = format_naive_time(max_length); let mut entries = (vec![], vec![]); let mut as_lines: Vec = String::from_utf8_lossy(raw.as_slice()) @@ -818,21 +821,33 @@ fn fix_subtitles(raw: &mut Vec, max_length: NaiveTime) { if line.trim() == "[Script Info]" { line.push_str("\nScaledBorderAndShadow: yes") } else if let Some(capture) = re.captures(line) { - let start = capture.name("start").map_or(NaiveTime::default(), |s| { + let mut start = capture.name("start").map_or(NaiveTime::default(), |s| { NaiveTime::parse_from_str(s.as_str(), "%H:%M:%S.%f").unwrap() }); - let end = capture.name("end").map_or(NaiveTime::default(), |s| { - NaiveTime::parse_from_str(s.as_str(), "%H:%M:%S.%f").unwrap() + let mut end = capture.name("end").map_or(NaiveTime::default(), |e| { + NaiveTime::parse_from_str(e.as_str(), "%H:%M:%S.%f").unwrap() }); - if end > max_length { + if start > max_length || end > max_length { + let layer = capture + .name("layer") + .map_or(0, |l| i32::from_str(l.as_str()).unwrap()); + + if start > max_length { + start = max_length; + } + if start > max_length || end > max_length { + end = max_length; + } + *line = re .replace( line, format!( - "Dialogue: {},{},", + "Dialogue: {},{},{},", + layer, format_naive_time(start), - &length_as_string + format_naive_time(end) ), ) .to_string() From 8f77028fcb933137d66c4f4fb2e0d3b7f4454843 Mon Sep 17 00:00:00 2001 From: bytedream Date: Fri, 1 Dec 2023 01:17:49 +0100 Subject: [PATCH 030/176] Show error message instead of panicking when capturing video length of invalid file (#258) --- crunchy-cli-core/src/utils/download.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index c65b4d2..de1aed0 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -768,7 +768,13 @@ pub fn get_video_length(path: &Path) -> Result { .args(["-i", path.to_str().unwrap()]) .output()?; let ffmpeg_output = String::from_utf8(ffmpeg.stderr)?; - let caps = video_length.captures(ffmpeg_output.as_str()).unwrap(); + let caps = video_length.captures(ffmpeg_output.as_str()).map_or( + Err(anyhow::anyhow!( + "failed to get video length: {}", + ffmpeg_output + )), + Ok, + )?; Ok(NaiveTime::parse_from_str(caps.name("time").unwrap().as_str(), "%H:%M:%S%.f").unwrap()) } From 9ca3b79291da1d4d4f3f6abb4c0728f641f75ae7 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 3 Dec 2023 00:15:57 +0100 Subject: [PATCH 031/176] Fix spelling --- crunchy-cli-core/src/utils/ffmpeg.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/crunchy-cli-core/src/utils/ffmpeg.rs b/crunchy-cli-core/src/utils/ffmpeg.rs index 7d77833..39678dc 100644 --- a/crunchy-cli-core/src/utils/ffmpeg.rs +++ b/crunchy-cli-core/src/utils/ffmpeg.rs @@ -229,7 +229,7 @@ impl FFmpegPreset { quality = Some(q) } else { return Err(format!( - "'{}' is not a valid ffmpeg preset (unknown token '{}'", + "'{}' is not a valid ffmpeg preset (unknown token '{}')", s, token )); } @@ -286,12 +286,10 @@ impl FFmpegPreset { output.extend(["-c:v", "h264_nvenc", "-c:a", "copy"]) } FFmpegHwAccel::Apple => { - // Apple's Video Toolbox encoders ignore `-crf`, - // use `-q:v` instead. It's on a scale of 1-100, - // 100 being lossless. Just did some math - // ((-a/51+1)*99+1 where `a` is the old crf value) - // so these settings very likely need some more - // tweeking. + // Apple's Video Toolbox encoders ignore `-crf`, use `-q:v` + // instead. It's on a scale of 1-100, 100 being lossless. Just + // did some math ((-a/51+1)*99+1 where `a` is the old crf value) + // so these settings very likely need some more tweaking match quality { FFmpegQuality::Lossless => output.extend(["-q:v", "65"]), FFmpegQuality::Normal => (), From 9487dd3dbffd4646713ce4e0ec1d6210fc1a1f14 Mon Sep 17 00:00:00 2001 From: bytedream Date: Fri, 8 Dec 2023 22:27:12 +0100 Subject: [PATCH 032/176] Show ffmpeg progress (#270) --- Cargo.lock | 1 + crunchy-cli-core/Cargo.toml | 5 +- crunchy-cli-core/src/utils/download.rs | 142 +++++++++++++++++++------ crunchy-cli-core/src/utils/os.rs | 60 +++++++++++ 4 files changed, 176 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 897731f..fa5723f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -397,6 +397,7 @@ dependencies = [ "indicatif", "lazy_static", "log", + "nix", "num_cpus", "regex", "reqwest", diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 5771888..4f0912f 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -34,8 +34,11 @@ serde_plain = "1.0" shlex = "1.2" sys-locale = "0.3" tempfile = "3.8" -tokio = { version = "1.34", features = ["macros", "rt-multi-thread", "time"] } +tokio = { version = "1.34", features = ["io-util", "macros", "net", "rt-multi-thread", "time"] } rustls-native-certs = { version = "0.6", optional = true } +[target.'cfg(not(target_os = "windows"))'.dependencies] +nix = { version = "0.27", features = ["fs"] } + [build-dependencies] chrono = "0.4" diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs index de1aed0..c154103 100644 --- a/crunchy-cli-core/src/utils/download.rs +++ b/crunchy-cli-core/src/utils/download.rs @@ -1,16 +1,14 @@ use crate::utils::context::Context; use crate::utils::ffmpeg::FFmpegPreset; -use crate::utils::log::progress; -use crate::utils::os::{is_special_file, temp_directory, tempfile}; +use crate::utils::os::{is_special_file, temp_directory, temp_named_pipe, tempfile}; use anyhow::{bail, Result}; use chrono::NaiveTime; use crunchyroll_rs::media::{Subtitle, VariantData, VariantSegment}; use crunchyroll_rs::Locale; -use indicatif::{ProgressBar, ProgressFinish, ProgressStyle}; +use indicatif::{ProgressBar, ProgressDrawTarget, ProgressFinish, ProgressStyle}; use log::{debug, warn, LevelFilter}; use regex::Regex; -use std::borrow::Borrow; -use std::borrow::BorrowMut; +use std::borrow::{Borrow, BorrowMut}; use std::cmp::Ordering; use std::collections::BTreeMap; use std::env; @@ -21,6 +19,7 @@ use std::str::FromStr; use std::sync::Arc; use std::time::Duration; use tempfile::TempPath; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader}; use tokio::sync::mpsc::unbounded_channel; use tokio::sync::Mutex; use tokio::task::JoinSet; @@ -177,15 +176,19 @@ impl Downloader { let mut videos = vec![]; let mut audios = vec![]; let mut subtitles = vec![]; + let mut max_frames = 0f64; + let fmt_space = self + .formats + .iter() + .flat_map(|f| { + f.audios + .iter() + .map(|(_, locale)| format!("Downloading {} audio", locale).len()) + }) + .max() + .unwrap(); for (i, format) in self.formats.iter().enumerate() { - let fmt_space = format - .audios - .iter() - .map(|(_, locale)| format!("Downloading {} audio", locale).len()) - .max() - .unwrap(); - let video_path = self .download_video( ctx, @@ -232,7 +235,11 @@ impl Downloader { None }; - let len = get_video_length(&video_path)?; + let (len, fps) = get_video_stats(&video_path)?; + let frames = len.signed_duration_since(NaiveTime::MIN).num_seconds() as f64 * fps; + if frames > max_frames { + max_frames = frames; + } for (subtitle, not_cc) in format.subtitles.iter() { if let Some(pb) = &progress_spinner { let mut progress_message = pb.message(); @@ -337,8 +344,14 @@ impl Downloader { } let (input_presets, mut output_presets) = self.ffmpeg_preset.into_input_output_args(); + let fifo = temp_named_pipe()?; - let mut command_args = vec!["-y".to_string(), "-hide_banner".to_string()]; + let mut command_args = vec![ + "-y".to_string(), + "-hide_banner".to_string(), + "-vstats_file".to_string(), + fifo.name(), + ]; command_args.extend(input_presets); command_args.extend(input); command_args.extend(maps); @@ -433,8 +446,6 @@ impl Downloader { } } - let progress_handler = progress!("Generating output file"); - let ffmpeg = Command::new("ffmpeg") // pass ffmpeg stdout to real stdout only if output file is stdout .stdout(if dst.to_str().unwrap() == "-" { @@ -444,14 +455,26 @@ impl Downloader { }) .stderr(Stdio::piped()) .args(command_args) - .output()?; - if !ffmpeg.status.success() { - bail!("{}", String::from_utf8_lossy(ffmpeg.stderr.as_slice())) + .spawn()?; + let ffmpeg_progress = tokio::spawn(async move { + ffmpeg_progress( + max_frames as u64, + fifo, + format!("{:<1$}", "Generating output file", fmt_space + 1), + ) + .await + }); + + let result = ffmpeg.wait_with_output()?; + if !result.status.success() { + bail!("{}", String::from_utf8_lossy(result.stderr.as_slice())) + } + ffmpeg_progress.abort(); + match ffmpeg_progress.await { + Ok(r) => Ok(r?), + Err(e) if e.is_cancelled() => Ok(()), + Err(e) => Err(anyhow::Error::from(e)), } - - progress_handler.stop("Output file generated"); - - Ok(()) } async fn check_free_space( @@ -752,13 +775,10 @@ fn estimate_variant_file_size(variant_data: &VariantData, segments: &[VariantSeg (variant_data.bandwidth / 8) * segments.iter().map(|s| s.length.as_secs()).sum::() } -/// Get the length of a video. This is required because sometimes subtitles have an unnecessary entry -/// long after the actual video ends with artificially extends the video length on some video players. -/// To prevent this, the video length must be hard set. See -/// [crunchy-labs/crunchy-cli#32](https://github.com/crunchy-labs/crunchy-cli/issues/32) for more -/// information. -pub fn get_video_length(path: &Path) -> Result { +/// Get the length and fps of a video. +pub fn get_video_stats(path: &Path) -> Result<(NaiveTime, f64)> { let video_length = Regex::new(r"Duration:\s(?P