From d0a8103e3def5a45c84bf7eafe2c8b43f8c7b98d Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 4 Jan 2023 00:12:13 +0100 Subject: [PATCH 001/342] Update dependencies & version --- Cargo.lock | 12 ++++++------ Cargo.toml | 2 +- crunchy-cli-core/Cargo.lock | 10 +++++----- crunchy-cli-core/Cargo.toml | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a564687..8da4e30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -279,7 +279,7 @@ dependencies = [ [[package]] name = "crunchy-cli" -version = "3.0.0-dev.5" +version = "3.0.0-dev.6" dependencies = [ "chrono", "clap", @@ -291,7 +291,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-dev.5" +version = "3.0.0-dev.6" dependencies = [ "anyhow", "async-trait", @@ -317,9 +317,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ec300b509afbd8977f71cd7feb7d0f20d38d8e38976b7fcd51f6128cbdefe6" +checksum = "dd742baacdfbf9caca8656262ac2397c334928ccd65fb332c95ee8e8b8d9223d" dependencies = [ "aes", "async-trait", @@ -342,9 +342,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d4b870818d5ce0993d70271bf803dbfbcc8a3a0fa7398b182f5f4b4e88509d" +checksum = "aadb66f2b5cdcc8b82f095aec6ca92ca4abb30016a13916f2358cf7bfd8a1997" dependencies = [ "darling", "quote", diff --git a/Cargo.toml b/Cargo.toml index c67d599..76ed950 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-dev.5" +version = "3.0.0-dev.6" edition = "2021" [dependencies] diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index 37ee2c3..af1086a 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -260,7 +260,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-dev.5" +version = "3.0.0-dev.6" dependencies = [ "anyhow", "async-trait", @@ -286,9 +286,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ec300b509afbd8977f71cd7feb7d0f20d38d8e38976b7fcd51f6128cbdefe6" +checksum = "dd742baacdfbf9caca8656262ac2397c334928ccd65fb332c95ee8e8b8d9223d" dependencies = [ "aes", "async-trait", @@ -311,9 +311,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d4b870818d5ce0993d70271bf803dbfbcc8a3a0fa7398b182f5f4b4e88509d" +checksum = "aadb66f2b5cdcc8b82f095aec6ca92ca4abb30016a13916f2358cf7bfd8a1997" dependencies = [ "darling", "quote", diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index e61031b..566b8c4 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-dev.5" +version = "3.0.0-dev.6" edition = "2021" [dependencies] @@ -15,7 +15,7 @@ ctrlc = "3.2" dirs = "4.0" indicatif = "0.17" log = { version = "0.4", features = ["std"] } -num_cpus = "1.14" +num_cpus = "1.15" regex = "1.7" sanitize-filename = "0.4" serde = "1.0" From 7726287859482f508dc584be405db7022ee31b22 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 4 Jan 2023 01:28:14 +0100 Subject: [PATCH 002/342] :) --- Cargo.lock | 8 ++++---- crunchy-cli-core/Cargo.lock | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8da4e30..eb7c6a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -317,9 +317,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd742baacdfbf9caca8656262ac2397c334928ccd65fb332c95ee8e8b8d9223d" +checksum = "184d0c725a09aec815316cbf41a2f362008ecb0e8c8e3b6b9930d01a89b5df21" dependencies = [ "aes", "async-trait", @@ -342,9 +342,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aadb66f2b5cdcc8b82f095aec6ca92ca4abb30016a13916f2358cf7bfd8a1997" +checksum = "5f3c82e1766339727fc2c10d66d0c4f001b1cf42e2993f9d93997b610f408776" dependencies = [ "darling", "quote", diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index af1086a..b5dd8a1 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -286,9 +286,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd742baacdfbf9caca8656262ac2397c334928ccd65fb332c95ee8e8b8d9223d" +checksum = "184d0c725a09aec815316cbf41a2f362008ecb0e8c8e3b6b9930d01a89b5df21" dependencies = [ "aes", "async-trait", @@ -311,9 +311,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aadb66f2b5cdcc8b82f095aec6ca92ca4abb30016a13916f2358cf7bfd8a1997" +checksum = "5f3c82e1766339727fc2c10d66d0c4f001b1cf42e2993f9d93997b610f408776" dependencies = [ "darling", "quote", From 404aa496e157162c35c3b5bf983cbc43c8689676 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Thu, 5 Jan 2023 22:28:23 +0100 Subject: [PATCH 003/342] Fix subtitle look and feel typo --- crunchy-cli-core/src/cli/archive.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index e3d5520..3910d24 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -481,7 +481,7 @@ fn fix_subtitle_look_and_feel(raw: Vec) -> Vec { for line in String::from_utf8_lossy(raw.as_slice()).split('\n') { if line.trim().starts_with('[') && script_info { - new.push_str("ScaledBorderAndShadows: yes\n"); + new.push_str("ScaledBorderAndShadow: yes\n"); script_info = false } else if line.trim() == "[Script Info]" { script_info = true From 892407d1f069b1dfcaa71c4eeca1c942ff40872c Mon Sep 17 00:00:00 2001 From: ByteDream Date: Fri, 6 Jan 2023 00:03:57 +0100 Subject: [PATCH 004/342] Fix --default-subtitle causing no such file error (#98) --- crunchy-cli-core/src/cli/archive.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 3910d24..3eb92a7 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -632,10 +632,10 @@ fn generate_mkv( if let Some(default_subtitle) = &archive.default_subtitle { // if `--default_subtitle ` is given set the default subtitle to the given locale if let Some(position) = subtitle_paths - .into_iter() - .position(|s| &s.1.locale == default_subtitle) + .iter() + .position(|(_, subtitle)| &subtitle.locale == default_subtitle) { - command_args.push(format!("-disposition:s:{}", position)) + command_args.extend([format!("-disposition:s:{}", position), "default".to_string()]) } else { command_args.extend(["-disposition:s:0".to_string(), "0".to_string()]) } From 06fd9a7a98989aa4a2c84c120b96b0477f9cf150 Mon Sep 17 00:00:00 2001 From: Hannes Braun Date: Wed, 4 Jan 2023 22:09:31 +0100 Subject: [PATCH 005/342] Archive subtitles of all versions of an episode --- crunchy-cli-core/src/cli/archive.rs | 95 +++++++++++++++++++------- crunchy-cli-core/src/utils/mod.rs | 1 + crunchy-cli-core/src/utils/subtitle.rs | 11 +++ 3 files changed, 82 insertions(+), 25 deletions(-) create mode 100644 crunchy-cli-core/src/utils/subtitle.rs diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 3eb92a7..8259ab3 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -6,6 +6,7 @@ use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file, tempfile}; use crate::utils::parse::{parse_url, UrlFilter}; use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number}; +use crate::utils::subtitle::Subtitle; use crate::Execute; use anyhow::{bail, Result}; use chrono::NaiveTime; @@ -232,7 +233,7 @@ impl Execute for Archive { } } - for (formats, subtitles) in archive_formats { + for (formats, mut subtitles) in archive_formats { let (primary, additionally) = formats.split_first().unwrap(); let mut path = PathBuf::from(&self.output); @@ -276,13 +277,14 @@ impl Execute for Archive { "Subtitle: {}", subtitles .iter() + .filter(|s| s.primary) // Don't print subtitles of non-primary streams. They might get removed depending on the merge behavior. .map(|s| { if let Some(default) = &self.default_subtitle { - if default == &s.locale { + if default == &s.stream_subtitle.locale { return format!("{} (primary)", default); } } - s.locale.to_string() + s.stream_subtitle.locale.to_string() }) .collect::>() .join(", ") @@ -309,13 +311,23 @@ impl Execute for Archive { } else { video_paths.push((path, additional)) } + + // Remove subtitles of deleted video + if only_audio { + subtitles.retain(|s| s.episode_id != additional.id); + } } let (primary_video, _) = video_paths.get(0).unwrap(); let primary_video_length = get_video_length(primary_video.to_path_buf()).unwrap(); for subtitle in subtitles { subtitle_paths.push(( - download_subtitle(&self, subtitle.clone(), primary_video_length).await?, + download_subtitle( + &self, + subtitle.stream_subtitle.clone(), + primary_video_length, + ) + .await?, subtitle, )) } @@ -334,7 +346,7 @@ async fn formats_from_series( archive: &Archive, series: Media, url_filter: &UrlFilter, -) -> Result, Vec)>> { +) -> Result, Vec)>> { let mut seasons = series.seasons().await?; // filter any season out which does not contain the specified audio languages @@ -367,8 +379,8 @@ async fn formats_from_series( } #[allow(clippy::type_complexity)] - let mut result: BTreeMap, Vec)>> = - BTreeMap::new(); + let mut result: BTreeMap, Vec)>> = BTreeMap::new(); + let mut primary_season = true; for season in series.seasons().await? { if !url_filter.is_season_valid(season.metadata.season_number) || !archive @@ -402,20 +414,26 @@ async fn formats_from_series( ) }; - let (ref mut formats, _) = result + let (ref mut formats, subtitles) = result .entry(season.metadata.season_number) .or_insert_with(BTreeMap::new) .entry(episode.metadata.episode_number) - .or_insert_with(|| { - let subtitles: Vec = archive - .subtitle - .iter() - .filter_map(|l| streams.subtitles.get(l).cloned()) - .collect(); - (vec![], subtitles) - }); + .or_insert_with(|| (vec![], vec![])); + subtitles.extend(archive.subtitle.iter().filter_map(|l| { + let stream_subtitle = streams.subtitles.get(l).cloned()?; + let subtitle = Subtitle { + stream_subtitle, + audio_locale: episode.metadata.audio_locale.clone(), + episode_id: episode.id.clone(), + forced: !episode.metadata.is_subbed, + primary: primary_season, + }; + Some(subtitle) + })); formats.push(Format::new_from_episode(episode, stream)); } + + primary_season = false; } Ok(result.into_values().flat_map(|v| v.into_values()).collect()) @@ -562,11 +580,12 @@ fn generate_mkv( target: PathBuf, video_paths: Vec<(TempPath, &Format)>, audio_paths: Vec<(TempPath, &Format)>, - subtitle_paths: Vec<(TempPath, StreamSubtitle)>, + subtitle_paths: Vec<(TempPath, Subtitle)>, ) -> Result<()> { let mut input = vec![]; let mut maps = vec![]; let mut metadata = vec![]; + let mut dispositions = vec![vec![]; subtitle_paths.len()]; for (i, (video_path, format)) in video_paths.iter().enumerate() { input.extend(["-i".to_string(), video_path.to_string_lossy().to_string()]); @@ -611,12 +630,26 @@ fn generate_mkv( ]); metadata.extend([ format!("-metadata:s:s:{}", i), - format!("language={}", subtitle.locale), + format!("language={}", subtitle.stream_subtitle.locale), ]); metadata.extend([ format!("-metadata:s:s:{}", i), - format!("title={}", subtitle.locale.to_human_readable()), + format!( + "title={}", + subtitle.stream_subtitle.locale.to_human_readable() + + if !subtitle.primary { + format!(" [Video: {}]", subtitle.audio_locale.to_human_readable()) + } else { + "".to_string() + } + .as_str() + ), ]); + + // mark forced subtitles + if subtitle.forced { + dispositions[i].push("forced"); + } } let (input_presets, output_presets) = @@ -633,16 +666,28 @@ fn generate_mkv( // if `--default_subtitle ` is given set the default subtitle to the given locale if let Some(position) = subtitle_paths .iter() - .position(|(_, subtitle)| &subtitle.locale == default_subtitle) + .position(|(_, subtitle)| &subtitle.stream_subtitle.locale == default_subtitle) { - command_args.extend([format!("-disposition:s:{}", position), "default".to_string()]) - } else { - command_args.extend(["-disposition:s:0".to_string(), "0".to_string()]) + dispositions[position].push("default"); } - } else { - command_args.extend(["-disposition:s:0".to_string(), "0".to_string()]) } + let disposition_args: Vec = dispositions + .iter() + .enumerate() + .flat_map(|(i, d)| { + vec![ + format!("-disposition:s:{}", i), + if !d.is_empty() { + d.join("+") + } else { + "0".to_string() + }, + ] + }) + .collect(); + command_args.extend(disposition_args); + command_args.extend(output_presets); command_args.extend([ "-f".to_string(), diff --git a/crunchy-cli-core/src/utils/mod.rs b/crunchy-cli-core/src/utils/mod.rs index 5f7a5d2..3b15a89 100644 --- a/crunchy-cli-core/src/utils/mod.rs +++ b/crunchy-cli-core/src/utils/mod.rs @@ -6,3 +6,4 @@ pub mod log; pub mod os; pub mod parse; pub mod sort; +pub mod subtitle; diff --git a/crunchy-cli-core/src/utils/subtitle.rs b/crunchy-cli-core/src/utils/subtitle.rs new file mode 100644 index 0000000..86b9359 --- /dev/null +++ b/crunchy-cli-core/src/utils/subtitle.rs @@ -0,0 +1,11 @@ +use crunchyroll_rs::media::StreamSubtitle; +use crunchyroll_rs::Locale; + +#[derive(Clone)] +pub struct Subtitle { + pub stream_subtitle: StreamSubtitle, + pub audio_locale: Locale, + pub episode_id: String, + pub forced: bool, + pub primary: bool, +} From 7588621f345e5b699985bda9a84cba2226151341 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sat, 7 Jan 2023 16:02:51 +0100 Subject: [PATCH 006/342] Add interactive input to choose season on duplicated season numbers (#55, #82) --- crunchy-cli-core/src/cli/archive.rs | 14 +++- crunchy-cli-core/src/cli/download.rs | 12 ++- crunchy-cli-core/src/cli/utils.rs | 106 ++++++++++++++++++++++++++- 3 files changed, 128 insertions(+), 4 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 8259ab3..2c952d6 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -1,5 +1,5 @@ use crate::cli::log::tab_info; -use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset}; +use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset, find_multiple_seasons_with_same_number, interactive_season_choosing}; use crate::utils::context::Context; use crate::utils::format::{format_string, Format}; use crate::utils::log::progress; @@ -120,6 +120,10 @@ pub struct Archive { #[arg(long)] no_subtitle_optimizations: bool, + #[arg(help = "Ignore interactive input")] + #[arg(short, long, default_value_t = false)] + yes: bool, + #[arg(help = "Crunchyroll series url(s)")] urls: Vec, } @@ -378,10 +382,16 @@ async fn formats_from_series( }) } + if !archive.yes && !find_multiple_seasons_with_same_number(&seasons).is_empty() { + info!(target: "progress_end", "Fetched seasons"); + seasons = interactive_season_choosing(seasons); + info!(target: "progress", "Fetching series details") + } + #[allow(clippy::type_complexity)] let mut result: BTreeMap, Vec)>> = BTreeMap::new(); let mut primary_season = true; - for season in series.seasons().await? { + for season in seasons { if !url_filter.is_season_valid(season.metadata.season_number) || !archive .locale diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index 0411a5e..8da833a 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -1,5 +1,5 @@ use crate::cli::log::tab_info; -use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset}; +use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset, interactive_season_choosing, find_multiple_seasons_with_same_number}; use crate::utils::context::Context; use crate::utils::format::{format_string, Format}; use crate::utils::log::progress; @@ -71,6 +71,10 @@ pub struct Download { #[arg(value_parser = FFmpegPreset::parse)] ffmpeg_preset: Vec, + #[arg(help = "Ignore interactive input")] + #[arg(short, long, default_value_t = false)] + yes: bool, + #[arg(help = "Url(s) to Crunchyroll episodes or series")] urls: Vec, } @@ -348,6 +352,12 @@ async fn formats_from_series( }) } + if !download.yes && !find_multiple_seasons_with_same_number(&seasons).is_empty() { + info!(target: "progress_end", "Fetched seasons"); + seasons = interactive_season_choosing(seasons); + info!(target: "progress", "Fetching series details") + } + let mut formats = vec![]; for season in seasons { if let Some(fmts) = formats_from_season(download, season, url_filter).await? { diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index 26d939c..fde1d08 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -5,9 +5,11 @@ use indicatif::{ProgressBar, ProgressFinish, ProgressStyle}; use log::{debug, LevelFilter}; use std::borrow::{Borrow, BorrowMut}; use std::collections::BTreeMap; -use std::io::Write; +use std::io::{BufRead, Write}; use std::sync::{mpsc, Arc, Mutex}; use std::time::Duration; +use crunchyroll_rs::{Media, Season}; +use regex::Regex; use tokio::task::JoinSet; pub fn find_resolution( @@ -327,3 +329,105 @@ impl FFmpegPreset { )) } } + +pub(crate) fn find_multiple_seasons_with_same_number(seasons: &Vec>) -> Vec { + let mut seasons_map: BTreeMap = BTreeMap::new(); + for season in seasons { + if let Some(s) = seasons_map.get_mut(&season.metadata.season_number) { + *s += 1; + } else { + seasons_map.insert(season.metadata.season_number, 1); + } + } + + seasons_map + .into_iter() + .filter_map(|(k, v)| if v > 1 { Some(k) } else { None }) + .collect() +} + +pub(crate) fn interactive_season_choosing(seasons: Vec>) -> Vec> { + let input_regex = + Regex::new(r"((?P\d+)|(?P\d+)-(?P\d+)?)(\s|$)").unwrap(); + + let mut seasons_map: BTreeMap>> = BTreeMap::new(); + for season in seasons { + if let Some(s) = seasons_map.get_mut(&season.metadata.season_number) { + s.push(season); + } else { + seasons_map.insert(season.metadata.season_number, vec![season]); + } + } + + for (num, season_vec) in seasons_map.iter_mut() { + if season_vec.len() == 1 { + continue; + } + println!(":: Found multiple seasons for season number {}", num); + println!(":: Select the number of the seasons you want to download (eg \"1 2 4\", \"1-3\", \"1-3 5\"):"); + for (i, season) in season_vec.iter().enumerate() { + println!(":: \t{}. {}", i + 1, season.title) + } + let mut stdout = std::io::stdout(); + let _ = write!(stdout, ":: => "); + let _ = stdout.flush(); + let mut user_input = String::new(); + std::io::stdin().lock() + .read_line(&mut user_input) + .expect("cannot open stdin"); + + let mut nums = vec![]; + for capture in input_regex.captures_iter(&user_input) { + if let Some(single) = capture.name("single") { + nums.push(single.as_str().parse().unwrap()); + } else { + let range_from = capture.name("range_from"); + let range_to = capture.name("range_to"); + + // input is '-' which means use all seasons + if range_from.is_none() && range_to.is_none() { + nums = vec![]; + break; + } + let from = range_from + .map(|f| f.as_str().parse::().unwrap() - 1) + .unwrap_or(usize::MIN); + let to = range_from + .map(|f| f.as_str().parse::().unwrap() - 1) + .unwrap_or(usize::MAX); + + nums.extend( + season_vec + .iter() + .enumerate() + .filter_map(|(i, _)| { + if i >= from && i <= to { + Some(i) + } else { + None + } + }) + .collect::>(), + ) + } + } + nums.dedup(); + + if !nums.is_empty() { + let mut remove_count = 0; + for i in 0..season_vec.len() - 1 { + if !nums.contains(&i) { + season_vec.remove(i - remove_count); + remove_count += 1 + } + } + } + } + + seasons_map + .into_values() + .into_iter() + .flatten() + .collect::>>() +} + From b991614dc38e8d36ffb2026a91429e0c686fc855 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sat, 7 Jan 2023 17:14:47 +0100 Subject: [PATCH 007/342] Fix output and download order on duplicated seasons --- crunchy-cli-core/src/utils/sort.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crunchy-cli-core/src/utils/sort.rs b/crunchy-cli-core/src/utils/sort.rs index 089fe18..21df74e 100644 --- a/crunchy-cli-core/src/utils/sort.rs +++ b/crunchy-cli-core/src/utils/sort.rs @@ -30,8 +30,11 @@ pub fn sort_formats_after_seasons(formats: Vec) -> Vec> { let mut as_map = BTreeMap::new(); for format in formats { - as_map.entry(format.season_number).or_insert_with(Vec::new); - as_map.get_mut(&format.season_number).unwrap().push(format); + // the season title is used as key instead of season number to distinguish duplicated season + // numbers which are actually two different seasons; season id is not used as this somehow + // messes up ordering when duplicated seasons exist + as_map.entry(format.season_title.clone()).or_insert_with(Vec::new); + as_map.get_mut(&format.season_title).unwrap().push(format); } let mut sorted = as_map @@ -41,7 +44,7 @@ pub fn sort_formats_after_seasons(formats: Vec) -> Vec> { values }) .collect::>>(); - sorted.sort_by(|a, b| a[0].series_id.cmp(&b[0].series_id)); + sorted.sort_by(|a, b| a[0].season_number.cmp(&b[0].season_number)); sorted } From b65c0e9dfdefb49add0294e136912729ae308ee4 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 8 Jan 2023 17:44:31 +0100 Subject: [PATCH 008/342] Update dependencies & version --- Cargo.lock | 32 ++++++++++++++++---------------- Cargo.toml | 4 ++-- crunchy-cli-core/Cargo.lock | 30 +++++++++++++++--------------- crunchy-cli-core/Cargo.toml | 4 ++-- 4 files changed, 35 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eb7c6a3..7177ac1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,9 +39,9 @@ checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" [[package]] name = "async-trait" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d1d8ab452a3936018a687b20e6f7cf5363d713b732b8884001317b0e48aa3" +checksum = "705339e0e4a9690e2908d2b3d049d85682cf19fbd5782494498fbf7003a6a282" dependencies = [ "proc-macro2", "quote", @@ -279,7 +279,7 @@ dependencies = [ [[package]] name = "crunchy-cli" -version = "3.0.0-dev.6" +version = "3.0.0-dev.7" dependencies = [ "chrono", "clap", @@ -291,7 +291,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-dev.6" +version = "3.0.0-dev.7" dependencies = [ "anyhow", "async-trait", @@ -395,9 +395,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add3fc1717409d029b20c5b6903fc0c0b02fa6741d820054f4a2efa5e5816fd" +checksum = "51d1075c37807dcf850c379432f0df05ba52cc30f279c5cfc43cc221ce7f8579" dependencies = [ "cc", "cxxbridge-flags", @@ -407,9 +407,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c87959ba14bc6fbc61df77c3fcfe180fc32b93538c4f1031dd802ccb5f2ff0" +checksum = "5044281f61b27bc598f2f6647d480aed48d2bf52d6eb0b627d84c0361b17aa70" dependencies = [ "cc", "codespan-reporting", @@ -422,15 +422,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69a3e162fde4e594ed2b07d0f83c6c67b745e7f28ce58c6df5e6b6bef99dfb59" +checksum = "61b50bc93ba22c27b0d31128d2d130a0a6b3d267ae27ef7e4fae2167dfe8781c" [[package]] name = "cxxbridge-macro" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e7e2adeb6a0d4a282e581096b06e1791532b7d576dcde5ccd9382acf55db8e6" +checksum = "39e61fda7e62115119469c7b3591fd913ecca96fb766cfd3f2e2502ab7bc87a5" dependencies = [ "proc-macro2", "quote", @@ -1669,9 +1669,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.23.0" +version = "1.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46" +checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae" dependencies = [ "autocfg", "bytes", @@ -1759,9 +1759,9 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "typenum" diff --git a/Cargo.toml b/Cargo.toml index 76ed950..ec8e4ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,11 +1,11 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-dev.6" +version = "3.0.0-dev.7" edition = "2021" [dependencies] -tokio = { version = "1.23", features = ["macros", "rt-multi-thread", "time"], default-features = false } +tokio = { version = "1.24", features = ["macros", "rt-multi-thread", "time"], default-features = false } crunchy-cli-core = { path = "./crunchy-cli-core" } diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index b5dd8a1..4b1a723 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -39,9 +39,9 @@ checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" [[package]] name = "async-trait" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d1d8ab452a3936018a687b20e6f7cf5363d713b732b8884001317b0e48aa3" +checksum = "705339e0e4a9690e2908d2b3d049d85682cf19fbd5782494498fbf7003a6a282" dependencies = [ "proc-macro2", "quote", @@ -260,7 +260,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-dev.6" +version = "3.0.0-dev.7" dependencies = [ "anyhow", "async-trait", @@ -364,9 +364,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add3fc1717409d029b20c5b6903fc0c0b02fa6741d820054f4a2efa5e5816fd" +checksum = "51d1075c37807dcf850c379432f0df05ba52cc30f279c5cfc43cc221ce7f8579" dependencies = [ "cc", "cxxbridge-flags", @@ -376,9 +376,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c87959ba14bc6fbc61df77c3fcfe180fc32b93538c4f1031dd802ccb5f2ff0" +checksum = "5044281f61b27bc598f2f6647d480aed48d2bf52d6eb0b627d84c0361b17aa70" dependencies = [ "cc", "codespan-reporting", @@ -391,15 +391,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69a3e162fde4e594ed2b07d0f83c6c67b745e7f28ce58c6df5e6b6bef99dfb59" +checksum = "61b50bc93ba22c27b0d31128d2d130a0a6b3d267ae27ef7e4fae2167dfe8781c" [[package]] name = "cxxbridge-macro" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e7e2adeb6a0d4a282e581096b06e1791532b7d576dcde5ccd9382acf55db8e6" +checksum = "39e61fda7e62115119469c7b3591fd913ecca96fb766cfd3f2e2502ab7bc87a5" dependencies = [ "proc-macro2", "quote", @@ -1632,9 +1632,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.23.0" +version = "1.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46" +checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae" dependencies = [ "autocfg", "bytes", @@ -1722,9 +1722,9 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "typenum" diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 566b8c4..a3ae301 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-dev.6" +version = "3.0.0-dev.7" edition = "2021" [dependencies] @@ -23,7 +23,7 @@ serde_json = "1.0" signal-hook = "0.3" tempfile = "3.3" terminal_size = "0.2" -tokio = { version = "1.23", features = ["macros", "rt-multi-thread", "time"] } +tokio = { version = "1.24", features = ["macros", "rt-multi-thread", "time"] } sys-locale = "0.2" [build-dependencies] From 13f54c0da636e7093d347bba5c126f8e895ebfe1 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 8 Jan 2023 18:06:37 +0100 Subject: [PATCH 009/342] Fix interactive season choosing activation on url filter excluded seasons --- crunchy-cli-core/src/cli/archive.rs | 5 ++++- crunchy-cli-core/src/cli/download.rs | 11 +++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 2c952d6..b9b7de0 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -379,7 +379,10 @@ async fn formats_from_series( .locale .iter() .any(|l| s.metadata.audio_locales.contains(l)) - }) + }); + // remove seasons which match the url filter. this is mostly done to not trigger the + // interactive season choosing when dupilcated seasons are excluded by the filter + seasons.retain(|s| url_filter.is_season_valid(s.metadata.season_number)) } if !archive.yes && !find_multiple_seasons_with_same_number(&seasons).is_empty() { diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index 8da833a..25c1262 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -349,7 +349,10 @@ async fn formats_from_series( seasons.retain(|s| { s.metadata.season_number != season.first().unwrap().metadata.season_number || s.metadata.audio_locales.contains(&download.audio) - }) + }); + // remove seasons which match the url filter. this is mostly done to not trigger the + // interactive season choosing when dupilcated seasons are excluded by the filter + seasons.retain(|s| url_filter.is_season_valid(s.metadata.season_number)) } if !download.yes && !find_multiple_seasons_with_same_number(&seasons).is_empty() { @@ -373,14 +376,14 @@ async fn formats_from_season( season: Media, url_filter: &UrlFilter, ) -> Result>> { - if !season.metadata.audio_locales.contains(&download.audio) { + if !url_filter.is_season_valid(season.metadata.season_number) { + return Ok(None); + } else if !season.metadata.audio_locales.contains(&download.audio) { error!( "Season {} ({}) is not available with {} audio", season.metadata.season_number, season.title, download.audio ); return Ok(None); - } else if !url_filter.is_season_valid(season.metadata.season_number) { - return Ok(None); } let mut formats = vec![]; From 4b33ef02c61d1ce8b60900785868a3445f779e90 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 9 Jan 2023 10:27:28 +0100 Subject: [PATCH 010/342] Fix output formatting for full path (#101) --- crunchy-cli-core/src/cli/archive.rs | 20 ++++++-------------- crunchy-cli-core/src/cli/download.rs | 20 ++++++-------------- crunchy-cli-core/src/utils/format.rs | 9 ++++++--- 3 files changed, 18 insertions(+), 31 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index b9b7de0..f9a6a04 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -1,7 +1,7 @@ use crate::cli::log::tab_info; use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset, find_multiple_seasons_with_same_number, interactive_season_choosing}; use crate::utils::context::Context; -use crate::utils::format::{format_string, Format}; +use crate::utils::format::{Format, format_path}; use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file, tempfile}; use crate::utils::parse::{parse_url, UrlFilter}; @@ -240,19 +240,11 @@ impl Execute for Archive { for (formats, mut subtitles) in archive_formats { let (primary, additionally) = formats.split_first().unwrap(); - let mut path = PathBuf::from(&self.output); - path = free_file( - path.with_file_name(format_string( - if let Some(fname) = path.file_name() { - fname.to_str().unwrap() - } else { - "{title}.mkv" - } - .to_string(), - primary, - true, - )), - ); + let path = free_file(format_path(if self.output.is_empty() { + "{title}.mkv" + } else { + &self.output + }.into(), &primary, true)); info!( "Downloading {} to '{}'", diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index 25c1262..a7ed773 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -1,7 +1,7 @@ use crate::cli::log::tab_info; use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset, interactive_season_choosing, find_multiple_seasons_with_same_number}; use crate::utils::context::Context; -use crate::utils::format::{format_string, Format}; +use crate::utils::format::{Format, format_path}; use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file}; use crate::utils::parse::{parse_url, UrlFilter}; @@ -206,19 +206,11 @@ impl Execute for Download { } for format in formats { - let mut path = PathBuf::from(&self.output); - path = free_file( - path.with_file_name(format_string( - if let Some(fname) = path.file_name() { - fname.to_str().unwrap() - } else { - "{title}.ts" - } - .to_string(), - &format, - true, - )), - ); + let path = free_file(format_path(if self.output.is_empty() { + "{title}.mkv" + } else { + &self.output + }.into(), &format, true)); info!( "Downloading {} to '{}'", diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index c463ea8..ee48e2a 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -1,3 +1,4 @@ +use std::path::PathBuf; use crunchyroll_rs::media::VariantData; use crunchyroll_rs::{Episode, Locale, Media, Movie}; use std::time::Duration; @@ -65,7 +66,7 @@ impl Format { /// Formats the given string if it has specific pattern in it. It's possible to sanitize it which /// removes characters which can cause failures if the output string is used as a file name. -pub fn format_string(s: String, format: &Format, sanitize: bool) -> String { +pub fn format_path(path: PathBuf, format: &Format, sanitize: bool) -> PathBuf { let sanitize_func = if sanitize { |s: &str| sanitize_filename::sanitize(s) } else { @@ -73,7 +74,9 @@ pub fn format_string(s: String, format: &Format, sanitize: bool) -> String { |s: &str| s.to_string() }; - s.replace("{title}", &sanitize_func(&format.title)) + let as_string = path.to_string_lossy().to_string(); + + PathBuf::from(as_string.replace("{title}", &sanitize_func(&format.title)) .replace("{series_name}", &sanitize_func(&format.series_name)) .replace("{season_name}", &sanitize_func(&format.season_title)) .replace("{audio}", &sanitize_func(&format.audio.to_string())) @@ -99,5 +102,5 @@ pub fn format_string(s: String, format: &Format, sanitize: bool) -> String { ) .replace("{series_id}", &sanitize_func(&format.series_id)) .replace("{season_id}", &sanitize_func(&format.season_id)) - .replace("{episode_id}", &sanitize_func(&format.id)) + .replace("{episode_id}", &sanitize_func(&format.id))) } From 12be16417ffdd0bda87b9c06aaf7f50d5454b178 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 9 Jan 2023 16:55:10 +0100 Subject: [PATCH 011/342] Fix interactive season choosing false-positive triggering --- Cargo.lock | 85 +++++++--------------------- crunchy-cli-core/Cargo.lock | 85 +++++++--------------------- crunchy-cli-core/Cargo.toml | 1 + crunchy-cli-core/src/cli/archive.rs | 31 +++++----- crunchy-cli-core/src/cli/download.rs | 22 ++++--- crunchy-cli-core/src/cli/utils.rs | 67 ++++++++++++++++------ crunchy-cli-core/src/utils/format.rs | 59 ++++++++++--------- crunchy-cli-core/src/utils/sort.rs | 4 +- 8 files changed, 157 insertions(+), 197 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7177ac1..77e1342 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -222,7 +222,7 @@ dependencies = [ "lazy_static", "libc", "unicode-width", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -302,6 +302,7 @@ dependencies = [ "ctrlc", "dirs", "indicatif", + "lazy_static", "log", "num_cpus", "regex", @@ -390,7 +391,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1631ca6e3c59112501a9d87fd86f21591ff77acd31331e8a73f8d80a65bbdd71" dependencies = [ "nix", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -859,14 +860,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" dependencies = [ "libc", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] name = "ipnet" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11b0d96e660696543b251e58030cf9787df56da39dab19ad60eae7353040917e" +checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" [[package]] name = "is-terminal" @@ -877,7 +878,7 @@ dependencies = [ "hermit-abi", "io-lifetimes", "rustix", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -980,7 +981,7 @@ dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1231,9 +1232,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" +checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" dependencies = [ "aho-corasick", "memchr", @@ -1338,7 +1339,7 @@ dependencies = [ "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1380,12 +1381,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" +checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" dependencies = [ - "lazy_static", - "windows-sys 0.36.1", + "windows-sys", ] [[package]] @@ -1591,7 +1591,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb20089a8ba2b69debd491f8d2d023761cbf196e999218c591fa1e7e15a21907" dependencies = [ "rustix", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1682,7 +1682,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio-macros", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1973,19 +1973,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" -dependencies = [ - "windows_aarch64_msvc 0.36.1", - "windows_i686_gnu 0.36.1", - "windows_i686_msvc 0.36.1", - "windows_x86_64_gnu 0.36.1", - "windows_x86_64_msvc 0.36.1", -] - [[package]] name = "windows-sys" version = "0.42.0" @@ -1993,12 +1980,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc 0.42.0", - "windows_i686_gnu 0.42.0", - "windows_i686_msvc 0.42.0", - "windows_x86_64_gnu 0.42.0", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", "windows_x86_64_gnullvm", - "windows_x86_64_msvc 0.42.0", + "windows_x86_64_msvc", ] [[package]] @@ -2007,48 +1994,24 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" -[[package]] -name = "windows_aarch64_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" - [[package]] name = "windows_aarch64_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" -[[package]] -name = "windows_i686_gnu" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" - [[package]] name = "windows_i686_gnu" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" -[[package]] -name = "windows_i686_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" - [[package]] name = "windows_i686_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" -[[package]] -name = "windows_x86_64_gnu" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" - [[package]] name = "windows_x86_64_gnu" version = "0.42.0" @@ -2061,12 +2024,6 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" -[[package]] -name = "windows_x86_64_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" - [[package]] name = "windows_x86_64_msvc" version = "0.42.0" diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index 4b1a723..5bdabf0 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -203,7 +203,7 @@ dependencies = [ "lazy_static", "libc", "unicode-width", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -271,6 +271,7 @@ dependencies = [ "ctrlc", "dirs", "indicatif", + "lazy_static", "log", "num_cpus", "regex", @@ -359,7 +360,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1631ca6e3c59112501a9d87fd86f21591ff77acd31331e8a73f8d80a65bbdd71" dependencies = [ "nix", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -828,14 +829,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" dependencies = [ "libc", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] name = "ipnet" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11b0d96e660696543b251e58030cf9787df56da39dab19ad60eae7353040917e" +checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" [[package]] name = "is-terminal" @@ -846,7 +847,7 @@ dependencies = [ "hermit-abi", "io-lifetimes", "rustix", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -949,7 +950,7 @@ dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1200,9 +1201,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" +checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" dependencies = [ "aho-corasick", "memchr", @@ -1301,7 +1302,7 @@ dependencies = [ "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1343,12 +1344,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" +checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" dependencies = [ - "lazy_static", - "windows-sys 0.36.1", + "windows-sys", ] [[package]] @@ -1554,7 +1554,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb20089a8ba2b69debd491f8d2d023761cbf196e999218c591fa1e7e15a21907" dependencies = [ "rustix", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1645,7 +1645,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio-macros", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1936,19 +1936,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" -dependencies = [ - "windows_aarch64_msvc 0.36.1", - "windows_i686_gnu 0.36.1", - "windows_i686_msvc 0.36.1", - "windows_x86_64_gnu 0.36.1", - "windows_x86_64_msvc 0.36.1", -] - [[package]] name = "windows-sys" version = "0.42.0" @@ -1956,12 +1943,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc 0.42.0", - "windows_i686_gnu 0.42.0", - "windows_i686_msvc 0.42.0", - "windows_x86_64_gnu 0.42.0", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", "windows_x86_64_gnullvm", - "windows_x86_64_msvc 0.42.0", + "windows_x86_64_msvc", ] [[package]] @@ -1970,48 +1957,24 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" -[[package]] -name = "windows_aarch64_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" - [[package]] name = "windows_aarch64_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" -[[package]] -name = "windows_i686_gnu" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" - [[package]] name = "windows_i686_gnu" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" -[[package]] -name = "windows_i686_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" - [[package]] name = "windows_i686_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" -[[package]] -name = "windows_x86_64_gnu" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" - [[package]] name = "windows_x86_64_gnu" version = "0.42.0" @@ -2024,12 +1987,6 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" -[[package]] -name = "windows_x86_64_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" - [[package]] name = "windows_x86_64_msvc" version = "0.42.0" diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index a3ae301..5bda1db 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -14,6 +14,7 @@ csv = "1.1" ctrlc = "3.2" dirs = "4.0" indicatif = "0.17" +lazy_static = "1.4" log = { version = "0.4", features = ["std"] } num_cpus = "1.15" regex = "1.7" diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index f9a6a04..7a96c05 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -1,7 +1,10 @@ use crate::cli::log::tab_info; -use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset, find_multiple_seasons_with_same_number, interactive_season_choosing}; +use crate::cli::utils::{ + download_segments, find_multiple_seasons_with_same_number, find_resolution, + interactive_season_choosing, FFmpegPreset, +}; use crate::utils::context::Context; -use crate::utils::format::{Format, format_path}; +use crate::utils::format::{format_path, Format}; use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file, tempfile}; use crate::utils::parse::{parse_url, UrlFilter}; @@ -240,11 +243,16 @@ impl Execute for Archive { for (formats, mut subtitles) in archive_formats { let (primary, additionally) = formats.split_first().unwrap(); - let path = free_file(format_path(if self.output.is_empty() { - "{title}.mkv" - } else { - &self.output - }.into(), &primary, true)); + let path = free_file(format_path( + if self.output.is_empty() { + "{title}.mkv" + } else { + &self.output + } + .into(), + &primary, + true, + )); info!( "Downloading {} to '{}'", @@ -387,15 +395,6 @@ async fn formats_from_series( let mut result: BTreeMap, Vec)>> = BTreeMap::new(); let mut primary_season = true; for season in seasons { - if !url_filter.is_season_valid(season.metadata.season_number) - || !archive - .locale - .iter() - .any(|l| season.metadata.audio_locales.contains(l)) - { - continue; - } - for episode in season.episodes().await? { if !url_filter.is_episode_valid( episode.metadata.episode_number, diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index a7ed773..bdca291 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -1,7 +1,10 @@ use crate::cli::log::tab_info; -use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset, interactive_season_choosing, find_multiple_seasons_with_same_number}; +use crate::cli::utils::{ + download_segments, find_multiple_seasons_with_same_number, find_resolution, + interactive_season_choosing, FFmpegPreset, +}; use crate::utils::context::Context; -use crate::utils::format::{Format, format_path}; +use crate::utils::format::{format_path, Format}; use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file}; use crate::utils::parse::{parse_url, UrlFilter}; @@ -206,11 +209,16 @@ impl Execute for Download { } for format in formats { - let path = free_file(format_path(if self.output.is_empty() { - "{title}.mkv" - } else { - &self.output - }.into(), &format, true)); + let path = free_file(format_path( + if self.output.is_empty() { + "{title}.mkv" + } else { + &self.output + } + .into(), + &format, + true, + )); info!( "Downloading {} to '{}'", diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index fde1d08..bc19b07 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -1,15 +1,16 @@ use crate::utils::context::Context; use anyhow::{bail, Result}; use crunchyroll_rs::media::{Resolution, VariantData, VariantSegment}; +use crunchyroll_rs::{Media, Season}; use indicatif::{ProgressBar, ProgressFinish, ProgressStyle}; +use lazy_static::lazy_static; use log::{debug, LevelFilter}; +use regex::Regex; use std::borrow::{Borrow, BorrowMut}; use std::collections::BTreeMap; use std::io::{BufRead, Write}; use std::sync::{mpsc, Arc, Mutex}; use std::time::Duration; -use crunchyroll_rs::{Media, Season}; -use regex::Regex; use tokio::task::JoinSet; pub fn find_resolution( @@ -111,7 +112,7 @@ pub async fn download_segments( }; buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec(); - + let mut c = thread_count.lock().unwrap(); debug!( "Downloaded and decrypted segment [{}/{} {:.2}%] {}", @@ -120,14 +121,14 @@ pub async fn download_segments( ((*c + 1) as f64 / total_segments as f64) * 100f64, segment.url ); - + thread_sender.send((num as i32 + (i * cpus) as i32, buf))?; - + *c += 1; } Ok(()) }; - + let result = download().await; if result.is_err() { @@ -149,7 +150,7 @@ pub async fn download_segments( for (pos, bytes) in receiver.iter() { // if the position is lower than 0, an error occured in the sending download thread if pos < 0 { - break + break; } if let Some(p) = &progress { @@ -330,6 +331,10 @@ impl FFmpegPreset { } } +lazy_static! { + static ref DUPLICATED_SEASONS_MULTILANG_REGEX: Regex = Regex::new(r"(-castilian|-english|-english-in|-french|-german|-hindi|-italian|-portuguese|-russian|-spanish)$").unwrap(); +} + pub(crate) fn find_multiple_seasons_with_same_number(seasons: &Vec>) -> Vec { let mut seasons_map: BTreeMap = BTreeMap::new(); for season in seasons { @@ -342,7 +347,25 @@ pub(crate) fn find_multiple_seasons_with_same_number(seasons: &Vec seasons_map .into_iter() - .filter_map(|(k, v)| if v > 1 { Some(k) } else { None }) + .filter_map(|(k, v)| { + if v > 1 { + // check if the different seasons are actual the same but with different dub languages + let mut multilang_season_vec: Vec = seasons + .iter() + .map(|s| { + DUPLICATED_SEASONS_MULTILANG_REGEX + .replace(s.slug_title.trim_end_matches("-dub"), "") + .to_string() + }) + .collect(); + multilang_season_vec.dedup(); + + if multilang_season_vec.len() > 1 { + return Some(k); + } + } + None + }) .collect() } @@ -363,6 +386,22 @@ pub(crate) fn interactive_season_choosing(seasons: Vec>) -> Vec = season_vec + .iter() + .map(|s| { + DUPLICATED_SEASONS_MULTILANG_REGEX + .replace(s.slug_title.trim_end_matches("-dub"), "") + .to_string() + }) + .collect(); + multilang_season_vec.dedup(); + + if multilang_season_vec.len() == 1 { + continue; + } + println!(":: Found multiple seasons for season number {}", num); println!(":: Select the number of the seasons you want to download (eg \"1 2 4\", \"1-3\", \"1-3 5\"):"); for (i, season) in season_vec.iter().enumerate() { @@ -372,7 +411,8 @@ pub(crate) fn interactive_season_choosing(seasons: Vec>) -> Vec "); let _ = stdout.flush(); let mut user_input = String::new(); - std::io::stdin().lock() + std::io::stdin() + .lock() .read_line(&mut user_input) .expect("cannot open stdin"); @@ -400,13 +440,7 @@ pub(crate) fn interactive_season_choosing(seasons: Vec>) -> Vec= from && i <= to { - Some(i) - } else { - None - } - }) + .filter_map(|(i, _)| if i >= from && i <= to { Some(i) } else { None }) .collect::>(), ) } @@ -430,4 +464,3 @@ pub(crate) fn interactive_season_choosing(seasons: Vec>) -> Vec>>() } - diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index ee48e2a..035a61c 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -1,6 +1,6 @@ -use std::path::PathBuf; use crunchyroll_rs::media::VariantData; use crunchyroll_rs::{Episode, Locale, Media, Movie}; +use std::path::PathBuf; use std::time::Duration; #[derive(Clone)] @@ -76,31 +76,34 @@ pub fn format_path(path: PathBuf, format: &Format, sanitize: bool) -> PathBuf { let as_string = path.to_string_lossy().to_string(); - PathBuf::from(as_string.replace("{title}", &sanitize_func(&format.title)) - .replace("{series_name}", &sanitize_func(&format.series_name)) - .replace("{season_name}", &sanitize_func(&format.season_title)) - .replace("{audio}", &sanitize_func(&format.audio.to_string())) - .replace( - "{resolution}", - &sanitize_func(&format.stream.resolution.to_string()), - ) - .replace( - "{padded_season_number}", - &sanitize_func(&format!("{:0>2}", format.season_number.to_string())), - ) - .replace( - "{season_number}", - &sanitize_func(&format.season_number.to_string()), - ) - .replace( - "{padded_episode_number}", - &sanitize_func(&format!("{:0>2}", format.number.to_string())), - ) - .replace( - "{episode_number}", - &sanitize_func(&format.number.to_string()), - ) - .replace("{series_id}", &sanitize_func(&format.series_id)) - .replace("{season_id}", &sanitize_func(&format.season_id)) - .replace("{episode_id}", &sanitize_func(&format.id))) + PathBuf::from( + as_string + .replace("{title}", &sanitize_func(&format.title)) + .replace("{series_name}", &sanitize_func(&format.series_name)) + .replace("{season_name}", &sanitize_func(&format.season_title)) + .replace("{audio}", &sanitize_func(&format.audio.to_string())) + .replace( + "{resolution}", + &sanitize_func(&format.stream.resolution.to_string()), + ) + .replace( + "{padded_season_number}", + &sanitize_func(&format!("{:0>2}", format.season_number.to_string())), + ) + .replace( + "{season_number}", + &sanitize_func(&format.season_number.to_string()), + ) + .replace( + "{padded_episode_number}", + &sanitize_func(&format!("{:0>2}", format.number.to_string())), + ) + .replace( + "{episode_number}", + &sanitize_func(&format.number.to_string()), + ) + .replace("{series_id}", &sanitize_func(&format.series_id)) + .replace("{season_id}", &sanitize_func(&format.season_id)) + .replace("{episode_id}", &sanitize_func(&format.id)), + ) } diff --git a/crunchy-cli-core/src/utils/sort.rs b/crunchy-cli-core/src/utils/sort.rs index 21df74e..9f8d81c 100644 --- a/crunchy-cli-core/src/utils/sort.rs +++ b/crunchy-cli-core/src/utils/sort.rs @@ -33,7 +33,9 @@ pub fn sort_formats_after_seasons(formats: Vec) -> Vec> { // the season title is used as key instead of season number to distinguish duplicated season // numbers which are actually two different seasons; season id is not used as this somehow // messes up ordering when duplicated seasons exist - as_map.entry(format.season_title.clone()).or_insert_with(Vec::new); + as_map + .entry(format.season_title.clone()) + .or_insert_with(Vec::new); as_map.get_mut(&format.season_title).unwrap().push(format); } From 29845ba6e53ce877d85aca47db840c52835fa61b Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 9 Jan 2023 17:26:04 +0100 Subject: [PATCH 012/342] Re-order instructions --- crunchy-cli-core/src/cli/archive.rs | 8 +++--- crunchy-cli-core/src/cli/download.rs | 10 +++++-- crunchy-cli-core/src/utils/format.rs | 43 +++++++++++++++------------- crunchy-cli-core/src/utils/sort.rs | 2 +- 4 files changed, 35 insertions(+), 28 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 7a96c05..1de0997 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -208,7 +208,7 @@ impl Execute for Archive { format.stream.resolution, format.stream.fps, format.season_number, - format.number, + format.episode_number, ) } } @@ -234,7 +234,7 @@ impl Execute for Archive { format.stream.resolution, format.stream.fps, format.season_number, - format.number + format.episode_number ) } } @@ -266,7 +266,7 @@ impl Execute for Archive { tab_info!( "Episode: S{:02}E{:02}", primary.season_number, - primary.number + primary.episode_number ); tab_info!( "Audio: {} (primary), {}", @@ -318,7 +318,7 @@ impl Execute for Archive { // Remove subtitles of deleted video if only_audio { - subtitles.retain(|s| s.episode_id != additional.id); + subtitles.retain(|s| s.episode_id != additional.episode_id); } } diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index bdca291..14f9503 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -182,7 +182,7 @@ impl Execute for Download { format.stream.resolution, format.stream.fps, format.season_number, - format.number, + format.episode_number, ) } } @@ -202,7 +202,7 @@ impl Execute for Download { format.stream.resolution, format.stream.fps, format.season_number, - format.number + format.episode_number ) } } @@ -229,7 +229,11 @@ impl Execute for Download { path.file_name().unwrap().to_str().unwrap() } ); - tab_info!("Episode: S{:02}E{:02}", format.season_number, format.number); + tab_info!( + "Episode: S{:02}E{:02}", + format.season_number, + format.episode_number + ); tab_info!("Audio: {}", format.audio); tab_info!( "Subtitles: {}", diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index 035a61c..5570960 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -5,10 +5,9 @@ use std::time::Duration; #[derive(Clone)] pub struct Format { - pub id: String, pub title: String, pub description: String, - pub number: u32, + pub audio: Locale, pub duration: Duration, @@ -20,15 +19,17 @@ pub struct Format { pub season_id: String, pub season_title: String, pub season_number: u32, + + pub episode_id: String, + pub episode_number: f32, } impl Format { pub fn new_from_episode(episode: Media, stream: VariantData) -> Self { Self { - id: episode.id, title: episode.title, description: episode.description, - number: episode.metadata.episode_number, + audio: episode.metadata.audio_locale, duration: episode.metadata.duration.to_std().unwrap(), @@ -40,15 +41,17 @@ impl Format { season_id: episode.metadata.season_id, season_title: episode.metadata.season_title, season_number: episode.metadata.season_number, + + episode_id: episode.id, + episode_number: episode.metadata.episode.parse().unwrap_or(episode.metadata.sequence_number), } } pub fn new_from_movie(movie: Media, stream: VariantData) -> Self { Self { - id: movie.id, title: movie.title, description: movie.description, - number: 1, + audio: Locale::ja_JP, duration: movie.metadata.duration.to_std().unwrap(), @@ -60,6 +63,9 @@ impl Format { season_id: movie.metadata.movie_listing_id, season_title: movie.metadata.movie_listing_title, season_number: 1, + + episode_id: movie.id, + episode_number: 1.0, } } } @@ -79,31 +85,28 @@ pub fn format_path(path: PathBuf, format: &Format, sanitize: bool) -> PathBuf { PathBuf::from( as_string .replace("{title}", &sanitize_func(&format.title)) - .replace("{series_name}", &sanitize_func(&format.series_name)) - .replace("{season_name}", &sanitize_func(&format.season_title)) .replace("{audio}", &sanitize_func(&format.audio.to_string())) .replace( "{resolution}", &sanitize_func(&format.stream.resolution.to_string()), ) - .replace( - "{padded_season_number}", - &sanitize_func(&format!("{:0>2}", format.season_number.to_string())), - ) + .replace("{series_id}", &sanitize_func(&format.series_id)) + .replace("{series_name}", &sanitize_func(&format.series_name)) + .replace("{season_id}", &sanitize_func(&format.season_id)) + .replace("{season_name}", &sanitize_func(&format.season_title)) .replace( "{season_number}", &sanitize_func(&format.season_number.to_string()), ) .replace( - "{padded_episode_number}", - &sanitize_func(&format!("{:0>2}", format.number.to_string())), + "{padded_season_number}", + &sanitize_func(&format!("{:0>2}", format.season_number.to_string())), ) + .replace("{episode_id}", &sanitize_func(&format.episode_id)) + .replace("{episode_number}", &sanitize_func(&format.episode_number.to_string())) .replace( - "{episode_number}", - &sanitize_func(&format.number.to_string()), - ) - .replace("{series_id}", &sanitize_func(&format.series_id)) - .replace("{season_id}", &sanitize_func(&format.season_id)) - .replace("{episode_id}", &sanitize_func(&format.id)), + "{padded_episode_number}", + &sanitize_func(&format!("{:0>2}", format.episode_number.to_string())), + ), ) } diff --git a/crunchy-cli-core/src/utils/sort.rs b/crunchy-cli-core/src/utils/sort.rs index 9f8d81c..1af0194 100644 --- a/crunchy-cli-core/src/utils/sort.rs +++ b/crunchy-cli-core/src/utils/sort.rs @@ -42,7 +42,7 @@ pub fn sort_formats_after_seasons(formats: Vec) -> Vec> { let mut sorted = as_map .into_iter() .map(|(_, mut values)| { - values.sort_by(|a, b| a.number.cmp(&b.number)); + values.sort_by(|a, b| a.episode_number.total_cmp(&b.episode_number)); values }) .collect::>>(); From 7d3a90e8112a4ce27158b217a564ded45c08c576 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 9 Jan 2023 19:12:00 +0100 Subject: [PATCH 013/342] Add relative episode number to format --- crunchy-cli-core/src/cli/archive.rs | 29 +++--- crunchy-cli-core/src/cli/download.rs | 54 +++++++--- crunchy-cli-core/src/utils/format.rs | 150 ++++++++++++++++----------- 3 files changed, 145 insertions(+), 88 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 1de0997..96d0737 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -4,7 +4,7 @@ use crate::cli::utils::{ interactive_season_choosing, FFmpegPreset, }; use crate::utils::context::Context; -use crate::utils::format::{format_path, Format}; +use crate::utils::format::Format; use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file, tempfile}; use crate::utils::parse::{parse_url, UrlFilter}; @@ -243,16 +243,17 @@ impl Execute for Archive { for (formats, mut subtitles) in archive_formats { let (primary, additionally) = formats.split_first().unwrap(); - let path = free_file(format_path( - if self.output.is_empty() { - "{title}.mkv" - } else { - &self.output - } - .into(), - &primary, - true, - )); + let path = free_file( + primary.format_path( + if self.output.is_empty() { + "{title}.mkv" + } else { + &self.output + } + .into(), + true, + ), + ); info!( "Downloading {} to '{}'", @@ -395,7 +396,9 @@ async fn formats_from_series( let mut result: BTreeMap, Vec)>> = BTreeMap::new(); let mut primary_season = true; for season in seasons { - for episode in season.episodes().await? { + let episodes = season.episodes().await?; + + for episode in episodes.iter() { if !url_filter.is_episode_valid( episode.metadata.episode_number, episode.metadata.season_number, @@ -434,7 +437,7 @@ async fn formats_from_series( }; Some(subtitle) })); - formats.push(Format::new_from_episode(episode, stream)); + formats.push(Format::new_from_episode(episode, &episodes, stream)); } primary_season = false; diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index 14f9503..d027df7 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -4,7 +4,7 @@ use crate::cli::utils::{ interactive_season_choosing, FFmpegPreset, }; use crate::utils::context::Context; -use crate::utils::format::{format_path, Format}; +use crate::utils::format::Format; use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file}; use crate::utils::parse::{parse_url, UrlFilter}; @@ -16,6 +16,7 @@ use crunchyroll_rs::{ Episode, Locale, Media, MediaCollection, Movie, MovieListing, Season, Series, }; use log::{debug, error, info, warn}; +use std::borrow::Cow; use std::fs::File; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; @@ -148,7 +149,7 @@ impl Execute for Download { episode.metadata.season_title, episode.metadata.series_title ); - format_from_episode(&self, episode, &url_filter, false) + format_from_episode(&self, &episode, &url_filter, None, false) .await? .map(|fmt| vec![fmt]) } @@ -209,16 +210,17 @@ impl Execute for Download { } for format in formats { - let path = free_file(format_path( - if self.output.is_empty() { - "{title}.mkv" - } else { - &self.output - } - .into(), - &format, - true, - )); + let path = free_file( + format.format_path( + if self.output.is_empty() { + "{title}.mkv" + } else { + &self.output + } + .into(), + true, + ), + ); info!( "Downloading {} to '{}'", @@ -392,8 +394,11 @@ async fn formats_from_season( let mut formats = vec![]; - for episode in season.episodes().await? { - if let Some(fmt) = format_from_episode(download, episode, url_filter, true).await? { + let episodes = season.episodes().await?; + for episode in episodes.iter() { + if let Some(fmt) = + format_from_episode(download, &episode, url_filter, Some(&episodes), true).await? + { formats.push(fmt) } } @@ -403,8 +408,9 @@ async fn formats_from_season( async fn format_from_episode( download: &Download, - episode: Media, + episode: &Media, url_filter: &UrlFilter, + season_episodes: Option<&Vec>>, filter_audio: bool, ) -> Result> { if filter_audio && episode.metadata.audio_locale != download.audio { @@ -457,7 +463,21 @@ async fn format_from_episode( ) }; - Ok(Some(Format::new_from_episode(episode, stream))) + let season_eps = if Format::has_relative_episodes_fmt(&download.output) { + if let Some(eps) = season_episodes { + Cow::from(eps) + } else { + Cow::from(episode.season().await?.episodes().await?) + } + } else { + Cow::from(vec![]) + }; + + Ok(Some(Format::new_from_episode( + episode, + &season_eps.to_vec(), + stream, + ))) } async fn format_from_movie_listing( @@ -515,7 +535,7 @@ async fn format_from_movie( } }; - Ok(Some(Format::new_from_movie(movie, stream))) + Ok(Some(Format::new_from_movie(&movie, stream))) } fn some_vec_or_none(v: Vec) -> Option> { diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index 5570960..3db28c3 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -1,5 +1,6 @@ use crunchyroll_rs::media::VariantData; use crunchyroll_rs::{Episode, Locale, Media, Movie}; +use log::warn; use std::path::PathBuf; use std::time::Duration; @@ -22,35 +23,56 @@ pub struct Format { pub episode_id: String, pub episode_number: f32, + pub relative_episode_number: f32, } impl Format { - pub fn new_from_episode(episode: Media, stream: VariantData) -> Self { + pub fn new_from_episode( + episode: &Media, + season_episodes: &Vec>, + stream: VariantData, + ) -> Self { Self { - title: episode.title, - description: episode.description, + title: episode.title.clone(), + description: episode.description.clone(), - audio: episode.metadata.audio_locale, + audio: episode.metadata.audio_locale.clone(), duration: episode.metadata.duration.to_std().unwrap(), stream, - series_id: episode.metadata.series_id, - series_name: episode.metadata.series_title, + series_id: episode.metadata.series_id.clone(), + series_name: episode.metadata.series_title.clone(), - season_id: episode.metadata.season_id, - season_title: episode.metadata.season_title, - season_number: episode.metadata.season_number, + season_id: episode.metadata.season_id.clone(), + season_title: episode.metadata.season_title.clone(), + season_number: episode.metadata.season_number.clone(), - episode_id: episode.id, - episode_number: episode.metadata.episode.parse().unwrap_or(episode.metadata.sequence_number), + episode_id: episode.id.clone(), + episode_number: episode + .metadata + .episode + .parse() + .unwrap_or(episode.metadata.sequence_number), + relative_episode_number: season_episodes + .iter() + .enumerate() + .find_map(|(i, e)| if e == episode { Some((i + 1) as f32) } else { None }) + .unwrap_or_else(|| { + warn!("Cannot find relative episode number for episode {} ({}) of season {} ({}) of {}, using normal episode number", episode.metadata.episode_number, episode.title, episode.metadata.season_number, episode.metadata.season_title, episode.metadata.series_title); + episode + .metadata + .episode + .parse() + .unwrap_or(episode.metadata.sequence_number) + }), } } - pub fn new_from_movie(movie: Media, stream: VariantData) -> Self { + pub fn new_from_movie(movie: &Media, stream: VariantData) -> Self { Self { - title: movie.title, - description: movie.description, + title: movie.title.clone(), + description: movie.description.clone(), audio: Locale::ja_JP, @@ -60,53 +82,65 @@ impl Format { series_id: movie.metadata.movie_listing_id.clone(), series_name: movie.metadata.movie_listing_title.clone(), - season_id: movie.metadata.movie_listing_id, - season_title: movie.metadata.movie_listing_title, + season_id: movie.metadata.movie_listing_id.clone(), + season_title: movie.metadata.movie_listing_title.clone(), season_number: 1, - episode_id: movie.id, + episode_id: movie.id.clone(), episode_number: 1.0, + relative_episode_number: 1.0, } } -} - -/// Formats the given string if it has specific pattern in it. It's possible to sanitize it which -/// removes characters which can cause failures if the output string is used as a file name. -pub fn format_path(path: PathBuf, format: &Format, sanitize: bool) -> PathBuf { - let sanitize_func = if sanitize { - |s: &str| sanitize_filename::sanitize(s) - } else { - // converting this to a string is actually unnecessary - |s: &str| s.to_string() - }; - - let as_string = path.to_string_lossy().to_string(); - - PathBuf::from( - as_string - .replace("{title}", &sanitize_func(&format.title)) - .replace("{audio}", &sanitize_func(&format.audio.to_string())) - .replace( - "{resolution}", - &sanitize_func(&format.stream.resolution.to_string()), - ) - .replace("{series_id}", &sanitize_func(&format.series_id)) - .replace("{series_name}", &sanitize_func(&format.series_name)) - .replace("{season_id}", &sanitize_func(&format.season_id)) - .replace("{season_name}", &sanitize_func(&format.season_title)) - .replace( - "{season_number}", - &sanitize_func(&format.season_number.to_string()), - ) - .replace( - "{padded_season_number}", - &sanitize_func(&format!("{:0>2}", format.season_number.to_string())), - ) - .replace("{episode_id}", &sanitize_func(&format.episode_id)) - .replace("{episode_number}", &sanitize_func(&format.episode_number.to_string())) - .replace( - "{padded_episode_number}", - &sanitize_func(&format!("{:0>2}", format.episode_number.to_string())), - ), - ) + + /// Formats the given string if it has specific pattern in it. It's possible to sanitize it which + /// removes characters which can cause failures if the output string is used as a file name. + pub fn format_path(&self, path: PathBuf, sanitize: bool) -> PathBuf { + let sanitize_func = if sanitize { + |s: &str| sanitize_filename::sanitize(s) + } else { + // converting this to a string is actually unnecessary + |s: &str| s.to_string() + }; + + let as_string = path.to_string_lossy().to_string(); + + PathBuf::from( + as_string + .replace("{title}", &sanitize_func(&self.title)) + .replace("{audio}", &sanitize_func(&self.audio.to_string())) + .replace( + "{resolution}", + &sanitize_func(&self.stream.resolution.to_string()), + ) + .replace("{series_id}", &sanitize_func(&self.series_id)) + .replace("{series_name}", &sanitize_func(&self.series_name)) + .replace("{season_id}", &sanitize_func(&self.season_id)) + .replace("{season_name}", &sanitize_func(&self.season_title)) + .replace( + "{season_number}", + &sanitize_func(&self.season_number.to_string()), + ) + .replace( + "{padded_season_number}", + &sanitize_func(&format!("{:0>2}", self.season_number.to_string())), + ) + .replace("{episode_id}", &sanitize_func(&self.episode_id)) + .replace( + "{episode_number}", + &sanitize_func(&self.episode_number.to_string()), + ) + .replace( + "{padded_episode_number}", + &sanitize_func(&format!("{:0>2}", self.episode_number.to_string())), + ) + .replace( + "{relative_episode_number}", + &sanitize_func(&format!("{:0>2}", self.relative_episode_number.to_string())), + ), + ) + } + + pub fn has_relative_episodes_fmt>(s: S) -> bool { + return s.as_ref().contains("{relative_episode_number}"); + } } From 2ea036d4c6b7faa5bdc33da1eb9a630126ef1e05 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 9 Jan 2023 19:12:31 +0100 Subject: [PATCH 014/342] Remove padded_*_number and make it default for *_number for output format --- crunchy-cli-core/src/utils/format.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index 3db28c3..60596ad 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -118,19 +118,11 @@ impl Format { .replace("{season_name}", &sanitize_func(&self.season_title)) .replace( "{season_number}", - &sanitize_func(&self.season_number.to_string()), - ) - .replace( - "{padded_season_number}", &sanitize_func(&format!("{:0>2}", self.season_number.to_string())), ) .replace("{episode_id}", &sanitize_func(&self.episode_id)) .replace( "{episode_number}", - &sanitize_func(&self.episode_number.to_string()), - ) - .replace( - "{padded_episode_number}", &sanitize_func(&format!("{:0>2}", self.episode_number.to_string())), ) .replace( From a0aab3bfb964195128abb25ed5c0cfa0f4841707 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 9 Jan 2023 23:25:16 +0100 Subject: [PATCH 015/342] Add arabic locale in duplicated seasons check --- crunchy-cli-core/src/cli/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index bc19b07..da56f5e 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -332,7 +332,7 @@ impl FFmpegPreset { } lazy_static! { - static ref DUPLICATED_SEASONS_MULTILANG_REGEX: Regex = Regex::new(r"(-castilian|-english|-english-in|-french|-german|-hindi|-italian|-portuguese|-russian|-spanish)$").unwrap(); + static ref DUPLICATED_SEASONS_MULTILANG_REGEX: Regex = Regex::new(r"(-arabic|-castilian|-english|-english-in|-french|-german|-hindi|-italian|-portuguese|-russian|-spanish)$").unwrap(); } pub(crate) fn find_multiple_seasons_with_same_number(seasons: &Vec>) -> Vec { From 3029325776de09ef159081df847ae67cea3be93d Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 9 Jan 2023 23:40:53 +0100 Subject: [PATCH 016/342] Add check if request locale is valid (#102) --- crunchy-cli-core/src/lib.rs | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index 52a4da1..6cef42e 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -6,7 +6,7 @@ use anyhow::bail; use anyhow::Result; use clap::{Parser, Subcommand}; use crunchyroll_rs::{Crunchyroll, Locale}; -use log::{debug, error, LevelFilter}; +use log::{debug, error, warn, LevelFilter}; use std::{env, fs}; mod cli; @@ -189,8 +189,41 @@ async fn create_ctx(cli: &Cli) -> Result { } async fn crunchyroll_session(cli: &Cli) -> Result { + let supported_langs = vec![ + Locale::ar_ME, + Locale::de_DE, + Locale::en_US, + Locale::es_ES, + Locale::es_419, + Locale::fr_FR, + Locale::it_IT, + Locale::pt_BR, + Locale::pt_PT, + Locale::ru_RU, + ]; + let locale = if let Some(lang) = &cli.lang { + if !supported_langs.contains(lang) { + bail!( + "Via `--lang` specified language is not supported. Supported languages: {}", + supported_langs + .iter() + .map(|l| format!("`{}` ({})", l.to_string(), l.to_human_readable())) + .collect::>() + .join(", ") + ) + } + lang.clone() + } else { + let mut lang = system_locale(); + if !supported_langs.contains(&lang) { + warn!("Recognized system locale is not supported. Using en-US as default. Use `--lang` to overwrite the used language"); + lang = Locale::en_US + } + lang + }; + let builder = Crunchyroll::builder() - .locale(cli.lang.clone().unwrap_or_else(system_locale)) + .locale(locale) .stabilization_locales(true); let login_methods_count = cli.login_method.credentials.is_some() as u8 From 5ce5b249c91fbfa752e4779bb0a9776c740ec88b Mon Sep 17 00:00:00 2001 From: ByteDream Date: Tue, 10 Jan 2023 19:20:08 +0100 Subject: [PATCH 017/342] Add relative episode number to cli help --- crunchy-cli-core/src/cli/archive.rs | 3 +-- crunchy-cli-core/src/cli/download.rs | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 96d0737..768c629 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -67,10 +67,9 @@ pub struct Archive { {season_name} → Name of the season\n \ {audio} → Audio language of the video\n \ {resolution} → Resolution of the video\n \ - {padded_season_number} → Number of the season padded to double digits\n \ {season_number} → Number of the season\n \ - {padded_episode_number} → Number of the episode padded to double digits\n \ {episode_number} → Number of the episode\n \ + {relative_episode_number} → Number of the episode relative to its season\ {series_id} → ID of the series\n \ {season_id} → ID of the season\n \ {episode_id} → ID of the episode")] diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index d027df7..7a49a4c 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -45,10 +45,9 @@ pub struct Download { {season_name} → Name of the season\n \ {audio} → Audio language of the video\n \ {resolution} → Resolution of the video\n \ - {padded_season_number} → Number of the season padded to double digits\n \ {season_number} → Number of the season\n \ - {padded_episode_number} → Number of the episode padded to double digits\n \ {episode_number} → Number of the episode\n \ + {relative_episode_number} → Number of the episode relative to its season\ {series_id} → ID of the series\n \ {season_id} → ID of the season\n \ {episode_id} → ID of the episode")] From 17233f2fd2721dc4f9e1b86957cb404d09f73249 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Tue, 10 Jan 2023 22:15:36 +0100 Subject: [PATCH 018/342] Update dependencies and version --- Cargo.lock | 26 ++++++++++++++++---------- Cargo.toml | 2 +- crunchy-cli-core/Cargo.lock | 24 +++++++++++++++--------- crunchy-cli-core/Cargo.toml | 2 +- 4 files changed, 33 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 77e1342..42fe778 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -60,6 +60,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" + [[package]] name = "bitflags" version = "1.3.2" @@ -279,7 +285,7 @@ dependencies = [ [[package]] name = "crunchy-cli" -version = "3.0.0-dev.7" +version = "3.0.0-dev.8" dependencies = [ "chrono", "clap", @@ -291,7 +297,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-dev.7" +version = "3.0.0-dev.8" dependencies = [ "anyhow", "async-trait", @@ -318,9 +324,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "184d0c725a09aec815316cbf41a2f362008ecb0e8c8e3b6b9930d01a89b5df21" +checksum = "f3770cda4c67e68c689c8e361af46bb9d017caf82263905358fd0751d10657a0" dependencies = [ "aes", "async-trait", @@ -343,9 +349,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f3c82e1766339727fc2c10d66d0c4f001b1cf42e2993f9d93997b610f408776" +checksum = "4a260a73e733bb0ce30343caaed5e968d3c1cc2ea0ab27c601481e9ef22a2fd7" dependencies = [ "darling", "quote", @@ -1268,7 +1274,7 @@ version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" dependencies = [ - "base64", + "base64 0.13.1", "bytes", "cookie", "cookie_store", @@ -1356,11 +1362,11 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0864aeff53f8c05aa08d86e5ef839d3dfcf07aeba2db32f12db0ef716e87bd55" +checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" dependencies = [ - "base64", + "base64 0.21.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index ec8e4ef..db1f129 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-dev.7" +version = "3.0.0-dev.8" edition = "2021" [dependencies] diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index 5bdabf0..b12b3a3 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -60,6 +60,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" + [[package]] name = "bitflags" version = "1.3.2" @@ -260,7 +266,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-dev.7" +version = "3.0.0-dev.8" dependencies = [ "anyhow", "async-trait", @@ -287,9 +293,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "184d0c725a09aec815316cbf41a2f362008ecb0e8c8e3b6b9930d01a89b5df21" +checksum = "f3770cda4c67e68c689c8e361af46bb9d017caf82263905358fd0751d10657a0" dependencies = [ "aes", "async-trait", @@ -312,9 +318,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f3c82e1766339727fc2c10d66d0c4f001b1cf42e2993f9d93997b610f408776" +checksum = "4a260a73e733bb0ce30343caaed5e968d3c1cc2ea0ab27c601481e9ef22a2fd7" dependencies = [ "darling", "quote", @@ -1237,7 +1243,7 @@ version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" dependencies = [ - "base64", + "base64 0.13.1", "bytes", "cookie", "cookie_store", @@ -1319,11 +1325,11 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0864aeff53f8c05aa08d86e5ef839d3dfcf07aeba2db32f12db0ef716e87bd55" +checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" dependencies = [ - "base64", + "base64 0.21.0", ] [[package]] diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 5bda1db..8eea6a8 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-dev.7" +version = "3.0.0-dev.8" edition = "2021" [dependencies] From 6d1f8d49f67aabbc9b1cf577fce38c3209993371 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Fri, 13 Jan 2023 15:21:23 +0100 Subject: [PATCH 019/342] Add hardsubs manually to download videos (#81) --- crunchy-cli-core/src/cli/archive.rs | 151 +------------------------ crunchy-cli-core/src/cli/download.rs | 134 +++++++++++++--------- crunchy-cli-core/src/cli/utils.rs | 20 ++-- crunchy-cli-core/src/utils/format.rs | 6 +- crunchy-cli-core/src/utils/mod.rs | 1 + crunchy-cli-core/src/utils/subtitle.rs | 108 ++++++++++++++++++ crunchy-cli-core/src/utils/video.rs | 25 ++++ 7 files changed, 233 insertions(+), 212 deletions(-) create mode 100644 crunchy-cli-core/src/utils/video.rs diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 768c629..c7030b4 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -9,16 +9,14 @@ use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file, tempfile}; use crate::utils::parse::{parse_url, UrlFilter}; use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number}; -use crate::utils::subtitle::Subtitle; +use crate::utils::subtitle::{download_subtitle, Subtitle}; +use crate::utils::video::get_video_length; use crate::Execute; use anyhow::{bail, Result}; -use chrono::NaiveTime; -use crunchyroll_rs::media::{Resolution, StreamSubtitle}; +use crunchyroll_rs::media::Resolution; use crunchyroll_rs::{Locale, Media, MediaCollection, Series}; use log::{debug, error, info, warn}; -use regex::Regex; use std::collections::BTreeMap; -use std::io::Write; use std::path::PathBuf; use std::process::{Command, Stdio}; use tempfile::TempPath; @@ -113,14 +111,6 @@ pub struct Archive { )] #[arg(long)] default_subtitle: Option, - #[arg(help = "Disable subtitle optimizations")] - #[arg( - long_help = "By default, Crunchyroll delivers subtitles in a format which may cause issues in some video players. \ - These issues are fixed internally by setting a flag which is not part of the official specification of the subtitle format. \ - If you do not want this fixes or they cause more trouble than they solve (for you), it can be disabled with this flag" - )] - #[arg(long)] - no_subtitle_optimizations: bool, #[arg(help = "Ignore interactive input")] #[arg(short, long, default_value_t = false)] @@ -326,12 +316,8 @@ impl Execute for Archive { let primary_video_length = get_video_length(primary_video.to_path_buf()).unwrap(); for subtitle in subtitles { subtitle_paths.push(( - download_subtitle( - &self, - subtitle.stream_subtitle.clone(), - primary_video_length, - ) - .await?, + download_subtitle(subtitle.stream_subtitle.clone(), primary_video_length) + .await?, subtitle, )) } @@ -436,7 +422,7 @@ async fn formats_from_series( }; Some(subtitle) })); - formats.push(Format::new_from_episode(episode, &episodes, stream)); + formats.push(Format::new_from_episode(episode, &episodes, stream, vec![])); } primary_season = false; @@ -476,111 +462,6 @@ async fn download_video(ctx: &Context, format: &Format, only_audio: bool) -> Res Ok(path) } -async fn download_subtitle( - archive: &Archive, - subtitle: StreamSubtitle, - max_length: NaiveTime, -) -> Result { - let tempfile = tempfile(".ass")?; - let (mut file, path) = tempfile.into_parts(); - - let mut buf = vec![]; - subtitle.write_to(&mut buf).await?; - if !archive.no_subtitle_optimizations { - buf = fix_subtitle_look_and_feel(buf) - } - buf = fix_subtitle_length(buf, max_length); - - file.write_all(buf.as_slice())?; - - Ok(path) -} - -/// Add `ScaledBorderAndShadows: yes` to subtitles; without it they look very messy on some video -/// players. See [crunchy-labs/crunchy-cli#66](https://github.com/crunchy-labs/crunchy-cli/issues/66) -/// for more information. -fn fix_subtitle_look_and_feel(raw: Vec) -> Vec { - let mut script_info = false; - let mut new = String::new(); - - for line in String::from_utf8_lossy(raw.as_slice()).split('\n') { - if line.trim().starts_with('[') && script_info { - new.push_str("ScaledBorderAndShadow: yes\n"); - script_info = false - } else if line.trim() == "[Script Info]" { - script_info = true - } - new.push_str(line); - new.push('\n') - } - - new.into_bytes() -} - -/// Fix the length of subtitles to a specified maximum amount. This is required because sometimes -/// subtitles have an unnecessary entry long after the actual video ends with artificially extends -/// the video length on some video players. To prevent this, the video length must be hard set. See -/// [crunchy-labs/crunchy-cli#32](https://github.com/crunchy-labs/crunchy-cli/issues/32) for more -/// information. -fn fix_subtitle_length(raw: Vec, max_length: NaiveTime) -> Vec { - let re = - Regex::new(r#"^Dialogue:\s\d+,(?P\d+:\d+:\d+\.\d+),(?P\d+:\d+:\d+\.\d+),"#) - .unwrap(); - - // chrono panics if we try to format NaiveTime with `%2f` and the nano seconds has more than 2 - // digits so them have to be reduced manually to avoid the panic - fn format_naive_time(native_time: NaiveTime) -> String { - let formatted_time = native_time.format("%f").to_string(); - format!( - "{}.{}", - native_time.format("%T"), - if formatted_time.len() <= 2 { - native_time.format("%2f").to_string() - } else { - formatted_time.split_at(2).0.parse().unwrap() - } - ) - } - - let length_as_string = format_naive_time(max_length); - let mut new = String::new(); - - for line in String::from_utf8_lossy(raw.as_slice()).split('\n') { - if let Some(capture) = re.captures(line) { - let start = capture.name("start").map_or(NaiveTime::default(), |s| { - NaiveTime::parse_from_str(s.as_str(), "%H:%M:%S.%f").unwrap() - }); - let end = capture.name("end").map_or(NaiveTime::default(), |s| { - NaiveTime::parse_from_str(s.as_str(), "%H:%M:%S.%f").unwrap() - }); - - if start > max_length { - continue; - } else if end > max_length { - new.push_str( - re.replace( - line, - format!( - "Dialogue: {},{},", - format_naive_time(start), - &length_as_string - ), - ) - .to_string() - .as_str(), - ) - } else { - new.push_str(line) - } - } else { - new.push_str(line) - } - new.push('\n') - } - - new.into_bytes() -} - fn generate_mkv( archive: &Archive, target: PathBuf, @@ -721,23 +602,3 @@ fn generate_mkv( Ok(()) } - -/// Get the length of a video. This is required because sometimes subtitles have an unnecessary entry -/// long after the actual video ends with artificially extends the video length on some video players. -/// To prevent this, the video length must be hard set. See -/// [crunchy-labs/crunchy-cli#32](https://github.com/crunchy-labs/crunchy-cli/issues/32) for more -/// information. -fn get_video_length(path: PathBuf) -> Result { - let video_length = Regex::new(r"Duration:\s(?P