Refactor

crunchy-cli-core/src/archive/command.rs

@@ -0,0 +1,184 @@
+use crate::archive::filter::ArchiveFilter;
+use crate::utils::context::Context;
+use crate::utils::download::MergeBehavior;
+use crate::utils::ffmpeg::FFmpegPreset;
+use crate::utils::filter::Filter;
+use crate::utils::format::formats_visual_output;
+use crate::utils::locale::all_locale_in_locales;
+use crate::utils::log::progress;
+use crate::utils::os::{free_file, has_ffmpeg, is_special_file};
+use crate::utils::parse::parse_url;
+use crate::Execute;
+use anyhow::bail;
+use anyhow::Result;
+use crunchyroll_rs::media::Resolution;
+use crunchyroll_rs::Locale;
+use log::debug;
+use std::path::PathBuf;
+
+#[derive(Clone, Debug, clap::Parser)]
+#[clap(about = "Archive a video")]
+#[command(arg_required_else_help(true))]
+#[command()]
+pub struct Archive {
+    #[arg(help = format!("Audio languages. Can be used multiple times. \
+    Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::<Vec<String>>().join(", ")))]
+    #[arg(long_help = format!("Audio languages. Can be used multiple times. \
+    Available languages are:\n{}", Locale::all().into_iter().map(|l| format!("{:<6} → {}", l.to_string(), l.to_human_readable())).collect::<Vec<String>>().join("\n  ")))]
+    #[arg(short, long, default_values_t = vec![Locale::ja_JP, crate::utils::locale::system_locale()])]
+    pub(crate) locale: Vec<Locale>,
+    #[arg(help = format!("Subtitle languages. Can be used multiple times. \
+    Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::<Vec<String>>().join(", ")))]
+    #[arg(long_help = format!("Subtitle languages. Can be used multiple times. \
+    Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::<Vec<String>>().join(", ")))]
+    #[arg(short, long, default_values_t = Locale::all())]
+    pub(crate) subtitle: Vec<Locale>,
+
+    #[arg(help = "Name of the output file")]
+    #[arg(long_help = "Name of the output file.\
+    If you use one of the following pattern they will get replaced:\n  \
+      {title}                   → Title of the video\n  \
+      {series_name}             → Name of the series\n  \
+      {season_name}             → Name of the season\n  \
+      {audio}                   → Audio language of the video\n  \
+      {resolution}              → Resolution of the video\n  \
+      {season_number}           → Number of the season\n  \
+      {episode_number}          → Number of the episode\n  \
+      {relative_episode_number} → Number of the episode relative to its season\
+      {series_id}               → ID of the series\n  \
+      {season_id}               → ID of the season\n  \
+      {episode_id}              → ID of the episode")]
+    #[arg(short, long, default_value = "{title}.mkv")]
+    pub(crate) output: String,
+
+    #[arg(help = "Video resolution")]
+    #[arg(long_help = "The video resolution.\
+    Can either be specified via the pixels (e.g. 1920x1080), the abbreviation for pixels (e.g. 1080p) or 'common-use' words (e.g. best). \
+    Specifying the exact pixels is not recommended, use one of the other options instead. \
+    Crunchyroll let you choose the quality with pixel abbreviation on their clients, so you might be already familiar with the available options. \
+    The available common-use words are 'best' (choose the best resolution available) and 'worst' (worst resolution available)")]
+    #[arg(short, long, default_value = "best")]
+    #[arg(value_parser = crate::utils::clap::clap_parse_resolution)]
+    pub(crate) resolution: Resolution,
+
+    #[arg(
+        help = "Sets the behavior of the stream merging. Valid behaviors are 'auto', 'audio' and 'video'"
+    )]
+    #[arg(
+        long_help = "Because of local restrictions (or other reasons) some episodes with different languages does not have the same length (e.g. when some scenes were cut out). \
+    With this flag you can set the behavior when handling multiple language.
+    Valid options are 'audio' (stores one video and all other languages as audio only), 'video' (stores the video + audio for every language) and 'auto' (detects if videos differ in length: if so, behave like 'video' else like 'audio')"
+    )]
+    #[arg(short, long, default_value = "auto")]
+    #[arg(value_parser = MergeBehavior::parse)]
+    pub(crate) merge: MergeBehavior,
+
+    #[arg(help = format!("Presets for video converting. Can be used multiple times. \
+    Available presets: \n  {}", FFmpegPreset::available_matches_human_readable().join("\n  ")))]
+    #[arg(long_help = format!("Presets for video converting. Can be used multiple times. \
+    Generally used to minify the file size with keeping (nearly) the same quality. \
+    It is recommended to only use this if you archive videos with high resolutions since low resolution videos tend to result in a larger file with any of the provided presets. \
+    Available presets: \n  {}", FFmpegPreset::available_matches_human_readable().join("\n  ")))]
+    #[arg(long)]
+    #[arg(value_parser = FFmpegPreset::parse)]
+    pub(crate) ffmpeg_preset: Option<FFmpegPreset>,
+
+    #[arg(
+        help = "Set which subtitle language should be set as default / auto shown when starting a video"
+    )]
+    #[arg(long)]
+    pub(crate) default_subtitle: Option<Locale>,
+
+    #[arg(help = "Skip files which are already existing")]
+    #[arg(long, default_value_t = false)]
+    pub(crate) skip_existing: bool,
+
+    #[arg(help = "Crunchyroll series url(s)")]
+    pub(crate) urls: Vec<String>,
+}
+
+#[async_trait::async_trait(?Send)]
+impl Execute for Archive {
+    fn pre_check(&mut self) -> Result<()> {
+        if !has_ffmpeg() {
+            bail!("FFmpeg is needed to run this command")
+        } else if PathBuf::from(&self.output)
+            .extension()
+            .unwrap_or_default()
+            .to_string_lossy()
+            != "mkv"
+            && !is_special_file(PathBuf::from(&self.output))
+        {
+            bail!("File extension is not '.mkv'. Currently only matroska / '.mkv' files are supported")
+        }
+
+        self.locale = all_locale_in_locales(self.locale.clone());
+        self.subtitle = all_locale_in_locales(self.subtitle.clone());
+
+        Ok(())
+    }
+
+    async fn execute(self, ctx: Context) -> Result<()> {
+        let mut parsed_urls = vec![];
+
+        for (i, url) in self.urls.clone().into_iter().enumerate() {
+            let progress_handler = progress!("Parsing url {}", i + 1);
+            match parse_url(&ctx.crunchy, url.clone(), true).await {
+                Ok((media_collection, url_filter)) => {
+                    progress_handler.stop(format!("Parsed url {}", i + 1));
+                    parsed_urls.push((media_collection, url_filter))
+                }
+                Err(e) => bail!("url {} could not be parsed: {}", url, e),
+            };
+        }
+
+        for (i, (media_collection, url_filter)) in parsed_urls.into_iter().enumerate() {
+            let progress_handler = progress!("Fetching series details");
+            let archive_formats = ArchiveFilter::new(url_filter, self.clone())
+                .visit(media_collection)
+                .await?;
+
+            if archive_formats.is_empty() {
+                progress_handler.stop(format!("Skipping url {} (no matching videos found)", i + 1));
+                continue;
+            }
+            progress_handler.stop(format!("Loaded series information for url {}", i + 1));
+
+            formats_visual_output(archive_formats.iter().map(|(_, f)| f).collect());
+
+            for (downloader, mut format) in archive_formats {
+                let formatted_path = format.format_path((&self.output).into(), true);
+                let (path, changed) = free_file(formatted_path.clone());
+
+                if changed && self.skip_existing {
+                    debug!(
+                        "Skipping already existing file '{}'",
+                        formatted_path.to_string_lossy()
+                    );
+                    continue;
+                }
+
+                format.locales.sort_by(|(a, _), (b, _)| {
+                    self.locale
+                        .iter()
+                        .position(|l| l == a)
+                        .cmp(&self.locale.iter().position(|l| l == b))
+                });
+                for (_, subtitles) in format.locales.iter_mut() {
+                    subtitles.sort_by(|a, b| {
+                        self.subtitle
+                            .iter()
+                            .position(|l| l == a)
+                            .cmp(&self.subtitle.iter().position(|l| l == b))
+                    })
+                }
+
+                format.visual_output(&path);
+
+                downloader.download(&ctx, &path).await?
+            }
+        }
+
+        Ok(())
+    }
+}

crunchy-cli-core/src/archive/filter.rs

@@ -0,0 +1,482 @@
+use crate::archive::command::Archive;
+use crate::utils::download::{DownloadBuilder, DownloadFormat, Downloader, MergeBehavior};
+use crate::utils::filter::{real_dedup_vec, Filter};
+use crate::utils::format::{Format, SingleFormat};
+use crate::utils::parse::UrlFilter;
+use crate::utils::video::variant_data_from_stream;
+use anyhow::{bail, Result};
+use chrono::Duration;
+use crunchyroll_rs::media::{Subtitle, VariantData};
+use crunchyroll_rs::{Concert, Episode, Locale, Movie, MovieListing, MusicVideo, Season, Series};
+use log::warn;
+use std::collections::HashMap;
+use std::hash::Hash;
+
+pub(crate) struct FilterResult {
+    format: SingleFormat,
+    video: VariantData,
+    audio: VariantData,
+    duration: Duration,
+    subtitles: Vec<Subtitle>,
+}
+
+enum Visited {
+    Series,
+    Season,
+    None,
+}
+
+pub(crate) struct ArchiveFilter {
+    url_filter: UrlFilter,
+    archive: Archive,
+    season_episode_count: HashMap<u32, Vec<String>>,
+    season_subtitles_missing: Vec<u32>,
+    visited: Visited,
+}
+
+impl ArchiveFilter {
+    pub(crate) fn new(url_filter: UrlFilter, archive: Archive) -> Self {
+        Self {
+            url_filter,
+            archive,
+            season_episode_count: HashMap::new(),
+            season_subtitles_missing: vec![],
+            visited: Visited::None,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl Filter for ArchiveFilter {
+    type T = Vec<FilterResult>;
+    type Output = (Downloader, Format);
+
+    async fn visit_series(&mut self, series: Series) -> Result<Vec<Season>> {
+        // `series.audio_locales` isn't always populated b/c of crunchyrolls api. so check if the
+        // audio is matching only if the field is populated
+        if !series.audio_locales.is_empty() {
+            let missing_audio = missing_locales(&series.audio_locales, &self.archive.locale);
+            if !missing_audio.is_empty() {
+                warn!(
+                    "Series {} is not available with {} audio",
+                    series.title,
+                    missing_audio
+                        .into_iter()
+                        .map(|l| l.to_string())
+                        .collect::<Vec<String>>()
+                        .join(", ")
+                )
+            }
+            let missing_subtitle =
+                missing_locales(&series.subtitle_locales, &self.archive.subtitle);
+            if !missing_subtitle.is_empty() {
+                warn!(
+                    "Series {} is not available with {} subtitles",
+                    series.title,
+                    missing_subtitle
+                        .into_iter()
+                        .map(|l| l.to_string())
+                        .collect::<Vec<String>>()
+                        .join(", ")
+                )
+            }
+            self.visited = Visited::Series
+        }
+        Ok(series.seasons().await?)
+    }
+
+    async fn visit_season(&mut self, mut season: Season) -> Result<Vec<Episode>> {
+        if !self.url_filter.is_season_valid(season.season_number) {
+            return Ok(vec![]);
+        }
+
+        let mut seasons = season.version(self.archive.locale.clone()).await?;
+        if self
+            .archive
+            .locale
+            .iter()
+            .any(|l| season.audio_locales.contains(l))
+        {
+            seasons.insert(0, season.clone());
+        }
+
+        if !matches!(self.visited, Visited::Series) {
+            let mut audio_locales: Vec<Locale> = seasons
+                .iter()
+                .map(|s| s.audio_locales.clone())
+                .flatten()
+                .collect();
+            real_dedup_vec(&mut audio_locales);
+            let missing_audio = missing_locales(&audio_locales, &self.archive.locale);
+            if !missing_audio.is_empty() {
+                warn!(
+                    "Season {} is not available with {} audio",
+                    season.season_number,
+                    missing_audio
+                        .into_iter()
+                        .map(|l| l.to_string())
+                        .collect::<Vec<String>>()
+                        .join(", ")
+                )
+            }
+
+            let subtitle_locales: Vec<Locale> = seasons
+                .iter()
+                .map(|s| s.subtitle_locales.clone())
+                .flatten()
+                .collect();
+            let missing_subtitle = missing_locales(&subtitle_locales, &self.archive.subtitle);
+            if !missing_subtitle.is_empty() {
+                warn!(
+                    "Season {} is not available with {} subtitles",
+                    season.season_number,
+                    missing_subtitle
+                        .into_iter()
+                        .map(|l| l.to_string())
+                        .collect::<Vec<String>>()
+                        .join(", ")
+                )
+            }
+            self.visited = Visited::Season
+        }
+
+        let mut episodes = vec![];
+        for season in seasons {
+            episodes.extend(season.episodes().await?)
+        }
+
+        if Format::has_relative_episodes_fmt(&self.archive.output) {
+            for episode in episodes.iter() {
+                self.season_episode_count
+                    .entry(episode.season_number)
+                    .or_insert(vec![])
+                    .push(episode.id.clone())
+            }
+        }
+
+        Ok(episodes)
+    }
+
+    async fn visit_episode(&mut self, mut episode: Episode) -> Result<Option<Self::T>> {
+        if !self
+            .url_filter
+            .is_episode_valid(episode.episode_number, episode.season_number)
+        {
+            return Ok(None);
+        }
+
+        let mut episodes = vec![];
+        if !matches!(self.visited, Visited::Series) && !matches!(self.visited, Visited::Season) {
+            episodes.extend(episode.version(self.archive.locale.clone()).await?);
+            let audio_locales: Vec<Locale> =
+                episodes.iter().map(|e| e.audio_locale.clone()).collect();
+            let missing_audio = missing_locales(&audio_locales, &self.archive.locale);
+            if !missing_audio.is_empty() {
+                warn!(
+                    "Episode {} is not available with {} audio",
+                    episode.episode_number,
+                    missing_audio
+                        .into_iter()
+                        .map(|l| l.to_string())
+                        .collect::<Vec<String>>()
+                        .join(", ")
+                )
+            }
+
+            let mut subtitle_locales: Vec<Locale> = episodes
+                .iter()
+                .map(|e| e.subtitle_locales.clone())
+                .flatten()
+                .collect();
+            real_dedup_vec(&mut subtitle_locales);
+            let missing_subtitles = missing_locales(&subtitle_locales, &self.archive.subtitle);
+            if !missing_subtitles.is_empty()
+                && !self
+                    .season_subtitles_missing
+                    .contains(&episode.season_number)
+            {
+                warn!(
+                    "Episode {} is not available with {} subtitles",
+                    episode.episode_number,
+                    missing_subtitles
+                        .into_iter()
+                        .map(|l| l.to_string())
+                        .collect::<Vec<String>>()
+                        .join(", ")
+                );
+                self.season_subtitles_missing.push(episode.season_number)
+            }
+        } else {
+            episodes.push(episode.clone())
+        }
+
+        let mut formats = vec![];
+        for episode in episodes {
+            let stream = episode.streams().await?;
+            let (video, audio) = if let Some((video, audio)) =
+                variant_data_from_stream(&stream, &self.archive.resolution).await?
+            {
+                (video, audio)
+            } else {
+                bail!(
+                    "Resolution ({}) is not available for episode {} ({}) of {} season {}",
+                    &self.archive.resolution,
+                    episode.episode_number,
+                    episode.title,
+                    episode.series_title,
+                    episode.season_number,
+                );
+            };
+            let subtitles: Vec<Subtitle> = self
+                .archive
+                .subtitle
+                .iter()
+                .filter_map(|s| stream.subtitles.get(s).cloned())
+                .collect();
+
+            let relative_episode_number = if Format::has_relative_episodes_fmt(&self.archive.output)
+            {
+                if self
+                    .season_episode_count
+                    .get(&episode.season_number)
+                    .is_none()
+                {
+                    let season_episodes = episode.season().await?.episodes().await?;
+                    self.season_episode_count.insert(
+                        episode.season_number,
+                        season_episodes.into_iter().map(|e| e.id).collect(),
+                    );
+                }
+                let relative_episode_number = self
+                    .season_episode_count
+                    .get(&episode.season_number)
+                    .unwrap()
+                    .iter()
+                    .position(|id| id == &episode.id);
+                if relative_episode_number.is_none() {
+                    warn!(
+                        "Failed to get relative episode number for episode {} ({}) of {} season {}",
+                        episode.episode_number,
+                        episode.title,
+                        episode.series_title,
+                        episode.season_number,
+                    )
+                }
+                relative_episode_number
+            } else {
+                None
+            };
+
+            formats.push(FilterResult {
+                format: SingleFormat::new_from_episode(
+                    &episode,
+                    &video,
+                    subtitles.iter().map(|s| s.locale.clone()).collect(),
+                    relative_episode_number.map(|n| n as u32),
+                ),
+                video,
+                audio,
+                duration: episode.duration.clone(),
+                subtitles,
+            })
+        }
+
+        Ok(Some(formats))
+    }
+
+    async fn visit_movie_listing(&mut self, movie_listing: MovieListing) -> Result<Vec<Movie>> {
+        Ok(movie_listing.movies().await?)
+    }
+
+    async fn visit_movie(&mut self, movie: Movie) -> Result<Option<Self::T>> {
+        let stream = movie.streams().await?;
+        let subtitles: Vec<&Subtitle> = self
+            .archive
+            .subtitle
+            .iter()
+            .filter_map(|l| stream.subtitles.get(l))
+            .collect();
+
+        let missing_subtitles = missing_locales(
+            &subtitles.iter().map(|&s| s.locale.clone()).collect(),
+            &self.archive.subtitle,
+        );
+        if !missing_subtitles.is_empty() {
+            warn!(
+                "Movie '{}' is not available with {} subtitles",
+                movie.title,
+                missing_subtitles
+                    .into_iter()
+                    .map(|l| l.to_string())
+                    .collect::<Vec<String>>()
+                    .join(", ")
+            )
+        }
+
+        let (video, audio) = if let Some((video, audio)) =
+            variant_data_from_stream(&stream, &self.archive.resolution).await?
+        {
+            (video, audio)
+        } else {
+            bail!(
+                "Resolution ({}) of movie {} is not available",
+                self.archive.resolution,
+                movie.title
+            )
+        };
+
+        Ok(Some(vec![FilterResult {
+            format: SingleFormat::new_from_movie(&movie, &video, vec![]),
+            video,
+            audio,
+            duration: movie.duration,
+            subtitles: vec![],
+        }]))
+    }
+
+    async fn visit_music_video(&mut self, music_video: MusicVideo) -> Result<Option<Self::T>> {
+        let stream = music_video.streams().await?;
+        let (video, audio) = if let Some((video, audio)) =
+            variant_data_from_stream(&stream, &self.archive.resolution).await?
+        {
+            (video, audio)
+        } else {
+            bail!(
+                "Resolution ({}) of music video {} is not available",
+                self.archive.resolution,
+                music_video.title
+            )
+        };
+
+        Ok(Some(vec![FilterResult {
+            format: SingleFormat::new_from_music_video(&music_video, &video),
+            video,
+            audio,
+            duration: music_video.duration,
+            subtitles: vec![],
+        }]))
+    }
+
+    async fn visit_concert(&mut self, concert: Concert) -> Result<Option<Self::T>> {
+        let stream = concert.streams().await?;
+        let (video, audio) = if let Some((video, audio)) =
+            variant_data_from_stream(&stream, &self.archive.resolution).await?
+        {
+            (video, audio)
+        } else {
+            bail!(
+                "Resolution ({}x{}) of music video {} is not available",
+                self.archive.resolution.width,
+                self.archive.resolution.height,
+                concert.title
+            )
+        };
+
+        Ok(Some(vec![FilterResult {
+            format: SingleFormat::new_from_concert(&concert, &video),
+            video,
+            audio,
+            duration: concert.duration,
+            subtitles: vec![],
+        }]))
+    }
+
+    async fn finish(self, input: Vec<Self::T>) -> Result<Vec<Self::Output>> {
+        let flatten_input: Vec<FilterResult> = input.into_iter().flatten().collect();
+
+        #[derive(Hash, Eq, PartialEq)]
+        struct SortKey {
+            season: u32,
+            episode: String,
+        }
+
+        let mut sorted: HashMap<SortKey, Vec<FilterResult>> = HashMap::new();
+        for data in flatten_input {
+            sorted
+                .entry(SortKey {
+                    season: data.format.season_number,
+                    episode: data.format.episode_number.to_string(),
+                })
+                .or_insert(vec![])
+                .push(data)
+        }
+
+        let mut values: Vec<Vec<FilterResult>> = sorted.into_values().collect();
+        values.sort_by(|a, b| {
+            a.first()
+                .unwrap()
+                .format
+                .sequence_number
+                .total_cmp(&b.first().unwrap().format.sequence_number)
+        });
+
+        let mut result = vec![];
+        for data in values {
+            let single_formats: Vec<SingleFormat> =
+                data.iter().map(|fr| fr.format.clone()).collect();
+            let format = Format::from_single_formats(single_formats);
+
+            let mut downloader = DownloadBuilder::new()
+                .default_subtitle(self.archive.default_subtitle.clone())
+                .ffmpeg_preset(self.archive.ffmpeg_preset.clone().unwrap_or_default())
+                .output_format(Some("matroska".to_string()))
+                .audio_sort(Some(self.archive.locale.clone()))
+                .subtitle_sort(Some(self.archive.subtitle.clone()))
+                .build();
+
+            match self.archive.merge.clone() {
+                MergeBehavior::Video => {
+                    for d in data {
+                        downloader.add_format(DownloadFormat {
+                            video: (d.video, d.format.audio.clone()),
+                            audios: vec![(d.audio, d.format.audio.clone())],
+                            subtitles: d.subtitles,
+                        })
+                    }
+                }
+                MergeBehavior::Audio => downloader.add_format(DownloadFormat {
+                    video: (
+                        data.first().unwrap().video.clone(),
+                        data.first().unwrap().format.audio.clone(),
+                    ),
+                    audios: data
+                        .iter()
+                        .map(|d| (d.audio.clone(), d.format.audio.clone()))
+                        .collect(),
+                    subtitles: data.iter().map(|d| d.subtitles.clone()).flatten().collect(),
+                }),
+                MergeBehavior::Auto => {
+                    let mut download_formats: HashMap<Duration, DownloadFormat> = HashMap::new();
+
+                    for d in data {
+                        if let Some(download_format) = download_formats.get_mut(&d.duration) {
+                            download_format.audios.push((d.audio, d.format.audio));
+                            download_format.subtitles.extend(d.subtitles)
+                        } else {
+                            download_formats.insert(
+                                d.duration,
+                                DownloadFormat {
+                                    video: (d.video, d.format.audio.clone()),
+                                    audios: vec![(d.audio, d.format.audio)],
+                                    subtitles: d.subtitles,
+                                },
+                            );
+                        }
+                    }
+
+                    for download_format in download_formats.into_values() {
+                        downloader.add_format(download_format)
+                    }
+                }
+            }
+
+            result.push((downloader, format))
+        }
+
+        Ok(result)
+    }
+}
+
+fn missing_locales<'a>(available: &Vec<Locale>, searched: &'a Vec<Locale>) -> Vec<&'a Locale> {
+    searched.iter().filter(|p| !available.contains(p)).collect()
+}

crunchy-cli-core/src/archive/mod.rs

@@ -0,0 +1,4 @@
+mod command;
+mod filter;
+
+pub use command::Archive;