Move stream download logic to fix cms error/rate limiting

crunchy-cli-core/src/archive/command.rs

@@ -1,19 +1,22 @@
 use crate::archive::filter::ArchiveFilter;
 use crate::utils::context::Context;
-use crate::utils::download::MergeBehavior;
+use crate::utils::download::{DownloadBuilder, DownloadFormat, MergeBehavior};
 use crate::utils::ffmpeg::FFmpegPreset;
 use crate::utils::filter::Filter;
-use crate::utils::format::formats_visual_output;
+use crate::utils::format::{Format, SingleFormat};
 use crate::utils::locale::all_locale_in_locales;
 use crate::utils::log::progress;
 use crate::utils::os::{free_file, has_ffmpeg, is_special_file};
 use crate::utils::parse::parse_url;
+use crate::utils::video::variant_data_from_stream;
 use crate::Execute;
 use anyhow::bail;
 use anyhow::Result;
-use crunchyroll_rs::media::Resolution;
+use chrono::Duration;
+use crunchyroll_rs::media::{Resolution, Subtitle};
 use crunchyroll_rs::Locale;
 use log::debug;
+use std::collections::HashMap;
 use std::path::PathBuf;
 
 #[derive(Clone, Debug, clap::Parser)]
@@ -135,19 +138,33 @@ impl Execute for Archive {
 
         for (i, (media_collection, url_filter)) in parsed_urls.into_iter().enumerate() {
             let progress_handler = progress!("Fetching series details");
-            let archive_formats = ArchiveFilter::new(url_filter, self.clone())
+            let single_format_collection = ArchiveFilter::new(url_filter, self.clone())
                 .visit(media_collection)
                 .await?;
 
-            if archive_formats.is_empty() {
+            if single_format_collection.is_empty() {
                 progress_handler.stop(format!("Skipping url {} (no matching videos found)", i + 1));
                 continue;
             }
             progress_handler.stop(format!("Loaded series information for url {}", i + 1));
 
-            formats_visual_output(archive_formats.iter().map(|(_, f)| f).collect());
+            single_format_collection.full_visual_output();
+
+            let download_builder = DownloadBuilder::new()
+                .default_subtitle(self.default_subtitle.clone())
+                .ffmpeg_preset(self.ffmpeg_preset.clone().unwrap_or_default())
+                .output_format(Some("matroska".to_string()))
+                .audio_sort(Some(self.locale.clone()))
+                .subtitle_sort(Some(self.subtitle.clone()));
+
+            for single_formats in single_format_collection.into_iter() {
+                let (download_formats, mut format) = get_format(&self, &single_formats).await?;
+
+                let mut downloader = download_builder.clone().build();
+                for download_format in download_formats {
+                    downloader.add_format(download_format)
+                }
 
-            for (downloader, mut format) in archive_formats {
                 let formatted_path = format.format_path((&self.output).into(), true);
                 let (path, changed) = free_file(formatted_path.clone());
 
@@ -183,3 +200,104 @@ impl Execute for Archive {
         Ok(())
     }
 }
+
+async fn get_format(
+    archive: &Archive,
+    single_formats: &Vec<SingleFormat>,
+) -> Result<(Vec<DownloadFormat>, Format)> {
+    let mut format_pairs = vec![];
+    let mut single_format_to_format_pairs = vec![];
+
+    for single_format in single_formats {
+        let stream = single_format.stream().await?;
+        let Some((video, audio)) = variant_data_from_stream(&stream, &archive.resolution).await? else {
+            if single_format.is_episode() {
+                bail!(
+                    "Resolution ({}) is not available for episode {} ({}) of {} season {}",
+                    archive.resolution,
+                    single_format.episode_number,
+                    single_format.title,
+                    single_format.series_name,
+                    single_format.season_number,
+                )
+            } else {
+                bail!(
+                    "Resolution ({}) is not available for {} ({})",
+                    archive.resolution,
+                    single_format.source_type(),
+                    single_format.title
+                )
+            }
+        };
+
+        let subtitles: Vec<Subtitle> = archive
+            .subtitle
+            .iter()
+            .filter_map(|s| stream.subtitles.get(s).cloned())
+            .collect();
+
+        format_pairs.push((single_format, video.clone(), audio, subtitles.clone()));
+        single_format_to_format_pairs.push((single_format.clone(), video, subtitles))
+    }
+
+    let mut download_formats = vec![];
+
+    match archive.merge {
+        MergeBehavior::Video => {
+            for (single_format, video, audio, subtitles) in format_pairs {
+                download_formats.push(DownloadFormat {
+                    video: (video, single_format.audio.clone()),
+                    audios: vec![(audio, single_format.audio.clone())],
+                    subtitles,
+                })
+            }
+        }
+        MergeBehavior::Audio => download_formats.push(DownloadFormat {
+            video: (
+                (*format_pairs.first().unwrap()).1.clone(),
+                (*format_pairs.first().unwrap()).0.audio.clone(),
+            ),
+            audios: format_pairs
+                .iter()
+                .map(|(single_format, _, audio, _)| (audio.clone(), single_format.audio.clone()))
+                .collect(),
+            // mix all subtitles together and then reduce them via a map so that only one subtitle
+            // per language exists
+            subtitles: format_pairs
+                .iter()
+                .flat_map(|(_, _, _, subtitles)| subtitles.clone())
+                .map(|s| (s.locale.clone(), s))
+                .collect::<HashMap<Locale, Subtitle>>()
+                .into_values()
+                .collect(),
+        }),
+        MergeBehavior::Auto => {
+            let mut d_formats: HashMap<Duration, DownloadFormat> = HashMap::new();
+
+            for (single_format, video, audio, subtitles) in format_pairs {
+                if let Some(d_format) = d_formats.get_mut(&single_format.duration) {
+                    d_format.audios.push((audio, single_format.audio.clone()));
+                    d_format.subtitles.extend(subtitles)
+                } else {
+                    d_formats.insert(
+                        single_format.duration,
+                        DownloadFormat {
+                            video: (video, single_format.audio.clone()),
+                            audios: vec![(audio, single_format.audio.clone())],
+                            subtitles,
+                        },
+                    );
+                }
+            }
+
+            for d_format in d_formats.into_values() {
+                download_formats.push(d_format)
+            }
+        }
+    }
+
+    Ok((
+        download_formats,
+        Format::from_single_formats(single_format_to_format_pairs),
+    ))
+}

crunchy-cli-core/src/archive/filter.rs

@@ -1,24 +1,11 @@
 use crate::archive::command::Archive;
-use crate::utils::download::{DownloadBuilder, DownloadFormat, Downloader, MergeBehavior};
 use crate::utils::filter::{real_dedup_vec, Filter};
-use crate::utils::format::{Format, SingleFormat};
+use crate::utils::format::{Format, SingleFormat, SingleFormatCollection};
 use crate::utils::parse::UrlFilter;
-use crate::utils::video::variant_data_from_stream;
-use anyhow::{bail, Result};
-use chrono::Duration;
-use crunchyroll_rs::media::{Subtitle, VariantData};
+use anyhow::Result;
 use crunchyroll_rs::{Concert, Episode, Locale, Movie, MovieListing, MusicVideo, Season, Series};
 use log::warn;
-use std::collections::HashMap;
-use std::hash::Hash;
-
-pub(crate) struct FilterResult {
-    format: SingleFormat,
-    video: VariantData,
-    audio: VariantData,
-    duration: Duration,
-    subtitles: Vec<Subtitle>,
-}
+use std::collections::{BTreeMap, HashMap};
 
 enum Visited {
     Series,
@@ -48,8 +35,8 @@ impl ArchiveFilter {
 
 #[async_trait::async_trait]
 impl Filter for ArchiveFilter {
-    type T = Vec<FilterResult>;
-    type Output = (Downloader, Format);
+    type T = Vec<SingleFormat>;
+    type Output = SingleFormatCollection;
 
     async fn visit_series(&mut self, series: Series) -> Result<Vec<Season>> {
         // `series.audio_locales` isn't always populated b/c of crunchyrolls api. so check if the
@@ -168,11 +155,19 @@ impl Filter for ArchiveFilter {
         let mut episodes = vec![];
         if !matches!(self.visited, Visited::Series) && !matches!(self.visited, Visited::Season) {
             if self.archive.locale.contains(&episode.audio_locale) {
-                episodes.push(episode.clone())
+                episodes.push((episode.clone(), episode.subtitle_locales.clone()))
             }
-            episodes.extend(episode.version(self.archive.locale.clone()).await?);
-            let audio_locales: Vec<Locale> =
-                episodes.iter().map(|e| e.audio_locale.clone()).collect();
+            episodes.extend(
+                episode
+                    .version(self.archive.locale.clone())
+                    .await?
+                    .into_iter()
+                    .map(|e| (e.clone(), e.subtitle_locales.clone())),
+            );
+            let audio_locales: Vec<Locale> = episodes
+                .iter()
+                .map(|(e, _)| e.audio_locale.clone())
+                .collect();
             let missing_audio = missing_locales(&audio_locales, &self.archive.locale);
             if !missing_audio.is_empty() {
                 warn!(
@@ -186,11 +181,8 @@ impl Filter for ArchiveFilter {
                 )
             }
 
-            let mut subtitle_locales: Vec<Locale> = episodes
-                .iter()
-                .map(|e| e.subtitle_locales.clone())
-                .flatten()
-                .collect();
+            let mut subtitle_locales: Vec<Locale> =
+                episodes.iter().map(|(_, s)| s.clone()).flatten().collect();
             real_dedup_vec(&mut subtitle_locales);
             let missing_subtitles = missing_locales(&subtitle_locales, &self.archive.subtitle);
             if !missing_subtitles.is_empty()
@@ -210,81 +202,49 @@ impl Filter for ArchiveFilter {
                 self.season_subtitles_missing.push(episode.season_number)
             }
         } else {
-            episodes.push(episode.clone())
+            episodes.push((episode.clone(), episode.subtitle_locales.clone()))
         }
 
-        let mut formats = vec![];
-        for episode in episodes {
-            let stream = episode.streams().await?;
-            let (video, audio) = if let Some((video, audio)) =
-                variant_data_from_stream(&stream, &self.archive.resolution).await?
+        let relative_episode_number = if Format::has_relative_episodes_fmt(&self.archive.output) {
+            if self
+                .season_episode_count
+                .get(&episode.season_number)
+                .is_none()
             {
-                (video, audio)
-            } else {
-                bail!(
-                    "Resolution ({}) is not available for episode {} ({}) of {} season {}",
-                    &self.archive.resolution,
+                let season_episodes = episode.season().await?.episodes().await?;
+                self.season_episode_count.insert(
+                    episode.season_number,
+                    season_episodes.into_iter().map(|e| e.id).collect(),
+                );
+            }
+            let relative_episode_number = self
+                .season_episode_count
+                .get(&episode.season_number)
+                .unwrap()
+                .iter()
+                .position(|id| id == &episode.id);
+            if relative_episode_number.is_none() {
+                warn!(
+                    "Failed to get relative episode number for episode {} ({}) of {} season {}",
                     episode.episode_number,
                     episode.title,
                     episode.series_title,
                     episode.season_number,
-                );
-            };
-            let subtitles: Vec<Subtitle> = self
-                .archive
-                .subtitle
-                .iter()
-                .filter_map(|s| stream.subtitles.get(s).cloned())
-                .collect();
+                )
+            }
+            relative_episode_number
+        } else {
+            None
+        };
 
-            let relative_episode_number = if Format::has_relative_episodes_fmt(&self.archive.output)
-            {
-                if self
-                    .season_episode_count
-                    .get(&episode.season_number)
-                    .is_none()
-                {
-                    let season_episodes = episode.season().await?.episodes().await?;
-                    self.season_episode_count.insert(
-                        episode.season_number,
-                        season_episodes.into_iter().map(|e| e.id).collect(),
-                    );
-                }
-                let relative_episode_number = self
-                    .season_episode_count
-                    .get(&episode.season_number)
-                    .unwrap()
-                    .iter()
-                    .position(|id| id == &episode.id);
-                if relative_episode_number.is_none() {
-                    warn!(
-                        "Failed to get relative episode number for episode {} ({}) of {} season {}",
-                        episode.episode_number,
-                        episode.title,
-                        episode.series_title,
-                        episode.season_number,
-                    )
-                }
-                relative_episode_number
-            } else {
-                None
-            };
-
-            formats.push(FilterResult {
-                format: SingleFormat::new_from_episode(
-                    &episode,
-                    &video,
-                    subtitles.iter().map(|s| s.locale.clone()).collect(),
-                    relative_episode_number.map(|n| n as u32),
-                ),
-                video,
-                audio,
-                duration: episode.duration.clone(),
-                subtitles,
-            })
-        }
-
-        Ok(Some(formats))
+        Ok(Some(
+            episodes
+                .into_iter()
+                .map(|(e, s)| {
+                    SingleFormat::new_from_episode(e, s, relative_episode_number.map(|n| n as u32))
+                })
+                .collect(),
+        ))
     }
 
     async fn visit_movie_listing(&mut self, movie_listing: MovieListing) -> Result<Vec<Movie>> {
@@ -292,199 +252,37 @@ impl Filter for ArchiveFilter {
     }
 
     async fn visit_movie(&mut self, movie: Movie) -> Result<Option<Self::T>> {
-        let stream = movie.streams().await?;
-        let subtitles: Vec<&Subtitle> = self
-            .archive
-            .subtitle
-            .iter()
-            .filter_map(|l| stream.subtitles.get(l))
-            .collect();
-
-        let missing_subtitles = missing_locales(
-            &subtitles.iter().map(|&s| s.locale.clone()).collect(),
-            &self.archive.subtitle,
-        );
-        if !missing_subtitles.is_empty() {
-            warn!(
-                "Movie '{}' is not available with {} subtitles",
-                movie.title,
-                missing_subtitles
-                    .into_iter()
-                    .map(|l| l.to_string())
-                    .collect::<Vec<String>>()
-                    .join(", ")
-            )
-        }
-
-        let (video, audio) = if let Some((video, audio)) =
-            variant_data_from_stream(&stream, &self.archive.resolution).await?
-        {
-            (video, audio)
-        } else {
-            bail!(
-                "Resolution ({}) of movie {} is not available",
-                self.archive.resolution,
-                movie.title
-            )
-        };
-
-        Ok(Some(vec![FilterResult {
-            format: SingleFormat::new_from_movie(&movie, &video, vec![]),
-            video,
-            audio,
-            duration: movie.duration,
-            subtitles: vec![],
-        }]))
+        Ok(Some(vec![SingleFormat::new_from_movie(movie, vec![])]))
     }
 
     async fn visit_music_video(&mut self, music_video: MusicVideo) -> Result<Option<Self::T>> {
-        let stream = music_video.streams().await?;
-        let (video, audio) = if let Some((video, audio)) =
-            variant_data_from_stream(&stream, &self.archive.resolution).await?
-        {
-            (video, audio)
-        } else {
-            bail!(
-                "Resolution ({}) of music video {} is not available",
-                self.archive.resolution,
-                music_video.title
-            )
-        };
-
-        Ok(Some(vec![FilterResult {
-            format: SingleFormat::new_from_music_video(&music_video, &video),
-            video,
-            audio,
-            duration: music_video.duration,
-            subtitles: vec![],
-        }]))
+        Ok(Some(vec![SingleFormat::new_from_music_video(music_video)]))
     }
 
     async fn visit_concert(&mut self, concert: Concert) -> Result<Option<Self::T>> {
-        let stream = concert.streams().await?;
-        let (video, audio) = if let Some((video, audio)) =
-            variant_data_from_stream(&stream, &self.archive.resolution).await?
-        {
-            (video, audio)
-        } else {
-            bail!(
-                "Resolution ({}x{}) of music video {} is not available",
-                self.archive.resolution.width,
-                self.archive.resolution.height,
-                concert.title
-            )
-        };
-
-        Ok(Some(vec![FilterResult {
-            format: SingleFormat::new_from_concert(&concert, &video),
-            video,
-            audio,
-            duration: concert.duration,
-            subtitles: vec![],
-        }]))
+        Ok(Some(vec![SingleFormat::new_from_concert(concert)]))
     }
 
-    async fn finish(self, input: Vec<Self::T>) -> Result<Vec<Self::Output>> {
-        let flatten_input: Vec<FilterResult> = input.into_iter().flatten().collect();
+    async fn finish(self, input: Vec<Self::T>) -> Result<Self::Output> {
+        let flatten_input: Self::T = input.into_iter().flatten().collect();
 
-        #[derive(Hash, Eq, PartialEq)]
-        struct SortKey {
-            season: u32,
-            episode: String,
-        }
+        let mut single_format_collection = SingleFormatCollection::new();
 
-        let mut sorted: HashMap<SortKey, Vec<FilterResult>> = HashMap::new();
+        struct SortKey(u32, String);
+
+        let mut sorted: BTreeMap<(u32, String), Self::T> = BTreeMap::new();
         for data in flatten_input {
             sorted
-                .entry(SortKey {
-                    season: data.format.season_number,
-                    episode: data.format.episode_number.to_string(),
-                })
+                .entry((data.season_number, data.sequence_number.to_string()))
                 .or_insert(vec![])
                 .push(data)
         }
 
-        let mut values: Vec<Vec<FilterResult>> = sorted.into_values().collect();
-        values.sort_by(|a, b| {
-            a.first()
-                .unwrap()
-                .format
-                .sequence_number
-                .total_cmp(&b.first().unwrap().format.sequence_number)
-        });
-
-        let mut result = vec![];
-        for data in values {
-            let single_formats: Vec<SingleFormat> =
-                data.iter().map(|fr| fr.format.clone()).collect();
-            let format = Format::from_single_formats(single_formats);
-
-            let mut downloader = DownloadBuilder::new()
-                .default_subtitle(self.archive.default_subtitle.clone())
-                .ffmpeg_preset(self.archive.ffmpeg_preset.clone().unwrap_or_default())
-                .output_format(Some("matroska".to_string()))
-                .audio_sort(Some(self.archive.locale.clone()))
-                .subtitle_sort(Some(self.archive.subtitle.clone()))
-                .build();
-
-            match self.archive.merge.clone() {
-                MergeBehavior::Video => {
-                    for d in data {
-                        downloader.add_format(DownloadFormat {
-                            video: (d.video, d.format.audio.clone()),
-                            audios: vec![(d.audio, d.format.audio.clone())],
-                            subtitles: d.subtitles,
-                        })
-                    }
-                }
-                MergeBehavior::Audio => downloader.add_format(DownloadFormat {
-                    video: (
-                        data.first().unwrap().video.clone(),
-                        data.first().unwrap().format.audio.clone(),
-                    ),
-                    audios: data
-                        .iter()
-                        .map(|d| (d.audio.clone(), d.format.audio.clone()))
-                        .collect(),
-                    // mix all subtitles together and then reduce them via a map so that only one
-                    // subtitle per language exists
-                    subtitles: data
-                        .iter()
-                        .flat_map(|d| d.subtitles.clone())
-                        .map(|s| (s.locale.clone(), s))
-                        .collect::<HashMap<Locale, Subtitle>>()
-                        .into_values()
-                        .collect(),
-                }),
-                MergeBehavior::Auto => {
-                    let mut download_formats: HashMap<Duration, DownloadFormat> = HashMap::new();
-
-                    for d in data {
-                        if let Some(download_format) = download_formats.get_mut(&d.duration) {
-                            download_format.audios.push((d.audio, d.format.audio));
-                            download_format.subtitles.extend(d.subtitles)
-                        } else {
-                            download_formats.insert(
-                                d.duration,
-                                DownloadFormat {
-                                    video: (d.video, d.format.audio.clone()),
-                                    audios: vec![(d.audio, d.format.audio)],
-                                    subtitles: d.subtitles,
-                                },
-                            );
-                        }
-                    }
-
-                    for download_format in download_formats.into_values() {
-                        downloader.add_format(download_format)
-                    }
-                }
-            }
-
-            result.push((downloader, format))
+        for data in sorted.into_values() {
+            single_format_collection.add_single_formats(data)
         }
 
-        Ok(result)
+        Ok(single_format_collection)
     }
 }