Add single-threaded option to downloading commands

This commit is contained in:
Valentine Briese 2023-10-13 12:21:38 -07:00
parent 13335c020b
commit 9ace4f3476
3 changed files with 171 additions and 149 deletions

View file

@ -98,6 +98,10 @@ pub struct Archive {
#[arg(short, long, default_value_t = false)] #[arg(short, long, default_value_t = false)]
pub(crate) yes: bool, pub(crate) yes: bool,
#[arg(help = "Download using only one thread")]
#[arg(short = 't', long, default_value_t = false)]
pub(crate) single_threaded: bool,
#[arg(help = "Crunchyroll series url(s)")] #[arg(help = "Crunchyroll series url(s)")]
#[arg(required = true)] #[arg(required = true)]
pub(crate) urls: Vec<String>, pub(crate) urls: Vec<String>,
@ -158,7 +162,8 @@ impl Execute for Archive {
.ffmpeg_preset(self.ffmpeg_preset.clone().unwrap_or_default()) .ffmpeg_preset(self.ffmpeg_preset.clone().unwrap_or_default())
.output_format(Some("matroska".to_string())) .output_format(Some("matroska".to_string()))
.audio_sort(Some(self.audio.clone())) .audio_sort(Some(self.audio.clone()))
.subtitle_sort(Some(self.subtitle.clone())); .subtitle_sort(Some(self.subtitle.clone()))
.single_threaded(self.single_threaded);
for single_formats in single_format_collection.into_iter() { for single_formats in single_format_collection.into_iter() {
let (download_formats, mut format) = get_format(&self, &single_formats).await?; let (download_formats, mut format) = get_format(&self, &single_formats).await?;

View file

@ -80,6 +80,10 @@ pub struct Download {
#[arg(long, default_value_t = false)] #[arg(long, default_value_t = false)]
pub(crate) force_hardsub: bool, pub(crate) force_hardsub: bool,
#[arg(help = "Download using only one thread")]
#[arg(short = 't', long, default_value_t = false)]
pub(crate) single_threaded: bool,
#[arg(help = "Url(s) to Crunchyroll episodes or series")] #[arg(help = "Url(s) to Crunchyroll episodes or series")]
#[arg(required = true)] #[arg(required = true)]
pub(crate) urls: Vec<String>, pub(crate) urls: Vec<String>,
@ -149,7 +153,8 @@ impl Execute for Download {
} else { } else {
None None
}) })
.ffmpeg_preset(self.ffmpeg_preset.clone().unwrap_or_default()); .ffmpeg_preset(self.ffmpeg_preset.clone().unwrap_or_default())
.single_threaded(self.single_threaded);
for mut single_formats in single_format_collection.into_iter() { for mut single_formats in single_format_collection.into_iter() {
// the vec contains always only one item // the vec contains always only one item

View file

@ -50,6 +50,7 @@ pub struct DownloadBuilder {
audio_sort: Option<Vec<Locale>>, audio_sort: Option<Vec<Locale>>,
subtitle_sort: Option<Vec<Locale>>, subtitle_sort: Option<Vec<Locale>>,
force_hardsub: bool, force_hardsub: bool,
single_threaded: bool,
} }
impl DownloadBuilder { impl DownloadBuilder {
@ -61,6 +62,7 @@ impl DownloadBuilder {
audio_sort: None, audio_sort: None,
subtitle_sort: None, subtitle_sort: None,
force_hardsub: false, force_hardsub: false,
single_threaded: false,
} }
} }
@ -73,6 +75,7 @@ impl DownloadBuilder {
subtitle_sort: self.subtitle_sort, subtitle_sort: self.subtitle_sort,
force_hardsub: self.force_hardsub, force_hardsub: self.force_hardsub,
single_threaded: self.single_threaded,
formats: vec![], formats: vec![],
} }
@ -99,6 +102,7 @@ pub struct Downloader {
subtitle_sort: Option<Vec<Locale>>, subtitle_sort: Option<Vec<Locale>>,
force_hardsub: bool, force_hardsub: bool,
single_threaded: bool,
formats: Vec<DownloadFormat>, formats: Vec<DownloadFormat>,
} }
@ -502,7 +506,8 @@ impl Downloader {
let tempfile = tempfile(".mp4")?; let tempfile = tempfile(".mp4")?;
let (mut file, path) = tempfile.into_parts(); let (mut file, path) = tempfile.into_parts();
download_segments(ctx, &mut file, message, variant_data).await?; self.download_segments(ctx, &mut file, message, variant_data)
.await?;
Ok(path) Ok(path)
} }
@ -516,7 +521,8 @@ impl Downloader {
let tempfile = tempfile(".m4a")?; let tempfile = tempfile(".m4a")?;
let (mut file, path) = tempfile.into_parts(); let (mut file, path) = tempfile.into_parts();
download_segments(ctx, &mut file, message, variant_data).await?; self.download_segments(ctx, &mut file, message, variant_data)
.await?;
Ok(path) Ok(path)
} }
@ -537,9 +543,9 @@ impl Downloader {
Ok(path) Ok(path)
} }
}
pub async fn download_segments( async fn download_segments(
&self,
ctx: &Context, ctx: &Context,
writer: &mut impl Write, writer: &mut impl Write,
message: String, message: String,
@ -569,7 +575,12 @@ pub async fn download_segments(
None None
}; };
let cpus = num_cpus::get(); // Only use 1 CPU (core?) if `single-threaded` option is enabled
let cpus = if self.single_threaded {
1
} else {
num_cpus::get()
};
let mut segs: Vec<Vec<VariantSegment>> = Vec::with_capacity(cpus); let mut segs: Vec<Vec<VariantSegment>> = Vec::with_capacity(cpus);
for _ in 0..cpus { for _ in 0..cpus {
segs.push(vec![]) segs.push(vec![])
@ -673,8 +684,8 @@ pub async fn download_segments(
if let Some(p) = &progress { if let Some(p) = &progress {
let progress_len = p.length().unwrap(); let progress_len = p.length().unwrap();
let estimated_segment_len = let estimated_segment_len = (variant_data.bandwidth / 8)
(variant_data.bandwidth / 8) * segments.get(pos as usize).unwrap().length.as_secs(); * segments.get(pos as usize).unwrap().length.as_secs();
let bytes_len = bytes.len() as u64; let bytes_len = bytes.len() as u64;
p.set_length(progress_len - estimated_segment_len + bytes_len); p.set_length(progress_len - estimated_segment_len + bytes_len);
@ -720,6 +731,7 @@ pub async fn download_segments(
Ok(()) Ok(())
} }
}
fn estimate_variant_file_size(variant_data: &VariantData, segments: &Vec<VariantSegment>) -> u64 { fn estimate_variant_file_size(variant_data: &VariantData, segments: &Vec<VariantSegment>) -> u64 {
(variant_data.bandwidth / 8) * segments.iter().map(|s| s.length.as_secs()).sum::<u64>() (variant_data.bandwidth / 8) * segments.iter().map(|s| s.length.as_secs()).sum::<u64>()