mirror of
https://github.com/crunchy-labs/crunchy-cli.git
synced 2026-01-21 12:12:00 -06:00
Add archive start sync flag
This commit is contained in:
bytedream
parent
fe49161e93
commit
1a511e12f9
8 changed files with 692 additions and 179 deletions
|
|
@ -1,11 +1,15 @@
|
|||
use crate::utils::ffmpeg::FFmpegPreset;
|
||||
use crate::utils::filter::real_dedup_vec;
|
||||
use crate::utils::os::{cache_dir, is_special_file, temp_directory, temp_named_pipe, tempfile};
|
||||
use crate::utils::log::progress;
|
||||
use crate::utils::os::{
|
||||
cache_dir, is_special_file, temp_directory, temp_named_pipe, tempdir, tempfile,
|
||||
};
|
||||
use crate::utils::rate_limit::RateLimiterService;
|
||||
use anyhow::{bail, Result};
|
||||
use chrono::NaiveTime;
|
||||
use chrono::{NaiveTime, TimeDelta};
|
||||
use crunchyroll_rs::media::{SkipEvents, SkipEventsEvent, StreamData, StreamSegment, Subtitle};
|
||||
use crunchyroll_rs::Locale;
|
||||
use image_hasher::{Hasher, HasherConfig, ImageHash};
|
||||
use indicatif::{ProgressBar, ProgressDrawTarget, ProgressFinish, ProgressStyle};
|
||||
use log::{debug, warn, LevelFilter};
|
||||
use regex::Regex;
|
||||
|
|
@ -59,6 +63,7 @@ pub struct DownloadBuilder {
|
|||
force_hardsub: bool,
|
||||
download_fonts: bool,
|
||||
no_closed_caption: bool,
|
||||
sync_start_value: Option<f64>,
|
||||
threads: usize,
|
||||
ffmpeg_threads: Option<usize>,
|
||||
audio_locale_output_map: HashMap<Locale, String>,
|
||||
|
|
@ -78,6 +83,7 @@ impl DownloadBuilder {
|
|||
force_hardsub: false,
|
||||
download_fonts: false,
|
||||
no_closed_caption: false,
|
||||
sync_start_value: None,
|
||||
threads: num_cpus::get(),
|
||||
ffmpeg_threads: None,
|
||||
audio_locale_output_map: HashMap::new(),
|
||||
|
|
@ -99,6 +105,8 @@ impl DownloadBuilder {
|
|||
download_fonts: self.download_fonts,
|
||||
no_closed_caption: self.no_closed_caption,
|
||||
|
||||
sync_start_value: self.sync_start_value,
|
||||
|
||||
download_threads: self.threads,
|
||||
ffmpeg_threads: self.ffmpeg_threads,
|
||||
|
||||
|
|
@ -110,10 +118,23 @@ impl DownloadBuilder {
|
|||
}
|
||||
}
|
||||
|
||||
struct FFmpegMeta {
|
||||
struct FFmpegVideoMeta {
|
||||
path: TempPath,
|
||||
language: Locale,
|
||||
title: String,
|
||||
length: TimeDelta,
|
||||
start_time: Option<TimeDelta>,
|
||||
}
|
||||
|
||||
struct FFmpegAudioMeta {
|
||||
path: TempPath,
|
||||
locale: Locale,
|
||||
start_time: Option<TimeDelta>,
|
||||
}
|
||||
|
||||
struct FFmpegSubtitleMeta {
|
||||
path: TempPath,
|
||||
locale: Locale,
|
||||
cc: bool,
|
||||
start_time: Option<TimeDelta>,
|
||||
}
|
||||
|
||||
pub struct DownloadFormat {
|
||||
|
|
@ -141,6 +162,8 @@ pub struct Downloader {
|
|||
download_fonts: bool,
|
||||
no_closed_caption: bool,
|
||||
|
||||
sync_start_value: Option<f64>,
|
||||
|
||||
download_threads: usize,
|
||||
ffmpeg_threads: Option<usize>,
|
||||
|
||||
|
|
@ -216,13 +239,16 @@ impl Downloader {
|
|||
}
|
||||
}
|
||||
|
||||
let mut video_offset = None;
|
||||
let mut audio_offsets = HashMap::new();
|
||||
let mut subtitle_offsets = HashMap::new();
|
||||
let mut videos = vec![];
|
||||
let mut audios = vec![];
|
||||
let mut subtitles = vec![];
|
||||
let mut fonts = vec![];
|
||||
let mut chapters = None;
|
||||
let mut max_len = NaiveTime::MIN;
|
||||
let mut max_frames = 0f64;
|
||||
let mut max_len = TimeDelta::min_value();
|
||||
let mut max_frames = 0;
|
||||
let fmt_space = self
|
||||
.formats
|
||||
.iter()
|
||||
|
|
@ -234,115 +260,252 @@ impl Downloader {
|
|||
.max()
|
||||
.unwrap();
|
||||
|
||||
for (i, format) in self.formats.iter().enumerate() {
|
||||
let video_path = self
|
||||
.download_video(
|
||||
&format.video.0,
|
||||
format!("{:<1$}", format!("Downloading video #{}", i + 1), fmt_space),
|
||||
)
|
||||
.await?;
|
||||
for (variant_data, locale) in format.audios.iter() {
|
||||
let audio_path = self
|
||||
.download_audio(
|
||||
variant_data,
|
||||
format!("{:<1$}", format!("Downloading {} audio", locale), fmt_space),
|
||||
if self.formats.len() > 1 && self.sync_start_value.is_some() {
|
||||
let all_segments_count: Vec<usize> = self
|
||||
.formats
|
||||
.iter()
|
||||
.map(|f| f.video.0.segments().len())
|
||||
.collect();
|
||||
let sync_segments = 11.max(
|
||||
all_segments_count.iter().max().unwrap() - all_segments_count.iter().min().unwrap(),
|
||||
);
|
||||
let mut sync_vids = vec![];
|
||||
for (i, format) in self.formats.iter().enumerate() {
|
||||
let path = self
|
||||
.download_video(
|
||||
&format.video.0,
|
||||
format!("Downloading video #{} sync segments", i + 1),
|
||||
Some(sync_segments),
|
||||
)
|
||||
.await?;
|
||||
audios.push(FFmpegMeta {
|
||||
path: audio_path,
|
||||
language: locale.clone(),
|
||||
title: if i == 0 {
|
||||
locale.to_human_readable()
|
||||
} else {
|
||||
format!("{} [Video: #{}]", locale.to_human_readable(), i + 1)
|
||||
},
|
||||
sync_vids.push(SyncVideo {
|
||||
path,
|
||||
length: len_from_segments(&format.video.0.segments()),
|
||||
available_frames: (len_from_segments(
|
||||
&format.video.0.segments()[0..sync_segments],
|
||||
)
|
||||
.num_milliseconds() as f64
|
||||
* format.video.0.fps().unwrap()
|
||||
/ 1000.0) as u64,
|
||||
idx: i,
|
||||
})
|
||||
}
|
||||
|
||||
let (len, fps) = get_video_stats(&video_path)?;
|
||||
let _progress_handler =
|
||||
progress!("Syncing video start times (this might take some time)");
|
||||
let mut offsets = sync_videos(sync_vids, self.sync_start_value.unwrap())?;
|
||||
drop(_progress_handler);
|
||||
|
||||
let mut offset_pre_checked = false;
|
||||
if let Some(tmp_offsets) = &offsets {
|
||||
let formats_with_offset: Vec<TimeDelta> = self
|
||||
.formats
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, f)| {
|
||||
len_from_segments(&f.video.0.segments())
|
||||
- TimeDelta::milliseconds(
|
||||
tmp_offsets
|
||||
.get(&i)
|
||||
.map(|o| (*o as f64 / f.video.0.fps().unwrap() * 1000.0) as i64)
|
||||
.unwrap_or_default(),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let min = formats_with_offset.iter().min().unwrap();
|
||||
let max = formats_with_offset.iter().max().unwrap();
|
||||
|
||||
if max.num_seconds() - min.num_seconds() > 15 {
|
||||
warn!("Found difference of >15 seconds after sync, skipping applying it");
|
||||
offsets = None;
|
||||
offset_pre_checked = true
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(offsets) = offsets {
|
||||
let mut root_format_idx = 0;
|
||||
let mut root_format_length = 0;
|
||||
let mut audio_count: usize = 0;
|
||||
let mut subtitle_count: usize = 0;
|
||||
for (i, format) in self.formats.iter().enumerate() {
|
||||
let format_fps = format.video.0.fps().unwrap();
|
||||
let format_len = format
|
||||
.video
|
||||
.0
|
||||
.segments()
|
||||
.iter()
|
||||
.map(|s| s.length.as_millis())
|
||||
.sum::<u128>() as u64
|
||||
- offsets.get(&i).map_or(0, |o| *o);
|
||||
if format_len > root_format_length {
|
||||
root_format_idx = i;
|
||||
root_format_length = format_len;
|
||||
}
|
||||
|
||||
for _ in &format.audios {
|
||||
if let Some(offset) = &offsets.get(&i) {
|
||||
audio_offsets.insert(
|
||||
audio_count,
|
||||
TimeDelta::milliseconds(
|
||||
(**offset as f64 / format_fps * 1000.0) as i64,
|
||||
),
|
||||
);
|
||||
}
|
||||
audio_count += 1
|
||||
}
|
||||
for _ in &format.subtitles {
|
||||
if let Some(offset) = &offsets.get(&i) {
|
||||
subtitle_offsets.insert(
|
||||
subtitle_count,
|
||||
TimeDelta::milliseconds(
|
||||
(**offset as f64 / format_fps * 1000.0) as i64,
|
||||
),
|
||||
);
|
||||
}
|
||||
subtitle_count += 1
|
||||
}
|
||||
}
|
||||
|
||||
let mut root_format = self.formats.remove(root_format_idx);
|
||||
|
||||
let mut audio_prepend = vec![];
|
||||
let mut subtitle_prepend = vec![];
|
||||
let mut audio_append = vec![];
|
||||
let mut subtitle_append = vec![];
|
||||
for (i, format) in self.formats.into_iter().enumerate() {
|
||||
if i < root_format_idx {
|
||||
audio_prepend.extend(format.audios);
|
||||
subtitle_prepend.extend(format.subtitles);
|
||||
} else {
|
||||
audio_append.extend(format.audios);
|
||||
subtitle_append.extend(format.subtitles);
|
||||
}
|
||||
}
|
||||
root_format.audios.splice(0..0, audio_prepend);
|
||||
root_format.subtitles.splice(0..0, subtitle_prepend);
|
||||
root_format.audios.extend(audio_append);
|
||||
root_format.subtitles.extend(subtitle_append);
|
||||
|
||||
self.formats = vec![root_format];
|
||||
video_offset = offsets.get(&root_format_idx).map(|o| {
|
||||
TimeDelta::milliseconds(
|
||||
(*o as f64 / self.formats[0].video.0.fps().unwrap() * 1000.0) as i64,
|
||||
)
|
||||
})
|
||||
} else if !offset_pre_checked {
|
||||
warn!("Couldn't find reliable sync positions")
|
||||
}
|
||||
}
|
||||
|
||||
// downloads all videos
|
||||
for (i, format) in self.formats.iter().enumerate() {
|
||||
let path = self
|
||||
.download_video(
|
||||
&format.video.0,
|
||||
format!("{:<1$}", format!("Downloading video #{}", i + 1), fmt_space),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let (len, fps) = get_video_stats(&path)?;
|
||||
if max_len < len {
|
||||
max_len = len
|
||||
}
|
||||
let frames = len.signed_duration_since(NaiveTime::MIN).num_seconds() as f64 * fps;
|
||||
if frames > max_frames {
|
||||
max_frames = frames;
|
||||
let frames = ((len.num_milliseconds() as f64
|
||||
- video_offset.unwrap_or_default().num_milliseconds() as f64)
|
||||
/ 1000.0
|
||||
* fps) as u64;
|
||||
if max_frames < frames {
|
||||
max_frames = frames
|
||||
}
|
||||
|
||||
if !format.subtitles.is_empty() {
|
||||
let progress_spinner = if log::max_level() == LevelFilter::Info {
|
||||
let progress_spinner = ProgressBar::new_spinner()
|
||||
.with_style(
|
||||
ProgressStyle::with_template(
|
||||
format!(
|
||||
":: {:<1$} {{msg}} {{spinner}}",
|
||||
"Downloading subtitles", fmt_space
|
||||
)
|
||||
.as_str(),
|
||||
videos.push(FFmpegVideoMeta {
|
||||
path,
|
||||
length: len,
|
||||
start_time: video_offset,
|
||||
})
|
||||
}
|
||||
|
||||
// downloads all audios
|
||||
for format in &self.formats {
|
||||
for (j, (stream_data, locale)) in format.audios.iter().enumerate() {
|
||||
let path = self
|
||||
.download_audio(
|
||||
stream_data,
|
||||
format!("{:<1$}", format!("Downloading {} audio", locale), fmt_space),
|
||||
)
|
||||
.await?;
|
||||
audios.push(FFmpegAudioMeta {
|
||||
path,
|
||||
locale: locale.clone(),
|
||||
start_time: audio_offsets.get(&j).cloned(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
for (i, format) in self.formats.iter().enumerate() {
|
||||
if format.subtitles.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let progress_spinner = if log::max_level() == LevelFilter::Info {
|
||||
let progress_spinner = ProgressBar::new_spinner()
|
||||
.with_style(
|
||||
ProgressStyle::with_template(
|
||||
format!(
|
||||
":: {:<1$} {{msg}} {{spinner}}",
|
||||
"Downloading subtitles", fmt_space
|
||||
)
|
||||
.unwrap()
|
||||
.tick_strings(&["—", "\\", "|", "/", ""]),
|
||||
.as_str(),
|
||||
)
|
||||
.with_finish(ProgressFinish::Abandon);
|
||||
progress_spinner.enable_steady_tick(Duration::from_millis(100));
|
||||
Some(progress_spinner)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
.unwrap()
|
||||
.tick_strings(&["—", "\\", "|", "/", ""]),
|
||||
)
|
||||
.with_finish(ProgressFinish::Abandon);
|
||||
progress_spinner.enable_steady_tick(Duration::from_millis(100));
|
||||
Some(progress_spinner)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
for (subtitle, not_cc) in format.subtitles.iter() {
|
||||
if !not_cc && self.no_closed_caption {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(pb) = &progress_spinner {
|
||||
let mut progress_message = pb.message();
|
||||
if !progress_message.is_empty() {
|
||||
progress_message += ", "
|
||||
}
|
||||
progress_message += &subtitle.locale.to_string();
|
||||
if !not_cc {
|
||||
progress_message += " (CC)";
|
||||
}
|
||||
if i != 0 {
|
||||
progress_message += &format!(" [Video: #{}]", i + 1);
|
||||
}
|
||||
pb.set_message(progress_message)
|
||||
}
|
||||
|
||||
let mut subtitle_title = subtitle.locale.to_human_readable();
|
||||
if !not_cc {
|
||||
subtitle_title += " (CC)"
|
||||
}
|
||||
if i != 0 {
|
||||
subtitle_title += &format!(" [Video: #{}]", i + 1)
|
||||
}
|
||||
|
||||
let subtitle_path = self.download_subtitle(subtitle.clone(), len).await?;
|
||||
debug!(
|
||||
"Downloaded {} subtitles{}{}",
|
||||
subtitle.locale,
|
||||
(!not_cc).then_some(" (cc)").unwrap_or_default(),
|
||||
(i != 0)
|
||||
.then_some(format!(" for video {}", i))
|
||||
.unwrap_or_default()
|
||||
);
|
||||
subtitles.push(FFmpegMeta {
|
||||
path: subtitle_path,
|
||||
language: subtitle.locale.clone(),
|
||||
title: subtitle_title,
|
||||
})
|
||||
for (j, (subtitle, not_cc)) in format.subtitles.iter().enumerate() {
|
||||
if !not_cc && self.no_closed_caption {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
videos.push(FFmpegMeta {
|
||||
path: video_path,
|
||||
language: format.video.1.clone(),
|
||||
title: if self.formats.len() == 1 {
|
||||
"Default".to_string()
|
||||
} else {
|
||||
format!("#{}", i + 1)
|
||||
},
|
||||
});
|
||||
|
||||
if let Some(pb) = &progress_spinner {
|
||||
let mut progress_message = pb.message();
|
||||
if !progress_message.is_empty() {
|
||||
progress_message += ", "
|
||||
}
|
||||
progress_message += &subtitle.locale.to_string();
|
||||
if !not_cc {
|
||||
progress_message += " (CC)";
|
||||
}
|
||||
if i.min(videos.len() - 1) != 0 {
|
||||
progress_message += &format!(" [Video: #{}]", i + 1);
|
||||
}
|
||||
pb.set_message(progress_message)
|
||||
}
|
||||
|
||||
let path = self
|
||||
.download_subtitle(subtitle.clone(), videos[i.min(videos.len() - 1)].length)
|
||||
.await?;
|
||||
debug!(
|
||||
"Downloaded {} subtitles{}",
|
||||
subtitle.locale,
|
||||
(!not_cc).then_some(" (cc)").unwrap_or_default(),
|
||||
);
|
||||
subtitles.push(FFmpegSubtitleMeta {
|
||||
path,
|
||||
locale: subtitle.locale.clone(),
|
||||
cc: !not_cc,
|
||||
start_time: subtitle_offsets.get(&j).cloned(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
for format in self.formats.iter() {
|
||||
if let Some(skip_events) = &format.metadata.skip_events {
|
||||
let (file, path) = tempfile(".chapter")?.into_parts();
|
||||
chapters = Some((
|
||||
|
|
@ -421,17 +584,30 @@ impl Downloader {
|
|||
let mut metadata = vec![];
|
||||
|
||||
for (i, meta) in videos.iter().enumerate() {
|
||||
if let Some(start_time) = meta.start_time {
|
||||
input.extend(["-ss".to_string(), format_time_delta(start_time)])
|
||||
}
|
||||
input.extend(["-i".to_string(), meta.path.to_string_lossy().to_string()]);
|
||||
maps.extend(["-map".to_string(), i.to_string()]);
|
||||
metadata.extend([
|
||||
format!("-metadata:s:v:{}", i),
|
||||
format!("title={}", meta.title),
|
||||
format!(
|
||||
"title={}",
|
||||
if videos.len() == 1 {
|
||||
"Default".to_string()
|
||||
} else {
|
||||
format!("#{}", i + 1)
|
||||
}
|
||||
),
|
||||
]);
|
||||
// the empty language metadata is created to avoid that metadata from the original track
|
||||
// is copied
|
||||
metadata.extend([format!("-metadata:s:v:{}", i), "language=".to_string()])
|
||||
}
|
||||
for (i, meta) in audios.iter().enumerate() {
|
||||
if let Some(start_time) = meta.start_time {
|
||||
input.extend(["-ss".to_string(), format_time_delta(start_time)])
|
||||
}
|
||||
input.extend(["-i".to_string(), meta.path.to_string_lossy().to_string()]);
|
||||
maps.extend(["-map".to_string(), (i + videos.len()).to_string()]);
|
||||
metadata.extend([
|
||||
|
|
@ -439,13 +615,20 @@ impl Downloader {
|
|||
format!(
|
||||
"language={}",
|
||||
self.audio_locale_output_map
|
||||
.get(&meta.language)
|
||||
.unwrap_or(&meta.language.to_string())
|
||||
.get(&meta.locale)
|
||||
.unwrap_or(&meta.locale.to_string())
|
||||
),
|
||||
]);
|
||||
metadata.extend([
|
||||
format!("-metadata:s:a:{}", i),
|
||||
format!("title={}", meta.title),
|
||||
format!(
|
||||
"title={}",
|
||||
if videos.len() == 1 {
|
||||
meta.locale.to_human_readable()
|
||||
} else {
|
||||
format!("{} [Video: #{}]", meta.locale.to_human_readable(), i + 1,)
|
||||
}
|
||||
),
|
||||
]);
|
||||
}
|
||||
|
||||
|
|
@ -465,6 +648,9 @@ impl Downloader {
|
|||
|
||||
if container_supports_softsubs {
|
||||
for (i, meta) in subtitles.iter().enumerate() {
|
||||
if let Some(start_time) = meta.start_time {
|
||||
input.extend(["-ss".to_string(), format_time_delta(start_time)])
|
||||
}
|
||||
input.extend(["-i".to_string(), meta.path.to_string_lossy().to_string()]);
|
||||
maps.extend([
|
||||
"-map".to_string(),
|
||||
|
|
@ -475,13 +661,22 @@ impl Downloader {
|
|||
format!(
|
||||
"language={}",
|
||||
self.subtitle_locale_output_map
|
||||
.get(&meta.language)
|
||||
.unwrap_or(&meta.language.to_string())
|
||||
.get(&meta.locale)
|
||||
.unwrap_or(&meta.locale.to_string())
|
||||
),
|
||||
]);
|
||||
metadata.extend([
|
||||
format!("-metadata:s:s:{}", i),
|
||||
format!("title={}", meta.title),
|
||||
format!("title={}", {
|
||||
let mut title = meta.locale.to_string();
|
||||
if meta.cc {
|
||||
title += " (CC)"
|
||||
}
|
||||
if videos.len() > 1 {
|
||||
title += &format!(" [Video: #{}]", i + 1)
|
||||
}
|
||||
title
|
||||
}),
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
|
@ -523,10 +718,7 @@ impl Downloader {
|
|||
|
||||
// set default subtitle
|
||||
if let Some(default_subtitle) = self.default_subtitle {
|
||||
if let Some(position) = subtitles
|
||||
.iter()
|
||||
.position(|m| m.language == default_subtitle)
|
||||
{
|
||||
if let Some(position) = subtitles.iter().position(|m| m.locale == default_subtitle) {
|
||||
if container_supports_softsubs {
|
||||
match dst.extension().unwrap_or_default().to_str().unwrap() {
|
||||
"mov" | "mp4" => output_presets.extend([
|
||||
|
|
@ -585,7 +777,7 @@ impl Downloader {
|
|||
if container_supports_softsubs {
|
||||
if let Some(position) = subtitles
|
||||
.iter()
|
||||
.position(|meta| meta.language == default_subtitle)
|
||||
.position(|meta| meta.locale == default_subtitle)
|
||||
{
|
||||
command_args.extend([
|
||||
format!("-disposition:s:s:{}", position),
|
||||
|
|
@ -597,9 +789,7 @@ impl Downloader {
|
|||
|
||||
// set the 'forced' flag to CC subtitles
|
||||
for (i, subtitle) in subtitles.iter().enumerate() {
|
||||
// well, checking if the title contains '(CC)' might not be the best solutions from a
|
||||
// performance perspective but easier than adjusting the `FFmpegMeta` struct
|
||||
if !subtitle.title.contains("(CC)") {
|
||||
if !subtitle.cc {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
@ -632,7 +822,7 @@ impl Downloader {
|
|||
// create parent directory if it does not exist
|
||||
if let Some(parent) = dst.parent() {
|
||||
if !parent.exists() {
|
||||
std::fs::create_dir_all(parent)?
|
||||
fs::create_dir_all(parent)?
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -650,7 +840,7 @@ impl Downloader {
|
|||
let ffmpeg_progress_cancellation_token = ffmpeg_progress_cancel.clone();
|
||||
let ffmpeg_progress = tokio::spawn(async move {
|
||||
ffmpeg_progress(
|
||||
max_frames as u64,
|
||||
max_frames,
|
||||
fifo,
|
||||
format!("{:<1$}", "Generating output file", fmt_space + 1),
|
||||
ffmpeg_progress_cancellation_token,
|
||||
|
|
@ -681,7 +871,7 @@ impl Downloader {
|
|||
let segments = stream_data.segments();
|
||||
|
||||
// sum the length of all streams up
|
||||
estimated_required_space += estimate_variant_file_size(stream_data, &segments);
|
||||
estimated_required_space += estimate_stream_data_file_size(stream_data, &segments);
|
||||
}
|
||||
|
||||
let tmp_stat = fs2::statvfs(temp_directory()).unwrap();
|
||||
|
|
@ -727,11 +917,16 @@ impl Downloader {
|
|||
Ok((tmp_required, dst_required))
|
||||
}
|
||||
|
||||
async fn download_video(&self, stream_data: &StreamData, message: String) -> Result<TempPath> {
|
||||
async fn download_video(
|
||||
&self,
|
||||
stream_data: &StreamData,
|
||||
message: String,
|
||||
max_segments: Option<usize>,
|
||||
) -> Result<TempPath> {
|
||||
let tempfile = tempfile(".mp4")?;
|
||||
let (mut file, path) = tempfile.into_parts();
|
||||
|
||||
self.download_segments(&mut file, message, stream_data)
|
||||
self.download_segments(&mut file, message, stream_data, max_segments)
|
||||
.await?;
|
||||
|
||||
Ok(path)
|
||||
|
|
@ -741,7 +936,7 @@ impl Downloader {
|
|||
let tempfile = tempfile(".m4a")?;
|
||||
let (mut file, path) = tempfile.into_parts();
|
||||
|
||||
self.download_segments(&mut file, message, stream_data)
|
||||
self.download_segments(&mut file, message, stream_data, None)
|
||||
.await?;
|
||||
|
||||
Ok(path)
|
||||
|
|
@ -750,7 +945,7 @@ impl Downloader {
|
|||
async fn download_subtitle(
|
||||
&self,
|
||||
subtitle: Subtitle,
|
||||
max_length: NaiveTime,
|
||||
max_length: TimeDelta,
|
||||
) -> Result<TempPath> {
|
||||
let tempfile = tempfile(".ass")?;
|
||||
let (mut file, path) = tempfile.into_parts();
|
||||
|
|
@ -796,14 +991,20 @@ impl Downloader {
|
|||
writer: &mut impl Write,
|
||||
message: String,
|
||||
stream_data: &StreamData,
|
||||
max_segments: Option<usize>,
|
||||
) -> Result<()> {
|
||||
let segments = stream_data.segments();
|
||||
let mut segments = stream_data.segments();
|
||||
if let Some(max_segments) = max_segments {
|
||||
segments = segments
|
||||
.drain(0..max_segments.min(segments.len() - 1))
|
||||
.collect();
|
||||
}
|
||||
let total_segments = segments.len();
|
||||
|
||||
let count = Arc::new(Mutex::new(0));
|
||||
|
||||
let progress = if log::max_level() == LevelFilter::Info {
|
||||
let estimated_file_size = estimate_variant_file_size(stream_data, &segments);
|
||||
let estimated_file_size = estimate_stream_data_file_size(stream_data, &segments);
|
||||
|
||||
let progress = ProgressBar::new(estimated_file_size)
|
||||
.with_style(
|
||||
|
|
@ -820,7 +1021,7 @@ impl Downloader {
|
|||
None
|
||||
};
|
||||
|
||||
let cpus = self.download_threads;
|
||||
let cpus = self.download_threads.min(segments.len());
|
||||
let mut segs: Vec<Vec<StreamSegment>> = Vec::with_capacity(cpus);
|
||||
for _ in 0..cpus {
|
||||
segs.push(vec![])
|
||||
|
|
@ -964,12 +1165,12 @@ impl Downloader {
|
|||
}
|
||||
}
|
||||
|
||||
fn estimate_variant_file_size(stream_data: &StreamData, segments: &[StreamSegment]) -> u64 {
|
||||
fn estimate_stream_data_file_size(stream_data: &StreamData, segments: &[StreamSegment]) -> u64 {
|
||||
(stream_data.bandwidth / 8) * segments.iter().map(|s| s.length.as_secs()).sum::<u64>()
|
||||
}
|
||||
|
||||
/// Get the length and fps of a video.
|
||||
fn get_video_stats(path: &Path) -> Result<(NaiveTime, f64)> {
|
||||
fn get_video_stats(path: &Path) -> Result<(TimeDelta, f64)> {
|
||||
let video_length = Regex::new(r"Duration:\s(?P<time>\d+:\d+:\d+\.\d+),")?;
|
||||
let video_fps = Regex::new(r"(?P<fps>[\d/.]+)\sfps")?;
|
||||
|
||||
|
|
@ -996,7 +1197,8 @@ fn get_video_stats(path: &Path) -> Result<(NaiveTime, f64)> {
|
|||
|
||||
Ok((
|
||||
NaiveTime::parse_from_str(length_caps.name("time").unwrap().as_str(), "%H:%M:%S%.f")
|
||||
.unwrap(),
|
||||
.unwrap()
|
||||
.signed_duration_since(NaiveTime::MIN),
|
||||
fps_caps.name("fps").unwrap().as_str().parse().unwrap(),
|
||||
))
|
||||
}
|
||||
|
|
@ -1125,28 +1327,12 @@ fn get_subtitle_stats(path: &Path) -> Result<Vec<String>> {
|
|||
/// players. To prevent this, the subtitle entries must be manually sorted. See
|
||||
/// [crunchy-labs/crunchy-cli#208](https://github.com/crunchy-labs/crunchy-cli/issues/208) for more
|
||||
/// information.
|
||||
fn fix_subtitles(raw: &mut Vec<u8>, max_length: NaiveTime) {
|
||||
fn fix_subtitles(raw: &mut Vec<u8>, max_length: TimeDelta) {
|
||||
let re = Regex::new(
|
||||
r"^Dialogue:\s(?P<layer>\d+),(?P<start>\d+:\d+:\d+\.\d+),(?P<end>\d+:\d+:\d+\.\d+),",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// chrono panics if we try to format NaiveTime with `%2f` and the nano seconds has more than 2
|
||||
// digits so them have to be reduced manually to avoid the panic
|
||||
fn format_naive_time(native_time: NaiveTime) -> String {
|
||||
let formatted_time = native_time.format("%f").to_string();
|
||||
format!(
|
||||
"{}.{}",
|
||||
native_time.format("%T"),
|
||||
if formatted_time.len() <= 2 {
|
||||
native_time.format("%2f").to_string()
|
||||
} else {
|
||||
formatted_time.split_at(2).0.parse().unwrap()
|
||||
}
|
||||
)
|
||||
.split_off(1) // <- in the ASS spec, the hour has only one digit
|
||||
}
|
||||
|
||||
let mut entries = (vec![], vec![]);
|
||||
|
||||
let mut as_lines: Vec<String> = String::from_utf8_lossy(raw.as_slice())
|
||||
|
|
@ -1158,12 +1344,18 @@ fn fix_subtitles(raw: &mut Vec<u8>, max_length: NaiveTime) {
|
|||
if line.trim() == "[Script Info]" {
|
||||
line.push_str("\nScaledBorderAndShadow: yes")
|
||||
} else if let Some(capture) = re.captures(line) {
|
||||
let mut start = capture.name("start").map_or(NaiveTime::default(), |s| {
|
||||
NaiveTime::parse_from_str(s.as_str(), "%H:%M:%S.%f").unwrap()
|
||||
});
|
||||
let mut end = capture.name("end").map_or(NaiveTime::default(), |e| {
|
||||
NaiveTime::parse_from_str(e.as_str(), "%H:%M:%S.%f").unwrap()
|
||||
});
|
||||
let mut start = capture
|
||||
.name("start")
|
||||
.map_or(NaiveTime::default(), |s| {
|
||||
NaiveTime::parse_from_str(s.as_str(), "%H:%M:%S.%f").unwrap()
|
||||
})
|
||||
.signed_duration_since(NaiveTime::MIN);
|
||||
let mut end = capture
|
||||
.name("end")
|
||||
.map_or(NaiveTime::default(), |e| {
|
||||
NaiveTime::parse_from_str(e.as_str(), "%H:%M:%S.%f").unwrap()
|
||||
})
|
||||
.signed_duration_since(NaiveTime::MIN);
|
||||
|
||||
if start > max_length || end > max_length {
|
||||
let layer = capture
|
||||
|
|
@ -1183,8 +1375,8 @@ fn fix_subtitles(raw: &mut Vec<u8>, max_length: NaiveTime) {
|
|||
format!(
|
||||
"Dialogue: {},{},{},",
|
||||
layer,
|
||||
format_naive_time(start),
|
||||
format_naive_time(end)
|
||||
format_time_delta(start),
|
||||
format_time_delta(end)
|
||||
),
|
||||
)
|
||||
.to_string()
|
||||
|
|
@ -1209,13 +1401,10 @@ fn fix_subtitles(raw: &mut Vec<u8>, max_length: NaiveTime) {
|
|||
|
||||
fn write_ffmpeg_chapters(
|
||||
file: &mut fs::File,
|
||||
video_len: NaiveTime,
|
||||
video_len: TimeDelta,
|
||||
events: &mut Vec<(&str, &SkipEventsEvent)>,
|
||||
) -> Result<()> {
|
||||
let video_len = video_len
|
||||
.signed_duration_since(NaiveTime::MIN)
|
||||
.num_milliseconds() as f32
|
||||
/ 1000.0;
|
||||
let video_len = video_len.num_milliseconds() as f32 / 1000.0;
|
||||
events.sort_by(|(_, event_a), (_, event_b)| event_a.start.total_cmp(&event_b.start));
|
||||
|
||||
writeln!(file, ";FFMETADATA1")?;
|
||||
|
|
@ -1332,3 +1521,149 @@ async fn ffmpeg_progress<R: AsyncReadExt + Unpin>(
|
|||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct SyncVideo {
|
||||
path: TempPath,
|
||||
length: TimeDelta,
|
||||
available_frames: u64,
|
||||
idx: usize,
|
||||
}
|
||||
|
||||
fn sync_videos(mut sync_videos: Vec<SyncVideo>, value: f64) -> Result<Option<HashMap<usize, u64>>> {
|
||||
let mut result = HashMap::new();
|
||||
let hasher = HasherConfig::new().to_hasher();
|
||||
let start_frame = 50;
|
||||
|
||||
sync_videos.sort_by_key(|sv| sv.length);
|
||||
|
||||
let sync_base = sync_videos.remove(0);
|
||||
let sync_hashes = extract_frame_hashes(&sync_base.path, start_frame, 100, &hasher)?;
|
||||
|
||||
for sync_video in sync_videos {
|
||||
let mut highest_frame_match = f64::INFINITY;
|
||||
let mut frame = start_frame;
|
||||
let mut hashes = vec![];
|
||||
|
||||
loop {
|
||||
if frame == sync_video.available_frames {
|
||||
debug!(
|
||||
"Failed to sync videos, end of stream {} reached (highest frame match: {})",
|
||||
sync_video.idx + 1,
|
||||
highest_frame_match
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
hashes.drain(0..(hashes.len() as i32 - sync_hashes.len() as i32).max(0) as usize);
|
||||
hashes.extend(extract_frame_hashes(
|
||||
&sync_video.path,
|
||||
frame,
|
||||
300 - hashes.len() as u64,
|
||||
&hasher,
|
||||
)?);
|
||||
|
||||
let check_frame_windows_result = check_frame_windows(&sync_hashes, &hashes);
|
||||
if let Some(offset) = check_frame_windows_result
|
||||
.iter()
|
||||
.enumerate()
|
||||
.find_map(|(i, cfw)| (*cfw <= value).then_some(i))
|
||||
{
|
||||
result.insert(sync_video.idx, frame + offset as u64 - start_frame);
|
||||
break;
|
||||
} else {
|
||||
let curr_highest_frame_match = *check_frame_windows_result
|
||||
.iter()
|
||||
.min_by(|a, b| a.total_cmp(b))
|
||||
.unwrap();
|
||||
if curr_highest_frame_match < highest_frame_match {
|
||||
highest_frame_match = curr_highest_frame_match
|
||||
}
|
||||
}
|
||||
|
||||
frame = (frame + 300 - sync_hashes.len() as u64).min(sync_video.available_frames)
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Some(result))
|
||||
}
|
||||
|
||||
fn extract_frame_hashes(
|
||||
input_file: &Path,
|
||||
start_frame: u64,
|
||||
frame_count: u64,
|
||||
hasher: &Hasher,
|
||||
) -> Result<Vec<ImageHash>> {
|
||||
let frame_dir = tempdir(format!(
|
||||
"{}_sync_frames",
|
||||
input_file
|
||||
.file_name()
|
||||
.unwrap_or_default()
|
||||
.to_string_lossy()
|
||||
.trim_end_matches(
|
||||
&input_file
|
||||
.file_stem()
|
||||
.unwrap_or_default()
|
||||
.to_string_lossy()
|
||||
.to_string()
|
||||
)
|
||||
))?;
|
||||
let extract_output = Command::new("ffmpeg")
|
||||
.arg("-hide_banner")
|
||||
.arg("-y")
|
||||
.args(["-i", input_file.to_string_lossy().to_string().as_str()])
|
||||
.args([
|
||||
"-vf",
|
||||
format!(
|
||||
r#"select=between(n\,{}\,{}),setpts=PTS-STARTPTS"#,
|
||||
start_frame,
|
||||
start_frame + frame_count
|
||||
)
|
||||
.as_str(),
|
||||
])
|
||||
.args(["-vframes", frame_count.to_string().as_str()])
|
||||
.arg(format!("{}/%03d.jpg", frame_dir.path().to_string_lossy()))
|
||||
.output()?;
|
||||
if !extract_output.status.success() {
|
||||
bail!(
|
||||
"{}",
|
||||
String::from_utf8_lossy(extract_output.stderr.as_slice())
|
||||
)
|
||||
}
|
||||
|
||||
let mut hashes = vec![];
|
||||
for file in frame_dir.path().read_dir()? {
|
||||
let file = file?;
|
||||
let img = image::open(file.path())?;
|
||||
hashes.push(hasher.hash_image(&img))
|
||||
}
|
||||
Ok(hashes)
|
||||
}
|
||||
|
||||
fn check_frame_windows(base_hashes: &[ImageHash], check_hashes: &[ImageHash]) -> Vec<f64> {
|
||||
let mut results = vec![];
|
||||
|
||||
for i in 0..(check_hashes.len() - base_hashes.len()) {
|
||||
let check_window = &check_hashes[i..(base_hashes.len() + i)];
|
||||
let sum = std::iter::zip(base_hashes, check_window)
|
||||
.map(|(a, b)| a.dist(b))
|
||||
.sum::<u32>();
|
||||
results.push(sum as f64 / check_window.len() as f64);
|
||||
}
|
||||
results
|
||||
}
|
||||
|
||||
fn format_time_delta(time_delta: TimeDelta) -> String {
|
||||
let hours = time_delta.num_hours();
|
||||
let minutes = time_delta.num_minutes() - time_delta.num_hours() * 60;
|
||||
let seconds = time_delta.num_seconds() - time_delta.num_minutes() * 60;
|
||||
let milliseconds = time_delta.num_milliseconds() - time_delta.num_seconds() * 1000;
|
||||
|
||||
format!(
|
||||
"{}:{:0>2}:{:0>2}.{:0>3}",
|
||||
hours, minutes, seconds, milliseconds
|
||||
)
|
||||
}
|
||||
|
||||
fn len_from_segments(segments: &[StreamSegment]) -> TimeDelta {
|
||||
TimeDelta::milliseconds(segments.iter().map(|s| s.length.as_millis()).sum::<u128>() as i64)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue