diff --git a/Cargo.lock b/Cargo.lock
index 61832fe..26f81b9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -179,18 +179,6 @@ version = "3.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"
-[[package]]
-name = "bytemuck"
-version = "1.15.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d6d68c57235a3a081186990eca2867354726650f42f7516ca50c28d6281fd15"
-
-[[package]]
-name = "byteorder"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
-
[[package]]
name = "bytes"
version = "1.6.0"
@@ -381,8 +369,6 @@ dependencies = [
"fs2",
"futures-util",
"http",
- "image",
- "image_hasher",
"indicatif",
"lazy_static",
"log",
@@ -391,6 +377,7 @@ dependencies = [
"regex",
"reqwest",
"rustls-native-certs",
+ "rusty-chromaprint",
"serde",
"serde_json",
"serde_plain",
@@ -951,32 +938,6 @@ dependencies = [
"unicode-normalization",
]
-[[package]]
-name = "image"
-version = "0.25.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd54d660e773627692c524beaad361aca785a4f9f5730ce91f42aabe5bce3d11"
-dependencies = [
- "bytemuck",
- "byteorder",
- "num-traits",
- "zune-core",
- "zune-jpeg",
-]
-
-[[package]]
-name = "image_hasher"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9481465fe767d92494987319b0b447a5829edf57f09c52bf8639396abaaeaf78"
-dependencies = [
- "base64 0.22.0",
- "image",
- "rustdct",
- "serde",
- "transpose",
-]
-
[[package]]
name = "indexmap"
version = "1.9.3"
@@ -1417,6 +1378,15 @@ dependencies = [
"proc-macro2",
]
+[[package]]
+name = "realfft"
+version = "3.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "953d9f7e5cdd80963547b456251296efc2626ed4e3cbf36c869d9564e0220571"
+dependencies = [
+ "rustfft",
+]
+
[[package]]
name = "redox_users"
version = "0.4.5"
@@ -1531,21 +1501,24 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b833d8d034ea094b1ea68aa6d5c740e0d04bad9d16568d08ba6f76823a114316"
+[[package]]
+name = "rubato"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6dd52e80cfc21894deadf554a5673002938ae4625f7a283e536f9cf7c17b0d5"
+dependencies = [
+ "num-complex",
+ "num-integer",
+ "num-traits",
+ "realfft",
+]
+
[[package]]
name = "rustc-demangle"
version = "0.1.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
-[[package]]
-name = "rustdct"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b61555105d6a9bf98797c063c362a1d24ed8ab0431655e38f1cf51e52089551"
-dependencies = [
- "rustfft",
-]
-
[[package]]
name = "rustfft"
version = "6.2.0"
@@ -1628,6 +1601,16 @@ dependencies = [
"untrusted",
]
+[[package]]
+name = "rusty-chromaprint"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1755646867c36ecb391776deaa0b557a76d3badf20c142de7282630c34b20440"
+dependencies = [
+ "rubato",
+ "rustfft",
+]
+
[[package]]
name = "ryu"
version = "1.0.17"
@@ -2501,18 +2484,3 @@ name = "zeroize"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63381fa6624bf92130a6b87c0d07380116f80b565c42cf0d754136f0238359ef"
-
-[[package]]
-name = "zune-core"
-version = "0.4.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a"
-
-[[package]]
-name = "zune-jpeg"
-version = "0.4.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec866b44a2a1fd6133d363f073ca1b179f438f99e7e5bfb1e33f7181facfe448"
-dependencies = [
- "zune-core",
-]
diff --git a/README.md b/README.md
index 7316f57..1fdf348 100644
--- a/README.md
+++ b/README.md
@@ -462,7 +462,7 @@ The `archive` command lets you download episodes with multiple audios and subtit
In the best case, when multiple audio & subtitle tracks are used, there is only one *video* track and all other languages can be stored as audio-only.
But, as said, this is not always the case.
With the `-m` / `--merge` flag you can define the behaviour when an episodes' video tracks differ in length.
- Valid options are `audio` - store one video and all other languages as audio only; `video` - store the video + audio for every language; `auto` - detect if videos differ in length: if so, behave like `video` - otherwise like `audio`.
+ Valid options are `audio` - store one video and all other languages as audio only; `video` - store the video + audio for every language; `auto` - detect if videos differ in length: if so, behave like `video` - otherwise like `audio`; `sync` - detect if videos differ in length: if so, it tries to find the offset of matching audio parts and removes the offset from the beginning, otherwise it behaves like `audio`.
Subtitles will always match the primary audio and video.
```shell
@@ -482,15 +482,18 @@ The `archive` command lets you download episodes with multiple audios and subtit
Default are `200` milliseconds.
-- Sync start
+- Sync tolerance
- If you want that all videos of the same episode should start at the same time and `--merge` doesn't fit your needs (e.g. one video has an intro, all other doesn't), you might consider using the `--sync-start`.
- It tries to sync the timing of all downloaded audios to match one video.
- This is done by downloading the first few segments/frames of all video tracks that differ in length and comparing them frame by frame.
- The flag takes an optional value determines how accurate the syncing is, generally speaking everything over 15 begins to be more inaccurate and everything below 6 is too accurate (and won't succeed).
- When the syncing fails, the command is continued as if `--sync-start` wasn't provided for this episode.
+ Sometimes two video tracks are downloaded with `--merge` set to `sync` because the audio fingerprinting fails to identify matching audio parts (e.g. opening).
+ To prevent this, you can use the `--sync-tolerance` flag to specify the difference by which two fingerprints are considered equal.
- Default is `7.5`.
+ Default is `6`.
+
+- Sync precision
+
+ If you use `--merge` set to `sync` and the syncing seems to be not accurate enough or takes to long, you can use the `--sync-precision` flag to specify the amount of offset determination runs from which the final offset is calculated.
+
+ Default is `4`.
- Language tagging
diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml
index e31ef46..517284a 100644
--- a/crunchy-cli-core/Cargo.toml
+++ b/crunchy-cli-core/Cargo.toml
@@ -24,14 +24,13 @@ derive_setters = "0.1"
futures-util = { version = "0.3", features = ["io"] }
fs2 = "0.4"
http = "1.1"
-image = { version = "0.25", features = ["jpeg"], default-features = false }
-image_hasher = "2.0"
indicatif = "0.17"
lazy_static = "1.4"
log = { version = "0.4", features = ["std"] }
num_cpus = "1.16"
regex = "1.10"
reqwest = { version = "0.12", features = ["socks", "stream"] }
+rusty-chromaprint = "0.2"
serde = "1.0"
serde_json = "1.0"
serde_plain = "1.0"
diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs
index 3525a4a..77cf50f 100644
--- a/crunchy-cli-core/src/archive/command.rs
+++ b/crunchy-cli-core/src/archive/command.rs
@@ -90,32 +90,31 @@ pub struct Archive {
pub(crate) resolution: Resolution,
#[arg(
- help = "Sets the behavior of the stream merging. Valid behaviors are 'auto', 'audio' and 'video'"
+ help = "Sets the behavior of the stream merging. Valid behaviors are 'auto', 'sync', 'audio' and 'video'"
)]
#[arg(
long_help = "Because of local restrictions (or other reasons) some episodes with different languages does not have the same length (e.g. when some scenes were cut out). \
With this flag you can set the behavior when handling multiple language.
- Valid options are 'audio' (stores one video and all other languages as audio only), 'video' (stores the video + audio for every language) and 'auto' (detects if videos differ in length: if so, behave like 'video' else like 'audio')"
+ Valid options are 'audio' (stores one video and all other languages as audio only), 'video' (stores the video + audio for every language), 'auto' (detects if videos differ in length: if so, behave like 'video' else like 'audio') and 'sync' (detects if videos differ in length: if so, tries to find the offset of matching audio parts and removes it from the beginning, otherwise it behaves like 'audio')"
)]
#[arg(short, long, default_value = "auto")]
#[arg(value_parser = MergeBehavior::parse)]
pub(crate) merge: MergeBehavior,
#[arg(
- help = "If the merge behavior is 'auto', only download multiple video tracks if their length difference is higher than the given milliseconds"
+ help = "If the merge behavior is 'auto' or 'sync', consider videos to be of equal lengths if the difference in length is smaller than the specified milliseconds"
)]
#[arg(long, default_value_t = 200)]
pub(crate) merge_time_tolerance: u32,
- #[arg(help = "Tries to sync the timing of all downloaded audios to match one video")]
#[arg(
- long_help = "Tries to sync the timing of all downloaded audios to match one video. \
- This is done by downloading the first few segments/frames of all video tracks that differ in length and comparing them frame by frame. \
- The value of this flag determines how accurate the syncing is, generally speaking everything over 15 begins to be more inaccurate and everything below 6 is too accurate (and won't succeed). \
- If you want to provide a custom value to this flag, you have to set it with an equals (e.g. `--sync-start=10` instead of `--sync-start 10`). \
- When the syncing fails, the command is continued as if `--sync-start` wasn't provided for this episode
- "
+ help = "If the merge behavior is 'sync', specify the difference by which two fingerprints are considered equal, higher values can help when the algorithm fails"
)]
- #[arg(long, require_equals = true, num_args = 0..=1, default_missing_value = "7.5")]
- pub(crate) sync_start: Option,
+ #[arg(long, default_value_t = 6)]
+ pub(crate) sync_tolerance: u32,
+ #[arg(
+ help = "If the merge behavior is 'sync', specify the amount of offset determination runs from which the final offset is calculated, higher values will increase the time required but lead to more precise offsets"
+ )]
+ #[arg(long, default_value_t = 4)]
+ pub(crate) sync_precision: u32,
#[arg(
help = "Specified which language tagging the audio and subtitle tracks and language specific format options should have. \
@@ -229,18 +228,10 @@ impl Execute for Archive {
}
if self.include_chapters
+ && !matches!(self.merge, MergeBehavior::Sync)
&& !matches!(self.merge, MergeBehavior::Audio)
- && self.sync_start.is_none()
{
- bail!("`--include-chapters` can only be used if `--merge` is set to 'audio' or `--sync-start` is set")
- }
-
- if !matches!(self.merge, MergeBehavior::Auto) && self.sync_start.is_some() {
- bail!("`--sync-start` can only be used if `--merge` is set to `auto`")
- }
-
- if self.sync_start.is_some() && self.ffmpeg_preset.is_none() {
- warn!("Using `--sync-start` without `--ffmpeg-preset` might produce worse sync results than with `--ffmpeg-preset` set")
+ bail!("`--include-chapters` can only be used if `--merge` is set to 'audio' or 'sync'")
}
self.audio = all_locale_in_locales(self.audio.clone());
@@ -317,7 +308,14 @@ impl Execute for Archive {
.audio_sort(Some(self.audio.clone()))
.subtitle_sort(Some(self.subtitle.clone()))
.no_closed_caption(self.no_closed_caption)
- .sync_start_value(self.sync_start)
+ .sync_tolerance(match self.merge {
+ MergeBehavior::Sync => Some(self.sync_tolerance),
+ _ => None,
+ })
+ .sync_precision(match self.merge {
+ MergeBehavior::Sync => Some(self.sync_precision),
+ _ => None,
+ })
.threads(self.threads)
.audio_locale_output_map(
zip(self.audio.clone(), self.output_audio_locales.clone()).collect(),
@@ -560,7 +558,7 @@ async fn get_format(
},
},
}),
- MergeBehavior::Auto => {
+ MergeBehavior::Auto | MergeBehavior::Sync => {
let mut d_formats: Vec<(Duration, DownloadFormat)> = vec![];
for (single_format, video, audio, subtitles) in format_pairs {
diff --git a/crunchy-cli-core/src/archive/filter.rs b/crunchy-cli-core/src/archive/filter.rs
index f638c50..b08fb6c 100644
--- a/crunchy-cli-core/src/archive/filter.rs
+++ b/crunchy-cli-core/src/archive/filter.rs
@@ -333,7 +333,7 @@ impl Filter for ArchiveFilter {
.unwrap()
.push(episode.season_number)
}
-
+
if episodes.is_empty() {
return Ok(None);
}
diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs
index bd7bf3d..cfec7b4 100644
--- a/crunchy-cli-core/src/utils/download.rs
+++ b/crunchy-cli-core/src/utils/download.rs
@@ -2,15 +2,13 @@ use crate::utils::ffmpeg::FFmpegPreset;
use crate::utils::filter::real_dedup_vec;
use crate::utils::fmt::format_time_delta;
use crate::utils::log::progress;
-use crate::utils::os::{
- cache_dir, is_special_file, temp_directory, temp_named_pipe, tempdir, tempfile,
-};
+use crate::utils::os::{cache_dir, is_special_file, temp_directory, temp_named_pipe, tempfile};
use crate::utils::rate_limit::RateLimiterService;
+use crate::utils::sync::{sync_audios, SyncAudio};
use anyhow::{bail, Result};
use chrono::{NaiveTime, TimeDelta};
use crunchyroll_rs::media::{SkipEvents, SkipEventsEvent, StreamData, StreamSegment, Subtitle};
use crunchyroll_rs::Locale;
-use image_hasher::{Hasher, HasherConfig, ImageHash};
use indicatif::{ProgressBar, ProgressDrawTarget, ProgressFinish, ProgressStyle};
use log::{debug, warn, LevelFilter};
use regex::Regex;
@@ -39,6 +37,7 @@ pub enum MergeBehavior {
Video,
Audio,
Auto,
+ Sync,
}
impl MergeBehavior {
@@ -47,6 +46,7 @@ impl MergeBehavior {
"video" => MergeBehavior::Video,
"audio" => MergeBehavior::Audio,
"auto" => MergeBehavior::Auto,
+ "sync" => MergeBehavior::Sync,
_ => return Err(format!("'{}' is not a valid merge behavior", s)),
})
}
@@ -64,7 +64,8 @@ pub struct DownloadBuilder {
force_hardsub: bool,
download_fonts: bool,
no_closed_caption: bool,
- sync_start_value: Option,
+ sync_tolerance: Option,
+ sync_precision: Option,
threads: usize,
ffmpeg_threads: Option,
audio_locale_output_map: HashMap,
@@ -84,7 +85,8 @@ impl DownloadBuilder {
force_hardsub: false,
download_fonts: false,
no_closed_caption: false,
- sync_start_value: None,
+ sync_tolerance: None,
+ sync_precision: None,
threads: num_cpus::get(),
ffmpeg_threads: None,
audio_locale_output_map: HashMap::new(),
@@ -106,7 +108,8 @@ impl DownloadBuilder {
download_fonts: self.download_fonts,
no_closed_caption: self.no_closed_caption,
- sync_start_value: self.sync_start_value,
+ sync_tolerance: self.sync_tolerance,
+ sync_precision: self.sync_precision,
download_threads: self.threads,
ffmpeg_threads: self.ffmpeg_threads,
@@ -165,7 +168,8 @@ pub struct Downloader {
download_fonts: bool,
no_closed_caption: bool,
- sync_start_value: Option,
+ sync_tolerance: Option,
+ sync_precision: Option,
download_threads: usize,
ffmpeg_threads: Option,
@@ -245,6 +249,7 @@ impl Downloader {
let mut video_offset = None;
let mut audio_offsets = HashMap::new();
let mut subtitle_offsets = HashMap::new();
+ let mut raw_audios = vec![];
let mut videos = vec![];
let mut audios = vec![];
let mut subtitles = vec![];
@@ -263,40 +268,33 @@ impl Downloader {
.max()
.unwrap();
- if self.formats.len() > 1 && self.sync_start_value.is_some() {
- let all_segments_count: Vec = self
- .formats
- .iter()
- .map(|f| f.video.0.segments().len())
- .collect();
- let sync_segments = 11.max(
- all_segments_count.iter().max().unwrap() - all_segments_count.iter().min().unwrap(),
- );
- let mut sync_vids = vec![];
- for (i, format) in self.formats.iter().enumerate() {
+ // downloads all audios
+ for (i, format) in self.formats.iter().enumerate() {
+ for (stream_data, locale) in &format.audios {
let path = self
- .download_video(
- &format.video.0,
- format!("Downloading video #{} sync segments", i + 1),
- Some(sync_segments),
+ .download_audio(
+ stream_data,
+ format!("{:<1$}", format!("Downloading {} audio", locale), fmt_space),
)
.await?;
- sync_vids.push(SyncVideo {
+ raw_audios.push(SyncAudio {
+ format_id: i,
path,
- length: len_from_segments(&format.video.0.segments()),
- available_frames: (len_from_segments(
- &format.video.0.segments()[0..sync_segments],
- )
- .num_milliseconds() as f64
- * format.video.0.fps().unwrap()
- / 1000.0) as u64,
- idx: i,
+ locale: locale.clone(),
+ sample_rate: stream_data.sampling_rate().unwrap(),
+ video_idx: i,
})
}
+ }
+ if self.formats.len() > 1 && self.sync_tolerance.is_some() {
let _progress_handler =
progress!("Syncing video start times (this might take some time)");
- let mut offsets = sync_videos(sync_vids, self.sync_start_value.unwrap())?;
+ let mut offsets = sync_audios(
+ &raw_audios,
+ self.sync_tolerance.unwrap(),
+ self.sync_precision.unwrap(),
+ )?;
drop(_progress_handler);
let mut offset_pre_checked = false;
@@ -307,19 +305,14 @@ impl Downloader {
.enumerate()
.map(|(i, f)| {
len_from_segments(&f.video.0.segments())
- - TimeDelta::milliseconds(
- tmp_offsets
- .get(&i)
- .map(|o| (*o as f64 / f.video.0.fps().unwrap() * 1000.0) as i64)
- .unwrap_or_default(),
- )
+ - tmp_offsets.get(&i).copied().unwrap_or_default()
})
.collect();
let min = formats_with_offset.iter().min().unwrap();
let max = formats_with_offset.iter().max().unwrap();
if max.num_seconds() - min.num_seconds() > 15 {
- warn!("Found difference of >15 seconds after sync, skipping applying it");
+ warn!("Found difference of >15 seconds after sync, so the application was skipped");
offsets = None;
offset_pre_checked = true
}
@@ -331,7 +324,7 @@ impl Downloader {
let mut audio_count: usize = 0;
let mut subtitle_count: usize = 0;
for (i, format) in self.formats.iter().enumerate() {
- let format_fps = format.video.0.fps().unwrap();
+ let offset = offsets.get(&i).copied().unwrap_or_default();
let format_len = format
.video
.0
@@ -339,7 +332,7 @@ impl Downloader {
.iter()
.map(|s| s.length.as_millis())
.sum::() as u64
- - offsets.get(&i).map_or(0, |o| *o);
+ - offset.num_milliseconds() as u64;
if format_len > root_format_length {
root_format_idx = i;
root_format_length = format_len;
@@ -347,23 +340,13 @@ impl Downloader {
for _ in &format.audios {
if let Some(offset) = &offsets.get(&i) {
- audio_offsets.insert(
- audio_count,
- TimeDelta::milliseconds(
- (**offset as f64 / format_fps * 1000.0) as i64,
- ),
- );
+ audio_offsets.insert(audio_count, **offset);
}
audio_count += 1
}
for _ in &format.subtitles {
if let Some(offset) = &offsets.get(&i) {
- subtitle_offsets.insert(
- subtitle_count,
- TimeDelta::milliseconds(
- (**offset as f64 / format_fps * 1000.0) as i64,
- ),
- );
+ subtitle_offsets.insert(subtitle_count, **offset);
}
subtitle_count += 1
}
@@ -390,20 +373,28 @@ impl Downloader {
root_format.subtitles.extend(subtitle_append);
self.formats = vec![root_format];
- video_offset = offsets.get(&root_format_idx).map(|o| {
- TimeDelta::milliseconds(
- (*o as f64 / self.formats[0].video.0.fps().unwrap() * 1000.0) as i64,
- )
- })
+ video_offset = offsets.get(&root_format_idx).copied();
+ for raw_audio in raw_audios.iter_mut() {
+ raw_audio.video_idx = root_format_idx;
+ }
} else {
for format in &mut self.formats {
format.metadata.skip_events = None
}
+ if !offset_pre_checked {
+ warn!("Couldn't find reliable sync positions")
+ }
}
+ }
- if !offset_pre_checked {
- warn!("Couldn't find reliable sync positions")
- }
+ // add audio metadata
+ for raw_audio in raw_audios {
+ audios.push(FFmpegAudioMeta {
+ path: raw_audio.path,
+ locale: raw_audio.locale,
+ start_time: audio_offsets.get(&raw_audio.format_id).copied(),
+ video_idx: raw_audio.video_idx,
+ })
}
// downloads all videos
@@ -435,24 +426,6 @@ impl Downloader {
})
}
- // downloads all audios
- for (i, format) in self.formats.iter().enumerate() {
- for (j, (stream_data, locale)) in format.audios.iter().enumerate() {
- let path = self
- .download_audio(
- stream_data,
- format!("{:<1$}", format!("Downloading {} audio", locale), fmt_space),
- )
- .await?;
- audios.push(FFmpegAudioMeta {
- path,
- locale: locale.clone(),
- start_time: audio_offsets.get(&j).cloned(),
- video_idx: i,
- })
- }
- }
-
for (i, format) in self.formats.iter().enumerate() {
if format.subtitles.is_empty() {
continue;
@@ -1538,134 +1511,6 @@ async fn ffmpeg_progress(
Ok(())
}
-struct SyncVideo {
- path: TempPath,
- length: TimeDelta,
- available_frames: u64,
- idx: usize,
-}
-
-fn sync_videos(mut sync_videos: Vec, value: f64) -> Result>> {
- let mut result = HashMap::new();
- let hasher = HasherConfig::new().preproc_dct().to_hasher();
- let start_frame = 300;
-
- sync_videos.sort_by_key(|sv| sv.length);
-
- let sync_base = sync_videos.remove(0);
- let sync_hashes = extract_frame_hashes(&sync_base.path, start_frame, 50, &hasher)?;
-
- for sync_video in sync_videos {
- let mut highest_frame_match = f64::INFINITY;
- let mut frame = start_frame;
- let mut hashes = vec![];
-
- loop {
- if frame == sync_video.available_frames {
- debug!(
- "Failed to sync videos, end of stream {} reached (highest frame match: {})",
- sync_video.idx + 1,
- highest_frame_match
- );
- return Ok(None);
- }
-
- hashes.drain(0..(hashes.len() as i32 - sync_hashes.len() as i32).max(0) as usize);
- hashes.extend(extract_frame_hashes(
- &sync_video.path,
- frame,
- 300 - hashes.len() as u64,
- &hasher,
- )?);
-
- let mut check_frame_windows_result: Vec<(usize, f64)> =
- check_frame_windows(&sync_hashes, &hashes)
- .into_iter()
- .enumerate()
- .collect();
- check_frame_windows_result.sort_by(|(_, a), (_, b)| a.partial_cmp(&b).unwrap());
- if check_frame_windows_result[0].1 <= value {
- result.insert(
- sync_video.idx,
- frame + check_frame_windows_result[0].0 as u64 - start_frame,
- );
- break;
- } else if check_frame_windows_result[0].1 < highest_frame_match {
- highest_frame_match = check_frame_windows_result[0].1
- }
-
- frame = (frame + 300 - sync_hashes.len() as u64).min(sync_video.available_frames)
- }
- }
-
- Ok(Some(result))
-}
-
-fn extract_frame_hashes(
- input_file: &Path,
- start_frame: u64,
- frame_count: u64,
- hasher: &Hasher,
-) -> Result> {
- let frame_dir = tempdir(format!(
- "{}_sync_frames",
- input_file
- .file_name()
- .unwrap_or_default()
- .to_string_lossy()
- .trim_end_matches(
- &input_file
- .file_stem()
- .unwrap_or_default()
- .to_string_lossy()
- .to_string()
- )
- ))?;
- let extract_output = Command::new("ffmpeg")
- .arg("-hide_banner")
- .arg("-y")
- .args(["-i", input_file.to_string_lossy().to_string().as_str()])
- .args([
- "-vf",
- format!(
- r#"select=between(n\,{}\,{}),setpts=PTS-STARTPTS,scale=-1:240"#,
- start_frame,
- start_frame + frame_count
- )
- .as_str(),
- ])
- .args(["-vframes", frame_count.to_string().as_str()])
- .arg(format!("{}/%03d.jpg", frame_dir.path().to_string_lossy()))
- .output()?;
- if !extract_output.status.success() {
- bail!(
- "{}",
- String::from_utf8_lossy(extract_output.stderr.as_slice())
- )
- }
-
- let mut hashes = vec![];
- for file in frame_dir.path().read_dir()? {
- let file = file?;
- let img = image::open(file.path())?;
- hashes.push(hasher.hash_image(&img))
- }
- Ok(hashes)
-}
-
-fn check_frame_windows(base_hashes: &[ImageHash], check_hashes: &[ImageHash]) -> Vec {
- let mut results = vec![];
-
- for i in 0..(check_hashes.len() - base_hashes.len()) {
- let check_window = &check_hashes[i..(base_hashes.len() + i)];
- let sum = std::iter::zip(base_hashes, check_window)
- .map(|(a, b)| a.dist(b))
- .sum::();
- results.push(sum as f64 / check_window.len() as f64);
- }
- results
-}
-
fn len_from_segments(segments: &[StreamSegment]) -> TimeDelta {
TimeDelta::milliseconds(segments.iter().map(|s| s.length.as_millis()).sum::() as i64)
}
diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs
index 1987e87..0a71838 100644
--- a/crunchy-cli-core/src/utils/format.rs
+++ b/crunchy-cli-core/src/utils/format.rs
@@ -544,7 +544,7 @@ impl Format {
path.set_file_name(format!("{}.{}", &name[..(255 - ext.len() - 1)], ext))
}
}
- path.into_iter()
+ path.iter()
.map(|s| {
if s.len() > 255 {
s.to_string_lossy()[..255].to_string()
diff --git a/crunchy-cli-core/src/utils/mod.rs b/crunchy-cli-core/src/utils/mod.rs
index 72a0908..6260047 100644
--- a/crunchy-cli-core/src/utils/mod.rs
+++ b/crunchy-cli-core/src/utils/mod.rs
@@ -11,4 +11,5 @@ pub mod log;
pub mod os;
pub mod parse;
pub mod rate_limit;
+pub mod sync;
pub mod video;
diff --git a/crunchy-cli-core/src/utils/os.rs b/crunchy-cli-core/src/utils/os.rs
index b65abc2..a216f87 100644
--- a/crunchy-cli-core/src/utils/os.rs
+++ b/crunchy-cli-core/src/utils/os.rs
@@ -7,7 +7,7 @@ use std::pin::Pin;
use std::process::{Command, Stdio};
use std::task::{Context, Poll};
use std::{env, fs, io};
-use tempfile::{Builder, NamedTempFile, TempDir, TempPath};
+use tempfile::{Builder, NamedTempFile, TempPath};
use tokio::io::{AsyncRead, ReadBuf};
pub fn has_ffmpeg() -> bool {
@@ -46,22 +46,6 @@ pub fn tempfile>(suffix: S) -> io::Result {
Ok(tempfile)
}
-/// Any tempdir should be created with this function. The prefix and directory of every directory
-/// created with this function stays the same which is helpful to query all existing tempdirs and
-/// e.g. remove them in a case of ctrl-c. Having one function also good to prevent mistakes like
-/// setting the wrong prefix if done manually.
-pub fn tempdir>(suffix: S) -> io::Result {
- let tempdir = Builder::default()
- .prefix(".crunchy-cli_")
- .suffix(suffix.as_ref())
- .tempdir_in(temp_directory())?;
- debug!(
- "Created temporary directory: {}",
- tempdir.path().to_string_lossy()
- );
- Ok(tempdir)
-}
-
pub fn cache_dir>(name: S) -> io::Result {
let cache_dir = temp_directory().join(format!(".crunchy-cli_{}_cache", name.as_ref()));
fs::create_dir_all(&cache_dir)?;
diff --git a/crunchy-cli-core/src/utils/sync.rs b/crunchy-cli-core/src/utils/sync.rs
new file mode 100644
index 0000000..1e9bc42
--- /dev/null
+++ b/crunchy-cli-core/src/utils/sync.rs
@@ -0,0 +1,432 @@
+use std::io::Read;
+use std::process::Stdio;
+use std::{
+ cmp,
+ collections::{HashMap, HashSet},
+ mem,
+ ops::Not,
+ path::Path,
+ process::Command,
+};
+
+use chrono::TimeDelta;
+use crunchyroll_rs::Locale;
+use log::debug;
+use tempfile::TempPath;
+
+use anyhow::{bail, Result};
+use rusty_chromaprint::{Configuration, Fingerprinter};
+
+use super::fmt::format_time_delta;
+
+pub struct SyncAudio {
+ pub format_id: usize,
+ pub path: TempPath,
+ pub locale: Locale,
+ pub sample_rate: u32,
+ pub video_idx: usize,
+}
+
+#[derive(Debug, Clone, Copy)]
+struct TimeRange {
+ start: f64,
+ end: f64,
+}
+
+pub fn sync_audios(
+ available_audios: &Vec,
+ sync_tolerance: u32,
+ sync_precision: u32,
+) -> Result>> {
+ let mut result: HashMap = HashMap::new();
+
+ let mut sync_audios = vec![];
+ let mut chromaprints = HashMap::new();
+ let mut formats = HashSet::new();
+ for audio in available_audios {
+ if formats.contains(&audio.format_id) {
+ continue;
+ }
+ formats.insert(audio.format_id);
+ sync_audios.push((audio.format_id, &audio.path, audio.sample_rate));
+ chromaprints.insert(
+ audio.format_id,
+ generate_chromaprint(
+ &audio.path,
+ audio.sample_rate,
+ &TimeDelta::zero(),
+ &TimeDelta::zero(),
+ &TimeDelta::zero(),
+ )?,
+ );
+ }
+ sync_audios.sort_by_key(|sync_audio| chromaprints.get(&sync_audio.0).unwrap().len());
+
+ let base_audio = sync_audios.remove(0);
+
+ let mut start = f64::MAX;
+ let mut end = f64::MIN;
+ let mut initial_offsets = HashMap::new();
+ for audio in &sync_audios {
+ debug!(
+ "Initial comparison of format {} to {}",
+ audio.0, &base_audio.0
+ );
+
+ let (lhs_ranges, rhs_ranges) = compare_chromaprints(
+ chromaprints.get(&base_audio.0).unwrap(),
+ chromaprints.get(&audio.0).unwrap(),
+ sync_tolerance,
+ );
+ if lhs_ranges.is_empty() || rhs_ranges.is_empty() {
+ bail!(
+ "Failed to sync videos, couldn't find matching audio parts between format {} and {}",
+ base_audio.0 + 1,
+ audio.0 + 1
+ );
+ }
+ let lhs_range = lhs_ranges[0];
+ let rhs_range = rhs_ranges[0];
+ start = start.min(lhs_range.start);
+ end = end.max(lhs_range.end);
+ start = start.min(rhs_range.start);
+ end = end.max(rhs_range.end);
+ let offset = TimeDelta::milliseconds(((rhs_range.start - lhs_range.start) * 1000.0) as i64);
+ initial_offsets.insert(audio.0, TimeDelta::zero().checked_sub(&offset).unwrap());
+ debug!(
+ "Found initial offset of {}ms ({} - {} {}s) ({} - {} {}s) for format {} to {}",
+ offset.num_milliseconds(),
+ lhs_range.start,
+ lhs_range.end,
+ lhs_range.end - lhs_range.start,
+ rhs_range.start,
+ rhs_range.end,
+ rhs_range.end - rhs_range.start,
+ audio.0,
+ base_audio.0
+ );
+ }
+
+ debug!(
+ "Found matching audio parts at {} - {}, narrowing search",
+ start, end
+ );
+
+ let start = TimeDelta::milliseconds((start * 1000.0) as i64 - 20000);
+ let end = TimeDelta::milliseconds((end * 1000.0) as i64 + 20000);
+
+ for sync_audio in &sync_audios {
+ let chromaprint = generate_chromaprint(
+ sync_audio.1,
+ sync_audio.2,
+ &start,
+ &end,
+ initial_offsets.get(&sync_audio.0).unwrap(),
+ )?;
+ chromaprints.insert(sync_audio.0, chromaprint);
+ }
+
+ let mut runs: HashMap = HashMap::new();
+ let iterator_range_limits: i64 = 2 ^ sync_precision as i64;
+ for i in -iterator_range_limits..=iterator_range_limits {
+ let base_offset = TimeDelta::milliseconds(
+ ((0.128 / iterator_range_limits as f64 * i as f64) * 1000.0) as i64,
+ );
+ chromaprints.insert(
+ base_audio.0,
+ generate_chromaprint(base_audio.1, base_audio.2, &start, &end, &base_offset)?,
+ );
+ for audio in &sync_audios {
+ let initial_offset = initial_offsets.get(&audio.0).copied().unwrap();
+ let offset = find_offset(
+ (&base_audio.0, chromaprints.get(&base_audio.0).unwrap()),
+ &base_offset,
+ (&audio.0, chromaprints.get(&audio.0).unwrap()),
+ &initial_offset,
+ &start,
+ sync_tolerance,
+ );
+ if offset.is_none() {
+ continue;
+ }
+ let offset = offset.unwrap();
+
+ result.insert(
+ audio.0,
+ result
+ .get(&audio.0)
+ .copied()
+ .unwrap_or_default()
+ .checked_add(&offset)
+ .unwrap(),
+ );
+ runs.insert(audio.0, runs.get(&audio.0).copied().unwrap_or_default() + 1);
+ }
+ }
+ let mut result: HashMap = result
+ .iter()
+ .map(|(format_id, offset)| {
+ (
+ *format_id,
+ TimeDelta::milliseconds(
+ offset.num_milliseconds() / runs.get(format_id).copied().unwrap(),
+ ),
+ )
+ })
+ .collect();
+ result.insert(base_audio.0, TimeDelta::milliseconds(0));
+
+ Ok(Some(result))
+}
+
+fn find_offset(
+ lhs: (&usize, &Vec),
+ lhs_shift: &TimeDelta,
+ rhs: (&usize, &Vec),
+ rhs_shift: &TimeDelta,
+ start: &TimeDelta,
+ sync_tolerance: u32,
+) -> Option {
+ let (lhs_ranges, rhs_ranges) = compare_chromaprints(lhs.1, rhs.1, sync_tolerance);
+ if lhs_ranges.is_empty() || rhs_ranges.is_empty() {
+ return None;
+ }
+ let lhs_range = lhs_ranges[0];
+ let rhs_range = rhs_ranges[0];
+ let offset = rhs_range.end - lhs_range.end;
+ let offset = TimeDelta::milliseconds((offset * 1000.0) as i64)
+ .checked_add(lhs_shift)?
+ .checked_sub(rhs_shift)?;
+ debug!(
+ "Found offset of {}ms ({} - {} {}s) ({} - {} {}s) for format {} to {}",
+ offset.num_milliseconds(),
+ lhs_range.start + start.num_milliseconds() as f64 / 1000.0,
+ lhs_range.end + start.num_milliseconds() as f64 / 1000.0,
+ lhs_range.end - lhs_range.start,
+ rhs_range.start + start.num_milliseconds() as f64 / 1000.0,
+ rhs_range.end + start.num_milliseconds() as f64 / 1000.0,
+ rhs_range.end - rhs_range.start,
+ rhs.0,
+ lhs.0
+ );
+ Some(offset)
+}
+
+fn generate_chromaprint(
+ input_file: &Path,
+ sample_rate: u32,
+ start: &TimeDelta,
+ end: &TimeDelta,
+ offset: &TimeDelta,
+) -> Result> {
+ let mut ss_argument: &TimeDelta = &start.checked_sub(offset).unwrap();
+ let mut offset_argument = &TimeDelta::zero();
+ if *offset < TimeDelta::zero() {
+ ss_argument = start;
+ offset_argument = offset;
+ };
+
+ let mut printer = Fingerprinter::new(&Configuration::preset_test1());
+ printer.start(sample_rate, 2)?;
+
+ let mut command = Command::new("ffmpeg");
+ command
+ .arg("-hide_banner")
+ .arg("-y")
+ .args(["-ss", format_time_delta(ss_argument).as_str()]);
+
+ if end.is_zero().not() {
+ command.args(["-to", format_time_delta(end).as_str()]);
+ }
+
+ command
+ .args(["-itsoffset", format_time_delta(offset_argument).as_str()])
+ .args(["-i", input_file.to_string_lossy().to_string().as_str()])
+ .args(["-ac", "2"])
+ .args([
+ "-f",
+ if cfg!(target_endian = "big") {
+ "s16be"
+ } else {
+ "s16le"
+ },
+ ])
+ .arg("-");
+
+ let mut handle = command
+ .stdout(Stdio::piped())
+ .stderr(Stdio::piped())
+ .spawn()?;
+
+ // the stdout is read in chunks because keeping all the raw audio data in memory would take up
+ // a significant amount of space
+ let mut stdout = handle.stdout.take().unwrap();
+ let mut buf: [u8; 128_000] = [0; 128_000];
+ while handle.try_wait()?.is_none() {
+ loop {
+ let read_bytes = stdout.read(&mut buf)?;
+ if read_bytes == 0 {
+ break;
+ }
+ let data: [i16; 64_000] = unsafe { mem::transmute(buf) };
+ printer.consume(&data[0..(read_bytes / 2)])
+ }
+ }
+
+ if !handle.wait()?.success() {
+ bail!("{}", std::io::read_to_string(handle.stderr.unwrap())?)
+ }
+
+ printer.finish();
+ return Ok(printer.fingerprint().into());
+}
+
+fn compare_chromaprints(
+ lhs_chromaprint: &Vec,
+ rhs_chromaprint: &Vec,
+ sync_tolerance: u32,
+) -> (Vec, Vec) {
+ let lhs_inverse_index = create_inverse_index(lhs_chromaprint);
+ let rhs_inverse_index = create_inverse_index(rhs_chromaprint);
+
+ let mut possible_shifts = HashSet::new();
+ for lhs_pair in lhs_inverse_index {
+ let original_point = lhs_pair.0;
+ for i in -2..=2 {
+ let modified_point = (original_point as i32 + i) as u32;
+ if rhs_inverse_index.contains_key(&modified_point) {
+ let rhs_index = rhs_inverse_index.get(&modified_point).copied().unwrap();
+ possible_shifts.insert(rhs_index as i32 - lhs_pair.1 as i32);
+ }
+ }
+ }
+
+ let mut all_lhs_time_ranges = vec![];
+ let mut all_rhs_time_ranges = vec![];
+ for shift_amount in possible_shifts {
+ let time_range_pair = find_time_ranges(
+ lhs_chromaprint,
+ rhs_chromaprint,
+ shift_amount,
+ sync_tolerance,
+ );
+ if time_range_pair.is_none() {
+ continue;
+ }
+ let (mut lhs_time_ranges, mut rhs_time_ranges) = time_range_pair.unwrap();
+ let mut lhs_time_ranges: Vec = lhs_time_ranges
+ .drain(..)
+ .filter(|time_range| {
+ (20.0 < (time_range.end - time_range.start))
+ && ((time_range.end - time_range.start) < 180.0)
+ && time_range.end > 0.0
+ })
+ .collect();
+ lhs_time_ranges.sort_by(|a, b| (b.end - b.start).total_cmp(&(a.end - a.start)));
+ let mut rhs_time_ranges: Vec = rhs_time_ranges
+ .drain(..)
+ .filter(|time_range| {
+ (20.0 < (time_range.end - time_range.start))
+ && ((time_range.end - time_range.start) < 180.0)
+ && time_range.end > 0.0
+ })
+ .collect();
+ rhs_time_ranges.sort_by(|a, b| (b.end - b.start).total_cmp(&(a.end - a.start)));
+ if lhs_time_ranges.is_empty() || rhs_time_ranges.is_empty() {
+ continue;
+ }
+
+ all_lhs_time_ranges.push(lhs_time_ranges[0]);
+ all_rhs_time_ranges.push(rhs_time_ranges[0]);
+ }
+ all_lhs_time_ranges.sort_by(|a, b| (a.end - a.start).total_cmp(&(b.end - b.start)));
+ all_lhs_time_ranges.reverse();
+ all_rhs_time_ranges.sort_by(|a, b| (a.end - a.start).total_cmp(&(b.end - b.start)));
+ all_rhs_time_ranges.reverse();
+
+ (all_lhs_time_ranges, all_rhs_time_ranges)
+}
+
+fn create_inverse_index(chromaprint: &Vec) -> HashMap {
+ let mut inverse_index = HashMap::with_capacity(chromaprint.capacity());
+ for (i, fingerprint) in chromaprint.iter().enumerate().take(chromaprint.capacity()) {
+ inverse_index.insert(*fingerprint, i);
+ }
+ inverse_index
+}
+
+fn find_time_ranges(
+ lhs_chromaprint: &[u32],
+ rhs_chromaprint: &[u32],
+ shift_amount: i32,
+ sync_tolerance: u32,
+) -> Option<(Vec, Vec)> {
+ let mut lhs_shift: i32 = 0;
+ let mut rhs_shift: i32 = 0;
+ if shift_amount < 0 {
+ lhs_shift -= shift_amount;
+ } else {
+ rhs_shift += shift_amount;
+ }
+
+ let mut lhs_matching_timestamps = vec![];
+ let mut rhs_matching_timestamps = vec![];
+ let upper_limit =
+ cmp::min(lhs_chromaprint.len(), rhs_chromaprint.len()) as i32 - shift_amount.abs();
+
+ for i in 0..upper_limit {
+ let lhs_position = i + lhs_shift;
+ let rhs_position = i + rhs_shift;
+ let difference = (lhs_chromaprint[lhs_position as usize]
+ ^ rhs_chromaprint[rhs_position as usize])
+ .count_ones();
+
+ if difference > sync_tolerance {
+ continue;
+ }
+
+ lhs_matching_timestamps.push(lhs_position as f64 * 0.128);
+ rhs_matching_timestamps.push(rhs_position as f64 * 0.128);
+ }
+ lhs_matching_timestamps.push(f64::MAX);
+ rhs_matching_timestamps.push(f64::MAX);
+
+ let lhs_time_ranges = timestamps_to_ranges(lhs_matching_timestamps);
+ lhs_time_ranges.as_ref()?;
+ let lhs_time_ranges = lhs_time_ranges.unwrap();
+ let rhs_time_ranges = timestamps_to_ranges(rhs_matching_timestamps).unwrap();
+
+ Some((lhs_time_ranges, rhs_time_ranges))
+}
+
+fn timestamps_to_ranges(mut timestamps: Vec) -> Option> {
+ if timestamps.is_empty() {
+ return None;
+ }
+
+ timestamps.sort_by(|a, b| a.total_cmp(b));
+
+ let mut time_ranges = vec![];
+ let mut current_range = TimeRange {
+ start: timestamps[0],
+ end: timestamps[0],
+ };
+
+ for i in 0..timestamps.len() - 1 {
+ let current = timestamps[i];
+ let next = timestamps[i + 1];
+ if next - current <= 1.0 {
+ current_range.end = next;
+ continue;
+ }
+
+ time_ranges.push(current_range);
+ current_range.start = next;
+ current_range.end = next;
+ }
+ if !time_ranges.is_empty() {
+ Some(time_ranges)
+ } else {
+ None
+ }
+}