NyxErinys/crunchy-cli
1
0
Fork 0
mirror of https://github.com/crunchy-labs/crunchy-cli.git synced 2026-01-21 04:02:00 -06:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #111 from crunchy-labs/fix/download-no-hardsub-videos

Browse source 
Manually add subtitles to download videos
This commit is contained in:
ByteDream 2023-01-14 00:03:18 +01:00 • committed by GitHub
commit b5bc36c4a2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 256 additions and 213 deletions
Download patch file Download diff file Expand all files Collapse all files

151
crunchy-cli-core/src/cli/archive.rs
View file

@ -9,16 +9,14 @@ use crate::utils::log::progress;
use crate::utils::os::{free_file, has_ffmpeg, is_special_file, tempfile};
use crate::utils::parse::{parse_url, UrlFilter};
use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number};
use crate::utils::subtitle::Subtitle;
use crate::utils::subtitle::{download_subtitle, Subtitle};
use crate::utils::video::get_video_length;
use crate::Execute;
use anyhow::{bail, Result};
use chrono::NaiveTime;
use crunchyroll_rs::media::{Resolution, StreamSubtitle};
use crunchyroll_rs::media::Resolution;
use crunchyroll_rs::{Locale, Media, MediaCollection, Series};
use log::{debug, error, info, warn};
use regex::Regex;
use std::collections::BTreeMap;
use std::io::Write;
use std::path::PathBuf;
use std::process::{Command, Stdio};
use tempfile::TempPath;
@ -113,14 +111,6 @@ pub struct Archive {
)]
#[arg(long)]
default_subtitle: Option<Locale>,
#[arg(help = "Disable subtitle optimizations")]
#[arg(
long_help = "By default, Crunchyroll delivers subtitles in a format which may cause issues in some video players. \
These issues are fixed internally by setting a flag which is not part of the official specification of the subtitle format. \
If you do not want this fixes or they cause more trouble than they solve (for you), it can be disabled with this flag"
)]
#[arg(long)]
no_subtitle_optimizations: bool,
#[arg(help = "Ignore interactive input")]
#[arg(short, long, default_value_t = false)]
@ -326,12 +316,8 @@ impl Execute for Archive {
let primary_video_length = get_video_length(primary_video.to_path_buf()).unwrap();
for subtitle in subtitles {
subtitle_paths.push((
download_subtitle(
&self,
subtitle.stream_subtitle.clone(),
primary_video_length,
)
.await?,
download_subtitle(subtitle.stream_subtitle.clone(), primary_video_length)
.await?,
subtitle,
))
}
@ -436,7 +422,7 @@ async fn formats_from_series(
};
Some(subtitle)
}));
formats.push(Format::new_from_episode(episode, &episodes, stream));
formats.push(Format::new_from_episode(episode, &episodes, stream, vec![]));
}
primary_season = false;
@ -476,111 +462,6 @@ async fn download_video(ctx: &Context, format: &Format, only_audio: bool) -> Res
Ok(path)
}
async fn download_subtitle(
archive: &Archive,
subtitle: StreamSubtitle,
max_length: NaiveTime,
) -> Result<TempPath> {
let tempfile = tempfile(".ass")?;
let (mut file, path) = tempfile.into_parts();
let mut buf = vec![];
subtitle.write_to(&mut buf).await?;
if !archive.no_subtitle_optimizations {
buf = fix_subtitle_look_and_feel(buf)
}
buf = fix_subtitle_length(buf, max_length);
file.write_all(buf.as_slice())?;
Ok(path)
}
/// Add `ScaledBorderAndShadows: yes` to subtitles; without it they look very messy on some video
/// players. See [crunchy-labs/crunchy-cli#66](https://github.com/crunchy-labs/crunchy-cli/issues/66)
/// for more information.
fn fix_subtitle_look_and_feel(raw: Vec<u8>) -> Vec<u8> {
let mut script_info = false;
let mut new = String::new();
for line in String::from_utf8_lossy(raw.as_slice()).split('\n') {
if line.trim().starts_with('[') && script_info {
new.push_str("ScaledBorderAndShadow: yes\n");
script_info = false
} else if line.trim() == "[Script Info]" {
script_info = true
}
new.push_str(line);
new.push('\n')
}
new.into_bytes()
}
/// Fix the length of subtitles to a specified maximum amount. This is required because sometimes
/// subtitles have an unnecessary entry long after the actual video ends with artificially extends
/// the video length on some video players. To prevent this, the video length must be hard set. See
/// [crunchy-labs/crunchy-cli#32](https://github.com/crunchy-labs/crunchy-cli/issues/32) for more
/// information.
fn fix_subtitle_length(raw: Vec<u8>, max_length: NaiveTime) -> Vec<u8> {
let re =
Regex::new(r#"^Dialogue:\s\d+,(?P<start>\d+:\d+:\d+\.\d+),(?P<end>\d+:\d+:\d+\.\d+),"#)
.unwrap();
// chrono panics if we try to format NaiveTime with `%2f` and the nano seconds has more than 2
// digits so them have to be reduced manually to avoid the panic
fn format_naive_time(native_time: NaiveTime) -> String {
let formatted_time = native_time.format("%f").to_string();
format!(
"{}.{}",
native_time.format("%T"),
if formatted_time.len() <= 2 {
native_time.format("%2f").to_string()
} else {
formatted_time.split_at(2).0.parse().unwrap()
}
)
}
let length_as_string = format_naive_time(max_length);
let mut new = String::new();
for line in String::from_utf8_lossy(raw.as_slice()).split('\n') {
if let Some(capture) = re.captures(line) {
let start = capture.name("start").map_or(NaiveTime::default(), |s| {
NaiveTime::parse_from_str(s.as_str(), "%H:%M:%S.%f").unwrap()
});
let end = capture.name("end").map_or(NaiveTime::default(), |s| {
NaiveTime::parse_from_str(s.as_str(), "%H:%M:%S.%f").unwrap()
});
if start > max_length {
continue;
} else if end > max_length {
new.push_str(
re.replace(
line,
format!(
"Dialogue: {},{},",
format_naive_time(start),
&length_as_string
),
)
.to_string()
.as_str(),
)
} else {
new.push_str(line)
}
} else {
new.push_str(line)
}
new.push('\n')
}
new.into_bytes()
}
fn generate_mkv(
archive: &Archive,
target: PathBuf,
@ -721,23 +602,3 @@ fn generate_mkv(
Ok(())
}
/// Get the length of a video. This is required because sometimes subtitles have an unnecessary entry
/// long after the actual video ends with artificially extends the video length on some video players.
/// To prevent this, the video length must be hard set. See
/// [crunchy-labs/crunchy-cli#32](https://github.com/crunchy-labs/crunchy-cli/issues/32) for more
/// information.
fn get_video_length(path: PathBuf) -> Result<NaiveTime> {
let video_length = Regex::new(r"Duration:\s(?P<time>\d+:\d+:\d+\.\d+),")?;
let ffmpeg = Command::new("ffmpeg")
.stdout(Stdio::null())
.stderr(Stdio::piped())
.arg("-y")
.args(["-i", path.to_str().unwrap()])
.output()?;
let ffmpeg_output = String::from_utf8(ffmpeg.stderr)?;
let caps = video_length.captures(ffmpeg_output.as_str()).unwrap();
Ok(NaiveTime::parse_from_str(caps.name("time").unwrap().as_str(), "%H:%M:%S%.f").unwrap())
}

158
crunchy-cli-core/src/cli/download.rs
View file

@ -6,18 +6,19 @@ use crate::cli::utils::{
use crate::utils::context::Context;
use crate::utils::format::Format;
use crate::utils::log::progress;
use crate::utils::os::{free_file, has_ffmpeg, is_special_file};
use crate::utils::os::{free_file, has_ffmpeg, is_special_file, tempfile};
use crate::utils::parse::{parse_url, UrlFilter};
use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number};
use crate::utils::subtitle::download_subtitle;
use crate::utils::video::get_video_length;
use crate::Execute;
use anyhow::{bail, Result};
use crunchyroll_rs::media::{Resolution, VariantData};
use crunchyroll_rs::media::{Resolution, StreamSubtitle, VariantData};
use crunchyroll_rs::{
Episode, Locale, Media, MediaCollection, Movie, MovieListing, Season, Series,
};
use log::{debug, error, info, warn};
use std::borrow::Cow;
use std::fs::File;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
@ -51,7 +52,7 @@ pub struct Download {
{series_id} → ID of the series\n \
{season_id} → ID of the season\n \
{episode_id} → ID of the episode")]
#[arg(short, long, default_value = "{title}.ts")]
#[arg(short, long, default_value = "{title}.mp4")]
output: String,
#[arg(help = "Video resolution")]
@ -85,17 +86,23 @@ pub struct Download {
#[async_trait::async_trait(?Send)]
impl Execute for Download {
fn pre_check(&self) -> Result<()> {
if has_ffmpeg() {
debug!("FFmpeg detected")
} else if PathBuf::from(&self.output)
if !has_ffmpeg() {
bail!("FFmpeg is needed to run this command")
} else if Path::new(&self.output)
.extension()
.unwrap_or_default()
.to_string_lossy()
!= "ts"
.is_empty()
&& self.output != "-"
{
bail!("File extension is not '.ts'. If you want to use a custom file format, please install ffmpeg")
} else if !self.ffmpeg_preset.is_empty() {
bail!("FFmpeg is required to use (ffmpeg) presets")
bail!("No file extension found. Please specify a file extension (via `-o`) for the output file")
}
if self.subtitle.is_some() {
if let Some(ext) = Path::new(&self.output).extension() {
if ext.to_string_lossy() != "mp4" {
warn!("Detected a non mp4 output container. Adding subtitles may take a while")
}
}
}
let _ = FFmpegPreset::ffmpeg_presets(self.ffmpeg_preset.clone())?;
@ -245,23 +252,14 @@ impl Execute for Download {
tab_info!("Resolution: {}", format.stream.resolution);
tab_info!("FPS: {:.2}", format.stream.fps);
let extension = path.extension().unwrap_or_default().to_string_lossy();
if (!extension.is_empty() && extension != "ts") || !self.ffmpeg_preset.is_empty() {
download_ffmpeg(&ctx, &self, format.stream, path.as_path()).await?;
} else if path.to_str().unwrap() == "-" {
let mut stdout = std::io::stdout().lock();
download_segments(&ctx, &mut stdout, None, format.stream).await?;
} else {
// create parent directory if it does not exist
if let Some(parent) = path.parent() {
if !parent.exists() {
std::fs::create_dir_all(parent)?
}
}
let mut file = File::options().create(true).write(true).open(&path)?;
download_segments(&ctx, &mut file, None, format.stream).await?
}
download_ffmpeg(
&ctx,
&self,
format.stream,
format.subtitles.get(0).cloned(),
path.to_path_buf(),
)
.await?;
}
}
@ -273,9 +271,10 @@ async fn download_ffmpeg(
ctx: &Context,
download: &Download,
variant_data: VariantData,
target: &Path,
subtitle: Option<StreamSubtitle>,
mut target: PathBuf,
) -> Result<()> {
let (input_presets, output_presets) =
let (input_presets, mut output_presets) =
FFmpegPreset::ffmpeg_presets(download.ffmpeg_preset.clone())?;
// create parent directory if it does not exist
@ -285,35 +284,83 @@ async fn download_ffmpeg(
}
}
let mut video_file = tempfile(".ts")?;
download_segments(ctx, &mut video_file, None, variant_data).await?;
let subtitle_file = if let Some(ref sub) = subtitle {
let video_len = get_video_length(video_file.path().to_path_buf())?;
Some(download_subtitle(sub.clone(), video_len).await?)
} else {
None
};
let stdout_tempfile = if target.to_string_lossy() == "-" {
let file = tempfile(".mp4")?;
target = file.path().to_path_buf();
Some(file)
} else {
None
};
let subtitle_presets = if let Some(sub_file) = &subtitle_file {
if target.extension().unwrap_or_default().to_string_lossy() == "mp4" {
vec![
"-i".to_string(),
sub_file.to_string_lossy().to_string(),
"-movflags".to_string(),
"faststart".to_string(),
"-c:s".to_string(),
"mov_text".to_string(),
"-disposition:s:s:0".to_string(),
"forced".to_string(),
]
} else {
// remove '-c:v copy' and '-c:a copy' from output presets as its causes issues with
// burning subs into the video
let mut last = String::new();
let mut remove_count = 0;
for (i, s) in output_presets.clone().iter().enumerate() {
if (last == "-c:v" || last == "-c:a") && s == "copy" {
// remove last
output_presets.remove(i - remove_count - 1);
remove_count += 1;
output_presets.remove(i - remove_count);
remove_count += 1;
}
last = s.clone();
}
vec![
"-vf".to_string(),
format!("subtitles={}", sub_file.to_string_lossy()),
]
}
} else {
vec![]
};
let mut ffmpeg = Command::new("ffmpeg")
.stdin(Stdio::piped())
.stdout(Stdio::null())
.stderr(Stdio::piped())
.arg("-y")
.args(input_presets)
.args(["-f", "mpegts", "-i", "pipe:"])
.args(
if target
.extension()
.unwrap_or_default()
.to_string_lossy()
.is_empty()
{
vec!["-f", "mpegts"]
} else {
vec![]
}
.as_slice(),
)
.args(["-i", video_file.path().to_string_lossy().as_ref()])
.args(subtitle_presets)
.args(output_presets)
.arg(target.to_str().unwrap())
.spawn()?;
download_segments(ctx, &mut ffmpeg.stdin.take().unwrap(), None, variant_data).await?;
let progress_handler = progress!("Generating output file");
if !ffmpeg.wait()?.success() {
bail!("{}", std::io::read_to_string(ffmpeg.stderr.unwrap())?)
}
progress_handler.stop("Output file generated");
let _progress_handler = progress!("Generating output file");
ffmpeg.wait()?;
info!("Output file generated");
if let Some(mut stdout_file) = stdout_tempfile {
let mut stdout = std::io::stdout();
std::io::copy(&mut stdout_file, &mut stdout)?;
}
Ok(())
}
@ -431,8 +478,11 @@ async fn format_from_episode(
}
let streams = episode.streams().await?;
let streaming_data = if let Some(subtitle) = &download.subtitle {
if !streams.subtitles.keys().cloned().any(|x| &x == subtitle) {
let streaming_data = streams.hls_streaming_data(None).await?;
let subtitle = if let Some(subtitle) = &download.subtitle {
if let Some(sub) = streams.subtitles.get(subtitle) {
Some(sub.clone())
} else {
error!(
"Episode {} ({}) of season {} ({}) of {} has no {} subtitles",
episode.metadata.episode_number,
@ -444,9 +494,8 @@ async fn format_from_episode(
);
return Ok(None);
}
streams.hls_streaming_data(Some(subtitle.clone())).await?
} else {
streams.hls_streaming_data(None).await?
None
};
let Some(stream) = find_resolution(streaming_data, &download.resolution) else {
@ -476,6 +525,7 @@ async fn format_from_episode(
episode,
&season_eps.to_vec(),
stream,
subtitle.map_or_else(|| vec![], |s| vec![s]),
)))
}

20
crunchy-cli-core/src/cli/utils.rs
View file

@ -290,38 +290,32 @@ impl FFmpegPreset {
FFmpegPreset::Av1 => bail!("'nvidia' hardware acceleration preset is not available in combination with the 'av1' codec preset"),
FFmpegPreset::H265 => {
input.extend(["-hwaccel", "cuvid", "-c:v", "h264_cuvid"]);
output.extend(["-c:v", "hevc_nvenc"]);
output.extend(["-c:v", "hevc_nvenc", "-c:a", "copy"]);
}
FFmpegPreset::H264 => {
input.extend(["-hwaccel", "cuvid", "-c:v", "h264_cuvid"]);
output.extend(["-c:v", "h264_nvenc"]);
output.extend(["-c:v", "h264_nvenc", "-c:a", "copy"]);
}
_ => ()
}
} else {
match preset {
FFmpegPreset::Av1 => {
output.extend(["-c:v", "libaom-av1"]);
output.extend(["-c:v", "libaom-av1", "-c:a", "copy"]);
}
FFmpegPreset::H265 => {
output.extend(["-c:v", "libx265"]);
output.extend(["-c:v", "libx265", "-c:a", "copy"]);
}
FFmpegPreset::H264 => {
output.extend(["-c:v", "libx264"]);
output.extend(["-c:v", "libx264", "-c:a", "copy"]);
}
_ => (),
}
}
}
if input.is_empty() && output.is_empty() {
output.extend(["-c", "copy"])
} else {
if output.is_empty() {
output.extend(["-c", "copy"])
} else {
output.extend(["-c:a", "copy", "-c:s", "copy"])
}
if output.is_empty() {
output.extend(["-c:v", "copy", "-c:a", "copy"])
}
Ok((

6
crunchy-cli-core/src/utils/format.rs
View file

@ -1,4 +1,4 @@
use crunchyroll_rs::media::VariantData;
use crunchyroll_rs::media::{StreamSubtitle, VariantData};
use crunchyroll_rs::{Episode, Locale, Media, Movie};
use log::warn;
use std::path::PathBuf;
@ -10,6 +10,7 @@ pub struct Format {
pub description: String,
pub audio: Locale,
pub subtitles: Vec<StreamSubtitle>,
pub duration: Duration,
pub stream: VariantData,
@ -31,12 +32,14 @@ impl Format {
episode: &Media<Episode>,
season_episodes: &Vec<Media<Episode>>,
stream: VariantData,
subtitles: Vec<StreamSubtitle>,
) -> Self {
Self {
title: episode.title.clone(),
description: episode.description.clone(),
audio: episode.metadata.audio_locale.clone(),
subtitles,
duration: episode.metadata.duration.to_std().unwrap(),
stream,
@ -78,6 +81,7 @@ impl Format {
duration: movie.metadata.duration.to_std().unwrap(),
stream,
subtitles: vec![],
series_id: movie.metadata.movie_listing_id.clone(),
series_name: movie.metadata.movie_listing_title.clone(),

1
crunchy-cli-core/src/utils/mod.rs
View file

@ -7,3 +7,4 @@ pub mod os;
pub mod parse;
pub mod sort;
pub mod subtitle;
pub mod video;

108
crunchy-cli-core/src/utils/subtitle.rs
View file

@ -1,5 +1,11 @@
use crate::utils::os::tempfile;
use anyhow::Result;
use chrono::NaiveTime;
use crunchyroll_rs::media::StreamSubtitle;
use crunchyroll_rs::Locale;
use regex::Regex;
use std::io::Write;
use tempfile::TempPath;
#[derive(Clone)]
pub struct Subtitle {
@ -9,3 +15,105 @@ pub struct Subtitle {
pub forced: bool,
pub primary: bool,
}
pub async fn download_subtitle(
subtitle: StreamSubtitle,
max_length: NaiveTime,
) -> Result<TempPath> {
let tempfile = tempfile(".ass")?;
let (mut file, path) = tempfile.into_parts();
let mut buf = vec![];
subtitle.write_to(&mut buf).await?;
buf = fix_subtitle_look_and_feel(buf);
buf = fix_subtitle_length(buf, max_length);
file.write_all(buf.as_slice())?;
Ok(path)
}
/// Add `ScaledBorderAndShadows: yes` to subtitles; without it they look very messy on some video
/// players. See [crunchy-labs/crunchy-cli#66](https://github.com/crunchy-labs/crunchy-cli/issues/66)
/// for more information.
fn fix_subtitle_look_and_feel(raw: Vec<u8>) -> Vec<u8> {
let mut script_info = false;
let mut new = String::new();
for line in String::from_utf8_lossy(raw.as_slice()).split('\n') {
if line.trim().starts_with('[') && script_info {
new.push_str("ScaledBorderAndShadow: yes\n");
script_info = false
} else if line.trim() == "[Script Info]" {
script_info = true
}
new.push_str(line);
new.push('\n')
}
new.into_bytes()
}
/// Fix the length of subtitles to a specified maximum amount. This is required because sometimes
/// subtitles have an unnecessary entry long after the actual video ends with artificially extends
/// the video length on some video players. To prevent this, the video length must be hard set. See
/// [crunchy-labs/crunchy-cli#32](https://github.com/crunchy-labs/crunchy-cli/issues/32) for more
/// information.
fn fix_subtitle_length(raw: Vec<u8>, max_length: NaiveTime) -> Vec<u8> {
let re =
Regex::new(r#"^Dialogue:\s\d+,(?P<start>\d+:\d+:\d+\.\d+),(?P<end>\d+:\d+:\d+\.\d+),"#)
.unwrap();
// chrono panics if we try to format NaiveTime with `%2f` and the nano seconds has more than 2
// digits so them have to be reduced manually to avoid the panic
fn format_naive_time(native_time: NaiveTime) -> String {
let formatted_time = native_time.format("%f").to_string();
format!(
"{}.{}",
native_time.format("%T"),
if formatted_time.len() <= 2 {
native_time.format("%2f").to_string()
} else {
formatted_time.split_at(2).0.parse().unwrap()
}
)
}
let length_as_string = format_naive_time(max_length);
let mut new = String::new();
for line in String::from_utf8_lossy(raw.as_slice()).split('\n') {
if let Some(capture) = re.captures(line) {
let start = capture.name("start").map_or(NaiveTime::default(), |s| {
NaiveTime::parse_from_str(s.as_str(), "%H:%M:%S.%f").unwrap()
});
let end = capture.name("end").map_or(NaiveTime::default(), |s| {
NaiveTime::parse_from_str(s.as_str(), "%H:%M:%S.%f").unwrap()
});
if start > max_length {
continue;
} else if end > max_length {
new.push_str(
re.replace(
line,
format!(
"Dialogue: {},{},",
format_naive_time(start),
&length_as_string
),
)
.to_string()
.as_str(),
)
} else {
new.push_str(line)
}
} else {
new.push_str(line)
}
new.push('\n')
}
new.into_bytes()
}

25
crunchy-cli-core/src/utils/video.rs Normal file
View file

@ -0,0 +1,25 @@
use anyhow::Result;
use chrono::NaiveTime;
use regex::Regex;
use std::path::PathBuf;
use std::process::{Command, Stdio};
/// Get the length of a video. This is required because sometimes subtitles have an unnecessary entry
/// long after the actual video ends with artificially extends the video length on some video players.
/// To prevent this, the video length must be hard set. See
/// [crunchy-labs/crunchy-cli#32](https://github.com/crunchy-labs/crunchy-cli/issues/32) for more
/// information.
pub fn get_video_length(path: PathBuf) -> Result<NaiveTime> {
let video_length = Regex::new(r"Duration:\s(?P<time>\d+:\d+:\d+\.\d+),")?;
let ffmpeg = Command::new("ffmpeg")
.stdout(Stdio::null())
.stderr(Stdio::piped())
.arg("-y")
.args(["-i", path.to_str().unwrap()])
.output()?;
let ffmpeg_output = String::from_utf8(ffmpeg.stderr)?;
let caps = video_length.captures(ffmpeg_output.as_str()).unwrap();
Ok(NaiveTime::parse_from_str(caps.name("time").unwrap().as_str(), "%H:%M:%S%.f").unwrap())
}