NyxErinys/crunchy-cli
1
0
Fork 0
mirror of https://github.com/crunchy-labs/crunchy-cli.git synced 2026-01-21 04:02:00 -06:00
Code Issues Projects Releases Packages Wiki Activity Actions

Move to new, DRM-free, endpoint

Browse source
This commit is contained in:
bytedream 2024-04-03 15:48:15 +02:00
parent ba8028737d
commit e694046b07
8 changed files with 245 additions and 331 deletions
Download patch file Download diff file Expand all files Collapse all files

6
crunchy-cli-core/Cargo.toml
View file

@ -16,20 +16,20 @@ anyhow = "1.0"
async-speed-limit = "0.4"
clap = { version = "4.5", features = ["derive", "string"] }
chrono = "0.4"
crunchyroll-rs = { version = "0.8.6", features = ["dash-stream", "experimental-stabilizations", "tower"] }
crunchyroll-rs = { version = "0.10.0", features = ["experimental-stabilizations", "tower"] }
ctrlc = "3.4"
dialoguer = { version = "0.11", default-features = false }
dirs = "5.0"
derive_setters = "0.1"
futures-util = { version = "0.3", features = ["io"] }
fs2 = "0.4"
http = "0.2"
http = "1.1"
indicatif = "0.17"
lazy_static = "1.4"
log = { version = "0.4", features = ["std"] }
num_cpus = "1.16"
regex = "1.10"
reqwest = { version = "0.11", default-features = false, features = ["socks", "stream"] }
reqwest = { version = "0.12", default-features = false, features = ["socks", "stream"] }
serde = "1.0"
serde_json = "1.0"
serde_plain = "1.0"

2
crunchy-cli-core/src/archive/command.rs
View file

@ -487,7 +487,7 @@ async fn get_format(
single_format.audio == Locale::ja_JP || stream.subtitles.len() > 1,
)
});
let cc = stream.closed_captions.get(s).cloned().map(|l| (l, false));
let cc = stream.captions.get(s).cloned().map(|l| (l, false));
subtitles
.into_iter()

2
crunchy-cli-core/src/download/command.rs
View file

@ -389,7 +389,7 @@ async fn get_format(
.get(subtitle_locale)
.cloned()
// use closed captions as fallback if no actual subtitles are found
.or_else(|| stream.closed_captions.get(subtitle_locale).cloned())
.or_else(|| stream.captions.get(subtitle_locale).cloned())
} else {
None
};

36
crunchy-cli-core/src/search/format.rs
View file

@ -163,37 +163,15 @@ impl From<&Concert> for FormatConcert {
struct FormatStream {
pub locale: Locale,
pub dash_url: String,
pub drm_dash_url: String,
pub hls_url: String,
pub drm_hls_url: String,
pub is_drm: bool,
}
impl From<&Stream> for FormatStream {
fn from(value: &Stream) -> Self {
let (dash_url, drm_dash_url, hls_url, drm_hls_url) =
value.variants.get(&Locale::Custom("".to_string())).map_or(
(
"".to_string(),
"".to_string(),
"".to_string(),
"".to_string(),
),
|v| {
(
v.adaptive_dash.clone().unwrap_or_default().url,
v.drm_adaptive_dash.clone().unwrap_or_default().url,
v.adaptive_hls.clone().unwrap_or_default().url,
v.drm_adaptive_hls.clone().unwrap_or_default().url,
)
},
);
Self {
locale: value.audio_locale.clone(),
dash_url,
drm_dash_url,
hls_url,
drm_hls_url,
dash_url: value.url.clone(),
is_drm: value.session.uses_stream_limits,
}
}
}
@ -441,7 +419,7 @@ impl Format {
if !stream_empty {
for (_, episodes) in tree.iter_mut() {
for (episode, streams) in episodes {
streams.push(episode.stream().await?)
streams.push(episode.stream_maybe_without_drm().await?)
}
}
} else {
@ -510,7 +488,7 @@ impl Format {
}
if !stream_empty {
for (movie, streams) in tree.iter_mut() {
streams.push(movie.stream().await?)
streams.push(movie.stream_maybe_without_drm().await?)
}
} else {
for (_, streams) in tree.iter_mut() {
@ -548,7 +526,7 @@ impl Format {
let stream_empty = self.check_pattern_count_empty(Scope::Stream);
let music_video = must_match_if_true!(!music_video_empty => media_collection|MediaCollection::MusicVideo(music_video) => music_video.clone()).unwrap_or_default();
let stream = must_match_if_true!(!stream_empty => media_collection|MediaCollection::MusicVideo(music_video) => music_video.stream().await?).unwrap_or_default();
let stream = must_match_if_true!(!stream_empty => media_collection|MediaCollection::MusicVideo(music_video) => music_video.stream_maybe_without_drm().await?).unwrap_or_default();
let music_video_map = self.serializable_to_json_map(FormatMusicVideo::from(&music_video));
let stream_map = self.serializable_to_json_map(FormatStream::from(&stream));
@ -570,7 +548,7 @@ impl Format {
let stream_empty = self.check_pattern_count_empty(Scope::Stream);
let concert = must_match_if_true!(!concert_empty => media_collection|MediaCollection::Concert(concert) => concert.clone()).unwrap_or_default();
let stream = must_match_if_true!(!stream_empty => media_collection|MediaCollection::Concert(concert) => concert.stream().await?).unwrap_or_default();
let stream = must_match_if_true!(!stream_empty => media_collection|MediaCollection::Concert(concert) => concert.stream_maybe_without_drm().await?).unwrap_or_default();
let concert_map = self.serializable_to_json_map(FormatConcert::from(&concert));
let stream_map = self.serializable_to_json_map(FormatStream::from(&stream));

57
crunchy-cli-core/src/utils/download.rs
View file

@ -4,7 +4,7 @@ use crate::utils::os::{cache_dir, is_special_file, temp_directory, temp_named_pi
use crate::utils::rate_limit::RateLimiterService;
use anyhow::{bail, Result};
use chrono::NaiveTime;
use crunchyroll_rs::media::{SkipEvents, SkipEventsEvent, Subtitle, VariantData, VariantSegment};
use crunchyroll_rs::media::{SkipEvents, SkipEventsEvent, StreamData, StreamSegment, Subtitle};
use crunchyroll_rs::Locale;
use indicatif::{ProgressBar, ProgressDrawTarget, ProgressFinish, ProgressStyle};
use log::{debug, warn, LevelFilter};
@ -117,8 +117,8 @@ struct FFmpegMeta {
}
pub struct DownloadFormat {
pub video: (VariantData, Locale),
pub audios: Vec<(VariantData, Locale)>,
pub video: (StreamData, Locale),
pub audios: Vec<(StreamData, Locale)>,
pub subtitles: Vec<(Subtitle, bool)>,
pub metadata: DownloadFormatMetadata,
}
@ -671,20 +671,17 @@ impl Downloader {
&self,
dst: &Path,
) -> Result<(Option<(PathBuf, u64)>, Option<(PathBuf, u64)>)> {
let mut all_variant_data = vec![];
let mut all_stream_data = vec![];
for format in &self.formats {
all_variant_data.push(&format.video.0);
all_variant_data.extend(format.audios.iter().map(|(a, _)| a))
all_stream_data.push(&format.video.0);
all_stream_data.extend(format.audios.iter().map(|(a, _)| a))
}
let mut estimated_required_space: u64 = 0;
for variant_data in all_variant_data {
// nearly no overhead should be generated with this call(s) as we're using dash as
// stream provider and generating the dash segments does not need any fetching of
// additional (http) resources as hls segments would
let segments = variant_data.segments().await?;
for stream_data in all_stream_data {
let segments = stream_data.segments();
// sum the length of all streams up
estimated_required_space += estimate_variant_file_size(variant_data, &segments);
estimated_required_space += estimate_variant_file_size(stream_data, &segments);
}
let tmp_stat = fs2::statvfs(temp_directory()).unwrap();
@ -730,29 +727,21 @@ impl Downloader {
Ok((tmp_required, dst_required))
}
async fn download_video(
&self,
variant_data: &VariantData,
message: String,
) -> Result<TempPath> {
async fn download_video(&self, stream_data: &StreamData, message: String) -> Result<TempPath> {
let tempfile = tempfile(".mp4")?;
let (mut file, path) = tempfile.into_parts();
self.download_segments(&mut file, message, variant_data)
self.download_segments(&mut file, message, stream_data)
.await?;
Ok(path)
}
async fn download_audio(
&self,
variant_data: &VariantData,
message: String,
) -> Result<TempPath> {
async fn download_audio(&self, stream_data: &StreamData, message: String) -> Result<TempPath> {
let tempfile = tempfile(".m4a")?;
let (mut file, path) = tempfile.into_parts();
self.download_segments(&mut file, message, variant_data)
self.download_segments(&mut file, message, stream_data)
.await?;
Ok(path)
@ -806,15 +795,15 @@ impl Downloader {
&self,
writer: &mut impl Write,
message: String,
variant_data: &VariantData,
stream_data: &StreamData,
) -> Result<()> {
let segments = variant_data.segments().await?;
let segments = stream_data.segments();
let total_segments = segments.len();
let count = Arc::new(Mutex::new(0));
let progress = if log::max_level() == LevelFilter::Info {
let estimated_file_size = estimate_variant_file_size(variant_data, &segments);
let estimated_file_size = estimate_variant_file_size(stream_data, &segments);
let progress = ProgressBar::new(estimated_file_size)
.with_style(
@ -832,7 +821,7 @@ impl Downloader {
};
let cpus = self.download_threads;
let mut segs: Vec<Vec<VariantSegment>> = Vec::with_capacity(cpus);
let mut segs: Vec<Vec<StreamSegment>> = Vec::with_capacity(cpus);
for _ in 0..cpus {
segs.push(vec![])
}
@ -858,7 +847,7 @@ impl Downloader {
let download = || async move {
for (i, segment) in thread_segments.into_iter().enumerate() {
let mut retry_count = 0;
let mut buf = loop {
let buf = loop {
let request = thread_client
.get(&segment.url)
.timeout(Duration::from_secs(60));
@ -884,11 +873,9 @@ impl Downloader {
retry_count += 1;
};
buf = VariantSegment::decrypt(&mut buf, segment.key)?.to_vec();
let mut c = thread_count.lock().await;
debug!(
"Downloaded and decrypted segment [{}/{} {:.2}%] {}",
"Downloaded segment [{}/{} {:.2}%] {}",
num + (i * cpus) + 1,
total_segments,
((*c + 1) as f64 / total_segments as f64) * 100f64,
@ -928,7 +915,7 @@ impl Downloader {
if let Some(p) = &progress {
let progress_len = p.length().unwrap();
let estimated_segment_len = (variant_data.bandwidth / 8)
let estimated_segment_len = (stream_data.bandwidth / 8)
* segments.get(pos as usize).unwrap().length.as_secs();
let bytes_len = bytes.len() as u64;
@ -977,8 +964,8 @@ impl Downloader {
}
}
fn estimate_variant_file_size(variant_data: &VariantData, segments: &[VariantSegment]) -> u64 {
(variant_data.bandwidth / 8) * segments.iter().map(|s| s.length.as_secs()).sum::<u64>()
fn estimate_variant_file_size(stream_data: &StreamData, segments: &[StreamSegment]) -> u64 {
(stream_data.bandwidth / 8) * segments.iter().map(|s| s.length.as_secs()).sum::<u64>()
}
/// Get the length and fps of a video.

31
crunchy-cli-core/src/utils/format.rs
View file

@ -2,9 +2,9 @@ use crate::utils::filter::real_dedup_vec;
use crate::utils::locale::LanguageTagging;
use crate::utils::log::tab_info;
use crate::utils::os::{is_special_file, sanitize};
use anyhow::Result;
use anyhow::{bail, Result};
use chrono::{Datelike, Duration};
use crunchyroll_rs::media::{Resolution, SkipEvents, Stream, Subtitle, VariantData};
use crunchyroll_rs::media::{Resolution, SkipEvents, Stream, StreamData, Subtitle};
use crunchyroll_rs::{Concert, Episode, Locale, MediaCollection, Movie, MusicVideo};
use log::{debug, info};
use std::cmp::Ordering;
@ -167,12 +167,17 @@ impl SingleFormat {
pub async fn stream(&self) -> Result<Stream> {
let stream = match &self.source {
MediaCollection::Episode(e) => e.stream().await?,
MediaCollection::Movie(m) => m.stream().await?,
MediaCollection::MusicVideo(mv) => mv.stream().await?,
MediaCollection::Concert(c) => c.stream().await?,
MediaCollection::Episode(e) => e.stream_maybe_without_drm().await?,
MediaCollection::Movie(m) => m.stream_maybe_without_drm().await?,
MediaCollection::MusicVideo(mv) => mv.stream_maybe_without_drm().await?,
MediaCollection::Concert(c) => c.stream_maybe_without_drm().await?,
_ => unreachable!(),
};
if stream.session.uses_stream_limits {
bail!("Found a stream which probably uses DRM. DRM downloads aren't supported")
}
Ok(stream)
}
@ -331,9 +336,7 @@ impl Iterator for SingleFormatCollectionIterator {
type Item = Vec<SingleFormat>;
fn next(&mut self) -> Option<Self::Item> {
let Some((_, episodes)) = self.0 .0.iter_mut().next() else {
return None;
};
let (_, episodes) = self.0 .0.iter_mut().next()?;
let value = episodes.pop_first().unwrap().1;
if episodes.is_empty() {
@ -377,7 +380,7 @@ pub struct Format {
impl Format {
#[allow(clippy::type_complexity)]
pub fn from_single_formats(
mut single_formats: Vec<(SingleFormat, VariantData, Vec<(Subtitle, bool)>)>,
mut single_formats: Vec<(SingleFormat, StreamData, Vec<(Subtitle, bool)>)>,
) -> Self {
let locales: Vec<(Locale, Vec<Locale>)> = single_formats
.iter()
@ -397,10 +400,10 @@ impl Format {
title: first_format.title,
description: first_format.description,
locales,
resolution: first_stream.resolution.clone(),
width: first_stream.resolution.width,
height: first_stream.resolution.height,
fps: first_stream.fps,
resolution: first_stream.resolution().unwrap(),
width: first_stream.resolution().unwrap().width,
height: first_stream.resolution().unwrap().height,
fps: first_stream.fps().unwrap(),
release_year: first_format.release_year,
release_month: first_format.release_month,
release_day: first_format.release_day,

38
crunchy-cli-core/src/utils/video.rs
View file

@ -1,17 +1,17 @@
use anyhow::{bail, Result};
use crunchyroll_rs::media::{Resolution, Stream, VariantData};
use crunchyroll_rs::media::{Resolution, Stream, StreamData};
use crunchyroll_rs::Locale;
pub async fn variant_data_from_stream(
stream: &Stream,
resolution: &Resolution,
subtitle: Option<Locale>,
) -> Result<Option<(VariantData, VariantData, bool)>> {
) -> Result<Option<(StreamData, StreamData, bool)>> {
// sometimes Crunchyroll marks episodes without real subtitles that they have subtitles and
// reports that only hardsub episode are existing. the following lines are trying to prevent
// potential errors which might get caused by this incorrect reporting
// (https://github.com/crunchy-labs/crunchy-cli/issues/231)
let mut hardsub_locales = stream.streaming_hardsub_locales();
let mut hardsub_locales: Vec<Locale> = stream.hard_subs.keys().cloned().collect();
let (hardsub_locale, mut contains_hardsub) = if !hardsub_locales
.contains(&Locale::Custom("".to_string()))
&& !hardsub_locales.contains(&Locale::Custom(":".to_string()))
@ -29,39 +29,29 @@ pub async fn variant_data_from_stream(
(subtitle, hardsubs_requested)
};
let mut streaming_data = match stream.dash_streaming_data(hardsub_locale).await {
let (mut videos, mut audios) = match stream.stream_data(hardsub_locale).await {
Ok(data) => data,
Err(e) => {
// the error variant is only `crunchyroll_rs::error::Error::Input` when the requested
// hardsub is not available
if let crunchyroll_rs::error::Error::Input { .. } = e {
contains_hardsub = false;
stream.dash_streaming_data(None).await?
stream.stream_data(None).await?
} else {
bail!(e)
}
}
};
streaming_data
.0
.sort_by(|a, b| a.bandwidth.cmp(&b.bandwidth).reverse());
streaming_data
.1
.sort_by(|a, b| a.bandwidth.cmp(&b.bandwidth).reverse());
}
.unwrap();
videos.sort_by(|a, b| a.bandwidth.cmp(&b.bandwidth).reverse());
audios.sort_by(|a, b| a.bandwidth.cmp(&b.bandwidth).reverse());
let video_variant = match resolution.height {
u64::MAX => Some(streaming_data.0.into_iter().next().unwrap()),
u64::MIN => Some(streaming_data.0.into_iter().last().unwrap()),
_ => streaming_data
.0
u64::MAX => Some(videos.into_iter().next().unwrap()),
u64::MIN => Some(videos.into_iter().last().unwrap()),
_ => videos
.into_iter()
.find(|v| resolution.height == v.resolution.height),
.find(|v| resolution.height == v.resolution().unwrap().height),
};
Ok(video_variant.map(|v| {
(
v,
streaming_data.1.first().unwrap().clone(),
contains_hardsub,
)
}))
Ok(video_variant.map(|v| (v, audios.first().unwrap().clone(), contains_hardsub)))
}