From dc7e5d564ea4ae7849bc2e593ab6493eb03db0f9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Oct 2022 01:56:32 +0000 Subject: [PATCH 001/441] Bump github.com/spf13/cobra from 1.6.0 to 1.6.1 Bumps [github.com/spf13/cobra](https://github.com/spf13/cobra) from 1.6.0 to 1.6.1. - [Release notes](https://github.com/spf13/cobra/releases) - [Commits](https://github.com/spf13/cobra/compare/v1.6.0...v1.6.1) --- updated-dependencies: - dependency-name: github.com/spf13/cobra dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e04d5ae..b884b5a 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( github.com/crunchy-labs/crunchyroll-go/v3 v3.0.4 github.com/grafov/m3u8 v0.11.1 - github.com/spf13/cobra v1.6.0 + github.com/spf13/cobra v1.6.1 ) require ( diff --git a/go.sum b/go.sum index 9ca4b9c..1881381 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/grafov/m3u8 v0.11.1/go.mod h1:nqzOkfBiZJENr52zTVd/Dcl03yzphIMbJqkXGu+ github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI= -github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 0572af4e07f9ec2bbefa79704a0833625095f630 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Fri, 28 Oct 2022 12:43:51 +0200 Subject: [PATCH 002/441] Change links from beta.crunchyroll.com to www.crunchyroll.com --- README.md | 12 ++++++------ cli/commands/login/login.go | 2 +- crunchy-cli.1 | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 7605a61..6e757e1 100644 --- a/README.md +++ b/README.md @@ -98,13 +98,13 @@ By default, the cli tries to download the episode with your system language as a **If your system language is not supported, an error message will be displayed and en-US (american english) will be chosen as language.** ```shell -$ crunchy download https://beta.crunchyroll.com/watch/GRDKJZ81Y/alone-and-lonesome +$ crunchy download https://www.crunchyroll.com/watch/GRDKJZ81Y/alone-and-lonesome ``` With `-r best` the video(s) will have the best available resolution (mostly 1920x1080 / Full HD). ```shell -$ crunchy download -r best https://beta.crunchyroll.com/watch/GRDKJZ81Y/alone-and-lonesome +$ crunchy download -r best https://www.crunchyroll.com/watch/GRDKJZ81Y/alone-and-lonesome ``` The file is by default saved as a `.ts` (mpeg transport stream) file. @@ -113,13 +113,13 @@ file, just name it `whatever.mp4`. **You need [ffmpeg](https://ffmpeg.org) to store the video in other file formats.** ```shell -$ crunchy download -o "daaaaaaaaaaaaaaaarling.ts" https://beta.crunchyroll.com/watch/GRDKJZ81Y/alone-and-lonesome +$ crunchy download -o "daaaaaaaaaaaaaaaarling.ts" https://www.crunchyroll.com/watch/GRDKJZ81Y/alone-and-lonesome ``` With the `--audio` flag you can specify which audio the video should have and with `--subtitle` which subtitle it should have. Type `crunchy help download` to see all available locales. ```shell -$ crunchy download --audio ja-JP --subtitle de-DE https://beta.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx +$ crunchy download --audio ja-JP --subtitle de-DE https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` ##### Flags @@ -143,13 +143,13 @@ the `--language` flag. Archive a file ```shell -$ crunchy archive https://beta.crunchyroll.com/watch/GRDKJZ81Y/alone-and-lonesome +$ crunchy archive https://www.crunchyroll.com/watch/GRDKJZ81Y/alone-and-lonesome ``` Downloads the first two episode of Darling in the FranXX and stores it compressed in a file. ```shell -$ crunchy archive -c "ditf.tar.gz" https://beta.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx +$ crunchy archive -c "ditf.tar.gz" https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` ##### Flags diff --git a/cli/commands/login/login.go b/cli/commands/login/login.go index 9fce181..47a595d 100644 --- a/cli/commands/login/login.go +++ b/cli/commands/login/login.go @@ -51,7 +51,7 @@ func init() { Cmd.Flags().BoolVar(&loginRefreshTokenFlag, "refresh-token", false, - "Use a refresh token to login instead of username and password. Can be obtained by copying the `etp-rt` cookie from beta.crunchyroll.com") + "Use a refresh token to login instead of username and password. Can be obtained by copying the `etp-rt` cookie from crunchyroll.com") Cmd.MarkFlagsMutuallyExclusive("session-id", "refresh-token") } diff --git a/crunchy-cli.1 b/crunchy-cli.1 index 91ba30c..eb3f16d 100644 --- a/crunchy-cli.1 +++ b/crunchy-cli.1 @@ -164,7 +164,7 @@ Use the list below to get a better overview what is possible ...[S3E4-] - Download all episodes from and including season 3, episode 4. ...[S1E4-S3] - Download all episodes from and including season 1, episode 4, until and including season 3. -In practise, it would look like this: \fIhttps://beta.crunchyroll.com/series/12345678/example[S1E5-S3E2]\fR. +In practise, it would look like this: \fIhttps://www.crunchyroll.com/series/12345678/example[S1E5-S3E2]\fR. The \fBS\fR, followed by the number indicates the season number, \fBE\fR, followed by the number indicates an episode number. It doesn't matter if \fBS\fR, \fBE\fR or both are missing. Theoretically \fB[-]\fR is a valid pattern too. Note that \fBS\fR must always stay before \fBE\fR when used. @@ -175,23 +175,23 @@ $ crunchy-cli login user@example.com 12345678 Download a episode normally. Your system locale will be used for the video's audio. .br -$ crunchy-cli download https://beta.crunchyroll.com/watch/GRDKJZ81Y/alone-and-lonesome +$ crunchy-cli download https://www.crunchyroll.com/watch/GRDKJZ81Y/alone-and-lonesome Download a episode with 720p and name it to 'darling.mp4'. Note that you need \fBffmpeg\fR to save files which do not have '.ts' as file extension. .br -$ crunchy-cli download -o "darling.mp4" -r 720p https://beta.crunchyroll.com/watch/GRDKJZ81Y/alone-and-lonesome +$ crunchy-cli download -o "darling.mp4" -r 720p https://www.crunchyroll.com/watch/GRDKJZ81Y/alone-and-lonesome Download a episode with japanese audio and american subtitles. .br -$ crunchy-cli download -a ja-JP -s en-US https://beta.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx[E3-E5] +$ crunchy-cli download -a ja-JP -s en-US https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx[E3-E5] Stores the episode in a .mkv file. .br -$ crunchy-cli archive https://beta.crunchyroll.com/watch/GRDKJZ81Y/alone-and-lonesome +$ crunchy-cli archive https://www.crunchyroll.com/watch/GRDKJZ81Y/alone-and-lonesome Downloads the first two episode of Darling in the FranXX and stores it compressed in a file. .br -$ crunchy-cli archive -c "ditf.tar.gz" https://beta.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx[E1-E2] +$ crunchy-cli archive -c "ditf.tar.gz" https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx[E1-E2] .SH BUGS If you notice any bug or want an enhancement, feel free to create a new issue or pull request in the GitHub repository. From 95b66c3ff51f7df23d9c4603400ce1356b917c6c Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 31 Oct 2022 21:19:24 +0100 Subject: [PATCH 003/441] Fix subtitle styling and size (#66) --- cli/commands/archive/archive.go | 34 ++++++++++++++++++++++++++++++++- crunchy-cli.1 | 4 ++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/cli/commands/archive/archive.go b/cli/commands/archive/archive.go index b46ea0f..03ec3ac 100644 --- a/cli/commands/archive/archive.go +++ b/cli/commands/archive/archive.go @@ -40,6 +40,8 @@ var ( archiveResolutionFlag string archiveGoroutinesFlag int + + archiveNoSubtitleOptimizations bool ) var Cmd = &cobra.Command{ @@ -203,6 +205,11 @@ func init() { "g", runtime.NumCPU(), "Number of parallel segment downloads") + + Cmd.Flags().BoolVar(&archiveNoSubtitleOptimizations, + "no-subtitle-optimizations", + false, + "Disable subtitle optimizations. See https://github.com/crunchy-labs/crunchy-cli/issues/66 for more information") } func archive(urls []string) error { @@ -540,13 +547,38 @@ func archiveDownloadSubtitles(filename string, subtitles ...*crunchyroll.Subtitl } files = append(files, f.Name()) - if err := subtitle.Save(f); err != nil { + buffer := &bytes.Buffer{} + + if err := subtitle.Save(buffer); err != nil { f.Close() for _, file := range files { os.Remove(file) } return nil, err } + + if !archiveNoSubtitleOptimizations { + buffer2 := &bytes.Buffer{} + var scriptInfo bool + for _, line := range strings.Split(buffer.String(), "\n") { + if scriptInfo && strings.HasPrefix(strings.TrimSpace(line), "[") { + buffer2.WriteString("ScaledBorderAndShadows: yes\n") + scriptInfo = false + } else if strings.TrimSpace(line) == "[Script Info]" { + scriptInfo = true + } + buffer2.WriteString(line + "\n") + } + + if _, err = io.Copy(f, buffer2); err != nil { + return nil, err + } + } else { + if _, err = io.Copy(f, buffer); err != nil { + return nil, err + } + } + f.Close() utils.Log.Debug("Downloaded '%s' subtitles", subtitle.Locale) diff --git a/crunchy-cli.1 b/crunchy-cli.1 index eb3f16d..3032c79 100644 --- a/crunchy-cli.1 +++ b/crunchy-cli.1 @@ -146,6 +146,10 @@ The video resolution. Can either be specified via the pixels (e.g. 1920x1080), t \fB-g, --goroutines GOROUTINES\fR Sets the number of parallel downloads for the segments the final video is made of. Default is the number of cores the computer has. +.TP + +\fB--no-subtitle-optimizations DISABLE\fR +Disable subtitle optimizations which caused subtitle sizing and layout issues (https://github.com/crunchy-labs/crunchy-cli/issues/66). .SH UPDATE COMMAND Checks if a newer version is available. From 10617df834cf41f468eacc290f42454a781da2b7 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 31 Oct 2022 22:18:44 +0100 Subject: [PATCH 004/441] Fix archive sorting (#63) --- cli/commands/archive/archive.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/cli/commands/archive/archive.go b/cli/commands/archive/archive.go index 03ec3ac..4ac4f82 100644 --- a/cli/commands/archive/archive.go +++ b/cli/commands/archive/archive.go @@ -824,17 +824,22 @@ func archiveExtractEpisodes(url string) ([][]utils.FormatInformation, error) { } var infoFormat [][]utils.FormatInformation - for _, e := range eps { + var keys []int + for e := range eps { + keys = append(keys, e) + } + sort.Ints(keys) + + for _, k := range keys { var tmpFormatInfo []utils.FormatInformation - - var keys []int - for episodeNumber := range e { - keys = append(keys, episodeNumber) + var kkey []int + for ee := range eps[k] { + kkey = append(kkey, ee) } - sort.Ints(keys) + sort.Ints(kkey) - for _, key := range keys { - tmpFormatInfo = append(tmpFormatInfo, *e[key]) + for _, kk := range kkey { + tmpFormatInfo = append(tmpFormatInfo, *eps[k][kk]) } infoFormat = append(infoFormat, tmpFormatInfo) From 59e8793a2f4c08192af1c1dd7e3433a1ddc2e8f8 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Tue, 1 Nov 2022 22:17:19 +0100 Subject: [PATCH 005/441] Set option to modify locale used (#60) --- cli/root.go | 14 ++++++++++++++ crunchy-cli.1 | 8 ++++++++ utils/locale.go | 4 ++++ 3 files changed, 26 insertions(+) diff --git a/cli/root.go b/cli/root.go index b589a03..c44ca23 100644 --- a/cli/root.go +++ b/cli/root.go @@ -10,6 +10,8 @@ import ( "github.com/crunchy-labs/crunchy-cli/cli/commands/login" "github.com/crunchy-labs/crunchy-cli/cli/commands/update" "github.com/crunchy-labs/crunchy-cli/utils" + "github.com/crunchy-labs/crunchyroll-go/v3" + crunchyUtils "github.com/crunchy-labs/crunchyroll-go/v3/utils" "github.com/spf13/cobra" "os" "runtime/debug" @@ -22,6 +24,8 @@ var ( proxyFlag string + langFlag string + useragentFlag string ) @@ -40,6 +44,14 @@ var RootCmd = &cobra.Command{ utils.Log = commands.NewLogger(false, false, false) } + if langFlag != "" { + if !crunchyUtils.ValidateLocale(crunchyroll.LOCALE(langFlag)) { + return fmt.Errorf("'%s' is not a valid language. Choose from %s", langFlag, strings.Join(utils.LocalesAsStrings(), ", ")) + } + + os.Setenv("CRUNCHY_LANG", langFlag) + } + utils.Log.Debug("Executing `%s` command with %d arg(s)", cmd.Name(), len(args)) utils.Client, err = utils.CreateOrDefaultClient(proxyFlag, useragentFlag) @@ -53,6 +65,8 @@ func init() { RootCmd.PersistentFlags().StringVarP(&proxyFlag, "proxy", "p", "", "Proxy to use") + RootCmd.PersistentFlags().StringVar(&langFlag, "lang", "", fmt.Sprintf("Set language to use. If not set, it's received from the system locale dynamically. Choose from: %s", strings.Join(utils.LocalesAsStrings(), ", "))) + RootCmd.PersistentFlags().StringVar(&useragentFlag, "useragent", fmt.Sprintf("crunchy-cli/%s", utils.Version), "Useragent to do all request with") RootCmd.AddCommand(archive.Cmd) diff --git a/crunchy-cli.1 b/crunchy-cli.1 index 3032c79..af697f3 100644 --- a/crunchy-cli.1 +++ b/crunchy-cli.1 @@ -44,6 +44,14 @@ Disables all output. Shows verbose output. .TP +\fB--lang\fR +Set language to use. If not set, it's received from the system locale dynamically. Choose from: ar-ME, ar-SA, de-DE, en-US, es-419, es-ES, es-LA, fr-FR, it-IT, ja-JP, pt-BR, pt-PT, ru-RU, zh-CN. +.TP + +\fB--useragent\fR +Useragent to do all request with. +.TP + \fB--version\fR Shows the current cli version. diff --git a/utils/locale.go b/utils/locale.go index 9769940..1e28f6a 100644 --- a/utils/locale.go +++ b/utils/locale.go @@ -14,6 +14,10 @@ import ( // SystemLocale receives the system locale // https://stackoverflow.com/questions/51829386/golang-get-system-language/51831590#51831590 func SystemLocale(verbose bool) crunchyroll.LOCALE { + if lang, ok := os.LookupEnv("CRUNCHY_LANG"); ok { + return crunchyroll.LOCALE(lang) + } + if runtime.GOOS != "windows" { if lang, ok := os.LookupEnv("LANG"); ok { var l crunchyroll.LOCALE From 039d7cfb81684d9de6e0311af8ff4b5c86ab1bcc Mon Sep 17 00:00:00 2001 From: ByteDream Date: Thu, 20 Oct 2022 18:52:08 +0200 Subject: [PATCH 006/441] Rewrite it in Rust --- .github/dependabot.yml | 15 +- .github/workflows/ci.yml | 20 - .github/workflows/codeql-analysis.yml | 67 - Cargo.lock | 1645 +++++++++++++++++++++++++ Cargo.toml | 33 + Makefile | 31 - README.md | 272 ++-- build.rs | 112 ++ cli/commands/archive/archive.go | 818 ------------ cli/commands/archive/compress.go | 136 -- cli/commands/download/download.go | 368 ------ cli/commands/info/info.go | 40 - cli/commands/logger.go | 196 --- cli/commands/login/login.go | 159 --- cli/commands/unix.go | 48 - cli/commands/update/update.go | 151 --- cli/commands/utils.go | 125 -- cli/commands/windows.go | 41 - cli/root.go | 85 -- crunchy-cli-core/Cargo.toml | 29 + crunchy-cli-core/src/cli/archive.rs | 567 +++++++++ crunchy-cli-core/src/cli/download.rs | 452 +++++++ crunchy-cli-core/src/cli/log.rs | 197 +++ crunchy-cli-core/src/cli/login.rs | 39 + crunchy-cli-core/src/cli/mod.rs | 5 + crunchy-cli-core/src/cli/utils.rs | 178 +++ crunchy-cli-core/src/lib.rs | 196 +++ crunchy-cli-core/src/utils/clap.rs | 6 + crunchy-cli-core/src/utils/context.rs | 6 + crunchy-cli-core/src/utils/format.rs | 77 ++ crunchy-cli-core/src/utils/locale.rs | 15 + crunchy-cli-core/src/utils/log.rs | 19 + crunchy-cli-core/src/utils/mod.rs | 8 + crunchy-cli-core/src/utils/os.rs | 52 + crunchy-cli-core/src/utils/parse.rs | 170 +++ crunchy-cli-core/src/utils/sort.rs | 47 + crunchy-cli.1 | 219 ---- go.mod | 14 - go.sum | 14 - main.go | 9 - src/main.rs | 4 + utils/extract.go | 99 -- utils/file.go | 49 - utils/format.go | 63 - utils/http.go | 51 - utils/locale.go | 59 - utils/logger.go | 12 - utils/save.go | 177 --- utils/std.go | 10 - utils/system.go | 7 - utils/vars.go | 14 - 51 files changed, 4018 insertions(+), 3208 deletions(-) delete mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/codeql-analysis.yml create mode 100644 Cargo.lock create mode 100644 Cargo.toml delete mode 100644 Makefile create mode 100644 build.rs delete mode 100644 cli/commands/archive/archive.go delete mode 100644 cli/commands/archive/compress.go delete mode 100644 cli/commands/download/download.go delete mode 100644 cli/commands/info/info.go delete mode 100644 cli/commands/logger.go delete mode 100644 cli/commands/login/login.go delete mode 100644 cli/commands/unix.go delete mode 100644 cli/commands/update/update.go delete mode 100644 cli/commands/utils.go delete mode 100644 cli/commands/windows.go delete mode 100644 cli/root.go create mode 100644 crunchy-cli-core/Cargo.toml create mode 100644 crunchy-cli-core/src/cli/archive.rs create mode 100644 crunchy-cli-core/src/cli/download.rs create mode 100644 crunchy-cli-core/src/cli/log.rs create mode 100644 crunchy-cli-core/src/cli/login.rs create mode 100644 crunchy-cli-core/src/cli/mod.rs create mode 100644 crunchy-cli-core/src/cli/utils.rs create mode 100644 crunchy-cli-core/src/lib.rs create mode 100644 crunchy-cli-core/src/utils/clap.rs create mode 100644 crunchy-cli-core/src/utils/context.rs create mode 100644 crunchy-cli-core/src/utils/format.rs create mode 100644 crunchy-cli-core/src/utils/locale.rs create mode 100644 crunchy-cli-core/src/utils/log.rs create mode 100644 crunchy-cli-core/src/utils/mod.rs create mode 100644 crunchy-cli-core/src/utils/os.rs create mode 100644 crunchy-cli-core/src/utils/parse.rs create mode 100644 crunchy-cli-core/src/utils/sort.rs delete mode 100644 crunchy-cli.1 delete mode 100644 go.mod delete mode 100644 go.sum delete mode 100644 main.go create mode 100644 src/main.rs delete mode 100644 utils/extract.go delete mode 100644 utils/file.go delete mode 100644 utils/format.go delete mode 100644 utils/http.go delete mode 100644 utils/locale.go delete mode 100644 utils/logger.go delete mode 100644 utils/save.go delete mode 100644 utils/std.go delete mode 100644 utils/system.go delete mode 100644 utils/vars.go diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3938344..40d73b8 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,6 +1,17 @@ version: 2 updates: - - package-ecosystem: "gomod" + - package-ecosystem: "cargo" directory: "/" schedule: - interval: "daily" + interval: "weekly" + ignore: + - dependency-name: "*" + update-types: [ "version-update:semver-patch" ] + + - package-ecosystem: "cargo" + directory: "/crunchy-cli-core" + schedule: + interval: "weekly" + ignore: + - dependency-name: "*" + update-types: [ "version-update:semver-patch" ] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index b2409a0..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: CI - -on: [ push, pull_request ] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Set up Go - uses: actions/setup-go@v3 - with: - go-version: 1.18 - - - name: Build - run: go build -v . - - - name: Test - run: go test -v . diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index e0f5470..0000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,67 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - push: - pull_request: - schedule: - - cron: '40 3 * * 2' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'go' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://git.io/codeql-language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v2 - - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..05b2d65 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1645 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aes" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "433cfd6710c9986c576a25ca913c39d66a6474107b406f34f91d4a8923395241" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "aho-corasick" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" + +[[package]] +name = "async-channel" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14485364214912d3b19cc3435dde4df66065127f05fa0d75c712f36f12c2f28" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + +[[package]] +name = "async-trait" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e805d94e6b5001b651426cf4cd446b1ab5f319d27bab5c644f61de0a804360c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "block-padding" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a90ec2df9600c28a01c56c4784c9207a96d2451833aeceb8cc97e4c9548bb78" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" + +[[package]] +name = "bytes" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" + +[[package]] +name = "cache-padded" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1db59621ec70f09c5e9b597b220c7a2b43611f4710dc03ceb8748637775692c" + +[[package]] +name = "castaway" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2698f953def977c68f935bb0dfa959375ad4638570e969e2f1e9f433cbf1af6" + +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + +[[package]] +name = "cc" +version = "1.0.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-integer", + "num-traits", + "serde", + "time", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "cipher" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1873270f8f7942c191139cb8a40fd228da6c3fd2fc376d7e92d47aa14aeb59e" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "clap" +version = "4.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2148adefda54e14492fb9bddcc600b4344c5d1a3123bd666dcb939c6f0e0e57e" +dependencies = [ + "atty", + "bitflags", + "clap_derive", + "clap_lex", + "once_cell", + "strsim", + "termcolor", +] + +[[package]] +name = "clap_complete" +version = "4.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b0fba905b035a30d25c1b585bf1171690712fbb0ad3ac47214963aa4acc36c" +dependencies = [ + "clap", +] + +[[package]] +name = "clap_derive" +version = "4.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "clap_mangen" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa149477df7854a1497db0def32b8a65bf98f72a14d04ac75b01938285d83420" +dependencies = [ + "clap", + "roff", +] + +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + +[[package]] +name = "concurrent-queue" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af4780a44ab5696ea9e28294517f1fffb421a83a25af521333c838635509db9c" +dependencies = [ + "cache-padded", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + +[[package]] +name = "cpufeatures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +dependencies = [ + "libc", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crunchy-cli" +version = "0.1.0" +dependencies = [ + "chrono", + "clap", + "clap_complete", + "clap_mangen", + "crunchy-cli-core", + "tokio", +] + +[[package]] +name = "crunchy-cli-core" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "chrono", + "clap", + "crunchyroll-rs", + "ctrlc", + "dirs", + "isahc", + "log", + "num_cpus", + "regex", + "signal-hook", + "sys-locale", + "tempfile", + "terminal_size", + "tokio", +] + +[[package]] +name = "crunchyroll-rs" +version = "0.1.0" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#7aedfc6c9a91a42ef46639ba9e99adba63cd0dda" +dependencies = [ + "aes", + "cbc", + "chrono", + "crunchyroll-rs-internal", + "http", + "isahc", + "m3u8-rs", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "smart-default", + "tokio", +] + +[[package]] +name = "crunchyroll-rs-internal" +version = "0.1.0" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#7aedfc6c9a91a42ef46639ba9e99adba63cd0dda" +dependencies = [ + "darling", + "quote", + "syn", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "ctrlc" +version = "3.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d91974fbbe88ec1df0c24a4f00f99583667a7e2e6272b2b92d294d81e462173" +dependencies = [ + "nix", + "winapi", +] + +[[package]] +name = "curl" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "509bd11746c7ac09ebd19f0b17782eae80aadee26237658a6b4808afb5c11a22" +dependencies = [ + "curl-sys", + "libc", + "openssl-probe", + "openssl-sys", + "schannel", + "socket2", + "winapi", +] + +[[package]] +name = "curl-sys" +version = "0.4.59+curl-7.86.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cfce34829f448b08f55b7db6d0009e23e2e86a34e8c2b366269bf5799b4a407" +dependencies = [ + "cc", + "libc", + "libnghttp2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", + "winapi", +] + +[[package]] +name = "cxx" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4a41a86530d0fe7f5d9ea779916b7cadd2d4f9add748b99c2c029cbbdfaf453" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06416d667ff3e3ad2df1cd8cd8afae5da26cf9cec4d0825040f88b5ca659a2f0" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "820a9a2af1669deeef27cb271f476ffd196a2c4b6731336011e0ba63e2c7cf71" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a08a6e2fcc370a089ad3b4aaf54db3b1b4cee38ddabce5896b33eb693275f470" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "darling" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0dd3cd20dc6b5a876612a6e5accfe7f3dd883db6d07acfbf14c128f61550dfa" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a784d2ccaf7c98501746bf0be29b2022ba41fd62a2e622af997a03e9f972859f" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7618812407e9402654622dd402b0a89dff9ba93badd6540781526117b92aab7e" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "encoding_rs" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "fastrand" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +dependencies = [ + "instant", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-core" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" + +[[package]] +name = "futures-io" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" + +[[package]] +name = "futures-lite" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + +[[package]] +name = "generic-array" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "http" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "iana-time-zone" +version = "0.1.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "io-lifetimes" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ce5ef949d49ee85593fc4d3f3f95ad61657076395cbbce23e2121fc5542074" + +[[package]] +name = "isahc" +version = "1.7.0" +source = "git+https://github.com/sagebind/isahc?rev=34f158ef#34f158ef9f87b2387bed2c81936916a29c1eaad1" +dependencies = [ + "async-channel", + "castaway", + "crossbeam-utils", + "curl", + "curl-sys", + "encoding_rs", + "event-listener", + "futures-lite", + "http", + "httpdate", + "log", + "mime", + "once_cell", + "polling", + "slab", + "sluice", + "tracing", + "tracing-futures", + "url", + "waker-fn", +] + +[[package]] +name = "itoa" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" + +[[package]] +name = "js-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.137" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" + +[[package]] +name = "libnghttp2-sys" +version = "0.1.7+1.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57ed28aba195b38d5ff02b9170cbff627e336a20925e43b4945390401c5dc93f" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "libz-sys" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "link-cplusplus" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369" +dependencies = [ + "cc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d" + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "m3u8-rs" +version = "5.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15d091887fd4a920417805283b7a838d0dcda68e8d632cd305a4439ee776d1ce" +dependencies = [ + "chrono", + "nom", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nix" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e322c04a9e3440c327fca7b6c8a63e6890a32fa2ad689db972425f07e0d22abb" +dependencies = [ + "autocfg", + "bitflags", + "cfg-if", + "libc", +] + +[[package]] +name = "nom" +version = "7.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-src" +version = "111.24.0+1.1.1s" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3498f259dab01178c6228c6b00dcef0ed2a2d5e20d648c017861227773ea4abd" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07d5c8cb6e57b3a3612064d7b18b117912b4ce70955c2504d4b741c9e244b132" +dependencies = [ + "autocfg", + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "os_str_bytes" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" + +[[package]] +name = "parking" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" + +[[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + +[[package]] +name = "pin-project" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pkg-config" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" + +[[package]] +name = "polling" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab4609a838d88b73d8238967b60dd115cc08d38e2bbaf51ee1e4b695f89122e2" +dependencies = [ + "autocfg", + "cfg-if", + "libc", + "log", + "wepoll-ffi", + "winapi", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom", + "redox_syscall", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "roff" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b833d8d034ea094b1ea68aa6d5c740e0d04bad9d16568d08ba6f76823a114316" + +[[package]] +name = "rustix" +version = "0.35.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "727a1a6d65f786ec22df8a81ca3121107f235970dc1705ed681d3e6e8b9cd5f9" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.42.0", +] + +[[package]] +name = "ryu" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" + +[[package]] +name = "schannel" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" +dependencies = [ + "lazy_static", + "windows-sys 0.36.1", +] + +[[package]] +name = "scratch" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" + +[[package]] +name = "serde" +version = "1.0.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "signal-hook" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg", +] + +[[package]] +name = "sluice" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7400c0eff44aa2fcb5e31a5f24ba9716ed90138769e4977a2ba6014ae63eb5" +dependencies = [ + "async-channel", + "futures-core", + "futures-io", +] + +[[package]] +name = "smart-default" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "133659a15339456eeeb07572eb02a91c91e9815e9cbc89566944d2c8d3efdbf6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "socket2" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sys-locale" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3358acbb4acd4146138b9bda219e904a6bb5aaaa237f8eed06f4d6bc1580ecee" +dependencies = [ + "js-sys", + "libc", + "wasm-bindgen", + "web-sys", + "winapi", +] + +[[package]] +name = "tempfile" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +dependencies = [ + "cfg-if", + "fastrand", + "libc", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "termcolor" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "terminal_size" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ca90c434fd12083d1a6bdcbe9f92a14f96c8a1ba600ba451734ac334521f7a" +dependencies = [ + "rustix", + "windows-sys 0.42.0", +] + +[[package]] +name = "thiserror" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "1.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d76ce4a75fb488c605c54bf610f221cea8b0dafb53333c1a67e8ee199dcd2ae3" +dependencies = [ + "autocfg", + "num_cpus", + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + +[[package]] +name = "typenum" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" + +[[package]] +name = "unicode-bidi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" + +[[package]] +name = "unicode-ident" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + +[[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "waker-fn" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" + +[[package]] +name = "web-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wepoll-ffi" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb" +dependencies = [ + "cc", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +dependencies = [ + "windows_aarch64_msvc 0.36.1", + "windows_i686_gnu 0.36.1", + "windows_i686_msvc 0.36.1", + "windows_x86_64_gnu 0.36.1", + "windows_x86_64_msvc 0.36.1", +] + +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc 0.42.0", + "windows_i686_gnu 0.42.0", + "windows_i686_msvc 0.42.0", + "windows_x86_64_gnu 0.42.0", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc 0.42.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" + +[[package]] +name = "windows_i686_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" + +[[package]] +name = "windows_i686_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..c99a2a9 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "crunchy-cli" +version = "0.1.0" +edition = "2021" + +[features] +default = ["static-curl"] + +# Embed a static curl library into the binary instead of just linking it. +static-curl = ["crunchy-cli-core/static-curl"] +# Embed a static openssl library into the binary instead of just linking it. If you want to compile this project against +# musl and have openssl issues, this might solve these issues. +static-ssl = ["crunchy-cli-core/static-ssl"] + +[dependencies] +tokio = { version = "1.22", features = ["macros", "rt-multi-thread", "time"], default-features = false } + +crunchy-cli-core = { path = "./crunchy-cli-core" } + +[build-dependencies] +chrono = "0.4" +clap = { version = "4.0", features = ["string"] } +clap_complete = "4.0" +clap_mangen = "0.2" + +# The static-* features must be used here since build dependency features cannot be manipulated from the features +# specified in this Cargo.toml [features]. +crunchy-cli-core = { path = "./crunchy-cli-core", features = ["static-curl", "static-ssl"] } + +[profile.release] +strip = true +opt-level = "z" +lto = true diff --git a/Makefile b/Makefile deleted file mode 100644 index 966872b..0000000 --- a/Makefile +++ /dev/null @@ -1,31 +0,0 @@ -VERSION=development -BINARY_NAME=crunchy -VERSION_BINARY_NAME=$(BINARY_NAME)-v$(VERSION) - -DESTDIR= -PREFIX=/usr - -build: - go build -ldflags "-X 'github.com/crunchy-labs/crunchy-cli/utils.Version=$(VERSION)'" -o $(BINARY_NAME) . - -clean: - rm -f $(BINARY_NAME) $(VERSION_BINARY_NAME)_* - -install: - install -Dm755 $(BINARY_NAME) $(DESTDIR)$(PREFIX)/bin/crunchy-cli - ln -sf ./crunchy-cli $(DESTDIR)$(PREFIX)/bin/crunchy - install -Dm644 crunchy-cli.1 $(DESTDIR)$(PREFIX)/share/man/man1/crunchy-cli.1 - install -Dm644 LICENSE $(DESTDIR)$(PREFIX)/share/licenses/crunchy-cli/LICENSE - -uninstall: - rm -f $(DESTDIR)$(PREFIX)/bin/crunchy-cli - rm -f $(DESTDIR)$(PREFIX)/bin/crunchy - rm -f $(DESTDIR)$(PREFIX)/share/man/man1/crunchy-cli.1 - rm -f $(DESTDIR)$(PREFIX)/share/licenses/crunchy-cli/LICENSE - -release: - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-X 'github.com/crunchy-labs/crunchy-cli/utils.Version=$(VERSION)'" -o $(VERSION_BINARY_NAME)_linux . - CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -ldflags "-X 'github.com/crunchy-labs/crunchy-cli/utils.Version=$(VERSION)'" -o $(VERSION_BINARY_NAME)_windows.exe . - CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -ldflags "-X 'github.com/crunchy-labs/crunchy-cli/utils.Version=$(VERSION)'" -o $(VERSION_BINARY_NAME)_darwin . - - strip $(VERSION_BINARY_NAME)_linux diff --git a/README.md b/README.md index cbe6a37..2229739 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # crunchy-cli -A [Go](https://golang.org) written cli client for [crunchyroll](https://www.crunchyroll.com). To use it, you need a crunchyroll premium account for full access & features. +A [Rust](https://www.rust-lang.org/) written cli client for [Crunchyroll](https://www.crunchyroll.com).

@@ -21,178 +21,202 @@ A [Go](https://golang.org) written cli client for [crunchyroll](https://www.crun

- CLI đŸ–Ĩī¸ + Usage đŸ–Ĩī¸ â€ĸ Disclaimer ☝ī¸ â€ĸ License ⚖

-_This repo was former known as **crunchyroll-go** (which still exists but now contains only the library part) but got split up into two separate repositories to provide more flexibility._ - -> This tool relies on the [crunchyroll-go](https://github.com/crunchy-labs/crunchyroll-go) library to communicate with crunchyroll. -> The library enters maintenance mode (only small fixes, no new features) with version v3 in favor of rewriting it completely in Rust. -> **crunchy-cli** follows it (with version v2.3.0) and won't have major updates until the Rust rewrite of the library reaches a good usable state. - -# đŸ–Ĩī¸ CLI +> We are in no way affiliated with, maintained, authorized, sponsored, or officially associated with Crunchyroll LLC or any of its subsidiaries or affiliates. +> The official Crunchyroll website can be found at https://crunchyroll.com/. ## ✨ Features -- Download single videos and entire series from [crunchyroll](https://www.crunchyroll.com). -- Archive episode or seasons in an `.mkv` file with multiple subtitles and audios and compress them to gzip or zip files. +- Download single videos and entire series from [Crunchyroll](https://www.crunchyroll.com). +- Archive episode or seasons in an `.mkv` file with multiple subtitles and audios. - Specify a range which episodes to download from an anime. ## 💾 Get the executable -- đŸ“Ĩ Download the latest binaries [here](https://github.com/crunchy-labs/crunchy-cli/releases/latest) or get it from below: - - [Linux (x64)](https://smartrelease.bytedream.org/github/crunchy-labs/crunchy-cli/crunchy-{tag}_linux) - - [Windows (x64)](https://smartrelease.bytedream.org/github/crunchy-labs/crunchy-cli/crunchy-{tag}_windows.exe) - - [MacOS (x64)](https://smartrelease.bytedream.org/github/crunchy-labs/crunchy-cli/crunchy-{tag}_darwin) -- If you use Arch btw. or any other Linux distro which is based on Arch Linux, you can download the package via the [AUR](https://aur.archlinux.org/packages/crunchyroll-go/): - ```shell - $ yay -S crunchy-cli - ``` -- On Windows [scoop](https://scoop.sh/) can be used to install it (added by [@AdmnJ](https://github.com/AdmnJ)): - ```shell - $ scoop bucket add extras # <- in case you haven't added the extra repository already - $ scoop install crunchyroll-go - ``` -- 🛠 Build it yourself. Must be done if your target platform is not covered by the [provided binaries](https://github.com/crunchy-labs/crunchy-cli/releases/latest) (like Raspberry Pi or M1 Mac): - - use `make` (requires `go` to be installed): - ```shell - $ git clone https://github.com/crunchy-labs/crunchy-cli - $ cd crunchy-cli - $ make - $ sudo make install # <- only if you want to install it on your system - ``` - - use `go`: - ```shell - $ git clone https://github.com/crunchy-labs/crunchy-cli - $ cd crunchy-cli - $ go build -o crunchy . - ``` +### đŸ“Ĩ Download the latest binaries -## 📝 Examples +Checkout the [releases](https://github.com/crunchy-labs/crunchy-cli/releases) tab and get the binary from the newest release. -_Before reading_: Because of the huge functionality not all cases can be covered in the README. Make sure to check the [wiki](https://github.com/crunchy-labs/crunchy-cli/wiki/Cli), further usages and options are described there. +### 🛠 Build it yourself + +Since we do not support every platform and architecture you may have to build the project yourself. +This requires [git](https://git-scm.com/) and [Cargo](https://doc.rust-lang.org/cargo). +```shell +$ git clone https://github.com/crunchy-labs/crunchy-cli +$ cd crunchy-cli +$ cargo build --release +``` +After the binary has built successfully it is available in `target/release`. + +## đŸ–Ĩī¸ Usage + +> All shown command are just examples + +Every command requires you to be logged in with an account. +It doesn't matter if this account is premium or not, both works (but as free user you do not have access to premium content). +You can pass your account via credentials (username & password) or refresh token. + +- Refresh Token + - To get the token you have to log in at [crunchyroll.com](https://www.crunchyroll.com/) and extract the `etp_rt` cookie. + The easiest way to get it is via a browser extension with lets you view your cookies, like [Cookie-Editor](https://cookie-editor.cgagnier.ca/) ([Firefox Store](https://addons.mozilla.org/en-US/firefox/addon/cookie-editor/); [Chrome Store](https://chrome.google.com/webstore/detail/cookie-editor/hlkenndednhfkekhgcdicdfddnkalmdm)). + If installed, search the `etp_rt` entry and extract the value. + - ```shell + $ crunchy --etp-rt "abcd1234-zyxw-9876-98zy-a1b2c3d4e5f6" + ``` +- Credentials + - Credentials must be provided as one single expression. + Username and password must be separated by a `:`. + - ```shell + $ crunchy --credentials "user:password" + ``` ### Login -Before you can do something, you have to log in first. - -This can be performed via crunchyroll account email and password. +If you do not want to provide your credentials every time you execute a command, they can be stored permanently on disk. +This can be done with the `login` subcommand. ```shell -$ crunchy login user@example.com password +$ crunchy --etp-rt "abcd1234-zyxw-9876-98zy-a1b2c3d4e5f6" login ``` -or via refresh token / `etp_rt` cookie - -```shell -$ crunchy login --refresh-token 7578ce50-5712-3gef-b97e-01332d6b588c -``` +Once set, you do not need to provide `--etp-rt` / `--credentials` anymore when using the cli. ### Download -By default, the cli tries to download the episode with your system language as audio. If no streams with your system language are available, the video will be downloaded with japanese audio and hardsubbed subtitles in your system language. -**If your system language is not supported, an error message will be displayed and en-US (american english) will be chosen as language.** +**Supported urls** +- Single episode + ```shell + $ crunchy download https://www.crunchyroll.com/watch/GRDQPM1ZY/alone-and-lonesome + ``` +- Episode range + + If you want only specific episodes / seasons of an anime you can easily provide the series url along with a _filter_. + The filter has to be attached to the url. See the [wiki](https://github.com/crunchy-labs/crunchy-cli/wiki/Cli#filter) for more information + ```shell + $ crunchy download https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx[E1] + ``` +- Series + ```shell + $ crunchy download https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx + ``` + +**Options** +- Audio language -```shell -$ crunchy download https://beta.crunchyroll.com/watch/GRDKJZ81Y/alone-and-lonesome -``` + Which audio the episode(s) should be can be set via the `-a` / `--audio` flag. + This only works if the url points to a series since episode urls are language specific. + ```shell + $ crunchy download -a de-DE https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx + ``` + Default is your system language. If not supported by Crunchyroll, `en-US` (American English) is the default. -With `-r best` the video(s) will have the best available resolution (mostly 1920x1080 / Full HD). +- Subtitle language -```shell -$ crunchy download -r best https://beta.crunchyroll.com/watch/GRDKJZ81Y/alone-and-lonesome -``` + Besides the audio, it's also possible to specify which language the subtitles should have with the `-s` / `--subtitle` flag. + The subtitle will be hardsubbed (burned into the video) and thus, can't be turned off or on. + ```shell + $ crunchy download -s de-DE https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx + ``` + Default is no subtitle. -The file is by default saved as a `.ts` (mpeg transport stream) file. -`.ts` files may can't be played or are looking very weird (it depends on the video player you are using). With the `-o` flag, you can change the name (and file ending) of the output file. So if you want to save it as, for example, `mp4` -file, just name it `whatever.mp4`. -**You need [ffmpeg](https://ffmpeg.org) to store the video in other file formats.** +- Output filename -```shell -$ crunchy download -o "daaaaaaaaaaaaaaaarling.ts" https://beta.crunchyroll.com/watch/GRDKJZ81Y/alone-and-lonesome -``` + You can specify the name of the output file with the `-o` / `--output` flag. + If you want to use any other file format than [`.ts`](https://en.wikipedia.org/wiki/MPEG_transport_stream) you need [ffmpeg](https://ffmpeg.org/). + ```shell + $ crunchy download -o "ditf.ts" https://www.crunchyroll.com/watch/GRDQPM1ZY/alone-and-lonesome + ``` + Default is `{title}.ts`. -With the `--audio` flag you can specify which audio the video should have and with `--subtitle` which subtitle it should have. Type `crunchy help download` to see all available locales. +- Resolution -```shell -$ crunchy download --audio ja-JP --subtitle de-DE https://beta.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx -``` - -##### Flags - -The following flags can be (optional) passed to modify the [download](#download) process. - -| Short | Extended | Description | -|-------|----------------|--------------------------------------------------------------------------------| -| `-a` | `--audio` | Forces audio of the video(s). | -| `-s` | `--subtitle` | Forces subtitle of the video(s). | -| `-d` | `--directory` | Directory to download the video(s) to. | -| `-o` | `--output` | Name of the output file. | -| `-r` | `--resolution` | The resolution of the video(s). `best` for best resolution, `worst` for worst. | -| `-g` | `--goroutines` | Sets how many parallel segment downloads should be used. | + The resolution for videos can be set via the `-r` / `--resolution` flag. + ```shell + $ crunchy download -r worst https://www.crunchyroll.com/watch/GRDQPM1ZY/alone-and-lonesome + ``` + Default is `best`. ### Archive -Archive works just like [download](#download). It downloads the given videos as `.mkv` files and stores all (soft) subtitles in it. Default audio locales are japanese and your system language (if available) but you can set more or less with -the `--language` flag. +**Supported urls** +- Series -Archive a file + Only series urls are supported since single episode urls are (audio) language locked. + ```shell + $ crunchy archive https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx + ``` + +**Options** +- Audio languages -```shell -$ crunchy archive https://beta.crunchyroll.com/watch/GRDKJZ81Y/alone-and-lonesome -``` + Which audios the episode(s) should be can be set via the `-a` / `--audio` flag. + ```shell + $ crunchy archive -a ja-JP -a de-DE https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx + ``` + Can be used multiple times. + Default is your system language (if not supported by Crunchyroll, `en-US` (American English) is the default) + `ja-JP` (Japanese). -Downloads the first two episode of Darling in the FranXX and stores it compressed in a file. +- Subtitle languages -```shell -$ crunchy archive -c "ditf.tar.gz" https://beta.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx -``` + Besides the audio, it's also possible to specify which languages the subtitles should have with the `-s` / `--subtitle` flag. + ```shell + $ crunchy archive -s de-DE https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx + ``` + Default is all subtitles. -##### Flags +- Output filename -The following flags can be (optional) passed to modify the [archive](#archive) process. + You can specify the name of the output file with the `-o` / `--output` flag. + The only supported file / container format is [`.mkv`](https://en.wikipedia.org/wiki/Matroska) since it stores / can store multiple audio, video and subtitle streams. + ```shell + $ crunchy archive -o "{title}.mkv" https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx + ``` + Default is `{title}.mkv`. -| Short | Extended | Description | -|-------|----------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `-l` | `--language` | Audio locale which should be downloaded. Can be used multiple times. | -| `-d` | `--directory` | Directory to download the video(s) to. | -| `-o` | `--output` | Name of the output file. | -| `-m` | `--merge` | Sets the behavior of the stream merging. Valid behaviors are 'auto', 'audio', 'video'. See the [wiki](https://github.com/crunchy-labs/crunchy-cli/wiki/Cli#archive) for more information. | -| `-c` | `--compress` | If is set, all output will be compresses into an archive. This flag sets the name of the compressed output file and the file ending specifies the compression algorithm (gzip, tar, zip are supported). | -| `-r` | `--resolution` | The resolution of the video(s). `best` for best resolution, `worst` for worst. | -| `-g` | `--goroutines` | Sets how many parallel segment downloads should be used. | +- Resolution + The resolution for videos can be set via the `-r` / `--resolution` flag. + ```shell + $ crunchy archive -r worst https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx + ``` + Default is `best`. -### Info +- Merge behavior -The `info` displays some information about the account which is used for the cli. + Because of local restrictions (or other reasons) some episodes with different languages does not have the same length (e.g. when some scenes were cut out). + The ideal state, when multiple audios & subtitles used, would be if only one _video_ has to be stored and all other languages can be stored as audio-only. + But, as said, this is not always the case. + With the `-m` / `--merge` flag you can set what you want to do if some video lengths differ. + Valid options are `audio` - store one video and all other languages as audio only; `video` - store the video + audio for every language; `auto` - detect if videos differ in length: if so, behave like `video` else like `audio`. + Subtitles will always match to the first / primary audio and video. + ```shell + $ crunchy archive -m audio https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx + ``` + Default is `auto`. -```shell -$ crunchy info -``` +- Default subtitle -### Update + `--default_subtitle` set which subtitle language should be set as default / auto appear when starting the downloaded video(s). + ```shell + $ crunchy archive --default_subtitle en-US https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx + ``` + Default is none. -If you want to update your local version of `crunchy-cli`, this command makes this easier. -It checks if a new version is available and if so, updates itself. +- No subtitle optimizations -```shell -$ crunchy update -``` - -### Global flags - -These flags you can use across every sub-command: - -| Flag | Description | -|------|------------------------------------------------------| -| `-q` | Disables all output. | -| `-v` | Shows additional debug output. | -| `-p` | Use a proxy to hide your ip / redirect your traffic. | + Subtitles, as Crunchyroll delivers them, look weird in some video players (#66). + This can be fixed by adding a specific entry to the subtitles. + But since this entry is only a de-factor standard and not represented in the official specification of the subtitle format ([`.ass`](https://en.wikipedia.org/wiki/SubStation_Alpha)) it could cause issues with some video players (but no issue got reported so far, so it's relatively safe to use). + `--no_subtitle_optimizations` can disable these optimizations. + ```shell + $ crunchy archive --no_subtitle_optimizations https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx + ``` # ☝ī¸ Disclaimer diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..cc6a0cc --- /dev/null +++ b/build.rs @@ -0,0 +1,112 @@ +use clap::{Command, CommandFactory}; +use clap_complete::shells; +use std::path::{Path, PathBuf}; + +// this build file generates completions for various shells as well as manual pages + +fn main() -> std::io::Result<()> { + // do not generate anything when building non release + if cfg!(debug_assertions) { + return Ok(()); + } + + // note that we're using an anti-pattern here / violate the rust conventions. build script are + // not supposed to write outside of 'OUT_DIR'. to have the generated files in the build "root" + // (the same directory where the output binary lives) is much simpler than in 'OUT_DIR' since + // its nested in sub directories and is difficult to find (at least more difficult than in the + // build root) + let unconventional_out_dir = + std::path::PathBuf::from(std::env::var_os("OUT_DIR").ok_or(std::io::ErrorKind::NotFound)?) + .parent() + .unwrap() + .parent() + .unwrap() + .parent() + .unwrap() + .to_path_buf(); + + let completions_dir = exist_or_create_dir(unconventional_out_dir.join("completions"))?; + let manpage_dir = exist_or_create_dir(unconventional_out_dir.join("manpages"))?; + + generate_completions(completions_dir)?; + generate_manpages(manpage_dir)?; + + Ok(()) +} + +fn exist_or_create_dir(path: PathBuf) -> std::io::Result { + if !path.exists() { + std::fs::create_dir(path.clone())? + } + Ok(path) +} + +fn generate_completions(out_dir: PathBuf) -> std::io::Result<()> { + let mut command: Command = crunchy_cli_core::Cli::command(); + + clap_complete::generate_to( + shells::Bash, + &mut command.clone(), + "crunchy-cli", + out_dir.clone(), + )?; + clap_complete::generate_to( + shells::Elvish, + &mut command.clone(), + "crunchy-cli", + out_dir.clone(), + )?; + println!( + "{}", + clap_complete::generate_to( + shells::Fish, + &mut command.clone(), + "crunchy-cli", + out_dir.clone(), + )? + .to_string_lossy() + ); + clap_complete::generate_to( + shells::PowerShell, + &mut command.clone(), + "crunchy-cli", + out_dir.clone(), + )?; + clap_complete::generate_to(shells::Zsh, &mut command, "crunchy-cli", out_dir)?; + + Ok(()) +} + +fn generate_manpages(out_dir: PathBuf) -> std::io::Result<()> { + fn generate_command_manpage( + mut command: Command, + base_path: &Path, + sub_name: &str, + ) -> std::io::Result<()> { + let (file_name, title) = if sub_name.is_empty() { + command = command.name("crunchy-cli"); + ("crunchy-cli.1".to_string(), "crunchy-cli".to_string()) + } else { + command = command.name(format!("crunchy-cli {}", sub_name)); + ( + format!("crunchy-cli-{}.1", sub_name), + format!("crunchy-cli-{}", sub_name), + ) + }; + + let mut command_buf = vec![]; + let man = clap_mangen::Man::new(command) + .title(title) + .date(chrono::Utc::now().format("%b %d, %Y").to_string()); + man.render(&mut command_buf)?; + + std::fs::write(base_path.join(file_name), command_buf) + } + + generate_command_manpage(crunchy_cli_core::Cli::command(), &out_dir, "")?; + generate_command_manpage(crunchy_cli_core::Archive::command(), &out_dir, "archive")?; + generate_command_manpage(crunchy_cli_core::Download::command(), &out_dir, "download")?; + generate_command_manpage(crunchy_cli_core::Login::command(), &out_dir, "login")?; + + Ok(()) +} diff --git a/cli/commands/archive/archive.go b/cli/commands/archive/archive.go deleted file mode 100644 index 744635f..0000000 --- a/cli/commands/archive/archive.go +++ /dev/null @@ -1,818 +0,0 @@ -package archive - -import ( - "bufio" - "bytes" - "context" - "fmt" - "io" - "math" - "os" - "os/exec" - "os/signal" - "path/filepath" - "regexp" - "runtime" - "sort" - "strconv" - "strings" - - "github.com/crunchy-labs/crunchy-cli/cli/commands" - "github.com/crunchy-labs/crunchy-cli/utils" - "github.com/crunchy-labs/crunchyroll-go/v3" - crunchyUtils "github.com/crunchy-labs/crunchyroll-go/v3/utils" - "github.com/grafov/m3u8" - "github.com/spf13/cobra" -) - -var ( - archiveLanguagesFlag []string - archiveSubLanguagesFlag []string - - archiveDirectoryFlag string - archiveOutputFlag string - archiveTempDirFlag string - - archiveMergeFlag string - - archiveCompressFlag string - - archiveResolutionFlag string - - archiveGoroutinesFlag int -) - -var Cmd = &cobra.Command{ - Use: "archive", - Short: "Stores the given videos with all subtitles and multiple audios in a .mkv file", - Args: cobra.MinimumNArgs(1), - - PreRunE: func(cmd *cobra.Command, args []string) error { - utils.Log.Debug("Validating arguments") - - if !utils.HasFFmpeg() { - return fmt.Errorf("ffmpeg is needed to run this command correctly") - } - utils.Log.Debug("FFmpeg detected") - - if filepath.Ext(archiveOutputFlag) != ".mkv" { - return fmt.Errorf("currently only matroska / .mkv files are supported") - } - - for _, locale := range archiveLanguagesFlag { - if !crunchyUtils.ValidateLocale(crunchyroll.LOCALE(locale)) { - // if locale is 'all', match all known locales - if locale == "all" { - archiveLanguagesFlag = utils.LocalesAsStrings() - break - } - return fmt.Errorf("%s is not a valid locale. Choose from: %s", locale, strings.Join(utils.LocalesAsStrings(), ", ")) - } - } - utils.Log.Debug("Using following audio locales: %s", strings.Join(archiveLanguagesFlag, ", ")) - - for _, locale := range archiveSubLanguagesFlag { - if !crunchyUtils.ValidateLocale(crunchyroll.LOCALE(locale)) { - // if locale is 'all', match all known locales - if locale == "all" { - archiveSubLanguagesFlag = utils.LocalesAsStrings() - break - } - return fmt.Errorf("%s is not a valid locale for Subtitels. Choose from: %s", locale, strings.Join(utils.LocalesAsStrings(), ", ")) - } - } - utils.Log.Debug("Using following subtitels locales: %s", strings.Join(archiveSubLanguagesFlag, ", ")) - - var found bool - for _, mode := range []string{"auto", "audio", "video"} { - if archiveMergeFlag == mode { - utils.Log.Debug("Using %s merge behavior", archiveMergeFlag) - found = true - break - } - } - if !found { - return fmt.Errorf("'%s' is no valid merge flag. Use 'auto', 'audio' or 'video'", archiveMergeFlag) - } - - if archiveCompressFlag != "" { - found = false - for _, algo := range []string{".tar", ".tar.gz", ".tgz", ".zip"} { - if strings.HasSuffix(archiveCompressFlag, algo) { - utils.Log.Debug("Using %s compression", algo) - found = true - break - } - } - if !found { - return fmt.Errorf("'%s' is no valid compress algorithm. Valid algorithms / file endings are '.tar', '.tar.gz', '.zip'", - archiveCompressFlag) - } - } - - switch archiveResolutionFlag { - case "1080p", "720p", "480p", "360p": - intRes, _ := strconv.ParseFloat(strings.TrimSuffix(archiveResolutionFlag, "p"), 84) - archiveResolutionFlag = fmt.Sprintf("%.0fx%s", math.Ceil(intRes*(float64(16)/float64(9))), strings.TrimSuffix(archiveResolutionFlag, "p")) - case "240p": - // 240p would round up to 427x240 if used in the case statement above, so it has to be handled separately - archiveResolutionFlag = "428x240" - case "1920x1080", "1280x720", "640x480", "480x360", "428x240", "best", "worst": - default: - return fmt.Errorf("'%s' is not a valid resolution", archiveResolutionFlag) - } - utils.Log.Debug("Using resolution '%s'", archiveResolutionFlag) - - return nil - }, - RunE: func(cmd *cobra.Command, args []string) error { - if err := commands.LoadCrunchy(); err != nil { - return err - } - - return archive(args) - }, -} - -func init() { - Cmd.Flags().StringSliceVarP(&archiveLanguagesFlag, - "language", - "l", - []string{string(utils.SystemLocale(false)), string(crunchyroll.JP)}, - "Audio locale which should be downloaded. Can be used multiple times") - - Cmd.Flags().StringSliceVarP(&archiveSubLanguagesFlag, - "sublang", - "s", - utils.LocalesAsStrings(), - "Subtitles langs which should be downloaded. Can be used multiple times") - - cwd, _ := os.Getwd() - - Cmd.Flags().StringVarP(&archiveDirectoryFlag, - "directory", - "d", - cwd, - "The directory to store the files into") - - Cmd.Flags().StringVarP(&archiveOutputFlag, - "output", - "o", - "{title}.mkv", - "Name of the output file. If you use the following things in the name, the will get replaced:\n"+ - "\t{title} Âģ Title of the video\n"+ - "\t{series_name} Âģ Name of the series\n"+ - "\t{season_name} Âģ Name of the season\n"+ - "\t{season_number} Âģ Number of the season\n"+ - "\t{episode_number} Âģ Number of the episode\n"+ - "\t{resolution} Âģ Resolution of the video\n"+ - "\t{fps} Âģ Frame Rate of the video\n"+ - "\t{audio} Âģ Audio locale of the video\n"+ - "\t{subtitle} Âģ Subtitle locale of the video") - - Cmd.Flags().StringVar(&archiveTempDirFlag, - "temp", - os.TempDir(), - "Directory to store temporary files in") - - Cmd.Flags().StringVarP(&archiveMergeFlag, - "merge", - "m", - "auto", - "Sets the behavior of the stream merging. Valid behaviors are 'auto', 'audio', 'video'") - - Cmd.Flags().StringVarP(&archiveCompressFlag, - "compress", - "c", - "", - "If is set, all output will be compresses into an archive (every url generates a new one). "+ - "This flag sets the name of the compressed output file. The file ending specifies the compression algorithm. "+ - "The following algorithms are supported: gzip, tar, zip") - - Cmd.Flags().StringVarP(&archiveResolutionFlag, - "resolution", - "r", - "best", - "The video resolution. Can either be specified via the pixels, the abbreviation for pixels, or 'common-use' words\n"+ - "\tAvailable pixels: 1920x1080, 1280x720, 640x480, 480x360, 428x240\n"+ - "\tAvailable abbreviations: 1080p, 720p, 480p, 360p, 240p\n"+ - "\tAvailable common-use words: best (best available resolution), worst (worst available resolution)") - - Cmd.Flags().IntVarP(&archiveGoroutinesFlag, - "goroutines", - "g", - runtime.NumCPU(), - "Number of parallel segment downloads") -} - -func archive(urls []string) error { - for i, url := range urls { - utils.Log.SetProcess("Parsing url %d", i+1) - episodes, err := archiveExtractEpisodes(url) - if err != nil { - utils.Log.StopProcess("Failed to parse url %d", i+1) - if utils.Crunchy.Config.Premium { - utils.Log.Debug("If the error says no episodes could be found but the passed url is correct and a crunchyroll classic url, " + - "try the corresponding crunchyroll beta url instead and try again. See https://github.com/crunchy-labs/crunchy-cli/issues/22 for more information") - } - return err - } - utils.Log.StopProcess("Parsed url %d", i+1) - - var compressFile *os.File - var c Compress - - if archiveCompressFlag != "" { - compressFile, err = os.Create(utils.GenerateFilename(archiveCompressFlag, "")) - if err != nil { - return fmt.Errorf("failed to create archive file: %v", err) - } - if strings.HasSuffix(archiveCompressFlag, ".tar") { - c = NewTarCompress(compressFile) - } else if strings.HasSuffix(archiveCompressFlag, ".tar.gz") || strings.HasSuffix(archiveCompressFlag, ".tgz") { - c = NewGzipCompress(compressFile) - } else if strings.HasSuffix(archiveCompressFlag, ".zip") { - c = NewZipCompress(compressFile) - } - } - - for _, season := range episodes { - utils.Log.Info("%s Season %d", season[0].SeriesName, season[0].SeasonNumber) - - for j, info := range season { - utils.Log.Info("\t%d. %s Âģ %spx, %.2f FPS (S%02dE%02d)", - j+1, - info.Title, - info.Resolution, - info.FPS, - info.SeasonNumber, - info.EpisodeNumber) - } - } - utils.Log.Empty() - - for j, season := range episodes { - for k, info := range season { - var filename string - var writeCloser io.WriteCloser - if c != nil { - filename = info.FormatString(archiveOutputFlag) - writeCloser, err = c.NewFile(info) - if err != nil { - return fmt.Errorf("failed to pre generate new archive file: %v", err) - } - } else { - dir := info.FormatString(archiveDirectoryFlag) - if _, err = os.Stat(dir); os.IsNotExist(err) { - if err = os.MkdirAll(dir, 0777); err != nil { - return fmt.Errorf("error while creating directory: %v", err) - } - } - filename = utils.GenerateFilename(info.FormatString(archiveOutputFlag), dir) - writeCloser, err = os.Create(filename) - if err != nil { - return fmt.Errorf("failed to create new file: %v", err) - } - } - - if err = archiveInfo(info, writeCloser, filename); err != nil { - writeCloser.Close() - if f, ok := writeCloser.(*os.File); ok { - os.Remove(f.Name()) - } else { - c.Close() - compressFile.Close() - os.RemoveAll(compressFile.Name()) - } - return err - } - writeCloser.Close() - - if i != len(urls)-1 || j != len(episodes)-1 || k != len(season)-1 { - utils.Log.Empty() - } - } - } - if c != nil { - c.Close() - } - if compressFile != nil { - compressFile.Close() - } - } - return nil -} - -func archiveInfo(info utils.FormatInformation, writeCloser io.WriteCloser, filename string) error { - utils.Log.Debug("Entering season %d, episode %d with %d additional formats", info.SeasonNumber, info.EpisodeNumber, len(info.AdditionalFormats)) - - dp, err := createArchiveProgress(info) - if err != nil { - return fmt.Errorf("error while setting up downloader: %v", err) - } - defer func() { - if dp.Total != dp.Current { - fmt.Println() - } - }() - - rootFile, err := os.CreateTemp("", fmt.Sprintf("%s_*.ts", strings.TrimSuffix(filepath.Base(filename), filepath.Ext(filename)))) - if err != nil { - return fmt.Errorf("failed to create temp file: %v", err) - } - defer os.Remove(rootFile.Name()) - defer rootFile.Close() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - downloader := crunchyroll.NewDownloader(ctx, rootFile, archiveGoroutinesFlag, func(segment *m3u8.MediaSegment, current, total int, file *os.File) error { - // check if the context was cancelled. - // must be done in to not print any progress messages if ctrl+c was pressed - if ctx.Err() != nil { - return nil - } - - if utils.Log.IsDev() { - dp.UpdateMessage(fmt.Sprintf("Downloading %d/%d (%.2f%%) Âģ %s", current, total, float32(current)/float32(total)*100, segment.URI), false) - } else { - dp.Update() - } - - if current == total { - dp.UpdateMessage("Merging segments", false) - } - return nil - }) - tmp, _ := os.MkdirTemp(archiveTempDirFlag, "crunchy_") - downloader.TempDir = tmp - - sig := make(chan os.Signal, 1) - signal.Notify(sig, os.Interrupt) - go func() { - select { - case <-sig: - signal.Stop(sig) - utils.Log.Err("Exiting... (may take a few seconds)") - utils.Log.Err("To force exit press ctrl+c (again)") - cancel() - // os.Exit(1) is not called since an immediate exit after the cancel function does not let - // the download process enough time to stop gratefully. A result of this is that the temporary - // directory where the segments are downloaded to will not be deleted - case <-ctx.Done(): - // this is just here to end the goroutine and prevent it from running forever without a reason - } - }() - utils.Log.Debug("Set up signal catcher") - - var additionalDownloaderOpts []string - var mergeMessage string - switch archiveMergeFlag { - case "auto": - additionalDownloaderOpts = []string{"-vn"} - for _, format := range info.AdditionalFormats { - if format.Video.Bandwidth != info.Format.Video.Bandwidth { - // revoke the changed FFmpegOpts above - additionalDownloaderOpts = []string{} - break - } - } - if len(additionalDownloaderOpts) > 0 { - mergeMessage = "merging audio for additional formats" - } else { - mergeMessage = "merging video for additional formats" - } - case "audio": - additionalDownloaderOpts = []string{"-vn"} - mergeMessage = "merging audio for additional formats" - case "video": - mergeMessage = "merging video for additional formats" - } - - utils.Log.Info("Downloading episode `%s` to `%s` (%s)", info.Title, filepath.Base(filename), mergeMessage) - utils.Log.Info("\tEpisode: S%02dE%02d", info.SeasonNumber, info.EpisodeNumber) - utils.Log.Info("\tAudio: %s", info.Audio) - utils.Log.Info("\tSubtitle: %s", info.Subtitle) - utils.Log.Info("\tResolution: %spx", info.Resolution) - utils.Log.Info("\tFPS: %.2f", info.FPS) - - var videoFiles, audioFiles, subtitleFiles []string - defer func() { - for _, f := range append(append(videoFiles, audioFiles...), subtitleFiles...) { - os.RemoveAll(f) - } - }() - - var f []string - if f, err = archiveDownloadVideos(downloader, filepath.Base(filename), true, info.Format); err != nil { - if err != ctx.Err() { - return fmt.Errorf("error while downloading: %v", err) - } - return err - } - videoFiles = append(videoFiles, f[0]) - - if len(additionalDownloaderOpts) == 0 { - var videos []string - downloader.FFmpegOpts = additionalDownloaderOpts - if videos, err = archiveDownloadVideos(downloader, filepath.Base(filename), true, info.AdditionalFormats...); err != nil { - return fmt.Errorf("error while downloading additional videos: %v", err) - } - downloader.FFmpegOpts = []string{} - videoFiles = append(videoFiles, videos...) - } else { - var audios []string - if audios, err = archiveDownloadVideos(downloader, filepath.Base(filename), false, info.AdditionalFormats...); err != nil { - return fmt.Errorf("error while downloading additional videos: %v", err) - } - audioFiles = append(audioFiles, audios...) - } - - sort.Sort(crunchyUtils.SubtitlesByLocale(info.Format.Subtitles)) - - sortSubtitles, _ := strconv.ParseBool(os.Getenv("SORT_SUBTITLES")) - if sortSubtitles && len(archiveLanguagesFlag) > 0 { - // this sort the subtitle locales after the languages which were specified - // with the `archiveLanguagesFlag` flag - for _, language := range archiveLanguagesFlag { - for i, subtitle := range info.Format.Subtitles { - if subtitle.Locale == crunchyroll.LOCALE(language) { - info.Format.Subtitles = append([]*crunchyroll.Subtitle{subtitle}, append(info.Format.Subtitles[:i], info.Format.Subtitles[i+1:]...)...) - break - } - } - } - } - - var subtitles []string - if subtitles, err = archiveDownloadSubtitles(filepath.Base(filename), info.Format.Subtitles...); err != nil { - return fmt.Errorf("error while downloading subtitles: %v", err) - } - subtitleFiles = append(subtitleFiles, subtitles...) - - if err = archiveFFmpeg(ctx, writeCloser, videoFiles, audioFiles, subtitleFiles); err != nil { - return fmt.Errorf("failed to merge files: %v", err) - } - - dp.UpdateMessage("Download finished", false) - - signal.Stop(sig) - utils.Log.Debug("Stopped signal catcher") - - utils.Log.Empty() - - return nil -} - -func createArchiveProgress(info utils.FormatInformation) (*commands.DownloadProgress, error) { - var progressCount int - if err := info.Format.InitVideo(); err != nil { - return nil, fmt.Errorf("error while initializing a video: %v", err) - } - // + number of segments a video has +1 is for merging - progressCount += int(info.Format.Video.Chunklist.Count()) + 1 - for _, f := range info.AdditionalFormats { - if f == info.Format { - continue - } - - if err := f.InitVideo(); err != nil { - return nil, err - } - // + number of segments a video has +1 is for merging - progressCount += int(f.Video.Chunklist.Count()) + 1 - } - - dp := &commands.DownloadProgress{ - Prefix: utils.Log.(*commands.Logger).InfoLog.Prefix(), - Message: "Downloading video", - // number of segments a video +1 is for the success message - Total: progressCount + 1, - Dev: utils.Log.IsDev(), - Quiet: utils.Log.(*commands.Logger).IsQuiet(), - } - if utils.Log.IsDev() { - dp.Prefix = utils.Log.(*commands.Logger).DebugLog.Prefix() - } - - return dp, nil -} - -func archiveDownloadVideos(downloader crunchyroll.Downloader, filename string, video bool, formats ...*crunchyroll.Format) ([]string, error) { - var files []string - - for _, format := range formats { - var name string - if video { - name = fmt.Sprintf("%s_%s_video_*.ts", filename, format.AudioLocale) - } else { - name = fmt.Sprintf("%s_%s_audio_*.aac", filename, format.AudioLocale) - } - - f, err := os.CreateTemp("", name) - if err != nil { - return nil, err - } - files = append(files, f.Name()) - - downloader.Writer = f - if err = format.Download(downloader); err != nil { - f.Close() - for _, file := range files { - os.Remove(file) - } - return nil, err - } - f.Close() - - utils.Log.Debug("Downloaded '%s' video", format.AudioLocale) - } - - return files, nil -} - -func archiveDownloadSubtitles(filename string, subtitles ...*crunchyroll.Subtitle) ([]string, error) { - var files []string - - for _, subtitle := range subtitles { - if !utils.ElementInSlice(string(subtitle.Locale), archiveSubLanguagesFlag) { - continue - } - - f, err := os.CreateTemp("", fmt.Sprintf("%s_%s_subtitle_*.ass", filename, subtitle.Locale)) - if err != nil { - return nil, err - } - files = append(files, f.Name()) - - if err := subtitle.Save(f); err != nil { - f.Close() - for _, file := range files { - os.Remove(file) - } - return nil, err - } - f.Close() - - utils.Log.Debug("Downloaded '%s' subtitles", subtitle.Locale) - } - - return files, nil -} - -func archiveFFmpeg(ctx context.Context, dst io.Writer, videoFiles, audioFiles, subtitleFiles []string) error { - var input, maps, metadata []string - re := regexp.MustCompile(`(?m)_([a-z]{2}-([A-Z]{2}|[0-9]{3}))_(video|audio|subtitle)`) - // https://github.com/crunchy-labs/crunchy-cli/issues/32 - videoLength32Fix := regexp.MustCompile(`Duration:\s?(\d+):(\d+):(\d+).(\d+),`) - - videoLength := [4]int{0, 0, 0, 0} - - for i, video := range videoFiles { - input = append(input, "-i", video) - maps = append(maps, "-map", strconv.Itoa(i)) - locale := crunchyroll.LOCALE(re.FindStringSubmatch(video)[1]) - metadata = append(metadata, fmt.Sprintf("-metadata:s:v:%d", i), fmt.Sprintf("language=%s", locale)) - metadata = append(metadata, fmt.Sprintf("-metadata:s:v:%d", i), fmt.Sprintf("title=%s", crunchyUtils.LocaleLanguage(locale))) - metadata = append(metadata, fmt.Sprintf("-metadata:s:a:%d", i), fmt.Sprintf("language=%s", locale)) - metadata = append(metadata, fmt.Sprintf("-metadata:s:a:%d", i), fmt.Sprintf("title=%s", crunchyUtils.LocaleLanguage(locale))) - - var errBuf bytes.Buffer - cmd := exec.CommandContext(ctx, "ffmpeg", "-i", video) - cmd.Stderr = &errBuf - cmd.Run() - - matches := videoLength32Fix.FindStringSubmatch(errBuf.String()) - hours, _ := strconv.Atoi(matches[1]) - minutes, _ := strconv.Atoi(matches[2]) - seconds, _ := strconv.Atoi(matches[3]) - millis, _ := strconv.Atoi(matches[4]) - - if hours > videoLength[0] { - videoLength = [4]int{hours, minutes, seconds, millis} - } else if hours == videoLength[0] && minutes > videoLength[1] { - videoLength = [4]int{hours, minutes, seconds, millis} - } else if hours == videoLength[0] && minutes == videoLength[1] && seconds > videoLength[2] { - videoLength = [4]int{hours, minutes, seconds, millis} - } else if hours == videoLength[0] && minutes == videoLength[1] && seconds == videoLength[2] && millis > videoLength[3] { - videoLength = [4]int{hours, minutes, seconds, millis} - } - } - - for i, audio := range audioFiles { - input = append(input, "-i", audio) - maps = append(maps, "-map", strconv.Itoa(i+len(videoFiles))+":1") - locale := crunchyroll.LOCALE(re.FindStringSubmatch(audio)[1]) - metadata = append(metadata, fmt.Sprintf("-metadata:s:a:%d", i+len(videoFiles)), fmt.Sprintf("language=%s", locale)) - metadata = append(metadata, fmt.Sprintf("-metadata:s:a:%d", i+len(videoFiles)), fmt.Sprintf("title=%s", crunchyUtils.LocaleLanguage(locale))) - } - - for i, subtitle := range subtitleFiles { - input = append(input, "-i", subtitle) - maps = append(maps, "-map", strconv.Itoa(i+len(videoFiles)+len(audioFiles))) - locale := crunchyroll.LOCALE(re.FindStringSubmatch(subtitle)[1]) - metadata = append(metadata, fmt.Sprintf("-metadata:s:s:%d", i), fmt.Sprintf("language=%s", locale)) - metadata = append(metadata, fmt.Sprintf("-metadata:s:s:%d", i), fmt.Sprintf("title=%s", crunchyUtils.LocaleLanguage(locale))) - } - - commandOptions := []string{"-y"} - commandOptions = append(commandOptions, input...) - commandOptions = append(commandOptions, maps...) - commandOptions = append(commandOptions, metadata...) - // we have to create a temporary file here because it must be seekable - // for ffmpeg. - // ffmpeg could write to dst too, but this would require to re-encode - // the audio which results in much higher time and resource consumption - // (0-1 second with the temp file, ~20 seconds with re-encoding on my system) - file, err := os.CreateTemp("", "") - if err != nil { - return err - } - file.Close() - defer os.Remove(file.Name()) - - commandOptions = append(commandOptions, "-disposition:s:0", "0", "-c", "copy", "-f", "matroska", file.Name()) - - // just a little nicer debug output to copy and paste the ffmpeg for debug reasons - if utils.Log.IsDev() { - var debugOptions []string - - for _, option := range commandOptions { - if strings.HasPrefix(option, "title=") { - debugOptions = append(debugOptions, "title=\""+strings.TrimPrefix(option, "title=")+"\"") - } else if strings.HasPrefix(option, "language=") { - debugOptions = append(debugOptions, "language=\""+strings.TrimPrefix(option, "language=")+"\"") - } else if strings.Contains(option, " ") { - debugOptions = append(debugOptions, "\""+option+"\"") - } else { - debugOptions = append(debugOptions, option) - } - } - utils.Log.Debug("FFmpeg merge command: ffmpeg %s", strings.Join(debugOptions, " ")) - } - - var errBuf bytes.Buffer - cmd := exec.CommandContext(ctx, "ffmpeg", commandOptions...) - cmd.Stderr = &errBuf - if err = cmd.Run(); err != nil { - return fmt.Errorf(errBuf.String()) - } - - file, err = os.Open(file.Name()) - if err != nil { - return err - } - defer file.Close() - - errBuf.Reset() - cmd = exec.CommandContext(ctx, "ffmpeg", "-i", file.Name()) - cmd.Stderr = &errBuf - cmd.Run() - - matches := videoLength32Fix.FindStringSubmatch(errBuf.String()) - hours, _ := strconv.Atoi(matches[1]) - minutes, _ := strconv.Atoi(matches[2]) - seconds, _ := strconv.Atoi(matches[3]) - millis, _ := strconv.Atoi(matches[4]) - - var reencode bool - if hours > videoLength[0] { - reencode = true - } else if hours == videoLength[0] && minutes > videoLength[1] { - reencode = true - } else if hours == videoLength[0] && minutes == videoLength[1] && seconds > videoLength[2] { - reencode = true - } else if hours == videoLength[0] && minutes == videoLength[1] && seconds == videoLength[2] && millis > videoLength[3] { - reencode = true - } - - // very dirty solution to https://github.com/crunchy-labs/crunchy-cli/issues/32. - // this might get triggered when not needed but there is currently no easy way to - // bypass this unwanted triggering - if reencode { - utils.Log.Debug("Re-encode to short video length") - - file.Close() - - tmpFile, _ := os.CreateTemp("", filepath.Base(file.Name())+"-32_fix") - tmpFile.Close() - - errBuf.Reset() - cmd = exec.CommandContext(ctx, "ffmpeg", - "-y", - "-i", file.Name(), - "-map", "0", - "-c", "copy", - "-disposition:s:0", "0", - "-t", fmt.Sprintf("%02d:%02d:%02d.%d", videoLength[0], videoLength[1], videoLength[2], videoLength[3]), - "-f", "matroska", - tmpFile.Name()) - cmd.Stderr = &errBuf - if err = cmd.Run(); err != nil { - return fmt.Errorf(errBuf.String()) - } - - os.Remove(file.Name()) - os.Rename(tmpFile.Name(), file.Name()) - - file, err = os.Open(file.Name()) - if err != nil { - return err - } - defer file.Close() - } - - _, err = bufio.NewWriter(dst).ReadFrom(file) - return err -} - -func archiveExtractEpisodes(url string) ([][]utils.FormatInformation, error) { - var hasJapanese bool - languagesAsLocale := []crunchyroll.LOCALE{crunchyroll.JP} - for _, language := range archiveLanguagesFlag { - locale := crunchyroll.LOCALE(language) - if locale == crunchyroll.JP { - hasJapanese = true - } else { - languagesAsLocale = append(languagesAsLocale, locale) - } - } - - if _, ok := crunchyroll.ParseBetaEpisodeURL(url); ok { - return nil, fmt.Errorf("archiving episodes by url is no longer supported (thx crunchyroll). use the series url instead and filter after the given episode (https://github.com/crunchy-labs/crunchy-cli/wiki/Cli#filter)") - } else if _, _, _, _, ok := crunchyroll.ParseEpisodeURL(url); ok { - return nil, fmt.Errorf("archiving episodes by url is no longer supported (thx crunchyroll). use the series url instead and filter after the given episode (https://github.com/crunchy-labs/crunchy-cli/wiki/Cli#filter)") - } - - episodes, err := utils.ExtractEpisodes(url, languagesAsLocale...) - if err != nil { - return nil, err - } - - if !hasJapanese && len(episodes[1:]) == 0 { - return nil, fmt.Errorf("no episodes found") - } - - for i, eps := range episodes { - if len(eps) == 0 { - utils.Log.SetProcess("%s has no matching episodes", languagesAsLocale[i]) - } else if len(episodes[0]) > len(eps) { - utils.Log.SetProcess("%s has %d less episodes than existing in japanese (%d)", languagesAsLocale[i], len(episodes[0])-len(eps), len(episodes[0])) - } - } - - if !hasJapanese { - episodes = episodes[1:] - } - - eps := make(map[int]map[int]*utils.FormatInformation) - for _, lang := range episodes { - for _, season := range crunchyUtils.SortEpisodesBySeason(lang) { - if _, ok := eps[season[0].SeasonNumber]; !ok { - eps[season[0].SeasonNumber] = map[int]*utils.FormatInformation{} - } - for _, episode := range season { - format, err := episode.GetFormat(archiveResolutionFlag, "", false) - if err != nil { - return nil, fmt.Errorf("error while receiving format for %s: %v", episode.Title, err) - } - - if _, ok := eps[episode.SeasonNumber][episode.EpisodeNumber]; !ok { - eps[episode.SeasonNumber][episode.EpisodeNumber] = &utils.FormatInformation{ - Format: format, - AdditionalFormats: make([]*crunchyroll.Format, 0), - - Title: episode.Title, - SeriesName: episode.SeriesTitle, - SeasonName: episode.SeasonTitle, - SeasonNumber: episode.SeasonNumber, - EpisodeNumber: episode.EpisodeNumber, - Resolution: format.Video.Resolution, - FPS: format.Video.FrameRate, - Audio: format.AudioLocale, - } - } else { - eps[episode.SeasonNumber][episode.EpisodeNumber].AdditionalFormats = append(eps[episode.SeasonNumber][episode.EpisodeNumber].AdditionalFormats, format) - } - } - } - } - - var infoFormat [][]utils.FormatInformation - for _, e := range eps { - var tmpFormatInfo []utils.FormatInformation - - var keys []int - for episodeNumber := range e { - keys = append(keys, episodeNumber) - } - sort.Ints(keys) - - for _, key := range keys { - tmpFormatInfo = append(tmpFormatInfo, *e[key]) - } - - infoFormat = append(infoFormat, tmpFormatInfo) - } - - return infoFormat, nil -} diff --git a/cli/commands/archive/compress.go b/cli/commands/archive/compress.go deleted file mode 100644 index e0b9ad4..0000000 --- a/cli/commands/archive/compress.go +++ /dev/null @@ -1,136 +0,0 @@ -package archive - -import ( - "archive/tar" - "archive/zip" - "bytes" - "compress/gzip" - "fmt" - "github.com/crunchy-labs/crunchy-cli/utils" - "io" - "os" - "path/filepath" - "sync" - "time" -) - -type Compress interface { - io.Closer - - NewFile(information utils.FormatInformation) (io.WriteCloser, error) -} - -func NewGzipCompress(file *os.File) *TarCompress { - gw := gzip.NewWriter(file) - return &TarCompress{ - parent: gw, - dst: tar.NewWriter(gw), - } -} - -func NewTarCompress(file *os.File) *TarCompress { - return &TarCompress{ - dst: tar.NewWriter(file), - } -} - -type TarCompress struct { - Compress - - wg sync.WaitGroup - - parent *gzip.Writer - dst *tar.Writer -} - -func (tc *TarCompress) Close() error { - // we have to wait here in case the actual content isn't copied completely into the - // writer yet - tc.wg.Wait() - - var err, err2 error - if tc.parent != nil { - err2 = tc.parent.Close() - } - err = tc.dst.Close() - - if err != nil && err2 != nil { - // best way to show double errors at once that I've found - return fmt.Errorf("%v\n%v", err, err2) - } else if err == nil && err2 != nil { - err = err2 - } - - return err -} - -func (tc *TarCompress) NewFile(information utils.FormatInformation) (io.WriteCloser, error) { - rp, wp := io.Pipe() - go func() { - tc.wg.Add(1) - defer tc.wg.Done() - var buf bytes.Buffer - io.Copy(&buf, rp) - - header := &tar.Header{ - Name: filepath.Join(fmt.Sprintf("S%2d", information.SeasonNumber), information.Title), - ModTime: time.Now(), - Mode: 0644, - Typeflag: tar.TypeReg, - // fun fact: I did not set the size for quiet some time because I thought that it isn't - // required. well because of this I debugged this part for multiple hours because without - // proper size information only a tiny amount gets copied into the tar (or zip) writer. - // this is also the reason why the file content is completely copied into a buffer before - // writing it to the writer. I could bypass this and save some memory but this requires - // some rewriting and im nearly at the (planned) finish for version 2 so nah in the future - // maybe - Size: int64(buf.Len()), - } - tc.dst.WriteHeader(header) - io.Copy(tc.dst, &buf) - }() - return wp, nil -} - -func NewZipCompress(file *os.File) *ZipCompress { - return &ZipCompress{ - dst: zip.NewWriter(file), - } -} - -type ZipCompress struct { - Compress - - wg sync.WaitGroup - - dst *zip.Writer -} - -func (zc *ZipCompress) Close() error { - zc.wg.Wait() - return zc.dst.Close() -} - -func (zc *ZipCompress) NewFile(information utils.FormatInformation) (io.WriteCloser, error) { - rp, wp := io.Pipe() - go func() { - zc.wg.Add(1) - defer zc.wg.Done() - - var buf bytes.Buffer - io.Copy(&buf, rp) - - header := &zip.FileHeader{ - Name: filepath.Join(fmt.Sprintf("S%2d", information.SeasonNumber), information.Title), - Modified: time.Now(), - Method: zip.Deflate, - UncompressedSize64: uint64(buf.Len()), - } - header.SetMode(0644) - - hw, _ := zc.dst.CreateHeader(header) - io.Copy(hw, &buf) - }() - - return wp, nil -} diff --git a/cli/commands/download/download.go b/cli/commands/download/download.go deleted file mode 100644 index 39273d0..0000000 --- a/cli/commands/download/download.go +++ /dev/null @@ -1,368 +0,0 @@ -package download - -import ( - "context" - "fmt" - "github.com/crunchy-labs/crunchy-cli/cli/commands" - "github.com/crunchy-labs/crunchy-cli/utils" - "github.com/crunchy-labs/crunchyroll-go/v3" - crunchyUtils "github.com/crunchy-labs/crunchyroll-go/v3/utils" - "github.com/grafov/m3u8" - "github.com/spf13/cobra" - "math" - "os" - "os/signal" - "path/filepath" - "runtime" - "sort" - "strconv" - "strings" -) - -var ( - downloadAudioFlag string - downloadSubtitleFlag string - - downloadDirectoryFlag string - downloadOutputFlag string - downloadTempDirFlag string - - downloadResolutionFlag string - - downloadGoroutinesFlag int -) - -var Cmd = &cobra.Command{ - Use: "download", - Short: "Download a video", - Args: cobra.MinimumNArgs(1), - - PreRunE: func(cmd *cobra.Command, args []string) error { - utils.Log.Debug("Validating arguments") - - if filepath.Ext(downloadOutputFlag) != ".ts" { - if !utils.HasFFmpeg() { - return fmt.Errorf("the file ending for the output file (%s) is not `.ts`. "+ - "Install ffmpeg (https://ffmpeg.org/download.html) to use other media file endings (e.g. `.mp4`)", downloadOutputFlag) - } else { - utils.Log.Debug("Custom file ending '%s' (ffmpeg is installed)", filepath.Ext(downloadOutputFlag)) - } - } - - if downloadAudioFlag != "" && !crunchyUtils.ValidateLocale(crunchyroll.LOCALE(downloadAudioFlag)) { - return fmt.Errorf("%s is not a valid audio locale. Choose from: %s", downloadAudioFlag, strings.Join(utils.LocalesAsStrings(), ", ")) - } else if downloadSubtitleFlag != "" && !crunchyUtils.ValidateLocale(crunchyroll.LOCALE(downloadSubtitleFlag)) { - return fmt.Errorf("%s is not a valid subtitle locale. Choose from: %s", downloadSubtitleFlag, strings.Join(utils.LocalesAsStrings(), ", ")) - } - utils.Log.Debug("Locales: audio: %s / subtitle: %s", downloadAudioFlag, downloadSubtitleFlag) - - switch downloadResolutionFlag { - case "1080p", "720p", "480p", "360p": - intRes, _ := strconv.ParseFloat(strings.TrimSuffix(downloadResolutionFlag, "p"), 84) - downloadResolutionFlag = fmt.Sprintf("%.0fx%s", math.Ceil(intRes*(float64(16)/float64(9))), strings.TrimSuffix(downloadResolutionFlag, "p")) - case "240p": - // 240p would round up to 427x240 if used in the case statement above, so it has to be handled separately - downloadResolutionFlag = "428x240" - case "1920x1080", "1280x720", "640x480", "480x360", "428x240", "best", "worst": - default: - return fmt.Errorf("'%s' is not a valid resolution", downloadResolutionFlag) - } - utils.Log.Debug("Using resolution '%s'", downloadResolutionFlag) - - return nil - }, - RunE: func(cmd *cobra.Command, args []string) error { - if err := commands.LoadCrunchy(); err != nil { - return err - } - - return download(args) - }, -} - -func init() { - Cmd.Flags().StringVarP(&downloadAudioFlag, "audio", - "a", - "", - "The locale of the audio. Available locales: "+strings.Join(utils.LocalesAsStrings(), ", ")) - Cmd.Flags().StringVarP(&downloadSubtitleFlag, - "subtitle", - "s", - "", - "The locale of the subtitle. Available locales: "+strings.Join(utils.LocalesAsStrings(), ", ")) - - cwd, _ := os.Getwd() - Cmd.Flags().StringVarP(&downloadDirectoryFlag, - "directory", - "d", - cwd, - "The directory to download the file(s) into") - Cmd.Flags().StringVarP(&downloadOutputFlag, - "output", - "o", - "{title}.ts", - "Name of the output file. "+ - "If you use the following things in the name, the will get replaced:\n"+ - "\t{title} Âģ Title of the video\n"+ - "\t{series_name} Âģ Name of the series\n"+ - "\t{season_name} Âģ Name of the season\n"+ - "\t{season_number} Âģ Number of the season\n"+ - "\t{episode_number} Âģ Number of the episode\n"+ - "\t{resolution} Âģ Resolution of the video\n"+ - "\t{fps} Âģ Frame Rate of the video\n"+ - "\t{audio} Âģ Audio locale of the video\n"+ - "\t{subtitle} Âģ Subtitle locale of the video") - Cmd.Flags().StringVar(&downloadTempDirFlag, - "temp", - os.TempDir(), - "Directory to store temporary files in") - - Cmd.Flags().StringVarP(&downloadResolutionFlag, - "resolution", - "r", - "best", - "The video resolution. Can either be specified via the pixels, the abbreviation for pixels, or 'common-use' words\n"+ - "\tAvailable pixels: 1920x1080, 1280x720, 640x480, 480x360, 428x240\n"+ - "\tAvailable abbreviations: 1080p, 720p, 480p, 360p, 240p\n"+ - "\tAvailable common-use words: best (best available resolution), worst (worst available resolution)") - - Cmd.Flags().IntVarP(&downloadGoroutinesFlag, - "goroutines", - "g", - runtime.NumCPU(), - "Sets how many parallel segment downloads should be used") -} - -func download(urls []string) error { - for i, url := range urls { - utils.Log.SetProcess("Parsing url %d", i+1) - episodes, err := downloadExtractEpisodes(url) - if err != nil { - utils.Log.StopProcess("Failed to parse url %d", i+1) - if utils.Crunchy.Config.Premium { - utils.Log.Debug("If the error says no episodes could be found but the passed url is correct and a crunchyroll classic url, " + - "try the corresponding crunchyroll beta url instead and try again. See https://github.com/crunchy-labs/crunchy-cli/issues/22 for more information") - } - return err - } - utils.Log.StopProcess("Parsed url %d", i+1) - - for _, season := range episodes { - utils.Log.Info("%s Season %d", season[0].SeriesName, season[0].SeasonNumber) - - for j, info := range season { - utils.Log.Info("\t%d. %s Âģ %spx, %.2f FPS (S%02dE%02d)", - j+1, - info.Title, - info.Resolution, - info.FPS, - info.SeasonNumber, - info.EpisodeNumber) - } - } - utils.Log.Empty() - - for j, season := range episodes { - for k, info := range season { - dir := info.FormatString(downloadDirectoryFlag) - if _, err = os.Stat(dir); os.IsNotExist(err) { - if err = os.MkdirAll(dir, 0777); err != nil { - return fmt.Errorf("error while creating directory: %v", err) - } - } - file, err := os.Create(utils.GenerateFilename(info.FormatString(downloadOutputFlag), dir)) - if err != nil { - return fmt.Errorf("failed to create output file: %v", err) - } - - if err = downloadInfo(info, file); err != nil { - file.Close() - os.Remove(file.Name()) - return err - } - file.Close() - - if i != len(urls)-1 || j != len(episodes)-1 || k != len(season)-1 { - utils.Log.Empty() - } - } - } - } - return nil -} - -func downloadInfo(info utils.FormatInformation, file *os.File) error { - utils.Log.Debug("Entering season %d, episode %d", info.SeasonNumber, info.EpisodeNumber) - - if err := info.Format.InitVideo(); err != nil { - return fmt.Errorf("error while initializing the video: %v", err) - } - - dp := &commands.DownloadProgress{ - Prefix: utils.Log.(*commands.Logger).InfoLog.Prefix(), - Message: "Downloading video", - // number of segments a video has +2 is for merging and the success message - Total: int(info.Format.Video.Chunklist.Count()) + 2, - Dev: utils.Log.IsDev(), - Quiet: utils.Log.(*commands.Logger).IsQuiet(), - } - if utils.Log.IsDev() { - dp.Prefix = utils.Log.(*commands.Logger).DebugLog.Prefix() - } - defer func() { - if dp.Total != dp.Current { - fmt.Println() - } - }() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - downloader := crunchyroll.NewDownloader(ctx, file, downloadGoroutinesFlag, func(segment *m3u8.MediaSegment, current, total int, file *os.File) error { - // check if the context was cancelled. - // must be done in to not print any progress messages if ctrl+c was pressed - if ctx.Err() != nil { - return nil - } - - if utils.Log.IsDev() { - dp.UpdateMessage(fmt.Sprintf("Downloading %d/%d (%.2f%%) Âģ %s", current, total, float32(current)/float32(total)*100, segment.URI), false) - } else { - dp.Update() - } - - if current == total { - dp.UpdateMessage("Merging segments", false) - } - return nil - }) - tmp, _ := os.MkdirTemp(downloadTempDirFlag, "crunchy_") - downloader.TempDir = tmp - if utils.HasFFmpeg() { - downloader.FFmpegOpts = make([]string, 0) - } - - sig := make(chan os.Signal, 1) - signal.Notify(sig, os.Interrupt) - go func() { - select { - case <-sig: - signal.Stop(sig) - utils.Log.Err("Exiting... (may take a few seconds)") - utils.Log.Err("To force exit press ctrl+c (again)") - cancel() - // os.Exit(1) is not called because an immediate exit after the cancel function does not let - // the download process enough time to stop gratefully. A result of this is that the temporary - // directory where the segments are downloaded to will not be deleted - case <-ctx.Done(): - // this is just here to end the goroutine and prevent it from running forever without a reason - } - }() - utils.Log.Debug("Set up signal catcher") - - utils.Log.Info("Downloading episode `%s` to `%s`", info.Title, filepath.Base(file.Name())) - utils.Log.Info("\tEpisode: S%02dE%02d", info.SeasonNumber, info.EpisodeNumber) - utils.Log.Info("\tAudio: %s", info.Audio) - utils.Log.Info("\tSubtitle: %s", info.Subtitle) - utils.Log.Info("\tResolution: %spx", info.Resolution) - utils.Log.Info("\tFPS: %.2f", info.FPS) - if err := info.Format.Download(downloader); err != nil { - return fmt.Errorf("error while downloading: %v", err) - } - - dp.UpdateMessage("Download finished", false) - - signal.Stop(sig) - utils.Log.Debug("Stopped signal catcher") - - utils.Log.Empty() - - return nil -} - -func downloadExtractEpisodes(url string) ([][]utils.FormatInformation, error) { - var episodes [][]*crunchyroll.Episode - var final []*crunchyroll.Episode - - if downloadAudioFlag != "" { - if _, ok := crunchyroll.ParseBetaEpisodeURL(url); ok { - return nil, fmt.Errorf("downloading episodes by url and specifying a language is no longer supported (thx crunchyroll). use the series url instead and filter after the given episode (https://github.com/crunchy-labs/crunchy-cli/wiki/Cli#filter)") - } else if _, _, _, _, ok := crunchyroll.ParseEpisodeURL(url); ok { - return nil, fmt.Errorf("downloading episodes by url and specifying a language is no longer supported (thx crunchyroll). use the series url instead and filter after the given episode (https://github.com/crunchy-labs/crunchy-cli/wiki/Cli#filter)") - } - - var err error - episodes, err = utils.ExtractEpisodes(url, crunchyroll.JP, crunchyroll.LOCALE(downloadAudioFlag)) - if err != nil { - return nil, err - } - japanese := episodes[0] - custom := episodes[1] - - sort.Sort(crunchyUtils.EpisodesByNumber(japanese)) - sort.Sort(crunchyUtils.EpisodesByNumber(custom)) - - var errMessages []string - - if len(japanese) == 0 || len(japanese) == len(custom) { - final = custom - } else { - for _, jp := range japanese { - before := len(final) - for _, episode := range custom { - if jp.SeasonNumber == episode.SeasonNumber && jp.EpisodeNumber == episode.EpisodeNumber { - final = append(final, episode) - } - } - if before == len(final) { - errMessages = append(errMessages, fmt.Sprintf("%s has no %s audio, using %s as fallback", jp.Title, crunchyroll.LOCALE(downloadAudioFlag), crunchyroll.JP)) - final = append(final, jp) - } - } - } - - if len(errMessages) > 10 { - for _, msg := range errMessages[:10] { - utils.Log.SetProcess(msg) - } - utils.Log.SetProcess("... and %d more", len(errMessages)-10) - } else { - for _, msg := range errMessages { - utils.Log.SetProcess(msg) - } - } - } else { - var err error - episodes, err = utils.ExtractEpisodes(url) - if err != nil { - return nil, err - } else if len(episodes) == 0 { - return nil, fmt.Errorf("cannot find any episode") - } - final = episodes[0] - } - - var infoFormat [][]utils.FormatInformation - for _, season := range crunchyUtils.SortEpisodesBySeason(final) { - tmpFormatInformation := make([]utils.FormatInformation, 0) - for _, episode := range season { - format, err := episode.GetFormat(downloadResolutionFlag, crunchyroll.LOCALE(downloadSubtitleFlag), true) - if err != nil { - return nil, fmt.Errorf("error while receiving format for %s: %v", episode.Title, err) - } - tmpFormatInformation = append(tmpFormatInformation, utils.FormatInformation{ - Format: format, - - Title: episode.Title, - SeriesName: episode.SeriesTitle, - SeasonName: episode.SeasonTitle, - SeasonNumber: episode.SeasonNumber, - EpisodeNumber: episode.EpisodeNumber, - Resolution: format.Video.Resolution, - FPS: format.Video.FrameRate, - Audio: format.AudioLocale, - }) - } - infoFormat = append(infoFormat, tmpFormatInformation) - } - return infoFormat, nil -} diff --git a/cli/commands/info/info.go b/cli/commands/info/info.go deleted file mode 100644 index 650d8c7..0000000 --- a/cli/commands/info/info.go +++ /dev/null @@ -1,40 +0,0 @@ -package info - -import ( - "fmt" - "github.com/crunchy-labs/crunchy-cli/cli/commands" - "github.com/crunchy-labs/crunchy-cli/utils" - crunchyUtils "github.com/crunchy-labs/crunchyroll-go/v3/utils" - "github.com/spf13/cobra" -) - -var Cmd = &cobra.Command{ - Use: "info", - Short: "Shows information about the logged in user", - Args: cobra.MinimumNArgs(0), - - RunE: func(cmd *cobra.Command, args []string) error { - if err := commands.LoadCrunchy(); err != nil { - return err - } - - return info() - }, -} - -func info() error { - account, err := utils.Crunchy.Account() - if err != nil { - return err - } - - fmt.Println("Username: ", account.Username) - fmt.Println("Email: ", account.Email) - fmt.Println("Premium: ", utils.Crunchy.Config.Premium) - fmt.Println("Interface language: ", crunchyUtils.LocaleLanguage(account.PreferredCommunicationLanguage)) - fmt.Println("Subtitle language: ", crunchyUtils.LocaleLanguage(account.PreferredContentSubtitleLanguage)) - fmt.Println("Created: ", account.Created) - fmt.Println("Account ID: ", account.AccountID) - - return nil -} diff --git a/cli/commands/logger.go b/cli/commands/logger.go deleted file mode 100644 index 89a4717..0000000 --- a/cli/commands/logger.go +++ /dev/null @@ -1,196 +0,0 @@ -package commands - -import ( - "fmt" - "github.com/crunchy-labs/crunchy-cli/utils" - "io" - "log" - "os" - "runtime" - "strings" - "sync" - "time" -) - -var prefix, progressDown, progressDownFinish string - -func initPrefixBecauseWindowsSucksBallsHard() { - // dear windows user, please change to a good OS, linux in the best case. - // MICROSHIT DOES NOT GET IT DONE TO SHOW THE SYMBOLS IN THE ELSE CLAUSE - // CORRECTLY. NOT IN THE CMD NOR POWERSHELL. WHY TF, IT IS ONE OF THE MOST - // PROFITABLE COMPANIES ON THIS PLANET AND CANNOT SHOW A PROPER UTF-8 SYMBOL - // IN THEIR OWN PRODUCT WHICH GETS USED MILLION TIMES A DAY - if runtime.GOOS == "windows" { - prefix = "=>" - progressDown = "|" - progressDownFinish = "->" - } else { - prefix = "➞" - progressDown = "↓" - progressDownFinish = "â†ŗ" - } -} - -type progress struct { - message string - stop bool -} - -func NewLogger(debug, info, err bool) *Logger { - initPrefixBecauseWindowsSucksBallsHard() - - debugLog, infoLog, errLog := log.New(io.Discard, prefix+" ", 0), log.New(io.Discard, prefix+" ", 0), log.New(io.Discard, prefix+" ", 0) - - if debug { - debugLog.SetOutput(os.Stdout) - } - if info { - infoLog.SetOutput(os.Stdout) - } - if err { - errLog.SetOutput(os.Stderr) - } - - if debug { - debugLog = log.New(debugLog.Writer(), "[debug] ", 0) - infoLog = log.New(infoLog.Writer(), "[info] ", 0) - errLog = log.New(errLog.Writer(), "[err] ", 0) - } - - return &Logger{ - DebugLog: debugLog, - InfoLog: infoLog, - ErrLog: errLog, - - devView: debug, - } -} - -type Logger struct { - utils.Logger - - DebugLog *log.Logger - InfoLog *log.Logger - ErrLog *log.Logger - - devView bool - - progress chan progress - done chan interface{} - lock sync.Mutex -} - -func (l *Logger) IsDev() bool { - return l.devView -} - -func (l *Logger) IsQuiet() bool { - return l.DebugLog.Writer() == io.Discard && l.InfoLog.Writer() == io.Discard && l.ErrLog.Writer() == io.Discard -} - -func (l *Logger) Debug(format string, v ...interface{}) { - l.DebugLog.Printf(format, v...) -} - -func (l *Logger) Info(format string, v ...interface{}) { - l.InfoLog.Printf(format, v...) -} - -func (l *Logger) Warn(format string, v ...interface{}) { - l.Err(format, v...) -} - -func (l *Logger) Err(format string, v ...interface{}) { - l.ErrLog.Printf(format, v...) -} - -func (l *Logger) Empty() { - if !l.devView && l.InfoLog.Writer() != io.Discard { - fmt.Println("") - } -} - -func (l *Logger) SetProcess(format string, v ...interface{}) { - if l.InfoLog.Writer() == io.Discard { - return - } else if l.devView { - l.Debug(format, v...) - return - } - - initialMessage := fmt.Sprintf(format, v...) - - p := progress{ - message: initialMessage, - } - - l.lock.Lock() - if l.done != nil { - l.progress <- p - return - } else { - l.progress = make(chan progress, 1) - l.progress <- p - l.done = make(chan interface{}) - } - - go func() { - states := []string{"-", "\\", "|", "/"} - - var count int - - for i := 0; ; i++ { - select { - case p := <-l.progress: - if p.stop { - fmt.Printf("\r" + strings.Repeat(" ", len(prefix)+len(initialMessage))) - if count > 1 { - fmt.Printf("\r%s %s\n", progressDownFinish, p.message) - } else { - fmt.Printf("\r%s %s\n", prefix, p.message) - } - - if l.done != nil { - l.done <- nil - } - l.progress = nil - - l.lock.Unlock() - return - } else { - if count > 0 { - fmt.Printf("\r%s %s\n", progressDown, p.message) - } - l.progress = make(chan progress, 1) - - count++ - - fmt.Printf("\r%s %s", states[i/10%4], initialMessage) - l.lock.Unlock() - } - default: - if i%10 == 0 { - fmt.Printf("\r%s %s", states[i/10%4], initialMessage) - } - time.Sleep(35 * time.Millisecond) - } - } - }() -} - -func (l *Logger) StopProcess(format string, v ...interface{}) { - if l.InfoLog.Writer() == io.Discard { - return - } else if l.devView { - l.Debug(format, v...) - return - } - - l.lock.Lock() - l.progress <- progress{ - message: fmt.Sprintf(format, v...), - stop: true, - } - <-l.done - l.done = nil -} diff --git a/cli/commands/login/login.go b/cli/commands/login/login.go deleted file mode 100644 index 9fce181..0000000 --- a/cli/commands/login/login.go +++ /dev/null @@ -1,159 +0,0 @@ -package login - -import ( - "bytes" - "fmt" - "github.com/crunchy-labs/crunchy-cli/cli/commands" - "github.com/crunchy-labs/crunchy-cli/utils" - "github.com/crunchy-labs/crunchyroll-go/v3" - "github.com/spf13/cobra" - "os" -) - -var ( - loginPersistentFlag bool - loginEncryptFlag bool - - loginSessionIDFlag bool - loginRefreshTokenFlag bool -) - -var Cmd = &cobra.Command{ - Use: "login", - Short: "Login to crunchyroll", - Args: cobra.RangeArgs(1, 2), - - RunE: func(cmd *cobra.Command, args []string) error { - if loginSessionIDFlag { - return loginSessionID(args[0]) - } else if loginRefreshTokenFlag { - return loginRefreshToken(args[0]) - } else { - return loginCredentials(args[0], args[1]) - } - }, -} - -func init() { - Cmd.Flags().BoolVar(&loginPersistentFlag, - "persistent", - false, - "If the given credential should be stored persistent") - Cmd.Flags().BoolVar(&loginEncryptFlag, - "encrypt", - false, - "Encrypt the given credentials (won't do anything if --session-id is given or --persistent is not given)") - - Cmd.Flags().BoolVar(&loginSessionIDFlag, - "session-id", - false, - "Use a session id to login instead of username and password") - Cmd.Flags().BoolVar(&loginRefreshTokenFlag, - "refresh-token", - false, - "Use a refresh token to login instead of username and password. Can be obtained by copying the `etp-rt` cookie from beta.crunchyroll.com") - - Cmd.MarkFlagsMutuallyExclusive("session-id", "refresh-token") -} - -func loginCredentials(user, password string) error { - utils.Log.Debug("Logging in via credentials") - c, err := crunchyroll.LoginWithCredentials(user, password, utils.SystemLocale(false), utils.Client) - if err != nil { - return err - } - - if loginPersistentFlag { - var passwd []byte - if loginEncryptFlag { - for { - fmt.Print("Enter password: ") - passwd, err = commands.ReadLineSilent() - if err != nil { - return err - } - fmt.Println() - - fmt.Print("Enter password again: ") - repasswd, err := commands.ReadLineSilent() - if err != nil { - return err - } - fmt.Println() - - if bytes.Equal(passwd, repasswd) { - break - } - fmt.Println("Passwords does not match, try again") - } - } - if err = utils.SaveCredentialsPersistent(user, password, passwd); err != nil { - return err - } - if !loginEncryptFlag { - utils.Log.Warn("The login information will be stored permanently UNENCRYPTED on your drive. " + - "To encrypt it, use the `--encrypt` flag") - } - } - if err = utils.SaveSession(c); err != nil { - return err - } - - if !loginPersistentFlag { - utils.Log.Info("Due to security reasons, you have to login again on the next reboot") - } - - return nil -} - -func loginSessionID(sessionID string) error { - utils.Log.Debug("Logging in via session id") - utils.Log.Warn("Logging in with session id is deprecated and not very reliable. Consider choosing another option (if it fails)") - var c *crunchyroll.Crunchyroll - var err error - if c, err = crunchyroll.LoginWithSessionID(sessionID, utils.SystemLocale(false), utils.Client); err != nil { - return err - } - - if loginPersistentFlag { - if err = utils.SaveSessionPersistent(c); err != nil { - return err - } - utils.Log.Warn("The login information will be stored permanently UNENCRYPTED on your drive") - } - if err = utils.SaveSession(c); err != nil { - return err - } - - if !loginPersistentFlag { - utils.Log.Info("Due to security reasons, you have to login again on the next reboot") - } - - return nil -} - -func loginRefreshToken(refreshToken string) error { - utils.Log.Debug("Logging in via refresh token") - var c *crunchyroll.Crunchyroll - var err error - if c, err = crunchyroll.LoginWithRefreshToken(refreshToken, utils.SystemLocale(false), utils.Client); err != nil { - utils.Log.Err(err.Error()) - os.Exit(1) - } - - if loginPersistentFlag { - if err = utils.SaveSessionPersistent(c); err != nil { - return err - } - utils.Log.Warn("The login information will be stored permanently UNENCRYPTED on your drive") - } - if err = utils.SaveSession(c); err != nil { - return err - } - - if !loginPersistentFlag { - utils.Log.Info("Due to security reasons, you have to login again on the next reboot") - } - - return nil -} diff --git a/cli/commands/unix.go b/cli/commands/unix.go deleted file mode 100644 index ec69180..0000000 --- a/cli/commands/unix.go +++ /dev/null @@ -1,48 +0,0 @@ -//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris || zos - -package commands - -import ( - "bufio" - "os" - "os/exec" - "syscall" -) - -// https://github.com/bgentry/speakeasy/blob/master/speakeasy_unix.go -var stty string - -func init() { - var err error - if stty, err = exec.LookPath("stty"); err != nil { - panic(err) - } -} - -func ReadLineSilent() ([]byte, error) { - pid, err := setEcho(false) - if err != nil { - return nil, err - } - defer setEcho(true) - - syscall.Wait4(pid, nil, 0, nil) - - l, _, err := bufio.NewReader(os.Stdin).ReadLine() - return l, err -} - -func setEcho(on bool) (pid int, err error) { - fds := []uintptr{os.Stdin.Fd(), os.Stdout.Fd(), os.Stderr.Fd()} - - if on { - pid, err = syscall.ForkExec(stty, []string{"stty", "echo"}, &syscall.ProcAttr{Files: fds}) - } else { - pid, err = syscall.ForkExec(stty, []string{"stty", "-echo"}, &syscall.ProcAttr{Files: fds}) - } - - if err != nil { - return 0, err - } - return -} diff --git a/cli/commands/update/update.go b/cli/commands/update/update.go deleted file mode 100644 index 6608f26..0000000 --- a/cli/commands/update/update.go +++ /dev/null @@ -1,151 +0,0 @@ -package update - -import ( - "encoding/json" - "fmt" - "github.com/crunchy-labs/crunchy-cli/utils" - "github.com/spf13/cobra" - "io" - "os" - "os/exec" - "path" - "runtime" - "strings" -) - -var ( - updateInstallFlag bool -) - -var Cmd = &cobra.Command{ - Use: "update", - Short: "Check if updates are available", - Args: cobra.MaximumNArgs(0), - - RunE: func(cmd *cobra.Command, args []string) error { - return update() - }, -} - -func init() { - Cmd.Flags().BoolVarP(&updateInstallFlag, - "install", - "i", - false, - "If set and a new version is available, the new version gets installed") -} - -func update() error { - var release map[string]interface{} - - resp, err := utils.Client.Get("https://api.github.com/repos/crunchy-labs/crunchy-cli/releases/latest") - if err != nil { - return err - } - defer resp.Body.Close() - if err = json.NewDecoder(resp.Body).Decode(&release); err != nil { - return err - } - releaseVersion := strings.TrimPrefix(release["tag_name"].(string), "v") - - if utils.Version == "development" { - utils.Log.Info("Development version, update service not available") - return nil - } - - latestRelease := strings.SplitN(releaseVersion, ".", 4) - if len(latestRelease) != 3 { - return fmt.Errorf("latest tag name (%s) is not parsable", releaseVersion) - } - - internalVersion := strings.SplitN(utils.Version, ".", 4) - if len(internalVersion) != 3 { - return fmt.Errorf("internal version (%s) is not parsable", utils.Version) - } - - utils.Log.Info("Installed version is %s", utils.Version) - - var hasUpdate bool - for i := 0; i < 3; i++ { - if latestRelease[i] < internalVersion[i] { - utils.Log.Info("Local version is newer than version in latest release (%s)", releaseVersion) - return nil - } else if latestRelease[i] > internalVersion[i] { - hasUpdate = true - } - } - - if !hasUpdate { - utils.Log.Info("Version is up-to-date") - return nil - } - - utils.Log.Info("A new version is available (%s): https://github.com/crunchy-labs/crunchy-cli/releases/tag/v%s", releaseVersion, releaseVersion) - - if updateInstallFlag { - if runtime.GOARCH != "amd64" { - return fmt.Errorf("invalid architecture found (%s), only amd64 is currently supported for automatic updating. "+ - "You have to update manually (https://github.com/crunchy-labs/crunchy-cli)", runtime.GOARCH) - } - - var downloadFile string - switch runtime.GOOS { - case "linux": - pacmanCommand := exec.Command("pacman -Q crunchy-cli") - if pacmanCommand.Run() == nil && pacmanCommand.ProcessState.Success() { - utils.Log.Info("crunchy-cli was probably installed via an Arch Linux AUR helper (like yay). Updating via this AUR helper is recommended") - return nil - } - downloadFile = fmt.Sprintf("crunchy-v%s_linux", releaseVersion) - case "darwin": - downloadFile = fmt.Sprintf("crunchy-v%s_darwin", releaseVersion) - case "windows": - downloadFile = fmt.Sprintf("crunchy-v%s_windows.exe", releaseVersion) - default: - return fmt.Errorf("invalid operation system found (%s), only linux, windows and darwin / macos are currently supported. "+ - "You have to update manually (https://github.com/crunchy-labs/crunchy-cli", runtime.GOOS) - } - - executePath := os.Args[0] - var perms os.FileInfo - // check if the path is relative, absolute or non (if so, the executable must be in PATH) - if strings.HasPrefix(executePath, "."+string(os.PathSeparator)) || path.IsAbs(executePath) { - if perms, err = os.Stat(os.Args[0]); err != nil { - return err - } - } else { - executePath, err = exec.LookPath(os.Args[0]) - if err != nil { - return err - } - if perms, err = os.Stat(executePath); err != nil { - return err - } - } - - utils.Log.SetProcess("Updating executable %s", executePath) - - if err = os.Remove(executePath); err != nil { - return err - } - executeFile, err := os.OpenFile(executePath, os.O_CREATE|os.O_WRONLY, perms.Mode()) - if err != nil { - return err - } - defer executeFile.Close() - - resp, err := utils.Client.Get(fmt.Sprintf("https://github.com/crunchy-labs/crunchy-cli/releases/download/v%s/%s", releaseVersion, downloadFile)) - if err != nil { - return err - } - defer resp.Body.Close() - - if _, err = io.Copy(executeFile, resp.Body); err != nil { - return err - } - - utils.Log.StopProcess("Updated executable %s", executePath) - } - - return nil -} diff --git a/cli/commands/utils.go b/cli/commands/utils.go deleted file mode 100644 index 53e51f2..0000000 --- a/cli/commands/utils.go +++ /dev/null @@ -1,125 +0,0 @@ -package commands - -import ( - "fmt" - "github.com/crunchy-labs/crunchy-cli/utils" - "os" - "os/exec" - "runtime" - "strconv" - "strings" - "sync" -) - -type DownloadProgress struct { - Prefix string - Message string - - Total int - Current int - - Dev bool - Quiet bool - - lock sync.Mutex -} - -func (dp *DownloadProgress) Update() { - dp.update("", false) -} - -func (dp *DownloadProgress) UpdateMessage(msg string, permanent bool) { - dp.update(msg, permanent) -} - -func (dp *DownloadProgress) update(msg string, permanent bool) { - if dp.Quiet { - return - } - - if dp.Current >= dp.Total { - return - } - - dp.lock.Lock() - defer dp.lock.Unlock() - dp.Current++ - - if msg == "" { - msg = dp.Message - } - if permanent { - dp.Message = msg - } - - if dp.Dev { - fmt.Printf("%s%s\n", dp.Prefix, msg) - return - } - - percentage := float32(dp.Current) / float32(dp.Total) * 100 - - pre := fmt.Sprintf("%s%s [", dp.Prefix, msg) - post := fmt.Sprintf("]%4d%% %8d/%d", int(percentage), dp.Current, dp.Total) - - // I don't really know why +2 is needed here but without it the Printf below would not print to the line end - progressWidth := terminalWidth() - len(pre) - len(post) + 2 - repeatCount := int(percentage / float32(100) * float32(progressWidth)) - // it can be lower than zero when the terminal is very tiny - if repeatCount < 0 { - repeatCount = 0 - } - progressPercentage := strings.Repeat("=", repeatCount) - if dp.Current != dp.Total { - progressPercentage += ">" - } - - fmt.Printf("\r%s%-"+fmt.Sprint(progressWidth)+"s%s", pre, progressPercentage, post) -} - -func terminalWidth() int { - if runtime.GOOS != "windows" { - cmd := exec.Command("stty", "size") - cmd.Stdin = os.Stdin - res, err := cmd.Output() - if err != nil { - return 60 - } - // on alpine linux the command `stty size` does not respond the terminal size - // but something like "stty: standard input". this may also apply to other systems - splitOutput := strings.SplitN(strings.ReplaceAll(string(res), "\n", ""), " ", 2) - if len(splitOutput) == 1 { - return 60 - } - width, err := strconv.Atoi(splitOutput[1]) - if err != nil { - return 60 - } - return width - } - return 60 -} - -func LoadCrunchy() error { - var encryptionKey []byte - - if utils.IsTempSession() { - encryptionKey = nil - } else { - if encrypted, err := utils.IsSavedSessionEncrypted(); err != nil { - if os.IsNotExist(err) { - return fmt.Errorf("to use this command, login first. Type `%s login -h` to get help", os.Args[0]) - } - return err - } else if encrypted { - encryptionKey, err = ReadLineSilent() - if err != nil { - return fmt.Errorf("failed to read password") - } - } - } - - var err error - utils.Crunchy, err = utils.LoadSession(encryptionKey) - return err -} diff --git a/cli/commands/windows.go b/cli/commands/windows.go deleted file mode 100644 index a9bce74..0000000 --- a/cli/commands/windows.go +++ /dev/null @@ -1,41 +0,0 @@ -//go:build windows - -package commands - -import ( - "bufio" - "os" - "syscall" -) - -// https://github.com/bgentry/speakeasy/blob/master/speakeasy_windows.go -func ReadLineSilent() ([]byte, error) { - var oldMode uint32 - - if err := syscall.GetConsoleMode(syscall.Stdin, &oldMode); err != nil { - return nil, err - } - - newMode := oldMode &^ 0x0004 - - err := setConsoleMode(syscall.Stdin, newMode) - defer setConsoleMode(syscall.Stdin, oldMode) - - if err != nil { - return nil, err - } - - l, _, err := bufio.NewReader(os.Stdin).ReadLine() - if err != nil { - return nil, err - } - return l, err -} - -func setConsoleMode(console syscall.Handle, mode uint32) error { - dll := syscall.MustLoadDLL("kernel32") - proc := dll.MustFindProc("SetConsoleMode") - _, _, err := proc.Call(uintptr(console), uintptr(mode)) - - return err -} diff --git a/cli/root.go b/cli/root.go deleted file mode 100644 index b589a03..0000000 --- a/cli/root.go +++ /dev/null @@ -1,85 +0,0 @@ -package cli - -import ( - "context" - "fmt" - "github.com/crunchy-labs/crunchy-cli/cli/commands" - "github.com/crunchy-labs/crunchy-cli/cli/commands/archive" - "github.com/crunchy-labs/crunchy-cli/cli/commands/download" - "github.com/crunchy-labs/crunchy-cli/cli/commands/info" - "github.com/crunchy-labs/crunchy-cli/cli/commands/login" - "github.com/crunchy-labs/crunchy-cli/cli/commands/update" - "github.com/crunchy-labs/crunchy-cli/utils" - "github.com/spf13/cobra" - "os" - "runtime/debug" - "strings" -) - -var ( - quietFlag bool - verboseFlag bool - - proxyFlag string - - useragentFlag string -) - -var RootCmd = &cobra.Command{ - Use: "crunchy-cli", - Version: utils.Version, - Short: "Download crunchyroll videos with ease. See the wiki for details about the cli and library: https://github.com/crunchy-labs/crunchy-cli/wiki", - - SilenceErrors: true, - SilenceUsage: true, - - PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) { - if verboseFlag { - utils.Log = commands.NewLogger(true, true, true) - } else if quietFlag { - utils.Log = commands.NewLogger(false, false, false) - } - - utils.Log.Debug("Executing `%s` command with %d arg(s)", cmd.Name(), len(args)) - - utils.Client, err = utils.CreateOrDefaultClient(proxyFlag, useragentFlag) - return - }, -} - -func init() { - RootCmd.PersistentFlags().BoolVarP(&quietFlag, "quiet", "q", false, "Disable all output") - RootCmd.PersistentFlags().BoolVarP(&verboseFlag, "verbose", "v", false, "Adds debug messages to the normal output") - - RootCmd.PersistentFlags().StringVarP(&proxyFlag, "proxy", "p", "", "Proxy to use") - - RootCmd.PersistentFlags().StringVar(&useragentFlag, "useragent", fmt.Sprintf("crunchy-cli/%s", utils.Version), "Useragent to do all request with") - - RootCmd.AddCommand(archive.Cmd) - RootCmd.AddCommand(download.Cmd) - RootCmd.AddCommand(info.Cmd) - RootCmd.AddCommand(login.Cmd) - RootCmd.AddCommand(update.Cmd) - - utils.Log = commands.NewLogger(false, true, true) -} - -func Execute() { - RootCmd.CompletionOptions.HiddenDefaultCmd = true - defer func() { - if r := recover(); r != nil { - if utils.Log.IsDev() { - utils.Log.Err("%v: %s", r, debug.Stack()) - } else { - utils.Log.Err("Unexpected error: %v", r) - } - os.Exit(1) - } - }() - if err := RootCmd.Execute(); err != nil { - if !strings.HasSuffix(err.Error(), context.Canceled.Error()) { - utils.Log.Err("An error occurred: %v", err) - } - os.Exit(1) - } -} diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml new file mode 100644 index 0000000..893b278 --- /dev/null +++ b/crunchy-cli-core/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "crunchy-cli-core" +version = "0.1.0" +edition = "2021" + +[features] +# Embed a static curl library into the binary instead of just linking it. +static-curl = ["crunchyroll-rs/static-curl"] +# Embed a static openssl library into the binary instead of just linking it. If you want to compile this project against +# musl and have openssl issues, this might solve these issues. +static-ssl = ["crunchyroll-rs/static-ssl"] + +[dependencies] +anyhow = "1.0" +async-trait = "0.1" +clap = { version = "4.0", features = ["derive"] } +chrono = "0.4" +crunchyroll-rs = { git = "https://github.com/crunchy-labs/crunchyroll-rs", default-features = false, features = ["stream", "parse"] } +ctrlc = "3.2" +dirs = "4.0" +isahc = { git = "https://github.com/sagebind/isahc", rev = "34f158ef" } +log = { version = "0.4", features = ["std"] } +num_cpus = "1.13" +regex = "1.6" +signal-hook = "0.3" +tempfile = "3.3" +terminal_size = "0.2" +tokio = { version = "1.21", features = ["macros", "rt-multi-thread", "time"] } +sys-locale = "0.2" diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs new file mode 100644 index 0000000..21ae13b --- /dev/null +++ b/crunchy-cli-core/src/cli/archive.rs @@ -0,0 +1,567 @@ +use crate::cli::log::tab_info; +use crate::cli::utils::{download_segments, find_resolution}; +use crate::utils::context::Context; +use crate::utils::format::{format_string, Format}; +use crate::utils::log::progress; +use crate::utils::os::{free_file, tempfile}; +use crate::utils::parse::{parse_url, UrlFilter}; +use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number}; +use crate::Execute; +use anyhow::{bail, Result}; +use crunchyroll_rs::media::{Resolution, StreamSubtitle}; +use crunchyroll_rs::{Locale, Media, MediaCollection, Series}; +use log::{debug, error, info}; +use regex::Regex; +use std::collections::BTreeMap; +use std::io::Write; +use std::path::PathBuf; +use std::process::{Command, Stdio}; +use tempfile::TempPath; + +#[derive(Clone, Debug)] +pub enum MergeBehavior { + Auto, + Audio, + Video, +} + +fn parse_merge_behavior(s: &str) -> Result { + Ok(match s.to_lowercase().as_str() { + "auto" => MergeBehavior::Auto, + "audio" => MergeBehavior::Audio, + "video" => MergeBehavior::Video, + _ => return Err(format!("'{}' is not a valid merge behavior", s)), + }) +} + +#[derive(Debug, clap::Parser)] +#[clap(about = "Archive a video")] +#[command(arg_required_else_help(true))] +#[command()] +pub struct Archive { + #[arg(help = format!("Audio languages. Can be used multiple times. \ + Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] + #[arg(long_help = format!("Audio languages. Can be used multiple times. \ + Available languages are:\n{}", Locale::all().into_iter().map(|l| format!("{:<6} → {}", l.to_string(), l.to_human_readable())).collect::>().join("\n ")))] + #[arg(short, long, default_values_t = vec![crate::utils::locale::system_locale(), Locale::ja_JP])] + audio: Vec, + #[arg(help = format!("Subtitle languages. Can be used multiple times. \ + Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] + #[arg(long_help = format!("Subtitle languages. Can be used multiple times. \ + Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] + #[arg(short, long, default_values_t = Locale::all())] + subtitle: Vec, + + #[arg(help = "Name of the output file")] + #[arg(long_help = "Name of the output file.\ + If you use one of the following pattern they will get replaced:\n \ + {title} → Title of the video\n \ + {series_name} → Name of the series\n \ + {season_name} → Name of the season\n \ + {audio} → Audio language of the video\n \ + {resolution} → Resolution of the video\n \ + {season_number} → Number of the season\n \ + {episode_number} → Number of the episode\n \ + {series_id} → ID of the series\n \ + {season_id} → ID of the season\n \ + {episode_id} → ID of the episode")] + #[arg(short, long, default_value = "{title}.mkv")] + output: String, + + #[arg(help = "Video resolution")] + #[arg(long_help = "The video resolution.\ + Can either be specified via the pixels (e.g. 1920x1080), the abbreviation for pixels (e.g. 1080p) or 'common-use' words (e.g. best). \ + Specifying the exact pixels is not recommended, use one of the other options instead. \ + Crunchyroll let you choose the quality with pixel abbreviation on their clients, so you might be already familiar with the available options. \ + The available common-use words are 'best' (choose the best resolution available) and 'worst' (worst resolution available)")] + #[arg(short, long, default_value = "best")] + #[arg(value_parser = crate::utils::clap::clap_parse_resolution)] + resolution: Resolution, + + #[arg( + help = "Sets the behavior of the stream merging. Valid behaviors are 'auto', 'audio' and 'video'" + )] + #[arg( + long_help = "Because of local restrictions (or other reasons) some episodes with different languages does not have the same length (e.g. when some scenes were cut out). \ + With this flag you can set the behavior when handling multiple language. + Valid options are 'audio' (stores one video and all other languages as audio only), 'video' (stores the video + audio for every language) and 'auto' (detects if videos differ in length: if so, behave like 'video' else like 'audio')" + )] + #[arg(short, long, default_value = "auto")] + #[arg(value_parser = parse_merge_behavior)] + merge: MergeBehavior, + + #[arg( + help = "Set which subtitle language should be set as default / auto shown when starting a video" + )] + #[arg(long)] + default_subtitle: Option, + #[arg(help = "Disable subtitle optimizations")] + #[arg( + long_help = "By default, Crunchyroll delivers subtitles in a format which may cause issues in some video players. \ + These issues are fixed internally by setting a flag which is not part of the official specification of the subtitle format. \ + If you do not want this fixes or they cause more trouble than they solve (for you), it can be disabled with this flag" + )] + #[arg(long)] + no_subtitle_optimizations: bool, + + #[arg(help = "Crunchyroll series url(s)")] + urls: Vec, +} + +#[async_trait::async_trait(?Send)] +impl Execute for Archive { + async fn execute(self, ctx: Context) -> Result<()> { + let mut parsed_urls = vec![]; + + for (i, url) in self.urls.iter().enumerate() { + let _progress_handler = progress!("Parsing url {}", i + 1); + match parse_url(&ctx.crunchy, url.clone(), true).await { + Ok((media_collection, url_filter)) => { + parsed_urls.push((media_collection, url_filter)); + info!("Parsed url {}", i + 1) + } + Err(e) => bail!("url {} could not be parsed: {}", url, e), + } + } + + for (i, (media_collection, url_filter)) in parsed_urls.into_iter().enumerate() { + let archive_formats = match media_collection { + MediaCollection::Series(series) => { + let _progress_handler = progress!("Fetching series details"); + formats_from_series(&self, series, &url_filter).await? + } + MediaCollection::Season(_) => bail!("Archiving a season is not supported"), + MediaCollection::Episode(episode) => bail!("Archiving a episode is not supported. Use url filtering instead to specify the episode (https://www.crunchyroll.com/series/{}/{}[S{}E{}])", episode.metadata.series_id, episode.metadata.series_slug_title, episode.metadata.season_number, episode.metadata.episode_number), + MediaCollection::MovieListing(_) => bail!("Archiving a movie listing is not supported"), + MediaCollection::Movie(_) => bail!("Archiving a movie is not supported") + }; + + if archive_formats.is_empty() { + info!("Skipping url {} (no matching episodes found)", i + 1); + continue; + } + info!("Loaded series information for url {}", i + 1); + + if log::max_level() == log::Level::Debug { + let seasons = sort_formats_after_seasons( + archive_formats + .clone() + .into_iter() + .map(|(a, _)| a.get(0).unwrap().clone()) + .collect(), + ); + debug!("Series has {} seasons", seasons.len()); + for (i, season) in seasons.into_iter().enumerate() { + info!("Season {} ({})", i + 1, season.get(0).unwrap().season_title); + for format in season { + info!( + "{}: {}px, {:.02} FPS (S{:02}E{:02})", + format.title, + format.stream.resolution, + format.stream.fps, + format.season_number, + format.number, + ) + } + } + } else { + for season in sort_formats_after_seasons( + archive_formats + .clone() + .into_iter() + .map(|(a, _)| a.get(0).unwrap().clone()) + .collect(), + ) { + let first = season.get(0).unwrap(); + info!( + "{} Season {} ({})", + first.series_name, first.season_number, first.season_title + ); + + for (i, format) in season.into_iter().enumerate() { + tab_info!( + "{}. {} Âģ {}px, {:.2} FPS (S{:02}E{:02})", + i + 1, + format.title, + format.stream.resolution, + format.stream.fps, + format.season_number, + format.number + ) + } + } + } + + for (formats, subtitles) in archive_formats { + let (primary, additionally) = formats.split_first().unwrap(); + + let mut path = PathBuf::from(&self.output); + path = free_file( + path.with_file_name(format_string( + if let Some(fname) = path.file_name() { + fname.to_str().unwrap() + } else { + "{title}.mkv" + } + .to_string(), + primary, + )), + ) + .0; + + info!( + "Downloading {} to '{}'", + primary.title, + path.to_str().unwrap() + ); + tab_info!( + "Episode: S{:02}E{:02}", + primary.season_number, + primary.number + ); + tab_info!( + "Audio: {} (primary), {}", + primary.audio, + additionally + .iter() + .map(|a| a.audio.to_string()) + .collect::>() + .join(", ") + ); + tab_info!( + "Subtitle: {}", + subtitles + .iter() + .map(|s| { + if let Some(default) = &self.default_subtitle { + if default == &s.locale { + return format!("{} (primary)", default); + } + } + s.locale.to_string() + }) + .collect::>() + .join(", ") + ); + tab_info!("Resolution: {}", primary.stream.resolution); + tab_info!("FPS: {:.2}", primary.stream.fps); + + let mut video_paths = vec![]; + let mut audio_paths = vec![]; + let mut subtitle_paths = vec![]; + + video_paths.push((download_video(&ctx, primary, false).await?, primary)); + for additional in additionally { + let only_audio = match self.merge { + MergeBehavior::Auto => additionally + .iter() + .all(|a| a.stream.bandwidth == primary.stream.bandwidth), + MergeBehavior::Audio => true, + MergeBehavior::Video => false, + }; + let path = download_video(&ctx, additional, only_audio).await?; + if only_audio { + audio_paths.push((path, additional)) + } else { + video_paths.push((path, additional)) + } + } + + for subtitle in subtitles { + subtitle_paths + .push((download_subtitle(&self, subtitle.clone()).await?, subtitle)) + } + + generate_mkv(&self, path, video_paths, audio_paths, subtitle_paths)? + } + } + + Ok(()) + } +} + +async fn formats_from_series( + archive: &Archive, + series: Media, + url_filter: &UrlFilter, +) -> Result, Vec)>> { + let mut seasons = series.seasons().await?; + + // filter any season out which does not contain the specified audio languages + for season in sort_seasons_after_number(seasons.clone()) { + // get all locales which are specified but not present in the current iterated season and + // print an error saying this + let not_present_audio = archive + .audio + .clone() + .into_iter() + .filter(|l| !season.iter().any(|s| &s.metadata.audio_locale == l)) + .collect::>(); + for not_present in not_present_audio { + error!( + "Season {} of series {} is not available with {} audio", + season.first().unwrap().metadata.season_number, + series.title, + not_present + ) + } + + // remove all seasons with the wrong audio for the current iterated season number + seasons.retain(|s| { + s.metadata.season_number != season.first().unwrap().metadata.season_number + || archive.audio.contains(&s.metadata.audio_locale) + }) + } + + #[allow(clippy::type_complexity)] + let mut result: BTreeMap, Vec)>> = + BTreeMap::new(); + for season in series.seasons().await? { + if !url_filter.is_season_valid(season.metadata.season_number) + || !archive.audio.contains(&season.metadata.audio_locale) + { + continue; + } + + for episode in season.episodes().await? { + if !url_filter.is_episode_valid( + episode.metadata.episode_number, + episode.metadata.season_number, + ) { + continue; + } + + let streams = episode.streams().await?; + let streaming_data = streams.streaming_data(None).await?; + let Some(stream) = find_resolution(streaming_data, &archive.resolution) else { + bail!( + "Resolution ({}x{}) is not available for episode {} ({}) of season {} ({}) of {}", + archive.resolution.width, + archive.resolution.height, + episode.metadata.episode_number, + episode.title, + episode.metadata.season_number, + episode.metadata.season_title, + episode.metadata.series_title + ) + }; + + let (ref mut formats, _) = result + .entry(season.metadata.season_number) + .or_insert_with(BTreeMap::new) + .entry(episode.metadata.episode_number) + .or_insert_with(|| { + let subtitles: Vec = archive + .subtitle + .iter() + .filter_map(|l| streams.subtitles.get(l).cloned()) + .collect(); + (vec![], subtitles) + }); + formats.push(Format::new_from_episode(episode, stream)); + } + } + + Ok(result.into_values().flat_map(|v| v.into_values()).collect()) +} + +async fn download_video(ctx: &Context, format: &Format, only_audio: bool) -> Result { + let tempfile = if only_audio { + tempfile(".aac")? + } else { + tempfile(".ts")? + }; + let (_, path) = tempfile.into_parts(); + + let ffmpeg = Command::new("ffmpeg") + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .arg("-y") + .args(["-f", "mpegts", "-i", "pipe:"]) + .args(if only_audio { vec!["-vn"] } else { vec![] }) + .arg(path.to_str().unwrap()) + .spawn()?; + + download_segments( + ctx, + &mut ffmpeg.stdin.unwrap(), + Some(format!("Download {}", format.audio)), + format.stream.segments().await?, + ) + .await?; + + Ok(path) +} + +async fn download_subtitle(archive: &Archive, subtitle: StreamSubtitle) -> Result { + let tempfile = tempfile(".ass")?; + let (mut file, path) = tempfile.into_parts(); + + let mut buf = vec![]; + subtitle.write_to(&mut buf).await?; + if !archive.no_subtitle_optimizations { + buf = fix_subtitle(buf) + } + + file.write_all(buf.as_slice())?; + + Ok(path) +} + +/// Add `ScaledBorderAndShadows: yes` to subtitles; without it they look very messy on some video +/// players. See [crunchy-labs/crunchy-cli#66](https://github.com/crunchy-labs/crunchy-cli/issues/66) +/// for more information. +fn fix_subtitle(raw: Vec) -> Vec { + let mut script_info = false; + let mut new = String::new(); + + for line in String::from_utf8_lossy(raw.as_slice()).split('\n') { + if line.trim().starts_with('[') && script_info { + new.push_str("ScaledBorderAndShadows: yes\n"); + script_info = false + } else if line.trim() == "[Script Info]" { + script_info = true + } + new.push_str(line); + new.push('\n') + } + + new.into_bytes() +} + +fn generate_mkv( + archive: &Archive, + target: PathBuf, + video_paths: Vec<(TempPath, &Format)>, + audio_paths: Vec<(TempPath, &Format)>, + subtitle_paths: Vec<(TempPath, StreamSubtitle)>, +) -> Result<()> { + let mut input = vec![]; + let mut maps = vec![]; + let mut metadata = vec![]; + + let mut video_length = (0, 0, 0, 0); + + for (i, (video_path, format)) in video_paths.iter().enumerate() { + input.extend(["-i".to_string(), video_path.to_string_lossy().to_string()]); + maps.extend(["-map".to_string(), i.to_string()]); + metadata.extend([ + format!("-metadata:s:v:{}", i), + format!("language={}", format.audio), + ]); + metadata.extend([ + format!("-metadata:s:v:{}", i), + format!("title={}", format.audio.to_human_readable()), + ]); + metadata.extend([ + format!("-metadata:s:a:{}", i), + format!("language={}", format.audio), + ]); + metadata.extend([ + format!("-metadata:s:a:{}", i), + format!("title={}", format.audio.to_human_readable()), + ]); + + let vid_len = get_video_length(video_path.to_path_buf())?; + if vid_len > video_length { + video_length = vid_len + } + } + for (i, (audio_path, format)) in audio_paths.iter().enumerate() { + input.extend(["-i".to_string(), audio_path.to_string_lossy().to_string()]); + maps.extend(["-map".to_string(), (i + video_paths.len()).to_string()]); + metadata.extend([ + format!("-metadata:s:a:{}", i + video_paths.len()), + format!("language={}", format.audio), + ]); + metadata.extend([ + format!("-metadata:s:a:{}", i + video_paths.len()), + format!("title={}", format.audio.to_human_readable()), + ]); + } + for (i, (subtitle_path, subtitle)) in subtitle_paths.iter().enumerate() { + input.extend([ + "-i".to_string(), + subtitle_path.to_string_lossy().to_string(), + ]); + maps.extend([ + "-map".to_string(), + (i + video_paths.len() + audio_paths.len()).to_string(), + ]); + metadata.extend([ + format!("-metadata:s:s:{}", i), + format!("language={}", subtitle.locale), + ]); + metadata.extend([ + format!("-metadata:s:s:{}", i), + format!("title={}", subtitle.locale.to_human_readable()), + ]); + } + + let mut command_args = vec!["-y".to_string()]; + command_args.extend(input); + command_args.extend(maps); + command_args.extend(metadata); + + // set default subtitle + if let Some(default_subtitle) = &archive.default_subtitle { + // if `--default_subtitle ` is given set the default subtitle to the given locale + if let Some(position) = subtitle_paths + .into_iter() + .position(|s| &s.1.locale == default_subtitle) + { + command_args.push(format!("-disposition:s:{}", position)) + } else { + command_args.extend(["-disposition:s:0".to_string(), "0".to_string()]) + } + } else { + command_args.extend(["-disposition:s:0".to_string(), "0".to_string()]) + } + + command_args.extend([ + "-c".to_string(), + "copy".to_string(), + "-f".to_string(), + "matroska".to_string(), + target.to_string_lossy().to_string(), + ]); + + debug!("ffmpeg {}", command_args.join(" ")); + + let ffmpeg = Command::new("ffmpeg") + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .args(command_args) + .output()?; + if !ffmpeg.status.success() { + bail!("{}", String::from_utf8_lossy(ffmpeg.stderr.as_slice())) + } + + Ok(()) +} + +/// Get the length of a video. This is required because sometimes subtitles have an unnecessary entry +/// long after the actual video ends with artificially extends the video length on some video players. +/// To prevent this, the video length must be hard set with ffmpeg. See +/// [crunchy-labs/crunchy-cli#32](https://github.com/crunchy-labs/crunchy-cli/issues/32) for more +/// information. +fn get_video_length(path: PathBuf) -> Result<(u32, u32, u32, u32)> { + let video_length = Regex::new(r"Duration:\s?(\d+):(\d+):(\d+).(\d+),")?; + + let ffmpeg = Command::new("ffmpeg") + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .arg("-y") + .args(["-i", path.to_str().unwrap()]) + .output()?; + let ffmpeg_output = String::from_utf8(ffmpeg.stderr)?; + let caps = video_length.captures(ffmpeg_output.as_str()).unwrap(); + + Ok(( + caps[1].parse()?, + caps[2].parse()?, + caps[3].parse()?, + caps[4].parse()?, + )) +} diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs new file mode 100644 index 0000000..7f9e3ec --- /dev/null +++ b/crunchy-cli-core/src/cli/download.rs @@ -0,0 +1,452 @@ +use crate::cli::log::tab_info; +use crate::cli::utils::{download_segments, find_resolution}; +use crate::utils::context::Context; +use crate::utils::format::{format_string, Format}; +use crate::utils::log::progress; +use crate::utils::os::{free_file, has_ffmpeg}; +use crate::utils::parse::{parse_url, UrlFilter}; +use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number}; +use crate::Execute; +use anyhow::{bail, Result}; +use crunchyroll_rs::media::{Resolution, VariantSegment}; +use crunchyroll_rs::{ + Episode, Locale, Media, MediaCollection, Movie, MovieListing, Season, Series, +}; +use log::{debug, error, info}; +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +#[derive(Debug, clap::Parser)] +#[clap(about = "Download a video")] +#[command(arg_required_else_help(true))] +pub struct Download { + #[arg(help = format!("Audio language. Can only be used if the provided url(s) point to a series. \ + Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] + #[arg(long_help = format!("Audio language. Can only be used if the provided url(s) point to a series. \ + Available languages are:\n{}", Locale::all().into_iter().map(|l| format!("{:<6} → {}", l.to_string(), l.to_human_readable())).collect::>().join("\n ")))] + #[arg(short, long, default_value_t = crate::utils::locale::system_locale())] + audio: Locale, + #[arg(help = format!("Subtitle language. Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] + #[arg(long_help = format!("Subtitle language. If set, the subtitle will be burned into the video and cannot be disabled. \ + Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] + #[arg(short, long)] + subtitle: Option, + + #[arg(help = "Name of the output file")] + #[arg(long_help = "Name of the output file.\ + If you use one of the following pattern they will get replaced:\n \ + {title} → Title of the video\n \ + {series_name} → Name of the series\n \ + {season_name} → Name of the season\n \ + {audio} → Audio language of the video\n \ + {resolution} → Resolution of the video\n \ + {season_number} → Number of the season\n \ + {episode_number} → Number of the episode\n \ + {series_id} → ID of the series\n \ + {season_id} → ID of the season\n \ + {episode_id} → ID of the episode")] + #[arg(short, long, default_value = "{title}.ts")] + output: String, + + #[arg(help = "Video resolution")] + #[arg(long_help = "The video resolution.\ + Can either be specified via the pixels (e.g. 1920x1080), the abbreviation for pixels (e.g. 1080p) or 'common-use' words (e.g. best). \ + Specifying the exact pixels is not recommended, use one of the other options instead. \ + Crunchyroll let you choose the quality with pixel abbreviation on their clients, so you might be already familiar with the available options. \ + The available common-use words are 'best' (choose the best resolution available) and 'worst' (worst resolution available)")] + #[arg(short, long, default_value = "best")] + #[arg(value_parser = crate::utils::clap::clap_parse_resolution)] + resolution: Resolution, + + #[arg(help = "Url(s) to Crunchyroll episodes or series")] + urls: Vec, +} + +#[async_trait::async_trait(?Send)] +impl Execute for Download { + async fn execute(self, ctx: Context) -> Result<()> { + let mut parsed_urls = vec![]; + + for (i, url) in self.urls.iter().enumerate() { + let _progress_handler = progress!("Parsing url {}", i + 1); + match parse_url(&ctx.crunchy, url.clone(), true).await { + Ok((media_collection, url_filter)) => { + parsed_urls.push((media_collection, url_filter)); + info!("Parsed url {}", i + 1) + } + Err(e) => bail!("url {} could not be parsed: {}", url, e), + } + } + + for (i, (media_collection, url_filter)) in parsed_urls.into_iter().enumerate() { + let _progress_handler = progress!("Fetching series details"); + let formats = match media_collection { + MediaCollection::Series(series) => { + debug!("Url {} is series ({})", i + 1, series.title); + formats_from_series(&self, series, &url_filter).await? + } + MediaCollection::Season(season) => { + debug!( + "Url {} is season {} ({})", + i + 1, + season.metadata.season_number, + season.title + ); + formats_from_season(&self, season, &url_filter).await? + } + MediaCollection::Episode(episode) => { + debug!( + "Url {} is episode {} ({}) of season {} ({}) of {}", + i + 1, + episode.metadata.episode_number, + episode.title, + episode.metadata.season_number, + episode.metadata.season_title, + episode.metadata.series_title + ); + format_from_episode(&self, episode, &url_filter, false) + .await? + .map(|fmt| vec![fmt]) + } + MediaCollection::MovieListing(movie_listing) => { + debug!("Url {} is movie listing ({})", i + 1, movie_listing.title); + format_from_movie_listing(&self, movie_listing, &url_filter).await? + } + MediaCollection::Movie(movie) => { + debug!("Url {} is movie ({})", i + 1, movie.title); + format_from_movie(&self, movie, &url_filter) + .await? + .map(|fmt| vec![fmt]) + } + }; + + let Some(formats) = formats else { + info!("Skipping url {} (no matching episodes found)", i + 1); + continue; + }; + info!("Loaded series information for url {}", i + 1); + drop(_progress_handler); + + if log::max_level() == log::Level::Debug { + let seasons = sort_formats_after_seasons(formats.clone()); + debug!("Series has {} seasons", seasons.len()); + for (i, season) in seasons.into_iter().enumerate() { + info!("Season {} ({})", i + 1, season.get(0).unwrap().season_title); + for format in season { + info!( + "{}: {}px, {:.02} FPS (S{:02}E{:02})", + format.title, + format.stream.resolution, + format.stream.fps, + format.season_number, + format.number, + ) + } + } + } else { + for season in sort_formats_after_seasons(formats.clone()) { + let first = season.get(0).unwrap(); + info!( + "{} Season {} ({})", + first.series_name, first.season_number, first.season_title + ); + + for (i, format) in season.into_iter().enumerate() { + tab_info!( + "{}. {} Âģ {}px, {:.2} FPS (S{:02}E{:02})", + i + 1, + format.title, + format.stream.resolution, + format.stream.fps, + format.season_number, + format.number + ) + } + } + } + + for format in formats { + let mut path = PathBuf::from(&self.output); + path = free_file( + path.with_file_name(format_string( + if let Some(fname) = path.file_name() { + fname.to_str().unwrap() + } else { + "{title}.ts" + } + .to_string(), + &format, + )), + ) + .0; + + let use_ffmpeg = if let Some(extension) = path.extension() { + if extension != "ts" { + if !has_ffmpeg() { + bail!( + "File ending is not `.ts`, ffmpeg is required to convert the video" + ) + } + true + } else { + false + } + } else { + false + }; + + info!( + "Downloading {} to '{}'", + format.title, + path.file_name().unwrap().to_str().unwrap() + ); + tab_info!("Episode: S{:02}E{:02}", format.season_number, format.number); + tab_info!("Audio: {}", format.audio); + tab_info!( + "Subtitles: {}", + self.subtitle + .clone() + .map_or("None".to_string(), |l| l.to_string()) + ); + tab_info!("Resolution: {}", format.stream.resolution); + tab_info!("FPS: {:.2}", format.stream.fps); + + let segments = format.stream.segments().await?; + + if use_ffmpeg { + download_ffmpeg(&ctx, segments, path.as_path()).await?; + } else if path.to_str().unwrap() == "-" { + let mut stdout = std::io::stdout().lock(); + download_segments(&ctx, &mut stdout, None, segments).await?; + } else { + let mut file = File::options().create(true).write(true).open(&path)?; + download_segments(&ctx, &mut file, None, segments).await? + } + } + } + + Ok(()) + } +} + +async fn download_ffmpeg( + ctx: &Context, + segments: Vec, + target: &Path, +) -> Result<()> { + let ffmpeg = Command::new("ffmpeg") + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .arg("-y") + .args(["-f", "mpegts", "-i", "pipe:"]) + .args(["-safe", "0"]) + .args(["-c", "copy"]) + .arg(target.to_str().unwrap()) + .spawn()?; + + download_segments(ctx, &mut ffmpeg.stdin.unwrap(), None, segments).await?; + + Ok(()) +} + +async fn formats_from_series( + download: &Download, + series: Media, + url_filter: &UrlFilter, +) -> Result>> { + if !series.metadata.audio_locales.is_empty() + && !series.metadata.audio_locales.contains(&download.audio) + { + error!( + "Series {} is not available with {} audio", + series.title, download.audio + ); + return Ok(None); + } + + let mut seasons = series.seasons().await?; + + // filter any season out which does not contain the specified audio language + for season in sort_seasons_after_number(seasons.clone()) { + // check if the current iterated season has the specified audio language + if !season + .iter() + .any(|s| s.metadata.audio_locale == download.audio) + { + error!( + "Season {} of series {} is not available with {} audio", + season.first().unwrap().metadata.season_number, + series.title, + download.audio + ); + } + + // remove all seasons with the wrong audio for the current iterated season number + seasons.retain(|s| { + s.metadata.season_number != season.first().unwrap().metadata.season_number + || s.metadata.audio_locale == download.audio + }) + } + + let mut formats = vec![]; + for season in seasons { + if let Some(fmts) = formats_from_season(download, season, url_filter).await? { + formats.extend(fmts) + } + } + + Ok(some_vec_or_none(formats)) +} + +async fn formats_from_season( + download: &Download, + season: Media, + url_filter: &UrlFilter, +) -> Result>> { + if season.metadata.audio_locale != download.audio { + error!( + "Season {} ({}) is not available with {} audio", + season.metadata.season_number, season.title, download.audio + ); + return Ok(None); + } else if !url_filter.is_season_valid(season.metadata.season_number) { + return Ok(None); + } + + let mut formats = vec![]; + + for episode in season.episodes().await? { + if let Some(fmt) = format_from_episode(download, episode, url_filter, true).await? { + formats.push(fmt) + } + } + + Ok(some_vec_or_none(formats)) +} + +async fn format_from_episode( + download: &Download, + episode: Media, + url_filter: &UrlFilter, + filter_audio: bool, +) -> Result> { + if filter_audio && episode.metadata.audio_locale != download.audio { + error!( + "Episode {} ({}) of season {} ({}) of {} has no {} audio", + episode.metadata.episode_number, + episode.title, + episode.metadata.season_number, + episode.metadata.season_title, + episode.metadata.series_title, + download.audio + ); + return Ok(None); + } else if !url_filter.is_episode_valid( + episode.metadata.episode_number, + episode.metadata.season_number, + ) { + return Ok(None); + } + + let streams = episode.streams().await?; + let streaming_data = if let Some(subtitle) = &download.subtitle { + if !streams.subtitles.keys().cloned().any(|x| &x == subtitle) { + error!( + "Episode {} ({}) of season {} ({}) of {} has no {} subtitles", + episode.metadata.episode_number, + episode.title, + episode.metadata.season_number, + episode.metadata.season_title, + episode.metadata.series_title, + subtitle + ); + return Ok(None); + } + streams.streaming_data(Some(subtitle.clone())).await? + } else { + streams.streaming_data(None).await? + }; + + let Some(stream) = find_resolution(streaming_data, &download.resolution) else { + bail!( + "Resolution ({}x{}) is not available for episode {} ({}) of season {} ({}) of {}", + download.resolution.width, + download.resolution.height, + episode.metadata.episode_number, + episode.title, + episode.metadata.season_number, + episode.metadata.season_title, + episode.metadata.series_title + ) + }; + + Ok(Some(Format::new_from_episode(episode, stream))) +} + +async fn format_from_movie_listing( + download: &Download, + movie_listing: Media, + url_filter: &UrlFilter, +) -> Result>> { + let mut formats = vec![]; + + for movie in movie_listing.movies().await? { + if let Some(fmt) = format_from_movie(download, movie, url_filter).await? { + formats.push(fmt) + } + } + + Ok(some_vec_or_none(formats)) +} + +async fn format_from_movie( + download: &Download, + movie: Media, + _: &UrlFilter, +) -> Result> { + let streams = movie.streams().await?; + let mut streaming_data = if let Some(subtitle) = &download.subtitle { + if !streams.subtitles.keys().cloned().any(|x| &x == subtitle) { + error!("Movie {} has no {} subtitles", movie.title, subtitle); + return Ok(None); + } + streams.streaming_data(Some(subtitle.clone())).await? + } else { + streams.streaming_data(None).await? + }; + + streaming_data.sort_by(|a, b| a.resolution.width.cmp(&b.resolution.width).reverse()); + let stream = { + match download.resolution.height { + u64::MAX => streaming_data.into_iter().next().unwrap(), + u64::MIN => streaming_data.into_iter().last().unwrap(), + _ => { + if let Some(streaming_data) = streaming_data.into_iter().find(|v| { + download.resolution.height == u64::MAX + || v.resolution.height == download.resolution.height + }) { + streaming_data + } else { + bail!( + "Resolution ({}x{}) is not available for movie {}", + download.resolution.width, + download.resolution.height, + movie.title + ) + } + } + } + }; + + Ok(Some(Format::new_from_movie(movie, stream))) +} + +fn some_vec_or_none(v: Vec) -> Option> { + if v.is_empty() { + None + } else { + Some(v) + } +} diff --git a/crunchy-cli-core/src/cli/log.rs b/crunchy-cli-core/src/cli/log.rs new file mode 100644 index 0000000..d4bb1b3 --- /dev/null +++ b/crunchy-cli-core/src/cli/log.rs @@ -0,0 +1,197 @@ +use log::{ + set_boxed_logger, set_max_level, Level, LevelFilter, Log, Metadata, Record, SetLoggerError, +}; +use std::io::{stdout, Write}; +use std::sync::{mpsc, Mutex}; +use std::thread; +use std::thread::JoinHandle; +use std::time::Duration; + +struct CliProgress { + handler: JoinHandle<()>, + sender: mpsc::SyncSender<(String, Level)>, +} + +impl CliProgress { + fn new(record: &Record) -> Self { + let (tx, rx) = mpsc::sync_channel(1); + + let init_message = format!("{}", record.args()); + let init_level = record.level(); + let handler = thread::spawn(move || { + let states = ["-", "\\", "|", "/"]; + + let mut old_message = init_message.clone(); + let mut latest_info_message = init_message; + let mut old_level = init_level; + for i in 0.. { + let (msg, level) = match rx.try_recv() { + Ok(payload) => payload, + Err(e) => match e { + mpsc::TryRecvError::Empty => (old_message.clone(), old_level), + mpsc::TryRecvError::Disconnected => break, + }, + }; + + // clear last line + // prefix (2), space (1), state (1), space (1), message(n) + let _ = write!(stdout(), "\r {}", " ".repeat(old_message.len())); + + if old_level != level || old_message != msg { + if old_level <= Level::Warn { + let _ = writeln!(stdout(), "\r:: â€ĸ {}", old_message); + } else if old_level == Level::Info && level <= Level::Warn { + let _ = writeln!(stdout(), "\r:: → {}", old_message); + } else if level == Level::Info { + latest_info_message = msg.clone(); + } + } + + let _ = write!( + stdout(), + "\r:: {} {}", + states[i / 2 % states.len()], + if level == Level::Info { + &msg + } else { + &latest_info_message + } + ); + let _ = stdout().flush(); + + old_message = msg; + old_level = level; + + thread::sleep(Duration::from_millis(100)); + } + + // clear last line + // prefix (2), space (1), state (1), space (1), message(n) + let _ = write!(stdout(), "\r {}", " ".repeat(old_message.len())); + let _ = writeln!(stdout(), "\r:: ✓ {}", old_message); + let _ = stdout().flush(); + }); + + Self { + handler, + sender: tx, + } + } + + fn send(&self, record: &Record) { + let _ = self + .sender + .send((format!("{}", record.args()), record.level())); + } + + fn stop(self) { + drop(self.sender); + let _ = self.handler.join(); + } +} + +#[allow(clippy::type_complexity)] +pub struct CliLogger { + level: LevelFilter, + progress: Mutex>, +} + +impl Log for CliLogger { + fn enabled(&self, metadata: &Metadata) -> bool { + metadata.level() <= self.level + } + + fn log(&self, record: &Record) { + if !self.enabled(record.metadata()) + || (record.target() != "progress" + && record.target() != "progress_end" + && !record.target().starts_with("crunchy_cli")) + { + return; + } + + if self.level >= LevelFilter::Debug { + self.extended(record); + return; + } + + match record.target() { + "progress" => self.progress(record, false), + "progress_end" => self.progress(record, true), + _ => { + if self.progress.lock().unwrap().is_some() { + self.progress(record, false); + } else if record.level() > Level::Warn { + self.normal(record) + } else { + self.error(record) + } + } + } + } + + fn flush(&self) { + let _ = stdout().flush(); + } +} + +impl CliLogger { + pub fn new(level: LevelFilter) -> Self { + Self { + level, + progress: Mutex::new(None), + } + } + + pub fn init(level: LevelFilter) -> Result<(), SetLoggerError> { + set_max_level(level); + set_boxed_logger(Box::new(CliLogger::new(level))) + } + + fn extended(&self, record: &Record) { + println!( + "[{}] {} {} ({}) {}", + chrono::Utc::now().format("%Y-%m-%d %H:%M:%S"), + record.level(), + // replace the 'progress' prefix if this function is invoked via 'progress!' + record + .target() + .replacen("progress", "crunchy_cli", 1) + .replacen("progress_end", "crunchy_cli", 1), + format!("{:?}", thread::current().id()) + .replace("ThreadId(", "") + .replace(')', ""), + record.args() + ) + } + + fn normal(&self, record: &Record) { + println!(":: {}", record.args()) + } + + fn error(&self, record: &Record) { + eprintln!(":: {}", record.args()) + } + + fn progress(&self, record: &Record, stop: bool) { + let mut progress_option = self.progress.lock().unwrap(); + if stop && progress_option.is_some() { + progress_option.take().unwrap().stop() + } else if let Some(p) = &*progress_option { + p.send(record); + } else { + *progress_option = Some(CliProgress::new(record)) + } + } +} + +macro_rules! tab_info { + ($($arg:tt)+) => { + if log::max_level() == log::LevelFilter::Debug { + info!($($arg)+) + } else { + info!("\t{}", format!($($arg)+)) + } + } +} +pub(crate) use tab_info; diff --git a/crunchy-cli-core/src/cli/login.rs b/crunchy-cli-core/src/cli/login.rs new file mode 100644 index 0000000..c6d876a --- /dev/null +++ b/crunchy-cli-core/src/cli/login.rs @@ -0,0 +1,39 @@ +use crate::utils::context::Context; +use crate::Execute; +use anyhow::bail; +use anyhow::Result; +use crunchyroll_rs::crunchyroll::SessionToken; +use std::fs; +use std::path::PathBuf; + +#[derive(Debug, clap::Parser)] +#[clap(about = "Save your login credentials persistent on disk")] +pub struct Login { + #[arg(help = "Remove your stored credentials (instead of save them)")] + #[arg(long)] + pub remove: bool, +} + +#[async_trait::async_trait(?Send)] +impl Execute for Login { + async fn execute(self, ctx: Context) -> Result<()> { + if let Some(login_file_path) = login_file_path() { + match ctx.crunchy.session_token().await { + SessionToken::RefreshToken(refresh_token) => Ok(fs::write( + login_file_path, + format!("refresh_token:{}", refresh_token), + )?), + SessionToken::EtpRt(etp_rt) => { + Ok(fs::write(login_file_path, format!("etp_rt:{}", etp_rt))?) + } + SessionToken::Anonymous => bail!("Anonymous login cannot be saved"), + } + } else { + bail!("Cannot find config path") + } + } +} + +pub fn login_file_path() -> Option { + dirs::config_dir().map(|config_dir| config_dir.join(".crunchy-cli-core")) +} diff --git a/crunchy-cli-core/src/cli/mod.rs b/crunchy-cli-core/src/cli/mod.rs new file mode 100644 index 0000000..c28f8e0 --- /dev/null +++ b/crunchy-cli-core/src/cli/mod.rs @@ -0,0 +1,5 @@ +pub mod archive; +pub mod download; +pub mod log; +pub mod login; +mod utils; diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs new file mode 100644 index 0000000..59816d1 --- /dev/null +++ b/crunchy-cli-core/src/cli/utils.rs @@ -0,0 +1,178 @@ +use crate::utils::context::Context; +use anyhow::Result; +use crunchyroll_rs::media::{Resolution, VariantData, VariantSegment}; +use isahc::AsyncReadResponseExt; +use log::{debug, LevelFilter}; +use std::borrow::{Borrow, BorrowMut}; +use std::collections::BTreeMap; +use std::io; +use std::io::Write; +use std::sync::{mpsc, Arc, Mutex}; +use std::time::Duration; +use tokio::task::JoinSet; + +pub fn find_resolution( + mut streaming_data: Vec, + resolution: &Resolution, +) -> Option { + streaming_data.sort_by(|a, b| a.resolution.width.cmp(&b.resolution.width).reverse()); + match resolution.height { + u64::MAX => Some(streaming_data.into_iter().next().unwrap()), + u64::MIN => Some(streaming_data.into_iter().last().unwrap()), + _ => streaming_data + .into_iter() + .find(|v| resolution.height == u64::MAX || v.resolution.height == resolution.height), + } +} + +pub async fn download_segments( + ctx: &Context, + writer: &mut impl Write, + message: Option, + segments: Vec, +) -> Result<()> { + let total_segments = segments.len(); + + let client = Arc::new(ctx.client.clone()); + let count = Arc::new(Mutex::new(0)); + let amount = Arc::new(Mutex::new(0)); + + // only print progress when log level is info + let output_handler = if log::max_level() == LevelFilter::Info { + let output_count = count.clone(); + let output_amount = amount.clone(); + Some(tokio::spawn(async move { + let sleep_time_ms = 100; + let iter_per_sec = 1000f64 / sleep_time_ms as f64; + + let mut bytes_start = 0f64; + let mut speed = 0f64; + let mut percentage = 0f64; + + while *output_count.lock().unwrap() < total_segments || percentage < 100f64 { + let tmp_amount = *output_amount.lock().unwrap() as f64; + + let tmp_speed = (tmp_amount - bytes_start) / 1024f64 / 1024f64; + if *output_count.lock().unwrap() < 3 { + speed = tmp_speed; + } else { + let (old_speed_ratio, new_speed_ratio) = if iter_per_sec <= 1f64 { + (0f64, 1f64) + } else { + (1f64 - (1f64 / iter_per_sec), (1f64 / iter_per_sec)) + }; + + // calculate the average download speed "smoother" + speed = (speed * old_speed_ratio) + (tmp_speed * new_speed_ratio); + } + + percentage = + (*output_count.lock().unwrap() as f64 / total_segments as f64) * 100f64; + + let size = terminal_size::terminal_size() + .unwrap_or((terminal_size::Width(60), terminal_size::Height(0))) + .0 + .0 as usize; + + let progress_available = size + - if let Some(msg) = &message { + 35 + msg.len() + } else { + 33 + }; + let progress_done_count = + (progress_available as f64 * (percentage / 100f64)).ceil() as usize; + let progress_to_do_count = progress_available - progress_done_count; + + let _ = write!( + io::stdout(), + "\r:: {}{:>5.1} MiB {:>5.2} MiB/s [{}{}] {:>3}%", + message.clone().map_or("".to_string(), |msg| msg + " "), + tmp_amount / 1024f64 / 1024f64, + speed * iter_per_sec, + "#".repeat(progress_done_count), + "-".repeat(progress_to_do_count), + percentage as usize + ); + + bytes_start = tmp_amount; + + tokio::time::sleep(Duration::from_millis(sleep_time_ms)).await; + } + println!() + })) + } else { + None + }; + + let cpus = num_cpus::get(); + let mut segs: Vec> = Vec::with_capacity(cpus); + for _ in 0..cpus { + segs.push(vec![]) + } + for (i, segment) in segments.into_iter().enumerate() { + segs[i - ((i / cpus) * cpus)].push(segment); + } + + let (sender, receiver) = mpsc::channel(); + + let mut join_set: JoinSet> = JoinSet::new(); + for num in 0..cpus { + let thread_client = client.clone(); + let thread_sender = sender.clone(); + let thread_segments = segs.remove(0); + let thread_amount = amount.clone(); + let thread_count = count.clone(); + join_set.spawn(async move { + for (i, segment) in thread_segments.into_iter().enumerate() { + let mut response = thread_client.get_async(&segment.url).await?; + let mut buf = response.bytes().await?.to_vec(); + + *thread_amount.lock().unwrap() += buf.len(); + + buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec(); + debug!( + "Downloaded and decrypted segment {} ({})", + num + (i * cpus), + segment.url + ); + thread_sender.send((num + (i * cpus), buf))?; + + *thread_count.lock().unwrap() += 1; + } + + Ok(()) + }); + } + + let mut data_pos = 0usize; + let mut buf: BTreeMap> = BTreeMap::new(); + loop { + // is always `Some` because `sender` does not get dropped when all threads are finished + let data = receiver.recv().unwrap(); + + if data_pos == data.0 { + writer.write_all(data.1.borrow())?; + data_pos += 1; + } else { + buf.insert(data.0, data.1); + } + while let Some(b) = buf.remove(&data_pos) { + writer.write_all(b.borrow())?; + data_pos += 1; + } + + if *count.lock().unwrap() >= total_segments { + break; + } + } + + while let Some(joined) = join_set.join_next().await { + joined?? + } + if let Some(handler) = output_handler { + handler.await? + } + + Ok(()) +} diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs new file mode 100644 index 0000000..faef927 --- /dev/null +++ b/crunchy-cli-core/src/lib.rs @@ -0,0 +1,196 @@ +use crate::cli::log::CliLogger; +use crate::utils::context::Context; +use crate::utils::locale::system_locale; +use crate::utils::log::progress; +use anyhow::bail; +use anyhow::Result; +use clap::{Parser, Subcommand}; +use crunchyroll_rs::{Crunchyroll, Locale}; +use log::{debug, error, info, LevelFilter}; +use std::{env, fs}; + +mod cli; +mod utils; + +pub use cli::{archive::Archive, download::Download, login::Login}; + +#[async_trait::async_trait(?Send)] +trait Execute { + async fn execute(self, ctx: Context) -> Result<()>; +} + +#[derive(Debug, Parser)] +#[clap(author, version, about)] +#[clap(name = "crunchy-cli")] +pub struct Cli { + #[clap(flatten)] + verbosity: Option, + + #[arg(help = "Overwrite the language in which results are returned. Default is your system language")] + #[arg(long)] + lang: Option, + + #[clap(flatten)] + login_method: LoginMethod, + + #[clap(subcommand)] + command: Command, +} + +#[derive(Debug, Subcommand)] +enum Command { + Archive(Archive), + Download(Download), + Login(Login), +} + +#[derive(Debug, Parser)] +struct Verbosity { + #[arg(help = "Verbose output")] + #[arg(short)] + v: bool, + + #[arg(help = "Quiet output. Does not print anything unless it's a error")] + #[arg(long_help = "Quiet output. Does not print anything unless it's a error. Can be helpful if you pipe the output to stdout")] + #[arg(short)] + q: bool, +} + +#[derive(Debug, Parser)] +struct LoginMethod { + #[arg(help = "Login with credentials (username or email and password)")] + #[arg(long_help = "Login with credentials (username or email and password). Must be provided as user:password")] + #[arg(long)] + credentials: Option, + #[arg(help = "Login with the etp-rt cookie")] + #[arg(long_help = "Login with the etp-rt cookie. This can be obtained when you login on crunchyroll.com and extract it from there")] + #[arg(long)] + etp_rt: Option, +} + +pub async fn cli_entrypoint() { + let cli: Cli = Cli::parse(); + + if let Some(verbosity) = &cli.verbosity { + if verbosity.v && verbosity.q { + eprintln!("Output cannot be verbose ('-v') and quiet ('-q') at the same time"); + std::process::exit(1) + } else if verbosity.v { + CliLogger::init(LevelFilter::Debug).unwrap() + } else if verbosity.q { + CliLogger::init(LevelFilter::Error).unwrap() + } + } else { + CliLogger::init(LevelFilter::Info).unwrap() + } + + debug!("cli input: {:?}", cli); + + let ctx = match create_ctx(&cli).await { + Ok(ctx) => ctx, + Err(e) => { + error!("{}", e); + std::process::exit(1) + } + }; + debug!("Created context"); + + ctrlc::set_handler(move || { + debug!("Ctrl-c detected"); + if let Ok(dir) = fs::read_dir(&env::temp_dir()) { + for file in dir.flatten() { + if file + .path() + .file_name() + .unwrap_or_default() + .to_str() + .unwrap_or_default() + .starts_with(".crunchy-cli_") + { + let result = fs::remove_file(file.path()); + debug!( + "Ctrl-c removed temporary file {} {}", + file.path().to_string_lossy(), + if result.is_ok() { + "successfully" + } else { + "not successfully" + } + ) + } + } + } + std::process::exit(1) + }) + .unwrap(); + debug!("Created ctrl-c handler"); + + let result = match cli.command { + Command::Archive(archive) => archive.execute(ctx).await, + Command::Download(download) => download.execute(ctx).await, + Command::Login(login) => { + if login.remove { + Ok(()) + } else { + login.execute(ctx).await + } + } + }; + if let Err(err) = result { + error!("{}", err); + std::process::exit(1) + } +} + +async fn create_ctx(cli: &Cli) -> Result { + let crunchy = crunchyroll_session(cli).await?; + // TODO: Use crunchy.client() when it's possible + // currently crunchy.client() has a cloudflare bypass built-in to access crunchyroll. the servers + // where crunchy stores their videos can't handle this bypass and simply refuses to connect + let client = isahc::HttpClient::new().unwrap(); + + Ok(Context { crunchy, client }) +} + +async fn crunchyroll_session(cli: &Cli) -> Result { + let mut builder = Crunchyroll::builder(); + builder.locale(cli.lang.clone().unwrap_or_else(system_locale)); + + let _progress_handler = progress!("Logging in"); + if cli.login_method.credentials.is_none() && cli.login_method.etp_rt.is_none() { + if let Some(login_file_path) = cli::login::login_file_path() { + if login_file_path.exists() { + let session = fs::read_to_string(login_file_path)?; + if let Some((token_type, token)) = session.split_once(':') { + match token_type { + "refresh_token" => { + return Ok(builder.login_with_refresh_token(token).await?) + } + "etp_rt" => return Ok(builder.login_with_etp_rt(token).await?), + _ => (), + } + } + bail!("Could not read stored session ('{}')", session) + } + } + bail!("Please use a login method ('--credentials' or '--etp_rt')") + } else if cli.login_method.credentials.is_some() && cli.login_method.etp_rt.is_some() { + bail!("Please use only one login method ('--credentials' or '--etp_rt')") + } + + let crunchy = if let Some(credentials) = &cli.login_method.credentials { + if let Some((user, password)) = credentials.split_once(':') { + builder.login_with_credentials(user, password).await? + } else { + bail!("Invalid credentials format. Please provide your credentials as user:password") + } + } else if let Some(etp_rt) = &cli.login_method.etp_rt { + builder.login_with_etp_rt(etp_rt).await? + } else { + bail!("should never happen") + }; + + info!("Logged in"); + + Ok(crunchy) +} diff --git a/crunchy-cli-core/src/utils/clap.rs b/crunchy-cli-core/src/utils/clap.rs new file mode 100644 index 0000000..4e27a5e --- /dev/null +++ b/crunchy-cli-core/src/utils/clap.rs @@ -0,0 +1,6 @@ +use crate::utils::parse::parse_resolution; +use crunchyroll_rs::media::Resolution; + +pub fn clap_parse_resolution(s: &str) -> Result { + parse_resolution(s.to_string()).map_err(|e| e.to_string()) +} diff --git a/crunchy-cli-core/src/utils/context.rs b/crunchy-cli-core/src/utils/context.rs new file mode 100644 index 0000000..6a2fd21 --- /dev/null +++ b/crunchy-cli-core/src/utils/context.rs @@ -0,0 +1,6 @@ +use crunchyroll_rs::Crunchyroll; + +pub struct Context { + pub crunchy: Crunchyroll, + pub client: isahc::HttpClient, +} diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs new file mode 100644 index 0000000..3dd7f74 --- /dev/null +++ b/crunchy-cli-core/src/utils/format.rs @@ -0,0 +1,77 @@ +use crunchyroll_rs::media::VariantData; +use crunchyroll_rs::{Episode, Locale, Media, Movie}; +use std::time::Duration; + +#[derive(Clone)] +pub struct Format { + pub id: String, + pub title: String, + pub description: String, + pub number: u32, + pub audio: Locale, + + pub duration: Duration, + pub stream: VariantData, + + pub series_id: String, + pub series_name: String, + + pub season_id: String, + pub season_title: String, + pub season_number: u32, +} + +impl Format { + pub fn new_from_episode(episode: Media, stream: VariantData) -> Self { + Self { + id: episode.id, + title: episode.title, + description: episode.description, + number: episode.metadata.episode_number, + audio: episode.metadata.audio_locale, + + duration: episode.metadata.duration.to_std().unwrap(), + stream, + + series_id: episode.metadata.series_id, + series_name: episode.metadata.series_title, + + season_id: episode.metadata.season_id, + season_title: episode.metadata.season_title, + season_number: episode.metadata.season_number, + } + } + + pub fn new_from_movie(movie: Media, stream: VariantData) -> Self { + Self { + id: movie.id, + title: movie.title, + description: movie.description, + number: 1, + audio: Locale::ja_JP, + + duration: movie.metadata.duration.to_std().unwrap(), + stream, + + series_id: movie.metadata.movie_listing_id.clone(), + series_name: movie.metadata.movie_listing_title.clone(), + + season_id: movie.metadata.movie_listing_id, + season_title: movie.metadata.movie_listing_title, + season_number: 1, + } + } +} + +pub fn format_string(s: String, format: &Format) -> String { + s.replace("{title}", &format.title) + .replace("{series_name}", &format.series_name) + .replace("{season_name}", &format.season_title) + .replace("{audio}", &format.audio.to_string()) + .replace("{resolution}", &format.stream.resolution.to_string()) + .replace("{season_number}", &format.season_number.to_string()) + .replace("{episode_number}", &format.number.to_string()) + .replace("{series_id}", &format.series_id) + .replace("{season_id}", &format.season_id) + .replace("{episode_id}", &format.id) +} diff --git a/crunchy-cli-core/src/utils/locale.rs b/crunchy-cli-core/src/utils/locale.rs new file mode 100644 index 0000000..300204d --- /dev/null +++ b/crunchy-cli-core/src/utils/locale.rs @@ -0,0 +1,15 @@ +use crunchyroll_rs::Locale; + +/// Return the locale of the system. +pub fn system_locale() -> Locale { + if let Some(system_locale) = sys_locale::get_locale() { + let locale = Locale::from(system_locale); + if let Locale::Custom(_) = locale { + Locale::en_US + } else { + locale + } + } else { + Locale::en_US + } +} diff --git a/crunchy-cli-core/src/utils/log.rs b/crunchy-cli-core/src/utils/log.rs new file mode 100644 index 0000000..b9fa939 --- /dev/null +++ b/crunchy-cli-core/src/utils/log.rs @@ -0,0 +1,19 @@ +use log::info; + +pub struct ProgressHandler; + +impl Drop for ProgressHandler { + fn drop(&mut self) { + info!(target: "progress_end", "") + } +} + +macro_rules! progress { + ($($arg:tt)+) => { + { + log::info!(target: "progress", $($arg)+); + $crate::utils::log::ProgressHandler{} + } + } +} +pub(crate) use progress; diff --git a/crunchy-cli-core/src/utils/mod.rs b/crunchy-cli-core/src/utils/mod.rs new file mode 100644 index 0000000..5f7a5d2 --- /dev/null +++ b/crunchy-cli-core/src/utils/mod.rs @@ -0,0 +1,8 @@ +pub mod clap; +pub mod context; +pub mod format; +pub mod locale; +pub mod log; +pub mod os; +pub mod parse; +pub mod sort; diff --git a/crunchy-cli-core/src/utils/os.rs b/crunchy-cli-core/src/utils/os.rs new file mode 100644 index 0000000..317381d --- /dev/null +++ b/crunchy-cli-core/src/utils/os.rs @@ -0,0 +1,52 @@ +use log::debug; +use std::io::ErrorKind; +use std::path::PathBuf; +use std::process::Command; +use std::{env, io}; +use tempfile::{Builder, NamedTempFile}; + +pub fn has_ffmpeg() -> bool { + if let Err(e) = Command::new("ffmpeg").spawn() { + if ErrorKind::NotFound != e.kind() { + debug!( + "unknown error occurred while checking if ffmpeg exists: {}", + e.kind() + ) + } + false + } else { + true + } +} + +/// Any tempfiles should be created with this function. The prefix and directory of every file +/// created with this method stays the same which is helpful to query all existing tempfiles and +/// e.g. remove them in a case of ctrl-c. Having one function also good to prevent mistakes like +/// setting the wrong prefix if done manually. +pub fn tempfile>(suffix: S) -> io::Result { + let tempfile = Builder::default() + .prefix(".crunchy-cli_") + .suffix(suffix.as_ref()) + .tempfile_in(&env::temp_dir())?; + debug!( + "Created temporary file: {}", + tempfile.path().to_string_lossy() + ); + Ok(tempfile) +} + +/// Check if the given path exists and rename it until the new (renamed) file does not exist. +pub fn free_file(mut path: PathBuf) -> (PathBuf, bool) { + let mut i = 0; + while path.exists() { + i += 1; + + let ext = path.extension().unwrap().to_str().unwrap(); + let mut filename = path.file_name().unwrap().to_str().unwrap(); + + filename = &filename[0..filename.len() - ext.len() - 1]; + + path.set_file_name(format!("{} ({}).{}", filename, i, ext)) + } + (path, i != 0) +} diff --git a/crunchy-cli-core/src/utils/parse.rs b/crunchy-cli-core/src/utils/parse.rs new file mode 100644 index 0000000..fbfea35 --- /dev/null +++ b/crunchy-cli-core/src/utils/parse.rs @@ -0,0 +1,170 @@ +use anyhow::{anyhow, bail, Result}; +use crunchyroll_rs::media::Resolution; +use crunchyroll_rs::{Crunchyroll, MediaCollection, UrlType}; +use log::debug; +use regex::Regex; + +/// Define a filter, based on season and episode number to filter episodes / movies. +/// If a struct instance equals the [`Default::default()`] it's considered that no filter is applied. +/// If `from_*` is [`None`] they're set to [`u32::MIN`]. +/// If `to_*` is [`None`] they're set to [`u32::MAX`]. +#[derive(Debug)] +pub struct InnerUrlFilter { + from_episode: Option, + to_episode: Option, + from_season: Option, + to_season: Option, +} + +#[derive(Debug, Default)] +pub struct UrlFilter { + inner: Vec, +} + +impl UrlFilter { + pub fn is_season_valid(&self, season: u32) -> bool { + self.inner.iter().any(|f| { + let from_season = f.from_season.unwrap_or(u32::MIN); + let to_season = f.to_season.unwrap_or(u32::MAX); + + season >= from_season && season <= to_season + }) + } + + pub fn is_episode_valid(&self, episode: u32, season: u32) -> bool { + self.inner.iter().any(|f| { + let from_episode = f.from_episode.unwrap_or(u32::MIN); + let to_episode = f.to_episode.unwrap_or(u32::MAX); + let from_season = f.from_season.unwrap_or(u32::MIN); + let to_season = f.to_season.unwrap_or(u32::MAX); + + episode >= from_episode + && episode <= to_episode + && season >= from_season + && season <= to_season + }) + } +} + +/// Parse a url and return all [`crunchyroll_rs::Media`] & +/// [`crunchyroll_rs::Media`] which could be related to it. +/// +/// The `with_filter` arguments says if filtering should be enabled for the url. Filtering is a +/// specific pattern at the end of the url which declares which parts of the url content should be +/// returned / filtered (out). _This only works if the url points to a series_. +/// +/// Examples how filtering works: +/// - `...[E5]` - Download the fifth episode. +/// - `...[S1]` - Download the full first season. +/// - `...[-S2]` - Download all seasons up to and including season 2. +/// - `...[S3E4-]` - Download all episodes from and including season 3, episode 4. +/// - `...[S1E4-S3]` - Download all episodes from and including season 1, episode 4, until andincluding season 3. +/// - `...[S3,S5]` - Download episode 3 and 5. +/// - `...[S1-S3,S4E2-S4E6]` - Download season 1 to 3 and episode 2 to episode 6 of season 4. + +/// In practice, it would look like this: `https://beta.crunchyroll.com/series/12345678/example[S1E5-S3E2]`. +pub async fn parse_url( + crunchy: &Crunchyroll, + mut url: String, + with_filter: bool, +) -> Result<(MediaCollection, UrlFilter)> { + let url_filter = if with_filter { + debug!("Url may contain filters"); + + let open_index = url.rfind('[').unwrap_or(0); + let close_index = url.rfind(']').unwrap_or(0); + + let filter = if open_index < close_index { + let filter = url.as_str()[open_index + 1..close_index].to_string(); + url = url.as_str()[0..open_index].to_string(); + filter + } else { + "".to_string() + }; + + let filter_regex = Regex::new(r"((S(?P\d+))?(E(?P\d+))?)(((?P-)((S(?P\d+))?(E(?P\d+))?))?)(,|$)").unwrap(); + + let mut filters = vec![]; + + for capture in filter_regex.captures_iter(&filter) { + let dash = capture.name("dash").is_some(); + let from_episode = capture + .name("from_episode") + .map_or(anyhow::Ok(None), |fe| Ok(Some(fe.as_str().parse()?)))?; + let to_episode = capture + .name("to_episode") + .map_or(anyhow::Ok(if dash { None } else { from_episode }), |te| { + Ok(Some(te.as_str().parse()?)) + })?; + let from_season = capture + .name("from_season") + .map_or(anyhow::Ok(None), |fs| Ok(Some(fs.as_str().parse()?)))?; + let to_season = capture + .name("to_season") + .map_or(anyhow::Ok(if dash { None } else { from_season }), |ts| { + Ok(Some(ts.as_str().parse()?)) + })?; + + filters.push(InnerUrlFilter { + from_episode, + to_episode, + from_season, + to_season, + }) + } + + let url_filter = UrlFilter { inner: filters }; + + debug!("Url filter: {:?}", url_filter); + + url_filter + } else { + UrlFilter::default() + }; + + let parsed_url = crunchyroll_rs::parse_url(url).map_or(Err(anyhow!("Invalid url")), Ok)?; + debug!("Url type: {:?}", parsed_url); + let media_collection = match parsed_url { + UrlType::Series(id) | UrlType::MovieListing(id) | UrlType::EpisodeOrMovie(id) => { + crunchy.media_collection_from_id(id).await? + } + }; + + Ok((media_collection, url_filter)) +} + +/// Parse a resolution given as a [`String`] to a [`crunchyroll_rs::media::Resolution`]. +pub fn parse_resolution(mut resolution: String) -> Result { + resolution = resolution.to_lowercase(); + + if resolution == "best" { + Ok(Resolution { + width: u64::MAX, + height: u64::MAX, + }) + } else if resolution == "worst" { + Ok(Resolution { + width: u64::MIN, + height: u64::MIN, + }) + } else if resolution.ends_with('p') { + let without_p = resolution.as_str()[0..resolution.len() - 2] + .parse() + .map_err(|_| anyhow!("Could not parse resolution"))?; + Ok(Resolution { + width: without_p * 16 / 9, + height: without_p, + }) + } else if let Some((w, h)) = resolution.split_once('x') { + Ok(Resolution { + width: w + .parse() + .map_err(|_| anyhow!("Could not parse resolution"))?, + height: h + .parse() + .map_err(|_| anyhow!("Could not parse resolution"))?, + }) + } else { + bail!("Could not parse resolution") + } +} diff --git a/crunchy-cli-core/src/utils/sort.rs b/crunchy-cli-core/src/utils/sort.rs new file mode 100644 index 0000000..089fe18 --- /dev/null +++ b/crunchy-cli-core/src/utils/sort.rs @@ -0,0 +1,47 @@ +use crate::utils::format::Format; +use crunchyroll_rs::{Media, Season}; +use std::collections::BTreeMap; + +/// Sort seasons after their season number. Crunchyroll may have multiple seasons for one season +/// number. They generally store different language in individual seasons with the same season number. +/// E.g. series X has one official season but crunchy has translations for it in 3 different languages +/// so there exist 3 different "seasons" on Crunchyroll which are actual the same season but with +/// different audio. +pub fn sort_seasons_after_number(seasons: Vec>) -> Vec>> { + let mut as_map = BTreeMap::new(); + + for season in seasons { + as_map + .entry(season.metadata.season_number) + .or_insert_with(Vec::new); + as_map + .get_mut(&season.metadata.season_number) + .unwrap() + .push(season) + } + + as_map.into_values().collect() +} + +/// Sort formats after their seasons and episodes (inside it) ascending. Make sure to have only +/// episodes from one series and in one language as argument since the function does not handle those +/// differences which could then lead to a semi messed up result. +pub fn sort_formats_after_seasons(formats: Vec) -> Vec> { + let mut as_map = BTreeMap::new(); + + for format in formats { + as_map.entry(format.season_number).or_insert_with(Vec::new); + as_map.get_mut(&format.season_number).unwrap().push(format); + } + + let mut sorted = as_map + .into_iter() + .map(|(_, mut values)| { + values.sort_by(|a, b| a.number.cmp(&b.number)); + values + }) + .collect::>>(); + sorted.sort_by(|a, b| a[0].series_id.cmp(&b[0].series_id)); + + sorted +} diff --git a/crunchy-cli.1 b/crunchy-cli.1 deleted file mode 100644 index 91ba30c..0000000 --- a/crunchy-cli.1 +++ /dev/null @@ -1,219 +0,0 @@ -.TH crunchy-cli 1 "27 June 2022" "crunchy-cli" "Crunchyroll Cli Client" - -.SH NAME -crunchy-cli - A cli for downloading videos and entire series from crunchyroll. - -.SH SYNOPSIS -crunchy-cli [\fB-h\fR] [\fB-p\fR \fIPROXY\fR] [\fB-q\fR] [\fB-v\fR] -.br -crunchy-cli help -.br -crunchy-cli login [\fB--persistent\fR] [\fB--session-id\fR \fISESSION_ID\fR] [\fIusername\fR, \fIpassword\fR] -.br -crunchy-cli download [\fB-a\fR \fIAUDIO\fR] [\fB-s\fR \fISUBTITLE\fR] [\fB-d\fR \fIDIRECTORY\fR] [\fB-o\fR \fIOUTPUT\fR] [\fB-r\fR \fIRESOLUTION\fR] [\fB-g\fR \fIGOROUTINES\fR] \fIURLs...\fR -.br -crunchy-cli archive [\fB-l\fR \fILANGUAGE\fR] [\fB-d\fR \fIDIRECTORY\fR] [\fB-o\fR \fIOUTPUT\fR] [\fB-m\fR \fIMERGE BEHAVIOR\fR] [\fB-c\fR \fICOMPRESS\fR] [\fB-r\fR \fIRESOLUTION\fR] [\fB-g\fR \fIGOROUTINES\fR] \fIURLs...\fR -.br -crunchy-cli update [\fB-i\fR \fIINSTALL\fR] - -.SH DESCRIPTION -.TP -With \fBcrunchy-cli\fR you can easily download video and series from crunchyroll. -.TP - -Note that you need an \fBcrunchyroll premium\fR account in order to use this tool! - -.SH GENERAL OPTIONS -.TP -This options can be passed to every action. -.TP - -\fB-h, --help\fR -Shows help. -.TP - -\fB-p, --proxy PROXY\fR -Sets a proxy through which all traffic will be routed. -.TP - -\fB-q, --quiet\fR -Disables all output. -.TP - -\fB-v, --verbose\fR -Shows verbose output. -.TP - -\fB--version\fR -Shows the current cli version. - -.SH LOGIN COMMAND -This command logs in to crunchyroll and stores the session id or credentials on the drive. This needs to be done before calling other commands since they need a valid login to operate. -.TP - -\fB--persistent\fR -Stores the given credentials permanent on the drive. The *nix path for it is $HOME/.config/crunchy. -.br -NOTE: The credentials are stored in plain text and if you not use \fB--session-id\fR your credentials are used (if you not use the \fB--persistent\fR flag only a session id gets stored regardless if you login with username/password or a session id). -.TP - -\fB--session-id SESSION_ID\fR -Login via a session id (which can be extracted from a crunchyroll browser cookie) instead of using username and password. - -.SH DOWNLOAD COMMAND -A command to simply download videos. The output file is stored as a \fI.ts\fR file. \fIffmpeg\fR has to be installed if you want to change the Format the videos are stored in. -.TP - -\fB-a, --audio AUDIO\fR -Forces to download videos with the given audio locale. If no video with this audio locale is available, nothing will be downloaded. Available locales are: ja-JP, en-US, es-419, es-LA, es-ES, fr-FR, pt-PT, pt-BR, it-IT, de-DE, ru-RU, ar-SA, ar-ME. -.TP - -\fB-s, --subtitle SUBTITLE\fR -Forces to download the videos with subtitles in the given locale / language. If no video with this subtitle locale is available, nothing will be downloaded. Available locales are: ja-JP, en-US, es-419, es-LA, es-ES, fr-FR, pt-PT, pt-BR, it-IT, de-DE, ru-RU, ar-SA, ar-ME. -.TP - -\fB-d, --directory DIRECTORY\fR -The directory to download all files to. -.TP - -\fB-o, --output OUTPUT\fR -Name of the output file. Formatting is also supported, so if the name contains one or more of the following things, they will get replaced. - {title} Âģ Title of the video. - {series_name} Âģ Name of the series. - {season_name} Âģ Name of the season. - {season_number} Âģ Number of the season. - {episode_number} Âģ Number of the episode. - {resolution} Âģ Resolution of the video. - {fps} Âģ Frame Rate of the video. - {audio} Âģ Audio locale of the video. - {subtitle} Âģ Subtitle locale of the video. -.TP - -\fB-r, resolution RESOLUTION\fR -The video resolution. Can either be specified via the pixels (e.g. 1920x1080), the abbreviation for pixels (e.g. 1080p) or "common-use" words (e.g. best). - Available pixels: 1920x1080, 1280x720, 640x480, 480x360, 426x240. - Available abbreviations: 1080p, 720p, 480p, 360p, 240p. - Available common-use words: best (best available resolution), worst (worst available resolution). -.TP - -\fB-g, --goroutines GOROUTINES\fR -Sets the number of parallel downloads for the segments the final video is made of. Default is the number of cores the computer has. - -.SH ARCHIVE COMMAND -This command behaves like \fBdownload\fR besides the fact that it requires \fIffmpeg\fR and stores the output only to .mkv files. -.TP - -\fB-l, --language LANGUAGE\fR -Audio locales which should be downloaded. Can be used multiple times. Available locales are: ja-JP, en-US, es-419, es-LA, es-ES, fr-FR, pt-PT, pt-BR, it-IT, de-DE, ru-RU, ar-SA, ar-ME. -.TP - -\fB-s, --sublang LANGUAGE\fR -Subtitle languages to use, by default all are included. Can be used multiple times. Available locales are: ja-JP, en-US, es-419, es-LA, es-ES, fr-FR, pt-PT, pt-BR, it-IT, de-DE, ru-RU, ar-SA, ar-ME. -.TP - -\fB-d, --directory DIRECTORY\fR -The directory to download all files to. -.TP - -\fB-o, --output OUTPUT\fR -Name of the output file. Formatting is also supported, so if the name contains one or more of the following things, they will get replaced. - {title} Âģ Title of the video. - {series_name} Âģ Name of the series. - {season_name} Âģ Name of the season. - {season_number} Âģ Number of the season. - {episode_number} Âģ Number of the episode. - {resolution} Âģ Resolution of the video. - {fps} Âģ Frame Rate of the video. - {audio} Âģ Audio locale of the video. - {subtitle} Âģ Subtitle locale of the video. -.TP - -\fB-m, --merge MERGE BEHAVIOR\fR -Sets the behavior of the stream merging. Valid behaviors are 'auto', 'audio', 'video'. \fB--audio\fR stores one video and only the audio of all other languages, \fBvideo\fR stores all videos of the given languages and their audio, \fBauto\fR (which is the default) only behaves like video if the length of two videos are different (and only for the two videos), else like audio. -.TP - -\fB-c, --compress COMPRESS\fR -If is set, all output will be compresses into an archive (every url generates a new one). This flag sets the name of the compressed output file. The file ending specifies the compression algorithm. The following algorithms are supported: gzip, tar, zip. -Just like \fB--output\fR the name can be formatted. But the only option available here is \fI{series_name}\fR. -.TP - -\fB-r, resolution RESOLUTION\fR -The video resolution. Can either be specified via the pixels (e.g. 1920x1080), the abbreviation for pixels (e.g. 1080p) or "common-use" words (e.g. best). - Available pixels: 1920x1080, 1280x720, 640x480, 480x360, 426x240. - Available abbreviations: 1080p, 720p, 480p, 360p, 240p. - Available common-use words: best (best available resolution), worst (worst available resolution). -.TP - -\fB-g, --goroutines GOROUTINES\fR -Sets the number of parallel downloads for the segments the final video is made of. Default is the number of cores the computer has. - -.SH UPDATE COMMAND -Checks if a newer version is available. -.TP - -\fB-i, --install INSTALL\fR -If given, the command tries to update the executable with the newer version (if a newer is available). - -.SH URL OPTIONS -If you want to download only specific episode of a series, you could either pass every single episode url to the downloader (which is fine for 1 - 3 episodes) or use filtering. -It works pretty simple, just put a specific pattern surrounded by square brackets at the end of the url from the anime you want to download. A season and / or episode as well as a range from where to where episodes should be downloaded can be specified. -Use the list below to get a better overview what is possible - ...[E5] - Download the fifth episode. - ...[S1] - Download the full first season. - ...[-S2] - Download all seasons up to and including season 2. - ...[S3E4-] - Download all episodes from and including season 3, episode 4. - ...[S1E4-S3] - Download all episodes from and including season 1, episode 4, until and including season 3. - -In practise, it would look like this: \fIhttps://beta.crunchyroll.com/series/12345678/example[S1E5-S3E2]\fR. - -The \fBS\fR, followed by the number indicates the season number, \fBE\fR, followed by the number indicates an episode number. It doesn't matter if \fBS\fR, \fBE\fR or both are missing. Theoretically \fB[-]\fR is a valid pattern too. Note that \fBS\fR must always stay before \fBE\fR when used. - -.SH EXAMPLES -Login via crunchyroll account email and password. -.br -$ crunchy-cli login user@example.com 12345678 - -Download a episode normally. Your system locale will be used for the video's audio. -.br -$ crunchy-cli download https://beta.crunchyroll.com/watch/GRDKJZ81Y/alone-and-lonesome - -Download a episode with 720p and name it to 'darling.mp4'. Note that you need \fBffmpeg\fR to save files which do not have '.ts' as file extension. -.br -$ crunchy-cli download -o "darling.mp4" -r 720p https://beta.crunchyroll.com/watch/GRDKJZ81Y/alone-and-lonesome - -Download a episode with japanese audio and american subtitles. -.br -$ crunchy-cli download -a ja-JP -s en-US https://beta.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx[E3-E5] - -Stores the episode in a .mkv file. -.br -$ crunchy-cli archive https://beta.crunchyroll.com/watch/GRDKJZ81Y/alone-and-lonesome - -Downloads the first two episode of Darling in the FranXX and stores it compressed in a file. -.br -$ crunchy-cli archive -c "ditf.tar.gz" https://beta.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx[E1-E2] - -.SH BUGS -If you notice any bug or want an enhancement, feel free to create a new issue or pull request in the GitHub repository. - -.SH AUTHOR -Crunchy Labs Maintainers -.br -Source: https://github.com/crunchy-labs/crunchy-cli - -.SH COPYRIGHT -Copyright (C) 2022 Crunchy Labs Maintainers - -This program is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License -as published by the Free Software Foundation, either version 3 -of the License, or (at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - diff --git a/go.mod b/go.mod deleted file mode 100644 index 162e7a1..0000000 --- a/go.mod +++ /dev/null @@ -1,14 +0,0 @@ -module github.com/crunchy-labs/crunchy-cli - -go 1.19 - -require ( - github.com/crunchy-labs/crunchyroll-go/v3 v3.0.2 - github.com/grafov/m3u8 v0.11.1 - github.com/spf13/cobra v1.5.0 -) - -require ( - github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect -) diff --git a/go.sum b/go.sum deleted file mode 100644 index 51fe7a6..0000000 --- a/go.sum +++ /dev/null @@ -1,14 +0,0 @@ -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/crunchy-labs/crunchyroll-go/v3 v3.0.2 h1:PG5++Gje126/xRtzZwCowoFU1Dl3qKzFjd3lWhVXoso= -github.com/crunchy-labs/crunchyroll-go/v3 v3.0.2/go.mod h1:SjTQD3IX7Z+MLsMSd2fP5ttsJ4KtpXY6r08bHLwrOLM= -github.com/grafov/m3u8 v0.11.1 h1:igZ7EBIB2IAsPPazKwRKdbhxcoBKO3lO1UY57PZDeNA= -github.com/grafov/m3u8 v0.11.1/go.mod h1:nqzOkfBiZJENr52zTVd/Dcl03yzphIMbJqkXGu+u080= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= -github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/main.go b/main.go deleted file mode 100644 index 72ec83b..0000000 --- a/main.go +++ /dev/null @@ -1,9 +0,0 @@ -package main - -import ( - "github.com/crunchy-labs/crunchy-cli/cli" -) - -func main() { - cli.Execute() -} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..5e1d695 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,4 @@ +#[tokio::main] +async fn main() { + crunchy_cli_core::cli_entrypoint().await +} diff --git a/utils/extract.go b/utils/extract.go deleted file mode 100644 index fa2d650..0000000 --- a/utils/extract.go +++ /dev/null @@ -1,99 +0,0 @@ -package utils - -import ( - "fmt" - "github.com/crunchy-labs/crunchyroll-go/v3" - "github.com/crunchy-labs/crunchyroll-go/v3/utils" - "regexp" - "strconv" - "strings" -) - -var urlFilter = regexp.MustCompile(`(S(\d+))?(E(\d+))?((-)(S(\d+))?(E(\d+))?)?(,|$)`) - -func ExtractEpisodes(url string, locales ...crunchyroll.LOCALE) ([][]*crunchyroll.Episode, error) { - var matches [][]string - - lastOpen := strings.LastIndex(url, "[") - if strings.HasSuffix(url, "]") && lastOpen != -1 && lastOpen < len(url) { - matches = urlFilter.FindAllStringSubmatch(url[lastOpen+1:len(url)-1], -1) - - var all string - for _, match := range matches { - all += match[0] - } - if all != url[lastOpen+1:len(url)-1] { - return nil, fmt.Errorf("invalid episode filter") - } - url = url[:lastOpen] - } - - episodes, err := Crunchy.ExtractEpisodesFromUrl(url, locales...) - if err != nil { - return nil, fmt.Errorf("failed to get episodes: %v", err) - } - - if len(episodes) == 0 { - return nil, fmt.Errorf("no episodes found") - } - - if matches != nil { - for _, match := range matches { - fromSeason, fromEpisode, toSeason, toEpisode := -1, -1, -1, -1 - if match[2] != "" { - fromSeason, _ = strconv.Atoi(match[2]) - } - if match[4] != "" { - fromEpisode, _ = strconv.Atoi(match[4]) - } - if match[8] != "" { - toSeason, _ = strconv.Atoi(match[8]) - } - if match[10] != "" { - toEpisode, _ = strconv.Atoi(match[10]) - } - - if match[6] != "-" { - toSeason = fromSeason - toEpisode = fromEpisode - } - - tmpEps := make([]*crunchyroll.Episode, 0) - for _, episode := range episodes { - if fromSeason != -1 && (episode.SeasonNumber < fromSeason || (fromEpisode != -1 && episode.EpisodeNumber < fromEpisode)) { - continue - } else if fromSeason == -1 && fromEpisode != -1 && episode.EpisodeNumber < fromEpisode { - continue - } else if toSeason != -1 && (episode.SeasonNumber > toSeason || (toEpisode != -1 && episode.EpisodeNumber > toEpisode)) { - continue - } else if toSeason == -1 && toEpisode != -1 && episode.EpisodeNumber > toEpisode { - continue - } else { - tmpEps = append(tmpEps, episode) - } - } - - if len(tmpEps) == 0 { - return nil, fmt.Errorf("no episodes are matching the given filter") - } - - episodes = tmpEps - } - } - - var final [][]*crunchyroll.Episode - if len(locales) > 0 { - final = make([][]*crunchyroll.Episode, len(locales)) - localeSorted, err := utils.SortEpisodesByAudio(episodes) - if err != nil { - return nil, fmt.Errorf("failed to get audio locale: %v", err) - } - for i, locale := range locales { - final[i] = append(final[i], localeSorted[locale]...) - } - } else { - final = [][]*crunchyroll.Episode{episodes} - } - - return final, nil -} diff --git a/utils/file.go b/utils/file.go deleted file mode 100644 index 183b06f..0000000 --- a/utils/file.go +++ /dev/null @@ -1,49 +0,0 @@ -package utils - -import ( - "fmt" - "os" - "path/filepath" - "runtime" - "strings" -) - -func FreeFileName(filename string) (string, bool) { - ext := filepath.Ext(filename) - base := strings.TrimSuffix(filename, ext) - // checks if a .tar stands before the "actual" file ending - if extraExt := filepath.Ext(base); extraExt == ".tar" { - ext = extraExt + ext - base = strings.TrimSuffix(base, extraExt) - } - j := 0 - for ; ; j++ { - if _, stat := os.Stat(filename); stat != nil && !os.IsExist(stat) { - break - } - filename = fmt.Sprintf("%s (%d)%s", base, j+1, ext) - } - return filename, j != 0 -} - -func GenerateFilename(name, directory string) string { - if runtime.GOOS != "windows" { - for _, char := range []string{"/"} { - name = strings.ReplaceAll(name, char, "") - } - Log.Debug("Replaced invalid characters (not windows)") - } else { - // ahh i love windows :))) - for _, char := range []string{"\\", "<", ">", ":", "\"", "/", "|", "?", "*"} { - name = strings.ReplaceAll(name, char, "") - } - Log.Debug("Replaced invalid characters (windows)") - } - - filename, changed := FreeFileName(filepath.Join(directory, name)) - if changed { - Log.Debug("File `%s` already exists, changing name to `%s`", filepath.Base(name), filepath.Base(filename)) - } - - return filename -} diff --git a/utils/format.go b/utils/format.go deleted file mode 100644 index 6d40142..0000000 --- a/utils/format.go +++ /dev/null @@ -1,63 +0,0 @@ -package utils - -import ( - "fmt" - "github.com/crunchy-labs/crunchyroll-go/v3" - "reflect" - "runtime" - "strings" -) - -type FormatInformation struct { - // the Format to download - Format *crunchyroll.Format - - // additional formats which are only used by archive.go - AdditionalFormats []*crunchyroll.Format - - Title string `json:"title"` - SeriesName string `json:"series_name"` - SeasonName string `json:"season_name"` - SeasonNumber int `json:"season_number"` - EpisodeNumber int `json:"episode_number"` - Resolution string `json:"resolution"` - FPS float64 `json:"fps"` - Audio crunchyroll.LOCALE `json:"audio"` - Subtitle crunchyroll.LOCALE `json:"subtitle"` -} - -func (fi FormatInformation) FormatString(source string) string { - fields := reflect.TypeOf(fi) - values := reflect.ValueOf(fi) - - for i := 0; i < fields.NumField(); i++ { - var valueAsString string - switch value := values.Field(i); value.Kind() { - case reflect.String: - valueAsString = value.String() - case reflect.Int: - valueAsString = fmt.Sprintf("%02d", value.Int()) - case reflect.Float64: - valueAsString = fmt.Sprintf("%.2f", value.Float()) - case reflect.Bool: - valueAsString = fields.Field(i).Tag.Get("json") - if !value.Bool() { - valueAsString = "no " + valueAsString - } - } - - if runtime.GOOS != "windows" { - for _, char := range []string{"/"} { - valueAsString = strings.ReplaceAll(valueAsString, char, "") - } - } else { - for _, char := range []string{"\\", "<", ">", ":", "\"", "/", "|", "?", "*"} { - valueAsString = strings.ReplaceAll(valueAsString, char, "") - } - } - - source = strings.ReplaceAll(source, "{"+fields.Field(i).Tag.Get("json")+"}", valueAsString) - } - - return source -} diff --git a/utils/http.go b/utils/http.go deleted file mode 100644 index f572bd9..0000000 --- a/utils/http.go +++ /dev/null @@ -1,51 +0,0 @@ -package utils - -import ( - "net/http" - "net/url" - "time" -) - -type headerRoundTripper struct { - http.RoundTripper - header map[string]string -} - -func (rht headerRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { - resp, err := rht.RoundTripper.RoundTrip(r) - if err != nil { - return nil, err - } - for k, v := range rht.header { - resp.Header.Set(k, v) - } - return resp, nil -} - -func CreateOrDefaultClient(proxy, useragent string) (*http.Client, error) { - if proxy == "" { - return http.DefaultClient, nil - } else { - proxyURL, err := url.Parse(proxy) - if err != nil { - return nil, err - } - - var rt http.RoundTripper = &http.Transport{ - DisableCompression: true, - Proxy: http.ProxyURL(proxyURL), - } - if useragent != "" { - rt = headerRoundTripper{ - RoundTripper: rt, - header: map[string]string{"User-Agent": useragent}, - } - } - - client := &http.Client{ - Transport: rt, - Timeout: 30 * time.Second, - } - return client, nil - } -} diff --git a/utils/locale.go b/utils/locale.go deleted file mode 100644 index 9769940..0000000 --- a/utils/locale.go +++ /dev/null @@ -1,59 +0,0 @@ -package utils - -import ( - "fmt" - "github.com/crunchy-labs/crunchyroll-go/v3" - "github.com/crunchy-labs/crunchyroll-go/v3/utils" - "os" - "os/exec" - "runtime" - "sort" - "strings" -) - -// SystemLocale receives the system locale -// https://stackoverflow.com/questions/51829386/golang-get-system-language/51831590#51831590 -func SystemLocale(verbose bool) crunchyroll.LOCALE { - if runtime.GOOS != "windows" { - if lang, ok := os.LookupEnv("LANG"); ok { - var l crunchyroll.LOCALE - if preSuffix := strings.Split(strings.Split(lang, ".")[0], "_"); len(preSuffix) == 1 { - l = crunchyroll.LOCALE(preSuffix[0]) - } else { - prefix := strings.Split(lang, "_")[0] - l = crunchyroll.LOCALE(fmt.Sprintf("%s-%s", prefix, preSuffix[1])) - } - if !utils.ValidateLocale(l) { - if verbose { - Log.Err("%s is not a supported locale, using %s as fallback", l, crunchyroll.US) - } - l = crunchyroll.US - } - return l - } - } else { - cmd := exec.Command("powershell", "Get-Culture | select -exp Name") - if output, err := cmd.Output(); err == nil { - l := crunchyroll.LOCALE(strings.Trim(string(output), "\r\n")) - if !utils.ValidateLocale(l) { - if verbose { - Log.Err("%s is not a supported locale, using %s as fallback", l, crunchyroll.US) - } - l = crunchyroll.US - } - return l - } - } - if verbose { - Log.Err("Failed to get locale, using %s", crunchyroll.US) - } - return crunchyroll.US -} - -func LocalesAsStrings() (locales []string) { - for _, locale := range utils.AllLocales { - locales = append(locales, string(locale)) - } - sort.Strings(locales) - return -} diff --git a/utils/logger.go b/utils/logger.go deleted file mode 100644 index 27ea344..0000000 --- a/utils/logger.go +++ /dev/null @@ -1,12 +0,0 @@ -package utils - -type Logger interface { - IsDev() bool - Debug(format string, v ...any) - Info(format string, v ...any) - Warn(format string, v ...any) - Err(format string, v ...any) - Empty() - SetProcess(format string, v ...any) - StopProcess(format string, v ...any) -} diff --git a/utils/save.go b/utils/save.go deleted file mode 100644 index 7660974..0000000 --- a/utils/save.go +++ /dev/null @@ -1,177 +0,0 @@ -package utils - -import ( - "crypto/aes" - "crypto/cipher" - "crypto/rand" - "crypto/sha256" - "fmt" - "github.com/crunchy-labs/crunchyroll-go/v3" - "io" - "os" - "path/filepath" - "strings" -) - -func SaveSession(crunchy *crunchyroll.Crunchyroll) error { - file := filepath.Join(os.TempDir(), ".crunchy") - return os.WriteFile(file, []byte(crunchy.RefreshToken), 0600) -} - -func SaveCredentialsPersistent(user, password string, encryptionKey []byte) error { - configDir, err := os.UserConfigDir() - if err != nil { - return err - } - file := filepath.Join(configDir, "crunchy-cli", "crunchy") - - var credentials []byte - if encryptionKey != nil { - hashedEncryptionKey := sha256.Sum256(encryptionKey) - block, err := aes.NewCipher(hashedEncryptionKey[:]) - if err != nil { - return err - } - gcm, err := cipher.NewGCM(block) - if err != nil { - return err - } - nonce := make([]byte, gcm.NonceSize()) - if _, err = io.ReadFull(rand.Reader, nonce); err != nil { - return err - } - b := gcm.Seal(nonce, nonce, []byte(fmt.Sprintf("%s\n%s", user, password)), nil) - credentials = append([]byte("aes:"), b...) - } else { - credentials = []byte(fmt.Sprintf("%s\n%s", user, password)) - } - - if err = os.MkdirAll(filepath.Join(configDir, "crunchy-cli"), 0755); err != nil { - return err - } - return os.WriteFile(file, credentials, 0600) -} - -func SaveSessionPersistent(crunchy *crunchyroll.Crunchyroll) error { - configDir, err := os.UserConfigDir() - if err != nil { - return err - } - file := filepath.Join(configDir, "crunchy-cli", "crunchy") - - if err = os.MkdirAll(filepath.Join(configDir, "crunchy-cli"), 0755); err != nil { - return err - } - return os.WriteFile(file, []byte(crunchy.RefreshToken), 0600) -} - -func IsTempSession() bool { - file := filepath.Join(os.TempDir(), ".crunchy") - if _, err := os.Stat(file); !os.IsNotExist(err) { - return true - } - return false -} - -func IsSavedSessionEncrypted() (bool, error) { - configDir, err := os.UserConfigDir() - if err != nil { - return false, err - } - file := filepath.Join(configDir, "crunchy-cli", "crunchy") - body, err := os.ReadFile(file) - if err != nil { - return false, err - } - return strings.HasPrefix(string(body), "aes:"), nil -} - -func LoadSession(encryptionKey []byte) (*crunchyroll.Crunchyroll, error) { - file := filepath.Join(os.TempDir(), ".crunchy") - crunchy, err := loadTempSession(file) - if err != nil { - return nil, err - } - if crunchy != nil { - return crunchy, nil - } - - configDir, err := os.UserConfigDir() - if err != nil { - return nil, err - } - file = filepath.Join(configDir, "crunchy-cli", "crunchy") - crunchy, err = loadPersistentSession(file, encryptionKey) - if err != nil { - return nil, err - } - if crunchy != nil { - return crunchy, nil - } - - return nil, fmt.Errorf("not logged in") -} - -func loadTempSession(file string) (*crunchyroll.Crunchyroll, error) { - if _, err := os.Stat(file); !os.IsNotExist(err) { - body, err := os.ReadFile(file) - if err != nil { - return nil, err - } - crunchy, err := crunchyroll.LoginWithRefreshToken(string(body), SystemLocale(true), Client) - if err != nil { - Log.Debug("Failed to login with temp refresh token: %v", err) - } else { - Log.Debug("Logged in with refresh token %s. BLANK THIS LINE OUT IF YOU'RE ASKED TO POST THE DEBUG OUTPUT SOMEWHERE", body) - return crunchy, nil - } - } - return nil, nil -} - -func loadPersistentSession(file string, encryptionKey []byte) (crunchy *crunchyroll.Crunchyroll, err error) { - if _, err = os.Stat(file); !os.IsNotExist(err) { - body, err := os.ReadFile(file) - if err != nil { - return nil, err - } - split := strings.SplitN(string(body), "\n", 2) - if len(split) == 1 || split[1] == "" && strings.HasPrefix(split[0], "aes:") { - encrypted := body[4:] - hashedEncryptionKey := sha256.Sum256(encryptionKey) - block, err := aes.NewCipher(hashedEncryptionKey[:]) - if err != nil { - return nil, err - } - gcm, err := cipher.NewGCM(block) - if err != nil { - return nil, err - } - nonce, cypherText := encrypted[:gcm.NonceSize()], encrypted[gcm.NonceSize():] - b, err := gcm.Open(nil, nonce, cypherText, nil) - if err != nil { - return nil, err - } - split = strings.SplitN(string(b), "\n", 2) - } - if len(split) == 2 { - if crunchy, err = crunchyroll.LoginWithCredentials(split[0], split[1], SystemLocale(true), Client); err != nil { - return nil, err - } - Log.Debug("Logged in with credentials") - } else { - if crunchy, err = crunchyroll.LoginWithRefreshToken(split[0], SystemLocale(true), Client); err != nil { - return nil, err - } - Log.Debug("Logged in with refresh token %s. BLANK THIS LINE OUT IF YOU'RE ASKED TO POST THE DEBUG OUTPUT SOMEWHERE", split[0]) - } - - // the refresh token is written to a temp file to reduce the amount of re-logging in. - // it seems like that crunchyroll has also a little cooldown if a user logs in too often in a short time - if err = os.WriteFile(filepath.Join(os.TempDir(), ".crunchy"), []byte(crunchy.RefreshToken), 0600); err != nil { - return nil, err - } - } - - return -} diff --git a/utils/std.go b/utils/std.go deleted file mode 100644 index 75b35f1..0000000 --- a/utils/std.go +++ /dev/null @@ -1,10 +0,0 @@ -package utils - -func ElementInSlice[T comparable](check T, toCheck []T) bool { - for _, item := range toCheck { - if check == item { - return true - } - } - return false -} diff --git a/utils/system.go b/utils/system.go deleted file mode 100644 index ac97706..0000000 --- a/utils/system.go +++ /dev/null @@ -1,7 +0,0 @@ -package utils - -import "os/exec" - -func HasFFmpeg() bool { - return exec.Command("ffmpeg", "-h").Run() == nil -} diff --git a/utils/vars.go b/utils/vars.go deleted file mode 100644 index d2893a5..0000000 --- a/utils/vars.go +++ /dev/null @@ -1,14 +0,0 @@ -package utils - -import ( - "github.com/crunchy-labs/crunchyroll-go/v3" - "net/http" -) - -var Version = "development" - -var ( - Crunchy *crunchyroll.Crunchyroll - Client *http.Client - Log Logger -) From 502cb399238f848a2e82c310fc889c10a5df4223 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Thu, 24 Nov 2022 15:54:17 +0100 Subject: [PATCH 007/441] Fix download binary description --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2229739..db06b99 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,9 @@ A [Rust](https://www.rust-lang.org/) written cli client for [Crunchyroll](https: ### đŸ“Ĩ Download the latest binaries -Checkout the [releases](https://github.com/crunchy-labs/crunchy-cli/releases) tab and get the binary from the newest release. +~~Checkout the [releases](https://github.com/crunchy-labs/crunchy-cli/releases) tab and get the binary from the newest release.~~ + +Currently, no pre-built binary of the rewrite / this branch is available. ### 🛠 Build it yourself From 4fd98723ea4e344f2d904054b2594bda3b9f54fb Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sat, 26 Nov 2022 22:37:58 +0100 Subject: [PATCH 008/441] Change archive flag name from audio to locale --- crunchy-cli-core/src/cli/archive.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 21ae13b..754d6cb 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -44,7 +44,7 @@ pub struct Archive { #[arg(long_help = format!("Audio languages. Can be used multiple times. \ Available languages are:\n{}", Locale::all().into_iter().map(|l| format!("{:<6} → {}", l.to_string(), l.to_human_readable())).collect::>().join("\n ")))] #[arg(short, long, default_values_t = vec![crate::utils::locale::system_locale(), Locale::ja_JP])] - audio: Vec, + locale: Vec, #[arg(help = format!("Subtitle languages. Can be used multiple times. \ Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] #[arg(long_help = format!("Subtitle languages. Can be used multiple times. \ @@ -292,7 +292,7 @@ async fn formats_from_series( // get all locales which are specified but not present in the current iterated season and // print an error saying this let not_present_audio = archive - .audio + .locale .clone() .into_iter() .filter(|l| !season.iter().any(|s| &s.metadata.audio_locale == l)) @@ -309,7 +309,7 @@ async fn formats_from_series( // remove all seasons with the wrong audio for the current iterated season number seasons.retain(|s| { s.metadata.season_number != season.first().unwrap().metadata.season_number - || archive.audio.contains(&s.metadata.audio_locale) + || archive.locale.contains(&s.metadata.audio_locale) }) } @@ -318,7 +318,7 @@ async fn formats_from_series( BTreeMap::new(); for season in series.seasons().await? { if !url_filter.is_season_valid(season.metadata.season_number) - || !archive.audio.contains(&season.metadata.audio_locale) + || !archive.locale.contains(&season.metadata.audio_locale) { continue; } From a7c2bbe807129e8844cd30817a2770f0879085c8 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 27 Nov 2022 15:18:55 +0100 Subject: [PATCH 009/441] Add ci workflow --- .github/workflows/ci.yml | 133 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..f9b30a5 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,133 @@ +name: ci + +on: + push: + branches: + - master + pull_request: + workflow_dispatch: + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-latest + toolchain: x86_64-unknown-linux-musl + - os: windows-latest + toolchain: x86_64-pc-windows-msvc + - os: macos-latest + toolchain: x86_64-apple-darwin + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Cargo cache # https://github.com/actions/cache/blob/main/examples.md#rust---cargo + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Install toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + target: ${{ matrix.toolchain }} + default: true + + - name: Test + uses: actions-rs/cargo@v1 + with: + command: test + args: --all-features + + build: + if: github.ref == 'refs/heads/master' + needs: + - test + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-latest + toolchain: x86_64-unknown-linux-musl + ext: + output: crunchy_linux + - os: windows-latest + toolchain: x86_64-pc-windows-msvc + ext: .exe + output: crunchy_windows.exe + - os: macos-latest + toolchain: x86_64-apple-darwin + ext: + output: crunchy_darwin + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Cargo cache # https://github.com/actions/cache/blob/main/examples.md#rust---cargo + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Install toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + target: ${{ matrix.toolchain }} + default: true + + - name: Build + uses: actions-rs/cargo@v1 + with: + command: build + args: --release --all-features + + - name: Bundle manpages + uses: thedoctor0/zip-release@0.6 + with: + type: zip + filename: manpages.zip + path: ./target/release/manpages + + - name: Bundle completions + uses: thedoctor0/zip-release@0.6 + with: + type: zip + filename: completions.zip + path: ./target/release/completions + + - name: Upload binary artifact + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.output }} + path: ./target/release/crunchy-cli${{ matrix.ext }} + if-no-files-found: error + + - name: Upload manpages artifact + uses: actions/upload-artifact@v3 + with: + name: + path: ./manpages.zip + if-no-files-found: error + + - name: Upload completions artifact + uses: actions/upload-artifact@v3 + with: + name: + path: ./completions.zip + if-no-files-found: error From 3f4ce3a0a92e4fa517b7dc03d66b014b5f7673dc Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 27 Nov 2022 17:11:26 +0100 Subject: [PATCH 010/441] Fix ci zip action version --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9b30a5..50a41e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,14 +98,14 @@ jobs: args: --release --all-features - name: Bundle manpages - uses: thedoctor0/zip-release@0.6 + uses: thedoctor0/zip-release@0:6 with: type: zip filename: manpages.zip path: ./target/release/manpages - name: Bundle completions - uses: thedoctor0/zip-release@0.6 + uses: thedoctor0/zip-release@0:6 with: type: zip filename: completions.zip From a7605884414afacf120a5ffdf94bf6d53de1a78a Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 27 Nov 2022 17:11:32 +0100 Subject: [PATCH 011/441] Add ci badge --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index db06b99..0dcbc63 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,9 @@ A [Rust](https://www.rust-lang.org/) written cli client for [Crunchyroll](https: Discord + + CI +

From 8d8333e41413546d6cf7ca66f2afa215d7c382b2 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 27 Nov 2022 17:16:12 +0100 Subject: [PATCH 012/441] Fix ci zip workflow (again) --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50a41e7..c27195a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,14 +98,14 @@ jobs: args: --release --all-features - name: Bundle manpages - uses: thedoctor0/zip-release@0:6 + uses: TheDoctor0/zip-release@0.6.2 with: type: zip filename: manpages.zip path: ./target/release/manpages - name: Bundle completions - uses: thedoctor0/zip-release@0:6 + uses: TheDoctor0/zip-release@0.6.2 with: type: zip filename: completions.zip From 45c315e9bb1008e54f1e6a2bb03e42d5d8c4f134 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 27 Nov 2022 17:27:12 +0100 Subject: [PATCH 013/441] Fix ci upload artifact action --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c27195a..01ad86f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,13 +121,13 @@ jobs: - name: Upload manpages artifact uses: actions/upload-artifact@v3 with: - name: + name: manpages.zip path: ./manpages.zip if-no-files-found: error - name: Upload completions artifact uses: actions/upload-artifact@v3 with: - name: + name: completions.zip path: ./completions.zip if-no-files-found: error From b118b74b994608a73a9495a8d8d2fcafb78db364 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 27 Nov 2022 19:57:37 +0100 Subject: [PATCH 014/441] Update README --- README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 0dcbc63..f62c988 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,8 @@ A [Rust](https://www.rust-lang.org/) written cli client for [Crunchyroll](https: License - - Release + + Release Discord @@ -44,9 +44,7 @@ A [Rust](https://www.rust-lang.org/) written cli client for [Crunchyroll](https: ### đŸ“Ĩ Download the latest binaries -~~Checkout the [releases](https://github.com/crunchy-labs/crunchy-cli/releases) tab and get the binary from the newest release.~~ - -Currently, no pre-built binary of the rewrite / this branch is available. +Checkout the [releases](https://github.com/crunchy-labs/crunchy-cli/releases) tab and get the binary from the newest (pre-)release. ### 🛠 Build it yourself From f3f41aa0a2c94b595eabfae119feed9cafa64045 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 27 Nov 2022 19:58:14 +0100 Subject: [PATCH 015/441] Add crunchy-cli-core/Cargo.lock --- crunchy-cli-core/Cargo.lock | 1608 +++++++++++++++++++++++++++++++++++ 1 file changed, 1608 insertions(+) create mode 100644 crunchy-cli-core/Cargo.lock diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock new file mode 100644 index 0000000..61ea0ff --- /dev/null +++ b/crunchy-cli-core/Cargo.lock @@ -0,0 +1,1608 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aes" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "433cfd6710c9986c576a25ca913c39d66a6474107b406f34f91d4a8923395241" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "aho-corasick" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" + +[[package]] +name = "async-channel" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14485364214912d3b19cc3435dde4df66065127f05fa0d75c712f36f12c2f28" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + +[[package]] +name = "async-trait" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e805d94e6b5001b651426cf4cd446b1ab5f319d27bab5c644f61de0a804360c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "block-padding" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a90ec2df9600c28a01c56c4784c9207a96d2451833aeceb8cc97e4c9548bb78" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" + +[[package]] +name = "bytes" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" + +[[package]] +name = "cache-padded" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1db59621ec70f09c5e9b597b220c7a2b43611f4710dc03ceb8748637775692c" + +[[package]] +name = "castaway" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2698f953def977c68f935bb0dfa959375ad4638570e969e2f1e9f433cbf1af6" + +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + +[[package]] +name = "cc" +version = "1.0.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-integer", + "num-traits", + "serde", + "time", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "cipher" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1873270f8f7942c191139cb8a40fd228da6c3fd2fc376d7e92d47aa14aeb59e" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "clap" +version = "4.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2148adefda54e14492fb9bddcc600b4344c5d1a3123bd666dcb939c6f0e0e57e" +dependencies = [ + "atty", + "bitflags", + "clap_derive", + "clap_lex", + "once_cell", + "strsim", + "termcolor", +] + +[[package]] +name = "clap_derive" +version = "4.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + +[[package]] +name = "concurrent-queue" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af4780a44ab5696ea9e28294517f1fffb421a83a25af521333c838635509db9c" +dependencies = [ + "cache-padded", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + +[[package]] +name = "cpufeatures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +dependencies = [ + "libc", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crunchy-cli-core" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "chrono", + "clap", + "crunchyroll-rs", + "ctrlc", + "dirs", + "isahc", + "log", + "num_cpus", + "regex", + "signal-hook", + "sys-locale", + "tempfile", + "terminal_size", + "tokio", +] + +[[package]] +name = "crunchyroll-rs" +version = "0.1.0" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#7aedfc6c9a91a42ef46639ba9e99adba63cd0dda" +dependencies = [ + "aes", + "cbc", + "chrono", + "crunchyroll-rs-internal", + "http", + "isahc", + "m3u8-rs", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "smart-default", + "tokio", +] + +[[package]] +name = "crunchyroll-rs-internal" +version = "0.1.0" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#7aedfc6c9a91a42ef46639ba9e99adba63cd0dda" +dependencies = [ + "darling", + "quote", + "syn", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "ctrlc" +version = "3.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d91974fbbe88ec1df0c24a4f00f99583667a7e2e6272b2b92d294d81e462173" +dependencies = [ + "nix", + "winapi", +] + +[[package]] +name = "curl" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "509bd11746c7ac09ebd19f0b17782eae80aadee26237658a6b4808afb5c11a22" +dependencies = [ + "curl-sys", + "libc", + "openssl-probe", + "openssl-sys", + "schannel", + "socket2", + "winapi", +] + +[[package]] +name = "curl-sys" +version = "0.4.59+curl-7.86.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cfce34829f448b08f55b7db6d0009e23e2e86a34e8c2b366269bf5799b4a407" +dependencies = [ + "cc", + "libc", + "libnghttp2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", + "winapi", +] + +[[package]] +name = "cxx" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4a41a86530d0fe7f5d9ea779916b7cadd2d4f9add748b99c2c029cbbdfaf453" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06416d667ff3e3ad2df1cd8cd8afae5da26cf9cec4d0825040f88b5ca659a2f0" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "820a9a2af1669deeef27cb271f476ffd196a2c4b6731336011e0ba63e2c7cf71" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a08a6e2fcc370a089ad3b4aaf54db3b1b4cee38ddabce5896b33eb693275f470" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "darling" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0dd3cd20dc6b5a876612a6e5accfe7f3dd883db6d07acfbf14c128f61550dfa" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a784d2ccaf7c98501746bf0be29b2022ba41fd62a2e622af997a03e9f972859f" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7618812407e9402654622dd402b0a89dff9ba93badd6540781526117b92aab7e" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "encoding_rs" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "fastrand" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +dependencies = [ + "instant", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-core" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" + +[[package]] +name = "futures-io" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" + +[[package]] +name = "futures-lite" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + +[[package]] +name = "generic-array" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "http" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "iana-time-zone" +version = "0.1.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "io-lifetimes" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ce5ef949d49ee85593fc4d3f3f95ad61657076395cbbce23e2121fc5542074" + +[[package]] +name = "isahc" +version = "1.7.0" +source = "git+https://github.com/sagebind/isahc?rev=34f158ef#34f158ef9f87b2387bed2c81936916a29c1eaad1" +dependencies = [ + "async-channel", + "castaway", + "crossbeam-utils", + "curl", + "curl-sys", + "encoding_rs", + "event-listener", + "futures-lite", + "http", + "httpdate", + "log", + "mime", + "once_cell", + "polling", + "slab", + "sluice", + "tracing", + "tracing-futures", + "url", + "waker-fn", +] + +[[package]] +name = "itoa" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" + +[[package]] +name = "js-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.137" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" + +[[package]] +name = "libnghttp2-sys" +version = "0.1.7+1.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57ed28aba195b38d5ff02b9170cbff627e336a20925e43b4945390401c5dc93f" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "libz-sys" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "link-cplusplus" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369" +dependencies = [ + "cc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d" + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "m3u8-rs" +version = "5.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15d091887fd4a920417805283b7a838d0dcda68e8d632cd305a4439ee776d1ce" +dependencies = [ + "chrono", + "nom", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nix" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e322c04a9e3440c327fca7b6c8a63e6890a32fa2ad689db972425f07e0d22abb" +dependencies = [ + "autocfg", + "bitflags", + "cfg-if", + "libc", +] + +[[package]] +name = "nom" +version = "7.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-src" +version = "111.24.0+1.1.1s" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3498f259dab01178c6228c6b00dcef0ed2a2d5e20d648c017861227773ea4abd" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07d5c8cb6e57b3a3612064d7b18b117912b4ce70955c2504d4b741c9e244b132" +dependencies = [ + "autocfg", + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "os_str_bytes" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" + +[[package]] +name = "parking" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" + +[[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + +[[package]] +name = "pin-project" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pkg-config" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" + +[[package]] +name = "polling" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab4609a838d88b73d8238967b60dd115cc08d38e2bbaf51ee1e4b695f89122e2" +dependencies = [ + "autocfg", + "cfg-if", + "libc", + "log", + "wepoll-ffi", + "winapi", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom", + "redox_syscall", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "rustix" +version = "0.35.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "727a1a6d65f786ec22df8a81ca3121107f235970dc1705ed681d3e6e8b9cd5f9" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.42.0", +] + +[[package]] +name = "ryu" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" + +[[package]] +name = "schannel" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" +dependencies = [ + "lazy_static", + "windows-sys 0.36.1", +] + +[[package]] +name = "scratch" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" + +[[package]] +name = "serde" +version = "1.0.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "signal-hook" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg", +] + +[[package]] +name = "sluice" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7400c0eff44aa2fcb5e31a5f24ba9716ed90138769e4977a2ba6014ae63eb5" +dependencies = [ + "async-channel", + "futures-core", + "futures-io", +] + +[[package]] +name = "smart-default" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "133659a15339456eeeb07572eb02a91c91e9815e9cbc89566944d2c8d3efdbf6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "socket2" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sys-locale" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3358acbb4acd4146138b9bda219e904a6bb5aaaa237f8eed06f4d6bc1580ecee" +dependencies = [ + "js-sys", + "libc", + "wasm-bindgen", + "web-sys", + "winapi", +] + +[[package]] +name = "tempfile" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +dependencies = [ + "cfg-if", + "fastrand", + "libc", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "termcolor" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "terminal_size" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ca90c434fd12083d1a6bdcbe9f92a14f96c8a1ba600ba451734ac334521f7a" +dependencies = [ + "rustix", + "windows-sys 0.42.0", +] + +[[package]] +name = "thiserror" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "1.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d76ce4a75fb488c605c54bf610f221cea8b0dafb53333c1a67e8ee199dcd2ae3" +dependencies = [ + "autocfg", + "num_cpus", + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + +[[package]] +name = "typenum" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" + +[[package]] +name = "unicode-bidi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" + +[[package]] +name = "unicode-ident" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + +[[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "waker-fn" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" + +[[package]] +name = "web-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wepoll-ffi" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb" +dependencies = [ + "cc", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +dependencies = [ + "windows_aarch64_msvc 0.36.1", + "windows_i686_gnu 0.36.1", + "windows_i686_msvc 0.36.1", + "windows_x86_64_gnu 0.36.1", + "windows_x86_64_msvc 0.36.1", +] + +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc 0.42.0", + "windows_i686_gnu 0.42.0", + "windows_i686_msvc 0.42.0", + "windows_x86_64_gnu 0.42.0", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc 0.42.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" + +[[package]] +name = "windows_i686_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" + +[[package]] +name = "windows_i686_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" From 9bfe6b0e54f996116e786cda2b59bdeba9204461 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 27 Nov 2022 20:02:37 +0100 Subject: [PATCH 016/441] Update authors --- Cargo.toml | 3 ++- crunchy-cli-core/Cargo.toml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index c99a2a9..b9adc07 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [package] name = "crunchy-cli" -version = "0.1.0" +authors = ["Crunchy Labs Maintainers"] +version = "3.0.0" edition = "2021" [features] diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 893b278..e6bba13 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,5 +1,6 @@ [package] name = "crunchy-cli-core" +authors = ["Crunchy Labs Maintainers"] version = "0.1.0" edition = "2021" From 7fe587a89168222ef1a6be7a29049fddcf5bbcb9 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 27 Nov 2022 20:22:59 +0100 Subject: [PATCH 017/441] Update version --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- crunchy-cli-core/Cargo.lock | 2 +- crunchy-cli-core/Cargo.toml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 05b2d65..4c3ba10 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -263,7 +263,7 @@ dependencies = [ [[package]] name = "crunchy-cli" -version = "0.1.0" +version = "3.0.0-beta.1" dependencies = [ "chrono", "clap", @@ -275,7 +275,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "0.1.0" +version = "3.0.0-beta.1" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index b9adc07..4cdf8dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0" +version = "3.0.0-beta.1" edition = "2021" [features] diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index 61ea0ff..1502091 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -244,7 +244,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "0.1.0" +version = "3.0.0-beta.1" dependencies = [ "anyhow", "async-trait", diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index e6bba13..e0e83be 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "0.1.0" +version = "3.0.0-beta.1" edition = "2021" [features] From 1487ba222eeb10adb99f3c7536a18eaca1cb031b Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 27 Nov 2022 20:24:10 +0100 Subject: [PATCH 018/441] Add short commit sha to ci build artifacts --- .github/workflows/ci.yml | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01ad86f..cdc3136 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,16 +58,16 @@ jobs: include: - os: ubuntu-latest toolchain: x86_64-unknown-linux-musl + platform: linux ext: - output: crunchy_linux - os: windows-latest toolchain: x86_64-pc-windows-msvc + platform: windows ext: .exe - output: crunchy_windows.exe - os: macos-latest toolchain: x86_64-apple-darwin + platform: darwin ext: - output: crunchy_darwin steps: - name: Checkout uses: actions/checkout@v3 @@ -98,36 +98,41 @@ jobs: args: --release --all-features - name: Bundle manpages - uses: TheDoctor0/zip-release@0.6.2 + uses: thedoctor0/zip-release@0.6 with: type: zip filename: manpages.zip path: ./target/release/manpages - name: Bundle completions - uses: TheDoctor0/zip-release@0.6.2 + uses: thedoctor0/zip-release@0.6 with: type: zip filename: completions.zip path: ./target/release/completions + - name: Get short commit SHA + id: short_commit_sha + shell: bash + run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + - name: Upload binary artifact uses: actions/upload-artifact@v3 with: - name: ${{ matrix.output }} + name: crunchy-${{ short_commit_sha.sha_short }}_${{ matrix.platform }}${{ matrix.ext }} path: ./target/release/crunchy-cli${{ matrix.ext }} if-no-files-found: error - name: Upload manpages artifact uses: actions/upload-artifact@v3 with: - name: manpages.zip + name: manpages-${{ short_commit_sha.sha_short }}.zip path: ./manpages.zip if-no-files-found: error - name: Upload completions artifact uses: actions/upload-artifact@v3 with: - name: completions.zip + name: completions-${{ short_commit_sha.sha_short }}.zip path: ./completions.zip if-no-files-found: error From 81931829b0859590512019a3147de447cc065001 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 27 Nov 2022 20:43:13 +0100 Subject: [PATCH 019/441] Fix invalid ci file syntax --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cdc3136..dcf1c56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,20 +119,20 @@ jobs: - name: Upload binary artifact uses: actions/upload-artifact@v3 with: - name: crunchy-${{ short_commit_sha.sha_short }}_${{ matrix.platform }}${{ matrix.ext }} + name: crunchy-${{ steps.short_commit_sha.sha_short }}_${{ matrix.platform }}${{ matrix.ext }} path: ./target/release/crunchy-cli${{ matrix.ext }} if-no-files-found: error - name: Upload manpages artifact uses: actions/upload-artifact@v3 with: - name: manpages-${{ short_commit_sha.sha_short }}.zip + name: manpages-${{ steps.short_commit_sha.sha_short }}.zip path: ./manpages.zip if-no-files-found: error - name: Upload completions artifact uses: actions/upload-artifact@v3 with: - name: completions-${{ short_commit_sha.sha_short }}.zip + name: completions-${{ steps.short_commit_sha.sha_short }}.zip path: ./completions.zip if-no-files-found: error From c99eedd7a7c0b29d22131813c2c08d085daa8907 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 27 Nov 2022 22:14:32 +0100 Subject: [PATCH 020/441] Re-fix ci zip workflow --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dcf1c56..4e67182 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,14 +98,14 @@ jobs: args: --release --all-features - name: Bundle manpages - uses: thedoctor0/zip-release@0.6 + uses: thedoctor0/zip-release@0.6.2 with: type: zip filename: manpages.zip path: ./target/release/manpages - name: Bundle completions - uses: thedoctor0/zip-release@0.6 + uses: thedoctor0/zip-release@0.6.2 with: type: zip filename: completions.zip From 24fbedc7d7a52bf96dec9f40b2313345a53b908b Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 27 Nov 2022 22:42:15 +0100 Subject: [PATCH 021/441] Fix ci short commit sha --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e67182..9c6dba9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,20 +119,20 @@ jobs: - name: Upload binary artifact uses: actions/upload-artifact@v3 with: - name: crunchy-${{ steps.short_commit_sha.sha_short }}_${{ matrix.platform }}${{ matrix.ext }} + name: crunchy-${{ steps.short_commit_sha.outputs.sha_short }}_${{ matrix.platform }}${{ matrix.ext }} path: ./target/release/crunchy-cli${{ matrix.ext }} if-no-files-found: error - name: Upload manpages artifact uses: actions/upload-artifact@v3 with: - name: manpages-${{ steps.short_commit_sha.sha_short }}.zip + name: manpages-${{ steps.short_commit_sha.outputs.sha_short }}.zip path: ./manpages.zip if-no-files-found: error - name: Upload completions artifact uses: actions/upload-artifact@v3 with: - name: completions-${{ steps.short_commit_sha.sha_short }}.zip + name: completions-${{ steps.short_commit_sha.outputs.sha_short }}.zip path: ./completions.zip if-no-files-found: error From 59b5e3d239c2ad0596f67a21bea248cb90aa2f01 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 28 Nov 2022 10:11:20 +0100 Subject: [PATCH 022/441] Update version --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- crunchy-cli-core/Cargo.lock | 2 +- crunchy-cli-core/Cargo.toml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4c3ba10..26ffd7f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -263,7 +263,7 @@ dependencies = [ [[package]] name = "crunchy-cli" -version = "3.0.0-beta.1" +version = "3.0.0-dev.1" dependencies = [ "chrono", "clap", @@ -275,7 +275,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-beta.1" +version = "3.0.0-dev.1" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 4cdf8dc..7614202 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-beta.1" +version = "3.0.0-dev.1" edition = "2021" [features] diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index 1502091..61bc6f6 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -244,7 +244,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-beta.1" +version = "3.0.0-dev.1" dependencies = [ "anyhow", "async-trait", diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index e0e83be..067679b 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-beta.1" +version = "3.0.0-dev.1" edition = "2021" [features] From b1182d4f7b23adc8042998980afa53b4c5c3c5f4 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 28 Nov 2022 11:54:04 +0100 Subject: [PATCH 023/441] Add (short) commit hash and build time to version hash --- build.rs | 7 ------- crunchy-cli-core/Cargo.toml | 5 ++++- crunchy-cli-core/build.rs | 33 +++++++++++++++++++++++++++++++++ crunchy-cli-core/src/lib.rs | 30 +++++++++++++++++++++++++----- 4 files changed, 62 insertions(+), 13 deletions(-) create mode 100644 crunchy-cli-core/build.rs diff --git a/build.rs b/build.rs index cc6a0cc..1e4d71f 100644 --- a/build.rs +++ b/build.rs @@ -2,14 +2,7 @@ use clap::{Command, CommandFactory}; use clap_complete::shells; use std::path::{Path, PathBuf}; -// this build file generates completions for various shells as well as manual pages - fn main() -> std::io::Result<()> { - // do not generate anything when building non release - if cfg!(debug_assertions) { - return Ok(()); - } - // note that we're using an anti-pattern here / violate the rust conventions. build script are // not supposed to write outside of 'OUT_DIR'. to have the generated files in the build "root" // (the same directory where the output binary lives) is much simpler than in 'OUT_DIR' since diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 067679b..00635b0 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -14,7 +14,7 @@ static-ssl = ["crunchyroll-rs/static-ssl"] [dependencies] anyhow = "1.0" async-trait = "0.1" -clap = { version = "4.0", features = ["derive"] } +clap = { version = "4.0", features = ["derive", "string"] } chrono = "0.4" crunchyroll-rs = { git = "https://github.com/crunchy-labs/crunchyroll-rs", default-features = false, features = ["stream", "parse"] } ctrlc = "3.2" @@ -28,3 +28,6 @@ tempfile = "3.3" terminal_size = "0.2" tokio = { version = "1.21", features = ["macros", "rt-multi-thread", "time"] } sys-locale = "0.2" + +[build-dependencies] +chrono = "0.4" diff --git a/crunchy-cli-core/build.rs b/crunchy-cli-core/build.rs new file mode 100644 index 0000000..f7d5974 --- /dev/null +++ b/crunchy-cli-core/build.rs @@ -0,0 +1,33 @@ +fn main() -> std::io::Result<()> { + println!( + "cargo:rustc-env=GIT_HASH={}", + get_short_commit_hash()?.unwrap_or_default() + ); + println!( + "cargo:rustc-env=BUILD_DATE={}", + chrono::Utc::now().format("%F") + ); + + Ok(()) +} + +fn get_short_commit_hash() -> std::io::Result> { + let git = std::process::Command::new("git") + .arg("rev-parse") + .arg("--short") + .arg("HEAD") + .output(); + + match git { + Ok(cmd) => Ok(Some( + String::from_utf8_lossy(cmd.stdout.as_slice()).to_string(), + )), + Err(e) => { + if e.kind() != std::io::ErrorKind::NotFound { + Err(e) + } else { + Ok(None) + } + } + } +} diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index faef927..4571337 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -20,13 +20,15 @@ trait Execute { } #[derive(Debug, Parser)] -#[clap(author, version, about)] +#[clap(author, version = version(), about)] #[clap(name = "crunchy-cli")] pub struct Cli { #[clap(flatten)] verbosity: Option, - #[arg(help = "Overwrite the language in which results are returned. Default is your system language")] + #[arg( + help = "Overwrite the language in which results are returned. Default is your system language" + )] #[arg(long)] lang: Option, @@ -37,6 +39,18 @@ pub struct Cli { command: Command, } +fn version() -> String { + let package_version = env!("CARGO_PKG_VERSION"); + let git_commit_hash = env!("GIT_HASH"); + let build_date = env!("BUILD_DATE"); + + if git_commit_hash.is_empty() { + format!("{}", package_version) + } else { + format!("{} ({} {})", package_version, git_commit_hash, build_date) + } +} + #[derive(Debug, Subcommand)] enum Command { Archive(Archive), @@ -51,7 +65,9 @@ struct Verbosity { v: bool, #[arg(help = "Quiet output. Does not print anything unless it's a error")] - #[arg(long_help = "Quiet output. Does not print anything unless it's a error. Can be helpful if you pipe the output to stdout")] + #[arg( + long_help = "Quiet output. Does not print anything unless it's a error. Can be helpful if you pipe the output to stdout" + )] #[arg(short)] q: bool, } @@ -59,11 +75,15 @@ struct Verbosity { #[derive(Debug, Parser)] struct LoginMethod { #[arg(help = "Login with credentials (username or email and password)")] - #[arg(long_help = "Login with credentials (username or email and password). Must be provided as user:password")] + #[arg( + long_help = "Login with credentials (username or email and password). Must be provided as user:password" + )] #[arg(long)] credentials: Option, #[arg(help = "Login with the etp-rt cookie")] - #[arg(long_help = "Login with the etp-rt cookie. This can be obtained when you login on crunchyroll.com and extract it from there")] + #[arg( + long_help = "Login with the etp-rt cookie. This can be obtained when you login on crunchyroll.com and extract it from there" + )] #[arg(long)] etp_rt: Option, } From 2d89a712039f314faf4a28774b3b9824278c5f04 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 28 Nov 2022 12:18:44 +0100 Subject: [PATCH 024/441] Remove commit sha in filename --- .github/workflows/ci.yml | 29 +++++------------------------ 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c6dba9..f5d01b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,42 +97,23 @@ jobs: command: build args: --release --all-features - - name: Bundle manpages - uses: thedoctor0/zip-release@0.6.2 - with: - type: zip - filename: manpages.zip - path: ./target/release/manpages - - - name: Bundle completions - uses: thedoctor0/zip-release@0.6.2 - with: - type: zip - filename: completions.zip - path: ./target/release/completions - - - name: Get short commit SHA - id: short_commit_sha - shell: bash - run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - name: Upload binary artifact uses: actions/upload-artifact@v3 with: - name: crunchy-${{ steps.short_commit_sha.outputs.sha_short }}_${{ matrix.platform }}${{ matrix.ext }} + name: crunchy-cli_${{ matrix.platform }} path: ./target/release/crunchy-cli${{ matrix.ext }} if-no-files-found: error - name: Upload manpages artifact uses: actions/upload-artifact@v3 with: - name: manpages-${{ steps.short_commit_sha.outputs.sha_short }}.zip - path: ./manpages.zip + name: manpages + path: ./target/release/manpages if-no-files-found: error - name: Upload completions artifact uses: actions/upload-artifact@v3 with: - name: completions-${{ steps.short_commit_sha.outputs.sha_short }}.zip - path: ./completions.zip + name: completions + path: ./target/release/completions if-no-files-found: error From 99002e606f11aca2f07d079ee60af555c4fc99f4 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 28 Nov 2022 22:14:57 +0100 Subject: [PATCH 025/441] Change windows ci toolchain to gnu --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5d01b6..9aba2cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: - os: ubuntu-latest toolchain: x86_64-unknown-linux-musl - os: windows-latest - toolchain: x86_64-pc-windows-msvc + toolchain: x86_64-pc-windows-gnu - os: macos-latest toolchain: x86_64-apple-darwin steps: @@ -61,7 +61,7 @@ jobs: platform: linux ext: - os: windows-latest - toolchain: x86_64-pc-windows-msvc + toolchain: x86_64-pc-windows-gnu platform: windows ext: .exe - os: macos-latest From 4095b80477f30788aa262a50655e29265adba9d0 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 30 Nov 2022 19:04:01 +0100 Subject: [PATCH 026/441] Update dependencies --- Cargo.lock | 196 +++++++++++++++++++++++++----------- crunchy-cli-core/Cargo.lock | 188 ++++++++++++++++++++++++---------- crunchy-cli-core/Cargo.toml | 1 + 3 files changed, 277 insertions(+), 108 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 26ffd7f..144cff4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,9 +39,9 @@ checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" [[package]] name = "async-channel" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14485364214912d3b19cc3435dde4df66065127f05fa0d75c712f36f12c2f28" +checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833" dependencies = [ "concurrent-queue", "event-listener", @@ -50,32 +50,27 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.58" +version = "0.1.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e805d94e6b5001b651426cf4cd446b1ab5f319d27bab5c644f61de0a804360c" +checksum = "31e6e93155431f3931513b243d371981bb2770112b370c82745a1d19d2f99364" dependencies = [ "proc-macro2", "quote", "syn", ] -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi", - "libc", - "winapi", -] - [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "bitflags" version = "1.3.2" @@ -103,12 +98,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" -[[package]] -name = "cache-padded" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1db59621ec70f09c5e9b597b220c7a2b43611f4710dc03ceb8748637775692c" - [[package]] name = "castaway" version = "0.1.2" @@ -164,14 +153,14 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.26" +version = "4.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2148adefda54e14492fb9bddcc600b4344c5d1a3123bd666dcb939c6f0e0e57e" +checksum = "4d63b9e9c07271b9957ad22c173bae2a4d9a81127680962039296abcd2f8251d" dependencies = [ - "atty", "bitflags", "clap_derive", "clap_lex", + "is-terminal", "once_cell", "strsim", "termcolor", @@ -179,9 +168,9 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.0.5" +version = "4.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96b0fba905b035a30d25c1b585bf1171690712fbb0ad3ac47214963aa4acc36c" +checksum = "b7b3c9eae0de7bf8e3f904a5e40612b21fb2e2e566456d177809a48b892d24da" dependencies = [ "clap", ] @@ -210,9 +199,9 @@ dependencies = [ [[package]] name = "clap_mangen" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa149477df7854a1497db0def32b8a65bf98f72a14d04ac75b01938285d83420" +checksum = "e503c3058af0a0854668ea01db55c622482a080092fede9dd2e00a00a9436504" dependencies = [ "clap", "roff", @@ -230,11 +219,21 @@ dependencies = [ [[package]] name = "concurrent-queue" -version = "1.2.4" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af4780a44ab5696ea9e28294517f1fffb421a83a25af521333c838635509db9c" +checksum = "bd7bef69dc86e3c610e4e7aed41035e2a7ed12e72dd7530f61327a6579a4390b" dependencies = [ - "cache-padded", + "crossbeam-utils", +] + +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", ] [[package]] @@ -298,27 +297,30 @@ dependencies = [ [[package]] name = "crunchyroll-rs" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#7aedfc6c9a91a42ef46639ba9e99adba63cd0dda" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#3e9f7bfaab68649ecdb486aee0b15fe658d51917" dependencies = [ "aes", "cbc", "chrono", "crunchyroll-rs-internal", + "curl-sys", "http", "isahc", "m3u8-rs", "regex", + "rustls-native-certs", "serde", "serde_json", "serde_urlencoded", "smart-default", + "static_vcruntime", "tokio", ] [[package]] name = "crunchyroll-rs-internal" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#7aedfc6c9a91a42ef46639ba9e99adba63cd0dda" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#3e9f7bfaab68649ecdb486aee0b15fe658d51917" dependencies = [ "darling", "quote", @@ -455,6 +457,12 @@ dependencies = [ "syn", ] +[[package]] +name = "data-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" + [[package]] name = "dirs" version = "4.0.0" @@ -598,6 +606,15 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + [[package]] name = "http" version = "0.2.8" @@ -676,9 +693,25 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "0.7.5" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ce5ef949d49ee85593fc4d3f3f95ad61657076395cbbce23e2121fc5542074" +checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" +dependencies = [ + "libc", + "windows-sys 0.42.0", +] + +[[package]] +name = "is-terminal" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "927609f78c2913a6f6ac3c27a4fe87f43e2a35367c0c4b0f8265e8f49a104330" +dependencies = [ + "hermit-abi 0.2.6", + "io-lifetimes", + "rustix", + "windows-sys 0.42.0", +] [[package]] name = "isahc" @@ -690,6 +723,7 @@ dependencies = [ "crossbeam-utils", "curl", "curl-sys", + "data-encoding", "encoding_rs", "event-listener", "futures-lite", @@ -767,9 +801,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.0.46" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d" +checksum = "8f9f08d8963a6c613f4b1a78f4f4a4dbfadf8e6545b2d72861731e4858b8b47f" [[package]] name = "log" @@ -855,7 +889,7 @@ version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", ] @@ -946,16 +980,16 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "polling" -version = "2.4.0" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab4609a838d88b73d8238967b60dd115cc08d38e2bbaf51ee1e4b695f89122e2" +checksum = "166ca89eb77fd403230b9c156612965a81e094ec6ec3aa13663d4c8b113fa748" dependencies = [ "autocfg", "cfg-if", "libc", "log", "wepoll-ffi", - "winapi", + "windows-sys 0.42.0", ] [[package]] @@ -1054,9 +1088,9 @@ checksum = "b833d8d034ea094b1ea68aa6d5c740e0d04bad9d16568d08ba6f76823a114316" [[package]] name = "rustix" -version = "0.35.13" +version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727a1a6d65f786ec22df8a81ca3121107f235970dc1705ed681d3e6e8b9cd5f9" +checksum = "cb93e85278e08bb5788653183213d3a60fc242b10cb9be96586f5a73dcb67c23" dependencies = [ "bitflags", "errno", @@ -1066,6 +1100,27 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0864aeff53f8c05aa08d86e5ef839d3dfcf07aeba2db32f12db0ef716e87bd55" +dependencies = [ + "base64", +] + [[package]] name = "ryu" version = "1.0.11" @@ -1089,19 +1144,42 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" [[package]] -name = "serde" -version = "1.0.147" +name = "security-framework" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" +checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.148" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e53f64bb4ba0191d6d0676e1b141ca55047d83b74f5607e6d8eb88126c52c2dc" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.147" +version = "1.0.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852" +checksum = "a55492425aa53521babf6137309e7d34c20bbfbbfcfe2c7f3a047fd1f6b92c0c" dependencies = [ "proc-macro2", "quote", @@ -1191,6 +1269,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "static_vcruntime" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "954e3e877803def9dc46075bf4060147c55cd70db97873077232eae0269dc89b" + [[package]] name = "strsim" version = "0.10.0" @@ -1199,9 +1283,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.103" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" +checksum = "4ae548ec36cf198c0ef7710d3c230987c2d6d7bd98ad6edc0274462724c585ce" dependencies = [ "proc-macro2", "quote", @@ -1246,9 +1330,9 @@ dependencies = [ [[package]] name = "terminal_size" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ca90c434fd12083d1a6bdcbe9f92a14f96c8a1ba600ba451734ac334521f7a" +checksum = "cb20089a8ba2b69debd491f8d2d023761cbf196e999218c591fa1e7e15a21907" dependencies = [ "rustix", "windows-sys 0.42.0", @@ -1276,9 +1360,9 @@ dependencies = [ [[package]] name = "time" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" dependencies = [ "libc", "wasi 0.10.0+wasi-snapshot-preview1", @@ -1314,9 +1398,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.8.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" +checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ "proc-macro2", "quote", diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index 61bc6f6..3ee8448 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -39,9 +39,9 @@ checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" [[package]] name = "async-channel" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14485364214912d3b19cc3435dde4df66065127f05fa0d75c712f36f12c2f28" +checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833" dependencies = [ "concurrent-queue", "event-listener", @@ -50,32 +50,27 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.58" +version = "0.1.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e805d94e6b5001b651426cf4cd446b1ab5f319d27bab5c644f61de0a804360c" +checksum = "31e6e93155431f3931513b243d371981bb2770112b370c82745a1d19d2f99364" dependencies = [ "proc-macro2", "quote", "syn", ] -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi", - "libc", - "winapi", -] - [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "bitflags" version = "1.3.2" @@ -103,12 +98,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" -[[package]] -name = "cache-padded" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1db59621ec70f09c5e9b597b220c7a2b43611f4710dc03ceb8748637775692c" - [[package]] name = "castaway" version = "0.1.2" @@ -164,14 +153,14 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.26" +version = "4.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2148adefda54e14492fb9bddcc600b4344c5d1a3123bd666dcb939c6f0e0e57e" +checksum = "4d63b9e9c07271b9957ad22c173bae2a4d9a81127680962039296abcd2f8251d" dependencies = [ - "atty", "bitflags", "clap_derive", "clap_lex", + "is-terminal", "once_cell", "strsim", "termcolor", @@ -211,11 +200,21 @@ dependencies = [ [[package]] name = "concurrent-queue" -version = "1.2.4" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af4780a44ab5696ea9e28294517f1fffb421a83a25af521333c838635509db9c" +checksum = "bd7bef69dc86e3c610e4e7aed41035e2a7ed12e72dd7530f61327a6579a4390b" dependencies = [ - "cache-padded", + "crossbeam-utils", +] + +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", ] [[package]] @@ -267,27 +266,30 @@ dependencies = [ [[package]] name = "crunchyroll-rs" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#7aedfc6c9a91a42ef46639ba9e99adba63cd0dda" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#3e9f7bfaab68649ecdb486aee0b15fe658d51917" dependencies = [ "aes", "cbc", "chrono", "crunchyroll-rs-internal", + "curl-sys", "http", "isahc", "m3u8-rs", "regex", + "rustls-native-certs", "serde", "serde_json", "serde_urlencoded", "smart-default", + "static_vcruntime", "tokio", ] [[package]] name = "crunchyroll-rs-internal" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#7aedfc6c9a91a42ef46639ba9e99adba63cd0dda" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#3e9f7bfaab68649ecdb486aee0b15fe658d51917" dependencies = [ "darling", "quote", @@ -424,6 +426,12 @@ dependencies = [ "syn", ] +[[package]] +name = "data-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" + [[package]] name = "dirs" version = "4.0.0" @@ -567,6 +575,15 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + [[package]] name = "http" version = "0.2.8" @@ -645,9 +662,25 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "0.7.5" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ce5ef949d49ee85593fc4d3f3f95ad61657076395cbbce23e2121fc5542074" +checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" +dependencies = [ + "libc", + "windows-sys 0.42.0", +] + +[[package]] +name = "is-terminal" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "927609f78c2913a6f6ac3c27a4fe87f43e2a35367c0c4b0f8265e8f49a104330" +dependencies = [ + "hermit-abi 0.2.6", + "io-lifetimes", + "rustix", + "windows-sys 0.42.0", +] [[package]] name = "isahc" @@ -659,6 +692,7 @@ dependencies = [ "crossbeam-utils", "curl", "curl-sys", + "data-encoding", "encoding_rs", "event-listener", "futures-lite", @@ -736,9 +770,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.0.46" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d" +checksum = "8f9f08d8963a6c613f4b1a78f4f4a4dbfadf8e6545b2d72861731e4858b8b47f" [[package]] name = "log" @@ -824,7 +858,7 @@ version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", ] @@ -915,16 +949,16 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "polling" -version = "2.4.0" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab4609a838d88b73d8238967b60dd115cc08d38e2bbaf51ee1e4b695f89122e2" +checksum = "166ca89eb77fd403230b9c156612965a81e094ec6ec3aa13663d4c8b113fa748" dependencies = [ "autocfg", "cfg-if", "libc", "log", "wepoll-ffi", - "winapi", + "windows-sys 0.42.0", ] [[package]] @@ -1017,9 +1051,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.35.13" +version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727a1a6d65f786ec22df8a81ca3121107f235970dc1705ed681d3e6e8b9cd5f9" +checksum = "cb93e85278e08bb5788653183213d3a60fc242b10cb9be96586f5a73dcb67c23" dependencies = [ "bitflags", "errno", @@ -1029,6 +1063,27 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0864aeff53f8c05aa08d86e5ef839d3dfcf07aeba2db32f12db0ef716e87bd55" +dependencies = [ + "base64", +] + [[package]] name = "ryu" version = "1.0.11" @@ -1052,19 +1107,42 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" [[package]] -name = "serde" -version = "1.0.147" +name = "security-framework" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" +checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.148" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e53f64bb4ba0191d6d0676e1b141ca55047d83b74f5607e6d8eb88126c52c2dc" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.147" +version = "1.0.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852" +checksum = "a55492425aa53521babf6137309e7d34c20bbfbbfcfe2c7f3a047fd1f6b92c0c" dependencies = [ "proc-macro2", "quote", @@ -1154,6 +1232,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "static_vcruntime" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "954e3e877803def9dc46075bf4060147c55cd70db97873077232eae0269dc89b" + [[package]] name = "strsim" version = "0.10.0" @@ -1162,9 +1246,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.103" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" +checksum = "4ae548ec36cf198c0ef7710d3c230987c2d6d7bd98ad6edc0274462724c585ce" dependencies = [ "proc-macro2", "quote", @@ -1209,9 +1293,9 @@ dependencies = [ [[package]] name = "terminal_size" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ca90c434fd12083d1a6bdcbe9f92a14f96c8a1ba600ba451734ac334521f7a" +checksum = "cb20089a8ba2b69debd491f8d2d023761cbf196e999218c591fa1e7e15a21907" dependencies = [ "rustix", "windows-sys 0.42.0", @@ -1239,9 +1323,9 @@ dependencies = [ [[package]] name = "time" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" dependencies = [ "libc", "wasi 0.10.0+wasi-snapshot-preview1", @@ -1277,9 +1361,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.8.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" +checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ "proc-macro2", "quote", diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 00635b0..f8c5007 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" static-curl = ["crunchyroll-rs/static-curl"] # Embed a static openssl library into the binary instead of just linking it. If you want to compile this project against # musl and have openssl issues, this might solve these issues. +# Has no effect on Windows, the library is always statically linked there. static-ssl = ["crunchyroll-rs/static-ssl"] [dependencies] From e200aab8ab47d746659c3c1d4969fd637f6a26c1 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 30 Nov 2022 19:04:11 +0100 Subject: [PATCH 027/441] Update ci --- .github/workflows/ci.yml | 58 ++++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9aba2cd..1fa94a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: command: test args: --all-features - build: + build-nix: if: github.ref == 'refs/heads/master' needs: - test @@ -59,15 +59,9 @@ jobs: - os: ubuntu-latest toolchain: x86_64-unknown-linux-musl platform: linux - ext: - - os: windows-latest - toolchain: x86_64-pc-windows-gnu - platform: windows - ext: .exe - os: macos-latest toolchain: x86_64-apple-darwin platform: darwin - ext: steps: - name: Checkout uses: actions/checkout@v3 @@ -101,7 +95,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: crunchy-cli_${{ matrix.platform }} - path: ./target/release/crunchy-cli${{ matrix.ext }} + path: ./target/release/crunchy-cli if-no-files-found: error - name: Upload manpages artifact @@ -117,3 +111,51 @@ jobs: name: completions path: ./target/release/completions if-no-files-found: error + + build-windows: + if: github.ref == 'refs/heads/master' + needs: + - test + runs-on: windows-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Install OpenSSL + run: vcpkg install openssl:x64-windows-static-md + + - name: Set env variables + shell: bash + run: echo "CFLAGS=-I$(echo $VCPKG_INSTALLATION_ROOT)\packages\openssl_x64-windows-static-md\include" >> $GITHUB_ENV + + - name: Cargo cache # https://github.com/actions/cache/blob/main/examples.md#rust---cargo + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Install toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + target: x86_64-pc-windows-msvc + default: true + + - name: Build + uses: actions-rs/cargo@v1 + with: + command: build + args: --release + + - name: Upload binary artifact + uses: actions/upload-artifact@v3 + with: + name: crunchy-cli_windows.exe + path: ./target/release/crunchy-cli.exe + if-no-files-found: error From 6ecd23bcd0c8b87fd49f2321fe09d08f623af9b4 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 30 Nov 2022 19:25:00 +0100 Subject: [PATCH 028/441] Fix ci --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1fa94a0..277a591 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,6 +121,8 @@ jobs: - name: Checkout uses: actions/checkout@v3 + - run: vcpkg integrate install + - name: Install OpenSSL run: vcpkg install openssl:x64-windows-static-md From 12d49a27e468632976f4c8d269c9e124e1e2a898 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 30 Nov 2022 19:25:21 +0100 Subject: [PATCH 029/441] Add static vc runtime --- Cargo.lock | 6 +++--- Cargo.toml | 3 +++ build.rs | 3 +++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 144cff4..2690925 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -269,6 +269,7 @@ dependencies = [ "clap_complete", "clap_mangen", "crunchy-cli-core", + "static_vcruntime", "tokio", ] @@ -297,7 +298,7 @@ dependencies = [ [[package]] name = "crunchyroll-rs" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#3e9f7bfaab68649ecdb486aee0b15fe658d51917" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#3d455ebca34cfa83df39777f3e1414cee4f84002" dependencies = [ "aes", "cbc", @@ -313,14 +314,13 @@ dependencies = [ "serde_json", "serde_urlencoded", "smart-default", - "static_vcruntime", "tokio", ] [[package]] name = "crunchyroll-rs-internal" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#3e9f7bfaab68649ecdb486aee0b15fe658d51917" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#3d455ebca34cfa83df39777f3e1414cee4f84002" dependencies = [ "darling", "quote", diff --git a/Cargo.toml b/Cargo.toml index 7614202..5e3b153 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,9 @@ clap_mangen = "0.2" # specified in this Cargo.toml [features]. crunchy-cli-core = { path = "./crunchy-cli-core", features = ["static-curl", "static-ssl"] } +[target.'cfg(all(windows, target_env = "msvc"))'.build-dependencies] +static_vcruntime = "2.0" + [profile.release] strip = true opt-level = "z" diff --git a/build.rs b/build.rs index 1e4d71f..927f4dd 100644 --- a/build.rs +++ b/build.rs @@ -3,6 +3,9 @@ use clap_complete::shells; use std::path::{Path, PathBuf}; fn main() -> std::io::Result<()> { + #[cfg(all(windows, target_env = "msvc"))] + static_vcruntime::metabuild(); + // note that we're using an anti-pattern here / violate the rust conventions. build script are // not supposed to write outside of 'OUT_DIR'. to have the generated files in the build "root" // (the same directory where the output binary lives) is much simpler than in 'OUT_DIR' since From a3c717dc1a4e9bd27e2bf508bfac8f02143268e0 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 30 Nov 2022 19:54:39 +0100 Subject: [PATCH 030/441] Split test job --- .github/workflows/ci.yml | 62 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 277a591..9f55b5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,15 +8,13 @@ on: workflow_dispatch: jobs: - test: + test-nix: runs-on: ${{ matrix.os }} strategy: matrix: include: - os: ubuntu-latest toolchain: x86_64-unknown-linux-musl - - os: windows-latest - toolchain: x86_64-pc-windows-gnu - os: macos-latest toolchain: x86_64-apple-darwin steps: @@ -48,10 +46,50 @@ jobs: command: test args: --all-features + test-windows: + runs-on: windows-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - run: vcpkg integrate install + + - name: Install OpenSSL + run: vcpkg install openssl:x64-windows-static-md + + - name: Set env variables + shell: bash + run: echo "CFLAGS=-I$(echo $VCPKG_INSTALLATION_ROOT)\packages\openssl_x64-windows-static-md\include" >> $GITHUB_ENV + + - name: Cargo cache # https://github.com/actions/cache/blob/main/examples.md#rust---cargo + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Install toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + target: x86_64-pc-windows-msvc + default: true + + - name: Test + uses: actions-rs/cargo@v1 + with: + command: test + args: + build-nix: if: github.ref == 'refs/heads/master' needs: - - test + - test-nix runs-on: ${{ matrix.os }} strategy: matrix: @@ -115,7 +153,7 @@ jobs: build-windows: if: github.ref == 'refs/heads/master' needs: - - test + - test-windows runs-on: windows-latest steps: - name: Checkout @@ -161,3 +199,17 @@ jobs: name: crunchy-cli_windows.exe path: ./target/release/crunchy-cli.exe if-no-files-found: error + + - name: Upload manpages artifact + uses: actions/upload-artifact@v3 + with: + name: manpages + path: ./target/release/manpages + if-no-files-found: error + + - name: Upload completions artifact + uses: actions/upload-artifact@v3 + with: + name: completions + path: ./target/release/completions + if-no-files-found: error From 0bb20d24a2bbbfcc882791a5aa25c70d7217ebcc Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 30 Nov 2022 21:34:20 +0100 Subject: [PATCH 031/441] Update dependency versions --- Cargo.lock | 59 +++++++++++++++++++++++++++++-------- Cargo.toml | 5 +--- crunchy-cli-core/Cargo.lock | 59 ++++++++++--------------------------- crunchy-cli-core/Cargo.toml | 10 +++---- 4 files changed, 68 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2690925..290b5d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -104,6 +104,15 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2698f953def977c68f935bb0dfa959375ad4638570e969e2f1e9f433cbf1af6" +[[package]] +name = "castaway" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a17ed5635fc8536268e5d4de1e22e81ac34419e5f052d4d51f4e01dcc263fcc" +dependencies = [ + "rustversion", +] + [[package]] name = "cbc" version = "0.1.2" @@ -284,10 +293,11 @@ dependencies = [ "crunchyroll-rs", "ctrlc", "dirs", - "isahc", + "isahc 1.7.0 (git+https://github.com/sagebind/isahc?rev=c39f6f8)", "log", "num_cpus", "regex", + "rustls-native-certs", "signal-hook", "sys-locale", "tempfile", @@ -306,7 +316,7 @@ dependencies = [ "crunchyroll-rs-internal", "curl-sys", "http", - "isahc", + "isahc 1.7.0 (git+https://github.com/sagebind/isahc?rev=34f158ef)", "m3u8-rs", "regex", "rustls-native-certs", @@ -719,7 +729,7 @@ version = "1.7.0" source = "git+https://github.com/sagebind/isahc?rev=34f158ef#34f158ef9f87b2387bed2c81936916a29c1eaad1" dependencies = [ "async-channel", - "castaway", + "castaway 0.1.2", "crossbeam-utils", "curl", "curl-sys", @@ -741,6 +751,33 @@ dependencies = [ "waker-fn", ] +[[package]] +name = "isahc" +version = "1.7.0" +source = "git+https://github.com/sagebind/isahc?rev=c39f6f8#c39f6f85aaa1f36c5857064c6c3c80f4d307d863" +dependencies = [ + "async-channel", + "castaway 0.2.2", + "crossbeam-utils", + "curl", + "curl-sys", + "data-encoding", + "encoding_rs", + "event-listener", + "futures-io", + "futures-lite", + "http", + "log", + "mime", + "once_cell", + "polling", + "sluice", + "tracing", + "tracing-futures", + "url", + "waker-fn", +] + [[package]] name = "itoa" version = "1.0.4" @@ -905,15 +942,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" -[[package]] -name = "openssl-src" -version = "111.24.0+1.1.1s" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3498f259dab01178c6228c6b00dcef0ed2a2d5e20d648c017861227773ea4abd" -dependencies = [ - "cc", -] - [[package]] name = "openssl-sys" version = "0.9.78" @@ -923,7 +951,6 @@ dependencies = [ "autocfg", "cc", "libc", - "openssl-src", "pkg-config", "vcpkg", ] @@ -1121,6 +1148,12 @@ dependencies = [ "base64", ] +[[package]] +name = "rustversion" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8" + [[package]] name = "ryu" version = "1.0.11" diff --git a/Cargo.toml b/Cargo.toml index 5e3b153..d08c47f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,9 +9,6 @@ default = ["static-curl"] # Embed a static curl library into the binary instead of just linking it. static-curl = ["crunchy-cli-core/static-curl"] -# Embed a static openssl library into the binary instead of just linking it. If you want to compile this project against -# musl and have openssl issues, this might solve these issues. -static-ssl = ["crunchy-cli-core/static-ssl"] [dependencies] tokio = { version = "1.22", features = ["macros", "rt-multi-thread", "time"], default-features = false } @@ -26,7 +23,7 @@ clap_mangen = "0.2" # The static-* features must be used here since build dependency features cannot be manipulated from the features # specified in this Cargo.toml [features]. -crunchy-cli-core = { path = "./crunchy-cli-core", features = ["static-curl", "static-ssl"] } +crunchy-cli-core = { path = "./crunchy-cli-core", features = ["static-curl"] } [target.'cfg(all(windows, target_env = "msvc"))'.build-dependencies] static_vcruntime = "2.0" diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index 3ee8448..cd16c1b 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -100,9 +100,12 @@ checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" [[package]] name = "castaway" -version = "0.1.2" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2698f953def977c68f935bb0dfa959375ad4638570e969e2f1e9f433cbf1af6" +checksum = "8a17ed5635fc8536268e5d4de1e22e81ac34419e5f052d4d51f4e01dcc263fcc" +dependencies = [ + "rustversion", +] [[package]] name = "cbc" @@ -256,6 +259,7 @@ dependencies = [ "log", "num_cpus", "regex", + "rustls-native-certs", "signal-hook", "sys-locale", "tempfile", @@ -266,7 +270,7 @@ dependencies = [ [[package]] name = "crunchyroll-rs" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#3e9f7bfaab68649ecdb486aee0b15fe658d51917" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#ffb054cbae4079bf2357a23e90a353ce875978af" dependencies = [ "aes", "cbc", @@ -282,14 +286,13 @@ dependencies = [ "serde_json", "serde_urlencoded", "smart-default", - "static_vcruntime", "tokio", ] [[package]] name = "crunchyroll-rs-internal" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#3e9f7bfaab68649ecdb486aee0b15fe658d51917" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#ffb054cbae4079bf2357a23e90a353ce875978af" dependencies = [ "darling", "quote", @@ -530,13 +533,8 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" dependencies = [ - "fastrand", "futures-core", - "futures-io", - "memchr", - "parking", "pin-project-lite", - "waker-fn", ] [[package]] @@ -685,7 +683,7 @@ dependencies = [ [[package]] name = "isahc" version = "1.7.0" -source = "git+https://github.com/sagebind/isahc?rev=34f158ef#34f158ef9f87b2387bed2c81936916a29c1eaad1" +source = "git+https://github.com/sagebind/isahc?rev=c39f6f8#c39f6f85aaa1f36c5857064c6c3c80f4d307d863" dependencies = [ "async-channel", "castaway", @@ -695,6 +693,7 @@ dependencies = [ "data-encoding", "encoding_rs", "event-listener", + "futures-io", "futures-lite", "http", "httpdate", @@ -702,7 +701,6 @@ dependencies = [ "mime", "once_cell", "polling", - "slab", "sluice", "tracing", "tracing-futures", @@ -874,15 +872,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" -[[package]] -name = "openssl-src" -version = "111.24.0+1.1.1s" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3498f259dab01178c6228c6b00dcef0ed2a2d5e20d648c017861227773ea4abd" -dependencies = [ - "cc", -] - [[package]] name = "openssl-sys" version = "0.9.78" @@ -892,7 +881,6 @@ dependencies = [ "autocfg", "cc", "libc", - "openssl-src", "pkg-config", "vcpkg", ] @@ -903,12 +891,6 @@ version = "6.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" -[[package]] -name = "parking" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" - [[package]] name = "percent-encoding" version = "2.2.0" @@ -1084,6 +1066,12 @@ dependencies = [ "base64", ] +[[package]] +name = "rustversion" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8" + [[package]] name = "ryu" version = "1.0.11" @@ -1191,15 +1179,6 @@ dependencies = [ "libc", ] -[[package]] -name = "slab" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" -dependencies = [ - "autocfg", -] - [[package]] name = "sluice" version = "0.5.5" @@ -1232,12 +1211,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "static_vcruntime" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "954e3e877803def9dc46075bf4060147c55cd70db97873077232eae0269dc89b" - [[package]] name = "strsim" version = "0.10.0" diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index f8c5007..30e6d44 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -7,10 +7,6 @@ edition = "2021" [features] # Embed a static curl library into the binary instead of just linking it. static-curl = ["crunchyroll-rs/static-curl"] -# Embed a static openssl library into the binary instead of just linking it. If you want to compile this project against -# musl and have openssl issues, this might solve these issues. -# Has no effect on Windows, the library is always statically linked there. -static-ssl = ["crunchyroll-rs/static-ssl"] [dependencies] anyhow = "1.0" @@ -20,7 +16,7 @@ chrono = "0.4" crunchyroll-rs = { git = "https://github.com/crunchy-labs/crunchyroll-rs", default-features = false, features = ["stream", "parse"] } ctrlc = "3.2" dirs = "4.0" -isahc = { git = "https://github.com/sagebind/isahc", rev = "34f158ef" } +isahc = { git = "https://github.com/sagebind/isahc", rev = "c39f6f8" } log = { version = "0.4", features = ["std"] } num_cpus = "1.13" regex = "1.6" @@ -30,5 +26,9 @@ terminal_size = "0.2" tokio = { version = "1.21", features = ["macros", "rt-multi-thread", "time"] } sys-locale = "0.2" +[target.'cfg(all(windows, target_env = "msvc"))'.dependencies] +isahc = { git = "https://github.com/sagebind/isahc", rev = "c39f6f8", features = ["data-encoding"] } +rustls-native-certs = "0.6" + [build-dependencies] chrono = "0.4" From 64bb39362e9f937bb852ded778e7ee315de72483 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 30 Nov 2022 21:34:30 +0100 Subject: [PATCH 032/441] Add Windows certificates --- crunchy-cli-core/src/lib.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index 4571337..c0cfce5 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -167,7 +167,21 @@ async fn create_ctx(cli: &Cli) -> Result { // TODO: Use crunchy.client() when it's possible // currently crunchy.client() has a cloudflare bypass built-in to access crunchyroll. the servers // where crunchy stores their videos can't handle this bypass and simply refuses to connect + #[cfg(not(all(windows, target_env = "msvc")))] let client = isahc::HttpClient::new().unwrap(); + #[cfg(all(windows, target_env = "msvc"))] + let client = isahc::HttpClientBuilder::default() + .proxy_tls_config( + isahc::tls::TlsConfigBuilder::default().root_cert_store( + isahc::tls::RootCertStore::custom( + rustls_native_certs::load_native_certs() + .unwrap() + .into_iter() + .map(|l| isahc::tls::Certificate::from_der(l.0)), + ), + ), + ) + .build(); Ok(Context { crunchy, client }) } From f687969f0456323dbffe2146e02657b2afa45a85 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 30 Nov 2022 21:53:23 +0100 Subject: [PATCH 033/441] Fix isahc tls config --- crunchy-cli-core/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index c0cfce5..d64dae5 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -171,7 +171,7 @@ async fn create_ctx(cli: &Cli) -> Result { let client = isahc::HttpClient::new().unwrap(); #[cfg(all(windows, target_env = "msvc"))] let client = isahc::HttpClientBuilder::default() - .proxy_tls_config( + .tls_config( isahc::tls::TlsConfigBuilder::default().root_cert_store( isahc::tls::RootCertStore::custom( rustls_native_certs::load_native_certs() From f6d6c9435c7b6c91001cfad4f53dd3cb7819be15 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 30 Nov 2022 22:28:51 +0100 Subject: [PATCH 034/441] Fix invalid Windows client builder --- crunchy-cli-core/src/lib.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index d64dae5..a1e8381 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -164,24 +164,27 @@ pub async fn cli_entrypoint() { async fn create_ctx(cli: &Cli) -> Result { let crunchy = crunchyroll_session(cli).await?; - // TODO: Use crunchy.client() when it's possible + // use crunchy.client() when it's possible // currently crunchy.client() has a cloudflare bypass built-in to access crunchyroll. the servers // where crunchy stores their videos can't handle this bypass and simply refuses to connect #[cfg(not(all(windows, target_env = "msvc")))] let client = isahc::HttpClient::new().unwrap(); #[cfg(all(windows, target_env = "msvc"))] + use isahc::config::Configurable; + #[cfg(all(windows, target_env = "msvc"))] let client = isahc::HttpClientBuilder::default() .tls_config( - isahc::tls::TlsConfigBuilder::default().root_cert_store( - isahc::tls::RootCertStore::custom( + isahc::tls::TlsConfigBuilder::default() + .root_cert_store(isahc::tls::RootCertStore::custom( rustls_native_certs::load_native_certs() .unwrap() .into_iter() .map(|l| isahc::tls::Certificate::from_der(l.0)), - ), - ), + )) + .build(), ) - .build(); + .build() + .unwrap(); Ok(Context { crunchy, client }) } From 474e9f5e31c0d07727b0ce39d8dc08373a7bd278 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 30 Nov 2022 23:43:57 +0100 Subject: [PATCH 035/441] Add very verbose output flag --- crunchy-cli-core/src/cli/log.rs | 10 ++++++---- crunchy-cli-core/src/lib.rs | 14 ++++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/crunchy-cli-core/src/cli/log.rs b/crunchy-cli-core/src/cli/log.rs index d4bb1b3..3539733 100644 --- a/crunchy-cli-core/src/cli/log.rs +++ b/crunchy-cli-core/src/cli/log.rs @@ -92,6 +92,7 @@ impl CliProgress { #[allow(clippy::type_complexity)] pub struct CliLogger { + all: bool, level: LevelFilter, progress: Mutex>, } @@ -105,7 +106,7 @@ impl Log for CliLogger { if !self.enabled(record.metadata()) || (record.target() != "progress" && record.target() != "progress_end" - && !record.target().starts_with("crunchy_cli")) + && (!self.all && !record.target().starts_with("crunchy_cli"))) { return; } @@ -136,16 +137,17 @@ impl Log for CliLogger { } impl CliLogger { - pub fn new(level: LevelFilter) -> Self { + pub fn new(all: bool, level: LevelFilter) -> Self { Self { + all, level, progress: Mutex::new(None), } } - pub fn init(level: LevelFilter) -> Result<(), SetLoggerError> { + pub fn init(all: bool, level: LevelFilter) -> Result<(), SetLoggerError> { set_max_level(level); - set_boxed_logger(Box::new(CliLogger::new(level))) + set_boxed_logger(Box::new(CliLogger::new(all, level))) } fn extended(&self, record: &Record) { diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index a1e8381..9b57859 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -64,6 +64,10 @@ struct Verbosity { #[arg(short)] v: bool, + #[arg(help = "Very verbose output. Generally not recommended, use '-v' instead")] + #[arg(long)] + vv: bool, + #[arg(help = "Quiet output. Does not print anything unless it's a error")] #[arg( long_help = "Quiet output. Does not print anything unless it's a error. Can be helpful if you pipe the output to stdout" @@ -92,16 +96,18 @@ pub async fn cli_entrypoint() { let cli: Cli = Cli::parse(); if let Some(verbosity) = &cli.verbosity { - if verbosity.v && verbosity.q { + if verbosity.v as u8 + verbosity.q as u8 + verbosity.vv as u8 > 1 { eprintln!("Output cannot be verbose ('-v') and quiet ('-q') at the same time"); std::process::exit(1) } else if verbosity.v { - CliLogger::init(LevelFilter::Debug).unwrap() + CliLogger::init(false, LevelFilter::Debug).unwrap() } else if verbosity.q { - CliLogger::init(LevelFilter::Error).unwrap() + CliLogger::init(false, LevelFilter::Error).unwrap() + } else if verbosity.vv { + CliLogger::init(true, LevelFilter::Debug).unwrap() } } else { - CliLogger::init(LevelFilter::Info).unwrap() + CliLogger::init(false, LevelFilter::Info).unwrap() } debug!("cli input: {:?}", cli); From 6aa4078be343c99503f62bd5e5179109a9706b2a Mon Sep 17 00:00:00 2001 From: ByteDream Date: Fri, 2 Dec 2022 17:28:54 +0100 Subject: [PATCH 036/441] Update dependencies --- Cargo.lock | 386 ++++++++++++++++++++++---- crunchy-cli-core/src/cli/archive.rs | 2 +- crunchy-cli-core/src/cli/download.rs | 14 +- crunchy-cli-core/src/cli/utils.rs | 7 +- crunchy-cli-core/src/lib.rs | 24 +- crunchy-cli-core/src/utils/context.rs | 1 - 6 files changed, 342 insertions(+), 92 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 290b5d1..fddbc62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -98,12 +98,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" -[[package]] -name = "castaway" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2698f953def977c68f935bb0dfa959375ad4638570e969e2f1e9f433cbf1af6" - [[package]] name = "castaway" version = "0.2.2" @@ -293,7 +287,7 @@ dependencies = [ "crunchyroll-rs", "ctrlc", "dirs", - "isahc 1.7.0 (git+https://github.com/sagebind/isahc?rev=c39f6f8)", + "isahc", "log", "num_cpus", "regex", @@ -308,7 +302,7 @@ dependencies = [ [[package]] name = "crunchyroll-rs" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#3d455ebca34cfa83df39777f3e1414cee4f84002" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#b05ab04af60547f4d5b8d008e3b3f829eb99aa0f" dependencies = [ "aes", "cbc", @@ -316,10 +310,10 @@ dependencies = [ "crunchyroll-rs-internal", "curl-sys", "http", - "isahc 1.7.0 (git+https://github.com/sagebind/isahc?rev=34f158ef)", + "isahc", "m3u8-rs", "regex", - "rustls-native-certs", + "reqwest", "serde", "serde_json", "serde_urlencoded", @@ -330,7 +324,7 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#3d455ebca34cfa83df39777f3e1414cee4f84002" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#b05ab04af60547f4d5b8d008e3b3f829eb99aa0f" dependencies = [ "darling", "quote", @@ -544,6 +538,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.1.0" @@ -553,6 +562,15 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures-channel" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" +dependencies = [ + "futures-core", +] + [[package]] name = "futures-core" version = "0.3.25" @@ -571,13 +589,35 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" dependencies = [ - "fastrand", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "futures-sink" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" + +[[package]] +name = "futures-task" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" + +[[package]] +name = "futures-util" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" +dependencies = [ "futures-core", "futures-io", + "futures-task", "memchr", - "parking", "pin-project-lite", - "waker-fn", + "pin-utils", + "slab", ] [[package]] @@ -601,6 +641,31 @@ dependencies = [ "wasi 0.11.0+wasi-snapshot-preview1", ] +[[package]] +name = "h2" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "heck" version = "0.4.0" @@ -636,12 +701,66 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + [[package]] name = "httpdate" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "hyper" +version = "0.14.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "iana-time-zone" version = "0.1.53" @@ -682,6 +801,16 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "indexmap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +dependencies = [ + "autocfg", + "hashbrown", +] + [[package]] name = "inout" version = "0.1.3" @@ -711,6 +840,12 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "ipnet" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f88c5561171189e69df9d98bcf18fd5f9558300f7ea7b801eb8a0fd748bd8745" + [[package]] name = "is-terminal" version = "0.4.1" @@ -723,41 +858,13 @@ dependencies = [ "windows-sys 0.42.0", ] -[[package]] -name = "isahc" -version = "1.7.0" -source = "git+https://github.com/sagebind/isahc?rev=34f158ef#34f158ef9f87b2387bed2c81936916a29c1eaad1" -dependencies = [ - "async-channel", - "castaway 0.1.2", - "crossbeam-utils", - "curl", - "curl-sys", - "data-encoding", - "encoding_rs", - "event-listener", - "futures-lite", - "http", - "httpdate", - "log", - "mime", - "once_cell", - "polling", - "slab", - "sluice", - "tracing", - "tracing-futures", - "url", - "waker-fn", -] - [[package]] name = "isahc" version = "1.7.0" source = "git+https://github.com/sagebind/isahc?rev=c39f6f8#c39f6f85aaa1f36c5857064c6c3c80f4d307d863" dependencies = [ "async-channel", - "castaway 0.2.2", + "castaway", "crossbeam-utils", "curl", "curl-sys", @@ -767,6 +874,7 @@ dependencies = [ "futures-io", "futures-lite", "http", + "httpdate", "log", "mime", "once_cell", @@ -853,9 +961,9 @@ dependencies = [ [[package]] name = "m3u8-rs" -version = "5.0.2" +version = "5.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15d091887fd4a920417805283b7a838d0dcda68e8d632cd305a4439ee776d1ce" +checksum = "2c3b9f971e885044cb57330d3c89a4f3bd45fe97d01c93b3dcec57096efacffc" dependencies = [ "chrono", "nom", @@ -880,10 +988,40 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] -name = "nix" -version = "0.25.0" +name = "mio" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e322c04a9e3440c327fca7b6c8a63e6890a32fa2ad689db972425f07e0d22abb" +checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +dependencies = [ + "libc", + "log", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.42.0", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nix" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4" dependencies = [ "autocfg", "bitflags", @@ -936,6 +1074,32 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +[[package]] +name = "openssl" +version = "0.10.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020433887e44c27ff16365eaa2d380547a94544ad509aff6eb5b6e3e0b27b376" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.5" @@ -961,12 +1125,6 @@ version = "6.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" -[[package]] -name = "parking" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" - [[package]] name = "percent-encoding" version = "2.2.0" @@ -999,6 +1157,12 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkg-config" version = "0.3.26" @@ -1107,6 +1271,43 @@ dependencies = [ "winapi", ] +[[package]] +name = "reqwest" +version = "0.11.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "roff" version = "0.2.1" @@ -1316,9 +1517,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.104" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae548ec36cf198c0ef7710d3c230987c2d6d7bd98ad6edc0274462724c585ce" +checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908" dependencies = [ "proc-macro2", "quote", @@ -1424,9 +1625,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d76ce4a75fb488c605c54bf610f221cea8b0dafb53333c1a67e8ee199dcd2ae3" dependencies = [ "autocfg", + "bytes", + "libc", + "memchr", + "mio", "num_cpus", "pin-project-lite", + "socket2", "tokio-macros", + "winapi", ] [[package]] @@ -1440,6 +1647,36 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + [[package]] name = "tracing" version = "0.1.37" @@ -1483,6 +1720,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + [[package]] name = "typenum" version = "1.15.0" @@ -1545,6 +1788,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.10.0+wasi-snapshot-preview1" @@ -1582,6 +1835,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.83" @@ -1760,3 +2025,12 @@ name = "windows_x86_64_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 754d6cb..a057aed 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -387,7 +387,7 @@ async fn download_video(ctx: &Context, format: &Format, only_audio: bool) -> Res ctx, &mut ffmpeg.stdin.unwrap(), Some(format!("Download {}", format.audio)), - format.stream.segments().await?, + format.stream.clone() ) .await?; diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index 7f9e3ec..8535869 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -8,7 +8,7 @@ use crate::utils::parse::{parse_url, UrlFilter}; use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number}; use crate::Execute; use anyhow::{bail, Result}; -use crunchyroll_rs::media::{Resolution, VariantSegment}; +use crunchyroll_rs::media::{Resolution, VariantData}; use crunchyroll_rs::{ Episode, Locale, Media, MediaCollection, Movie, MovieListing, Season, Series, }; @@ -212,16 +212,14 @@ impl Execute for Download { tab_info!("Resolution: {}", format.stream.resolution); tab_info!("FPS: {:.2}", format.stream.fps); - let segments = format.stream.segments().await?; - if use_ffmpeg { - download_ffmpeg(&ctx, segments, path.as_path()).await?; + download_ffmpeg(&ctx, format.stream, path.as_path()).await?; } else if path.to_str().unwrap() == "-" { let mut stdout = std::io::stdout().lock(); - download_segments(&ctx, &mut stdout, None, segments).await?; + download_segments(&ctx, &mut stdout, None, format.stream).await?; } else { let mut file = File::options().create(true).write(true).open(&path)?; - download_segments(&ctx, &mut file, None, segments).await? + download_segments(&ctx, &mut file, None, format.stream).await? } } } @@ -232,7 +230,7 @@ impl Execute for Download { async fn download_ffmpeg( ctx: &Context, - segments: Vec, + variant_data: VariantData, target: &Path, ) -> Result<()> { let ffmpeg = Command::new("ffmpeg") @@ -246,7 +244,7 @@ async fn download_ffmpeg( .arg(target.to_str().unwrap()) .spawn()?; - download_segments(ctx, &mut ffmpeg.stdin.unwrap(), None, segments).await?; + download_segments(ctx, &mut ffmpeg.stdin.unwrap(), None, variant_data).await?; Ok(()) } diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index 59816d1..e513101 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -26,14 +26,15 @@ pub fn find_resolution( } pub async fn download_segments( - ctx: &Context, + _ctx: &Context, writer: &mut impl Write, message: Option, - segments: Vec, + variant_data: VariantData, ) -> Result<()> { + let segments = variant_data.segments().await?; let total_segments = segments.len(); - let client = Arc::new(ctx.client.clone()); + let client = Arc::new(variant_data.download_client()); let count = Arc::new(Mutex::new(0)); let amount = Arc::new(Mutex::new(0)); diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index 9b57859..083b401 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -170,29 +170,7 @@ pub async fn cli_entrypoint() { async fn create_ctx(cli: &Cli) -> Result { let crunchy = crunchyroll_session(cli).await?; - // use crunchy.client() when it's possible - // currently crunchy.client() has a cloudflare bypass built-in to access crunchyroll. the servers - // where crunchy stores their videos can't handle this bypass and simply refuses to connect - #[cfg(not(all(windows, target_env = "msvc")))] - let client = isahc::HttpClient::new().unwrap(); - #[cfg(all(windows, target_env = "msvc"))] - use isahc::config::Configurable; - #[cfg(all(windows, target_env = "msvc"))] - let client = isahc::HttpClientBuilder::default() - .tls_config( - isahc::tls::TlsConfigBuilder::default() - .root_cert_store(isahc::tls::RootCertStore::custom( - rustls_native_certs::load_native_certs() - .unwrap() - .into_iter() - .map(|l| isahc::tls::Certificate::from_der(l.0)), - )) - .build(), - ) - .build() - .unwrap(); - - Ok(Context { crunchy, client }) + Ok(Context { crunchy }) } async fn crunchyroll_session(cli: &Cli) -> Result { diff --git a/crunchy-cli-core/src/utils/context.rs b/crunchy-cli-core/src/utils/context.rs index 6a2fd21..f8df024 100644 --- a/crunchy-cli-core/src/utils/context.rs +++ b/crunchy-cli-core/src/utils/context.rs @@ -2,5 +2,4 @@ use crunchyroll_rs::Crunchyroll; pub struct Context { pub crunchy: Crunchyroll, - pub client: isahc::HttpClient, } From 3fcb512c187e968df987b4c954925e514c223d56 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Fri, 2 Dec 2022 18:19:13 +0100 Subject: [PATCH 037/441] Merge test and build ci --- .github/workflows/ci.yml | 96 ++++++---------------------------------- 1 file changed, 13 insertions(+), 83 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f55b5d..d0e5960 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,88 +8,8 @@ on: workflow_dispatch: jobs: - test-nix: - runs-on: ${{ matrix.os }} - strategy: - matrix: - include: - - os: ubuntu-latest - toolchain: x86_64-unknown-linux-musl - - os: macos-latest - toolchain: x86_64-apple-darwin - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Cargo cache # https://github.com/actions/cache/blob/main/examples.md#rust---cargo - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - name: Install toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - target: ${{ matrix.toolchain }} - default: true - - - name: Test - uses: actions-rs/cargo@v1 - with: - command: test - args: --all-features - - test-windows: - runs-on: windows-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - - run: vcpkg integrate install - - - name: Install OpenSSL - run: vcpkg install openssl:x64-windows-static-md - - - name: Set env variables - shell: bash - run: echo "CFLAGS=-I$(echo $VCPKG_INSTALLATION_ROOT)\packages\openssl_x64-windows-static-md\include" >> $GITHUB_ENV - - - name: Cargo cache # https://github.com/actions/cache/blob/main/examples.md#rust---cargo - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - name: Install toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - target: x86_64-pc-windows-msvc - default: true - - - name: Test - uses: actions-rs/cargo@v1 - with: - command: test - args: - build-nix: if: github.ref == 'refs/heads/master' - needs: - - test-nix runs-on: ${{ matrix.os }} strategy: matrix: @@ -123,6 +43,12 @@ jobs: target: ${{ matrix.toolchain }} default: true + - name: Test + uses: actions-rs/cargo@v1 + with: + command: test + args: --release --all-features + - name: Build uses: actions-rs/cargo@v1 with: @@ -152,8 +78,6 @@ jobs: build-windows: if: github.ref == 'refs/heads/master' - needs: - - test-windows runs-on: windows-latest steps: - name: Checkout @@ -187,11 +111,17 @@ jobs: target: x86_64-pc-windows-msvc default: true + - name: Test + uses: actions-rs/cargo@v1 + with: + command: test + args: --release --all-features + - name: Build uses: actions-rs/cargo@v1 with: command: build - args: --release + args: --release --all-features - name: Upload binary artifact uses: actions/upload-artifact@v3 From 33e27504f2ba2b981caa90103f2c29badfd1b0ed Mon Sep 17 00:00:00 2001 From: ByteDream Date: Fri, 2 Dec 2022 19:41:50 +0100 Subject: [PATCH 038/441] Update dependencies --- Cargo.lock | 4 +- crunchy-cli-core/Cargo.lock | 347 ++++++++++++++++++++++++++++++++++-- 2 files changed, 339 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fddbc62..ffdd71b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -302,7 +302,7 @@ dependencies = [ [[package]] name = "crunchyroll-rs" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#b05ab04af60547f4d5b8d008e3b3f829eb99aa0f" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#66678d49bc071f49c83a7fa73ab280d1c1de5b2f" dependencies = [ "aes", "cbc", @@ -324,7 +324,7 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#b05ab04af60547f4d5b8d008e3b3f829eb99aa0f" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#66678d49bc071f49c83a7fa73ab280d1c1de5b2f" dependencies = [ "darling", "quote", diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index cd16c1b..4fa983a 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -270,7 +270,7 @@ dependencies = [ [[package]] name = "crunchyroll-rs" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#ffb054cbae4079bf2357a23e90a353ce875978af" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#66678d49bc071f49c83a7fa73ab280d1c1de5b2f" dependencies = [ "aes", "cbc", @@ -281,7 +281,7 @@ dependencies = [ "isahc", "m3u8-rs", "regex", - "rustls-native-certs", + "reqwest", "serde", "serde_json", "serde_urlencoded", @@ -292,7 +292,7 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#ffb054cbae4079bf2357a23e90a353ce875978af" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#66678d49bc071f49c83a7fa73ab280d1c1de5b2f" dependencies = [ "darling", "quote", @@ -506,6 +506,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.1.0" @@ -515,6 +530,15 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures-channel" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" +dependencies = [ + "futures-core", +] + [[package]] name = "futures-core" version = "0.3.25" @@ -537,6 +561,33 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "futures-sink" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" + +[[package]] +name = "futures-task" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" + +[[package]] +name = "futures-util" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" +dependencies = [ + "futures-core", + "futures-io", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "generic-array" version = "0.14.6" @@ -558,6 +609,31 @@ dependencies = [ "wasi 0.11.0+wasi-snapshot-preview1", ] +[[package]] +name = "h2" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "heck" version = "0.4.0" @@ -593,12 +669,66 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + [[package]] name = "httpdate" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "hyper" +version = "0.14.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "iana-time-zone" version = "0.1.53" @@ -639,6 +769,16 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "indexmap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +dependencies = [ + "autocfg", + "hashbrown", +] + [[package]] name = "inout" version = "0.1.3" @@ -668,6 +808,12 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "ipnet" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f88c5561171189e69df9d98bcf18fd5f9558300f7ea7b801eb8a0fd748bd8745" + [[package]] name = "is-terminal" version = "0.4.1" @@ -783,9 +929,9 @@ dependencies = [ [[package]] name = "m3u8-rs" -version = "5.0.2" +version = "5.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15d091887fd4a920417805283b7a838d0dcda68e8d632cd305a4439ee776d1ce" +checksum = "2c3b9f971e885044cb57330d3c89a4f3bd45fe97d01c93b3dcec57096efacffc" dependencies = [ "chrono", "nom", @@ -810,10 +956,40 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] -name = "nix" -version = "0.25.0" +name = "mio" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e322c04a9e3440c327fca7b6c8a63e6890a32fa2ad689db972425f07e0d22abb" +checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +dependencies = [ + "libc", + "log", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.42.0", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nix" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4" dependencies = [ "autocfg", "bitflags", @@ -866,6 +1042,32 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +[[package]] +name = "openssl" +version = "0.10.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020433887e44c27ff16365eaa2d380547a94544ad509aff6eb5b6e3e0b27b376" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.5" @@ -923,6 +1125,12 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkg-config" version = "0.3.26" @@ -1031,6 +1239,43 @@ dependencies = [ "winapi", ] +[[package]] +name = "reqwest" +version = "0.11.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "rustix" version = "0.36.4" @@ -1179,6 +1424,15 @@ dependencies = [ "libc", ] +[[package]] +name = "slab" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg", +] + [[package]] name = "sluice" version = "0.5.5" @@ -1219,9 +1473,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.104" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae548ec36cf198c0ef7710d3c230987c2d6d7bd98ad6edc0274462724c585ce" +checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908" dependencies = [ "proc-macro2", "quote", @@ -1327,9 +1581,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d76ce4a75fb488c605c54bf610f221cea8b0dafb53333c1a67e8ee199dcd2ae3" dependencies = [ "autocfg", + "bytes", + "libc", + "memchr", + "mio", "num_cpus", "pin-project-lite", + "socket2", "tokio-macros", + "winapi", ] [[package]] @@ -1343,6 +1603,36 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + [[package]] name = "tracing" version = "0.1.37" @@ -1386,6 +1676,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + [[package]] name = "typenum" version = "1.15.0" @@ -1448,6 +1744,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.10.0+wasi-snapshot-preview1" @@ -1485,6 +1791,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.83" @@ -1663,3 +1981,12 @@ name = "windows_x86_64_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] From cd9c69baf1ecb47865da2db8902b580155c9278e Mon Sep 17 00:00:00 2001 From: ByteDream Date: Fri, 2 Dec 2022 20:23:30 +0100 Subject: [PATCH 039/441] Fix windows artifact name --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0e5960..3900976 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -126,7 +126,7 @@ jobs: - name: Upload binary artifact uses: actions/upload-artifact@v3 with: - name: crunchy-cli_windows.exe + name: crunchy-cli_windows path: ./target/release/crunchy-cli.exe if-no-files-found: error From e9b3088cde86aa8468a47602e2694b5799fdc081 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Fri, 2 Dec 2022 21:42:24 +0100 Subject: [PATCH 040/441] Fix windows ok output character --- crunchy-cli-core/src/cli/log.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/cli/log.rs b/crunchy-cli-core/src/cli/log.rs index 3539733..d1f0225 100644 --- a/crunchy-cli-core/src/cli/log.rs +++ b/crunchy-cli-core/src/cli/log.rs @@ -19,6 +19,12 @@ impl CliProgress { let init_message = format!("{}", record.args()); let init_level = record.level(); let handler = thread::spawn(move || { + #[cfg(not(windows))] + let ok = '✔'; + #[cfg(windows)] + // windows does not support all unicode characters by default in their consoles, so + // we're using this (square root?) symbol instead. microsoft. + let ok = '√'; let states = ["-", "\\", "|", "/"]; let mut old_message = init_message.clone(); @@ -68,7 +74,7 @@ impl CliProgress { // clear last line // prefix (2), space (1), state (1), space (1), message(n) let _ = write!(stdout(), "\r {}", " ".repeat(old_message.len())); - let _ = writeln!(stdout(), "\r:: ✓ {}", old_message); + let _ = writeln!(stdout(), "\r:: {} {}", ok, old_message); let _ = stdout().flush(); }); From afab3826c945d5906aaaf07e0d6b43749e7a338d Mon Sep 17 00:00:00 2001 From: ByteDream Date: Fri, 2 Dec 2022 22:06:08 +0100 Subject: [PATCH 041/441] Extend function to get free file --- Cargo.lock | 11 +++++++++++ crunchy-cli-core/Cargo.lock | 11 +++++++++++ crunchy-cli-core/Cargo.toml | 1 + crunchy-cli-core/src/cli/archive.rs | 5 ++--- crunchy-cli-core/src/cli/download.rs | 9 ++------- crunchy-cli-core/src/utils/os.rs | 17 ++++++++++------- 6 files changed, 37 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ffdd71b..ff1f3dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -292,6 +292,7 @@ dependencies = [ "num_cpus", "regex", "rustls-native-certs", + "sanitize-filename", "signal-hook", "sys-locale", "tempfile", @@ -1361,6 +1362,16 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +[[package]] +name = "sanitize-filename" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c502bdb638f1396509467cb0580ef3b29aa2a45c5d43e5d84928241280296c" +dependencies = [ + "lazy_static", + "regex", +] + [[package]] name = "schannel" version = "0.1.20" diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index 4fa983a..b842f38 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -260,6 +260,7 @@ dependencies = [ "num_cpus", "regex", "rustls-native-certs", + "sanitize-filename", "signal-hook", "sys-locale", "tempfile", @@ -1323,6 +1324,16 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +[[package]] +name = "sanitize-filename" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c502bdb638f1396509467cb0580ef3b29aa2a45c5d43e5d84928241280296c" +dependencies = [ + "lazy_static", + "regex", +] + [[package]] name = "schannel" version = "0.1.20" diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 30e6d44..d797098 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -20,6 +20,7 @@ isahc = { git = "https://github.com/sagebind/isahc", rev = "c39f6f8" } log = { version = "0.4", features = ["std"] } num_cpus = "1.13" regex = "1.6" +sanitize-filename = "0.4" signal-hook = "0.3" tempfile = "3.3" terminal_size = "0.2" diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index a057aed..e798e75 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -206,8 +206,7 @@ impl Execute for Archive { .to_string(), primary, )), - ) - .0; + ); info!( "Downloading {} to '{}'", @@ -387,7 +386,7 @@ async fn download_video(ctx: &Context, format: &Format, only_audio: bool) -> Res ctx, &mut ffmpeg.stdin.unwrap(), Some(format!("Download {}", format.audio)), - format.stream.clone() + format.stream.clone(), ) .await?; diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index 8535869..acb086b 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -178,8 +178,7 @@ impl Execute for Download { .to_string(), &format, )), - ) - .0; + ); let use_ffmpeg = if let Some(extension) = path.extension() { if extension != "ts" { @@ -228,11 +227,7 @@ impl Execute for Download { } } -async fn download_ffmpeg( - ctx: &Context, - variant_data: VariantData, - target: &Path, -) -> Result<()> { +async fn download_ffmpeg(ctx: &Context, variant_data: VariantData, target: &Path) -> Result<()> { let ffmpeg = Command::new("ffmpeg") .stdin(Stdio::piped()) .stdout(Stdio::null()) diff --git a/crunchy-cli-core/src/utils/os.rs b/crunchy-cli-core/src/utils/os.rs index 317381d..92e4e9f 100644 --- a/crunchy-cli-core/src/utils/os.rs +++ b/crunchy-cli-core/src/utils/os.rs @@ -19,7 +19,7 @@ pub fn has_ffmpeg() -> bool { } } -/// Any tempfiles should be created with this function. The prefix and directory of every file +/// Any tempfile should be created with this function. The prefix and directory of every file /// created with this method stays the same which is helpful to query all existing tempfiles and /// e.g. remove them in a case of ctrl-c. Having one function also good to prevent mistakes like /// setting the wrong prefix if done manually. @@ -36,17 +36,20 @@ pub fn tempfile>(suffix: S) -> io::Result { } /// Check if the given path exists and rename it until the new (renamed) file does not exist. -pub fn free_file(mut path: PathBuf) -> (PathBuf, bool) { +pub fn free_file(mut path: PathBuf) -> PathBuf { let mut i = 0; while path.exists() { i += 1; - let ext = path.extension().unwrap().to_str().unwrap(); - let mut filename = path.file_name().unwrap().to_str().unwrap(); - - filename = &filename[0..filename.len() - ext.len() - 1]; + let ext = path.extension().unwrap().to_string_lossy(); + let filename = path.file_stem().unwrap().to_string_lossy(); path.set_file_name(format!("{} ({}).{}", filename, i, ext)) } - (path, i != 0) + sanitize_file(path) +} + +/// Sanitizes the given path to not contain any invalid file character. +pub fn sanitize_file(path: PathBuf) -> PathBuf { + path.with_file_name(sanitize_filename::sanitize(path.file_name().unwrap().to_string_lossy())) } From 9d45995e86c4c2b1c8979fa71e6e7abba14937fd Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sat, 3 Dec 2022 00:26:52 +0100 Subject: [PATCH 042/441] Set progress width with message to use complete space --- crunchy-cli-core/src/cli/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index e513101..6197684 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -77,7 +77,7 @@ pub async fn download_segments( let progress_available = size - if let Some(msg) = &message { - 35 + msg.len() + 34 + msg.len() } else { 33 }; From 135d59ce8b3ac8230c385c3a9469564bd460ce5b Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sat, 3 Dec 2022 00:39:52 +0100 Subject: [PATCH 043/441] Add pre-check function --- crunchy-cli-core/src/cli/archive.rs | 12 +++++++++++- crunchy-cli-core/src/cli/download.rs | 27 +++++++++++---------------- crunchy-cli-core/src/lib.rs | 3 ++- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index e798e75..d4b0a5d 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -3,7 +3,7 @@ use crate::cli::utils::{download_segments, find_resolution}; use crate::utils::context::Context; use crate::utils::format::{format_string, Format}; use crate::utils::log::progress; -use crate::utils::os::{free_file, tempfile}; +use crate::utils::os::{free_file, has_ffmpeg, tempfile}; use crate::utils::parse::{parse_url, UrlFilter}; use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number}; use crate::Execute; @@ -110,6 +110,16 @@ pub struct Archive { #[async_trait::async_trait(?Send)] impl Execute for Archive { + fn pre_check(&self) -> Result<()> { + if !has_ffmpeg() { + bail!("FFmpeg is needed to run this command") + } else if PathBuf::from(&self.output).extension().unwrap_or_default().to_string_lossy() != "mkv" { + bail!("File extension is not '.mkv'. Currently only matroska / '.mkv' files are supported") + } + + Ok(()) + } + async fn execute(self, ctx: Context) -> Result<()> { let mut parsed_urls = vec![]; diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index acb086b..b0d0ffa 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -65,6 +65,16 @@ pub struct Download { #[async_trait::async_trait(?Send)] impl Execute for Download { + fn pre_check(&self) -> Result<()> { + if has_ffmpeg() { + debug!("FFmpeg detected") + } else if PathBuf::from(&self.output).extension().unwrap_or_default().to_string_lossy() != "ts" { + bail!("File extension is not '.ts'. If you want to use a custom file format, please install ffmpeg") + } + + Ok(()) + } + async fn execute(self, ctx: Context) -> Result<()> { let mut parsed_urls = vec![]; @@ -180,21 +190,6 @@ impl Execute for Download { )), ); - let use_ffmpeg = if let Some(extension) = path.extension() { - if extension != "ts" { - if !has_ffmpeg() { - bail!( - "File ending is not `.ts`, ffmpeg is required to convert the video" - ) - } - true - } else { - false - } - } else { - false - }; - info!( "Downloading {} to '{}'", format.title, @@ -211,7 +206,7 @@ impl Execute for Download { tab_info!("Resolution: {}", format.stream.resolution); tab_info!("FPS: {:.2}", format.stream.fps); - if use_ffmpeg { + if path.extension().unwrap_or_default().to_string_lossy() != "ts" { download_ffmpeg(&ctx, format.stream, path.as_path()).await?; } else if path.to_str().unwrap() == "-" { let mut stdout = std::io::stdout().lock(); diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index 083b401..c895d51 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -16,6 +16,7 @@ pub use cli::{archive::Archive, download::Download, login::Login}; #[async_trait::async_trait(?Send)] trait Execute { + fn pre_check(&self) -> Result<()> { Ok(()) } async fn execute(self, ctx: Context) -> Result<()>; } @@ -163,7 +164,7 @@ pub async fn cli_entrypoint() { } }; if let Err(err) = result { - error!("{}", err); + error!("a unexpected error occurred: {}", err); std::process::exit(1) } } From 5826d95e6aa2964dfabf189a3b785bab89972c14 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sat, 3 Dec 2022 01:04:29 +0100 Subject: [PATCH 044/441] Add progress width offset --- crunchy-cli-core/src/cli/utils.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index 6197684..0bf6b45 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -75,11 +75,17 @@ pub async fn download_segments( .0 .0 as usize; + // there is a offset of 1 "length" (idk how to describe it), so removing 1 from + // `progress_available` would fill the terminal width completely. on multiple + // systems there is a bug that printing until the end of the line causes a newline + // even though technically there shouldn't be one. on my tests, this only happens on + // windows and mac machines and (at the addressed environments) only with release + // builds. so maybe an unwanted optimization? let progress_available = size - if let Some(msg) = &message { - 34 + msg.len() + 35 + msg.len() } else { - 33 + 34 }; let progress_done_count = (progress_available as f64 * (percentage / 100f64)).ceil() as usize; From cd1308426ee6611292fcdb6bfcf2d24da1a18ea0 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sat, 3 Dec 2022 12:26:36 +0100 Subject: [PATCH 045/441] Re-add static-ssl feature --- Cargo.lock | 30 ++++++++++++++++++++---------- Cargo.toml | 3 +++ crunchy-cli-core/Cargo.lock | 30 ++++++++++++++++++++---------- crunchy-cli-core/Cargo.toml | 2 +- 4 files changed, 44 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ff1f3dd..720a1c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -385,9 +385,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4a41a86530d0fe7f5d9ea779916b7cadd2d4f9add748b99c2c029cbbdfaf453" +checksum = "bdf07d07d6531bfcdbe9b8b739b104610c6508dcc4d63b410585faf338241daf" dependencies = [ "cc", "cxxbridge-flags", @@ -397,9 +397,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06416d667ff3e3ad2df1cd8cd8afae5da26cf9cec4d0825040f88b5ca659a2f0" +checksum = "d2eb5b96ecdc99f72657332953d4d9c50135af1bac34277801cc3937906ebd39" dependencies = [ "cc", "codespan-reporting", @@ -412,15 +412,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "820a9a2af1669deeef27cb271f476ffd196a2c4b6731336011e0ba63e2c7cf71" +checksum = "ac040a39517fd1674e0f32177648334b0f4074625b5588a64519804ba0553b12" [[package]] name = "cxxbridge-macro" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08a6e2fcc370a089ad3b4aaf54db3b1b4cee38ddabce5896b33eb693275f470" +checksum = "1362b0ddcfc4eb0a1f57b68bd77dd99f0e826958a96abd0ae9bd092e114ffed6" dependencies = [ "proc-macro2", "quote", @@ -910,9 +910,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.137" +version = "0.2.138" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" +checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8" [[package]] name = "libnghttp2-sys" @@ -1107,6 +1107,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-src" +version = "111.24.0+1.1.1s" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3498f259dab01178c6228c6b00dcef0ed2a2d5e20d648c017861227773ea4abd" +dependencies = [ + "cc", +] + [[package]] name = "openssl-sys" version = "0.9.78" @@ -1116,6 +1125,7 @@ dependencies = [ "autocfg", "cc", "libc", + "openssl-src", "pkg-config", "vcpkg", ] diff --git a/Cargo.toml b/Cargo.toml index d08c47f..1b815c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,9 @@ default = ["static-curl"] # Embed a static curl library into the binary instead of just linking it. static-curl = ["crunchy-cli-core/static-curl"] +# Embed a static tls library into the binary instead of just linking it. +# has no effect on Windows, always activated there. +static-ssl = ["crunchy-cli-core/static-ssl"] [dependencies] tokio = { version = "1.22", features = ["macros", "rt-multi-thread", "time"], default-features = false } diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index b842f38..2612b36 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -353,9 +353,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4a41a86530d0fe7f5d9ea779916b7cadd2d4f9add748b99c2c029cbbdfaf453" +checksum = "bdf07d07d6531bfcdbe9b8b739b104610c6508dcc4d63b410585faf338241daf" dependencies = [ "cc", "cxxbridge-flags", @@ -365,9 +365,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06416d667ff3e3ad2df1cd8cd8afae5da26cf9cec4d0825040f88b5ca659a2f0" +checksum = "d2eb5b96ecdc99f72657332953d4d9c50135af1bac34277801cc3937906ebd39" dependencies = [ "cc", "codespan-reporting", @@ -380,15 +380,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "820a9a2af1669deeef27cb271f476ffd196a2c4b6731336011e0ba63e2c7cf71" +checksum = "ac040a39517fd1674e0f32177648334b0f4074625b5588a64519804ba0553b12" [[package]] name = "cxxbridge-macro" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08a6e2fcc370a089ad3b4aaf54db3b1b4cee38ddabce5896b33eb693275f470" +checksum = "1362b0ddcfc4eb0a1f57b68bd77dd99f0e826958a96abd0ae9bd092e114ffed6" dependencies = [ "proc-macro2", "quote", @@ -878,9 +878,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.137" +version = "0.2.138" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" +checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8" [[package]] name = "libnghttp2-sys" @@ -1075,6 +1075,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-src" +version = "111.24.0+1.1.1s" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3498f259dab01178c6228c6b00dcef0ed2a2d5e20d648c017861227773ea4abd" +dependencies = [ + "cc", +] + [[package]] name = "openssl-sys" version = "0.9.78" @@ -1084,6 +1093,7 @@ dependencies = [ "autocfg", "cc", "libc", + "openssl-src", "pkg-config", "vcpkg", ] diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index d797098..395030a 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -5,8 +5,8 @@ version = "3.0.0-dev.1" edition = "2021" [features] -# Embed a static curl library into the binary instead of just linking it. static-curl = ["crunchyroll-rs/static-curl"] +static-ssl = ["isahc/native-tls-static"] [dependencies] anyhow = "1.0" From 64717fd405256daa3a764107da92c8bfad309138 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sat, 3 Dec 2022 12:32:45 +0100 Subject: [PATCH 046/441] Update dependencies --- Cargo.lock | 4 ++-- crunchy-cli-core/Cargo.lock | 4 ++-- crunchy-cli-core/Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 720a1c4..108a431 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -303,7 +303,7 @@ dependencies = [ [[package]] name = "crunchyroll-rs" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#66678d49bc071f49c83a7fa73ab280d1c1de5b2f" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#7644b24864608c059d702db8b373be81ffcec643" dependencies = [ "aes", "cbc", @@ -325,7 +325,7 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#66678d49bc071f49c83a7fa73ab280d1c1de5b2f" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#7644b24864608c059d702db8b373be81ffcec643" dependencies = [ "darling", "quote", diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index 2612b36..d0f951a 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -271,7 +271,7 @@ dependencies = [ [[package]] name = "crunchyroll-rs" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#66678d49bc071f49c83a7fa73ab280d1c1de5b2f" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#7644b24864608c059d702db8b373be81ffcec643" dependencies = [ "aes", "cbc", @@ -293,7 +293,7 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#66678d49bc071f49c83a7fa73ab280d1c1de5b2f" +source = "git+https://github.com/crunchy-labs/crunchyroll-rs#7644b24864608c059d702db8b373be81ffcec643" dependencies = [ "darling", "quote", diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 395030a..a1d898e 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" [features] static-curl = ["crunchyroll-rs/static-curl"] -static-ssl = ["isahc/native-tls-static"] +static-ssl = ["crunchyroll-rs/static-ssl"] [dependencies] anyhow = "1.0" From 7c3bbfc1737ecdaf291d25ca817f1d35eccd21e3 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 4 Dec 2022 18:54:19 +0100 Subject: [PATCH 047/441] Wait until buffer is empty when downloading --- crunchy-cli-core/src/cli/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index 0bf6b45..758636d 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -169,7 +169,7 @@ pub async fn download_segments( data_pos += 1; } - if *count.lock().unwrap() >= total_segments { + if *count.lock().unwrap() >= total_segments && buf.is_empty() { break; } } From 285d27772c10b288022bda19f4b027505f96c284 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 4 Dec 2022 18:54:22 +0100 Subject: [PATCH 048/441] Add manually .ass file editing to fix #32 --- crunchy-cli-core/src/cli/archive.rs | 108 ++++++++++++++++++++++------ 1 file changed, 86 insertions(+), 22 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index d4b0a5d..d586725 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -8,6 +8,7 @@ use crate::utils::parse::{parse_url, UrlFilter}; use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number}; use crate::Execute; use anyhow::{bail, Result}; +use chrono::{NaiveTime, Timelike}; use crunchyroll_rs::media::{Resolution, StreamSubtitle}; use crunchyroll_rs::{Locale, Media, MediaCollection, Series}; use log::{debug, error, info}; @@ -113,7 +114,12 @@ impl Execute for Archive { fn pre_check(&self) -> Result<()> { if !has_ffmpeg() { bail!("FFmpeg is needed to run this command") - } else if PathBuf::from(&self.output).extension().unwrap_or_default().to_string_lossy() != "mkv" { + } else if PathBuf::from(&self.output) + .extension() + .unwrap_or_default() + .to_string_lossy() + != "mkv" + { bail!("File extension is not '.mkv'. Currently only matroska / '.mkv' files are supported") } @@ -276,9 +282,18 @@ impl Execute for Archive { } } + let (primary_video, _) = video_paths.get(0).unwrap(); + let primary_video_length = get_video_length(primary_video.to_path_buf()).unwrap(); for subtitle in subtitles { - subtitle_paths - .push((download_subtitle(&self, subtitle.clone()).await?, subtitle)) + subtitle_paths.push(( + download_subtitle( + &self, + subtitle.clone(), + primary_video_length + ) + .await?, + subtitle, + )) } generate_mkv(&self, path, video_paths, audio_paths, subtitle_paths)? @@ -403,15 +418,20 @@ async fn download_video(ctx: &Context, format: &Format, only_audio: bool) -> Res Ok(path) } -async fn download_subtitle(archive: &Archive, subtitle: StreamSubtitle) -> Result { +async fn download_subtitle( + archive: &Archive, + subtitle: StreamSubtitle, + max_length: NaiveTime, +) -> Result { let tempfile = tempfile(".ass")?; let (mut file, path) = tempfile.into_parts(); let mut buf = vec![]; subtitle.write_to(&mut buf).await?; if !archive.no_subtitle_optimizations { - buf = fix_subtitle(buf) + buf = fix_subtitle_look_and_feel(buf) } + buf = fix_subtitle_length(buf, max_length); file.write_all(buf.as_slice())?; @@ -421,7 +441,7 @@ async fn download_subtitle(archive: &Archive, subtitle: StreamSubtitle) -> Resul /// Add `ScaledBorderAndShadows: yes` to subtitles; without it they look very messy on some video /// players. See [crunchy-labs/crunchy-cli#66](https://github.com/crunchy-labs/crunchy-cli/issues/66) /// for more information. -fn fix_subtitle(raw: Vec) -> Vec { +fn fix_subtitle_look_and_feel(raw: Vec) -> Vec { let mut script_info = false; let mut new = String::new(); @@ -439,6 +459,62 @@ fn fix_subtitle(raw: Vec) -> Vec { new.into_bytes() } +/// Fix the length of subtitles to a specified maximum amount. This is required because sometimes +/// subtitles have an unnecessary entry long after the actual video ends with artificially extends +/// the video length on some video players. To prevent this, the video length must be hard set. See +/// [crunchy-labs/crunchy-cli#32](https://github.com/crunchy-labs/crunchy-cli/issues/32) for more +/// information. +fn fix_subtitle_length(raw: Vec, max_length: NaiveTime) -> Vec { + let re = + Regex::new(r#"^Dialogue:\s\d+,(?P\d+:\d+:\d+\.\d+),(?P\d+:\d+:\d+\.\d+),"#) + .unwrap(); + + // chrono panics if we try to format NaiveTime with `%2f` and the nano seconds has more than 2 + // digits so them have to be reduced manually to avoid the panic + fn format_naive_time(native_time: NaiveTime) -> String { + let formatted_time = native_time.format("%f").to_string(); + format!("{}.{}", native_time.format("%T"), if formatted_time.len() <= 2 { + native_time.format("%2f").to_string() + } else { + formatted_time.split_at(2).0.parse().unwrap() + }) + } + + let length_as_string = format_naive_time(max_length); + let mut new = String::new(); + + for line in String::from_utf8_lossy(raw.as_slice()).split('\n') { + if let Some(capture) = re.captures(line) { + let start = capture.name("start").map_or(NaiveTime::default(), |s| { + NaiveTime::parse_from_str(s.as_str(), "%H:%M:%S.%f").unwrap() + }); + let end = capture.name("end").map_or(NaiveTime::default(), |s| { + NaiveTime::parse_from_str(s.as_str(), "%H:%M:%S.%f").unwrap() + }); + + if start > max_length { + continue + } else if end > max_length { + new.push_str(re.replace( + line, + format!( + "Dialogue: {},{},", + format_naive_time(start), + &length_as_string + ), + ).to_string().as_str()) + } else { + new.push_str(line) + } + } else { + new.push_str(line) + } + new.push('\n') + } + + new.into_bytes() +} + fn generate_mkv( archive: &Archive, target: PathBuf, @@ -450,8 +526,6 @@ fn generate_mkv( let mut maps = vec![]; let mut metadata = vec![]; - let mut video_length = (0, 0, 0, 0); - for (i, (video_path, format)) in video_paths.iter().enumerate() { input.extend(["-i".to_string(), video_path.to_string_lossy().to_string()]); maps.extend(["-map".to_string(), i.to_string()]); @@ -471,11 +545,6 @@ fn generate_mkv( format!("-metadata:s:a:{}", i), format!("title={}", format.audio.to_human_readable()), ]); - - let vid_len = get_video_length(video_path.to_path_buf())?; - if vid_len > video_length { - video_length = vid_len - } } for (i, (audio_path, format)) in audio_paths.iter().enumerate() { input.extend(["-i".to_string(), audio_path.to_string_lossy().to_string()]); @@ -552,11 +621,11 @@ fn generate_mkv( /// Get the length of a video. This is required because sometimes subtitles have an unnecessary entry /// long after the actual video ends with artificially extends the video length on some video players. -/// To prevent this, the video length must be hard set with ffmpeg. See +/// To prevent this, the video length must be hard set. See /// [crunchy-labs/crunchy-cli#32](https://github.com/crunchy-labs/crunchy-cli/issues/32) for more /// information. -fn get_video_length(path: PathBuf) -> Result<(u32, u32, u32, u32)> { - let video_length = Regex::new(r"Duration:\s?(\d+):(\d+):(\d+).(\d+),")?; +fn get_video_length(path: PathBuf) -> Result { + let video_length = Regex::new(r"Duration:\s(?P - CI + CI

From cc9342cd0acb8809410ae6b42953690ce4bc76bb Mon Sep 17 00:00:00 2001 From: bytedream Date: Fri, 16 Dec 2022 10:09:41 +0100 Subject: [PATCH 072/441] Update dependencies --- .github/workflows/ci.yml | 75 +--- Cargo.lock | 506 +++++++++------------------ Cargo.toml | 16 +- build.rs | 3 - crunchy-cli-core/Cargo.lock | 505 ++++++++++---------------- crunchy-cli-core/Cargo.toml | 17 +- crunchy-cli-core/src/cli/archive.rs | 6 +- crunchy-cli-core/src/cli/download.rs | 6 +- crunchy-cli-core/src/cli/utils.rs | 7 +- 9 files changed, 373 insertions(+), 768 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eddc7b2..caef473 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,9 +16,15 @@ jobs: - os: ubuntu-latest toolchain: x86_64-unknown-linux-musl platform: linux + ext: + - os: windows-latest + toolchain: x86_64-pc-windows-gnu + platform: windows + ext: .exe - os: macos-latest toolchain: x86_64-apple-darwin platform: darwin + ext: steps: - name: Checkout uses: actions/checkout@v3 @@ -58,74 +64,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: crunchy-cli_${{ matrix.platform }} - path: ./target/release/crunchy-cli - if-no-files-found: error - - - name: Upload manpages artifact - uses: actions/upload-artifact@v3 - with: - name: manpages - path: ./target/release/manpages - if-no-files-found: error - - - name: Upload completions artifact - uses: actions/upload-artifact@v3 - with: - name: completions - path: ./target/release/completions - if-no-files-found: error - - build-windows: - runs-on: windows-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - - run: vcpkg integrate install - - - name: Install OpenSSL - run: vcpkg install openssl:x64-windows-static-md - - - name: Set env variables - shell: bash - run: echo "CFLAGS=-I$(echo $VCPKG_INSTALLATION_ROOT)\packages\openssl_x64-windows-static-md\include" >> $GITHUB_ENV - - - name: Cargo cache # https://github.com/actions/cache/blob/main/examples.md#rust---cargo - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - name: Install toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - target: x86_64-pc-windows-msvc - default: true - - - name: Test - uses: actions-rs/cargo@v1 - with: - command: test - args: --release --all-features - - - name: Build - uses: actions-rs/cargo@v1 - with: - command: build - args: --release --all-features - - - name: Upload binary artifact - uses: actions/upload-artifact@v3 - with: - name: crunchy-cli_windows - path: ./target/release/crunchy-cli.exe + path: ./target/release/crunchy-cli${{ matrix.ext }} if-no-files-found: error - name: Upload manpages artifact diff --git a/Cargo.lock b/Cargo.lock index a31d2fa..15acee9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,17 +37,6 @@ version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" -[[package]] -name = "async-channel" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833" -dependencies = [ - "concurrent-queue", - "event-listener", - "futures-core", -] - [[package]] name = "async-trait" version = "0.1.59" @@ -110,15 +99,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" -[[package]] -name = "castaway" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a17ed5635fc8536268e5d4de1e22e81ac34419e5f052d4d51f4e01dcc263fcc" -dependencies = [ - "rustversion", -] - [[package]] name = "cbc" version = "0.1.2" @@ -130,9 +110,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.77" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4" +checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" [[package]] name = "cfg-if" @@ -233,12 +213,30 @@ dependencies = [ ] [[package]] -name = "concurrent-queue" -version = "2.0.0" +name = "cookie" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd7bef69dc86e3c610e4e7aed41035e2a7ed12e72dd7530f61327a6579a4390b" +checksum = "344adc371239ef32293cb1c4fe519592fcf21206c79c02854320afcdf3ab4917" dependencies = [ - "crossbeam-utils", + "percent-encoding", + "time 0.3.17", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e4b6aa369f41f5faa04bb80c9b1f4216ea81646ed6124d76ba5c49a7aafd9cd" +dependencies = [ + "cookie", + "idna 0.2.3", + "log", + "publicsuffix", + "serde", + "serde_json", + "time 0.3.17", + "url", ] [[package]] @@ -266,15 +264,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crossbeam-utils" -version = "0.8.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" -dependencies = [ - "cfg-if", -] - [[package]] name = "crunchy-cli" version = "3.0.0-dev.3" @@ -284,7 +273,6 @@ dependencies = [ "clap_complete", "clap_mangen", "crunchy-cli-core", - "static_vcruntime", "tokio", ] @@ -300,11 +288,9 @@ dependencies = [ "csv", "ctrlc", "dirs", - "isahc", "log", "num_cpus", "regex", - "rustls-native-certs", "sanitize-filename", "serde", "serde_json", @@ -317,31 +303,32 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#86fb8307a531aedec708dd1c8c88b76bcf2a8c38" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5032be36dc6c4757af4f28152659e7a772b48f6925f0e0ac81b5c384e291314" dependencies = [ "aes", "cbc", "chrono", "crunchyroll-rs-internal", - "curl-sys", - "dash-mpd", "http", - "isahc", "m3u8-rs", "regex", "reqwest", + "rustls", "serde", "serde_json", "serde_urlencoded", "smart-default", "tokio", + "webpki-roots", ] [[package]] name = "crunchyroll-rs-internal" -version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#86fb8307a531aedec708dd1c8c88b76bcf2a8c38" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e4aa1f09fd76f44329455abfac35e22c654ac782e93bc8a7d3ee1be27509f4" dependencies = [ "darling", "quote", @@ -382,43 +369,12 @@ dependencies = [ [[package]] name = "ctrlc" -version = "3.2.3" +version = "3.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d91974fbbe88ec1df0c24a4f00f99583667a7e2e6272b2b92d294d81e462173" +checksum = "1631ca6e3c59112501a9d87fd86f21591ff77acd31331e8a73f8d80a65bbdd71" dependencies = [ "nix", - "winapi", -] - -[[package]] -name = "curl" -version = "0.4.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "509bd11746c7ac09ebd19f0b17782eae80aadee26237658a6b4808afb5c11a22" -dependencies = [ - "curl-sys", - "libc", - "openssl-probe", - "openssl-sys", - "schannel", - "socket2", - "winapi", -] - -[[package]] -name = "curl-sys" -version = "0.4.59+curl-7.86.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cfce34829f448b08f55b7db6d0009e23e2e86a34e8c2b366269bf5799b4a407" -dependencies = [ - "cc", - "libc", - "libnghttp2-sys", - "libz-sys", - "openssl-sys", - "pkg-config", - "vcpkg", - "winapi", + "windows-sys 0.42.0", ] [[package]] @@ -500,28 +456,6 @@ dependencies = [ "syn", ] -[[package]] -name = "dash-mpd" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6a181b3a4a1ef2cb4dd72f2a85e36a2e54445b8b9513635d3fad23e9b9a7c4c" -dependencies = [ - "chrono", - "log", - "quick-xml", - "regex", - "serde", - "serde_with", - "thiserror", - "xattr", -] - -[[package]] -name = "data-encoding" -version = "2.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb" - [[package]] name = "dirs" version = "4.0.0" @@ -572,12 +506,6 @@ dependencies = [ "libc", ] -[[package]] -name = "event-listener" -version = "2.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" - [[package]] name = "fastrand" version = "1.8.0" @@ -638,16 +566,6 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" -[[package]] -name = "futures-lite" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" -dependencies = [ - "futures-core", - "pin-project-lite", -] - [[package]] name = "futures-sink" version = "0.3.25" @@ -745,12 +663,6 @@ dependencies = [ "libc", ] -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - [[package]] name = "http" version = "0.2.8" @@ -809,6 +721,19 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" +dependencies = [ + "http", + "hyper", + "rustls", + "tokio", + "tokio-rustls", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -852,6 +777,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "idna" version = "0.3.0" @@ -870,7 +806,6 @@ checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" dependencies = [ "autocfg", "hashbrown", - "serde", ] [[package]] @@ -904,9 +839,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec947b7a4ce12e3b87e353abae7ce124d025b6c7d6c5aea5cc0bcf92e9510ded" +checksum = "11b0d96e660696543b251e58030cf9787df56da39dab19ad60eae7353040917e" [[package]] name = "is-terminal" @@ -920,34 +855,6 @@ dependencies = [ "windows-sys 0.42.0", ] -[[package]] -name = "isahc" -version = "1.7.0" -source = "git+https://github.com/sagebind/isahc?rev=c39f6f8#c39f6f85aaa1f36c5857064c6c3c80f4d307d863" -dependencies = [ - "async-channel", - "castaway", - "crossbeam-utils", - "curl", - "curl-sys", - "data-encoding", - "encoding_rs", - "event-listener", - "futures-io", - "futures-lite", - "http", - "httpdate", - "log", - "mime", - "once_cell", - "polling", - "sluice", - "tracing", - "tracing-futures", - "url", - "waker-fn", -] - [[package]] name = "itoa" version = "0.4.8" @@ -981,28 +888,6 @@ version = "0.2.138" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8" -[[package]] -name = "libnghttp2-sys" -version = "0.1.7+1.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57ed28aba195b38d5ff02b9170cbff627e336a20925e43b4945390401c5dc93f" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "libz-sys" -version = "1.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "link-cplusplus" version = "1.0.7" @@ -1014,9 +899,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f9f08d8963a6c613f4b1a78f4f4a4dbfadf8e6545b2d72861731e4858b8b47f" +checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" [[package]] name = "log" @@ -1037,6 +922,12 @@ dependencies = [ "nom", ] +[[package]] +name = "matches" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" + [[package]] name = "memchr" version = "2.5.0" @@ -1087,14 +978,14 @@ dependencies = [ [[package]] name = "nix" -version = "0.25.1" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4" +checksum = "46a58d1d356c6597d08cde02c2f09d785b09e28711837b1ed667dc652c08a694" dependencies = [ - "autocfg", "bitflags", "cfg-if", "libc", + "static_assertions", ] [[package]] @@ -1174,15 +1065,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" -[[package]] -name = "openssl-src" -version = "111.24.0+1.1.1s" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3498f259dab01178c6228c6b00dcef0ed2a2d5e20d648c017861227773ea4abd" -dependencies = [ - "cc", -] - [[package]] name = "openssl-sys" version = "0.9.79" @@ -1192,7 +1074,6 @@ dependencies = [ "autocfg", "cc", "libc", - "openssl-src", "pkg-config", "vcpkg", ] @@ -1209,26 +1090,6 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" -[[package]] -name = "pin-project" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "pin-project-lite" version = "0.2.9" @@ -1247,20 +1108,6 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" -[[package]] -name = "polling" -version = "2.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "166ca89eb77fd403230b9c156612965a81e094ec6ec3aa13663d4c8b113fa748" -dependencies = [ - "autocfg", - "cfg-if", - "libc", - "log", - "wepoll-ffi", - "windows-sys 0.42.0", -] - [[package]] name = "proc-macro-error" version = "1.0.4" @@ -1285,6 +1132,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + [[package]] name = "proc-macro2" version = "1.0.47" @@ -1295,13 +1148,19 @@ dependencies = [ ] [[package]] -name = "quick-xml" -version = "0.26.0" +name = "psl-types" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" +checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac" + +[[package]] +name = "publicsuffix" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a8c1bda5ae1af7f99a2962e49df150414a43d62404644d98dd5c3a93d07457" dependencies = [ - "memchr", - "serde", + "idna 0.3.0", + "psl-types", ] [[package]] @@ -1373,6 +1232,8 @@ checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" dependencies = [ "base64", "bytes", + "cookie", + "cookie_store", "encoding_rs", "futures-core", "futures-util", @@ -1380,6 +1241,7 @@ dependencies = [ "http", "http-body", "hyper", + "hyper-rustls", "hyper-tls", "ipnet", "js-sys", @@ -1389,19 +1251,39 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", + "proc-macro-hack", + "rustls", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", "tokio", "tokio-native-tls", + "tokio-rustls", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", + "webpki-roots", "winreg", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + [[package]] name = "roff" version = "0.2.1" @@ -1423,15 +1305,15 @@ dependencies = [ ] [[package]] -name = "rustls-native-certs" -version = "0.6.2" +name = "rustls" +version = "0.20.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" +checksum = "539a2bfe908f471bfa933876bd1eb6a19cf2176d375f82ef7f99530a40e48c2c" dependencies = [ - "openssl-probe", - "rustls-pemfile", - "schannel", - "security-framework", + "log", + "ring", + "sct", + "webpki", ] [[package]] @@ -1443,12 +1325,6 @@ dependencies = [ "base64", ] -[[package]] -name = "rustversion" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8" - [[package]] name = "ryu" version = "1.0.11" @@ -1481,6 +1357,16 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "security-framework" version = "2.7.0" @@ -1506,18 +1392,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "256b9932320c590e707b94576e3cc1f7c9024d0ee6612dfbcf1cb106cbe8e055" +checksum = "e326c9ec8042f1b5da33252c8a37e9ffbd2c9bef0155215b6e6c80c790e05f91" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4eae9b04cbffdfd550eb462ed33bc6a1b68c935127d008b27444d08380f94e4" +checksum = "42a3df25b0713732468deadad63ab9da1f1fd75a48a15024b50363f128db627e" dependencies = [ "proc-macro2", "quote", @@ -1547,34 +1433,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_with" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25bf4a5a814902cd1014dbccfa4d4560fb8432c779471e96e035602519f82eef" -dependencies = [ - "base64", - "chrono", - "hex", - "indexmap", - "serde", - "serde_json", - "serde_with_macros", - "time 0.3.17", -] - -[[package]] -name = "serde_with_macros" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3452b4c0f6c1e357f73fdb87cd1efabaa12acf328c7a528e252893baeb3f4aa" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "signal-hook" version = "0.3.14" @@ -1603,17 +1461,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "sluice" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d7400c0eff44aa2fcb5e31a5f24ba9716ed90138769e4977a2ba6014ae63eb5" -dependencies = [ - "async-channel", - "futures-core", - "futures-io", -] - [[package]] name = "smart-default" version = "0.6.0" @@ -1636,10 +1483,16 @@ dependencies = [ ] [[package]] -name = "static_vcruntime" -version = "2.0.0" +name = "spin" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "954e3e877803def9dc46075bf4060147c55cd70db97873077232eae0269dc89b" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "strsim" @@ -1816,6 +1669,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +dependencies = [ + "rustls", + "tokio", + "webpki", +] + [[package]] name = "tokio-util" version = "0.7.4" @@ -1843,23 +1707,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", - "log", "pin-project-lite", - "tracing-attributes", "tracing-core", ] -[[package]] -name = "tracing-attributes" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "tracing-core" version = "0.1.30" @@ -1869,16 +1720,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "tracing-futures" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" -dependencies = [ - "pin-project", - "tracing", -] - [[package]] name = "try-lock" version = "0.2.3" @@ -1918,6 +1759,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "2.3.1" @@ -1925,7 +1772,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" dependencies = [ "form_urlencoded", - "idna", + "idna 0.3.0", "percent-encoding", ] @@ -1941,12 +1788,6 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" -[[package]] -name = "waker-fn" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" - [[package]] name = "want" version = "0.3.0" @@ -2046,12 +1887,22 @@ dependencies = [ ] [[package]] -name = "wepoll-ffi" -version = "0.1.2" +name = "webpki" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" dependencies = [ - "cc", + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" +dependencies = [ + "webpki", ] [[package]] @@ -2193,12 +2044,3 @@ checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ "winapi", ] - -[[package]] -name = "xattr" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc" -dependencies = [ - "libc", -] diff --git a/Cargo.toml b/Cargo.toml index a853f11..b2af93c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,17 +4,8 @@ authors = ["Crunchy Labs Maintainers"] version = "3.0.0-dev.3" edition = "2021" -[features] -default = ["static-curl"] - -# Embed a static curl library into the binary instead of just linking it. -static-curl = ["crunchy-cli-core/static-curl"] -# Embed a static tls library into the binary instead of just linking it. -# has no effect on Windows, always activated there. -static-ssl = ["crunchy-cli-core/static-ssl"] - [dependencies] -tokio = { version = "1.22", features = ["macros", "rt-multi-thread", "time"], default-features = false } +tokio = { version = "1.23", features = ["macros", "rt-multi-thread", "time"], default-features = false } crunchy-cli-core = { path = "./crunchy-cli-core" } @@ -26,10 +17,7 @@ clap_mangen = "0.2" # The static-* features must be used here since build dependency features cannot be manipulated from the features # specified in this Cargo.toml [features]. -crunchy-cli-core = { path = "./crunchy-cli-core", features = ["static-curl"] } - -[target.'cfg(all(windows, target_env = "msvc"))'.build-dependencies] -static_vcruntime = "2.0" +crunchy-cli-core = { path = "./crunchy-cli-core" } [profile.release] strip = true diff --git a/build.rs b/build.rs index 927f4dd..1e4d71f 100644 --- a/build.rs +++ b/build.rs @@ -3,9 +3,6 @@ use clap_complete::shells; use std::path::{Path, PathBuf}; fn main() -> std::io::Result<()> { - #[cfg(all(windows, target_env = "msvc"))] - static_vcruntime::metabuild(); - // note that we're using an anti-pattern here / violate the rust conventions. build script are // not supposed to write outside of 'OUT_DIR'. to have the generated files in the build "root" // (the same directory where the output binary lives) is much simpler than in 'OUT_DIR' since diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index 43acaea..aba7b4a 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -37,17 +37,6 @@ version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" -[[package]] -name = "async-channel" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833" -dependencies = [ - "concurrent-queue", - "event-listener", - "futures-core", -] - [[package]] name = "async-trait" version = "0.1.59" @@ -110,15 +99,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" -[[package]] -name = "castaway" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a17ed5635fc8536268e5d4de1e22e81ac34419e5f052d4d51f4e01dcc263fcc" -dependencies = [ - "rustversion", -] - [[package]] name = "cbc" version = "0.1.2" @@ -130,9 +110,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.77" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4" +checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" [[package]] name = "cfg-if" @@ -214,12 +194,30 @@ dependencies = [ ] [[package]] -name = "concurrent-queue" -version = "2.0.0" +name = "cookie" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd7bef69dc86e3c610e4e7aed41035e2a7ed12e72dd7530f61327a6579a4390b" +checksum = "344adc371239ef32293cb1c4fe519592fcf21206c79c02854320afcdf3ab4917" dependencies = [ - "crossbeam-utils", + "percent-encoding", + "time 0.3.17", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e4b6aa369f41f5faa04bb80c9b1f4216ea81646ed6124d76ba5c49a7aafd9cd" +dependencies = [ + "cookie", + "idna 0.2.3", + "log", + "publicsuffix", + "serde", + "serde_json", + "time 0.3.17", + "url", ] [[package]] @@ -247,15 +245,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crossbeam-utils" -version = "0.8.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" -dependencies = [ - "cfg-if", -] - [[package]] name = "crunchy-cli-core" version = "3.0.0-dev.3" @@ -268,11 +257,9 @@ dependencies = [ "csv", "ctrlc", "dirs", - "isahc", "log", "num_cpus", "regex", - "rustls-native-certs", "sanitize-filename", "serde", "serde_json", @@ -285,31 +272,32 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#86fb8307a531aedec708dd1c8c88b76bcf2a8c38" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5032be36dc6c4757af4f28152659e7a772b48f6925f0e0ac81b5c384e291314" dependencies = [ "aes", "cbc", "chrono", "crunchyroll-rs-internal", - "curl-sys", - "dash-mpd", "http", - "isahc", "m3u8-rs", "regex", "reqwest", + "rustls", "serde", "serde_json", "serde_urlencoded", "smart-default", "tokio", + "webpki-roots", ] [[package]] name = "crunchyroll-rs-internal" -version = "0.1.0" -source = "git+https://github.com/crunchy-labs/crunchyroll-rs#86fb8307a531aedec708dd1c8c88b76bcf2a8c38" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e4aa1f09fd76f44329455abfac35e22c654ac782e93bc8a7d3ee1be27509f4" dependencies = [ "darling", "quote", @@ -350,43 +338,12 @@ dependencies = [ [[package]] name = "ctrlc" -version = "3.2.3" +version = "3.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d91974fbbe88ec1df0c24a4f00f99583667a7e2e6272b2b92d294d81e462173" +checksum = "1631ca6e3c59112501a9d87fd86f21591ff77acd31331e8a73f8d80a65bbdd71" dependencies = [ "nix", - "winapi", -] - -[[package]] -name = "curl" -version = "0.4.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "509bd11746c7ac09ebd19f0b17782eae80aadee26237658a6b4808afb5c11a22" -dependencies = [ - "curl-sys", - "libc", - "openssl-probe", - "openssl-sys", - "schannel", - "socket2", - "winapi", -] - -[[package]] -name = "curl-sys" -version = "0.4.59+curl-7.86.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cfce34829f448b08f55b7db6d0009e23e2e86a34e8c2b366269bf5799b4a407" -dependencies = [ - "cc", - "libc", - "libnghttp2-sys", - "libz-sys", - "openssl-sys", - "pkg-config", - "vcpkg", - "winapi", + "windows-sys 0.42.0", ] [[package]] @@ -468,28 +425,6 @@ dependencies = [ "syn", ] -[[package]] -name = "dash-mpd" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6a181b3a4a1ef2cb4dd72f2a85e36a2e54445b8b9513635d3fad23e9b9a7c4c" -dependencies = [ - "chrono", - "log", - "quick-xml", - "regex", - "serde", - "serde_with", - "thiserror", - "xattr", -] - -[[package]] -name = "data-encoding" -version = "2.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb" - [[package]] name = "dirs" version = "4.0.0" @@ -540,12 +475,6 @@ dependencies = [ "libc", ] -[[package]] -name = "event-listener" -version = "2.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" - [[package]] name = "fastrand" version = "1.8.0" @@ -606,16 +535,6 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" -[[package]] -name = "futures-lite" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" -dependencies = [ - "futures-core", - "pin-project-lite", -] - [[package]] name = "futures-sink" version = "0.3.25" @@ -713,12 +632,6 @@ dependencies = [ "libc", ] -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - [[package]] name = "http" version = "0.2.8" @@ -777,6 +690,19 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" +dependencies = [ + "http", + "hyper", + "rustls", + "tokio", + "tokio-rustls", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -820,6 +746,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "idna" version = "0.3.0" @@ -838,7 +775,6 @@ checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" dependencies = [ "autocfg", "hashbrown", - "serde", ] [[package]] @@ -872,9 +808,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec947b7a4ce12e3b87e353abae7ce124d025b6c7d6c5aea5cc0bcf92e9510ded" +checksum = "11b0d96e660696543b251e58030cf9787df56da39dab19ad60eae7353040917e" [[package]] name = "is-terminal" @@ -888,34 +824,6 @@ dependencies = [ "windows-sys 0.42.0", ] -[[package]] -name = "isahc" -version = "1.7.0" -source = "git+https://github.com/sagebind/isahc?rev=c39f6f8#c39f6f85aaa1f36c5857064c6c3c80f4d307d863" -dependencies = [ - "async-channel", - "castaway", - "crossbeam-utils", - "curl", - "curl-sys", - "data-encoding", - "encoding_rs", - "event-listener", - "futures-io", - "futures-lite", - "http", - "httpdate", - "log", - "mime", - "once_cell", - "polling", - "sluice", - "tracing", - "tracing-futures", - "url", - "waker-fn", -] - [[package]] name = "itoa" version = "0.4.8" @@ -949,28 +857,6 @@ version = "0.2.138" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8" -[[package]] -name = "libnghttp2-sys" -version = "0.1.7+1.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57ed28aba195b38d5ff02b9170cbff627e336a20925e43b4945390401c5dc93f" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "libz-sys" -version = "1.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "link-cplusplus" version = "1.0.7" @@ -982,9 +868,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f9f08d8963a6c613f4b1a78f4f4a4dbfadf8e6545b2d72861731e4858b8b47f" +checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" [[package]] name = "log" @@ -1005,6 +891,12 @@ dependencies = [ "nom", ] +[[package]] +name = "matches" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" + [[package]] name = "memchr" version = "2.5.0" @@ -1055,14 +947,14 @@ dependencies = [ [[package]] name = "nix" -version = "0.25.1" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4" +checksum = "46a58d1d356c6597d08cde02c2f09d785b09e28711837b1ed667dc652c08a694" dependencies = [ - "autocfg", "bitflags", "cfg-if", "libc", + "static_assertions", ] [[package]] @@ -1142,15 +1034,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" -[[package]] -name = "openssl-src" -version = "111.24.0+1.1.1s" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3498f259dab01178c6228c6b00dcef0ed2a2d5e20d648c017861227773ea4abd" -dependencies = [ - "cc", -] - [[package]] name = "openssl-sys" version = "0.9.79" @@ -1160,7 +1043,6 @@ dependencies = [ "autocfg", "cc", "libc", - "openssl-src", "pkg-config", "vcpkg", ] @@ -1177,26 +1059,6 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" -[[package]] -name = "pin-project" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "pin-project-lite" version = "0.2.9" @@ -1215,20 +1077,6 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" -[[package]] -name = "polling" -version = "2.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "166ca89eb77fd403230b9c156612965a81e094ec6ec3aa13663d4c8b113fa748" -dependencies = [ - "autocfg", - "cfg-if", - "libc", - "log", - "wepoll-ffi", - "windows-sys 0.42.0", -] - [[package]] name = "proc-macro-error" version = "1.0.4" @@ -1253,6 +1101,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + [[package]] name = "proc-macro2" version = "1.0.47" @@ -1263,13 +1117,19 @@ dependencies = [ ] [[package]] -name = "quick-xml" -version = "0.26.0" +name = "psl-types" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" +checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac" + +[[package]] +name = "publicsuffix" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a8c1bda5ae1af7f99a2962e49df150414a43d62404644d98dd5c3a93d07457" dependencies = [ - "memchr", - "serde", + "idna 0.3.0", + "psl-types", ] [[package]] @@ -1341,6 +1201,8 @@ checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" dependencies = [ "base64", "bytes", + "cookie", + "cookie_store", "encoding_rs", "futures-core", "futures-util", @@ -1348,6 +1210,7 @@ dependencies = [ "http", "http-body", "hyper", + "hyper-rustls", "hyper-tls", "ipnet", "js-sys", @@ -1357,19 +1220,39 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", + "proc-macro-hack", + "rustls", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", "tokio", "tokio-native-tls", + "tokio-rustls", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", + "webpki-roots", "winreg", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + [[package]] name = "rustix" version = "0.36.5" @@ -1385,15 +1268,15 @@ dependencies = [ ] [[package]] -name = "rustls-native-certs" -version = "0.6.2" +name = "rustls" +version = "0.20.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" +checksum = "539a2bfe908f471bfa933876bd1eb6a19cf2176d375f82ef7f99530a40e48c2c" dependencies = [ - "openssl-probe", - "rustls-pemfile", - "schannel", - "security-framework", + "log", + "ring", + "sct", + "webpki", ] [[package]] @@ -1405,12 +1288,6 @@ dependencies = [ "base64", ] -[[package]] -name = "rustversion" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8" - [[package]] name = "ryu" version = "1.0.11" @@ -1443,6 +1320,16 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "security-framework" version = "2.7.0" @@ -1468,18 +1355,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "256b9932320c590e707b94576e3cc1f7c9024d0ee6612dfbcf1cb106cbe8e055" +checksum = "e326c9ec8042f1b5da33252c8a37e9ffbd2c9bef0155215b6e6c80c790e05f91" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4eae9b04cbffdfd550eb462ed33bc6a1b68c935127d008b27444d08380f94e4" +checksum = "42a3df25b0713732468deadad63ab9da1f1fd75a48a15024b50363f128db627e" dependencies = [ "proc-macro2", "quote", @@ -1509,34 +1396,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_with" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25bf4a5a814902cd1014dbccfa4d4560fb8432c779471e96e035602519f82eef" -dependencies = [ - "base64", - "chrono", - "hex", - "indexmap", - "serde", - "serde_json", - "serde_with_macros", - "time 0.3.17", -] - -[[package]] -name = "serde_with_macros" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3452b4c0f6c1e357f73fdb87cd1efabaa12acf328c7a528e252893baeb3f4aa" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "signal-hook" version = "0.3.14" @@ -1565,17 +1424,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "sluice" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d7400c0eff44aa2fcb5e31a5f24ba9716ed90138769e4977a2ba6014ae63eb5" -dependencies = [ - "async-channel", - "futures-core", - "futures-io", -] - [[package]] name = "smart-default" version = "0.6.0" @@ -1597,6 +1445,18 @@ dependencies = [ "winapi", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "strsim" version = "0.10.0" @@ -1772,6 +1632,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +dependencies = [ + "rustls", + "tokio", + "webpki", +] + [[package]] name = "tokio-util" version = "0.7.4" @@ -1799,23 +1670,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", - "log", "pin-project-lite", - "tracing-attributes", "tracing-core", ] -[[package]] -name = "tracing-attributes" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "tracing-core" version = "0.1.30" @@ -1825,16 +1683,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "tracing-futures" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" -dependencies = [ - "pin-project", - "tracing", -] - [[package]] name = "try-lock" version = "0.2.3" @@ -1874,6 +1722,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "2.3.1" @@ -1881,7 +1735,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" dependencies = [ "form_urlencoded", - "idna", + "idna 0.3.0", "percent-encoding", ] @@ -1897,12 +1751,6 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" -[[package]] -name = "waker-fn" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" - [[package]] name = "want" version = "0.3.0" @@ -2002,12 +1850,22 @@ dependencies = [ ] [[package]] -name = "wepoll-ffi" -version = "0.1.2" +name = "webpki" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" dependencies = [ - "cc", + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" +dependencies = [ + "webpki", ] [[package]] @@ -2149,12 +2007,3 @@ checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ "winapi", ] - -[[package]] -name = "xattr" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc" -dependencies = [ - "libc", -] diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index bc34907..bd04867 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -4,35 +4,26 @@ authors = ["Crunchy Labs Maintainers"] version = "3.0.0-dev.3" edition = "2021" -[features] -static-curl = ["crunchyroll-rs/static-curl"] -static-ssl = ["crunchyroll-rs/static-ssl"] - [dependencies] anyhow = "1.0" async-trait = "0.1" clap = { version = "4.0", features = ["derive", "string"] } chrono = "0.4" -crunchyroll-rs = { git = "https://github.com/crunchy-labs/crunchyroll-rs", default-features = false, features = ["parse", "hls-stream", "dash-stream"] } +crunchyroll-rs = "0.2" csv = "1.1" ctrlc = "3.2" dirs = "4.0" -isahc = { git = "https://github.com/sagebind/isahc", rev = "c39f6f8" } log = { version = "0.4", features = ["std"] } -num_cpus = "1.13" -regex = "1.6" +num_cpus = "1.14" +regex = "1.7" sanitize-filename = "0.4" serde = "1.0" serde_json = "1.0" signal-hook = "0.3" tempfile = "3.3" terminal_size = "0.2" -tokio = { version = "1.21", features = ["macros", "rt-multi-thread", "time"] } +tokio = { version = "1.23", features = ["macros", "rt-multi-thread", "time"] } sys-locale = "0.2" -[target.'cfg(all(windows, target_env = "msvc"))'.dependencies] -isahc = { git = "https://github.com/sagebind/isahc", rev = "c39f6f8", features = ["data-encoding"] } -rustls-native-certs = "0.6" - [build-dependencies] chrono = "0.4" diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index f9bb6a5..5aabbd8 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -332,7 +332,7 @@ async fn formats_from_series( .locale .clone() .into_iter() - .filter(|l| !season.iter().any(|s| &s.metadata.audio_locale == l)) + .filter(|l| !season.iter().any(|s| s.metadata.audio_locales.contains(l))) .collect::>(); for not_present in not_present_audio { error!( @@ -346,7 +346,7 @@ async fn formats_from_series( // remove all seasons with the wrong audio for the current iterated season number seasons.retain(|s| { s.metadata.season_number != season.first().unwrap().metadata.season_number - || archive.locale.contains(&s.metadata.audio_locale) + || archive.locale.iter().any(|l| s.metadata.audio_locales.contains(l)) }) } @@ -355,7 +355,7 @@ async fn formats_from_series( BTreeMap::new(); for season in series.seasons().await? { if !url_filter.is_season_valid(season.metadata.season_number) - || !archive.locale.contains(&season.metadata.audio_locale) + || !archive.locale.iter().any(|l| season.metadata.audio_locales.contains(l)) { continue; } diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index b7093aa..31f8cda 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -290,7 +290,7 @@ async fn formats_from_series( // check if the current iterated season has the specified audio language if !season .iter() - .any(|s| s.metadata.audio_locale == download.audio) + .any(|s| s.metadata.audio_locales.contains(&download.audio)) { error!( "Season {} of series {} is not available with {} audio", @@ -303,7 +303,7 @@ async fn formats_from_series( // remove all seasons with the wrong audio for the current iterated season number seasons.retain(|s| { s.metadata.season_number != season.first().unwrap().metadata.season_number - || s.metadata.audio_locale == download.audio + || s.metadata.audio_locales.contains(&download.audio) }) } @@ -322,7 +322,7 @@ async fn formats_from_season( season: Media, url_filter: &UrlFilter, ) -> Result>> { - if season.metadata.audio_locale != download.audio { + if !season.metadata.audio_locales.contains(&download.audio) { error!( "Season {} ({}) is not available with {} audio", season.metadata.season_number, season.title, download.audio diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index 9b817e4..a64057a 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -1,7 +1,6 @@ use crate::utils::context::Context; use anyhow::{bail, Result}; use crunchyroll_rs::media::{Resolution, VariantData, VariantSegment}; -use isahc::AsyncReadResponseExt; use log::{debug, LevelFilter}; use std::borrow::{Borrow, BorrowMut}; use std::collections::BTreeMap; @@ -26,7 +25,7 @@ pub fn find_resolution( } pub async fn download_segments( - _ctx: &Context, + ctx: &Context, writer: &mut impl Write, message: Option, variant_data: VariantData, @@ -34,7 +33,7 @@ pub async fn download_segments( let segments = variant_data.segments().await?; let total_segments = segments.len(); - let client = Arc::new(variant_data.download_client()); + let client = Arc::new(ctx.crunchy.client()); let count = Arc::new(Mutex::new(0)); let amount = Arc::new(Mutex::new(0)); @@ -132,7 +131,7 @@ pub async fn download_segments( let thread_count = count.clone(); join_set.spawn(async move { for (i, segment) in thread_segments.into_iter().enumerate() { - let mut response = thread_client.get_async(&segment.url).await?; + let response = thread_client.get(&segment.url).send().await?; let mut buf = response.bytes().await?.to_vec(); *thread_amount.lock().unwrap() += buf.len(); From d49f2d8eaa9c2a3b88224c3504a5fd2b3013615e Mon Sep 17 00:00:00 2001 From: Alexandru Dracea Date: Fri, 16 Dec 2022 17:09:52 +0200 Subject: [PATCH 073/441] add minimal .gitignore (#83) * add .gitignore * more ignore * newline * readd .lock Co-authored-by: Alexandru.Dracea --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5bb2fd7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/.idea +/target +/.vscode From 03fe0c6f01d08ed11e9dde70b622ee655214ce5e Mon Sep 17 00:00:00 2001 From: ByteDream Date: Fri, 16 Dec 2022 21:57:26 +0100 Subject: [PATCH 074/441] Add additional command --- .github/workflows/ci.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index caef473..3ddf65d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ on: workflow_dispatch: jobs: - build-nix: + build: runs-on: ${{ matrix.os }} strategy: matrix: @@ -16,20 +16,23 @@ jobs: - os: ubuntu-latest toolchain: x86_64-unknown-linux-musl platform: linux + command: sudo apt install -y musl ext: - os: windows-latest toolchain: x86_64-pc-windows-gnu platform: windows + command: ext: .exe - os: macos-latest toolchain: x86_64-apple-darwin platform: darwin + command: ext: steps: - name: Checkout uses: actions/checkout@v3 - - name: Cargo cache # https://github.com/actions/cache/blob/main/examples.md#rust---cargo + - name: Cargo cache uses: actions/cache@v3 with: path: | @@ -48,6 +51,9 @@ jobs: target: ${{ matrix.toolchain }} default: true + - name: Additional command + run: ${{ matrix.command }} + - name: Test uses: actions-rs/cargo@v1 with: From 2451e33639c7449fd5958e6caf4b04099cb2c43f Mon Sep 17 00:00:00 2001 From: ByteDream Date: Fri, 16 Dec 2022 22:07:54 +0100 Subject: [PATCH 075/441] Fix ubuntu musl package --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ddf65d..fbb064e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: - os: ubuntu-latest toolchain: x86_64-unknown-linux-musl platform: linux - command: sudo apt install -y musl + command: sudo apt install -y musl-tools ext: - os: windows-latest toolchain: x86_64-pc-windows-gnu @@ -43,6 +43,9 @@ jobs: target/ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + - name: Additional command + run: ${{ matrix.command }} + - name: Install toolchain uses: actions-rs/toolchain@v1 with: @@ -51,9 +54,6 @@ jobs: target: ${{ matrix.toolchain }} default: true - - name: Additional command - run: ${{ matrix.command }} - - name: Test uses: actions-rs/cargo@v1 with: From 306019d8b88cc5a1e7fb8a05428c1c80714f3dac Mon Sep 17 00:00:00 2001 From: ByteDream Date: Fri, 16 Dec 2022 23:28:30 +0100 Subject: [PATCH 076/441] Fix linux ci --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fbb064e..0a2047f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,7 @@ jobs: key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Additional command + if: matrix.command != '' run: ${{ matrix.command }} - name: Install toolchain @@ -53,6 +54,7 @@ jobs: toolchain: stable target: ${{ matrix.toolchain }} default: true + overwrite: true - name: Test uses: actions-rs/cargo@v1 From 5de4a83e5deeab68d9d05b5bb1f400849d400e89 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sat, 17 Dec 2022 17:13:56 +0100 Subject: [PATCH 077/441] Change rust actions used --- .github/workflows/ci.yml | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a2047f..eb4c0c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,17 +16,14 @@ jobs: - os: ubuntu-latest toolchain: x86_64-unknown-linux-musl platform: linux - command: sudo apt install -y musl-tools ext: - os: windows-latest toolchain: x86_64-pc-windows-gnu platform: windows - command: ext: .exe - os: macos-latest toolchain: x86_64-apple-darwin platform: darwin - command: ext: steps: - name: Checkout @@ -43,48 +40,39 @@ jobs: target/ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - name: Additional command - if: matrix.command != '' - run: ${{ matrix.command }} + - name: Install system dependencies + if: matrix.platform == 'linux' + run: sudo apt-get install musl-tools - name: Install toolchain - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@stable with: - profile: minimal toolchain: stable target: ${{ matrix.toolchain }} - default: true - overwrite: true - name: Test - uses: actions-rs/cargo@v1 - with: - command: test - args: --release --all-features + run: cargo test --release --all-features --target ${{ matrix.toolchain }} - name: Build - uses: actions-rs/cargo@v1 - with: - command: build - args: --release --all-features + run: cargo build --release --all-features --target ${{ matrix.toolchain }} - name: Upload binary artifact uses: actions/upload-artifact@v3 with: name: crunchy-cli_${{ matrix.platform }} - path: ./target/release/crunchy-cli${{ matrix.ext }} + path: ./target/${{ matrix.toolchain }}/release/crunchy-cli${{ matrix.ext }} if-no-files-found: error - name: Upload manpages artifact uses: actions/upload-artifact@v3 with: name: manpages - path: ./target/release/manpages + path: ./target/${{ matrix.toolchain }}/release/manpages if-no-files-found: error - name: Upload completions artifact uses: actions/upload-artifact@v3 with: name: completions - path: ./target/release/completions + path: ./target/${{ matrix.toolchain }}/release/completions if-no-files-found: error From 4f107d8cf24854b94615ab4917f9f99b0fbcec29 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 18 Dec 2022 12:51:36 +0100 Subject: [PATCH 078/441] Update version and dependencies --- Cargo.lock | 94 ++++++++++++++++++------------------- Cargo.toml | 2 +- crunchy-cli-core/Cargo.lock | 92 ++++++++++++++++++------------------ crunchy-cli-core/Cargo.toml | 2 +- 4 files changed, 95 insertions(+), 95 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 15acee9..f205883 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -33,15 +33,15 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.66" +version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" +checksum = "7724808837b77f4b4de9d283820f9d98bcf496d5692934b857a2399d31ff22e6" [[package]] name = "async-trait" -version = "0.1.59" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6e93155431f3931513b243d371981bb2770112b370c82745a1d19d2f99364" +checksum = "677d1d8ab452a3936018a687b20e6f7cf5363d713b732b8884001317b0e48aa3" dependencies = [ "proc-macro2", "quote", @@ -214,9 +214,9 @@ dependencies = [ [[package]] name = "cookie" -version = "0.16.1" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "344adc371239ef32293cb1c4fe519592fcf21206c79c02854320afcdf3ab4917" +checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" dependencies = [ "percent-encoding", "time 0.3.17", @@ -266,7 +266,7 @@ dependencies = [ [[package]] name = "crunchy-cli" -version = "3.0.0-dev.3" +version = "3.0.0-dev.4" dependencies = [ "chrono", "clap", @@ -278,7 +278,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-dev.3" +version = "3.0.0-dev.4" dependencies = [ "anyhow", "async-trait", @@ -379,9 +379,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.83" +version = "1.0.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdf07d07d6531bfcdbe9b8b739b104610c6508dcc4d63b410585faf338241daf" +checksum = "27874566aca772cb515af4c6e997b5fe2119820bca447689145e39bb734d19a0" dependencies = [ "cc", "cxxbridge-flags", @@ -391,9 +391,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.83" +version = "1.0.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2eb5b96ecdc99f72657332953d4d9c50135af1bac34277801cc3937906ebd39" +checksum = "e7bb951f2523a49533003656a72121306b225ec16a49a09dc6b0ba0d6f3ec3c0" dependencies = [ "cc", "codespan-reporting", @@ -406,15 +406,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.83" +version = "1.0.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac040a39517fd1674e0f32177648334b0f4074625b5588a64519804ba0553b12" +checksum = "be778b6327031c1c7b61dd2e48124eee5361e6aa76b8de93692f011b08870ab4" [[package]] name = "cxxbridge-macro" -version = "1.0.83" +version = "1.0.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1362b0ddcfc4eb0a1f57b68bd77dd99f0e826958a96abd0ae9bd092e114ffed6" +checksum = "7b8a2b87662fe5a0a0b38507756ab66aff32638876a0866e5a5fc82ceb07ee49" dependencies = [ "proc-macro2", "quote", @@ -671,7 +671,7 @@ checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" dependencies = [ "bytes", "fnv", - "itoa 1.0.4", + "itoa 1.0.5", ] [[package]] @@ -712,7 +712,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa 1.0.4", + "itoa 1.0.5", "pin-project-lite", "socket2", "tokio", @@ -863,9 +863,9 @@ checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" [[package]] name = "itoa" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" +checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" [[package]] name = "js-sys" @@ -890,9 +890,9 @@ checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8" [[package]] name = "link-cplusplus" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369" +checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" dependencies = [ "cc", ] @@ -1140,9 +1140,9 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro2" -version = "1.0.47" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +checksum = "e9d89e5dba24725ae5678020bf8f1357a9aa7ff10736b551adbcd3f8d17d766f" dependencies = [ "unicode-ident", ] @@ -1165,9 +1165,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +checksum = "556d0f47a940e895261e77dc200d5eadfc6ef644c179c6f5edfc105e3a2292c8" dependencies = [ "proc-macro2", ] @@ -1327,9 +1327,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" [[package]] name = "sanitize-filename" @@ -1353,9 +1353,9 @@ dependencies = [ [[package]] name = "scratch" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" +checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" [[package]] name = "sct" @@ -1392,18 +1392,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e326c9ec8042f1b5da33252c8a37e9ffbd2c9bef0155215b6e6c80c790e05f91" +checksum = "97fed41fc1a24994d044e6db6935e69511a1153b52c15eb42493b26fa87feba0" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42a3df25b0713732468deadad63ab9da1f1fd75a48a15024b50363f128db627e" +checksum = "255abe9a125a985c05190d687b320c12f9b1f0b99445e608c21ba0782c719ad8" dependencies = [ "proc-macro2", "quote", @@ -1412,11 +1412,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.89" +version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db" +checksum = "8778cc0b528968fe72abec38b5db5a20a70d148116cd9325d2bc5f5180ca3faf" dependencies = [ - "itoa 1.0.4", + "itoa 1.0.5", "ryu", "serde", ] @@ -1428,7 +1428,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.4", + "itoa 1.0.5", "ryu", "serde", ] @@ -1502,9 +1502,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.105" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908" +checksum = "09ee3a69cd2c7e06684677e5629b3878b253af05e4714964204279c6bc02cf0b" dependencies = [ "proc-macro2", "quote", @@ -1559,18 +1559,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" dependencies = [ "proc-macro2", "quote", @@ -1594,7 +1594,7 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" dependencies = [ - "itoa 1.0.4", + "itoa 1.0.5", "serde", "time-core", "time-macros", @@ -1740,9 +1740,9 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-ident" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" [[package]] name = "unicode-normalization" diff --git a/Cargo.toml b/Cargo.toml index b2af93c..f5f988b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-dev.3" +version = "3.0.0-dev.4" edition = "2021" [dependencies] diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index aba7b4a..a85defe 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -33,15 +33,15 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.66" +version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" +checksum = "7724808837b77f4b4de9d283820f9d98bcf496d5692934b857a2399d31ff22e6" [[package]] name = "async-trait" -version = "0.1.59" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6e93155431f3931513b243d371981bb2770112b370c82745a1d19d2f99364" +checksum = "677d1d8ab452a3936018a687b20e6f7cf5363d713b732b8884001317b0e48aa3" dependencies = [ "proc-macro2", "quote", @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "cookie" -version = "0.16.1" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "344adc371239ef32293cb1c4fe519592fcf21206c79c02854320afcdf3ab4917" +checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" dependencies = [ "percent-encoding", "time 0.3.17", @@ -247,7 +247,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-dev.3" +version = "3.0.0-dev.4" dependencies = [ "anyhow", "async-trait", @@ -348,9 +348,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.83" +version = "1.0.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdf07d07d6531bfcdbe9b8b739b104610c6508dcc4d63b410585faf338241daf" +checksum = "27874566aca772cb515af4c6e997b5fe2119820bca447689145e39bb734d19a0" dependencies = [ "cc", "cxxbridge-flags", @@ -360,9 +360,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.83" +version = "1.0.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2eb5b96ecdc99f72657332953d4d9c50135af1bac34277801cc3937906ebd39" +checksum = "e7bb951f2523a49533003656a72121306b225ec16a49a09dc6b0ba0d6f3ec3c0" dependencies = [ "cc", "codespan-reporting", @@ -375,15 +375,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.83" +version = "1.0.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac040a39517fd1674e0f32177648334b0f4074625b5588a64519804ba0553b12" +checksum = "be778b6327031c1c7b61dd2e48124eee5361e6aa76b8de93692f011b08870ab4" [[package]] name = "cxxbridge-macro" -version = "1.0.83" +version = "1.0.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1362b0ddcfc4eb0a1f57b68bd77dd99f0e826958a96abd0ae9bd092e114ffed6" +checksum = "7b8a2b87662fe5a0a0b38507756ab66aff32638876a0866e5a5fc82ceb07ee49" dependencies = [ "proc-macro2", "quote", @@ -640,7 +640,7 @@ checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" dependencies = [ "bytes", "fnv", - "itoa 1.0.4", + "itoa 1.0.5", ] [[package]] @@ -681,7 +681,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa 1.0.4", + "itoa 1.0.5", "pin-project-lite", "socket2", "tokio", @@ -832,9 +832,9 @@ checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" [[package]] name = "itoa" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" +checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" [[package]] name = "js-sys" @@ -859,9 +859,9 @@ checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8" [[package]] name = "link-cplusplus" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369" +checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" dependencies = [ "cc", ] @@ -1109,9 +1109,9 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro2" -version = "1.0.47" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +checksum = "e9d89e5dba24725ae5678020bf8f1357a9aa7ff10736b551adbcd3f8d17d766f" dependencies = [ "unicode-ident", ] @@ -1134,9 +1134,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +checksum = "556d0f47a940e895261e77dc200d5eadfc6ef644c179c6f5edfc105e3a2292c8" dependencies = [ "proc-macro2", ] @@ -1290,9 +1290,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" [[package]] name = "sanitize-filename" @@ -1316,9 +1316,9 @@ dependencies = [ [[package]] name = "scratch" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" +checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" [[package]] name = "sct" @@ -1355,18 +1355,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e326c9ec8042f1b5da33252c8a37e9ffbd2c9bef0155215b6e6c80c790e05f91" +checksum = "97fed41fc1a24994d044e6db6935e69511a1153b52c15eb42493b26fa87feba0" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42a3df25b0713732468deadad63ab9da1f1fd75a48a15024b50363f128db627e" +checksum = "255abe9a125a985c05190d687b320c12f9b1f0b99445e608c21ba0782c719ad8" dependencies = [ "proc-macro2", "quote", @@ -1375,11 +1375,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.89" +version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db" +checksum = "8778cc0b528968fe72abec38b5db5a20a70d148116cd9325d2bc5f5180ca3faf" dependencies = [ - "itoa 1.0.4", + "itoa 1.0.5", "ryu", "serde", ] @@ -1391,7 +1391,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.4", + "itoa 1.0.5", "ryu", "serde", ] @@ -1465,9 +1465,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.105" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908" +checksum = "09ee3a69cd2c7e06684677e5629b3878b253af05e4714964204279c6bc02cf0b" dependencies = [ "proc-macro2", "quote", @@ -1522,18 +1522,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" dependencies = [ "proc-macro2", "quote", @@ -1557,7 +1557,7 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" dependencies = [ - "itoa 1.0.4", + "itoa 1.0.5", "serde", "time-core", "time-macros", @@ -1703,9 +1703,9 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-ident" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" [[package]] name = "unicode-normalization" diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index bd04867..6dccd4d 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-dev.3" +version = "3.0.0-dev.4" edition = "2021" [dependencies] From 4bfc6f22e111ff53b6ad73811bbab692b938331c Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 18 Dec 2022 13:50:20 +0100 Subject: [PATCH 079/441] Fix discord link --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 21f84a6..8008de2 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ A [Rust](https://www.rust-lang.org/) written cli client for [Crunchyroll](https: Release - - Discord + + Discord CI From 67bbc00d87c3fd5c0f64f6ae537a331bc5b54702 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 18 Dec 2022 15:03:16 +0100 Subject: [PATCH 080/441] Add pre-release notice --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 8008de2..bf57145 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,10 @@ A [Rust](https://www.rust-lang.org/) written cli client for [Crunchyroll](https: > We are in no way affiliated with, maintained, authorized, sponsored, or officially associated with Crunchyroll LLC or any of its subsidiaries or affiliates. > The official Crunchyroll website can be found at https://crunchyroll.com/. +> This README belongs to the _master_ branch which is currently under heavy development towards the next major version (3.0). +> It is mostly stable but some issues may still occur. +> If you do not want to use an under-development / pre-release version, head over to the _[golang](https://github.com/crunchy-labs/crunchy-cli/tree/golang)_ branch which contains the EOL but last stable version (and documentation for it). + ## ✨ Features - Download single videos and entire series from [Crunchyroll](https://www.crunchyroll.com). From 8bb2c9c7501cf4382d3b0117d55e4f6c7b42d93a Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 19 Dec 2022 15:22:45 +0100 Subject: [PATCH 081/441] Fix file name sanitizing --- crunchy-cli-core/src/cli/archive.rs | 1 + crunchy-cli-core/src/cli/download.rs | 1 + crunchy-cli-core/src/utils/format.rs | 31 ++++++++++++++++++---------- crunchy-cli-core/src/utils/os.rs | 9 +------- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 5aabbd8..a551bd1 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -237,6 +237,7 @@ impl Execute for Archive { } .to_string(), primary, + true, )), ); diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index 31f8cda..e13d3f0 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -209,6 +209,7 @@ impl Execute for Download { } .to_string(), &format, + true, )), ); diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index 3dd7f74..b111dce 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -63,15 +63,24 @@ impl Format { } } -pub fn format_string(s: String, format: &Format) -> String { - s.replace("{title}", &format.title) - .replace("{series_name}", &format.series_name) - .replace("{season_name}", &format.season_title) - .replace("{audio}", &format.audio.to_string()) - .replace("{resolution}", &format.stream.resolution.to_string()) - .replace("{season_number}", &format.season_number.to_string()) - .replace("{episode_number}", &format.number.to_string()) - .replace("{series_id}", &format.series_id) - .replace("{season_id}", &format.season_id) - .replace("{episode_id}", &format.id) +/// Formats the given string if it has specific pattern in it. It's possible to sanitize it which +/// removes characters which can cause failures if the output string is used as a file name. +pub fn format_string(s: String, format: &Format, sanitize: bool) -> String { + let sanitize_func = if sanitize { + |s: &str| sanitize_filename::sanitize(s) + } else { + // converting this to a string is actually unnecessary + |s: &str| s.to_string() + }; + + s.replace("{title}", &sanitize_func(&format.title)) + .replace("{series_name}", &sanitize_func(&format.series_name)) + .replace("{season_name}", &sanitize_func(&format.season_title)) + .replace("{audio}", &sanitize_func(&format.audio.to_string())) + .replace("{resolution}", &sanitize_func(&format.stream.resolution.to_string())) + .replace("{season_number}", &sanitize_func(&format.season_number.to_string())) + .replace("{episode_number}", &sanitize_func(&format.number.to_string())) + .replace("{series_id}", &sanitize_func(&format.series_id)) + .replace("{season_id}", &sanitize_func(&format.season_id)) + .replace("{episode_id}", &sanitize_func(&format.id)) } diff --git a/crunchy-cli-core/src/utils/os.rs b/crunchy-cli-core/src/utils/os.rs index abdf151..01d0f89 100644 --- a/crunchy-cli-core/src/utils/os.rs +++ b/crunchy-cli-core/src/utils/os.rs @@ -50,12 +50,5 @@ pub fn free_file(mut path: PathBuf) -> PathBuf { path.set_file_name(format!("{} ({}).{}", filename, i, ext)) } - sanitize_file(path) -} - -/// Sanitizes the given path to not contain any invalid file character. -pub fn sanitize_file(path: PathBuf) -> PathBuf { - path.with_file_name(sanitize_filename::sanitize( - path.file_name().unwrap().to_string_lossy(), - )) + path } From af9aca4d0cfc3608369507766d63ca3e40fe449a Mon Sep 17 00:00:00 2001 From: Alexandru Dracea Date: Mon, 19 Dec 2022 18:35:37 +0200 Subject: [PATCH 082/441] Add padding --- crunchy-cli-core/src/cli/archive.rs | 22 ++++++++++++---------- crunchy-cli-core/src/cli/download.rs | 22 ++++++++++++---------- crunchy-cli-core/src/utils/format.rs | 23 ++++++++++++++++++++--- 3 files changed, 44 insertions(+), 23 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index a551bd1..6e56e81 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -58,16 +58,18 @@ pub struct Archive { #[arg(help = "Name of the output file")] #[arg(long_help = "Name of the output file.\ If you use one of the following pattern they will get replaced:\n \ - {title} → Title of the video\n \ - {series_name} → Name of the series\n \ - {season_name} → Name of the season\n \ - {audio} → Audio language of the video\n \ - {resolution} → Resolution of the video\n \ - {season_number} → Number of the season\n \ - {episode_number} → Number of the episode\n \ - {series_id} → ID of the series\n \ - {season_id} → ID of the season\n \ - {episode_id} → ID of the episode")] + {title} → Title of the video\n \ + {series_name} → Name of the series\n \ + {season_name} → Name of the season\n \ + {audio} → Audio language of the video\n \ + {resolution} → Resolution of the video\n \ + {padded_season_number} → Number of the season padded to double digits\n \ + {season_number} → Number of the season\n \ + {padded_episode_number} → Number of the episode padded to double digits\n \ + {episode_number} → Number of the episode\n \ + {series_id} → ID of the series\n \ + {season_id} → ID of the season\n \ + {episode_id} → ID of the episode")] #[arg(short, long, default_value = "{title}.mkv")] output: String, diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index e13d3f0..4e4eb3c 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -36,16 +36,18 @@ pub struct Download { #[arg(help = "Name of the output file")] #[arg(long_help = "Name of the output file.\ If you use one of the following pattern they will get replaced:\n \ - {title} → Title of the video\n \ - {series_name} → Name of the series\n \ - {season_name} → Name of the season\n \ - {audio} → Audio language of the video\n \ - {resolution} → Resolution of the video\n \ - {season_number} → Number of the season\n \ - {episode_number} → Number of the episode\n \ - {series_id} → ID of the series\n \ - {season_id} → ID of the season\n \ - {episode_id} → ID of the episode")] + {title} → Title of the video\n \ + {series_name} → Name of the series\n \ + {season_name} → Name of the season\n \ + {audio} → Audio language of the video\n \ + {resolution} → Resolution of the video\n \ + {padded_season_number} → Number of the season padded to double digits\n \ + {season_number} → Number of the season\n \ + {padded_episode_number} → Number of the episode padded to double digits\n \ + {episode_number} → Number of the episode\n \ + {series_id} → ID of the series\n \ + {season_id} → ID of the season\n \ + {episode_id} → ID of the episode")] #[arg(short, long, default_value = "{title}.ts")] output: String, diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index b111dce..c463ea8 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -77,9 +77,26 @@ pub fn format_string(s: String, format: &Format, sanitize: bool) -> String { .replace("{series_name}", &sanitize_func(&format.series_name)) .replace("{season_name}", &sanitize_func(&format.season_title)) .replace("{audio}", &sanitize_func(&format.audio.to_string())) - .replace("{resolution}", &sanitize_func(&format.stream.resolution.to_string())) - .replace("{season_number}", &sanitize_func(&format.season_number.to_string())) - .replace("{episode_number}", &sanitize_func(&format.number.to_string())) + .replace( + "{resolution}", + &sanitize_func(&format.stream.resolution.to_string()), + ) + .replace( + "{padded_season_number}", + &sanitize_func(&format!("{:0>2}", format.season_number.to_string())), + ) + .replace( + "{season_number}", + &sanitize_func(&format.season_number.to_string()), + ) + .replace( + "{padded_episode_number}", + &sanitize_func(&format!("{:0>2}", format.number.to_string())), + ) + .replace( + "{episode_number}", + &sanitize_func(&format.number.to_string()), + ) .replace("{series_id}", &sanitize_func(&format.series_id)) .replace("{season_id}", &sanitize_func(&format.season_id)) .replace("{episode_id}", &sanitize_func(&format.id)) From 17fa045c32e103a13535a819dad4da6b6731347a Mon Sep 17 00:00:00 2001 From: ByteDream Date: Thu, 22 Dec 2022 14:45:56 +0100 Subject: [PATCH 083/441] Use library for progress --- Cargo.lock | 130 ++++++++++++++++++--------- crunchy-cli-core/Cargo.lock | 130 ++++++++++++++++++--------- crunchy-cli-core/Cargo.toml | 1 + crunchy-cli-core/src/cli/archive.rs | 33 ++++--- crunchy-cli-core/src/cli/download.rs | 28 +++--- crunchy-cli-core/src/cli/log.rs | 130 +++++++-------------------- crunchy-cli-core/src/cli/utils.rs | 127 +++++++++----------------- crunchy-cli-core/src/lib.rs | 6 +- crunchy-cli-core/src/utils/log.rs | 17 +++- 9 files changed, 307 insertions(+), 295 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f205883..a7432c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -33,9 +33,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.67" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7724808837b77f4b4de9d283820f9d98bcf496d5692934b857a2399d31ff22e6" +checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" [[package]] name = "async-trait" @@ -148,9 +148,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.29" +version = "4.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d63b9e9c07271b9957ad22c173bae2a4d9a81127680962039296abcd2f8251d" +checksum = "656ad1e55e23d287773f7d8192c300dc715c3eeded93b3da651d11c42cfd74d2" dependencies = [ "bitflags", "clap_derive", @@ -212,6 +212,20 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "console" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c050367d967ced717c04b65d8c619d863ef9292ce0c5760028655a2fb298718c" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "terminal_size 0.1.17", + "unicode-width", + "winapi", +] + [[package]] name = "cookie" version = "0.16.2" @@ -288,6 +302,7 @@ dependencies = [ "csv", "ctrlc", "dirs", + "indicatif", "log", "num_cpus", "regex", @@ -297,7 +312,7 @@ dependencies = [ "signal-hook", "sys-locale", "tempfile", - "terminal_size", + "terminal_size 0.2.3", "tokio", ] @@ -379,9 +394,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.84" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27874566aca772cb515af4c6e997b5fe2119820bca447689145e39bb734d19a0" +checksum = "5add3fc1717409d029b20c5b6903fc0c0b02fa6741d820054f4a2efa5e5816fd" dependencies = [ "cc", "cxxbridge-flags", @@ -391,9 +406,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.84" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7bb951f2523a49533003656a72121306b225ec16a49a09dc6b0ba0d6f3ec3c0" +checksum = "b4c87959ba14bc6fbc61df77c3fcfe180fc32b93538c4f1031dd802ccb5f2ff0" dependencies = [ "cc", "codespan-reporting", @@ -406,15 +421,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.84" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be778b6327031c1c7b61dd2e48124eee5361e6aa76b8de93692f011b08870ab4" +checksum = "69a3e162fde4e594ed2b07d0f83c6c67b745e7f28ce58c6df5e6b6bef99dfb59" [[package]] name = "cxxbridge-macro" -version = "1.0.84" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b8a2b87662fe5a0a0b38507756ab66aff32638876a0866e5a5fc82ceb07ee49" +checksum = "3e7e2adeb6a0d4a282e581096b06e1791532b7d576dcde5ccd9382acf55db8e6" dependencies = [ "proc-macro2", "quote", @@ -476,6 +491,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" version = "0.8.31" @@ -645,15 +666,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.2.6" @@ -808,6 +820,18 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "indicatif" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4295cbb7573c16d310e99e713cf9e75101eb190ab31fccd35f2d2691b4352b19" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "unicode-width", +] + [[package]] name = "inout" version = "0.1.3" @@ -845,11 +869,11 @@ checksum = "11b0d96e660696543b251e58030cf9787df56da39dab19ad60eae7353040917e" [[package]] name = "is-terminal" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927609f78c2913a6f6ac3c27a4fe87f43e2a35367c0c4b0f8265e8f49a104330" +checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi", "io-lifetimes", "rustix", "windows-sys 0.42.0", @@ -1019,14 +1043,20 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi 0.1.19", + "hermit-abi", "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "once_cell" version = "1.16.0" @@ -1035,9 +1065,9 @@ checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" [[package]] name = "openssl" -version = "0.10.44" +version = "0.10.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29d971fd5722fec23977260f6e81aa67d2f22cadbdc2aa049f1022d9a3be1566" +checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" dependencies = [ "bitflags", "cfg-if", @@ -1067,9 +1097,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.79" +version = "0.9.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5454462c0eced1e97f2ec09036abc8da362e66802f66fd20f86854d9d8cbcbc4" +checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" dependencies = [ "autocfg", "cc", @@ -1108,6 +1138,12 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +[[package]] +name = "portable-atomic" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bdd679d533107e090c2704a35982fc06302e30898e63ffa26a81155c012e92" + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -1134,15 +1170,15 @@ dependencies = [ [[package]] name = "proc-macro-hack" -version = "0.5.19" +version = "0.5.20+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" +checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.48" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9d89e5dba24725ae5678020bf8f1357a9aa7ff10736b551adbcd3f8d17d766f" +checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" dependencies = [ "unicode-ident", ] @@ -1165,9 +1201,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556d0f47a940e895261e77dc200d5eadfc6ef644c179c6f5edfc105e3a2292c8" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" dependencies = [ "proc-macro2", ] @@ -1412,9 +1448,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.90" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8778cc0b528968fe72abec38b5db5a20a70d148116cd9325d2bc5f5180ca3faf" +checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" dependencies = [ "itoa 1.0.5", "ryu", @@ -1502,9 +1538,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ee3a69cd2c7e06684677e5629b3878b253af05e4714964204279c6bc02cf0b" +checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" dependencies = [ "proc-macro2", "quote", @@ -1547,6 +1583,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "terminal_size" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "terminal_size" version = "0.2.3" diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index a85defe..8347aa7 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -33,9 +33,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.67" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7724808837b77f4b4de9d283820f9d98bcf496d5692934b857a2399d31ff22e6" +checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" [[package]] name = "async-trait" @@ -148,9 +148,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.29" +version = "4.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d63b9e9c07271b9957ad22c173bae2a4d9a81127680962039296abcd2f8251d" +checksum = "656ad1e55e23d287773f7d8192c300dc715c3eeded93b3da651d11c42cfd74d2" dependencies = [ "bitflags", "clap_derive", @@ -193,6 +193,20 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "console" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c050367d967ced717c04b65d8c619d863ef9292ce0c5760028655a2fb298718c" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "terminal_size 0.1.17", + "unicode-width", + "winapi", +] + [[package]] name = "cookie" version = "0.16.2" @@ -257,6 +271,7 @@ dependencies = [ "csv", "ctrlc", "dirs", + "indicatif", "log", "num_cpus", "regex", @@ -266,7 +281,7 @@ dependencies = [ "signal-hook", "sys-locale", "tempfile", - "terminal_size", + "terminal_size 0.2.3", "tokio", ] @@ -348,9 +363,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.84" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27874566aca772cb515af4c6e997b5fe2119820bca447689145e39bb734d19a0" +checksum = "5add3fc1717409d029b20c5b6903fc0c0b02fa6741d820054f4a2efa5e5816fd" dependencies = [ "cc", "cxxbridge-flags", @@ -360,9 +375,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.84" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7bb951f2523a49533003656a72121306b225ec16a49a09dc6b0ba0d6f3ec3c0" +checksum = "b4c87959ba14bc6fbc61df77c3fcfe180fc32b93538c4f1031dd802ccb5f2ff0" dependencies = [ "cc", "codespan-reporting", @@ -375,15 +390,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.84" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be778b6327031c1c7b61dd2e48124eee5361e6aa76b8de93692f011b08870ab4" +checksum = "69a3e162fde4e594ed2b07d0f83c6c67b745e7f28ce58c6df5e6b6bef99dfb59" [[package]] name = "cxxbridge-macro" -version = "1.0.84" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b8a2b87662fe5a0a0b38507756ab66aff32638876a0866e5a5fc82ceb07ee49" +checksum = "3e7e2adeb6a0d4a282e581096b06e1791532b7d576dcde5ccd9382acf55db8e6" dependencies = [ "proc-macro2", "quote", @@ -445,6 +460,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" version = "0.8.31" @@ -614,15 +635,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.2.6" @@ -777,6 +789,18 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "indicatif" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4295cbb7573c16d310e99e713cf9e75101eb190ab31fccd35f2d2691b4352b19" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "unicode-width", +] + [[package]] name = "inout" version = "0.1.3" @@ -814,11 +838,11 @@ checksum = "11b0d96e660696543b251e58030cf9787df56da39dab19ad60eae7353040917e" [[package]] name = "is-terminal" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927609f78c2913a6f6ac3c27a4fe87f43e2a35367c0c4b0f8265e8f49a104330" +checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi", "io-lifetimes", "rustix", "windows-sys 0.42.0", @@ -988,14 +1012,20 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi 0.1.19", + "hermit-abi", "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "once_cell" version = "1.16.0" @@ -1004,9 +1034,9 @@ checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" [[package]] name = "openssl" -version = "0.10.44" +version = "0.10.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29d971fd5722fec23977260f6e81aa67d2f22cadbdc2aa049f1022d9a3be1566" +checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" dependencies = [ "bitflags", "cfg-if", @@ -1036,9 +1066,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.79" +version = "0.9.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5454462c0eced1e97f2ec09036abc8da362e66802f66fd20f86854d9d8cbcbc4" +checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" dependencies = [ "autocfg", "cc", @@ -1077,6 +1107,12 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +[[package]] +name = "portable-atomic" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bdd679d533107e090c2704a35982fc06302e30898e63ffa26a81155c012e92" + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -1103,15 +1139,15 @@ dependencies = [ [[package]] name = "proc-macro-hack" -version = "0.5.19" +version = "0.5.20+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" +checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.48" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9d89e5dba24725ae5678020bf8f1357a9aa7ff10736b551adbcd3f8d17d766f" +checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" dependencies = [ "unicode-ident", ] @@ -1134,9 +1170,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556d0f47a940e895261e77dc200d5eadfc6ef644c179c6f5edfc105e3a2292c8" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" dependencies = [ "proc-macro2", ] @@ -1375,9 +1411,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.90" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8778cc0b528968fe72abec38b5db5a20a70d148116cd9325d2bc5f5180ca3faf" +checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" dependencies = [ "itoa 1.0.5", "ryu", @@ -1465,9 +1501,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ee3a69cd2c7e06684677e5629b3878b253af05e4714964204279c6bc02cf0b" +checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" dependencies = [ "proc-macro2", "quote", @@ -1510,6 +1546,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "terminal_size" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "terminal_size" version = "0.2.3" diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 6dccd4d..780716f 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -13,6 +13,7 @@ crunchyroll-rs = "0.2" csv = "1.1" ctrlc = "3.2" dirs = "4.0" +indicatif = "0.17" log = { version = "0.4", features = ["std"] } num_cpus = "1.14" regex = "1.7" diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 6e56e81..b5244f7 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -1,5 +1,5 @@ use crate::cli::log::tab_info; -use crate::cli::utils::{download_segments, FFmpegPreset, find_resolution}; +use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset}; use crate::utils::context::Context; use crate::utils::format::{format_string, Format}; use crate::utils::log::progress; @@ -137,7 +137,9 @@ impl Execute for Archive { bail!("File extension is not '.mkv'. Currently only matroska / '.mkv' files are supported") } let _ = FFmpegPreset::ffmpeg_presets(self.ffmpeg_preset.clone())?; - if self.ffmpeg_preset.len() == 1 && self.ffmpeg_preset.get(0).unwrap() == &FFmpegPreset::Nvidia { + if self.ffmpeg_preset.len() == 1 + && self.ffmpeg_preset.get(0).unwrap() == &FFmpegPreset::Nvidia + { warn!("Skipping 'nvidia' hardware acceleration preset since no other codec preset was specified") } @@ -148,20 +150,20 @@ impl Execute for Archive { let mut parsed_urls = vec![]; for (i, url) in self.urls.iter().enumerate() { - let _progress_handler = progress!("Parsing url {}", i + 1); + let progress_handler = progress!("Parsing url {}", i + 1); match parse_url(&ctx.crunchy, url.clone(), true).await { Ok((media_collection, url_filter)) => { parsed_urls.push((media_collection, url_filter)); - info!("Parsed url {}", i + 1) + progress_handler.stop(format!("Parsed url {}", i + 1)) } Err(e) => bail!("url {} could not be parsed: {}", url, e), } } for (i, (media_collection, url_filter)) in parsed_urls.into_iter().enumerate() { + let progress_handler = progress!("Fetching series details"); let archive_formats = match media_collection { MediaCollection::Series(series) => { - let _progress_handler = progress!("Fetching series details"); formats_from_series(&self, series, &url_filter).await? } MediaCollection::Season(_) => bail!("Archiving a season is not supported"), @@ -171,10 +173,13 @@ impl Execute for Archive { }; if archive_formats.is_empty() { - info!("Skipping url {} (no matching episodes found)", i + 1); + progress_handler.stop(format!( + "Skipping url {} (no matching episodes found)", + i + 1 + )); continue; } - info!("Loaded series information for url {}", i + 1); + progress_handler.stop(format!("Loaded series information for url {}", i + 1)); if log::max_level() == log::Level::Debug { let seasons = sort_formats_after_seasons( @@ -310,9 +315,9 @@ impl Execute for Archive { )) } - let _progess_handler = progress!("Generating mkv"); + let progess_handler = progress!("Generating mkv"); generate_mkv(&self, path, video_paths, audio_paths, subtitle_paths)?; - info!("Mkv generated") + progess_handler.stop("Mkv generated") } } @@ -349,7 +354,10 @@ async fn formats_from_series( // remove all seasons with the wrong audio for the current iterated season number seasons.retain(|s| { s.metadata.season_number != season.first().unwrap().metadata.season_number - || archive.locale.iter().any(|l| s.metadata.audio_locales.contains(l)) + || archive + .locale + .iter() + .any(|l| s.metadata.audio_locales.contains(l)) }) } @@ -358,7 +366,10 @@ async fn formats_from_series( BTreeMap::new(); for season in series.seasons().await? { if !url_filter.is_season_valid(season.metadata.season_number) - || !archive.locale.iter().any(|l| season.metadata.audio_locales.contains(l)) + || !archive + .locale + .iter() + .any(|l| season.metadata.audio_locales.contains(l)) { continue; } diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index 4e4eb3c..d8a4ada 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -1,5 +1,5 @@ use crate::cli::log::tab_info; -use crate::cli::utils::{download_segments, FFmpegPreset, find_resolution}; +use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset}; use crate::utils::context::Context; use crate::utils::format::{format_string, Format}; use crate::utils::log::progress; @@ -92,7 +92,9 @@ impl Execute for Download { } let _ = FFmpegPreset::ffmpeg_presets(self.ffmpeg_preset.clone())?; - if self.ffmpeg_preset.len() == 1 && self.ffmpeg_preset.get(0).unwrap() == &FFmpegPreset::Nvidia { + if self.ffmpeg_preset.len() == 1 + && self.ffmpeg_preset.get(0).unwrap() == &FFmpegPreset::Nvidia + { warn!("Skipping 'nvidia' hardware acceleration preset since no other codec preset was specified") } @@ -103,18 +105,18 @@ impl Execute for Download { let mut parsed_urls = vec![]; for (i, url) in self.urls.iter().enumerate() { - let _progress_handler = progress!("Parsing url {}", i + 1); + let progress_handler = progress!("Parsing url {}", i + 1); match parse_url(&ctx.crunchy, url.clone(), true).await { Ok((media_collection, url_filter)) => { parsed_urls.push((media_collection, url_filter)); - info!("Parsed url {}", i + 1) + progress_handler.stop(format!("Parsed url {}", i + 1)) } Err(e) => bail!("url {} could not be parsed: {}", url, e), } } for (i, (media_collection, url_filter)) in parsed_urls.into_iter().enumerate() { - let _progress_handler = progress!("Fetching series details"); + let progress_handler = progress!("Fetching series details"); let formats = match media_collection { MediaCollection::Series(series) => { debug!("Url {} is series ({})", i + 1, series.title); @@ -156,11 +158,10 @@ impl Execute for Download { }; let Some(formats) = formats else { - info!("Skipping url {} (no matching episodes found)", i + 1); + progress_handler.stop(format!("Skipping url {} (no matching episodes found)", i + 1)); continue; }; - info!("Loaded series information for url {}", i + 1); - drop(_progress_handler); + progress_handler.stop(format!("Loaded series information for url {}", i + 1)); if log::max_level() == log::Level::Debug { let seasons = sort_formats_after_seasons(formats.clone()); @@ -231,7 +232,9 @@ impl Execute for Download { tab_info!("Resolution: {}", format.stream.resolution); tab_info!("FPS: {:.2}", format.stream.fps); - if path.extension().unwrap_or_default().to_string_lossy() != "ts" || !self.ffmpeg_preset.is_empty() { + if path.extension().unwrap_or_default().to_string_lossy() != "ts" + || !self.ffmpeg_preset.is_empty() + { download_ffmpeg(&ctx, &self, format.stream, path.as_path()).await?; } else if path.to_str().unwrap() == "-" { let mut stdout = std::io::stdout().lock(); @@ -247,7 +250,12 @@ impl Execute for Download { } } -async fn download_ffmpeg(ctx: &Context, download: &Download, variant_data: VariantData, target: &Path) -> Result<()> { +async fn download_ffmpeg( + ctx: &Context, + download: &Download, + variant_data: VariantData, + target: &Path, +) -> Result<()> { let (input_presets, output_presets) = FFmpegPreset::ffmpeg_presets(download.ffmpeg_preset.clone())?; diff --git a/crunchy-cli-core/src/cli/log.rs b/crunchy-cli-core/src/cli/log.rs index d1f0225..377685b 100644 --- a/crunchy-cli-core/src/cli/log.rs +++ b/crunchy-cli-core/src/cli/log.rs @@ -1,106 +1,17 @@ +use indicatif::{ProgressBar, ProgressStyle}; use log::{ set_boxed_logger, set_max_level, Level, LevelFilter, Log, Metadata, Record, SetLoggerError, }; use std::io::{stdout, Write}; -use std::sync::{mpsc, Mutex}; +use std::sync::Mutex; use std::thread; -use std::thread::JoinHandle; use std::time::Duration; -struct CliProgress { - handler: JoinHandle<()>, - sender: mpsc::SyncSender<(String, Level)>, -} - -impl CliProgress { - fn new(record: &Record) -> Self { - let (tx, rx) = mpsc::sync_channel(1); - - let init_message = format!("{}", record.args()); - let init_level = record.level(); - let handler = thread::spawn(move || { - #[cfg(not(windows))] - let ok = '✔'; - #[cfg(windows)] - // windows does not support all unicode characters by default in their consoles, so - // we're using this (square root?) symbol instead. microsoft. - let ok = '√'; - let states = ["-", "\\", "|", "/"]; - - let mut old_message = init_message.clone(); - let mut latest_info_message = init_message; - let mut old_level = init_level; - for i in 0.. { - let (msg, level) = match rx.try_recv() { - Ok(payload) => payload, - Err(e) => match e { - mpsc::TryRecvError::Empty => (old_message.clone(), old_level), - mpsc::TryRecvError::Disconnected => break, - }, - }; - - // clear last line - // prefix (2), space (1), state (1), space (1), message(n) - let _ = write!(stdout(), "\r {}", " ".repeat(old_message.len())); - - if old_level != level || old_message != msg { - if old_level <= Level::Warn { - let _ = writeln!(stdout(), "\r:: â€ĸ {}", old_message); - } else if old_level == Level::Info && level <= Level::Warn { - let _ = writeln!(stdout(), "\r:: → {}", old_message); - } else if level == Level::Info { - latest_info_message = msg.clone(); - } - } - - let _ = write!( - stdout(), - "\r:: {} {}", - states[i / 2 % states.len()], - if level == Level::Info { - &msg - } else { - &latest_info_message - } - ); - let _ = stdout().flush(); - - old_message = msg; - old_level = level; - - thread::sleep(Duration::from_millis(100)); - } - - // clear last line - // prefix (2), space (1), state (1), space (1), message(n) - let _ = write!(stdout(), "\r {}", " ".repeat(old_message.len())); - let _ = writeln!(stdout(), "\r:: {} {}", ok, old_message); - let _ = stdout().flush(); - }); - - Self { - handler, - sender: tx, - } - } - - fn send(&self, record: &Record) { - let _ = self - .sender - .send((format!("{}", record.args()), record.level())); - } - - fn stop(self) { - drop(self.sender); - let _ = self.handler.join(); - } -} - #[allow(clippy::type_complexity)] pub struct CliLogger { all: bool, level: LevelFilter, - progress: Mutex>, + progress: Mutex>, } impl Log for CliLogger { @@ -127,7 +38,7 @@ impl Log for CliLogger { "progress_end" => self.progress(record, true), _ => { if self.progress.lock().unwrap().is_some() { - self.progress(record, false); + self.progress(record, false) } else if record.level() > Level::Warn { self.normal(record) } else { @@ -182,13 +93,34 @@ impl CliLogger { } fn progress(&self, record: &Record, stop: bool) { - let mut progress_option = self.progress.lock().unwrap(); - if stop && progress_option.is_some() { - progress_option.take().unwrap().stop() - } else if let Some(p) = &*progress_option { - p.send(record); + let mut progress = self.progress.lock().unwrap(); + + let msg = format!("{}", record.args()); + if stop && progress.is_some() { + if msg.is_empty() { + progress.take().unwrap().finish() + } else { + progress.take().unwrap().finish_with_message(msg) + } + } else if let Some(p) = &*progress { + p.println(format!(":: → {}", msg)) } else { - *progress_option = Some(CliProgress::new(record)) + #[cfg(not(windows))] + let finish_str = "✔"; + #[cfg(windows)] + // windows does not support all unicode characters by default in their consoles, so + // we're using this (square root?) symbol instead. microsoft. + let finish_str = "√"; + + let pb = ProgressBar::new_spinner(); + pb.set_style( + ProgressStyle::with_template(":: {spinner} {msg}") + .unwrap() + .tick_strings(&["-", "\\", "|", "/", finish_str]), + ); + pb.enable_steady_tick(Duration::from_millis(200)); + pb.set_message(msg); + *progress = Some(pb) } } } diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index a64057a..6045ce4 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -1,13 +1,12 @@ use crate::utils::context::Context; use anyhow::{bail, Result}; use crunchyroll_rs::media::{Resolution, VariantData, VariantSegment}; +use indicatif::{ProgressBar, ProgressFinish, ProgressStyle}; use log::{debug, LevelFilter}; use std::borrow::{Borrow, BorrowMut}; use std::collections::BTreeMap; -use std::io; use std::io::Write; use std::sync::{mpsc, Arc, Mutex}; -use std::time::Duration; use tokio::task::JoinSet; pub fn find_resolution( @@ -35,78 +34,25 @@ pub async fn download_segments( let client = Arc::new(ctx.crunchy.client()); let count = Arc::new(Mutex::new(0)); - let amount = Arc::new(Mutex::new(0)); - // only print progress when log level is info - let output_handler = if log::max_level() == LevelFilter::Info { - let output_count = count.clone(); - let output_amount = amount.clone(); - Some(tokio::spawn(async move { - let sleep_time_ms = 100; - let iter_per_sec = 1000f64 / sleep_time_ms as f64; + let progress = if log::max_level() == LevelFilter::Info { + let estimated_file_size = (variant_data.bandwidth / 8) + * segments + .iter() + .map(|s| s.length.unwrap_or_default().as_secs()) + .sum::(); - let mut bytes_start = 0f64; - let mut speed = 0f64; - let mut percentage = 0f64; - - while *output_count.lock().unwrap() < total_segments || percentage < 100f64 { - let tmp_amount = *output_amount.lock().unwrap() as f64; - - let tmp_speed = (tmp_amount - bytes_start) / 1024f64 / 1024f64; - if *output_count.lock().unwrap() < 3 { - speed = tmp_speed; - } else { - let (old_speed_ratio, new_speed_ratio) = if iter_per_sec <= 1f64 { - (0f64, 1f64) - } else { - (1f64 - (1f64 / iter_per_sec), (1f64 / iter_per_sec)) - }; - - // calculate the average download speed "smoother" - speed = (speed * old_speed_ratio) + (tmp_speed * new_speed_ratio); - } - - percentage = - (*output_count.lock().unwrap() as f64 / total_segments as f64) * 100f64; - - let size = terminal_size::terminal_size() - .unwrap_or((terminal_size::Width(60), terminal_size::Height(0))) - .0 - .0 as usize; - - // there is a offset of 1 "length" (idk how to describe it), so removing 1 from - // `progress_available` would fill the terminal width completely. on multiple - // systems there is a bug that printing until the end of the line causes a newline - // even though technically there shouldn't be one. on my tests, this only happens on - // windows and mac machines and (at the addressed environments) only with release - // builds. so maybe an unwanted optimization? - let progress_available = size - - if let Some(msg) = &message { - 35 + msg.len() - } else { - 34 - }; - let progress_done_count = - (progress_available as f64 * (percentage / 100f64)).ceil() as usize; - let progress_to_do_count = progress_available - progress_done_count; - - let _ = write!( - io::stdout(), - "\r:: {}{:>5.1} MiB {:>5.2} MiB/s [{}{}] {:>3}%", - message.clone().map_or("".to_string(), |msg| msg + " "), - tmp_amount / 1024f64 / 1024f64, - speed * iter_per_sec, - "#".repeat(progress_done_count), - "-".repeat(progress_to_do_count), - percentage as usize - ); - - bytes_start = tmp_amount; - - tokio::time::sleep(Duration::from_millis(sleep_time_ms)).await; - } - println!() - })) + let progress = ProgressBar::new(estimated_file_size) + .with_style( + ProgressStyle::with_template( + ":: {msg}{bytes:>10} {bytes_per_sec:>12} [{wide_bar}] {percent:>3}%", + ) + .unwrap() + .progress_chars("##-"), + ) + .with_message(message.map(|m| m + " ").unwrap_or_default()) + .with_finish(ProgressFinish::Abandon); + Some(progress) } else { None }; @@ -116,7 +62,7 @@ pub async fn download_segments( for _ in 0..cpus { segs.push(vec![]) } - for (i, segment) in segments.into_iter().enumerate() { + for (i, segment) in segments.clone().into_iter().enumerate() { segs[i - ((i / cpus) * cpus)].push(segment); } @@ -127,15 +73,12 @@ pub async fn download_segments( let thread_client = client.clone(); let thread_sender = sender.clone(); let thread_segments = segs.remove(0); - let thread_amount = amount.clone(); let thread_count = count.clone(); join_set.spawn(async move { for (i, segment) in thread_segments.into_iter().enumerate() { let response = thread_client.get(&segment.url).send().await?; let mut buf = response.bytes().await?.to_vec(); - *thread_amount.lock().unwrap() += buf.len(); - buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec(); debug!( "Downloaded and decrypted segment {} ({})", @@ -155,13 +98,28 @@ pub async fn download_segments( let mut buf: BTreeMap> = BTreeMap::new(); loop { // is always `Some` because `sender` does not get dropped when all threads are finished - let data = receiver.recv().unwrap(); + let (pos, bytes) = receiver.recv().unwrap(); - if data_pos == data.0 { - writer.write_all(data.1.borrow())?; + if let Some(p) = &progress { + let progress_len = p.length().unwrap(); + let estimated_segment_len = (variant_data.bandwidth / 8) + * segments + .get(pos) + .unwrap() + .length + .unwrap_or_default() + .as_secs(); + let bytes_len = bytes.len() as u64; + + p.set_length(progress_len - estimated_segment_len + bytes_len); + p.inc(bytes_len) + } + + if data_pos == pos { + writer.write_all(bytes.borrow())?; data_pos += 1; } else { - buf.insert(data.0, data.1); + buf.insert(pos, bytes); } while let Some(b) = buf.remove(&data_pos) { writer.write_all(b.borrow())?; @@ -176,9 +134,6 @@ pub async fn download_segments( while let Some(joined) = join_set.join_next().await { joined?? } - if let Some(handler) = output_handler { - handler.await? - } Ok(()) } @@ -200,7 +155,7 @@ impl ToString for FFmpegPreset { &FFmpegPreset::H265 => "h265", &FFmpegPreset::H264 => "h264", } - .to_string() + .to_string() } } @@ -233,7 +188,9 @@ impl FFmpegPreset { }) } - pub(crate) fn ffmpeg_presets(mut presets: Vec) -> Result<(Vec, Vec)> { + pub(crate) fn ffmpeg_presets( + mut presets: Vec, + ) -> Result<(Vec, Vec)> { fn preset_check_remove(presets: &mut Vec, preset: FFmpegPreset) -> bool { if let Some(i) = presets.iter().position(|p| p == &preset) { presets.remove(i); diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index d669b64..00699e4 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -6,7 +6,7 @@ use anyhow::bail; use anyhow::Result; use clap::{Parser, Subcommand}; use crunchyroll_rs::{Crunchyroll, Locale}; -use log::{debug, error, info, LevelFilter}; +use log::{debug, error, LevelFilter}; use std::{env, fs}; mod cli; @@ -196,7 +196,7 @@ async fn crunchyroll_session(cli: &Cli) -> Result { + cli.login_method.etp_rt.is_some() as u8 + cli.login_method.anonymous as u8; - let _progress_handler = progress!("Logging in"); + let progress_handler = progress!("Logging in"); if login_methods_count == 0 { if let Some(login_file_path) = cli::login::login_file_path() { if login_file_path.exists() { @@ -232,7 +232,7 @@ async fn crunchyroll_session(cli: &Cli) -> Result { bail!("should never happen") }; - info!("Logged in"); + progress_handler.stop("Logged in"); Ok(crunchy) } diff --git a/crunchy-cli-core/src/utils/log.rs b/crunchy-cli-core/src/utils/log.rs index b9fa939..24fed5d 100644 --- a/crunchy-cli-core/src/utils/log.rs +++ b/crunchy-cli-core/src/utils/log.rs @@ -1,10 +1,21 @@ use log::info; -pub struct ProgressHandler; +pub struct ProgressHandler { + pub(crate) stopped: bool, +} impl Drop for ProgressHandler { fn drop(&mut self) { - info!(target: "progress_end", "") + if !self.stopped { + info!(target: "progress_end", "") + } + } +} + +impl ProgressHandler { + pub(crate) fn stop>(mut self, msg: S) { + self.stopped = true; + info!(target: "progress_end", "{}", msg.as_ref()) } } @@ -12,7 +23,7 @@ macro_rules! progress { ($($arg:tt)+) => { { log::info!(target: "progress", $($arg)+); - $crate::utils::log::ProgressHandler{} + $crate::utils::log::ProgressHandler{stopped: false} } } } From 2c3bd78fc1a03450d792733820dddf0d8da92368 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Tue, 27 Dec 2022 20:37:45 +0100 Subject: [PATCH 084/441] Leave special files untouched from renaming --- crunchy-cli-core/src/utils/os.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crunchy-cli-core/src/utils/os.rs b/crunchy-cli-core/src/utils/os.rs index 01d0f89..e3320fb 100644 --- a/crunchy-cli-core/src/utils/os.rs +++ b/crunchy-cli-core/src/utils/os.rs @@ -37,6 +37,12 @@ pub fn tempfile>(suffix: S) -> io::Result { /// Check if the given path exists and rename it until the new (renamed) file does not exist. pub fn free_file(mut path: PathBuf) -> PathBuf { + // if path is not a file and not a dir it's probably a pipe on linux which reguarly is intended + // and thus does not need to be renamed. what it is on windows ¯\_(ツ)_/¯ + if !path.is_file() && !path.is_dir() { + return path; + } + let mut i = 0; while path.exists() { i += 1; From c37e2495e1e66d9ee7a5d17d3570dcb5f081e137 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Tue, 27 Dec 2022 20:49:53 +0100 Subject: [PATCH 085/441] Create output parent directory if it doesn't exists (#91) --- crunchy-cli-core/src/cli/archive.rs | 7 +++++++ crunchy-cli-core/src/cli/download.rs | 13 +++++++++++++ 2 files changed, 20 insertions(+) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index b5244f7..70240cf 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -647,6 +647,13 @@ fn generate_mkv( debug!("ffmpeg {}", command_args.join(" ")); + // create parent directory if it does not exist + if let Some(parent) = target.parent() { + if !parent.exists() { + std::fs::create_dir_all(parent)? + } + } + let ffmpeg = Command::new("ffmpeg") .stdout(Stdio::null()) .stderr(Stdio::piped()) diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index d8a4ada..99da758 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -240,6 +240,12 @@ impl Execute for Download { let mut stdout = std::io::stdout().lock(); download_segments(&ctx, &mut stdout, None, format.stream).await?; } else { + // create parent directory if it does not exist + if let Some(parent) = path.parent() { + if !parent.exists() { + std::fs::create_dir_all(parent)? + } + } let mut file = File::options().create(true).write(true).open(&path)?; download_segments(&ctx, &mut file, None, format.stream).await? } @@ -259,6 +265,13 @@ async fn download_ffmpeg( let (input_presets, output_presets) = FFmpegPreset::ffmpeg_presets(download.ffmpeg_preset.clone())?; + // create parent directory if it does not exist + if let Some(parent) = target.parent() { + if !parent.exists() { + std::fs::create_dir_all(parent)? + } + } + let mut ffmpeg = Command::new("ffmpeg") .stdin(Stdio::piped()) .stdout(Stdio::null()) From 14f42833cb5257e0f9ef10f4b243c0ec51cbf28e Mon Sep 17 00:00:00 2001 From: ByteDream Date: Tue, 27 Dec 2022 22:59:35 +0100 Subject: [PATCH 086/441] Fix output to special file (pipes etc.) --- crunchy-cli-core/src/cli/archive.rs | 9 +++++++-- crunchy-cli-core/src/cli/download.rs | 27 ++++++++++++++++++++++----- crunchy-cli-core/src/utils/os.rs | 13 +++++++++---- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 70240cf..e3d5520 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -3,7 +3,7 @@ use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset}; use crate::utils::context::Context; use crate::utils::format::{format_string, Format}; use crate::utils::log::progress; -use crate::utils::os::{free_file, has_ffmpeg, tempfile}; +use crate::utils::os::{free_file, has_ffmpeg, is_special_file, tempfile}; use crate::utils::parse::{parse_url, UrlFilter}; use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number}; use crate::Execute; @@ -133,6 +133,7 @@ impl Execute for Archive { .unwrap_or_default() .to_string_lossy() != "mkv" + && !is_special_file(PathBuf::from(&self.output)) { bail!("File extension is not '.mkv'. Currently only matroska / '.mkv' files are supported") } @@ -251,7 +252,11 @@ impl Execute for Archive { info!( "Downloading {} to '{}'", primary.title, - path.to_str().unwrap() + if is_special_file(&path) { + path.to_str().unwrap() + } else { + path.file_name().unwrap().to_str().unwrap() + } ); tab_info!( "Episode: S{:02}E{:02}", diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index 99da758..0411a5e 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -3,7 +3,7 @@ use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset}; use crate::utils::context::Context; use crate::utils::format::{format_string, Format}; use crate::utils::log::progress; -use crate::utils::os::{free_file, has_ffmpeg}; +use crate::utils::os::{free_file, has_ffmpeg, is_special_file}; use crate::utils::parse::{parse_url, UrlFilter}; use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number}; use crate::Execute; @@ -219,7 +219,11 @@ impl Execute for Download { info!( "Downloading {} to '{}'", format.title, - path.file_name().unwrap().to_str().unwrap() + if is_special_file(&path) { + path.to_str().unwrap() + } else { + path.file_name().unwrap().to_str().unwrap() + } ); tab_info!("Episode: S{:02}E{:02}", format.season_number, format.number); tab_info!("Audio: {}", format.audio); @@ -232,9 +236,9 @@ impl Execute for Download { tab_info!("Resolution: {}", format.stream.resolution); tab_info!("FPS: {:.2}", format.stream.fps); - if path.extension().unwrap_or_default().to_string_lossy() != "ts" - || !self.ffmpeg_preset.is_empty() - { + let extension = path.extension().unwrap_or_default().to_string_lossy(); + + if (!extension.is_empty() && extension != "ts") || !self.ffmpeg_preset.is_empty() { download_ffmpeg(&ctx, &self, format.stream, path.as_path()).await?; } else if path.to_str().unwrap() == "-" { let mut stdout = std::io::stdout().lock(); @@ -279,6 +283,19 @@ async fn download_ffmpeg( .arg("-y") .args(input_presets) .args(["-f", "mpegts", "-i", "pipe:"]) + .args( + if target + .extension() + .unwrap_or_default() + .to_string_lossy() + .is_empty() + { + vec!["-f", "mpegts"] + } else { + vec![] + } + .as_slice(), + ) .args(output_presets) .arg(target.to_str().unwrap()) .spawn()?; diff --git a/crunchy-cli-core/src/utils/os.rs b/crunchy-cli-core/src/utils/os.rs index e3320fb..d2d4c3c 100644 --- a/crunchy-cli-core/src/utils/os.rs +++ b/crunchy-cli-core/src/utils/os.rs @@ -1,6 +1,6 @@ use log::debug; use std::io::ErrorKind; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::{env, io}; use tempfile::{Builder, NamedTempFile}; @@ -37,9 +37,8 @@ pub fn tempfile>(suffix: S) -> io::Result { /// Check if the given path exists and rename it until the new (renamed) file does not exist. pub fn free_file(mut path: PathBuf) -> PathBuf { - // if path is not a file and not a dir it's probably a pipe on linux which reguarly is intended - // and thus does not need to be renamed. what it is on windows ¯\_(ツ)_/¯ - if !path.is_file() && !path.is_dir() { + // if it's a special file does not rename it + if is_special_file(&path) { return path; } @@ -58,3 +57,9 @@ pub fn free_file(mut path: PathBuf) -> PathBuf { } path } + +/// Check if the given path is a special file. On Linux this is probably a pipe and on Windows +/// ¯\_(ツ)_/¯ +pub fn is_special_file>(path: P) -> bool { + path.as_ref().exists() && !path.as_ref().is_file() && !path.as_ref().is_dir() +} From c5940a240c8b2a6c39d6b3033ea6b5f6ee761335 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 28 Dec 2022 02:18:17 +0100 Subject: [PATCH 087/441] Slightly change download process to be more verbose in error situations --- crunchy-cli-core/src/cli/utils.rs | 37 ++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index 6045ce4..1f6bf40 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -93,13 +93,16 @@ pub async fn download_segments( Ok(()) }); } + // drop the sender already here so it does not outlive all (download) threads which are the only + // real consumers of it + drop(sender); + // this is the main loop which writes the data. it uses a BTreeMap as a buffer as the write + // happens synchronized. the download consist of multiple segments. the map keys are representing + // the segment number and the values the corresponding bytes let mut data_pos = 0usize; let mut buf: BTreeMap> = BTreeMap::new(); - loop { - // is always `Some` because `sender` does not get dropped when all threads are finished - let (pos, bytes) = receiver.recv().unwrap(); - + for (pos, bytes) in receiver.iter() { if let Some(p) = &progress { let progress_len = p.length().unwrap(); let estimated_segment_len = (variant_data.bandwidth / 8) @@ -115,26 +118,44 @@ pub async fn download_segments( p.inc(bytes_len) } + // check if the currently sent bytes are the next in the buffer. if so, write them directly + // to the target without first adding them to the buffer. + // if not, add them to the buffer if data_pos == pos { writer.write_all(bytes.borrow())?; data_pos += 1; } else { buf.insert(pos, bytes); } + // check if the buffer contains the next segment(s) while let Some(b) = buf.remove(&data_pos) { writer.write_all(b.borrow())?; data_pos += 1; } - - if *count.lock().unwrap() >= total_segments && buf.is_empty() { - break; - } } + // write the remaining buffer, if existent + while let Some(b) = buf.remove(&data_pos) { + writer.write_all(b.borrow())?; + data_pos += 1; + } + + // if any error has occured while downloading it gets returned here. maybe a little late, if one + // out of, for example 12, threads has the error while let Some(joined) = join_set.join_next().await { joined?? } + if !buf.is_empty() { + bail!( + "Download buffer is not empty. Remaining segments: {}", + buf.into_keys() + .map(|k| k.to_string()) + .collect::>() + .join(", ") + ) + } + Ok(()) } From 240e5563a3db767fccccfe523af74a8d862f5da0 Mon Sep 17 00:00:00 2001 From: Alexandru Dracea Date: Wed, 28 Dec 2022 15:44:45 +0200 Subject: [PATCH 088/441] Add error handling and retry attempts Handles cases where the segments fail to download and sometimes get stuck by introducing a timeout and retrying on failure. --- crunchy-cli-core/src/cli/utils.rs | 51 +++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index 1f6bf40..3d14318 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -76,8 +76,55 @@ pub async fn download_segments( let thread_count = count.clone(); join_set.spawn(async move { for (i, segment) in thread_segments.into_iter().enumerate() { - let response = thread_client.get(&segment.url).send().await?; - let mut buf = response.bytes().await?.to_vec(); + let response_res = thread_client + .get(&segment.url) + .timeout(Duration::from_secs(60u64)) + .send() + .await; + let verfified_response = match response_res { + Ok(x) => x, + Err(y) => panic!("This is likely a netowrking error: {}", y), + }; + let possible_error_in_response = verfified_response.bytes().await; + let mut buf = if let Ok(r) = possible_error_in_response { + r.to_vec() + } else { + debug!( + "Segment Failed to download: {}, retrying.", + num + (i * cpus) + ); + let mut resp = thread_client + .get(&segment.url) + .timeout(Duration::from_secs(60u64)) + .send() + .await + .unwrap() + .bytes() + .await; + if resp.is_err() { + let mut retry_ctr = 1; + loop { + debug!( + "Segment Failed to download: {}, retry {}.", + num + (i * cpus), + retry_ctr + ); + resp = thread_client + .get(&segment.url) + .timeout(Duration::from_secs(60u64)) + .send() + .await + .unwrap() + .bytes() + .await; + if resp.is_ok() { + break; + } + retry_ctr += 1; + } + } + resp.unwrap().to_vec() + }; buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec(); debug!( From 8a3c0132e7e0ec4206d001670fd062bcf9673052 Mon Sep 17 00:00:00 2001 From: Alexandru Dracea Date: Wed, 28 Dec 2022 15:59:55 +0200 Subject: [PATCH 089/441] Update utils.rs --- crunchy-cli-core/src/cli/utils.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index 3d14318..afc431f 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -3,7 +3,8 @@ use anyhow::{bail, Result}; use crunchyroll_rs::media::{Resolution, VariantData, VariantSegment}; use indicatif::{ProgressBar, ProgressFinish, ProgressStyle}; use log::{debug, LevelFilter}; -use std::borrow::{Borrow, BorrowMut}; +use std::borrow::Borrow; +use std::time::Duration; use std::collections::BTreeMap; use std::io::Write; use std::sync::{mpsc, Arc, Mutex}; From c2ae622d01c1913ff51264858a4c6b0f28353690 Mon Sep 17 00:00:00 2001 From: Alexandru Dracea Date: Wed, 28 Dec 2022 16:01:55 +0200 Subject: [PATCH 090/441] Update utils.rs --- crunchy-cli-core/src/cli/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index afc431f..cd45859 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -3,7 +3,7 @@ use anyhow::{bail, Result}; use crunchyroll_rs::media::{Resolution, VariantData, VariantSegment}; use indicatif::{ProgressBar, ProgressFinish, ProgressStyle}; use log::{debug, LevelFilter}; -use std::borrow::Borrow; +use std::borrow::{Borrow, BorrowMut}; use std::time::Duration; use std::collections::BTreeMap; use std::io::Write; From d0681c7f6cc01caa684b39f30c459a20d5f09c24 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 28 Dec 2022 15:18:12 +0100 Subject: [PATCH 091/441] Simplify retry segment download --- crunchy-cli-core/src/cli/utils.rs | 62 ++++++++++--------------------- 1 file changed, 19 insertions(+), 43 deletions(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index cd45859..7c32e86 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -4,10 +4,10 @@ use crunchyroll_rs::media::{Resolution, VariantData, VariantSegment}; use indicatif::{ProgressBar, ProgressFinish, ProgressStyle}; use log::{debug, LevelFilter}; use std::borrow::{Borrow, BorrowMut}; -use std::time::Duration; use std::collections::BTreeMap; use std::io::Write; use std::sync::{mpsc, Arc, Mutex}; +use std::time::Duration; use tokio::task::JoinSet; pub fn find_resolution( @@ -77,54 +77,30 @@ pub async fn download_segments( let thread_count = count.clone(); join_set.spawn(async move { for (i, segment) in thread_segments.into_iter().enumerate() { - let response_res = thread_client - .get(&segment.url) - .timeout(Duration::from_secs(60u64)) - .send() - .await; - let verfified_response = match response_res { - Ok(x) => x, - Err(y) => panic!("This is likely a netowrking error: {}", y), - }; - let possible_error_in_response = verfified_response.bytes().await; - let mut buf = if let Ok(r) = possible_error_in_response { - r.to_vec() - } else { - debug!( - "Segment Failed to download: {}, retrying.", - num + (i * cpus) - ); - let mut resp = thread_client + let mut retry_count = 0; + let mut buf = loop { + let response = thread_client .get(&segment.url) - .timeout(Duration::from_secs(60u64)) + .timeout(Duration::from_secs(10)) .send() .await - .unwrap() - .bytes() - .await; - if resp.is_err() { - let mut retry_ctr = 1; - loop { - debug!( - "Segment Failed to download: {}, retry {}.", - num + (i * cpus), - retry_ctr - ); - resp = thread_client - .get(&segment.url) - .timeout(Duration::from_secs(60u64)) - .send() - .await - .unwrap() - .bytes() - .await; - if resp.is_ok() { - break; + .unwrap(); + + match response.bytes().await { + Ok(b) => break b.to_vec(), + Err(e) => { + if e.is_body() { + if retry_count == 5 { + panic!("Max retry count reached ({}), multiple errors occured while receiving segment {}: {}", retry_count, num + (i * cpus), e) + } + debug!("Failed to download segment {}. Retrying ({} out of 5 retries left)", num + (i * cpus), 5 - retry_count) + } else { + panic!("{}", e) } - retry_ctr += 1; } } - resp.unwrap().to_vec() + + retry_count += 1; }; buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec(); From 7115c5546d1c71c496c26b82d85b2f1aaf114aca Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 28 Dec 2022 15:25:10 +0100 Subject: [PATCH 092/441] Show error message on segment download retry --- crunchy-cli-core/src/cli/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index 7c32e86..e2bbc63 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -93,7 +93,7 @@ pub async fn download_segments( if retry_count == 5 { panic!("Max retry count reached ({}), multiple errors occured while receiving segment {}: {}", retry_count, num + (i * cpus), e) } - debug!("Failed to download segment {}. Retrying ({} out of 5 retries left)", num + (i * cpus), 5 - retry_count) + debug!("Failed to download segment {} ({}). Retrying, {} out of 5 retries left", num + (i * cpus), e, 5 - retry_count) } else { panic!("{}", e) } From b8e46099f9035c609b376c2cfa1b0d28719564c9 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 28 Dec 2022 15:35:38 +0100 Subject: [PATCH 093/441] Re-increase segment request timeout --- crunchy-cli-core/src/cli/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index e2bbc63..841dbef 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -81,7 +81,7 @@ pub async fn download_segments( let mut buf = loop { let response = thread_client .get(&segment.url) - .timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(60)) .send() .await .unwrap(); From 03db38b31ce85f7264fe90d259c09e7ad3ee8679 Mon Sep 17 00:00:00 2001 From: ByteDream <63594396+ByteDream@users.noreply.github.com> Date: Wed, 28 Dec 2022 15:45:33 +0100 Subject: [PATCH 094/441] Add debug segment percentage (#93) * Fix file extension unwrap panic * Change log output name from crunchy_cli_core to crunchy_cli * Add percentage output --- crunchy-cli-core/src/cli/log.rs | 5 +++-- crunchy-cli-core/src/cli/utils.rs | 8 ++++++-- crunchy-cli-core/src/utils/os.rs | 4 ++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/crunchy-cli-core/src/cli/log.rs b/crunchy-cli-core/src/cli/log.rs index 377685b..3f63eb0 100644 --- a/crunchy-cli-core/src/cli/log.rs +++ b/crunchy-cli-core/src/cli/log.rs @@ -75,8 +75,9 @@ impl CliLogger { // replace the 'progress' prefix if this function is invoked via 'progress!' record .target() - .replacen("progress", "crunchy_cli", 1) - .replacen("progress_end", "crunchy_cli", 1), + .replacen("crunchy_cli_core", "crunchy_cli", 1) + .replacen("progress_end", "crunchy_cli", 1) + .replacen("progress", "crunchy_cli", 1), format!("{:?}", thread::current().id()) .replace("ThreadId(", "") .replace(')', ""), diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index 6045ce4..4c8c10e 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -80,14 +80,18 @@ pub async fn download_segments( let mut buf = response.bytes().await?.to_vec(); buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec(); + + let mut c = thread_count.lock().unwrap(); debug!( - "Downloaded and decrypted segment {} ({})", + "Downloaded and decrypted segment [{}/{} {:.2}%] {}", num + (i * cpus), + total_segments, + ((*c + 1) as f64 / total_segments as f64) * 100f64, segment.url ); thread_sender.send((num + (i * cpus), buf))?; - *thread_count.lock().unwrap() += 1; + *c += 1; } Ok(()) diff --git a/crunchy-cli-core/src/utils/os.rs b/crunchy-cli-core/src/utils/os.rs index d2d4c3c..a7b3fbf 100644 --- a/crunchy-cli-core/src/utils/os.rs +++ b/crunchy-cli-core/src/utils/os.rs @@ -46,8 +46,8 @@ pub fn free_file(mut path: PathBuf) -> PathBuf { while path.exists() { i += 1; - let ext = path.extension().unwrap().to_string_lossy(); - let mut filename = path.file_stem().unwrap().to_str().unwrap(); + let ext = path.extension().unwrap_or_default().to_string_lossy(); + let mut filename = path.file_stem().unwrap_or_default().to_str().unwrap(); if filename.ends_with(&format!(" ({})", i - 1)) { filename = filename.strip_suffix(&format!(" ({})", i - 1)).unwrap(); From 0c139420165e412516313d612124dd62fe2b117f Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 2 Jan 2023 17:53:54 +0100 Subject: [PATCH 095/441] Update dependencies --- Cargo.lock | 69 ++++++++++++++++--------------------- crunchy-cli-core/Cargo.lock | 61 ++++++++++++++------------------ 2 files changed, 56 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a7432c8..8221757 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,9 +148,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.30" +version = "4.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "656ad1e55e23d287773f7d8192c300dc715c3eeded93b3da651d11c42cfd74d2" +checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39" dependencies = [ "bitflags", "clap_derive", @@ -163,9 +163,9 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.0.6" +version = "4.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7b3c9eae0de7bf8e3f904a5e40612b21fb2e2e566456d177809a48b892d24da" +checksum = "10861370d2ba66b0f5989f83ebf35db6421713fd92351790e7fdd6c36774c56b" dependencies = [ "clap", ] @@ -194,9 +194,9 @@ dependencies = [ [[package]] name = "clap_mangen" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e503c3058af0a0854668ea01db55c622482a080092fede9dd2e00a00a9436504" +checksum = "904eb24d05ad587557e0f484ddce5c737c30cf81372badb16d13e41c4b8340b1" dependencies = [ "clap", "roff", @@ -214,16 +214,15 @@ dependencies = [ [[package]] name = "console" -version = "0.15.2" +version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c050367d967ced717c04b65d8c619d863ef9292ce0c5760028655a2fb298718c" +checksum = "5556015fe3aad8b968e5d4124980fbe2f6aaee7aeec6b749de1faaa2ca5d0a4c" dependencies = [ "encode_unicode", "lazy_static", "libc", - "terminal_size 0.1.17", "unicode-width", - "winapi", + "windows-sys 0.42.0", ] [[package]] @@ -312,21 +311,23 @@ dependencies = [ "signal-hook", "sys-locale", "tempfile", - "terminal_size 0.2.3", + "terminal_size", "tokio", ] [[package]] name = "crunchyroll-rs" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5032be36dc6c4757af4f28152659e7a772b48f6925f0e0ac81b5c384e291314" +checksum = "5ea0bc79ecd9fafc95c6049d0b38e0e00c8d4665314ae3ee675db67b622ce3b7" dependencies = [ "aes", + "async-trait", "cbc", "chrono", "crunchyroll-rs-internal", "http", + "lazy_static", "m3u8-rs", "regex", "reqwest", @@ -341,9 +342,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e4aa1f09fd76f44329455abfac35e22c654ac782e93bc8a7d3ee1be27509f4" +checksum = "66be957a34f7498e4bc324af67c58bb7ace299c214c9353281545f92dbc45d78" dependencies = [ "darling", "quote", @@ -908,9 +909,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.138" +version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" [[package]] name = "link-cplusplus" @@ -1014,9 +1015,9 @@ dependencies = [ [[package]] name = "nom" -version = "7.1.1" +version = "7.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +checksum = "e5507769c4919c998e69e49c839d9dc6e693ede4cc4290d6ad8b41d4f09c548c" dependencies = [ "memchr", "minimal-lexical", @@ -1059,9 +1060,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "once_cell" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" [[package]] name = "openssl" @@ -1140,9 +1141,9 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "portable-atomic" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bdd679d533107e090c2704a35982fc06302e30898e63ffa26a81155c012e92" +checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b" [[package]] name = "proc-macro-error" @@ -1328,9 +1329,9 @@ checksum = "b833d8d034ea094b1ea68aa6d5c740e0d04bad9d16568d08ba6f76823a114316" [[package]] name = "rustix" -version = "0.36.5" +version = "0.36.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3807b5d10909833d3e9acd1eb5fb988f79376ff10fce42937de71a449c4c588" +checksum = "4feacf7db682c6c329c4ede12649cd36ecab0f3be5b7d74e6a20304725db4549" dependencies = [ "bitflags", "errno", @@ -1428,18 +1429,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.151" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fed41fc1a24994d044e6db6935e69511a1153b52c15eb42493b26fa87feba0" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.151" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "255abe9a125a985c05190d687b320c12f9b1f0b99445e608c21ba0782c719ad8" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2", "quote", @@ -1583,16 +1584,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "terminal_size" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "terminal_size" version = "0.2.3" diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index 8347aa7..6a805d3 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -148,9 +148,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.30" +version = "4.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "656ad1e55e23d287773f7d8192c300dc715c3eeded93b3da651d11c42cfd74d2" +checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39" dependencies = [ "bitflags", "clap_derive", @@ -195,16 +195,15 @@ dependencies = [ [[package]] name = "console" -version = "0.15.2" +version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c050367d967ced717c04b65d8c619d863ef9292ce0c5760028655a2fb298718c" +checksum = "5556015fe3aad8b968e5d4124980fbe2f6aaee7aeec6b749de1faaa2ca5d0a4c" dependencies = [ "encode_unicode", "lazy_static", "libc", - "terminal_size 0.1.17", "unicode-width", - "winapi", + "windows-sys 0.42.0", ] [[package]] @@ -281,21 +280,23 @@ dependencies = [ "signal-hook", "sys-locale", "tempfile", - "terminal_size 0.2.3", + "terminal_size", "tokio", ] [[package]] name = "crunchyroll-rs" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5032be36dc6c4757af4f28152659e7a772b48f6925f0e0ac81b5c384e291314" +checksum = "5ea0bc79ecd9fafc95c6049d0b38e0e00c8d4665314ae3ee675db67b622ce3b7" dependencies = [ "aes", + "async-trait", "cbc", "chrono", "crunchyroll-rs-internal", "http", + "lazy_static", "m3u8-rs", "regex", "reqwest", @@ -310,9 +311,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e4aa1f09fd76f44329455abfac35e22c654ac782e93bc8a7d3ee1be27509f4" +checksum = "66be957a34f7498e4bc324af67c58bb7ace299c214c9353281545f92dbc45d78" dependencies = [ "darling", "quote", @@ -877,9 +878,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.138" +version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" [[package]] name = "link-cplusplus" @@ -983,9 +984,9 @@ dependencies = [ [[package]] name = "nom" -version = "7.1.1" +version = "7.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +checksum = "e5507769c4919c998e69e49c839d9dc6e693ede4cc4290d6ad8b41d4f09c548c" dependencies = [ "memchr", "minimal-lexical", @@ -1028,9 +1029,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "once_cell" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" [[package]] name = "openssl" @@ -1109,9 +1110,9 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "portable-atomic" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bdd679d533107e090c2704a35982fc06302e30898e63ffa26a81155c012e92" +checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b" [[package]] name = "proc-macro-error" @@ -1291,9 +1292,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.5" +version = "0.36.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3807b5d10909833d3e9acd1eb5fb988f79376ff10fce42937de71a449c4c588" +checksum = "4feacf7db682c6c329c4ede12649cd36ecab0f3be5b7d74e6a20304725db4549" dependencies = [ "bitflags", "errno", @@ -1391,18 +1392,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.151" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fed41fc1a24994d044e6db6935e69511a1153b52c15eb42493b26fa87feba0" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.151" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "255abe9a125a985c05190d687b320c12f9b1f0b99445e608c21ba0782c719ad8" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2", "quote", @@ -1546,16 +1547,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "terminal_size" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "terminal_size" version = "0.2.3" From fae5d699331fcb9ced5856f0ab521982edc0fb76 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 2 Jan 2023 17:56:50 +0100 Subject: [PATCH 096/441] Apply stabilizations fixes (#89) --- crunchy-cli-core/src/lib.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index 00699e4..4f2ce32 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -189,8 +189,9 @@ async fn create_ctx(cli: &Cli) -> Result { } async fn crunchyroll_session(cli: &Cli) -> Result { - let mut builder = Crunchyroll::builder(); - builder.locale(cli.lang.clone().unwrap_or_else(system_locale)); + let mut builder = Crunchyroll::builder() + .locale(cli.lang.clone().unwrap_or_else(system_locale)) + .stabilization_locales(true); let login_methods_count = cli.login_method.credentials.is_some() as u8 + cli.login_method.etp_rt.is_some() as u8 From 3c3b7b65662e4f315ff34634d2c7b92a2445a212 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Tue, 3 Jan 2023 01:24:17 +0100 Subject: [PATCH 097/441] Fix panic on specific filenames --- crunchy-cli-core/src/utils/os.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crunchy-cli-core/src/utils/os.rs b/crunchy-cli-core/src/utils/os.rs index d2d4c3c..a7b3fbf 100644 --- a/crunchy-cli-core/src/utils/os.rs +++ b/crunchy-cli-core/src/utils/os.rs @@ -46,8 +46,8 @@ pub fn free_file(mut path: PathBuf) -> PathBuf { while path.exists() { i += 1; - let ext = path.extension().unwrap().to_string_lossy(); - let mut filename = path.file_stem().unwrap().to_str().unwrap(); + let ext = path.extension().unwrap_or_default().to_string_lossy(); + let mut filename = path.file_stem().unwrap_or_default().to_str().unwrap(); if filename.ends_with(&format!(" ({})", i - 1)) { filename = filename.strip_suffix(&format!(" ({})", i - 1)).unwrap(); From b365bda5dca57754a383bdb3de0484f600c9697c Mon Sep 17 00:00:00 2001 From: ByteDream Date: Tue, 3 Jan 2023 01:28:42 +0100 Subject: [PATCH 098/441] Fix download threads to properly return errors --- crunchy-cli-core/src/cli/utils.rs | 93 ++++++++++++++++++------------- 1 file changed, 53 insertions(+), 40 deletions(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index 841dbef..99d4f67 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -74,47 +74,56 @@ pub async fn download_segments( let thread_client = client.clone(); let thread_sender = sender.clone(); let thread_segments = segs.remove(0); - let thread_count = count.clone(); join_set.spawn(async move { - for (i, segment) in thread_segments.into_iter().enumerate() { - let mut retry_count = 0; - let mut buf = loop { - let response = thread_client - .get(&segment.url) - .timeout(Duration::from_secs(60)) - .send() - .await - .unwrap(); + let after_download_sender = thread_sender.clone(); - match response.bytes().await { - Ok(b) => break b.to_vec(), - Err(e) => { - if e.is_body() { - if retry_count == 5 { - panic!("Max retry count reached ({}), multiple errors occured while receiving segment {}: {}", retry_count, num + (i * cpus), e) + // the download process is encapsulated in its own function. this is done to easily + // catch errors which get returned with `...?` and `bail!(...)` and that the thread + // itself can report that an error has occured + let download = || async move { + for (i, segment) in thread_segments.into_iter().enumerate() { + let mut retry_count = 0; + let mut buf = loop { + let response = thread_client + .get(&segment.url) + .timeout(Duration::from_secs(60)) + .send() + .await?; + + match response.bytes().await { + Ok(b) => break b.to_vec(), + Err(e) => { + if e.is_body() { + if retry_count == 5 { + bail!("Max retry count reached ({}), multiple errors occured while receiving segment {}: {}", retry_count, num + (i * cpus), e) + } + debug!("Failed to download segment {} ({}). Retrying, {} out of 5 retries left", num + (i * cpus), e, 5 - retry_count) + } else { + bail!("{}", e) } - debug!("Failed to download segment {} ({}). Retrying, {} out of 5 retries left", num + (i * cpus), e, 5 - retry_count) - } else { - panic!("{}", e) } } - } - retry_count += 1; - }; + retry_count += 1; + }; - buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec(); - debug!( - "Downloaded and decrypted segment {} ({})", - num + (i * cpus), - segment.url - ); - thread_sender.send((num + (i * cpus), buf))?; + buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec(); + debug!( + "Downloaded and decrypted segment {} ({})", + num + (i * cpus), + segment.url + ); + thread_sender.send((num as i32 + (i * cpus) as i32, buf))?; + } + Ok(()) + }; - *thread_count.lock().unwrap() += 1; + let result = download().await; + if result.is_err() { + after_download_sender.send((-1 as i32, vec![]))?; } - Ok(()) + result }); } // drop the sender already here so it does not outlive all (download) threads which are the only @@ -124,14 +133,19 @@ pub async fn download_segments( // this is the main loop which writes the data. it uses a BTreeMap as a buffer as the write // happens synchronized. the download consist of multiple segments. the map keys are representing // the segment number and the values the corresponding bytes - let mut data_pos = 0usize; - let mut buf: BTreeMap> = BTreeMap::new(); + let mut data_pos = 0; + let mut buf: BTreeMap> = BTreeMap::new(); for (pos, bytes) in receiver.iter() { + // if the position is lower than 0, an error occured in the sending download thread + if pos < 0 { + break + } + if let Some(p) = &progress { let progress_len = p.length().unwrap(); let estimated_segment_len = (variant_data.bandwidth / 8) * segments - .get(pos) + .get(pos as usize) .unwrap() .length .unwrap_or_default() @@ -158,18 +172,17 @@ pub async fn download_segments( } } + // if any error has occured while downloading it gets returned here + while let Some(joined) = join_set.join_next().await { + joined?? + } + // write the remaining buffer, if existent while let Some(b) = buf.remove(&data_pos) { writer.write_all(b.borrow())?; data_pos += 1; } - // if any error has occured while downloading it gets returned here. maybe a little late, if one - // out of, for example 12, threads has the error - while let Some(joined) = join_set.join_next().await { - joined?? - } - if !buf.is_empty() { bail!( "Download buffer is not empty. Remaining segments: {}", From 29c6129e6efbb18a1ac1ae7cbbc46893e4870603 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Tue, 3 Jan 2023 14:50:12 +0100 Subject: [PATCH 099/441] Update dependencies & version --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 2 +- crunchy-cli-core/Cargo.lock | 14 +++++++------- crunchy-cli-core/Cargo.toml | 2 +- crunchy-cli-core/src/lib.rs | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8221757..a564687 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -214,9 +214,9 @@ dependencies = [ [[package]] name = "console" -version = "0.15.3" +version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5556015fe3aad8b968e5d4124980fbe2f6aaee7aeec6b749de1faaa2ca5d0a4c" +checksum = "c9b6515d269224923b26b5febea2ed42b2d5f2ce37284a4dd670fedd6cb8347a" dependencies = [ "encode_unicode", "lazy_static", @@ -279,7 +279,7 @@ dependencies = [ [[package]] name = "crunchy-cli" -version = "3.0.0-dev.4" +version = "3.0.0-dev.5" dependencies = [ "chrono", "clap", @@ -291,7 +291,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-dev.4" +version = "3.0.0-dev.5" dependencies = [ "anyhow", "async-trait", @@ -317,9 +317,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea0bc79ecd9fafc95c6049d0b38e0e00c8d4665314ae3ee675db67b622ce3b7" +checksum = "f8ec300b509afbd8977f71cd7feb7d0f20d38d8e38976b7fcd51f6128cbdefe6" dependencies = [ "aes", "async-trait", @@ -342,9 +342,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66be957a34f7498e4bc324af67c58bb7ace299c214c9353281545f92dbc45d78" +checksum = "10d4b870818d5ce0993d70271bf803dbfbcc8a3a0fa7398b182f5f4b4e88509d" dependencies = [ "darling", "quote", diff --git a/Cargo.toml b/Cargo.toml index f5f988b..c67d599 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-dev.4" +version = "3.0.0-dev.5" edition = "2021" [dependencies] diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index 6a805d3..37ee2c3 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "console" -version = "0.15.3" +version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5556015fe3aad8b968e5d4124980fbe2f6aaee7aeec6b749de1faaa2ca5d0a4c" +checksum = "c9b6515d269224923b26b5febea2ed42b2d5f2ce37284a4dd670fedd6cb8347a" dependencies = [ "encode_unicode", "lazy_static", @@ -260,7 +260,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-dev.4" +version = "3.0.0-dev.5" dependencies = [ "anyhow", "async-trait", @@ -286,9 +286,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea0bc79ecd9fafc95c6049d0b38e0e00c8d4665314ae3ee675db67b622ce3b7" +checksum = "f8ec300b509afbd8977f71cd7feb7d0f20d38d8e38976b7fcd51f6128cbdefe6" dependencies = [ "aes", "async-trait", @@ -311,9 +311,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66be957a34f7498e4bc324af67c58bb7ace299c214c9353281545f92dbc45d78" +checksum = "10d4b870818d5ce0993d70271bf803dbfbcc8a3a0fa7398b182f5f4b4e88509d" dependencies = [ "darling", "quote", diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 780716f..e61031b 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-dev.4" +version = "3.0.0-dev.5" edition = "2021" [dependencies] diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index 4f2ce32..52a4da1 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -189,7 +189,7 @@ async fn create_ctx(cli: &Cli) -> Result { } async fn crunchyroll_session(cli: &Cli) -> Result { - let mut builder = Crunchyroll::builder() + let builder = Crunchyroll::builder() .locale(cli.lang.clone().unwrap_or_else(system_locale)) .stabilization_locales(true); From d0a8103e3def5a45c84bf7eafe2c8b43f8c7b98d Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 4 Jan 2023 00:12:13 +0100 Subject: [PATCH 100/441] Update dependencies & version --- Cargo.lock | 12 ++++++------ Cargo.toml | 2 +- crunchy-cli-core/Cargo.lock | 10 +++++----- crunchy-cli-core/Cargo.toml | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a564687..8da4e30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -279,7 +279,7 @@ dependencies = [ [[package]] name = "crunchy-cli" -version = "3.0.0-dev.5" +version = "3.0.0-dev.6" dependencies = [ "chrono", "clap", @@ -291,7 +291,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-dev.5" +version = "3.0.0-dev.6" dependencies = [ "anyhow", "async-trait", @@ -317,9 +317,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ec300b509afbd8977f71cd7feb7d0f20d38d8e38976b7fcd51f6128cbdefe6" +checksum = "dd742baacdfbf9caca8656262ac2397c334928ccd65fb332c95ee8e8b8d9223d" dependencies = [ "aes", "async-trait", @@ -342,9 +342,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d4b870818d5ce0993d70271bf803dbfbcc8a3a0fa7398b182f5f4b4e88509d" +checksum = "aadb66f2b5cdcc8b82f095aec6ca92ca4abb30016a13916f2358cf7bfd8a1997" dependencies = [ "darling", "quote", diff --git a/Cargo.toml b/Cargo.toml index c67d599..76ed950 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-dev.5" +version = "3.0.0-dev.6" edition = "2021" [dependencies] diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index 37ee2c3..af1086a 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -260,7 +260,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-dev.5" +version = "3.0.0-dev.6" dependencies = [ "anyhow", "async-trait", @@ -286,9 +286,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ec300b509afbd8977f71cd7feb7d0f20d38d8e38976b7fcd51f6128cbdefe6" +checksum = "dd742baacdfbf9caca8656262ac2397c334928ccd65fb332c95ee8e8b8d9223d" dependencies = [ "aes", "async-trait", @@ -311,9 +311,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d4b870818d5ce0993d70271bf803dbfbcc8a3a0fa7398b182f5f4b4e88509d" +checksum = "aadb66f2b5cdcc8b82f095aec6ca92ca4abb30016a13916f2358cf7bfd8a1997" dependencies = [ "darling", "quote", diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index e61031b..566b8c4 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-dev.5" +version = "3.0.0-dev.6" edition = "2021" [dependencies] @@ -15,7 +15,7 @@ ctrlc = "3.2" dirs = "4.0" indicatif = "0.17" log = { version = "0.4", features = ["std"] } -num_cpus = "1.14" +num_cpus = "1.15" regex = "1.7" sanitize-filename = "0.4" serde = "1.0" From 7726287859482f508dc584be405db7022ee31b22 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Wed, 4 Jan 2023 01:28:14 +0100 Subject: [PATCH 101/441] :) --- Cargo.lock | 8 ++++---- crunchy-cli-core/Cargo.lock | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8da4e30..eb7c6a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -317,9 +317,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd742baacdfbf9caca8656262ac2397c334928ccd65fb332c95ee8e8b8d9223d" +checksum = "184d0c725a09aec815316cbf41a2f362008ecb0e8c8e3b6b9930d01a89b5df21" dependencies = [ "aes", "async-trait", @@ -342,9 +342,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aadb66f2b5cdcc8b82f095aec6ca92ca4abb30016a13916f2358cf7bfd8a1997" +checksum = "5f3c82e1766339727fc2c10d66d0c4f001b1cf42e2993f9d93997b610f408776" dependencies = [ "darling", "quote", diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index af1086a..b5dd8a1 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -286,9 +286,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd742baacdfbf9caca8656262ac2397c334928ccd65fb332c95ee8e8b8d9223d" +checksum = "184d0c725a09aec815316cbf41a2f362008ecb0e8c8e3b6b9930d01a89b5df21" dependencies = [ "aes", "async-trait", @@ -311,9 +311,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aadb66f2b5cdcc8b82f095aec6ca92ca4abb30016a13916f2358cf7bfd8a1997" +checksum = "5f3c82e1766339727fc2c10d66d0c4f001b1cf42e2993f9d93997b610f408776" dependencies = [ "darling", "quote", From 404aa496e157162c35c3b5bf983cbc43c8689676 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Thu, 5 Jan 2023 22:28:23 +0100 Subject: [PATCH 102/441] Fix subtitle look and feel typo --- crunchy-cli-core/src/cli/archive.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index e3d5520..3910d24 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -481,7 +481,7 @@ fn fix_subtitle_look_and_feel(raw: Vec) -> Vec { for line in String::from_utf8_lossy(raw.as_slice()).split('\n') { if line.trim().starts_with('[') && script_info { - new.push_str("ScaledBorderAndShadows: yes\n"); + new.push_str("ScaledBorderAndShadow: yes\n"); script_info = false } else if line.trim() == "[Script Info]" { script_info = true From 892407d1f069b1dfcaa71c4eeca1c942ff40872c Mon Sep 17 00:00:00 2001 From: ByteDream Date: Fri, 6 Jan 2023 00:03:57 +0100 Subject: [PATCH 103/441] Fix --default-subtitle causing no such file error (#98) --- crunchy-cli-core/src/cli/archive.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 3910d24..3eb92a7 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -632,10 +632,10 @@ fn generate_mkv( if let Some(default_subtitle) = &archive.default_subtitle { // if `--default_subtitle ` is given set the default subtitle to the given locale if let Some(position) = subtitle_paths - .into_iter() - .position(|s| &s.1.locale == default_subtitle) + .iter() + .position(|(_, subtitle)| &subtitle.locale == default_subtitle) { - command_args.push(format!("-disposition:s:{}", position)) + command_args.extend([format!("-disposition:s:{}", position), "default".to_string()]) } else { command_args.extend(["-disposition:s:0".to_string(), "0".to_string()]) } From 06fd9a7a98989aa4a2c84c120b96b0477f9cf150 Mon Sep 17 00:00:00 2001 From: Hannes Braun Date: Wed, 4 Jan 2023 22:09:31 +0100 Subject: [PATCH 104/441] Archive subtitles of all versions of an episode --- crunchy-cli-core/src/cli/archive.rs | 95 +++++++++++++++++++------- crunchy-cli-core/src/utils/mod.rs | 1 + crunchy-cli-core/src/utils/subtitle.rs | 11 +++ 3 files changed, 82 insertions(+), 25 deletions(-) create mode 100644 crunchy-cli-core/src/utils/subtitle.rs diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 3eb92a7..8259ab3 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -6,6 +6,7 @@ use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file, tempfile}; use crate::utils::parse::{parse_url, UrlFilter}; use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number}; +use crate::utils::subtitle::Subtitle; use crate::Execute; use anyhow::{bail, Result}; use chrono::NaiveTime; @@ -232,7 +233,7 @@ impl Execute for Archive { } } - for (formats, subtitles) in archive_formats { + for (formats, mut subtitles) in archive_formats { let (primary, additionally) = formats.split_first().unwrap(); let mut path = PathBuf::from(&self.output); @@ -276,13 +277,14 @@ impl Execute for Archive { "Subtitle: {}", subtitles .iter() + .filter(|s| s.primary) // Don't print subtitles of non-primary streams. They might get removed depending on the merge behavior. .map(|s| { if let Some(default) = &self.default_subtitle { - if default == &s.locale { + if default == &s.stream_subtitle.locale { return format!("{} (primary)", default); } } - s.locale.to_string() + s.stream_subtitle.locale.to_string() }) .collect::>() .join(", ") @@ -309,13 +311,23 @@ impl Execute for Archive { } else { video_paths.push((path, additional)) } + + // Remove subtitles of deleted video + if only_audio { + subtitles.retain(|s| s.episode_id != additional.id); + } } let (primary_video, _) = video_paths.get(0).unwrap(); let primary_video_length = get_video_length(primary_video.to_path_buf()).unwrap(); for subtitle in subtitles { subtitle_paths.push(( - download_subtitle(&self, subtitle.clone(), primary_video_length).await?, + download_subtitle( + &self, + subtitle.stream_subtitle.clone(), + primary_video_length, + ) + .await?, subtitle, )) } @@ -334,7 +346,7 @@ async fn formats_from_series( archive: &Archive, series: Media, url_filter: &UrlFilter, -) -> Result, Vec)>> { +) -> Result, Vec)>> { let mut seasons = series.seasons().await?; // filter any season out which does not contain the specified audio languages @@ -367,8 +379,8 @@ async fn formats_from_series( } #[allow(clippy::type_complexity)] - let mut result: BTreeMap, Vec)>> = - BTreeMap::new(); + let mut result: BTreeMap, Vec)>> = BTreeMap::new(); + let mut primary_season = true; for season in series.seasons().await? { if !url_filter.is_season_valid(season.metadata.season_number) || !archive @@ -402,20 +414,26 @@ async fn formats_from_series( ) }; - let (ref mut formats, _) = result + let (ref mut formats, subtitles) = result .entry(season.metadata.season_number) .or_insert_with(BTreeMap::new) .entry(episode.metadata.episode_number) - .or_insert_with(|| { - let subtitles: Vec = archive - .subtitle - .iter() - .filter_map(|l| streams.subtitles.get(l).cloned()) - .collect(); - (vec![], subtitles) - }); + .or_insert_with(|| (vec![], vec![])); + subtitles.extend(archive.subtitle.iter().filter_map(|l| { + let stream_subtitle = streams.subtitles.get(l).cloned()?; + let subtitle = Subtitle { + stream_subtitle, + audio_locale: episode.metadata.audio_locale.clone(), + episode_id: episode.id.clone(), + forced: !episode.metadata.is_subbed, + primary: primary_season, + }; + Some(subtitle) + })); formats.push(Format::new_from_episode(episode, stream)); } + + primary_season = false; } Ok(result.into_values().flat_map(|v| v.into_values()).collect()) @@ -562,11 +580,12 @@ fn generate_mkv( target: PathBuf, video_paths: Vec<(TempPath, &Format)>, audio_paths: Vec<(TempPath, &Format)>, - subtitle_paths: Vec<(TempPath, StreamSubtitle)>, + subtitle_paths: Vec<(TempPath, Subtitle)>, ) -> Result<()> { let mut input = vec![]; let mut maps = vec![]; let mut metadata = vec![]; + let mut dispositions = vec![vec![]; subtitle_paths.len()]; for (i, (video_path, format)) in video_paths.iter().enumerate() { input.extend(["-i".to_string(), video_path.to_string_lossy().to_string()]); @@ -611,12 +630,26 @@ fn generate_mkv( ]); metadata.extend([ format!("-metadata:s:s:{}", i), - format!("language={}", subtitle.locale), + format!("language={}", subtitle.stream_subtitle.locale), ]); metadata.extend([ format!("-metadata:s:s:{}", i), - format!("title={}", subtitle.locale.to_human_readable()), + format!( + "title={}", + subtitle.stream_subtitle.locale.to_human_readable() + + if !subtitle.primary { + format!(" [Video: {}]", subtitle.audio_locale.to_human_readable()) + } else { + "".to_string() + } + .as_str() + ), ]); + + // mark forced subtitles + if subtitle.forced { + dispositions[i].push("forced"); + } } let (input_presets, output_presets) = @@ -633,16 +666,28 @@ fn generate_mkv( // if `--default_subtitle ` is given set the default subtitle to the given locale if let Some(position) = subtitle_paths .iter() - .position(|(_, subtitle)| &subtitle.locale == default_subtitle) + .position(|(_, subtitle)| &subtitle.stream_subtitle.locale == default_subtitle) { - command_args.extend([format!("-disposition:s:{}", position), "default".to_string()]) - } else { - command_args.extend(["-disposition:s:0".to_string(), "0".to_string()]) + dispositions[position].push("default"); } - } else { - command_args.extend(["-disposition:s:0".to_string(), "0".to_string()]) } + let disposition_args: Vec = dispositions + .iter() + .enumerate() + .flat_map(|(i, d)| { + vec![ + format!("-disposition:s:{}", i), + if !d.is_empty() { + d.join("+") + } else { + "0".to_string() + }, + ] + }) + .collect(); + command_args.extend(disposition_args); + command_args.extend(output_presets); command_args.extend([ "-f".to_string(), diff --git a/crunchy-cli-core/src/utils/mod.rs b/crunchy-cli-core/src/utils/mod.rs index 5f7a5d2..3b15a89 100644 --- a/crunchy-cli-core/src/utils/mod.rs +++ b/crunchy-cli-core/src/utils/mod.rs @@ -6,3 +6,4 @@ pub mod log; pub mod os; pub mod parse; pub mod sort; +pub mod subtitle; diff --git a/crunchy-cli-core/src/utils/subtitle.rs b/crunchy-cli-core/src/utils/subtitle.rs new file mode 100644 index 0000000..86b9359 --- /dev/null +++ b/crunchy-cli-core/src/utils/subtitle.rs @@ -0,0 +1,11 @@ +use crunchyroll_rs::media::StreamSubtitle; +use crunchyroll_rs::Locale; + +#[derive(Clone)] +pub struct Subtitle { + pub stream_subtitle: StreamSubtitle, + pub audio_locale: Locale, + pub episode_id: String, + pub forced: bool, + pub primary: bool, +} From 7588621f345e5b699985bda9a84cba2226151341 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sat, 7 Jan 2023 16:02:51 +0100 Subject: [PATCH 105/441] Add interactive input to choose season on duplicated season numbers (#55, #82) --- crunchy-cli-core/src/cli/archive.rs | 14 +++- crunchy-cli-core/src/cli/download.rs | 12 ++- crunchy-cli-core/src/cli/utils.rs | 106 ++++++++++++++++++++++++++- 3 files changed, 128 insertions(+), 4 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 8259ab3..2c952d6 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -1,5 +1,5 @@ use crate::cli::log::tab_info; -use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset}; +use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset, find_multiple_seasons_with_same_number, interactive_season_choosing}; use crate::utils::context::Context; use crate::utils::format::{format_string, Format}; use crate::utils::log::progress; @@ -120,6 +120,10 @@ pub struct Archive { #[arg(long)] no_subtitle_optimizations: bool, + #[arg(help = "Ignore interactive input")] + #[arg(short, long, default_value_t = false)] + yes: bool, + #[arg(help = "Crunchyroll series url(s)")] urls: Vec, } @@ -378,10 +382,16 @@ async fn formats_from_series( }) } + if !archive.yes && !find_multiple_seasons_with_same_number(&seasons).is_empty() { + info!(target: "progress_end", "Fetched seasons"); + seasons = interactive_season_choosing(seasons); + info!(target: "progress", "Fetching series details") + } + #[allow(clippy::type_complexity)] let mut result: BTreeMap, Vec)>> = BTreeMap::new(); let mut primary_season = true; - for season in series.seasons().await? { + for season in seasons { if !url_filter.is_season_valid(season.metadata.season_number) || !archive .locale diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index 0411a5e..8da833a 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -1,5 +1,5 @@ use crate::cli::log::tab_info; -use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset}; +use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset, interactive_season_choosing, find_multiple_seasons_with_same_number}; use crate::utils::context::Context; use crate::utils::format::{format_string, Format}; use crate::utils::log::progress; @@ -71,6 +71,10 @@ pub struct Download { #[arg(value_parser = FFmpegPreset::parse)] ffmpeg_preset: Vec, + #[arg(help = "Ignore interactive input")] + #[arg(short, long, default_value_t = false)] + yes: bool, + #[arg(help = "Url(s) to Crunchyroll episodes or series")] urls: Vec, } @@ -348,6 +352,12 @@ async fn formats_from_series( }) } + if !download.yes && !find_multiple_seasons_with_same_number(&seasons).is_empty() { + info!(target: "progress_end", "Fetched seasons"); + seasons = interactive_season_choosing(seasons); + info!(target: "progress", "Fetching series details") + } + let mut formats = vec![]; for season in seasons { if let Some(fmts) = formats_from_season(download, season, url_filter).await? { diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index 26d939c..fde1d08 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -5,9 +5,11 @@ use indicatif::{ProgressBar, ProgressFinish, ProgressStyle}; use log::{debug, LevelFilter}; use std::borrow::{Borrow, BorrowMut}; use std::collections::BTreeMap; -use std::io::Write; +use std::io::{BufRead, Write}; use std::sync::{mpsc, Arc, Mutex}; use std::time::Duration; +use crunchyroll_rs::{Media, Season}; +use regex::Regex; use tokio::task::JoinSet; pub fn find_resolution( @@ -327,3 +329,105 @@ impl FFmpegPreset { )) } } + +pub(crate) fn find_multiple_seasons_with_same_number(seasons: &Vec>) -> Vec { + let mut seasons_map: BTreeMap = BTreeMap::new(); + for season in seasons { + if let Some(s) = seasons_map.get_mut(&season.metadata.season_number) { + *s += 1; + } else { + seasons_map.insert(season.metadata.season_number, 1); + } + } + + seasons_map + .into_iter() + .filter_map(|(k, v)| if v > 1 { Some(k) } else { None }) + .collect() +} + +pub(crate) fn interactive_season_choosing(seasons: Vec>) -> Vec> { + let input_regex = + Regex::new(r"((?P\d+)|(?P\d+)-(?P\d+)?)(\s|$)").unwrap(); + + let mut seasons_map: BTreeMap>> = BTreeMap::new(); + for season in seasons { + if let Some(s) = seasons_map.get_mut(&season.metadata.season_number) { + s.push(season); + } else { + seasons_map.insert(season.metadata.season_number, vec![season]); + } + } + + for (num, season_vec) in seasons_map.iter_mut() { + if season_vec.len() == 1 { + continue; + } + println!(":: Found multiple seasons for season number {}", num); + println!(":: Select the number of the seasons you want to download (eg \"1 2 4\", \"1-3\", \"1-3 5\"):"); + for (i, season) in season_vec.iter().enumerate() { + println!(":: \t{}. {}", i + 1, season.title) + } + let mut stdout = std::io::stdout(); + let _ = write!(stdout, ":: => "); + let _ = stdout.flush(); + let mut user_input = String::new(); + std::io::stdin().lock() + .read_line(&mut user_input) + .expect("cannot open stdin"); + + let mut nums = vec![]; + for capture in input_regex.captures_iter(&user_input) { + if let Some(single) = capture.name("single") { + nums.push(single.as_str().parse().unwrap()); + } else { + let range_from = capture.name("range_from"); + let range_to = capture.name("range_to"); + + // input is '-' which means use all seasons + if range_from.is_none() && range_to.is_none() { + nums = vec![]; + break; + } + let from = range_from + .map(|f| f.as_str().parse::().unwrap() - 1) + .unwrap_or(usize::MIN); + let to = range_from + .map(|f| f.as_str().parse::().unwrap() - 1) + .unwrap_or(usize::MAX); + + nums.extend( + season_vec + .iter() + .enumerate() + .filter_map(|(i, _)| { + if i >= from && i <= to { + Some(i) + } else { + None + } + }) + .collect::>(), + ) + } + } + nums.dedup(); + + if !nums.is_empty() { + let mut remove_count = 0; + for i in 0..season_vec.len() - 1 { + if !nums.contains(&i) { + season_vec.remove(i - remove_count); + remove_count += 1 + } + } + } + } + + seasons_map + .into_values() + .into_iter() + .flatten() + .collect::>>() +} + From b991614dc38e8d36ffb2026a91429e0c686fc855 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sat, 7 Jan 2023 17:14:47 +0100 Subject: [PATCH 106/441] Fix output and download order on duplicated seasons --- crunchy-cli-core/src/utils/sort.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crunchy-cli-core/src/utils/sort.rs b/crunchy-cli-core/src/utils/sort.rs index 089fe18..21df74e 100644 --- a/crunchy-cli-core/src/utils/sort.rs +++ b/crunchy-cli-core/src/utils/sort.rs @@ -30,8 +30,11 @@ pub fn sort_formats_after_seasons(formats: Vec) -> Vec> { let mut as_map = BTreeMap::new(); for format in formats { - as_map.entry(format.season_number).or_insert_with(Vec::new); - as_map.get_mut(&format.season_number).unwrap().push(format); + // the season title is used as key instead of season number to distinguish duplicated season + // numbers which are actually two different seasons; season id is not used as this somehow + // messes up ordering when duplicated seasons exist + as_map.entry(format.season_title.clone()).or_insert_with(Vec::new); + as_map.get_mut(&format.season_title).unwrap().push(format); } let mut sorted = as_map @@ -41,7 +44,7 @@ pub fn sort_formats_after_seasons(formats: Vec) -> Vec> { values }) .collect::>>(); - sorted.sort_by(|a, b| a[0].series_id.cmp(&b[0].series_id)); + sorted.sort_by(|a, b| a[0].season_number.cmp(&b[0].season_number)); sorted } From b65c0e9dfdefb49add0294e136912729ae308ee4 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 8 Jan 2023 17:44:31 +0100 Subject: [PATCH 107/441] Update dependencies & version --- Cargo.lock | 32 ++++++++++++++++---------------- Cargo.toml | 4 ++-- crunchy-cli-core/Cargo.lock | 30 +++++++++++++++--------------- crunchy-cli-core/Cargo.toml | 4 ++-- 4 files changed, 35 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eb7c6a3..7177ac1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,9 +39,9 @@ checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" [[package]] name = "async-trait" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d1d8ab452a3936018a687b20e6f7cf5363d713b732b8884001317b0e48aa3" +checksum = "705339e0e4a9690e2908d2b3d049d85682cf19fbd5782494498fbf7003a6a282" dependencies = [ "proc-macro2", "quote", @@ -279,7 +279,7 @@ dependencies = [ [[package]] name = "crunchy-cli" -version = "3.0.0-dev.6" +version = "3.0.0-dev.7" dependencies = [ "chrono", "clap", @@ -291,7 +291,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-dev.6" +version = "3.0.0-dev.7" dependencies = [ "anyhow", "async-trait", @@ -395,9 +395,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add3fc1717409d029b20c5b6903fc0c0b02fa6741d820054f4a2efa5e5816fd" +checksum = "51d1075c37807dcf850c379432f0df05ba52cc30f279c5cfc43cc221ce7f8579" dependencies = [ "cc", "cxxbridge-flags", @@ -407,9 +407,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c87959ba14bc6fbc61df77c3fcfe180fc32b93538c4f1031dd802ccb5f2ff0" +checksum = "5044281f61b27bc598f2f6647d480aed48d2bf52d6eb0b627d84c0361b17aa70" dependencies = [ "cc", "codespan-reporting", @@ -422,15 +422,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69a3e162fde4e594ed2b07d0f83c6c67b745e7f28ce58c6df5e6b6bef99dfb59" +checksum = "61b50bc93ba22c27b0d31128d2d130a0a6b3d267ae27ef7e4fae2167dfe8781c" [[package]] name = "cxxbridge-macro" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e7e2adeb6a0d4a282e581096b06e1791532b7d576dcde5ccd9382acf55db8e6" +checksum = "39e61fda7e62115119469c7b3591fd913ecca96fb766cfd3f2e2502ab7bc87a5" dependencies = [ "proc-macro2", "quote", @@ -1669,9 +1669,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.23.0" +version = "1.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46" +checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae" dependencies = [ "autocfg", "bytes", @@ -1759,9 +1759,9 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "typenum" diff --git a/Cargo.toml b/Cargo.toml index 76ed950..ec8e4ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,11 +1,11 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-dev.6" +version = "3.0.0-dev.7" edition = "2021" [dependencies] -tokio = { version = "1.23", features = ["macros", "rt-multi-thread", "time"], default-features = false } +tokio = { version = "1.24", features = ["macros", "rt-multi-thread", "time"], default-features = false } crunchy-cli-core = { path = "./crunchy-cli-core" } diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index b5dd8a1..4b1a723 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -39,9 +39,9 @@ checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" [[package]] name = "async-trait" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d1d8ab452a3936018a687b20e6f7cf5363d713b732b8884001317b0e48aa3" +checksum = "705339e0e4a9690e2908d2b3d049d85682cf19fbd5782494498fbf7003a6a282" dependencies = [ "proc-macro2", "quote", @@ -260,7 +260,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-dev.6" +version = "3.0.0-dev.7" dependencies = [ "anyhow", "async-trait", @@ -364,9 +364,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add3fc1717409d029b20c5b6903fc0c0b02fa6741d820054f4a2efa5e5816fd" +checksum = "51d1075c37807dcf850c379432f0df05ba52cc30f279c5cfc43cc221ce7f8579" dependencies = [ "cc", "cxxbridge-flags", @@ -376,9 +376,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c87959ba14bc6fbc61df77c3fcfe180fc32b93538c4f1031dd802ccb5f2ff0" +checksum = "5044281f61b27bc598f2f6647d480aed48d2bf52d6eb0b627d84c0361b17aa70" dependencies = [ "cc", "codespan-reporting", @@ -391,15 +391,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69a3e162fde4e594ed2b07d0f83c6c67b745e7f28ce58c6df5e6b6bef99dfb59" +checksum = "61b50bc93ba22c27b0d31128d2d130a0a6b3d267ae27ef7e4fae2167dfe8781c" [[package]] name = "cxxbridge-macro" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e7e2adeb6a0d4a282e581096b06e1791532b7d576dcde5ccd9382acf55db8e6" +checksum = "39e61fda7e62115119469c7b3591fd913ecca96fb766cfd3f2e2502ab7bc87a5" dependencies = [ "proc-macro2", "quote", @@ -1632,9 +1632,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.23.0" +version = "1.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46" +checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae" dependencies = [ "autocfg", "bytes", @@ -1722,9 +1722,9 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "typenum" diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 566b8c4..a3ae301 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-dev.6" +version = "3.0.0-dev.7" edition = "2021" [dependencies] @@ -23,7 +23,7 @@ serde_json = "1.0" signal-hook = "0.3" tempfile = "3.3" terminal_size = "0.2" -tokio = { version = "1.23", features = ["macros", "rt-multi-thread", "time"] } +tokio = { version = "1.24", features = ["macros", "rt-multi-thread", "time"] } sys-locale = "0.2" [build-dependencies] From 13f54c0da636e7093d347bba5c126f8e895ebfe1 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Sun, 8 Jan 2023 18:06:37 +0100 Subject: [PATCH 108/441] Fix interactive season choosing activation on url filter excluded seasons --- crunchy-cli-core/src/cli/archive.rs | 5 ++++- crunchy-cli-core/src/cli/download.rs | 11 +++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 2c952d6..b9b7de0 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -379,7 +379,10 @@ async fn formats_from_series( .locale .iter() .any(|l| s.metadata.audio_locales.contains(l)) - }) + }); + // remove seasons which match the url filter. this is mostly done to not trigger the + // interactive season choosing when dupilcated seasons are excluded by the filter + seasons.retain(|s| url_filter.is_season_valid(s.metadata.season_number)) } if !archive.yes && !find_multiple_seasons_with_same_number(&seasons).is_empty() { diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index 8da833a..25c1262 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -349,7 +349,10 @@ async fn formats_from_series( seasons.retain(|s| { s.metadata.season_number != season.first().unwrap().metadata.season_number || s.metadata.audio_locales.contains(&download.audio) - }) + }); + // remove seasons which match the url filter. this is mostly done to not trigger the + // interactive season choosing when dupilcated seasons are excluded by the filter + seasons.retain(|s| url_filter.is_season_valid(s.metadata.season_number)) } if !download.yes && !find_multiple_seasons_with_same_number(&seasons).is_empty() { @@ -373,14 +376,14 @@ async fn formats_from_season( season: Media, url_filter: &UrlFilter, ) -> Result>> { - if !season.metadata.audio_locales.contains(&download.audio) { + if !url_filter.is_season_valid(season.metadata.season_number) { + return Ok(None); + } else if !season.metadata.audio_locales.contains(&download.audio) { error!( "Season {} ({}) is not available with {} audio", season.metadata.season_number, season.title, download.audio ); return Ok(None); - } else if !url_filter.is_season_valid(season.metadata.season_number) { - return Ok(None); } let mut formats = vec![]; From 4b33ef02c61d1ce8b60900785868a3445f779e90 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 9 Jan 2023 10:27:28 +0100 Subject: [PATCH 109/441] Fix output formatting for full path (#101) --- crunchy-cli-core/src/cli/archive.rs | 20 ++++++-------------- crunchy-cli-core/src/cli/download.rs | 20 ++++++-------------- crunchy-cli-core/src/utils/format.rs | 9 ++++++--- 3 files changed, 18 insertions(+), 31 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index b9b7de0..f9a6a04 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -1,7 +1,7 @@ use crate::cli::log::tab_info; use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset, find_multiple_seasons_with_same_number, interactive_season_choosing}; use crate::utils::context::Context; -use crate::utils::format::{format_string, Format}; +use crate::utils::format::{Format, format_path}; use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file, tempfile}; use crate::utils::parse::{parse_url, UrlFilter}; @@ -240,19 +240,11 @@ impl Execute for Archive { for (formats, mut subtitles) in archive_formats { let (primary, additionally) = formats.split_first().unwrap(); - let mut path = PathBuf::from(&self.output); - path = free_file( - path.with_file_name(format_string( - if let Some(fname) = path.file_name() { - fname.to_str().unwrap() - } else { - "{title}.mkv" - } - .to_string(), - primary, - true, - )), - ); + let path = free_file(format_path(if self.output.is_empty() { + "{title}.mkv" + } else { + &self.output + }.into(), &primary, true)); info!( "Downloading {} to '{}'", diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index 25c1262..a7ed773 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -1,7 +1,7 @@ use crate::cli::log::tab_info; use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset, interactive_season_choosing, find_multiple_seasons_with_same_number}; use crate::utils::context::Context; -use crate::utils::format::{format_string, Format}; +use crate::utils::format::{Format, format_path}; use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file}; use crate::utils::parse::{parse_url, UrlFilter}; @@ -206,19 +206,11 @@ impl Execute for Download { } for format in formats { - let mut path = PathBuf::from(&self.output); - path = free_file( - path.with_file_name(format_string( - if let Some(fname) = path.file_name() { - fname.to_str().unwrap() - } else { - "{title}.ts" - } - .to_string(), - &format, - true, - )), - ); + let path = free_file(format_path(if self.output.is_empty() { + "{title}.mkv" + } else { + &self.output + }.into(), &format, true)); info!( "Downloading {} to '{}'", diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index c463ea8..ee48e2a 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -1,3 +1,4 @@ +use std::path::PathBuf; use crunchyroll_rs::media::VariantData; use crunchyroll_rs::{Episode, Locale, Media, Movie}; use std::time::Duration; @@ -65,7 +66,7 @@ impl Format { /// Formats the given string if it has specific pattern in it. It's possible to sanitize it which /// removes characters which can cause failures if the output string is used as a file name. -pub fn format_string(s: String, format: &Format, sanitize: bool) -> String { +pub fn format_path(path: PathBuf, format: &Format, sanitize: bool) -> PathBuf { let sanitize_func = if sanitize { |s: &str| sanitize_filename::sanitize(s) } else { @@ -73,7 +74,9 @@ pub fn format_string(s: String, format: &Format, sanitize: bool) -> String { |s: &str| s.to_string() }; - s.replace("{title}", &sanitize_func(&format.title)) + let as_string = path.to_string_lossy().to_string(); + + PathBuf::from(as_string.replace("{title}", &sanitize_func(&format.title)) .replace("{series_name}", &sanitize_func(&format.series_name)) .replace("{season_name}", &sanitize_func(&format.season_title)) .replace("{audio}", &sanitize_func(&format.audio.to_string())) @@ -99,5 +102,5 @@ pub fn format_string(s: String, format: &Format, sanitize: bool) -> String { ) .replace("{series_id}", &sanitize_func(&format.series_id)) .replace("{season_id}", &sanitize_func(&format.season_id)) - .replace("{episode_id}", &sanitize_func(&format.id)) + .replace("{episode_id}", &sanitize_func(&format.id))) } From 12be16417ffdd0bda87b9c06aaf7f50d5454b178 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 9 Jan 2023 16:55:10 +0100 Subject: [PATCH 110/441] Fix interactive season choosing false-positive triggering --- Cargo.lock | 85 +++++++--------------------- crunchy-cli-core/Cargo.lock | 85 +++++++--------------------- crunchy-cli-core/Cargo.toml | 1 + crunchy-cli-core/src/cli/archive.rs | 31 +++++----- crunchy-cli-core/src/cli/download.rs | 22 ++++--- crunchy-cli-core/src/cli/utils.rs | 67 ++++++++++++++++------ crunchy-cli-core/src/utils/format.rs | 59 ++++++++++--------- crunchy-cli-core/src/utils/sort.rs | 4 +- 8 files changed, 157 insertions(+), 197 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7177ac1..77e1342 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -222,7 +222,7 @@ dependencies = [ "lazy_static", "libc", "unicode-width", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -302,6 +302,7 @@ dependencies = [ "ctrlc", "dirs", "indicatif", + "lazy_static", "log", "num_cpus", "regex", @@ -390,7 +391,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1631ca6e3c59112501a9d87fd86f21591ff77acd31331e8a73f8d80a65bbdd71" dependencies = [ "nix", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -859,14 +860,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" dependencies = [ "libc", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] name = "ipnet" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11b0d96e660696543b251e58030cf9787df56da39dab19ad60eae7353040917e" +checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" [[package]] name = "is-terminal" @@ -877,7 +878,7 @@ dependencies = [ "hermit-abi", "io-lifetimes", "rustix", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -980,7 +981,7 @@ dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1231,9 +1232,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" +checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" dependencies = [ "aho-corasick", "memchr", @@ -1338,7 +1339,7 @@ dependencies = [ "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1380,12 +1381,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" +checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" dependencies = [ - "lazy_static", - "windows-sys 0.36.1", + "windows-sys", ] [[package]] @@ -1591,7 +1591,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb20089a8ba2b69debd491f8d2d023761cbf196e999218c591fa1e7e15a21907" dependencies = [ "rustix", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1682,7 +1682,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio-macros", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1973,19 +1973,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" -dependencies = [ - "windows_aarch64_msvc 0.36.1", - "windows_i686_gnu 0.36.1", - "windows_i686_msvc 0.36.1", - "windows_x86_64_gnu 0.36.1", - "windows_x86_64_msvc 0.36.1", -] - [[package]] name = "windows-sys" version = "0.42.0" @@ -1993,12 +1980,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc 0.42.0", - "windows_i686_gnu 0.42.0", - "windows_i686_msvc 0.42.0", - "windows_x86_64_gnu 0.42.0", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", "windows_x86_64_gnullvm", - "windows_x86_64_msvc 0.42.0", + "windows_x86_64_msvc", ] [[package]] @@ -2007,48 +1994,24 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" -[[package]] -name = "windows_aarch64_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" - [[package]] name = "windows_aarch64_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" -[[package]] -name = "windows_i686_gnu" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" - [[package]] name = "windows_i686_gnu" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" -[[package]] -name = "windows_i686_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" - [[package]] name = "windows_i686_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" -[[package]] -name = "windows_x86_64_gnu" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" - [[package]] name = "windows_x86_64_gnu" version = "0.42.0" @@ -2061,12 +2024,6 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" -[[package]] -name = "windows_x86_64_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" - [[package]] name = "windows_x86_64_msvc" version = "0.42.0" diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index 4b1a723..5bdabf0 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -203,7 +203,7 @@ dependencies = [ "lazy_static", "libc", "unicode-width", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -271,6 +271,7 @@ dependencies = [ "ctrlc", "dirs", "indicatif", + "lazy_static", "log", "num_cpus", "regex", @@ -359,7 +360,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1631ca6e3c59112501a9d87fd86f21591ff77acd31331e8a73f8d80a65bbdd71" dependencies = [ "nix", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -828,14 +829,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" dependencies = [ "libc", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] name = "ipnet" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11b0d96e660696543b251e58030cf9787df56da39dab19ad60eae7353040917e" +checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" [[package]] name = "is-terminal" @@ -846,7 +847,7 @@ dependencies = [ "hermit-abi", "io-lifetimes", "rustix", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -949,7 +950,7 @@ dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1200,9 +1201,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" +checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" dependencies = [ "aho-corasick", "memchr", @@ -1301,7 +1302,7 @@ dependencies = [ "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1343,12 +1344,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" +checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" dependencies = [ - "lazy_static", - "windows-sys 0.36.1", + "windows-sys", ] [[package]] @@ -1554,7 +1554,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb20089a8ba2b69debd491f8d2d023761cbf196e999218c591fa1e7e15a21907" dependencies = [ "rustix", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1645,7 +1645,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio-macros", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1936,19 +1936,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" -dependencies = [ - "windows_aarch64_msvc 0.36.1", - "windows_i686_gnu 0.36.1", - "windows_i686_msvc 0.36.1", - "windows_x86_64_gnu 0.36.1", - "windows_x86_64_msvc 0.36.1", -] - [[package]] name = "windows-sys" version = "0.42.0" @@ -1956,12 +1943,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc 0.42.0", - "windows_i686_gnu 0.42.0", - "windows_i686_msvc 0.42.0", - "windows_x86_64_gnu 0.42.0", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", "windows_x86_64_gnullvm", - "windows_x86_64_msvc 0.42.0", + "windows_x86_64_msvc", ] [[package]] @@ -1970,48 +1957,24 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" -[[package]] -name = "windows_aarch64_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" - [[package]] name = "windows_aarch64_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" -[[package]] -name = "windows_i686_gnu" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" - [[package]] name = "windows_i686_gnu" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" -[[package]] -name = "windows_i686_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" - [[package]] name = "windows_i686_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" -[[package]] -name = "windows_x86_64_gnu" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" - [[package]] name = "windows_x86_64_gnu" version = "0.42.0" @@ -2024,12 +1987,6 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" -[[package]] -name = "windows_x86_64_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" - [[package]] name = "windows_x86_64_msvc" version = "0.42.0" diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index a3ae301..5bda1db 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -14,6 +14,7 @@ csv = "1.1" ctrlc = "3.2" dirs = "4.0" indicatif = "0.17" +lazy_static = "1.4" log = { version = "0.4", features = ["std"] } num_cpus = "1.15" regex = "1.7" diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index f9a6a04..7a96c05 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -1,7 +1,10 @@ use crate::cli::log::tab_info; -use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset, find_multiple_seasons_with_same_number, interactive_season_choosing}; +use crate::cli::utils::{ + download_segments, find_multiple_seasons_with_same_number, find_resolution, + interactive_season_choosing, FFmpegPreset, +}; use crate::utils::context::Context; -use crate::utils::format::{Format, format_path}; +use crate::utils::format::{format_path, Format}; use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file, tempfile}; use crate::utils::parse::{parse_url, UrlFilter}; @@ -240,11 +243,16 @@ impl Execute for Archive { for (formats, mut subtitles) in archive_formats { let (primary, additionally) = formats.split_first().unwrap(); - let path = free_file(format_path(if self.output.is_empty() { - "{title}.mkv" - } else { - &self.output - }.into(), &primary, true)); + let path = free_file(format_path( + if self.output.is_empty() { + "{title}.mkv" + } else { + &self.output + } + .into(), + &primary, + true, + )); info!( "Downloading {} to '{}'", @@ -387,15 +395,6 @@ async fn formats_from_series( let mut result: BTreeMap, Vec)>> = BTreeMap::new(); let mut primary_season = true; for season in seasons { - if !url_filter.is_season_valid(season.metadata.season_number) - || !archive - .locale - .iter() - .any(|l| season.metadata.audio_locales.contains(l)) - { - continue; - } - for episode in season.episodes().await? { if !url_filter.is_episode_valid( episode.metadata.episode_number, diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index a7ed773..bdca291 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -1,7 +1,10 @@ use crate::cli::log::tab_info; -use crate::cli::utils::{download_segments, find_resolution, FFmpegPreset, interactive_season_choosing, find_multiple_seasons_with_same_number}; +use crate::cli::utils::{ + download_segments, find_multiple_seasons_with_same_number, find_resolution, + interactive_season_choosing, FFmpegPreset, +}; use crate::utils::context::Context; -use crate::utils::format::{Format, format_path}; +use crate::utils::format::{format_path, Format}; use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file}; use crate::utils::parse::{parse_url, UrlFilter}; @@ -206,11 +209,16 @@ impl Execute for Download { } for format in formats { - let path = free_file(format_path(if self.output.is_empty() { - "{title}.mkv" - } else { - &self.output - }.into(), &format, true)); + let path = free_file(format_path( + if self.output.is_empty() { + "{title}.mkv" + } else { + &self.output + } + .into(), + &format, + true, + )); info!( "Downloading {} to '{}'", diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index fde1d08..bc19b07 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -1,15 +1,16 @@ use crate::utils::context::Context; use anyhow::{bail, Result}; use crunchyroll_rs::media::{Resolution, VariantData, VariantSegment}; +use crunchyroll_rs::{Media, Season}; use indicatif::{ProgressBar, ProgressFinish, ProgressStyle}; +use lazy_static::lazy_static; use log::{debug, LevelFilter}; +use regex::Regex; use std::borrow::{Borrow, BorrowMut}; use std::collections::BTreeMap; use std::io::{BufRead, Write}; use std::sync::{mpsc, Arc, Mutex}; use std::time::Duration; -use crunchyroll_rs::{Media, Season}; -use regex::Regex; use tokio::task::JoinSet; pub fn find_resolution( @@ -111,7 +112,7 @@ pub async fn download_segments( }; buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec(); - + let mut c = thread_count.lock().unwrap(); debug!( "Downloaded and decrypted segment [{}/{} {:.2}%] {}", @@ -120,14 +121,14 @@ pub async fn download_segments( ((*c + 1) as f64 / total_segments as f64) * 100f64, segment.url ); - + thread_sender.send((num as i32 + (i * cpus) as i32, buf))?; - + *c += 1; } Ok(()) }; - + let result = download().await; if result.is_err() { @@ -149,7 +150,7 @@ pub async fn download_segments( for (pos, bytes) in receiver.iter() { // if the position is lower than 0, an error occured in the sending download thread if pos < 0 { - break + break; } if let Some(p) = &progress { @@ -330,6 +331,10 @@ impl FFmpegPreset { } } +lazy_static! { + static ref DUPLICATED_SEASONS_MULTILANG_REGEX: Regex = Regex::new(r"(-castilian|-english|-english-in|-french|-german|-hindi|-italian|-portuguese|-russian|-spanish)$").unwrap(); +} + pub(crate) fn find_multiple_seasons_with_same_number(seasons: &Vec>) -> Vec { let mut seasons_map: BTreeMap = BTreeMap::new(); for season in seasons { @@ -342,7 +347,25 @@ pub(crate) fn find_multiple_seasons_with_same_number(seasons: &Vec seasons_map .into_iter() - .filter_map(|(k, v)| if v > 1 { Some(k) } else { None }) + .filter_map(|(k, v)| { + if v > 1 { + // check if the different seasons are actual the same but with different dub languages + let mut multilang_season_vec: Vec = seasons + .iter() + .map(|s| { + DUPLICATED_SEASONS_MULTILANG_REGEX + .replace(s.slug_title.trim_end_matches("-dub"), "") + .to_string() + }) + .collect(); + multilang_season_vec.dedup(); + + if multilang_season_vec.len() > 1 { + return Some(k); + } + } + None + }) .collect() } @@ -363,6 +386,22 @@ pub(crate) fn interactive_season_choosing(seasons: Vec>) -> Vec = season_vec + .iter() + .map(|s| { + DUPLICATED_SEASONS_MULTILANG_REGEX + .replace(s.slug_title.trim_end_matches("-dub"), "") + .to_string() + }) + .collect(); + multilang_season_vec.dedup(); + + if multilang_season_vec.len() == 1 { + continue; + } + println!(":: Found multiple seasons for season number {}", num); println!(":: Select the number of the seasons you want to download (eg \"1 2 4\", \"1-3\", \"1-3 5\"):"); for (i, season) in season_vec.iter().enumerate() { @@ -372,7 +411,8 @@ pub(crate) fn interactive_season_choosing(seasons: Vec>) -> Vec "); let _ = stdout.flush(); let mut user_input = String::new(); - std::io::stdin().lock() + std::io::stdin() + .lock() .read_line(&mut user_input) .expect("cannot open stdin"); @@ -400,13 +440,7 @@ pub(crate) fn interactive_season_choosing(seasons: Vec>) -> Vec= from && i <= to { - Some(i) - } else { - None - } - }) + .filter_map(|(i, _)| if i >= from && i <= to { Some(i) } else { None }) .collect::>(), ) } @@ -430,4 +464,3 @@ pub(crate) fn interactive_season_choosing(seasons: Vec>) -> Vec>>() } - diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index ee48e2a..035a61c 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -1,6 +1,6 @@ -use std::path::PathBuf; use crunchyroll_rs::media::VariantData; use crunchyroll_rs::{Episode, Locale, Media, Movie}; +use std::path::PathBuf; use std::time::Duration; #[derive(Clone)] @@ -76,31 +76,34 @@ pub fn format_path(path: PathBuf, format: &Format, sanitize: bool) -> PathBuf { let as_string = path.to_string_lossy().to_string(); - PathBuf::from(as_string.replace("{title}", &sanitize_func(&format.title)) - .replace("{series_name}", &sanitize_func(&format.series_name)) - .replace("{season_name}", &sanitize_func(&format.season_title)) - .replace("{audio}", &sanitize_func(&format.audio.to_string())) - .replace( - "{resolution}", - &sanitize_func(&format.stream.resolution.to_string()), - ) - .replace( - "{padded_season_number}", - &sanitize_func(&format!("{:0>2}", format.season_number.to_string())), - ) - .replace( - "{season_number}", - &sanitize_func(&format.season_number.to_string()), - ) - .replace( - "{padded_episode_number}", - &sanitize_func(&format!("{:0>2}", format.number.to_string())), - ) - .replace( - "{episode_number}", - &sanitize_func(&format.number.to_string()), - ) - .replace("{series_id}", &sanitize_func(&format.series_id)) - .replace("{season_id}", &sanitize_func(&format.season_id)) - .replace("{episode_id}", &sanitize_func(&format.id))) + PathBuf::from( + as_string + .replace("{title}", &sanitize_func(&format.title)) + .replace("{series_name}", &sanitize_func(&format.series_name)) + .replace("{season_name}", &sanitize_func(&format.season_title)) + .replace("{audio}", &sanitize_func(&format.audio.to_string())) + .replace( + "{resolution}", + &sanitize_func(&format.stream.resolution.to_string()), + ) + .replace( + "{padded_season_number}", + &sanitize_func(&format!("{:0>2}", format.season_number.to_string())), + ) + .replace( + "{season_number}", + &sanitize_func(&format.season_number.to_string()), + ) + .replace( + "{padded_episode_number}", + &sanitize_func(&format!("{:0>2}", format.number.to_string())), + ) + .replace( + "{episode_number}", + &sanitize_func(&format.number.to_string()), + ) + .replace("{series_id}", &sanitize_func(&format.series_id)) + .replace("{season_id}", &sanitize_func(&format.season_id)) + .replace("{episode_id}", &sanitize_func(&format.id)), + ) } diff --git a/crunchy-cli-core/src/utils/sort.rs b/crunchy-cli-core/src/utils/sort.rs index 21df74e..9f8d81c 100644 --- a/crunchy-cli-core/src/utils/sort.rs +++ b/crunchy-cli-core/src/utils/sort.rs @@ -33,7 +33,9 @@ pub fn sort_formats_after_seasons(formats: Vec) -> Vec> { // the season title is used as key instead of season number to distinguish duplicated season // numbers which are actually two different seasons; season id is not used as this somehow // messes up ordering when duplicated seasons exist - as_map.entry(format.season_title.clone()).or_insert_with(Vec::new); + as_map + .entry(format.season_title.clone()) + .or_insert_with(Vec::new); as_map.get_mut(&format.season_title).unwrap().push(format); } From 29845ba6e53ce877d85aca47db840c52835fa61b Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 9 Jan 2023 17:26:04 +0100 Subject: [PATCH 111/441] Re-order instructions --- crunchy-cli-core/src/cli/archive.rs | 8 +++--- crunchy-cli-core/src/cli/download.rs | 10 +++++-- crunchy-cli-core/src/utils/format.rs | 43 +++++++++++++++------------- crunchy-cli-core/src/utils/sort.rs | 2 +- 4 files changed, 35 insertions(+), 28 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 7a96c05..1de0997 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -208,7 +208,7 @@ impl Execute for Archive { format.stream.resolution, format.stream.fps, format.season_number, - format.number, + format.episode_number, ) } } @@ -234,7 +234,7 @@ impl Execute for Archive { format.stream.resolution, format.stream.fps, format.season_number, - format.number + format.episode_number ) } } @@ -266,7 +266,7 @@ impl Execute for Archive { tab_info!( "Episode: S{:02}E{:02}", primary.season_number, - primary.number + primary.episode_number ); tab_info!( "Audio: {} (primary), {}", @@ -318,7 +318,7 @@ impl Execute for Archive { // Remove subtitles of deleted video if only_audio { - subtitles.retain(|s| s.episode_id != additional.id); + subtitles.retain(|s| s.episode_id != additional.episode_id); } } diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index bdca291..14f9503 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -182,7 +182,7 @@ impl Execute for Download { format.stream.resolution, format.stream.fps, format.season_number, - format.number, + format.episode_number, ) } } @@ -202,7 +202,7 @@ impl Execute for Download { format.stream.resolution, format.stream.fps, format.season_number, - format.number + format.episode_number ) } } @@ -229,7 +229,11 @@ impl Execute for Download { path.file_name().unwrap().to_str().unwrap() } ); - tab_info!("Episode: S{:02}E{:02}", format.season_number, format.number); + tab_info!( + "Episode: S{:02}E{:02}", + format.season_number, + format.episode_number + ); tab_info!("Audio: {}", format.audio); tab_info!( "Subtitles: {}", diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index 035a61c..5570960 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -5,10 +5,9 @@ use std::time::Duration; #[derive(Clone)] pub struct Format { - pub id: String, pub title: String, pub description: String, - pub number: u32, + pub audio: Locale, pub duration: Duration, @@ -20,15 +19,17 @@ pub struct Format { pub season_id: String, pub season_title: String, pub season_number: u32, + + pub episode_id: String, + pub episode_number: f32, } impl Format { pub fn new_from_episode(episode: Media, stream: VariantData) -> Self { Self { - id: episode.id, title: episode.title, description: episode.description, - number: episode.metadata.episode_number, + audio: episode.metadata.audio_locale, duration: episode.metadata.duration.to_std().unwrap(), @@ -40,15 +41,17 @@ impl Format { season_id: episode.metadata.season_id, season_title: episode.metadata.season_title, season_number: episode.metadata.season_number, + + episode_id: episode.id, + episode_number: episode.metadata.episode.parse().unwrap_or(episode.metadata.sequence_number), } } pub fn new_from_movie(movie: Media, stream: VariantData) -> Self { Self { - id: movie.id, title: movie.title, description: movie.description, - number: 1, + audio: Locale::ja_JP, duration: movie.metadata.duration.to_std().unwrap(), @@ -60,6 +63,9 @@ impl Format { season_id: movie.metadata.movie_listing_id, season_title: movie.metadata.movie_listing_title, season_number: 1, + + episode_id: movie.id, + episode_number: 1.0, } } } @@ -79,31 +85,28 @@ pub fn format_path(path: PathBuf, format: &Format, sanitize: bool) -> PathBuf { PathBuf::from( as_string .replace("{title}", &sanitize_func(&format.title)) - .replace("{series_name}", &sanitize_func(&format.series_name)) - .replace("{season_name}", &sanitize_func(&format.season_title)) .replace("{audio}", &sanitize_func(&format.audio.to_string())) .replace( "{resolution}", &sanitize_func(&format.stream.resolution.to_string()), ) - .replace( - "{padded_season_number}", - &sanitize_func(&format!("{:0>2}", format.season_number.to_string())), - ) + .replace("{series_id}", &sanitize_func(&format.series_id)) + .replace("{series_name}", &sanitize_func(&format.series_name)) + .replace("{season_id}", &sanitize_func(&format.season_id)) + .replace("{season_name}", &sanitize_func(&format.season_title)) .replace( "{season_number}", &sanitize_func(&format.season_number.to_string()), ) .replace( - "{padded_episode_number}", - &sanitize_func(&format!("{:0>2}", format.number.to_string())), + "{padded_season_number}", + &sanitize_func(&format!("{:0>2}", format.season_number.to_string())), ) + .replace("{episode_id}", &sanitize_func(&format.episode_id)) + .replace("{episode_number}", &sanitize_func(&format.episode_number.to_string())) .replace( - "{episode_number}", - &sanitize_func(&format.number.to_string()), - ) - .replace("{series_id}", &sanitize_func(&format.series_id)) - .replace("{season_id}", &sanitize_func(&format.season_id)) - .replace("{episode_id}", &sanitize_func(&format.id)), + "{padded_episode_number}", + &sanitize_func(&format!("{:0>2}", format.episode_number.to_string())), + ), ) } diff --git a/crunchy-cli-core/src/utils/sort.rs b/crunchy-cli-core/src/utils/sort.rs index 9f8d81c..1af0194 100644 --- a/crunchy-cli-core/src/utils/sort.rs +++ b/crunchy-cli-core/src/utils/sort.rs @@ -42,7 +42,7 @@ pub fn sort_formats_after_seasons(formats: Vec) -> Vec> { let mut sorted = as_map .into_iter() .map(|(_, mut values)| { - values.sort_by(|a, b| a.number.cmp(&b.number)); + values.sort_by(|a, b| a.episode_number.total_cmp(&b.episode_number)); values }) .collect::>>(); From 7d3a90e8112a4ce27158b217a564ded45c08c576 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 9 Jan 2023 19:12:00 +0100 Subject: [PATCH 112/441] Add relative episode number to format --- crunchy-cli-core/src/cli/archive.rs | 29 +++--- crunchy-cli-core/src/cli/download.rs | 54 +++++++--- crunchy-cli-core/src/utils/format.rs | 150 ++++++++++++++++----------- 3 files changed, 145 insertions(+), 88 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 1de0997..96d0737 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -4,7 +4,7 @@ use crate::cli::utils::{ interactive_season_choosing, FFmpegPreset, }; use crate::utils::context::Context; -use crate::utils::format::{format_path, Format}; +use crate::utils::format::Format; use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file, tempfile}; use crate::utils::parse::{parse_url, UrlFilter}; @@ -243,16 +243,17 @@ impl Execute for Archive { for (formats, mut subtitles) in archive_formats { let (primary, additionally) = formats.split_first().unwrap(); - let path = free_file(format_path( - if self.output.is_empty() { - "{title}.mkv" - } else { - &self.output - } - .into(), - &primary, - true, - )); + let path = free_file( + primary.format_path( + if self.output.is_empty() { + "{title}.mkv" + } else { + &self.output + } + .into(), + true, + ), + ); info!( "Downloading {} to '{}'", @@ -395,7 +396,9 @@ async fn formats_from_series( let mut result: BTreeMap, Vec)>> = BTreeMap::new(); let mut primary_season = true; for season in seasons { - for episode in season.episodes().await? { + let episodes = season.episodes().await?; + + for episode in episodes.iter() { if !url_filter.is_episode_valid( episode.metadata.episode_number, episode.metadata.season_number, @@ -434,7 +437,7 @@ async fn formats_from_series( }; Some(subtitle) })); - formats.push(Format::new_from_episode(episode, stream)); + formats.push(Format::new_from_episode(episode, &episodes, stream)); } primary_season = false; diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index 14f9503..d027df7 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -4,7 +4,7 @@ use crate::cli::utils::{ interactive_season_choosing, FFmpegPreset, }; use crate::utils::context::Context; -use crate::utils::format::{format_path, Format}; +use crate::utils::format::Format; use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file}; use crate::utils::parse::{parse_url, UrlFilter}; @@ -16,6 +16,7 @@ use crunchyroll_rs::{ Episode, Locale, Media, MediaCollection, Movie, MovieListing, Season, Series, }; use log::{debug, error, info, warn}; +use std::borrow::Cow; use std::fs::File; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; @@ -148,7 +149,7 @@ impl Execute for Download { episode.metadata.season_title, episode.metadata.series_title ); - format_from_episode(&self, episode, &url_filter, false) + format_from_episode(&self, &episode, &url_filter, None, false) .await? .map(|fmt| vec![fmt]) } @@ -209,16 +210,17 @@ impl Execute for Download { } for format in formats { - let path = free_file(format_path( - if self.output.is_empty() { - "{title}.mkv" - } else { - &self.output - } - .into(), - &format, - true, - )); + let path = free_file( + format.format_path( + if self.output.is_empty() { + "{title}.mkv" + } else { + &self.output + } + .into(), + true, + ), + ); info!( "Downloading {} to '{}'", @@ -392,8 +394,11 @@ async fn formats_from_season( let mut formats = vec![]; - for episode in season.episodes().await? { - if let Some(fmt) = format_from_episode(download, episode, url_filter, true).await? { + let episodes = season.episodes().await?; + for episode in episodes.iter() { + if let Some(fmt) = + format_from_episode(download, &episode, url_filter, Some(&episodes), true).await? + { formats.push(fmt) } } @@ -403,8 +408,9 @@ async fn formats_from_season( async fn format_from_episode( download: &Download, - episode: Media, + episode: &Media, url_filter: &UrlFilter, + season_episodes: Option<&Vec>>, filter_audio: bool, ) -> Result> { if filter_audio && episode.metadata.audio_locale != download.audio { @@ -457,7 +463,21 @@ async fn format_from_episode( ) }; - Ok(Some(Format::new_from_episode(episode, stream))) + let season_eps = if Format::has_relative_episodes_fmt(&download.output) { + if let Some(eps) = season_episodes { + Cow::from(eps) + } else { + Cow::from(episode.season().await?.episodes().await?) + } + } else { + Cow::from(vec![]) + }; + + Ok(Some(Format::new_from_episode( + episode, + &season_eps.to_vec(), + stream, + ))) } async fn format_from_movie_listing( @@ -515,7 +535,7 @@ async fn format_from_movie( } }; - Ok(Some(Format::new_from_movie(movie, stream))) + Ok(Some(Format::new_from_movie(&movie, stream))) } fn some_vec_or_none(v: Vec) -> Option> { diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index 5570960..3db28c3 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -1,5 +1,6 @@ use crunchyroll_rs::media::VariantData; use crunchyroll_rs::{Episode, Locale, Media, Movie}; +use log::warn; use std::path::PathBuf; use std::time::Duration; @@ -22,35 +23,56 @@ pub struct Format { pub episode_id: String, pub episode_number: f32, + pub relative_episode_number: f32, } impl Format { - pub fn new_from_episode(episode: Media, stream: VariantData) -> Self { + pub fn new_from_episode( + episode: &Media, + season_episodes: &Vec>, + stream: VariantData, + ) -> Self { Self { - title: episode.title, - description: episode.description, + title: episode.title.clone(), + description: episode.description.clone(), - audio: episode.metadata.audio_locale, + audio: episode.metadata.audio_locale.clone(), duration: episode.metadata.duration.to_std().unwrap(), stream, - series_id: episode.metadata.series_id, - series_name: episode.metadata.series_title, + series_id: episode.metadata.series_id.clone(), + series_name: episode.metadata.series_title.clone(), - season_id: episode.metadata.season_id, - season_title: episode.metadata.season_title, - season_number: episode.metadata.season_number, + season_id: episode.metadata.season_id.clone(), + season_title: episode.metadata.season_title.clone(), + season_number: episode.metadata.season_number.clone(), - episode_id: episode.id, - episode_number: episode.metadata.episode.parse().unwrap_or(episode.metadata.sequence_number), + episode_id: episode.id.clone(), + episode_number: episode + .metadata + .episode + .parse() + .unwrap_or(episode.metadata.sequence_number), + relative_episode_number: season_episodes + .iter() + .enumerate() + .find_map(|(i, e)| if e == episode { Some((i + 1) as f32) } else { None }) + .unwrap_or_else(|| { + warn!("Cannot find relative episode number for episode {} ({}) of season {} ({}) of {}, using normal episode number", episode.metadata.episode_number, episode.title, episode.metadata.season_number, episode.metadata.season_title, episode.metadata.series_title); + episode + .metadata + .episode + .parse() + .unwrap_or(episode.metadata.sequence_number) + }), } } - pub fn new_from_movie(movie: Media, stream: VariantData) -> Self { + pub fn new_from_movie(movie: &Media, stream: VariantData) -> Self { Self { - title: movie.title, - description: movie.description, + title: movie.title.clone(), + description: movie.description.clone(), audio: Locale::ja_JP, @@ -60,53 +82,65 @@ impl Format { series_id: movie.metadata.movie_listing_id.clone(), series_name: movie.metadata.movie_listing_title.clone(), - season_id: movie.metadata.movie_listing_id, - season_title: movie.metadata.movie_listing_title, + season_id: movie.metadata.movie_listing_id.clone(), + season_title: movie.metadata.movie_listing_title.clone(), season_number: 1, - episode_id: movie.id, + episode_id: movie.id.clone(), episode_number: 1.0, + relative_episode_number: 1.0, } } -} - -/// Formats the given string if it has specific pattern in it. It's possible to sanitize it which -/// removes characters which can cause failures if the output string is used as a file name. -pub fn format_path(path: PathBuf, format: &Format, sanitize: bool) -> PathBuf { - let sanitize_func = if sanitize { - |s: &str| sanitize_filename::sanitize(s) - } else { - // converting this to a string is actually unnecessary - |s: &str| s.to_string() - }; - - let as_string = path.to_string_lossy().to_string(); - - PathBuf::from( - as_string - .replace("{title}", &sanitize_func(&format.title)) - .replace("{audio}", &sanitize_func(&format.audio.to_string())) - .replace( - "{resolution}", - &sanitize_func(&format.stream.resolution.to_string()), - ) - .replace("{series_id}", &sanitize_func(&format.series_id)) - .replace("{series_name}", &sanitize_func(&format.series_name)) - .replace("{season_id}", &sanitize_func(&format.season_id)) - .replace("{season_name}", &sanitize_func(&format.season_title)) - .replace( - "{season_number}", - &sanitize_func(&format.season_number.to_string()), - ) - .replace( - "{padded_season_number}", - &sanitize_func(&format!("{:0>2}", format.season_number.to_string())), - ) - .replace("{episode_id}", &sanitize_func(&format.episode_id)) - .replace("{episode_number}", &sanitize_func(&format.episode_number.to_string())) - .replace( - "{padded_episode_number}", - &sanitize_func(&format!("{:0>2}", format.episode_number.to_string())), - ), - ) + + /// Formats the given string if it has specific pattern in it. It's possible to sanitize it which + /// removes characters which can cause failures if the output string is used as a file name. + pub fn format_path(&self, path: PathBuf, sanitize: bool) -> PathBuf { + let sanitize_func = if sanitize { + |s: &str| sanitize_filename::sanitize(s) + } else { + // converting this to a string is actually unnecessary + |s: &str| s.to_string() + }; + + let as_string = path.to_string_lossy().to_string(); + + PathBuf::from( + as_string + .replace("{title}", &sanitize_func(&self.title)) + .replace("{audio}", &sanitize_func(&self.audio.to_string())) + .replace( + "{resolution}", + &sanitize_func(&self.stream.resolution.to_string()), + ) + .replace("{series_id}", &sanitize_func(&self.series_id)) + .replace("{series_name}", &sanitize_func(&self.series_name)) + .replace("{season_id}", &sanitize_func(&self.season_id)) + .replace("{season_name}", &sanitize_func(&self.season_title)) + .replace( + "{season_number}", + &sanitize_func(&self.season_number.to_string()), + ) + .replace( + "{padded_season_number}", + &sanitize_func(&format!("{:0>2}", self.season_number.to_string())), + ) + .replace("{episode_id}", &sanitize_func(&self.episode_id)) + .replace( + "{episode_number}", + &sanitize_func(&self.episode_number.to_string()), + ) + .replace( + "{padded_episode_number}", + &sanitize_func(&format!("{:0>2}", self.episode_number.to_string())), + ) + .replace( + "{relative_episode_number}", + &sanitize_func(&format!("{:0>2}", self.relative_episode_number.to_string())), + ), + ) + } + + pub fn has_relative_episodes_fmt>(s: S) -> bool { + return s.as_ref().contains("{relative_episode_number}"); + } } From 2ea036d4c6b7faa5bdc33da1eb9a630126ef1e05 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 9 Jan 2023 19:12:31 +0100 Subject: [PATCH 113/441] Remove padded_*_number and make it default for *_number for output format --- crunchy-cli-core/src/utils/format.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index 3db28c3..60596ad 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -118,19 +118,11 @@ impl Format { .replace("{season_name}", &sanitize_func(&self.season_title)) .replace( "{season_number}", - &sanitize_func(&self.season_number.to_string()), - ) - .replace( - "{padded_season_number}", &sanitize_func(&format!("{:0>2}", self.season_number.to_string())), ) .replace("{episode_id}", &sanitize_func(&self.episode_id)) .replace( "{episode_number}", - &sanitize_func(&self.episode_number.to_string()), - ) - .replace( - "{padded_episode_number}", &sanitize_func(&format!("{:0>2}", self.episode_number.to_string())), ) .replace( From a0aab3bfb964195128abb25ed5c0cfa0f4841707 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 9 Jan 2023 23:25:16 +0100 Subject: [PATCH 114/441] Add arabic locale in duplicated seasons check --- crunchy-cli-core/src/cli/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs index bc19b07..da56f5e 100644 --- a/crunchy-cli-core/src/cli/utils.rs +++ b/crunchy-cli-core/src/cli/utils.rs @@ -332,7 +332,7 @@ impl FFmpegPreset { } lazy_static! { - static ref DUPLICATED_SEASONS_MULTILANG_REGEX: Regex = Regex::new(r"(-castilian|-english|-english-in|-french|-german|-hindi|-italian|-portuguese|-russian|-spanish)$").unwrap(); + static ref DUPLICATED_SEASONS_MULTILANG_REGEX: Regex = Regex::new(r"(-arabic|-castilian|-english|-english-in|-french|-german|-hindi|-italian|-portuguese|-russian|-spanish)$").unwrap(); } pub(crate) fn find_multiple_seasons_with_same_number(seasons: &Vec>) -> Vec { From 3029325776de09ef159081df847ae67cea3be93d Mon Sep 17 00:00:00 2001 From: ByteDream Date: Mon, 9 Jan 2023 23:40:53 +0100 Subject: [PATCH 115/441] Add check if request locale is valid (#102) --- crunchy-cli-core/src/lib.rs | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index 52a4da1..6cef42e 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -6,7 +6,7 @@ use anyhow::bail; use anyhow::Result; use clap::{Parser, Subcommand}; use crunchyroll_rs::{Crunchyroll, Locale}; -use log::{debug, error, LevelFilter}; +use log::{debug, error, warn, LevelFilter}; use std::{env, fs}; mod cli; @@ -189,8 +189,41 @@ async fn create_ctx(cli: &Cli) -> Result { } async fn crunchyroll_session(cli: &Cli) -> Result { + let supported_langs = vec![ + Locale::ar_ME, + Locale::de_DE, + Locale::en_US, + Locale::es_ES, + Locale::es_419, + Locale::fr_FR, + Locale::it_IT, + Locale::pt_BR, + Locale::pt_PT, + Locale::ru_RU, + ]; + let locale = if let Some(lang) = &cli.lang { + if !supported_langs.contains(lang) { + bail!( + "Via `--lang` specified language is not supported. Supported languages: {}", + supported_langs + .iter() + .map(|l| format!("`{}` ({})", l.to_string(), l.to_human_readable())) + .collect::>() + .join(", ") + ) + } + lang.clone() + } else { + let mut lang = system_locale(); + if !supported_langs.contains(&lang) { + warn!("Recognized system locale is not supported. Using en-US as default. Use `--lang` to overwrite the used language"); + lang = Locale::en_US + } + lang + }; + let builder = Crunchyroll::builder() - .locale(cli.lang.clone().unwrap_or_else(system_locale)) + .locale(locale) .stabilization_locales(true); let login_methods_count = cli.login_method.credentials.is_some() as u8 From 5ce5b249c91fbfa752e4779bb0a9776c740ec88b Mon Sep 17 00:00:00 2001 From: ByteDream Date: Tue, 10 Jan 2023 19:20:08 +0100 Subject: [PATCH 116/441] Add relative episode number to cli help --- crunchy-cli-core/src/cli/archive.rs | 3 +-- crunchy-cli-core/src/cli/download.rs | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 96d0737..768c629 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -67,10 +67,9 @@ pub struct Archive { {season_name} → Name of the season\n \ {audio} → Audio language of the video\n \ {resolution} → Resolution of the video\n \ - {padded_season_number} → Number of the season padded to double digits\n \ {season_number} → Number of the season\n \ - {padded_episode_number} → Number of the episode padded to double digits\n \ {episode_number} → Number of the episode\n \ + {relative_episode_number} → Number of the episode relative to its season\ {series_id} → ID of the series\n \ {season_id} → ID of the season\n \ {episode_id} → ID of the episode")] diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs index d027df7..7a49a4c 100644 --- a/crunchy-cli-core/src/cli/download.rs +++ b/crunchy-cli-core/src/cli/download.rs @@ -45,10 +45,9 @@ pub struct Download { {season_name} → Name of the season\n \ {audio} → Audio language of the video\n \ {resolution} → Resolution of the video\n \ - {padded_season_number} → Number of the season padded to double digits\n \ {season_number} → Number of the season\n \ - {padded_episode_number} → Number of the episode padded to double digits\n \ {episode_number} → Number of the episode\n \ + {relative_episode_number} → Number of the episode relative to its season\ {series_id} → ID of the series\n \ {season_id} → ID of the season\n \ {episode_id} → ID of the episode")] From 17233f2fd2721dc4f9e1b86957cb404d09f73249 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Tue, 10 Jan 2023 22:15:36 +0100 Subject: [PATCH 117/441] Update dependencies and version --- Cargo.lock | 26 ++++++++++++++++---------- Cargo.toml | 2 +- crunchy-cli-core/Cargo.lock | 24 +++++++++++++++--------- crunchy-cli-core/Cargo.toml | 2 +- 4 files changed, 33 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 77e1342..42fe778 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -60,6 +60,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" + [[package]] name = "bitflags" version = "1.3.2" @@ -279,7 +285,7 @@ dependencies = [ [[package]] name = "crunchy-cli" -version = "3.0.0-dev.7" +version = "3.0.0-dev.8" dependencies = [ "chrono", "clap", @@ -291,7 +297,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-dev.7" +version = "3.0.0-dev.8" dependencies = [ "anyhow", "async-trait", @@ -318,9 +324,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "184d0c725a09aec815316cbf41a2f362008ecb0e8c8e3b6b9930d01a89b5df21" +checksum = "f3770cda4c67e68c689c8e361af46bb9d017caf82263905358fd0751d10657a0" dependencies = [ "aes", "async-trait", @@ -343,9 +349,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f3c82e1766339727fc2c10d66d0c4f001b1cf42e2993f9d93997b610f408776" +checksum = "4a260a73e733bb0ce30343caaed5e968d3c1cc2ea0ab27c601481e9ef22a2fd7" dependencies = [ "darling", "quote", @@ -1268,7 +1274,7 @@ version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" dependencies = [ - "base64", + "base64 0.13.1", "bytes", "cookie", "cookie_store", @@ -1356,11 +1362,11 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0864aeff53f8c05aa08d86e5ef839d3dfcf07aeba2db32f12db0ef716e87bd55" +checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" dependencies = [ - "base64", + "base64 0.21.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index ec8e4ef..db1f129 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-dev.7" +version = "3.0.0-dev.8" edition = "2021" [dependencies] diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index 5bdabf0..b12b3a3 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -60,6 +60,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" + [[package]] name = "bitflags" version = "1.3.2" @@ -260,7 +266,7 @@ dependencies = [ [[package]] name = "crunchy-cli-core" -version = "3.0.0-dev.7" +version = "3.0.0-dev.8" dependencies = [ "anyhow", "async-trait", @@ -287,9 +293,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "184d0c725a09aec815316cbf41a2f362008ecb0e8c8e3b6b9930d01a89b5df21" +checksum = "f3770cda4c67e68c689c8e361af46bb9d017caf82263905358fd0751d10657a0" dependencies = [ "aes", "async-trait", @@ -312,9 +318,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f3c82e1766339727fc2c10d66d0c4f001b1cf42e2993f9d93997b610f408776" +checksum = "4a260a73e733bb0ce30343caaed5e968d3c1cc2ea0ab27c601481e9ef22a2fd7" dependencies = [ "darling", "quote", @@ -1237,7 +1243,7 @@ version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" dependencies = [ - "base64", + "base64 0.13.1", "bytes", "cookie", "cookie_store", @@ -1319,11 +1325,11 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0864aeff53f8c05aa08d86e5ef839d3dfcf07aeba2db32f12db0ef716e87bd55" +checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" dependencies = [ - "base64", + "base64 0.21.0", ] [[package]] diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 5bda1db..8eea6a8 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "crunchy-cli-core" authors = ["Crunchy Labs Maintainers"] -version = "3.0.0-dev.7" +version = "3.0.0-dev.8" edition = "2021" [dependencies] From 6d1f8d49f67aabbc9b1cf577fce38c3209993371 Mon Sep 17 00:00:00 2001 From: ByteDream Date: Fri, 13 Jan 2023 15:21:23 +0100 Subject: [PATCH 118/441] Add hardsubs manually to download videos (#81) --- crunchy-cli-core/src/cli/archive.rs | 151 +------------------------ crunchy-cli-core/src/cli/download.rs | 134 +++++++++++++--------- crunchy-cli-core/src/cli/utils.rs | 20 ++-- crunchy-cli-core/src/utils/format.rs | 6 +- crunchy-cli-core/src/utils/mod.rs | 1 + crunchy-cli-core/src/utils/subtitle.rs | 108 ++++++++++++++++++ crunchy-cli-core/src/utils/video.rs | 25 ++++ 7 files changed, 233 insertions(+), 212 deletions(-) create mode 100644 crunchy-cli-core/src/utils/video.rs diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs index 768c629..c7030b4 100644 --- a/crunchy-cli-core/src/cli/archive.rs +++ b/crunchy-cli-core/src/cli/archive.rs @@ -9,16 +9,14 @@ use crate::utils::log::progress; use crate::utils::os::{free_file, has_ffmpeg, is_special_file, tempfile}; use crate::utils::parse::{parse_url, UrlFilter}; use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number}; -use crate::utils::subtitle::Subtitle; +use crate::utils::subtitle::{download_subtitle, Subtitle}; +use crate::utils::video::get_video_length; use crate::Execute; use anyhow::{bail, Result}; -use chrono::NaiveTime; -use crunchyroll_rs::media::{Resolution, StreamSubtitle}; +use crunchyroll_rs::media::Resolution; use crunchyroll_rs::{Locale, Media, MediaCollection, Series}; use log::{debug, error, info, warn}; -use regex::Regex; use std::collections::BTreeMap; -use std::io::Write; use std::path::PathBuf; use std::process::{Command, Stdio}; use tempfile::TempPath; @@ -113,14 +111,6 @@ pub struct Archive { )] #[arg(long)] default_subtitle: Option, - #[arg(help = "Disable subtitle optimizations")] - #[arg( - long_help = "By default, Crunchyroll delivers subtitles in a format which may cause issues in some video players. \ - These issues are fixed internally by setting a flag which is not part of the official specification of the subtitle format. \ - If you do not want this fixes or they cause more trouble than they solve (for you), it can be disabled with this flag" - )] - #[arg(long)] - no_subtitle_optimizations: bool, #[arg(help = "Ignore interactive input")] #[arg(short, long, default_value_t = false)] @@ -326,12 +316,8 @@ impl Execute for Archive { let primary_video_length = get_video_length(primary_video.to_path_buf()).unwrap(); for subtitle in subtitles { subtitle_paths.push(( - download_subtitle( - &self, - subtitle.stream_subtitle.clone(), - primary_video_length, - ) - .await?, + download_subtitle(subtitle.stream_subtitle.clone(), primary_video_length) + .await?, subtitle, )) } @@ -436,7 +422,7 @@ async fn formats_from_series( }; Some(subtitle) })); - formats.push(Format::new_from_episode(episode, &episodes, stream)); + formats.push(Format::new_from_episode(episode, &episodes, stream, vec![])); } primary_season = false; @@ -476,111 +462,6 @@ async fn download_video(ctx: &Context, format: &Format, only_audio: bool) -> Res Ok(path) } -async fn download_subtitle( - archive: &Archive, - subtitle: StreamSubtitle, - max_length: NaiveTime, -) -> Result { - let tempfile = tempfile(".ass")?; - let (mut file, path) = tempfile.into_parts(); - - let mut buf = vec![]; - subtitle.write_to(&mut buf).await?; - if !archive.no_subtitle_optimizations { - buf = fix_subtitle_look_and_feel(buf) - } - buf = fix_subtitle_length(buf, max_length); - - file.write_all(buf.as_slice())?; - - Ok(path) -} - -/// Add `ScaledBorderAndShadows: yes` to subtitles; without it they look very messy on some video -/// players. See [crunchy-labs/crunchy-cli#66](https://github.com/crunchy-labs/crunchy-cli/issues/66) -/// for more information. -fn fix_subtitle_look_and_feel(raw: Vec) -> Vec { - let mut script_info = false; - let mut new = String::new(); - - for line in String::from_utf8_lossy(raw.as_slice()).split('\n') { - if line.trim().starts_with('[') && script_info { - new.push_str("ScaledBorderAndShadow: yes\n"); - script_info = false - } else if line.trim() == "[Script Info]" { - script_info = true - } - new.push_str(line); - new.push('\n') - } - - new.into_bytes() -} - -/// Fix the length of subtitles to a specified maximum amount. This is required because sometimes -/// subtitles have an unnecessary entry long after the actual video ends with artificially extends -/// the video length on some video players. To prevent this, the video length must be hard set. See -/// [crunchy-labs/crunchy-cli#32](https://github.com/crunchy-labs/crunchy-cli/issues/32) for more -/// information. -fn fix_subtitle_length(raw: Vec, max_length: NaiveTime) -> Vec { - let re = - Regex::new(r#"^Dialogue:\s\d+,(?P\d+:\d+:\d+\.\d+),(?P\d+:\d+:\d+\.\d+),"#) - .unwrap(); - - // chrono panics if we try to format NaiveTime with `%2f` and the nano seconds has more than 2 - // digits so them have to be reduced manually to avoid the panic - fn format_naive_time(native_time: NaiveTime) -> String { - let formatted_time = native_time.format("%f").to_string(); - format!( - "{}.{}", - native_time.format("%T"), - if formatted_time.len() <= 2 { - native_time.format("%2f").to_string() - } else { - formatted_time.split_at(2).0.parse().unwrap() - } - ) - } - - let length_as_string = format_naive_time(max_length); - let mut new = String::new(); - - for line in String::from_utf8_lossy(raw.as_slice()).split('\n') { - if let Some(capture) = re.captures(line) { - let start = capture.name("start").map_or(NaiveTime::default(), |s| { - NaiveTime::parse_from_str(s.as_str(), "%H:%M:%S.%f").unwrap() - }); - let end = capture.name("end").map_or(NaiveTime::default(), |s| { - NaiveTime::parse_from_str(s.as_str(), "%H:%M:%S.%f").unwrap() - }); - - if start > max_length { - continue; - } else if end > max_length { - new.push_str( - re.replace( - line, - format!( - "Dialogue: {},{},", - format_naive_time(start), - &length_as_string - ), - ) - .to_string() - .as_str(), - ) - } else { - new.push_str(line) - } - } else { - new.push_str(line) - } - new.push('\n') - } - - new.into_bytes() -} - fn generate_mkv( archive: &Archive, target: PathBuf, @@ -721,23 +602,3 @@ fn generate_mkv( Ok(()) } - -/// Get the length of a video. This is required because sometimes subtitles have an unnecessary entry -/// long after the actual video ends with artificially extends the video length on some video players. -/// To prevent this, the video length must be hard set. See -/// [crunchy-labs/crunchy-cli#32](https://github.com/crunchy-labs/crunchy-cli/issues/32) for more -/// information. -fn get_video_length(path: PathBuf) -> Result { - let video_length = Regex::new(r"Duration:\s(?P

@@ -26,13 +26,13 @@ A [Rust](https://www.rust-lang.org/) written cli client for [Crunchyroll](https:

Usage đŸ–Ĩī¸ â€ĸ - Disclaimer ☝ī¸ + Disclaimer 📜 â€ĸ License ⚖

> We are in no way affiliated with, maintained, authorized, sponsored, or officially associated with Crunchyroll LLC or any of its subsidiaries or affiliates. -> The official Crunchyroll website can be found at https://crunchyroll.com/. +> The official Crunchyroll website can be found at [crunchyroll.com](https://crunchyroll.com/). > This README belongs to the _master_ branch which is currently under heavy development towards the next major version (3.0). > It is mostly stable but some issues may still occur. @@ -41,14 +41,14 @@ A [Rust](https://www.rust-lang.org/) written cli client for [Crunchyroll](https: ## ✨ Features - Download single videos and entire series from [Crunchyroll](https://www.crunchyroll.com). -- Archive episode or seasons in an `.mkv` file with multiple subtitles and audios. -- Specify a range which episodes to download from an anime. +- Archive episodes or seasons in an `.mkv` file with multiple subtitles and audios. +- Specify a range of episodes to download from an anime. ## 💾 Get the executable ### đŸ“Ĩ Download the latest binaries -Checkout the [releases](https://github.com/crunchy-labs/crunchy-cli/releases) tab and get the binary from the newest (pre-)release. +Check out the [releases](https://github.com/crunchy-labs/crunchy-cli/releases) tab and get the binary from the latest (pre-)release. ### 🛠 Build it yourself @@ -58,47 +58,45 @@ This requires [git](https://git-scm.com/) and [Cargo](https://doc.rust-lang.org/ $ git clone https://github.com/crunchy-labs/crunchy-cli $ cd crunchy-cli $ cargo build --release +$ cargo install --force --path . ``` -After the binary has built successfully it is available in `target/release`. ## đŸ–Ĩī¸ Usage -> All shown command are just examples +> All shown commands are examples 🧑đŸŧ‍đŸŗ -Every command requires you to be logged in with an account. -It doesn't matter if this account is premium or not, both works (but as free user you do not have access to premium content). -You can pass your account via credentials (username & password) or refresh token. +crunchy-cli requires you to log in. +Though you can use a non-premium account, you will not have access to premium content without a subscription. +You can authenticate with your credentials (username:password) or by using a refresh token. -- Refresh Token - - To get the token you have to log in at [crunchyroll.com](https://www.crunchyroll.com/) and extract the `etp_rt` cookie. - The easiest way to get it is via a browser extension with lets you view your cookies, like [Cookie-Editor](https://cookie-editor.cgagnier.ca/) ([Firefox Store](https://addons.mozilla.org/en-US/firefox/addon/cookie-editor/); [Chrome Store](https://chrome.google.com/webstore/detail/cookie-editor/hlkenndednhfkekhgcdicdfddnkalmdm)). - If installed, search the `etp_rt` entry and extract the value. - - ```shell - $ crunchy --etp-rt "abcd1234-zyxw-9876-98zy-a1b2c3d4e5f6" - ``` - Credentials - - Credentials must be provided as one single expression. - Username and password must be separated by a `:`. - ```shell $ crunchy --credentials "user:password" ``` -- Anonymous - - Login without an account at all is also possible. +- Refresh Token + - To obtain a refresh token, you have to log in at [crunchyroll.com](https://www.crunchyroll.com/) and extract the `etp_rt` cookie. + The easiest way to get it is via a browser extension which lets you export your cookies, like [Cookie-Editor](https://cookie-editor.cgagnier.ca/) ([Firefox](https://addons.mozilla.org/en-US/firefox/addon/cookie-editor/) / [Chrome](https://chrome.google.com/webstore/detail/cookie-editor/hlkenndednhfkekhgcdicdfddnkalmdm)). + When installed, look for the `etp_rt` entry and extract its value. + - ```shell + $ crunchy --etp-rt "4ebf1690-53a4-491a-a2ac-488309120f5d" + ``` +- Stay Anonymous + - Skip the login check: - ```shell $ crunchy --anonymous ``` ### Login -If you do not want to provide your credentials every time you execute a command, they can be stored permanently on disk. -This can be done with the `login` subcommand. +crunchy-cli can store your session, so you don't have to authenticate every time you execute a command. + +Note that the `login` keyword has to be used *last*. ```shell -$ crunchy --etp-rt "abcd1234-zyxw-9876-98zy-a1b2c3d4e5f6" login +$ crunchy --etp-rt "4ebf1690-53a4-491a-a2ac-488309120f5d" login ``` -Once set, you do not need to provide `--etp-rt` / `--credentials` anymore when using the cli. -This does not work if you've using this with `--anonymous`. +With the session stored, you do not need to use `--credentials` / `--etp-rt` anymore. This does not work with `--anonymous`. ### Download @@ -115,25 +113,25 @@ This does not work if you've using this with `--anonymous`. **Options** - Audio language - Which audio the episode(s) should be can be set via the `-a` / `--audio` flag. + Set the audio language with the `-a` / `--audio` flag. This only works if the url points to a series since episode urls are language specific. ```shell $ crunchy download -a de-DE https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` - Default is your system language. If not supported by Crunchyroll, `en-US` (American English) is the default. + Default is your system locale. If not supported by Crunchyroll, `en-US` (American English) is the default. - Subtitle language - Besides the audio, it's also possible to specify which language the subtitles should have with the `-s` / `--subtitle` flag. - The subtitle will be hardsubbed (burned into the video) and thus, can't be turned off or on. + Besides the audio, you can specify the subtitle language by using the `-s` / `--subtitle` flag. + The subtitles will be burned into the video track (cf. [hardsub](https://www.urbandictionary.com/define.php?term=hardsub)) and thus can not be turned off. ```shell $ crunchy download -s de-DE https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` - Default is no subtitle. + Default is none. -- Output filename +- Output template - You can specify the name of the output file with the `-o` / `--output` flag. + Define an output template by using the `-o` / `--output` flag. If you want to use any other file format than [`.ts`](https://en.wikipedia.org/wiki/MPEG_transport_stream) you need [ffmpeg](https://ffmpeg.org/). ```shell $ crunchy download -o "ditf.ts" https://www.crunchyroll.com/watch/GRDQPM1ZY/alone-and-lonesome @@ -153,7 +151,7 @@ This does not work if you've using this with `--anonymous`. **Supported urls** - Series - Only series urls are supported since single episode urls are (audio) language locked. + Only series urls are supported, because episode urls are locked to a single audio language. ```shell $ crunchy archive https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` @@ -161,25 +159,24 @@ This does not work if you've using this with `--anonymous`. **Options** - Audio languages - Which audios the episode(s) should be can be set via the `-a` / `--audio` flag. + Set the audio language with the `-a` / `--audio` flag. Can be used multiple times. ```shell $ crunchy archive -a ja-JP -a de-DE https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` - Can be used multiple times. - Default is your system language (if not supported by Crunchyroll, `en-US` (American English) is the default) + `ja-JP` (Japanese). + Default is your system locale (if not supported by Crunchyroll, `en-US` (American English) and `ja-JP` (Japanese) are used). - Subtitle languages - Besides the audio, it's also possible to specify which languages the subtitles should have with the `-s` / `--subtitle` flag. + Besides the audio, you can specify the subtitle language by using the `-s` / `--subtitle` flag. ```shell $ crunchy archive -s de-DE https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` - Default is all subtitles. + Default is `all` subtitles. -- Output filename +- Output template - You can specify the name of the output file with the `-o` / `--output` flag. - The only supported file / container format is [`.mkv`](https://en.wikipedia.org/wiki/Matroska) since it stores / can store multiple audio, video and subtitle streams. + Define an output template by using the `-o` / `--output` flag. + crunchy-cli uses the [`.mkv`](https://en.wikipedia.org/wiki/Matroska) container format, because of it's ability to store multiple audio, video and subtitle tracks at once. ```shell $ crunchy archive -o "{title}.mkv" https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` @@ -195,12 +192,12 @@ This does not work if you've using this with `--anonymous`. - Merge behavior - Because of local restrictions (or other reasons) some episodes with different languages does not have the same length (e.g. when some scenes were cut out). - The ideal state, when multiple audios & subtitles used, would be if only one _video_ has to be stored and all other languages can be stored as audio-only. + Due to censorship, some episodes have multiple lengths for different languages. + In the best case, when multiple audio & subtitle tracks are used, there is only one *video* track and all other languages can be stored as audio-only. But, as said, this is not always the case. - With the `-m` / `--merge` flag you can set what you want to do if some video lengths differ. - Valid options are `audio` - store one video and all other languages as audio only; `video` - store the video + audio for every language; `auto` - detect if videos differ in length: if so, behave like `video` else like `audio`. - Subtitles will always match to the first / primary audio and video. + With the `-m` / `--merge` flag you can define the behaviour when an episodes' video tracks differ in length. + Valid options are `audio` - store one video and all other languages as audio only; `video` - store the video + audio for every language; `auto` - detect if videos differ in length: if so, behave like `video` - otherwise like `audio`. + Subtitles will always match the primary audio and video. ```shell $ crunchy archive -m audio https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` @@ -208,52 +205,45 @@ This does not work if you've using this with `--anonymous`. - Default subtitle - `--default_subtitle` set which subtitle language should be set as default / auto appear when starting the downloaded video(s). + `--default_subtitle` Set which subtitle language is to be flagged as **default** and **forced**. ```shell $ crunchy archive --default_subtitle en-US https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` Default is none. -- No subtitle optimizations +- Subtitle optimizations - Subtitles, as Crunchyroll delivers them, look weird in some video players (#66). + Crunchyroll's subtitles look weird in some players (#66). This can be fixed by adding a specific entry to the subtitles. - But since this entry is only a de-factor standard and not represented in the official specification of the subtitle format ([`.ass`](https://en.wikipedia.org/wiki/SubStation_Alpha)) it could cause issues with some video players (but no issue got reported so far, so it's relatively safe to use). - `--no_subtitle_optimizations` can disable these optimizations. + Even though this entry is a de facto standard, it is not defined in the official specification for the `.ass` format (cf. [Advanced SubStation Subtitles](https://wiki.videolan.org/SubStation_Alpha)). This could cause compatibility issues, but no issues have been reported yet. + `--no_subtitle_optimizations` disables these optimizations. ```shell $ crunchy archive --no_subtitle_optimizations https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx ``` -### Url Filtering +### Episode filtering -If you want to download only specific episode of a series, you could either pass every single episode url to the downloader (which is fine for 1 - 3 episodes) or use _filtering_. +Filters patterns can be used to download a specific range of episodes from a single series. -It works pretty simple, just put a specific pattern surrounded by square brackets at the end of the url from the anime you want to download. -A season and / or episode as well as a range from where to where episodes should be downloaded can be specified. -Use the list below to get a better overview what is possible +A filter pattern may consist of either a season, an episode, or a combination of the two. +When used in combination, seasons `S` must be defined before episodes `E`. + +There are many possible patterns, for example: - `...[E5]` - Download the fifth episode. -- `...[S1]` - Download the full first season. -- `...[-S2]` - Download all seasons up to and including season 2. -- `...[S3E4-]` - Download all episodes from and including season 3, episode 4. -- `...[S1E4-S3]` - Download all episodes from and including season 1, episode 4, until and including season 3. -- `...[S3,S5]` - Download episode 3 and 5. -- `...[S1-S3,S4E2-S4E6]` - Download season 1 to 3 and episode 2 to episode 6 of season 4. +- `...[S1]` - Download the whole first season. +- `...[-S2]` - Download the first two seasons. +- `...[S3E4-]` - Download everything from season three, episode four, onwards. +- `...[S1E4-S3]` - Download season one, starting at episode four, then download season two and three. +- `...[S3,S5]` - Download season three and five. +- `...[S1-S3,S4E2-S4E6]` - Download season one to three, then episodes two to six from season four. -In practice, it would look like this: `https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx[E1-E5]`. +In practice, it would look like this: `https://www.crunchyroll.com/series/GY8VEQ95Y/darling-in-the-franxx[E1-E5]` -The `S`, followed by the number indicates the _season_ number, `E`, followed by the number indicates an _episode_ number. -It doesn't matter if `S`, `E` or both are missing. -Note that `S` must always stay before `E` when used. +# 📜 Disclaimer -There is also a regex available at [regex101.com](https://regex101.com/r/SDZyZM) where you can test if your pattern is correct. -Just put in your pattern without square brackets into the big empty field and if the full pattern is highlighted this means it is valid. -If none or only some parts are highlighted, it's not valid not. +This tool is **ONLY** meant for private use. You need a subscription to [`đŸ’ŗ Crunchyroll Premium đŸ’ŗ`](https://www.crunchyroll.com/welcome#plans) to download premium content. -# ☝ī¸ Disclaimer - -This tool is **ONLY** meant to be used for private purposes. To use this tool you need crunchyroll premium anyway, so there is no reason why rip and share the episodes. - -**The responsibility for what happens to the downloaded videos lies entirely with the user who downloaded them.** +**You are entirely responsible for what happens to files you downloaded through crunchy-cli.** # ⚖ License From 0a40f3c40f7eaeca06a677debcb1813f69e45b3d Mon Sep 17 00:00:00 2001 From: ByteDream Date: Thu, 23 Mar 2023 01:17:41 +0100 Subject: [PATCH 141/441] Refactor --- Cargo.lock | 775 ++++++++++------- Cargo.toml | 8 +- crunchy-cli-core/Cargo.lock | 776 +++++++++++------- crunchy-cli-core/Cargo.toml | 12 +- crunchy-cli-core/src/archive/command.rs | 184 +++++ crunchy-cli-core/src/archive/filter.rs | 482 +++++++++++ crunchy-cli-core/src/archive/mod.rs | 4 + crunchy-cli-core/src/cli/archive.rs | 618 -------------- crunchy-cli-core/src/cli/download.rs | 603 -------------- crunchy-cli-core/src/cli/log.rs | 138 ---- crunchy-cli-core/src/cli/mod.rs | 5 - crunchy-cli-core/src/cli/utils.rs | 693 ---------------- crunchy-cli-core/src/download/command.rs | 151 ++++ crunchy-cli-core/src/download/filter.rs | 349 ++++++++ crunchy-cli-core/src/download/mod.rs | 4 + crunchy-cli-core/src/lib.rs | 22 +- .../src/{cli/login.rs => login/command.rs} | 2 + crunchy-cli-core/src/login/mod.rs | 4 + crunchy-cli-core/src/utils/download.rs | 657 +++++++++++++++ crunchy-cli-core/src/utils/ffmpeg.rs | 344 ++++++++ crunchy-cli-core/src/utils/filter.rs | 95 +++ crunchy-cli-core/src/utils/format.rs | 309 +++++-- crunchy-cli-core/src/utils/locale.rs | 14 + crunchy-cli-core/src/utils/log.rs | 140 +++- crunchy-cli-core/src/utils/mod.rs | 5 +- crunchy-cli-core/src/utils/os.rs | 2 +- crunchy-cli-core/src/utils/parse.rs | 24 +- crunchy-cli-core/src/utils/sort.rs | 52 -- crunchy-cli-core/src/utils/subtitle.rs | 119 --- crunchy-cli-core/src/utils/video.rs | 42 +- 30 files changed, 3651 insertions(+), 2982 deletions(-) create mode 100644 crunchy-cli-core/src/archive/command.rs create mode 100644 crunchy-cli-core/src/archive/filter.rs create mode 100644 crunchy-cli-core/src/archive/mod.rs delete mode 100644 crunchy-cli-core/src/cli/archive.rs delete mode 100644 crunchy-cli-core/src/cli/download.rs delete mode 100644 crunchy-cli-core/src/cli/log.rs delete mode 100644 crunchy-cli-core/src/cli/mod.rs delete mode 100644 crunchy-cli-core/src/cli/utils.rs create mode 100644 crunchy-cli-core/src/download/command.rs create mode 100644 crunchy-cli-core/src/download/filter.rs create mode 100644 crunchy-cli-core/src/download/mod.rs rename crunchy-cli-core/src/{cli/login.rs => login/command.rs} (94%) create mode 100644 crunchy-cli-core/src/login/mod.rs create mode 100644 crunchy-cli-core/src/utils/download.rs create mode 100644 crunchy-cli-core/src/utils/ffmpeg.rs create mode 100644 crunchy-cli-core/src/utils/filter.rs delete mode 100644 crunchy-cli-core/src/utils/sort.rs delete mode 100644 crunchy-cli-core/src/utils/subtitle.rs diff --git a/Cargo.lock b/Cargo.lock index ae46f99..2651e95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -33,19 +33,19 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.68" +version = "1.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" +checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" [[package]] name = "async-trait" -version = "0.1.61" +version = "0.1.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "705339e0e4a9690e2908d2b3d049d85682cf19fbd5782494498fbf7003a6a282" +checksum = "86ea188f25f0255d8f92797797c97ebf5631fa88178beb1a46fdf5622c9a00e4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.5", ] [[package]] @@ -72,6 +72,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487f1e0fcbe47deb8b0574e646def1c903389d95241dd1bbcc6ce4a715dfc0c1" + [[package]] name = "block-padding" version = "0.3.2" @@ -81,29 +87,17 @@ dependencies = [ "generic-array", ] -[[package]] -name = "bstr" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" -dependencies = [ - "lazy_static", - "memchr", - "regex-automata", - "serde", -] - [[package]] name = "bumpalo" -version = "3.11.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "bytes" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "cbc" @@ -116,9 +110,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" [[package]] name = "cfg-if" @@ -128,9 +122,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.23" +version = "0.4.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" +checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" dependencies = [ "iana-time-zone", "js-sys", @@ -144,9 +138,9 @@ dependencies = [ [[package]] name = "cipher" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1873270f8f7942c191139cb8a40fd228da6c3fd2fc376d7e92d47aa14aeb59e" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ "crypto-common", "inout", @@ -154,55 +148,55 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.32" +version = "4.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39" +checksum = "42dfd32784433290c51d92c438bb72ea5063797fc3cc9a21a8c4346bebbb2098" dependencies = [ - "bitflags", + "bitflags 2.0.2", "clap_derive", "clap_lex", "is-terminal", "once_cell", - "strsim", + "strsim 0.10.0", "termcolor", ] [[package]] name = "clap_complete" -version = "4.0.7" +version = "4.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10861370d2ba66b0f5989f83ebf35db6421713fd92351790e7fdd6c36774c56b" +checksum = "37686beaba5ac9f3ab01ee3172f792fc6ffdd685bfb9e63cfef02c0571a4e8e1" dependencies = [ "clap", ] [[package]] name = "clap_derive" -version = "4.0.21" +version = "4.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" +checksum = "fddf67631444a3a3e3e5ac51c36a5e01335302de677bd78759eaa90ab1f46644" dependencies = [ "heck", "proc-macro-error", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "clap_lex" -version = "0.3.0" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" +checksum = "033f6b7a4acb1f358c742aaca805c939ee73b4c6209ae4318ec7aca81c42e646" dependencies = [ "os_str_bytes", ] [[package]] name = "clap_mangen" -version = "0.2.6" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904eb24d05ad587557e0f484ddce5c737c30cf81372badb16d13e41c4b8340b1" +checksum = "4237e29de9c6949982ba87d51709204504fb8ed2fd38232fcb1e5bf7d4ba48c8" dependencies = [ "clap", "roff", @@ -220,15 +214,15 @@ dependencies = [ [[package]] name = "console" -version = "0.15.4" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9b6515d269224923b26b5febea2ed42b2d5f2ce37284a4dd670fedd6cb8347a" +checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60" dependencies = [ "encode_unicode", "lazy_static", "libc", "unicode-width", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -238,7 +232,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" dependencies = [ "percent-encoding", - "time 0.3.17", + "time 0.3.20", "version_check", ] @@ -254,7 +248,7 @@ dependencies = [ "publicsuffix", "serde", "serde_json", - "time 0.3.17", + "time 0.3.20", "url", ] @@ -304,8 +298,8 @@ dependencies = [ "chrono", "clap", "crunchyroll-rs", - "csv", "ctrlc", + "derive_setters", "dirs", "indicatif", "lazy_static", @@ -325,15 +319,17 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.5" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3770cda4c67e68c689c8e361af46bb9d017caf82263905358fd0751d10657a0" +checksum = "bddaff98c25bdedca0f1ed2b68c28b12a3e52054144b1b11380236d23051c751" dependencies = [ "aes", "async-trait", "cbc", "chrono", "crunchyroll-rs-internal", + "dash-mpd", + "futures-util", "http", "lazy_static", "m3u8-rs", @@ -350,13 +346,13 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.5" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a260a73e733bb0ce30343caaed5e968d3c1cc2ea0ab27c601481e9ef22a2fd7" +checksum = "3f26b71b36db3139ce788545c2bffa7e69d4fd7b689f143086c6283e847a4668" dependencies = [ - "darling", + "darling 0.14.4", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -369,43 +365,21 @@ dependencies = [ "typenum", ] -[[package]] -name = "csv" -version = "1.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" -dependencies = [ - "bstr", - "csv-core", - "itoa 0.4.8", - "ryu", - "serde", -] - -[[package]] -name = "csv-core" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" -dependencies = [ - "memchr", -] - [[package]] name = "ctrlc" -version = "3.2.4" +version = "3.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1631ca6e3c59112501a9d87fd86f21591ff77acd31331e8a73f8d80a65bbdd71" +checksum = "bbcf33c2a618cbe41ee43ae6e9f2e48368cd9f9db2896f10167d8d762679f639" dependencies = [ "nix", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] name = "cxx" -version = "1.0.86" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d1075c37807dcf850c379432f0df05ba52cc30f279c5cfc43cc221ce7f8579" +checksum = "a9c00419335c41018365ddf7e4d5f1c12ee3659ddcf3e01974650ba1de73d038" dependencies = [ "cc", "cxxbridge-flags", @@ -415,9 +389,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.86" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5044281f61b27bc598f2f6647d480aed48d2bf52d6eb0b627d84c0361b17aa70" +checksum = "fb8307ad413a98fff033c8545ecf133e3257747b3bae935e7602aab8aa92d4ca" dependencies = [ "cc", "codespan-reporting", @@ -425,79 +399,145 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn", + "syn 2.0.5", ] [[package]] name = "cxxbridge-flags" -version = "1.0.86" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61b50bc93ba22c27b0d31128d2d130a0a6b3d267ae27ef7e4fae2167dfe8781c" +checksum = "edc52e2eb08915cb12596d29d55f0b5384f00d697a646dbd269b6ecb0fbd9d31" [[package]] name = "cxxbridge-macro" -version = "1.0.86" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e61fda7e62115119469c7b3591fd913ecca96fb766cfd3f2e2502ab7bc87a5" +checksum = "631569015d0d8d54e6c241733f944042623ab6df7bc3be7466874b05fcdb1c5f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.5", ] [[package]] name = "darling" -version = "0.14.2" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0dd3cd20dc6b5a876612a6e5accfe7f3dd883db6d07acfbf14c128f61550dfa" +checksum = "0d706e75d87e35569db781a9b5e2416cff1236a47ed380831f959382ccd5f858" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.10.2", + "darling_macro 0.10.2", +] + +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core 0.14.4", + "darling_macro 0.14.4", ] [[package]] name = "darling_core" -version = "0.14.2" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a784d2ccaf7c98501746bf0be29b2022ba41fd62a2e622af997a03e9f972859f" +checksum = "f0c960ae2da4de88a91b2d920c2a7233b400bc33cb28453a2987822d8392519b" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", - "strsim", - "syn", + "strsim 0.9.3", + "syn 1.0.109", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn 1.0.109", ] [[package]] name = "darling_macro" -version = "0.14.2" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7618812407e9402654622dd402b0a89dff9ba93badd6540781526117b92aab7e" +checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72" dependencies = [ - "darling_core", + "darling_core 0.10.2", "quote", - "syn", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core 0.14.4", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "dash-mpd" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df35f3b3b0fde2747a01530de0a81f7a27620d050f656e518ef0550557f564de" +dependencies = [ + "chrono", + "fs-err", + "iso8601", + "log", + "num-traits", + "quick-xml", + "regex", + "serde", + "serde_with", + "thiserror", + "xattr", +] + +[[package]] +name = "derive_setters" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1cf41b4580a37cca5ef2ada2cc43cf5d6be3983f4522e83010d67ab6925e84b" +dependencies = [ + "darling 0.10.2", + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] name = "dirs" -version = "4.0.0" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +checksum = "dece029acd3353e3a58ac2e3eb3c8d6c35827a892edc6cc4138ef9c33df46ecd" dependencies = [ "dirs-sys", ] [[package]] name = "dirs-sys" -version = "0.3.7" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +checksum = "04414300db88f70d74c5ff54e50f9e1d1737d9a5b90f53fcf2e95ca2a9ab554b" dependencies = [ "libc", "redox_users", - "winapi", + "windows-sys 0.45.0", ] [[package]] @@ -508,9 +548,9 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" [[package]] name = "encoding_rs" -version = "0.8.31" +version = "0.8.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" dependencies = [ "cfg-if", ] @@ -538,9 +578,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" dependencies = [ "instant", ] @@ -576,43 +616,49 @@ dependencies = [ ] [[package]] -name = "futures-channel" -version = "0.3.25" +name = "fs-err" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" +checksum = "0845fa252299212f0389d64ba26f34fa32cfe41588355f21ed507c59a0f64541" + +[[package]] +name = "futures-channel" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "164713a5a0dcc3e7b4b1ed7d3b433cabc18025386f9339346e8daf15963cf7ac" dependencies = [ "futures-core", ] [[package]] name = "futures-core" -version = "0.3.25" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" +checksum = "86d7a0c1aa76363dac491de0ee99faf6941128376f1cf96f07db7603b7de69dd" [[package]] name = "futures-io" -version = "0.3.25" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" +checksum = "89d422fa3cbe3b40dca574ab087abb5bc98258ea57eea3fd6f1fa7162c778b91" [[package]] name = "futures-sink" -version = "0.3.25" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" +checksum = "ec93083a4aecafb2a80a885c9de1f0ccae9dbd32c2bb54b0c3a65690e0b8d2f2" [[package]] name = "futures-task" -version = "0.3.25" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" +checksum = "fd65540d33b37b16542a0438c12e6aeead10d4ac5d05bd3f805b8f35ab592879" [[package]] name = "futures-util" -version = "0.3.25" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" +checksum = "3ef6b17e481503ec85211fed8f39d1970f128935ca1f814cd32ac4a6842e84ab" dependencies = [ "futures-core", "futures-io", @@ -646,9 +692,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" dependencies = [ "bytes", "fnv", @@ -671,9 +717,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "heck" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" @@ -685,14 +731,26 @@ dependencies = [ ] [[package]] -name = "http" -version = "0.2.8" +name = "hermit-abi" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ "bytes", "fnv", - "itoa 1.0.5", + "itoa", ] [[package]] @@ -720,9 +778,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.23" +version = "0.14.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c" +checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" dependencies = [ "bytes", "futures-channel", @@ -733,7 +791,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa 1.0.5", + "itoa", "pin-project-lite", "socket2", "tokio", @@ -770,16 +828,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.53" +version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +checksum = "0c17cc76786e99f8d2f055c11159e7f0091c42474dcc3189fbab96072e873e6d" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "winapi", + "windows", ] [[package]] @@ -827,13 +885,14 @@ checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" dependencies = [ "autocfg", "hashbrown", + "serde", ] [[package]] name = "indicatif" -version = "0.17.2" +version = "0.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4295cbb7573c16d310e99e713cf9e75101eb190ab31fccd35f2d2691b4352b19" +checksum = "cef509aa9bc73864d6756f0d34d35504af3cf0844373afe9b8669a5b8005a729" dependencies = [ "console", "number_prefix", @@ -862,12 +921,13 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.3" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" +checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" dependencies = [ + "hermit-abi 0.3.1", "libc", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -878,33 +938,36 @@ checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" [[package]] name = "is-terminal" -version = "0.4.2" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189" +checksum = "8687c819457e979cc940d09cb16e42a1bf70aa6b60a549de6d3a62a0ee90c69e" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.1", "io-lifetimes", "rustix", - "windows-sys", + "windows-sys 0.45.0", +] + +[[package]] +name = "iso8601" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924e5d73ea28f59011fec52a0d12185d496a9b075d360657aed2a5707f701153" +dependencies = [ + "nom", ] [[package]] name = "itoa" -version = "0.4.8" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" - -[[package]] -name = "itoa" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "js-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" dependencies = [ "wasm-bindgen", ] @@ -917,9 +980,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.139" +version = "0.2.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" [[package]] name = "link-cplusplus" @@ -957,9 +1020,9 @@ dependencies = [ [[package]] name = "matches" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" +checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" [[package]] name = "memchr" @@ -969,9 +1032,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "mime" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "minimal-lexical" @@ -981,14 +1044,14 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mio" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -1011,11 +1074,11 @@ dependencies = [ [[package]] name = "nix" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46a58d1d356c6597d08cde02c2f09d785b09e28711837b1ed667dc652c08a694" +checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if", "libc", "static_assertions", @@ -1023,9 +1086,9 @@ dependencies = [ [[package]] name = "nom" -version = "7.1.2" +version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5507769c4919c998e69e49c839d9dc6e693ede4cc4290d6ad8b41d4f09c548c" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" dependencies = [ "memchr", "minimal-lexical", @@ -1056,7 +1119,7 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi", + "hermit-abi 0.2.6", "libc", ] @@ -1068,17 +1131,17 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "once_cell" -version = "1.17.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "openssl" -version = "0.10.45" +version = "0.10.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" +checksum = "d8b277f87dacc05a6b709965d1cbafac4649d6ce9f3ce9ceb88508b5666dfec9" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if", "foreign-types", "libc", @@ -1095,7 +1158,7 @@ checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1106,9 +1169,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.80" +version = "0.9.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" +checksum = "a95792af3c4e0153c3914df2261bedd30a98476f94dc892b67dfe1d89d433a04" dependencies = [ "autocfg", "cc", @@ -1119,9 +1182,9 @@ dependencies = [ [[package]] name = "os_str_bytes" -version = "6.4.1" +version = "6.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" +checksum = "ceedf44fb00f2d1984b0bc98102627ce622e083e49a5bacdb3e514fa4238e267" [[package]] name = "percent-encoding" @@ -1162,7 +1225,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "version_check", ] @@ -1177,17 +1240,11 @@ dependencies = [ "version_check", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" - [[package]] name = "proc-macro2" -version = "1.0.49" +version = "1.0.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" +checksum = "ba466839c78239c09faf015484e5cc04860f88242cff4d03eb038f04b4699b73" dependencies = [ "unicode-ident", ] @@ -1209,10 +1266,20 @@ dependencies = [ ] [[package]] -name = "quote" -version = "1.0.23" +name = "quick-xml" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "e5c1a97b1bc42b1d550bfb48d4262153fe400a12bab1511821736f7eac76d7e2" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quote" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" dependencies = [ "proc-macro2", ] @@ -1223,7 +1290,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -1239,43 +1306,28 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" +checksum = "cce168fea28d3e05f158bda4576cf0c844d5045bc2cc3620fa0292ed5bb5814c" dependencies = [ "aho-corasick", "memchr", "regex-syntax", ] -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" - [[package]] name = "regex-syntax" -version = "0.6.28" +version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" - -[[package]] -name = "remove_dir_all" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "reqwest" -version = "0.11.13" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" +checksum = "0ba30cc2c0cd02af1222ed216ba659cdb2f879dfe3181852fe7c50b1d0005949" dependencies = [ - "base64 0.13.1", + "base64 0.21.0", "bytes", "cookie", "cookie_store", @@ -1296,7 +1348,6 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "proc-macro-hack", "rustls", "rustls-pemfile", "serde", @@ -1337,23 +1388,23 @@ checksum = "b833d8d034ea094b1ea68aa6d5c740e0d04bad9d16568d08ba6f76823a114316" [[package]] name = "rustix" -version = "0.36.6" +version = "0.36.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4feacf7db682c6c329c4ede12649cd36ecab0f3be5b7d74e6a20304725db4549" +checksum = "db4165c9963ab29e422d6c26fbc1d37f15bace6b2810221f9d925023480fcf0e" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] name = "rustls" -version = "0.20.7" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "539a2bfe908f471bfa933876bd1eb6a19cf2176d375f82ef7f99530a40e48c2c" +checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" dependencies = [ "log", "ring", @@ -1372,9 +1423,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" [[package]] name = "sanitize-filename" @@ -1392,14 +1443,14 @@ version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" dependencies = [ - "windows-sys", + "windows-sys 0.42.0", ] [[package]] name = "scratch" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" +checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" [[package]] name = "sct" @@ -1413,11 +1464,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.7.0" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" +checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "core-foundation-sys", "libc", @@ -1426,9 +1477,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.6.1" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" dependencies = [ "core-foundation-sys", "libc", @@ -1436,31 +1487,31 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.152" +version = "1.0.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "771d4d9c4163ee138805e12c710dd365e4f44be8be0503cb1bb9eb989425d9c9" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "e801c1712f48475582b7696ac71e0ca34ebb30e09338425384269d9717c62cad" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.5", ] [[package]] name = "serde_json" -version = "1.0.91" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" +checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea" dependencies = [ - "itoa 1.0.5", + "itoa", "ryu", "serde", ] @@ -1472,11 +1523,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.5", + "itoa", "ryu", "serde", ] +[[package]] +name = "serde_with" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85456ffac572dc8826334164f2fb6fb40a7c766aebe195a2a21ee69ee2885ecf" +dependencies = [ + "base64 0.13.1", + "chrono", + "hex", + "indexmap", + "serde", + "serde_json", + "serde_with_macros", + "time 0.3.20", +] + +[[package]] +name = "serde_with_macros" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cbcd6104f8a4ab6af7f6be2a0da6be86b9de3c401f6e86bb856ab2af739232f" +dependencies = [ + "darling 0.14.4", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "shlex" version = "1.1.0" @@ -1485,9 +1564,9 @@ checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" [[package]] name = "signal-hook" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d" +checksum = "732768f1176d21d09e076c23a93123d40bba92d50c4058da34d45c8de8e682b9" dependencies = [ "libc", "signal-hook-registry", @@ -1495,18 +1574,18 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" dependencies = [ "libc", ] [[package]] name = "slab" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" dependencies = [ "autocfg", ] @@ -1519,14 +1598,14 @@ checksum = "133659a15339456eeeb07572eb02a91c91e9815e9cbc89566944d2c8d3efdbf6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", @@ -1544,6 +1623,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strsim" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" + [[package]] name = "strsim" version = "0.10.0" @@ -1552,9 +1637,20 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.107" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89c2d1c76a26822187a1fbb5964e3fff108bc208f02e820ab9dac1234f6b388a" dependencies = [ "proc-macro2", "quote", @@ -1563,68 +1659,67 @@ dependencies = [ [[package]] name = "sys-locale" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3358acbb4acd4146138b9bda219e904a6bb5aaaa237f8eed06f4d6bc1580ecee" +checksum = "f8a11bd9c338fdba09f7881ab41551932ad42e405f61d01e8406baea71c07aee" dependencies = [ "js-sys", "libc", "wasm-bindgen", "web-sys", - "winapi", + "windows-sys 0.45.0", ] [[package]] name = "tempfile" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95" dependencies = [ "cfg-if", "fastrand", - "libc", "redox_syscall", - "remove_dir_all", - "winapi", + "rustix", + "windows-sys 0.42.0", ] [[package]] name = "termcolor" -version = "1.1.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" dependencies = [ "winapi-util", ] [[package]] name = "terminal_size" -version = "0.2.3" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb20089a8ba2b69debd491f8d2d023761cbf196e999218c591fa1e7e15a21907" +checksum = "4c9afddd2cec1c0909f06b00ef33f94ab2cc0578c4a610aa208ddfec8aa2b43a" dependencies = [ "rustix", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] name = "thiserror" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.5", ] [[package]] @@ -1640,11 +1735,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.17" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" +checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" dependencies = [ - "itoa 1.0.5", + "itoa", "serde", "time-core", "time-macros", @@ -1658,9 +1753,9 @@ checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" [[package]] name = "time-macros" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" +checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" dependencies = [ "time-core", ] @@ -1676,15 +1771,15 @@ dependencies = [ [[package]] name = "tinyvec_macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.24.1" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae" +checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" dependencies = [ "autocfg", "bytes", @@ -1695,7 +1790,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -1706,14 +1801,14 @@ checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "tokio-native-tls" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", "tokio", @@ -1732,9 +1827,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.4" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" dependencies = [ "bytes", "futures-core", @@ -1784,15 +1879,15 @@ checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "unicode-bidi" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" [[package]] name = "unicode-normalization" @@ -1862,9 +1957,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1872,24 +1967,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" dependencies = [ "cfg-if", "js-sys", @@ -1899,9 +1994,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1909,28 +2004,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" [[package]] name = "web-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" dependencies = [ "js-sys", "wasm-bindgen", @@ -1986,6 +2081,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.42.0" @@ -2002,46 +2106,70 @@ dependencies = [ ] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.0" +name = "windows-sys" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_msvc" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_i686_gnu" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_msvc" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_x86_64_gnu" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_msvc" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "winreg" @@ -2051,3 +2179,12 @@ checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ "winapi", ] + +[[package]] +name = "xattr" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea263437ca03c1522846a4ddafbca2542d0ad5ed9b784909d4b27b76f62bc34a" +dependencies = [ + "libc", +] diff --git a/Cargo.toml b/Cargo.toml index db1f129..62ff7fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,18 +5,16 @@ version = "3.0.0-dev.8" edition = "2021" [dependencies] -tokio = { version = "1.24", features = ["macros", "rt-multi-thread", "time"], default-features = false } +tokio = { version = "1.26", features = ["macros", "rt-multi-thread", "time"], default-features = false } crunchy-cli-core = { path = "./crunchy-cli-core" } [build-dependencies] chrono = "0.4" -clap = { version = "4.0", features = ["string"] } -clap_complete = "4.0" +clap = { version = "4.1", features = ["string"] } +clap_complete = "4.1" clap_mangen = "0.2" -# The static-* features must be used here since build dependency features cannot be manipulated from the features -# specified in this Cargo.toml [features]. crunchy-cli-core = { path = "./crunchy-cli-core" } [profile.release] diff --git a/crunchy-cli-core/Cargo.lock b/crunchy-cli-core/Cargo.lock index b12b3a3..c4e2235 100644 --- a/crunchy-cli-core/Cargo.lock +++ b/crunchy-cli-core/Cargo.lock @@ -33,19 +33,19 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.68" +version = "1.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" +checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" [[package]] name = "async-trait" -version = "0.1.61" +version = "0.1.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "705339e0e4a9690e2908d2b3d049d85682cf19fbd5782494498fbf7003a6a282" +checksum = "86ea188f25f0255d8f92797797c97ebf5631fa88178beb1a46fdf5622c9a00e4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.6", ] [[package]] @@ -72,6 +72,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487f1e0fcbe47deb8b0574e646def1c903389d95241dd1bbcc6ce4a715dfc0c1" + [[package]] name = "block-padding" version = "0.3.2" @@ -81,29 +87,17 @@ dependencies = [ "generic-array", ] -[[package]] -name = "bstr" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" -dependencies = [ - "lazy_static", - "memchr", - "regex-automata", - "serde", -] - [[package]] name = "bumpalo" -version = "3.11.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "bytes" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "cbc" @@ -116,9 +110,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" [[package]] name = "cfg-if" @@ -128,9 +122,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.23" +version = "0.4.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" +checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" dependencies = [ "iana-time-zone", "js-sys", @@ -144,9 +138,9 @@ dependencies = [ [[package]] name = "cipher" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1873270f8f7942c191139cb8a40fd228da6c3fd2fc376d7e92d47aa14aeb59e" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ "crypto-common", "inout", @@ -154,37 +148,37 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.32" +version = "4.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39" +checksum = "42dfd32784433290c51d92c438bb72ea5063797fc3cc9a21a8c4346bebbb2098" dependencies = [ - "bitflags", + "bitflags 2.0.2", "clap_derive", "clap_lex", "is-terminal", "once_cell", - "strsim", + "strsim 0.10.0", "termcolor", ] [[package]] name = "clap_derive" -version = "4.0.21" +version = "4.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" +checksum = "fddf67631444a3a3e3e5ac51c36a5e01335302de677bd78759eaa90ab1f46644" dependencies = [ "heck", "proc-macro-error", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "clap_lex" -version = "0.3.0" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" +checksum = "033f6b7a4acb1f358c742aaca805c939ee73b4c6209ae4318ec7aca81c42e646" dependencies = [ "os_str_bytes", ] @@ -201,15 +195,15 @@ dependencies = [ [[package]] name = "console" -version = "0.15.4" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9b6515d269224923b26b5febea2ed42b2d5f2ce37284a4dd670fedd6cb8347a" +checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60" dependencies = [ "encode_unicode", "lazy_static", "libc", "unicode-width", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -219,7 +213,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" dependencies = [ "percent-encoding", - "time 0.3.17", + "time 0.3.20", "version_check", ] @@ -235,7 +229,7 @@ dependencies = [ "publicsuffix", "serde", "serde_json", - "time 0.3.17", + "time 0.3.20", "url", ] @@ -273,8 +267,8 @@ dependencies = [ "chrono", "clap", "crunchyroll-rs", - "csv", "ctrlc", + "derive_setters", "dirs", "indicatif", "lazy_static", @@ -284,6 +278,7 @@ dependencies = [ "sanitize-filename", "serde", "serde_json", + "shlex", "signal-hook", "sys-locale", "tempfile", @@ -293,15 +288,17 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.2.5" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3770cda4c67e68c689c8e361af46bb9d017caf82263905358fd0751d10657a0" +checksum = "bddaff98c25bdedca0f1ed2b68c28b12a3e52054144b1b11380236d23051c751" dependencies = [ "aes", "async-trait", "cbc", "chrono", "crunchyroll-rs-internal", + "dash-mpd", + "futures-util", "http", "lazy_static", "m3u8-rs", @@ -318,13 +315,13 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.2.5" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a260a73e733bb0ce30343caaed5e968d3c1cc2ea0ab27c601481e9ef22a2fd7" +checksum = "3f26b71b36db3139ce788545c2bffa7e69d4fd7b689f143086c6283e847a4668" dependencies = [ - "darling", + "darling 0.14.4", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -337,43 +334,21 @@ dependencies = [ "typenum", ] -[[package]] -name = "csv" -version = "1.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" -dependencies = [ - "bstr", - "csv-core", - "itoa 0.4.8", - "ryu", - "serde", -] - -[[package]] -name = "csv-core" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" -dependencies = [ - "memchr", -] - [[package]] name = "ctrlc" -version = "3.2.4" +version = "3.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1631ca6e3c59112501a9d87fd86f21591ff77acd31331e8a73f8d80a65bbdd71" +checksum = "bbcf33c2a618cbe41ee43ae6e9f2e48368cd9f9db2896f10167d8d762679f639" dependencies = [ "nix", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] name = "cxx" -version = "1.0.86" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d1075c37807dcf850c379432f0df05ba52cc30f279c5cfc43cc221ce7f8579" +checksum = "a9c00419335c41018365ddf7e4d5f1c12ee3659ddcf3e01974650ba1de73d038" dependencies = [ "cc", "cxxbridge-flags", @@ -383,9 +358,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.86" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5044281f61b27bc598f2f6647d480aed48d2bf52d6eb0b627d84c0361b17aa70" +checksum = "fb8307ad413a98fff033c8545ecf133e3257747b3bae935e7602aab8aa92d4ca" dependencies = [ "cc", "codespan-reporting", @@ -393,79 +368,145 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn", + "syn 2.0.6", ] [[package]] name = "cxxbridge-flags" -version = "1.0.86" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61b50bc93ba22c27b0d31128d2d130a0a6b3d267ae27ef7e4fae2167dfe8781c" +checksum = "edc52e2eb08915cb12596d29d55f0b5384f00d697a646dbd269b6ecb0fbd9d31" [[package]] name = "cxxbridge-macro" -version = "1.0.86" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e61fda7e62115119469c7b3591fd913ecca96fb766cfd3f2e2502ab7bc87a5" +checksum = "631569015d0d8d54e6c241733f944042623ab6df7bc3be7466874b05fcdb1c5f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.6", ] [[package]] name = "darling" -version = "0.14.2" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0dd3cd20dc6b5a876612a6e5accfe7f3dd883db6d07acfbf14c128f61550dfa" +checksum = "0d706e75d87e35569db781a9b5e2416cff1236a47ed380831f959382ccd5f858" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.10.2", + "darling_macro 0.10.2", +] + +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core 0.14.4", + "darling_macro 0.14.4", ] [[package]] name = "darling_core" -version = "0.14.2" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a784d2ccaf7c98501746bf0be29b2022ba41fd62a2e622af997a03e9f972859f" +checksum = "f0c960ae2da4de88a91b2d920c2a7233b400bc33cb28453a2987822d8392519b" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", - "strsim", - "syn", + "strsim 0.9.3", + "syn 1.0.109", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn 1.0.109", ] [[package]] name = "darling_macro" -version = "0.14.2" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7618812407e9402654622dd402b0a89dff9ba93badd6540781526117b92aab7e" +checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72" dependencies = [ - "darling_core", + "darling_core 0.10.2", "quote", - "syn", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core 0.14.4", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "dash-mpd" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df35f3b3b0fde2747a01530de0a81f7a27620d050f656e518ef0550557f564de" +dependencies = [ + "chrono", + "fs-err", + "iso8601", + "log", + "num-traits", + "quick-xml", + "regex", + "serde", + "serde_with", + "thiserror", + "xattr", +] + +[[package]] +name = "derive_setters" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1cf41b4580a37cca5ef2ada2cc43cf5d6be3983f4522e83010d67ab6925e84b" +dependencies = [ + "darling 0.10.2", + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] name = "dirs" -version = "4.0.0" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +checksum = "dece029acd3353e3a58ac2e3eb3c8d6c35827a892edc6cc4138ef9c33df46ecd" dependencies = [ "dirs-sys", ] [[package]] name = "dirs-sys" -version = "0.3.7" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +checksum = "04414300db88f70d74c5ff54e50f9e1d1737d9a5b90f53fcf2e95ca2a9ab554b" dependencies = [ "libc", "redox_users", - "winapi", + "windows-sys 0.45.0", ] [[package]] @@ -476,9 +517,9 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" [[package]] name = "encoding_rs" -version = "0.8.31" +version = "0.8.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" dependencies = [ "cfg-if", ] @@ -506,9 +547,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" dependencies = [ "instant", ] @@ -544,43 +585,49 @@ dependencies = [ ] [[package]] -name = "futures-channel" -version = "0.3.25" +name = "fs-err" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" +checksum = "0845fa252299212f0389d64ba26f34fa32cfe41588355f21ed507c59a0f64541" + +[[package]] +name = "futures-channel" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "164713a5a0dcc3e7b4b1ed7d3b433cabc18025386f9339346e8daf15963cf7ac" dependencies = [ "futures-core", ] [[package]] name = "futures-core" -version = "0.3.25" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" +checksum = "86d7a0c1aa76363dac491de0ee99faf6941128376f1cf96f07db7603b7de69dd" [[package]] name = "futures-io" -version = "0.3.25" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" +checksum = "89d422fa3cbe3b40dca574ab087abb5bc98258ea57eea3fd6f1fa7162c778b91" [[package]] name = "futures-sink" -version = "0.3.25" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" +checksum = "ec93083a4aecafb2a80a885c9de1f0ccae9dbd32c2bb54b0c3a65690e0b8d2f2" [[package]] name = "futures-task" -version = "0.3.25" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" +checksum = "fd65540d33b37b16542a0438c12e6aeead10d4ac5d05bd3f805b8f35ab592879" [[package]] name = "futures-util" -version = "0.3.25" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" +checksum = "3ef6b17e481503ec85211fed8f39d1970f128935ca1f814cd32ac4a6842e84ab" dependencies = [ "futures-core", "futures-io", @@ -614,9 +661,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" dependencies = [ "bytes", "fnv", @@ -639,9 +686,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "heck" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" @@ -653,14 +700,26 @@ dependencies = [ ] [[package]] -name = "http" -version = "0.2.8" +name = "hermit-abi" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ "bytes", "fnv", - "itoa 1.0.5", + "itoa", ] [[package]] @@ -688,9 +747,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.23" +version = "0.14.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c" +checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" dependencies = [ "bytes", "futures-channel", @@ -701,7 +760,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa 1.0.5", + "itoa", "pin-project-lite", "socket2", "tokio", @@ -738,16 +797,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.53" +version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +checksum = "0c17cc76786e99f8d2f055c11159e7f0091c42474dcc3189fbab96072e873e6d" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "winapi", + "windows", ] [[package]] @@ -795,13 +854,14 @@ checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" dependencies = [ "autocfg", "hashbrown", + "serde", ] [[package]] name = "indicatif" -version = "0.17.2" +version = "0.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4295cbb7573c16d310e99e713cf9e75101eb190ab31fccd35f2d2691b4352b19" +checksum = "cef509aa9bc73864d6756f0d34d35504af3cf0844373afe9b8669a5b8005a729" dependencies = [ "console", "number_prefix", @@ -830,12 +890,13 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.3" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" +checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" dependencies = [ + "hermit-abi 0.3.1", "libc", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -846,33 +907,36 @@ checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" [[package]] name = "is-terminal" -version = "0.4.2" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189" +checksum = "8687c819457e979cc940d09cb16e42a1bf70aa6b60a549de6d3a62a0ee90c69e" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.1", "io-lifetimes", "rustix", - "windows-sys", + "windows-sys 0.45.0", +] + +[[package]] +name = "iso8601" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924e5d73ea28f59011fec52a0d12185d496a9b075d360657aed2a5707f701153" +dependencies = [ + "nom", ] [[package]] name = "itoa" -version = "0.4.8" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" - -[[package]] -name = "itoa" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "js-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" dependencies = [ "wasm-bindgen", ] @@ -885,9 +949,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.139" +version = "0.2.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" [[package]] name = "link-cplusplus" @@ -925,9 +989,9 @@ dependencies = [ [[package]] name = "matches" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" +checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" [[package]] name = "memchr" @@ -937,9 +1001,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "mime" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "minimal-lexical" @@ -949,14 +1013,14 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mio" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -979,11 +1043,11 @@ dependencies = [ [[package]] name = "nix" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46a58d1d356c6597d08cde02c2f09d785b09e28711837b1ed667dc652c08a694" +checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if", "libc", "static_assertions", @@ -991,9 +1055,9 @@ dependencies = [ [[package]] name = "nom" -version = "7.1.2" +version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5507769c4919c998e69e49c839d9dc6e693ede4cc4290d6ad8b41d4f09c548c" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" dependencies = [ "memchr", "minimal-lexical", @@ -1024,7 +1088,7 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi", + "hermit-abi 0.2.6", "libc", ] @@ -1036,17 +1100,17 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "once_cell" -version = "1.17.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "openssl" -version = "0.10.45" +version = "0.10.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" +checksum = "d8b277f87dacc05a6b709965d1cbafac4649d6ce9f3ce9ceb88508b5666dfec9" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if", "foreign-types", "libc", @@ -1063,7 +1127,7 @@ checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1074,9 +1138,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.80" +version = "0.9.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" +checksum = "a95792af3c4e0153c3914df2261bedd30a98476f94dc892b67dfe1d89d433a04" dependencies = [ "autocfg", "cc", @@ -1087,9 +1151,9 @@ dependencies = [ [[package]] name = "os_str_bytes" -version = "6.4.1" +version = "6.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" +checksum = "ceedf44fb00f2d1984b0bc98102627ce622e083e49a5bacdb3e514fa4238e267" [[package]] name = "percent-encoding" @@ -1130,7 +1194,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "version_check", ] @@ -1145,17 +1209,11 @@ dependencies = [ "version_check", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" - [[package]] name = "proc-macro2" -version = "1.0.49" +version = "1.0.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" +checksum = "ba466839c78239c09faf015484e5cc04860f88242cff4d03eb038f04b4699b73" dependencies = [ "unicode-ident", ] @@ -1177,10 +1235,20 @@ dependencies = [ ] [[package]] -name = "quote" -version = "1.0.23" +name = "quick-xml" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "e5c1a97b1bc42b1d550bfb48d4262153fe400a12bab1511821736f7eac76d7e2" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quote" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" dependencies = [ "proc-macro2", ] @@ -1191,7 +1259,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -1207,43 +1275,28 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" +checksum = "cce168fea28d3e05f158bda4576cf0c844d5045bc2cc3620fa0292ed5bb5814c" dependencies = [ "aho-corasick", "memchr", "regex-syntax", ] -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" - [[package]] name = "regex-syntax" -version = "0.6.28" +version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" - -[[package]] -name = "remove_dir_all" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "reqwest" -version = "0.11.13" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" +checksum = "0ba30cc2c0cd02af1222ed216ba659cdb2f879dfe3181852fe7c50b1d0005949" dependencies = [ - "base64 0.13.1", + "base64 0.21.0", "bytes", "cookie", "cookie_store", @@ -1264,7 +1317,6 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "proc-macro-hack", "rustls", "rustls-pemfile", "serde", @@ -1299,23 +1351,23 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.6" +version = "0.36.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4feacf7db682c6c329c4ede12649cd36ecab0f3be5b7d74e6a20304725db4549" +checksum = "db4165c9963ab29e422d6c26fbc1d37f15bace6b2810221f9d925023480fcf0e" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] name = "rustls" -version = "0.20.7" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "539a2bfe908f471bfa933876bd1eb6a19cf2176d375f82ef7f99530a40e48c2c" +checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" dependencies = [ "log", "ring", @@ -1334,9 +1386,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" [[package]] name = "sanitize-filename" @@ -1354,14 +1406,14 @@ version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" dependencies = [ - "windows-sys", + "windows-sys 0.42.0", ] [[package]] name = "scratch" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" +checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" [[package]] name = "sct" @@ -1375,11 +1427,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.7.0" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" +checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "core-foundation-sys", "libc", @@ -1388,9 +1440,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.6.1" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" dependencies = [ "core-foundation-sys", "libc", @@ -1398,31 +1450,31 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.152" +version = "1.0.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "771d4d9c4163ee138805e12c710dd365e4f44be8be0503cb1bb9eb989425d9c9" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "e801c1712f48475582b7696ac71e0ca34ebb30e09338425384269d9717c62cad" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.6", ] [[package]] name = "serde_json" -version = "1.0.91" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" +checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea" dependencies = [ - "itoa 1.0.5", + "itoa", "ryu", "serde", ] @@ -1434,16 +1486,50 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.5", + "itoa", "ryu", "serde", ] [[package]] -name = "signal-hook" -version = "0.3.14" +name = "serde_with" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d" +checksum = "85456ffac572dc8826334164f2fb6fb40a7c766aebe195a2a21ee69ee2885ecf" +dependencies = [ + "base64 0.13.1", + "chrono", + "hex", + "indexmap", + "serde", + "serde_json", + "serde_with_macros", + "time 0.3.20", +] + +[[package]] +name = "serde_with_macros" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cbcd6104f8a4ab6af7f6be2a0da6be86b9de3c401f6e86bb856ab2af739232f" +dependencies = [ + "darling 0.14.4", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "shlex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" + +[[package]] +name = "signal-hook" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "732768f1176d21d09e076c23a93123d40bba92d50c4058da34d45c8de8e682b9" dependencies = [ "libc", "signal-hook-registry", @@ -1451,18 +1537,18 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" dependencies = [ "libc", ] [[package]] name = "slab" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" dependencies = [ "autocfg", ] @@ -1475,14 +1561,14 @@ checksum = "133659a15339456eeeb07572eb02a91c91e9815e9cbc89566944d2c8d3efdbf6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", @@ -1500,6 +1586,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strsim" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" + [[package]] name = "strsim" version = "0.10.0" @@ -1508,9 +1600,20 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.107" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece519cfaf36269ea69d16c363fa1d59ceba8296bbfbfc003c3176d01f2816ee" dependencies = [ "proc-macro2", "quote", @@ -1519,68 +1622,67 @@ dependencies = [ [[package]] name = "sys-locale" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3358acbb4acd4146138b9bda219e904a6bb5aaaa237f8eed06f4d6bc1580ecee" +checksum = "f8a11bd9c338fdba09f7881ab41551932ad42e405f61d01e8406baea71c07aee" dependencies = [ "js-sys", "libc", "wasm-bindgen", "web-sys", - "winapi", + "windows-sys 0.45.0", ] [[package]] name = "tempfile" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95" dependencies = [ "cfg-if", "fastrand", - "libc", "redox_syscall", - "remove_dir_all", - "winapi", + "rustix", + "windows-sys 0.42.0", ] [[package]] name = "termcolor" -version = "1.1.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" dependencies = [ "winapi-util", ] [[package]] name = "terminal_size" -version = "0.2.3" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb20089a8ba2b69debd491f8d2d023761cbf196e999218c591fa1e7e15a21907" +checksum = "4c9afddd2cec1c0909f06b00ef33f94ab2cc0578c4a610aa208ddfec8aa2b43a" dependencies = [ "rustix", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] name = "thiserror" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.6", ] [[package]] @@ -1596,11 +1698,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.17" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" +checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" dependencies = [ - "itoa 1.0.5", + "itoa", "serde", "time-core", "time-macros", @@ -1614,9 +1716,9 @@ checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" [[package]] name = "time-macros" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" +checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" dependencies = [ "time-core", ] @@ -1632,15 +1734,15 @@ dependencies = [ [[package]] name = "tinyvec_macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.24.1" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae" +checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" dependencies = [ "autocfg", "bytes", @@ -1651,7 +1753,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -1662,14 +1764,14 @@ checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "tokio-native-tls" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", "tokio", @@ -1688,9 +1790,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.4" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" dependencies = [ "bytes", "futures-core", @@ -1740,15 +1842,15 @@ checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "unicode-bidi" -version = "0.3.8" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" [[package]] name = "unicode-normalization" @@ -1818,9 +1920,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1828,24 +1930,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" dependencies = [ "cfg-if", "js-sys", @@ -1855,9 +1957,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1865,28 +1967,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" [[package]] name = "web-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" dependencies = [ "js-sys", "wasm-bindgen", @@ -1942,6 +2044,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.42.0" @@ -1958,46 +2069,70 @@ dependencies = [ ] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.0" +name = "windows-sys" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_msvc" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_i686_gnu" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_msvc" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_x86_64_gnu" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_msvc" -version = "0.42.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "winreg" @@ -2007,3 +2142,12 @@ checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ "winapi", ] + +[[package]] +name = "xattr" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea263437ca03c1522846a4ddafbca2542d0ad5ed9b784909d4b27b76f62bc34a" +dependencies = [ + "libc", +] diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index e56cf71..f86a98f 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -7,12 +7,12 @@ edition = "2021" [dependencies] anyhow = "1.0" async-trait = "0.1" -clap = { version = "4.0", features = ["derive", "string"] } +clap = { version = "4.1", features = ["derive", "string"] } chrono = "0.4" -crunchyroll-rs = "0.2" -csv = "1.1" +crunchyroll-rs = { version = "0.3", features = ["dash-stream"] } ctrlc = "3.2" -dirs = "4.0" +dirs = "5.0" +derive_setters = "0.1" indicatif = "0.17" lazy_static = "1.4" log = { version = "0.4", features = ["std"] } @@ -23,9 +23,9 @@ serde = "1.0" serde_json = "1.0" shlex = "1.1" signal-hook = "0.3" -tempfile = "3.3" +tempfile = "3.4" terminal_size = "0.2" -tokio = { version = "1.24", features = ["macros", "rt-multi-thread", "time"] } +tokio = { version = "1.26", features = ["macros", "rt-multi-thread", "time"] } sys-locale = "0.2" [build-dependencies] diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs new file mode 100644 index 0000000..3583996 --- /dev/null +++ b/crunchy-cli-core/src/archive/command.rs @@ -0,0 +1,184 @@ +use crate::archive::filter::ArchiveFilter; +use crate::utils::context::Context; +use crate::utils::download::MergeBehavior; +use crate::utils::ffmpeg::FFmpegPreset; +use crate::utils::filter::Filter; +use crate::utils::format::formats_visual_output; +use crate::utils::locale::all_locale_in_locales; +use crate::utils::log::progress; +use crate::utils::os::{free_file, has_ffmpeg, is_special_file}; +use crate::utils::parse::parse_url; +use crate::Execute; +use anyhow::bail; +use anyhow::Result; +use crunchyroll_rs::media::Resolution; +use crunchyroll_rs::Locale; +use log::debug; +use std::path::PathBuf; + +#[derive(Clone, Debug, clap::Parser)] +#[clap(about = "Archive a video")] +#[command(arg_required_else_help(true))] +#[command()] +pub struct Archive { + #[arg(help = format!("Audio languages. Can be used multiple times. \ + Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] + #[arg(long_help = format!("Audio languages. Can be used multiple times. \ + Available languages are:\n{}", Locale::all().into_iter().map(|l| format!("{:<6} → {}", l.to_string(), l.to_human_readable())).collect::>().join("\n ")))] + #[arg(short, long, default_values_t = vec![Locale::ja_JP, crate::utils::locale::system_locale()])] + pub(crate) locale: Vec, + #[arg(help = format!("Subtitle languages. Can be used multiple times. \ + Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] + #[arg(long_help = format!("Subtitle languages. Can be used multiple times. \ + Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] + #[arg(short, long, default_values_t = Locale::all())] + pub(crate) subtitle: Vec, + + #[arg(help = "Name of the output file")] + #[arg(long_help = "Name of the output file.\ + If you use one of the following pattern they will get replaced:\n \ + {title} → Title of the video\n \ + {series_name} → Name of the series\n \ + {season_name} → Name of the season\n \ + {audio} → Audio language of the video\n \ + {resolution} → Resolution of the video\n \ + {season_number} → Number of the season\n \ + {episode_number} → Number of the episode\n \ + {relative_episode_number} → Number of the episode relative to its season\ + {series_id} → ID of the series\n \ + {season_id} → ID of the season\n \ + {episode_id} → ID of the episode")] + #[arg(short, long, default_value = "{title}.mkv")] + pub(crate) output: String, + + #[arg(help = "Video resolution")] + #[arg(long_help = "The video resolution.\ + Can either be specified via the pixels (e.g. 1920x1080), the abbreviation for pixels (e.g. 1080p) or 'common-use' words (e.g. best). \ + Specifying the exact pixels is not recommended, use one of the other options instead. \ + Crunchyroll let you choose the quality with pixel abbreviation on their clients, so you might be already familiar with the available options. \ + The available common-use words are 'best' (choose the best resolution available) and 'worst' (worst resolution available)")] + #[arg(short, long, default_value = "best")] + #[arg(value_parser = crate::utils::clap::clap_parse_resolution)] + pub(crate) resolution: Resolution, + + #[arg( + help = "Sets the behavior of the stream merging. Valid behaviors are 'auto', 'audio' and 'video'" + )] + #[arg( + long_help = "Because of local restrictions (or other reasons) some episodes with different languages does not have the same length (e.g. when some scenes were cut out). \ + With this flag you can set the behavior when handling multiple language. + Valid options are 'audio' (stores one video and all other languages as audio only), 'video' (stores the video + audio for every language) and 'auto' (detects if videos differ in length: if so, behave like 'video' else like 'audio')" + )] + #[arg(short, long, default_value = "auto")] + #[arg(value_parser = MergeBehavior::parse)] + pub(crate) merge: MergeBehavior, + + #[arg(help = format!("Presets for video converting. Can be used multiple times. \ + Available presets: \n {}", FFmpegPreset::available_matches_human_readable().join("\n ")))] + #[arg(long_help = format!("Presets for video converting. Can be used multiple times. \ + Generally used to minify the file size with keeping (nearly) the same quality. \ + It is recommended to only use this if you archive videos with high resolutions since low resolution videos tend to result in a larger file with any of the provided presets. \ + Available presets: \n {}", FFmpegPreset::available_matches_human_readable().join("\n ")))] + #[arg(long)] + #[arg(value_parser = FFmpegPreset::parse)] + pub(crate) ffmpeg_preset: Option, + + #[arg( + help = "Set which subtitle language should be set as default / auto shown when starting a video" + )] + #[arg(long)] + pub(crate) default_subtitle: Option, + + #[arg(help = "Skip files which are already existing")] + #[arg(long, default_value_t = false)] + pub(crate) skip_existing: bool, + + #[arg(help = "Crunchyroll series url(s)")] + pub(crate) urls: Vec, +} + +#[async_trait::async_trait(?Send)] +impl Execute for Archive { + fn pre_check(&mut self) -> Result<()> { + if !has_ffmpeg() { + bail!("FFmpeg is needed to run this command") + } else if PathBuf::from(&self.output) + .extension() + .unwrap_or_default() + .to_string_lossy() + != "mkv" + && !is_special_file(PathBuf::from(&self.output)) + { + bail!("File extension is not '.mkv'. Currently only matroska / '.mkv' files are supported") + } + + self.locale = all_locale_in_locales(self.locale.clone()); + self.subtitle = all_locale_in_locales(self.subtitle.clone()); + + Ok(()) + } + + async fn execute(self, ctx: Context) -> Result<()> { + let mut parsed_urls = vec![]; + + for (i, url) in self.urls.clone().into_iter().enumerate() { + let progress_handler = progress!("Parsing url {}", i + 1); + match parse_url(&ctx.crunchy, url.clone(), true).await { + Ok((media_collection, url_filter)) => { + progress_handler.stop(format!("Parsed url {}", i + 1)); + parsed_urls.push((media_collection, url_filter)) + } + Err(e) => bail!("url {} could not be parsed: {}", url, e), + }; + } + + for (i, (media_collection, url_filter)) in parsed_urls.into_iter().enumerate() { + let progress_handler = progress!("Fetching series details"); + let archive_formats = ArchiveFilter::new(url_filter, self.clone()) + .visit(media_collection) + .await?; + + if archive_formats.is_empty() { + progress_handler.stop(format!("Skipping url {} (no matching videos found)", i + 1)); + continue; + } + progress_handler.stop(format!("Loaded series information for url {}", i + 1)); + + formats_visual_output(archive_formats.iter().map(|(_, f)| f).collect()); + + for (downloader, mut format) in archive_formats { + let formatted_path = format.format_path((&self.output).into(), true); + let (path, changed) = free_file(formatted_path.clone()); + + if changed && self.skip_existing { + debug!( + "Skipping already existing file '{}'", + formatted_path.to_string_lossy() + ); + continue; + } + + format.locales.sort_by(|(a, _), (b, _)| { + self.locale + .iter() + .position(|l| l == a) + .cmp(&self.locale.iter().position(|l| l == b)) + }); + for (_, subtitles) in format.locales.iter_mut() { + subtitles.sort_by(|a, b| { + self.subtitle + .iter() + .position(|l| l == a) + .cmp(&self.subtitle.iter().position(|l| l == b)) + }) + } + + format.visual_output(&path); + + downloader.download(&ctx, &path).await? + } + } + + Ok(()) + } +} diff --git a/crunchy-cli-core/src/archive/filter.rs b/crunchy-cli-core/src/archive/filter.rs new file mode 100644 index 0000000..d0db3d5 --- /dev/null +++ b/crunchy-cli-core/src/archive/filter.rs @@ -0,0 +1,482 @@ +use crate::archive::command::Archive; +use crate::utils::download::{DownloadBuilder, DownloadFormat, Downloader, MergeBehavior}; +use crate::utils::filter::{real_dedup_vec, Filter}; +use crate::utils::format::{Format, SingleFormat}; +use crate::utils::parse::UrlFilter; +use crate::utils::video::variant_data_from_stream; +use anyhow::{bail, Result}; +use chrono::Duration; +use crunchyroll_rs::media::{Subtitle, VariantData}; +use crunchyroll_rs::{Concert, Episode, Locale, Movie, MovieListing, MusicVideo, Season, Series}; +use log::warn; +use std::collections::HashMap; +use std::hash::Hash; + +pub(crate) struct FilterResult { + format: SingleFormat, + video: VariantData, + audio: VariantData, + duration: Duration, + subtitles: Vec, +} + +enum Visited { + Series, + Season, + None, +} + +pub(crate) struct ArchiveFilter { + url_filter: UrlFilter, + archive: Archive, + season_episode_count: HashMap>, + season_subtitles_missing: Vec, + visited: Visited, +} + +impl ArchiveFilter { + pub(crate) fn new(url_filter: UrlFilter, archive: Archive) -> Self { + Self { + url_filter, + archive, + season_episode_count: HashMap::new(), + season_subtitles_missing: vec![], + visited: Visited::None, + } + } +} + +#[async_trait::async_trait] +impl Filter for ArchiveFilter { + type T = Vec; + type Output = (Downloader, Format); + + async fn visit_series(&mut self, series: Series) -> Result> { + // `series.audio_locales` isn't always populated b/c of crunchyrolls api. so check if the + // audio is matching only if the field is populated + if !series.audio_locales.is_empty() { + let missing_audio = missing_locales(&series.audio_locales, &self.archive.locale); + if !missing_audio.is_empty() { + warn!( + "Series {} is not available with {} audio", + series.title, + missing_audio + .into_iter() + .map(|l| l.to_string()) + .collect::>() + .join(", ") + ) + } + let missing_subtitle = + missing_locales(&series.subtitle_locales, &self.archive.subtitle); + if !missing_subtitle.is_empty() { + warn!( + "Series {} is not available with {} subtitles", + series.title, + missing_subtitle + .into_iter() + .map(|l| l.to_string()) + .collect::>() + .join(", ") + ) + } + self.visited = Visited::Series + } + Ok(series.seasons().await?) + } + + async fn visit_season(&mut self, mut season: Season) -> Result> { + if !self.url_filter.is_season_valid(season.season_number) { + return Ok(vec![]); + } + + let mut seasons = season.version(self.archive.locale.clone()).await?; + if self + .archive + .locale + .iter() + .any(|l| season.audio_locales.contains(l)) + { + seasons.insert(0, season.clone()); + } + + if !matches!(self.visited, Visited::Series) { + let mut audio_locales: Vec = seasons + .iter() + .map(|s| s.audio_locales.clone()) + .flatten() + .collect(); + real_dedup_vec(&mut audio_locales); + let missing_audio = missing_locales(&audio_locales, &self.archive.locale); + if !missing_audio.is_empty() { + warn!( + "Season {} is not available with {} audio", + season.season_number, + missing_audio + .into_iter() + .map(|l| l.to_string()) + .collect::>() + .join(", ") + ) + } + + let subtitle_locales: Vec = seasons + .iter() + .map(|s| s.subtitle_locales.clone()) + .flatten() + .collect(); + let missing_subtitle = missing_locales(&subtitle_locales, &self.archive.subtitle); + if !missing_subtitle.is_empty() { + warn!( + "Season {} is not available with {} subtitles", + season.season_number, + missing_subtitle + .into_iter() + .map(|l| l.to_string()) + .collect::>() + .join(", ") + ) + } + self.visited = Visited::Season + } + + let mut episodes = vec![]; + for season in seasons { + episodes.extend(season.episodes().await?) + } + + if Format::has_relative_episodes_fmt(&self.archive.output) { + for episode in episodes.iter() { + self.season_episode_count + .entry(episode.season_number) + .or_insert(vec![]) + .push(episode.id.clone()) + } + } + + Ok(episodes) + } + + async fn visit_episode(&mut self, mut episode: Episode) -> Result> { + if !self + .url_filter + .is_episode_valid(episode.episode_number, episode.season_number) + { + return Ok(None); + } + + let mut episodes = vec![]; + if !matches!(self.visited, Visited::Series) && !matches!(self.visited, Visited::Season) { + episodes.extend(episode.version(self.archive.locale.clone()).await?); + let audio_locales: Vec = + episodes.iter().map(|e| e.audio_locale.clone()).collect(); + let missing_audio = missing_locales(&audio_locales, &self.archive.locale); + if !missing_audio.is_empty() { + warn!( + "Episode {} is not available with {} audio", + episode.episode_number, + missing_audio + .into_iter() + .map(|l| l.to_string()) + .collect::>() + .join(", ") + ) + } + + let mut subtitle_locales: Vec = episodes + .iter() + .map(|e| e.subtitle_locales.clone()) + .flatten() + .collect(); + real_dedup_vec(&mut subtitle_locales); + let missing_subtitles = missing_locales(&subtitle_locales, &self.archive.subtitle); + if !missing_subtitles.is_empty() + && !self + .season_subtitles_missing + .contains(&episode.season_number) + { + warn!( + "Episode {} is not available with {} subtitles", + episode.episode_number, + missing_subtitles + .into_iter() + .map(|l| l.to_string()) + .collect::>() + .join(", ") + ); + self.season_subtitles_missing.push(episode.season_number) + } + } else { + episodes.push(episode.clone()) + } + + let mut formats = vec![]; + for episode in episodes { + let stream = episode.streams().await?; + let (video, audio) = if let Some((video, audio)) = + variant_data_from_stream(&stream, &self.archive.resolution).await? + { + (video, audio) + } else { + bail!( + "Resolution ({}) is not available for episode {} ({}) of {} season {}", + &self.archive.resolution, + episode.episode_number, + episode.title, + episode.series_title, + episode.season_number, + ); + }; + let subtitles: Vec = self + .archive + .subtitle + .iter() + .filter_map(|s| stream.subtitles.get(s).cloned()) + .collect(); + + let relative_episode_number = if Format::has_relative_episodes_fmt(&self.archive.output) + { + if self + .season_episode_count + .get(&episode.season_number) + .is_none() + { + let season_episodes = episode.season().await?.episodes().await?; + self.season_episode_count.insert( + episode.season_number, + season_episodes.into_iter().map(|e| e.id).collect(), + ); + } + let relative_episode_number = self + .season_episode_count + .get(&episode.season_number) + .unwrap() + .iter() + .position(|id| id == &episode.id); + if relative_episode_number.is_none() { + warn!( + "Failed to get relative episode number for episode {} ({}) of {} season {}", + episode.episode_number, + episode.title, + episode.series_title, + episode.season_number, + ) + } + relative_episode_number + } else { + None + }; + + formats.push(FilterResult { + format: SingleFormat::new_from_episode( + &episode, + &video, + subtitles.iter().map(|s| s.locale.clone()).collect(), + relative_episode_number.map(|n| n as u32), + ), + video, + audio, + duration: episode.duration.clone(), + subtitles, + }) + } + + Ok(Some(formats)) + } + + async fn visit_movie_listing(&mut self, movie_listing: MovieListing) -> Result> { + Ok(movie_listing.movies().await?) + } + + async fn visit_movie(&mut self, movie: Movie) -> Result> { + let stream = movie.streams().await?; + let subtitles: Vec<&Subtitle> = self + .archive + .subtitle + .iter() + .filter_map(|l| stream.subtitles.get(l)) + .collect(); + + let missing_subtitles = missing_locales( + &subtitles.iter().map(|&s| s.locale.clone()).collect(), + &self.archive.subtitle, + ); + if !missing_subtitles.is_empty() { + warn!( + "Movie '{}' is not available with {} subtitles", + movie.title, + missing_subtitles + .into_iter() + .map(|l| l.to_string()) + .collect::>() + .join(", ") + ) + } + + let (video, audio) = if let Some((video, audio)) = + variant_data_from_stream(&stream, &self.archive.resolution).await? + { + (video, audio) + } else { + bail!( + "Resolution ({}) of movie {} is not available", + self.archive.resolution, + movie.title + ) + }; + + Ok(Some(vec![FilterResult { + format: SingleFormat::new_from_movie(&movie, &video, vec![]), + video, + audio, + duration: movie.duration, + subtitles: vec![], + }])) + } + + async fn visit_music_video(&mut self, music_video: MusicVideo) -> Result> { + let stream = music_video.streams().await?; + let (video, audio) = if let Some((video, audio)) = + variant_data_from_stream(&stream, &self.archive.resolution).await? + { + (video, audio) + } else { + bail!( + "Resolution ({}) of music video {} is not available", + self.archive.resolution, + music_video.title + ) + }; + + Ok(Some(vec![FilterResult { + format: SingleFormat::new_from_music_video(&music_video, &video), + video, + audio, + duration: music_video.duration, + subtitles: vec![], + }])) + } + + async fn visit_concert(&mut self, concert: Concert) -> Result> { + let stream = concert.streams().await?; + let (video, audio) = if let Some((video, audio)) = + variant_data_from_stream(&stream, &self.archive.resolution).await? + { + (video, audio) + } else { + bail!( + "Resolution ({}x{}) of music video {} is not available", + self.archive.resolution.width, + self.archive.resolution.height, + concert.title + ) + }; + + Ok(Some(vec![FilterResult { + format: SingleFormat::new_from_concert(&concert, &video), + video, + audio, + duration: concert.duration, + subtitles: vec![], + }])) + } + + async fn finish(self, input: Vec) -> Result> { + let flatten_input: Vec = input.into_iter().flatten().collect(); + + #[derive(Hash, Eq, PartialEq)] + struct SortKey { + season: u32, + episode: String, + } + + let mut sorted: HashMap> = HashMap::new(); + for data in flatten_input { + sorted + .entry(SortKey { + season: data.format.season_number, + episode: data.format.episode_number.to_string(), + }) + .or_insert(vec![]) + .push(data) + } + + let mut values: Vec> = sorted.into_values().collect(); + values.sort_by(|a, b| { + a.first() + .unwrap() + .format + .sequence_number + .total_cmp(&b.first().unwrap().format.sequence_number) + }); + + let mut result = vec![]; + for data in values { + let single_formats: Vec = + data.iter().map(|fr| fr.format.clone()).collect(); + let format = Format::from_single_formats(single_formats); + + let mut downloader = DownloadBuilder::new() + .default_subtitle(self.archive.default_subtitle.clone()) + .ffmpeg_preset(self.archive.ffmpeg_preset.clone().unwrap_or_default()) + .output_format(Some("matroska".to_string())) + .audio_sort(Some(self.archive.locale.clone())) + .subtitle_sort(Some(self.archive.subtitle.clone())) + .build(); + + match self.archive.merge.clone() { + MergeBehavior::Video => { + for d in data { + downloader.add_format(DownloadFormat { + video: (d.video, d.format.audio.clone()), + audios: vec![(d.audio, d.format.audio.clone())], + subtitles: d.subtitles, + }) + } + } + MergeBehavior::Audio => downloader.add_format(DownloadFormat { + video: ( + data.first().unwrap().video.clone(), + data.first().unwrap().format.audio.clone(), + ), + audios: data + .iter() + .map(|d| (d.audio.clone(), d.format.audio.clone())) + .collect(), + subtitles: data.iter().map(|d| d.subtitles.clone()).flatten().collect(), + }), + MergeBehavior::Auto => { + let mut download_formats: HashMap = HashMap::new(); + + for d in data { + if let Some(download_format) = download_formats.get_mut(&d.duration) { + download_format.audios.push((d.audio, d.format.audio)); + download_format.subtitles.extend(d.subtitles) + } else { + download_formats.insert( + d.duration, + DownloadFormat { + video: (d.video, d.format.audio.clone()), + audios: vec![(d.audio, d.format.audio)], + subtitles: d.subtitles, + }, + ); + } + } + + for download_format in download_formats.into_values() { + downloader.add_format(download_format) + } + } + } + + result.push((downloader, format)) + } + + Ok(result) + } +} + +fn missing_locales<'a>(available: &Vec, searched: &'a Vec) -> Vec<&'a Locale> { + searched.iter().filter(|p| !available.contains(p)).collect() +} diff --git a/crunchy-cli-core/src/archive/mod.rs b/crunchy-cli-core/src/archive/mod.rs new file mode 100644 index 0000000..c3544a4 --- /dev/null +++ b/crunchy-cli-core/src/archive/mod.rs @@ -0,0 +1,4 @@ +mod command; +mod filter; + +pub use command::Archive; diff --git a/crunchy-cli-core/src/cli/archive.rs b/crunchy-cli-core/src/cli/archive.rs deleted file mode 100644 index e500f21..0000000 --- a/crunchy-cli-core/src/cli/archive.rs +++ /dev/null @@ -1,618 +0,0 @@ -use crate::cli::log::tab_info; -use crate::cli::utils::{ - all_locale_in_locales, download_segments, find_multiple_seasons_with_same_number, - find_resolution, interactive_season_choosing, FFmpegPreset, -}; -use crate::utils::context::Context; -use crate::utils::format::Format; -use crate::utils::log::progress; -use crate::utils::os::{free_file, has_ffmpeg, is_special_file, tempfile}; -use crate::utils::parse::{parse_url, UrlFilter}; -use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number}; -use crate::utils::subtitle::{download_subtitle, Subtitle}; -use crate::utils::video::get_video_length; -use crate::Execute; -use anyhow::{bail, Result}; -use crunchyroll_rs::media::Resolution; -use crunchyroll_rs::{Locale, Media, MediaCollection, Series}; -use log::{debug, error, info}; -use std::path::PathBuf; -use std::process::{Command, Stdio}; -use tempfile::TempPath; - -#[derive(Clone, Debug)] -pub enum MergeBehavior { - Auto, - Audio, - Video, -} - -impl MergeBehavior { - fn parse(s: &str) -> Result { - Ok(match s.to_lowercase().as_str() { - "auto" => MergeBehavior::Auto, - "audio" => MergeBehavior::Audio, - "video" => MergeBehavior::Video, - _ => return Err(format!("'{}' is not a valid merge behavior", s)), - }) - } -} - -#[derive(Debug, clap::Parser)] -#[clap(about = "Archive a video")] -#[command(arg_required_else_help(true))] -#[command()] -pub struct Archive { - #[arg(help = format!("Audio languages. Can be used multiple times. \ - Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] - #[arg(long_help = format!("Audio languages. Can be used multiple times. \ - Available languages are:\n{}", Locale::all().into_iter().map(|l| format!("{:<6} → {}", l.to_string(), l.to_human_readable())).collect::>().join("\n ")))] - #[arg(short, long, default_values_t = vec![crate::utils::locale::system_locale(), Locale::ja_JP])] - locale: Vec, - #[arg(help = format!("Subtitle languages. Can be used multiple times. \ - Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] - #[arg(long_help = format!("Subtitle languages. Can be used multiple times. \ - Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] - #[arg(short, long, default_values_t = Locale::all())] - subtitle: Vec, - - #[arg(help = "Name of the output file")] - #[arg(long_help = "Name of the output file.\ - If you use one of the following pattern they will get replaced:\n \ - {title} → Title of the video\n \ - {series_name} → Name of the series\n \ - {season_name} → Name of the season\n \ - {audio} → Audio language of the video\n \ - {resolution} → Resolution of the video\n \ - {season_number} → Number of the season\n \ - {episode_number} → Number of the episode\n \ - {relative_episode_number} → Number of the episode relative to its season\ - {series_id} → ID of the series\n \ - {season_id} → ID of the season\n \ - {episode_id} → ID of the episode")] - #[arg(short, long, default_value = "{title}.mkv")] - output: String, - - #[arg(help = "Video resolution")] - #[arg(long_help = "The video resolution.\ - Can either be specified via the pixels (e.g. 1920x1080), the abbreviation for pixels (e.g. 1080p) or 'common-use' words (e.g. best). \ - Specifying the exact pixels is not recommended, use one of the other options instead. \ - Crunchyroll let you choose the quality with pixel abbreviation on their clients, so you might be already familiar with the available options. \ - The available common-use words are 'best' (choose the best resolution available) and 'worst' (worst resolution available)")] - #[arg(short, long, default_value = "best")] - #[arg(value_parser = crate::utils::clap::clap_parse_resolution)] - resolution: Resolution, - - #[arg( - help = "Sets the behavior of the stream merging. Valid behaviors are 'auto', 'audio' and 'video'" - )] - #[arg( - long_help = "Because of local restrictions (or other reasons) some episodes with different languages does not have the same length (e.g. when some scenes were cut out). \ - With this flag you can set the behavior when handling multiple language. - Valid options are 'audio' (stores one video and all other languages as audio only), 'video' (stores the video + audio for every language) and 'auto' (detects if videos differ in length: if so, behave like 'video' else like 'audio')" - )] - #[arg(short, long, default_value = "auto")] - #[arg(value_parser = MergeBehavior::parse)] - merge: MergeBehavior, - - #[arg(help = format!("Presets for video converting. Can be used multiple times. \ - Available presets: \n {}", FFmpegPreset::available_matches_human_readable().join("\n ")))] - #[arg(long_help = format!("Presets for video converting. Can be used multiple times. \ - Generally used to minify the file size with keeping (nearly) the same quality. \ - It is recommended to only use this if you archive videos with high resolutions since low resolution videos tend to result in a larger file with any of the provided presets. \ - Available presets: \n {}", FFmpegPreset::available_matches_human_readable().join("\n ")))] - #[arg(long)] - #[arg(value_parser = FFmpegPreset::parse)] - ffmpeg_preset: Option, - - #[arg( - help = "Set which subtitle language should be set as default / auto shown when starting a video" - )] - #[arg(long)] - default_subtitle: Option, - - #[arg(help = "Skip files which are already existing")] - #[arg(long, default_value_t = false)] - skip_existing: bool, - - #[arg(help = "Ignore interactive input")] - #[arg(short, long, default_value_t = false)] - yes: bool, - - #[arg(help = "Crunchyroll series url(s)")] - urls: Vec, -} - -#[async_trait::async_trait(?Send)] -impl Execute for Archive { - fn pre_check(&mut self) -> Result<()> { - if !has_ffmpeg() { - bail!("FFmpeg is needed to run this command") - } else if PathBuf::from(&self.output) - .extension() - .unwrap_or_default() - .to_string_lossy() - != "mkv" - && !is_special_file(PathBuf::from(&self.output)) - { - bail!("File extension is not '.mkv'. Currently only matroska / '.mkv' files are supported") - } - - self.locale = all_locale_in_locales(self.locale.clone()); - self.subtitle = all_locale_in_locales(self.subtitle.clone()); - - Ok(()) - } - - async fn execute(self, ctx: Context) -> Result<()> { - let mut parsed_urls = vec![]; - - for (i, url) in self.urls.iter().enumerate() { - let progress_handler = progress!("Parsing url {}", i + 1); - match parse_url(&ctx.crunchy, url.clone(), true).await { - Ok((media_collection, url_filter)) => { - parsed_urls.push((media_collection, url_filter)); - progress_handler.stop(format!("Parsed url {}", i + 1)) - } - Err(e) => bail!("url {} could not be parsed: {}", url, e), - } - } - - for (i, (media_collection, url_filter)) in parsed_urls.into_iter().enumerate() { - let progress_handler = progress!("Fetching series details"); - let archive_formats = match media_collection { - MediaCollection::Series(series) => { - formats_from_series(&self, series, &url_filter).await? - } - MediaCollection::Season(_) => bail!("Archiving a season is not supported"), - MediaCollection::Episode(episode) => bail!("Archiving a episode is not supported. Use url filtering instead to specify the episode (https://www.crunchyroll.com/series/{}/{}[S{}E{}])", episode.metadata.series_id, episode.metadata.series_slug_title, episode.metadata.season_number, episode.metadata.episode_number), - MediaCollection::MovieListing(_) => bail!("Archiving a movie listing is not supported"), - MediaCollection::Movie(_) => bail!("Archiving a movie is not supported") - }; - - if archive_formats.is_empty() { - progress_handler.stop(format!( - "Skipping url {} (no matching episodes found)", - i + 1 - )); - continue; - } - progress_handler.stop(format!("Loaded series information for url {}", i + 1)); - - if log::max_level() == log::Level::Debug { - let seasons = sort_formats_after_seasons( - archive_formats - .clone() - .into_iter() - .map(|(a, _)| a.get(0).unwrap().clone()) - .collect(), - ); - debug!("Series has {} seasons", seasons.len()); - for (i, season) in seasons.into_iter().enumerate() { - info!("Season {} ({})", i + 1, season.get(0).unwrap().season_title); - for format in season { - info!( - "{}: {}px, {:.02} FPS (S{:02}E{:02})", - format.title, - format.stream.resolution, - format.stream.fps, - format.season_number, - format.episode_number, - ) - } - } - } else { - for season in sort_formats_after_seasons( - archive_formats - .clone() - .into_iter() - .map(|(a, _)| a.get(0).unwrap().clone()) - .collect(), - ) { - let first = season.get(0).unwrap(); - info!( - "{} Season {} ({})", - first.series_name, first.season_number, first.season_title - ); - - for (i, format) in season.into_iter().enumerate() { - tab_info!( - "{}. {} Âģ {}px, {:.2} FPS (S{:02}E{:02})", - i + 1, - format.title, - format.stream.resolution, - format.stream.fps, - format.season_number, - format.episode_number - ) - } - } - } - - for (formats, mut subtitles) in archive_formats { - let (primary, additionally) = formats.split_first().unwrap(); - - let formatted_path = primary.format_path((&self.output).into(), true); - let (path, changed) = free_file(formatted_path.clone()); - - if changed && self.skip_existing { - debug!( - "Skipping already existing file '{}'", - formatted_path.to_string_lossy() - ); - continue; - } - - info!( - "Downloading {} to '{}'", - primary.title, - if is_special_file(&path) { - path.to_str().unwrap() - } else { - path.file_name().unwrap().to_str().unwrap() - } - ); - tab_info!( - "Episode: S{:02}E{:02}", - primary.season_number, - primary.episode_number - ); - tab_info!( - "Audio: {} (primary), {}", - primary.audio, - additionally - .iter() - .map(|a| a.audio.to_string()) - .collect::>() - .join(", ") - ); - tab_info!( - "Subtitle: {}", - subtitles - .iter() - .filter(|s| s.primary) // Don't print subtitles of non-primary streams. They might get removed depending on the merge behavior. - .map(|s| { - if let Some(default) = &self.default_subtitle { - if default == &s.stream_subtitle.locale { - return format!("{} (primary)", default); - } - } - s.stream_subtitle.locale.to_string() - }) - .collect::>() - .join(", ") - ); - tab_info!("Resolution: {}", primary.stream.resolution); - tab_info!("FPS: {:.2}", primary.stream.fps); - - let mut video_paths = vec![]; - let mut audio_paths = vec![]; - let mut subtitle_paths = vec![]; - - video_paths.push((download_video(&ctx, primary, false).await?, primary)); - for additional in additionally { - let identical_video = additionally - .iter() - .all(|a| a.stream.bandwidth == primary.stream.bandwidth); - let only_audio = match self.merge { - MergeBehavior::Auto => identical_video, - MergeBehavior::Audio => true, - MergeBehavior::Video => false, - }; - let path = download_video(&ctx, additional, only_audio).await?; - if only_audio { - audio_paths.push((path, additional)) - } else { - video_paths.push((path, additional)) - } - - // Remove subtitles of forcibly deleted video - if matches!(self.merge, MergeBehavior::Audio) && !identical_video { - subtitles.retain(|s| s.episode_id != additional.episode_id); - } - } - - let (primary_video, _) = video_paths.get(0).unwrap(); - let primary_video_length = get_video_length(primary_video.to_path_buf()).unwrap(); - for subtitle in subtitles { - subtitle_paths.push(( - download_subtitle(subtitle.stream_subtitle.clone(), primary_video_length) - .await?, - subtitle, - )) - } - - let progess_handler = progress!("Generating mkv"); - generate_mkv(&self, path, video_paths, audio_paths, subtitle_paths)?; - progess_handler.stop("Mkv generated") - } - } - - Ok(()) - } -} - -async fn formats_from_series( - archive: &Archive, - series: Media, - url_filter: &UrlFilter, -) -> Result, Vec)>> { - let mut seasons = series.seasons().await?; - - // filter any season out which does not contain the specified audio languages - for season in sort_seasons_after_number(seasons.clone()) { - // get all locales which are specified but not present in the current iterated season and - // print an error saying this - let not_present_audio = archive - .locale - .clone() - .into_iter() - .filter(|l| !season.iter().any(|s| s.metadata.audio_locales.contains(l))) - .collect::>(); - if !not_present_audio.is_empty() { - error!( - "Season {} of series {} is not available with {} audio", - season.first().unwrap().metadata.season_number, - series.title, - not_present_audio - .into_iter() - .map(|l| l.to_string()) - .collect::>() - .join(", ") - ) - } - - // remove all seasons with the wrong audio for the current iterated season number - seasons.retain(|s| { - s.metadata.season_number != season.first().unwrap().metadata.season_number - || archive - .locale - .iter() - .any(|l| s.metadata.audio_locales.contains(l)) - }); - // remove seasons which match the url filter. this is mostly done to not trigger the - // interactive season choosing when dupilcated seasons are excluded by the filter - seasons.retain(|s| url_filter.is_season_valid(s.metadata.season_number)) - } - - if !archive.yes && !find_multiple_seasons_with_same_number(&seasons).is_empty() { - info!(target: "progress_end", "Fetched seasons"); - seasons = interactive_season_choosing(seasons); - info!(target: "progress", "Fetching series details") - } - - #[allow(clippy::type_complexity)] - let mut result: Vec<(Vec, Vec)> = Vec::new(); - let mut primary_season = true; - for season in seasons { - let episodes = season.episodes().await?; - - for episode in episodes.iter() { - if !url_filter.is_episode_valid( - episode.metadata.episode_number, - episode.metadata.season_number, - ) { - continue; - } - - let streams = episode.streams().await?; - let streaming_data = streams.hls_streaming_data(None).await?; - let Some(stream) = find_resolution(streaming_data, &archive.resolution) else { - bail!( - "Resolution ({}x{}) is not available for episode {} ({}) of season {} ({}) of {}", - archive.resolution.width, - archive.resolution.height, - episode.metadata.episode_number, - episode.title, - episode.metadata.season_number, - episode.metadata.season_title, - episode.metadata.series_title - ) - }; - - let mut formats: Vec = Vec::new(); - let mut subtitles: Vec = Vec::new(); - subtitles.extend(archive.subtitle.iter().filter_map(|l| { - let stream_subtitle = streams.subtitles.get(l).cloned()?; - let subtitle = Subtitle { - stream_subtitle, - audio_locale: episode.metadata.audio_locale.clone(), - episode_id: episode.id.clone(), - forced: !episode.metadata.is_subbed, - primary: primary_season, - }; - Some(subtitle) - })); - formats.push(Format::new_from_episode(episode, &episodes, stream, vec![])); - - result.push((formats, subtitles)); - } - - primary_season = false; - } - - Ok(result) -} - -async fn download_video(ctx: &Context, format: &Format, only_audio: bool) -> Result { - let tempfile = if only_audio { - tempfile(".aac")? - } else { - tempfile(".ts")? - }; - let (_, path) = tempfile.into_parts(); - - let ffmpeg = Command::new("ffmpeg") - .stdin(Stdio::piped()) - .stdout(Stdio::null()) - .stderr(Stdio::piped()) - .arg("-y") - .args(["-f", "mpegts"]) - .args(["-i", "pipe:"]) - .args(["-c", "copy"]) - .args(if only_audio { vec!["-vn"] } else { vec![] }) - .arg(path.to_str().unwrap()) - .spawn()?; - - download_segments( - ctx, - &mut ffmpeg.stdin.unwrap(), - Some(format!("Download {}", format.audio)), - format.stream.clone(), - ) - .await?; - - Ok(path) -} - -fn generate_mkv( - archive: &Archive, - target: PathBuf, - video_paths: Vec<(TempPath, &Format)>, - audio_paths: Vec<(TempPath, &Format)>, - subtitle_paths: Vec<(TempPath, Subtitle)>, -) -> Result<()> { - let mut input = vec![]; - let mut maps = vec![]; - let mut metadata = vec![]; - let mut dispositions = vec![vec![]; subtitle_paths.len()]; - - for (i, (video_path, format)) in video_paths.iter().enumerate() { - input.extend(["-i".to_string(), video_path.to_string_lossy().to_string()]); - maps.extend(["-map".to_string(), i.to_string()]); - metadata.extend([ - format!("-metadata:s:v:{}", i), - format!("language={}", format.audio), - ]); - metadata.extend([ - format!("-metadata:s:v:{}", i), - format!("title={}", format.audio.to_human_readable()), - ]); - metadata.extend([ - format!("-metadata:s:a:{}", i), - format!("language={}", format.audio), - ]); - metadata.extend([ - format!("-metadata:s:a:{}", i), - format!("title={}", format.audio.to_human_readable()), - ]); - } - for (i, (audio_path, format)) in audio_paths.iter().enumerate() { - input.extend(["-i".to_string(), audio_path.to_string_lossy().to_string()]); - maps.extend(["-map".to_string(), (i + video_paths.len()).to_string()]); - metadata.extend([ - format!("-metadata:s:a:{}", i + video_paths.len()), - format!("language={}", format.audio), - ]); - metadata.extend([ - format!("-metadata:s:a:{}", i + video_paths.len()), - format!("title={}", format.audio.to_human_readable()), - ]); - } - for (i, (subtitle_path, subtitle)) in subtitle_paths.iter().enumerate() { - input.extend([ - "-i".to_string(), - subtitle_path.to_string_lossy().to_string(), - ]); - maps.extend([ - "-map".to_string(), - (i + video_paths.len() + audio_paths.len()).to_string(), - ]); - metadata.extend([ - format!("-metadata:s:s:{}", i), - format!("language={}", subtitle.stream_subtitle.locale), - ]); - metadata.extend([ - format!("-metadata:s:s:{}", i), - format!( - "title={}", - subtitle.stream_subtitle.locale.to_human_readable() - + if !subtitle.primary { - format!(" [Video: {}]", subtitle.audio_locale.to_human_readable()) - } else { - "".to_string() - } - .as_str() - ), - ]); - - // mark forced subtitles - if subtitle.forced { - dispositions[i].push("forced"); - } - } - - let (input_presets, output_presets) = if let Some(preset) = archive.ffmpeg_preset.clone() { - preset.to_input_output_args() - } else { - ( - vec![], - vec![ - "-c:v".to_string(), - "copy".to_string(), - "-c:a".to_string(), - "copy".to_string(), - ], - ) - }; - - let mut command_args = vec!["-y".to_string()]; - command_args.extend(input_presets); - command_args.extend(input); - command_args.extend(maps); - command_args.extend(metadata); - - // set default subtitle - if let Some(default_subtitle) = &archive.default_subtitle { - // if `--default_subtitle ` is given set the default subtitle to the given locale - if let Some(position) = subtitle_paths - .iter() - .position(|(_, subtitle)| &subtitle.stream_subtitle.locale == default_subtitle) - { - dispositions[position].push("default"); - } - } - - let disposition_args: Vec = dispositions - .iter() - .enumerate() - .flat_map(|(i, d)| { - vec![ - format!("-disposition:s:{}", i), - if !d.is_empty() { - d.join("+") - } else { - "0".to_string() - }, - ] - }) - .collect(); - command_args.extend(disposition_args); - - command_args.extend(output_presets); - command_args.extend([ - "-f".to_string(), - "matroska".to_string(), - target.to_string_lossy().to_string(), - ]); - - debug!("ffmpeg {}", command_args.join(" ")); - - // create parent directory if it does not exist - if let Some(parent) = target.parent() { - if !parent.exists() { - std::fs::create_dir_all(parent)? - } - } - - let ffmpeg = Command::new("ffmpeg") - .stdout(Stdio::null()) - .stderr(Stdio::piped()) - .args(command_args) - .output()?; - if !ffmpeg.status.success() { - bail!("{}", String::from_utf8_lossy(ffmpeg.stderr.as_slice())) - } - - Ok(()) -} diff --git a/crunchy-cli-core/src/cli/download.rs b/crunchy-cli-core/src/cli/download.rs deleted file mode 100644 index b4c037c..0000000 --- a/crunchy-cli-core/src/cli/download.rs +++ /dev/null @@ -1,603 +0,0 @@ -use crate::cli::log::tab_info; -use crate::cli::utils::{ - download_segments, find_multiple_seasons_with_same_number, find_resolution, - interactive_season_choosing, FFmpegPreset, -}; -use crate::utils::context::Context; -use crate::utils::format::Format; -use crate::utils::log::progress; -use crate::utils::os::{free_file, has_ffmpeg, is_special_file, tempfile}; -use crate::utils::parse::{parse_url, UrlFilter}; -use crate::utils::sort::{sort_formats_after_seasons, sort_seasons_after_number}; -use crate::utils::subtitle::download_subtitle; -use crate::utils::video::get_video_length; -use crate::Execute; -use anyhow::{bail, Result}; -use crunchyroll_rs::media::{Resolution, StreamSubtitle, VariantData}; -use crunchyroll_rs::{ - Episode, Locale, Media, MediaCollection, Movie, MovieListing, Season, Series, -}; -use log::{debug, error, info, warn}; -use std::borrow::Cow; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; - -#[derive(Debug, clap::Parser)] -#[clap(about = "Download a video")] -#[command(arg_required_else_help(true))] -pub struct Download { - #[arg(help = format!("Audio language. Can only be used if the provided url(s) point to a series. \ - Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] - #[arg(long_help = format!("Audio language. Can only be used if the provided url(s) point to a series. \ - Available languages are:\n{}", Locale::all().into_iter().map(|l| format!("{:<6} → {}", l.to_string(), l.to_human_readable())).collect::>().join("\n ")))] - #[arg(short, long, default_value_t = crate::utils::locale::system_locale())] - audio: Locale, - #[arg(help = format!("Subtitle language. Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] - #[arg(long_help = format!("Subtitle language. If set, the subtitle will be burned into the video and cannot be disabled. \ - Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] - #[arg(short, long)] - subtitle: Option, - - #[arg(help = "Name of the output file")] - #[arg(long_help = "Name of the output file.\ - If you use one of the following pattern they will get replaced:\n \ - {title} → Title of the video\n \ - {series_name} → Name of the series\n \ - {season_name} → Name of the season\n \ - {audio} → Audio language of the video\n \ - {resolution} → Resolution of the video\n \ - {season_number} → Number of the season\n \ - {episode_number} → Number of the episode\n \ - {relative_episode_number} → Number of the episode relative to its season\ - {series_id} → ID of the series\n \ - {season_id} → ID of the season\n \ - {episode_id} → ID of the episode")] - #[arg(short, long, default_value = "{title}.mp4")] - output: String, - - #[arg(help = "Video resolution")] - #[arg(long_help = "The video resolution.\ - Can either be specified via the pixels (e.g. 1920x1080), the abbreviation for pixels (e.g. 1080p) or 'common-use' words (e.g. best). \ - Specifying the exact pixels is not recommended, use one of the other options instead. \ - Crunchyroll let you choose the quality with pixel abbreviation on their clients, so you might be already familiar with the available options. \ - The available common-use words are 'best' (choose the best resolution available) and 'worst' (worst resolution available)")] - #[arg(short, long, default_value = "best")] - #[arg(value_parser = crate::utils::clap::clap_parse_resolution)] - resolution: Resolution, - - #[arg(help = format!("Presets for video converting. Can be used multiple times. \ - Available presets: \n {}", FFmpegPreset::available_matches_human_readable().join("\n ")))] - #[arg(long_help = format!("Presets for video converting. Can be used multiple times. \ - Generally used to minify the file size with keeping (nearly) the same quality. \ - It is recommended to only use this if you download videos with high resolutions since low resolution videos tend to result in a larger file with any of the provided presets. \ - Available presets: \n {}", FFmpegPreset::available_matches_human_readable().join("\n ")))] - #[arg(long)] - #[arg(value_parser = FFmpegPreset::parse)] - ffmpeg_preset: Option, - - #[arg(help = "Skip files which are already existing")] - #[arg(long, default_value_t = false)] - skip_existing: bool, - - #[arg(help = "Ignore interactive input")] - #[arg(short, long, default_value_t = false)] - yes: bool, - - #[arg(help = "Url(s) to Crunchyroll episodes or series")] - urls: Vec, -} - -#[async_trait::async_trait(?Send)] -impl Execute for Download { - fn pre_check(&mut self) -> Result<()> { - if !has_ffmpeg() { - bail!("FFmpeg is needed to run this command") - } else if Path::new(&self.output) - .extension() - .unwrap_or_default() - .is_empty() - && self.output != "-" - { - bail!("No file extension found. Please specify a file extension (via `-o`) for the output file") - } - - if self.subtitle.is_some() { - if let Some(ext) = Path::new(&self.output).extension() { - if ext.to_string_lossy() != "mp4" { - warn!("Detected a non mp4 output container. Adding subtitles may take a while") - } - } - } - - Ok(()) - } - - async fn execute(self, ctx: Context) -> Result<()> { - let mut parsed_urls = vec![]; - - for (i, url) in self.urls.iter().enumerate() { - let progress_handler = progress!("Parsing url {}", i + 1); - match parse_url(&ctx.crunchy, url.clone(), true).await { - Ok((media_collection, url_filter)) => { - parsed_urls.push((media_collection, url_filter)); - progress_handler.stop(format!("Parsed url {}", i + 1)) - } - Err(e) => bail!("url {} could not be parsed: {}", url, e), - } - } - - for (i, (media_collection, url_filter)) in parsed_urls.into_iter().enumerate() { - let progress_handler = progress!("Fetching series details"); - let formats = match media_collection { - MediaCollection::Series(series) => { - debug!("Url {} is series ({})", i + 1, series.title); - formats_from_series(&self, series, &url_filter).await? - } - MediaCollection::Season(season) => { - debug!( - "Url {} is season {} ({})", - i + 1, - season.metadata.season_number, - season.title - ); - formats_from_season(&self, season, &url_filter).await? - } - MediaCollection::Episode(episode) => { - debug!( - "Url {} is episode {} ({}) of season {} ({}) of {}", - i + 1, - episode.metadata.episode_number, - episode.title, - episode.metadata.season_number, - episode.metadata.season_title, - episode.metadata.series_title - ); - format_from_episode(&self, &episode, &url_filter, None, false) - .await? - .map(|fmt| vec![fmt]) - } - MediaCollection::MovieListing(movie_listing) => { - debug!("Url {} is movie listing ({})", i + 1, movie_listing.title); - format_from_movie_listing(&self, movie_listing, &url_filter).await? - } - MediaCollection::Movie(movie) => { - debug!("Url {} is movie ({})", i + 1, movie.title); - format_from_movie(&self, movie, &url_filter) - .await? - .map(|fmt| vec![fmt]) - } - }; - - let Some(formats) = formats else { - progress_handler.stop(format!("Skipping url {} (no matching episodes found)", i + 1)); - continue; - }; - progress_handler.stop(format!("Loaded series information for url {}", i + 1)); - - if log::max_level() == log::Level::Debug { - let seasons = sort_formats_after_seasons(formats.clone()); - debug!("Series has {} seasons", seasons.len()); - for (i, season) in seasons.into_iter().enumerate() { - info!("Season {} ({})", i + 1, season.get(0).unwrap().season_title); - for format in season { - info!( - "{}: {}px, {:.02} FPS (S{:02}E{:02})", - format.title, - format.stream.resolution, - format.stream.fps, - format.season_number, - format.episode_number, - ) - } - } - } else { - for season in sort_formats_after_seasons(formats.clone()) { - let first = season.get(0).unwrap(); - info!( - "{} Season {} ({})", - first.series_name, first.season_number, first.season_title - ); - - for (i, format) in season.into_iter().enumerate() { - tab_info!( - "{}. {} Âģ {}px, {:.2} FPS (S{:02}E{:02})", - i + 1, - format.title, - format.stream.resolution, - format.stream.fps, - format.season_number, - format.episode_number - ) - } - } - } - - for format in formats { - let formatted_path = format.format_path((&self.output).into(), true); - let (path, changed) = free_file(formatted_path.clone()); - - if changed && self.skip_existing { - debug!( - "Skipping already existing file '{}'", - formatted_path.to_string_lossy() - ); - continue; - } - - info!( - "Downloading {} to '{}'", - format.title, - if is_special_file(&path) { - path.to_str().unwrap() - } else { - path.file_name().unwrap().to_str().unwrap() - } - ); - tab_info!( - "Episode: S{:02}E{:02}", - format.season_number, - format.episode_number - ); - tab_info!("Audio: {}", format.audio); - tab_info!( - "Subtitles: {}", - self.subtitle - .clone() - .map_or("None".to_string(), |l| l.to_string()) - ); - tab_info!("Resolution: {}", format.stream.resolution); - tab_info!("FPS: {:.2}", format.stream.fps); - - download_ffmpeg( - &ctx, - &self, - format.stream, - format.subtitles.get(0).cloned(), - path.to_path_buf(), - ) - .await?; - } - } - - Ok(()) - } -} - -async fn download_ffmpeg( - ctx: &Context, - download: &Download, - variant_data: VariantData, - subtitle: Option, - mut target: PathBuf, -) -> Result<()> { - let (input_presets, mut output_presets) = if let Some(preset) = download.ffmpeg_preset.clone() { - preset.to_input_output_args() - } else { - ( - vec![], - vec![ - "-c:v".to_string(), - "copy".to_string(), - "-c:a".to_string(), - "copy".to_string(), - ], - ) - }; - - // create parent directory if it does not exist - if let Some(parent) = target.parent() { - if !parent.exists() { - std::fs::create_dir_all(parent)? - } - } - - let mut video_file = tempfile(".ts")?; - download_segments(ctx, &mut video_file, None, variant_data).await?; - let subtitle_file = if let Some(ref sub) = subtitle { - let video_len = get_video_length(video_file.path().to_path_buf())?; - Some(download_subtitle(sub.clone(), video_len).await?) - } else { - None - }; - - let stdout_tempfile = if target.to_string_lossy() == "-" { - let file = tempfile(".mp4")?; - target = file.path().to_path_buf(); - - Some(file) - } else { - None - }; - - let subtitle_presets = if let Some(sub_file) = &subtitle_file { - if target.extension().unwrap_or_default().to_string_lossy() == "mp4" { - vec![ - "-i".to_string(), - sub_file.to_string_lossy().to_string(), - "-movflags".to_string(), - "faststart".to_string(), - "-c:s".to_string(), - "mov_text".to_string(), - "-disposition:s:s:0".to_string(), - "forced".to_string(), - ] - } else { - // remove '-c:v copy' and '-c:a copy' from output presets as its causes issues with - // burning subs into the video - let mut last = String::new(); - let mut remove_count = 0; - for (i, s) in output_presets.clone().iter().enumerate() { - if (last == "-c:v" || last == "-c:a") && s == "copy" { - // remove last - output_presets.remove(i - remove_count - 1); - remove_count += 1; - output_presets.remove(i - remove_count); - remove_count += 1; - } - last = s.clone(); - } - - vec![ - "-vf".to_string(), - format!("subtitles={}", sub_file.to_string_lossy()), - ] - } - } else { - vec![] - }; - - let mut ffmpeg = Command::new("ffmpeg") - .stdout(Stdio::null()) - .stderr(Stdio::piped()) - .arg("-y") - .args(input_presets) - .args(["-i", video_file.path().to_string_lossy().as_ref()]) - .args(subtitle_presets) - .args(output_presets) - .arg(target.to_str().unwrap()) - .spawn()?; - - let progress_handler = progress!("Generating output file"); - if !ffmpeg.wait()?.success() { - bail!("{}", std::io::read_to_string(ffmpeg.stderr.unwrap())?) - } - progress_handler.stop("Output file generated"); - - if let Some(mut stdout_file) = stdout_tempfile { - let mut stdout = std::io::stdout(); - - std::io::copy(&mut stdout_file, &mut stdout)?; - } - - Ok(()) -} - -async fn formats_from_series( - download: &Download, - series: Media, - url_filter: &UrlFilter, -) -> Result>> { - if !series.metadata.audio_locales.is_empty() - && !series.metadata.audio_locales.contains(&download.audio) - { - error!( - "Series {} is not available with {} audio", - series.title, download.audio - ); - return Ok(None); - } - - let mut seasons = series.seasons().await?; - - // filter any season out which does not contain the specified audio language - for season in sort_seasons_after_number(seasons.clone()) { - // check if the current iterated season has the specified audio language - if !season - .iter() - .any(|s| s.metadata.audio_locales.contains(&download.audio)) - { - error!( - "Season {} of series {} is not available with {} audio", - season.first().unwrap().metadata.season_number, - series.title, - download.audio - ); - } - - // remove all seasons with the wrong audio for the current iterated season number - seasons.retain(|s| { - s.metadata.season_number != season.first().unwrap().metadata.season_number - || s.metadata.audio_locales.contains(&download.audio) - }); - // remove seasons which match the url filter. this is mostly done to not trigger the - // interactive season choosing when dupilcated seasons are excluded by the filter - seasons.retain(|s| url_filter.is_season_valid(s.metadata.season_number)) - } - - if !download.yes && !find_multiple_seasons_with_same_number(&seasons).is_empty() { - info!(target: "progress_end", "Fetched seasons"); - seasons = interactive_season_choosing(seasons); - info!(target: "progress", "Fetching series details") - } - - let mut formats = vec![]; - for season in seasons { - if let Some(fmts) = formats_from_season(download, season, url_filter).await? { - formats.extend(fmts) - } - } - - Ok(some_vec_or_none(formats)) -} - -async fn formats_from_season( - download: &Download, - season: Media, - url_filter: &UrlFilter, -) -> Result>> { - if !url_filter.is_season_valid(season.metadata.season_number) { - return Ok(None); - } else if !season.metadata.audio_locales.contains(&download.audio) { - error!( - "Season {} ({}) is not available with {} audio", - season.metadata.season_number, season.title, download.audio - ); - return Ok(None); - } - - let mut formats = vec![]; - - let episodes = season.episodes().await?; - for episode in episodes.iter() { - if let Some(fmt) = - format_from_episode(download, &episode, url_filter, Some(&episodes), true).await? - { - formats.push(fmt) - } - } - - Ok(some_vec_or_none(formats)) -} - -async fn format_from_episode( - download: &Download, - episode: &Media, - url_filter: &UrlFilter, - season_episodes: Option<&Vec>>, - filter_audio: bool, -) -> Result> { - if filter_audio && episode.metadata.audio_locale != download.audio { - error!( - "Episode {} ({}) of season {} ({}) of {} has no {} audio", - episode.metadata.episode_number, - episode.title, - episode.metadata.season_number, - episode.metadata.season_title, - episode.metadata.series_title, - download.audio - ); - return Ok(None); - } else if !url_filter.is_episode_valid( - episode.metadata.episode_number, - episode.metadata.season_number, - ) { - return Ok(None); - } - - let streams = episode.streams().await?; - let streaming_data = streams.hls_streaming_data(None).await?; - let subtitle = if let Some(subtitle) = &download.subtitle { - if let Some(sub) = streams.subtitles.get(subtitle) { - Some(sub.clone()) - } else { - error!( - "Episode {} ({}) of season {} ({}) of {} has no {} subtitles", - episode.metadata.episode_number, - episode.title, - episode.metadata.season_number, - episode.metadata.season_title, - episode.metadata.series_title, - subtitle - ); - return Ok(None); - } - } else { - None - }; - - let Some(stream) = find_resolution(streaming_data, &download.resolution) else { - bail!( - "Resolution ({}x{}) is not available for episode {} ({}) of season {} ({}) of {}", - download.resolution.width, - download.resolution.height, - episode.metadata.episode_number, - episode.title, - episode.metadata.season_number, - episode.metadata.season_title, - episode.metadata.series_title - ) - }; - - let season_eps = if Format::has_relative_episodes_fmt(&download.output) { - if let Some(eps) = season_episodes { - Cow::from(eps) - } else { - Cow::from(episode.season().await?.episodes().await?) - } - } else { - Cow::from(vec![]) - }; - - Ok(Some(Format::new_from_episode( - episode, - &season_eps.to_vec(), - stream, - subtitle.map_or_else(|| vec![], |s| vec![s]), - ))) -} - -async fn format_from_movie_listing( - download: &Download, - movie_listing: Media, - url_filter: &UrlFilter, -) -> Result>> { - let mut formats = vec![]; - - for movie in movie_listing.movies().await? { - if let Some(fmt) = format_from_movie(download, movie, url_filter).await? { - formats.push(fmt) - } - } - - Ok(some_vec_or_none(formats)) -} - -async fn format_from_movie( - download: &Download, - movie: Media, - _: &UrlFilter, -) -> Result> { - let streams = movie.streams().await?; - let mut streaming_data = if let Some(subtitle) = &download.subtitle { - if !streams.subtitles.keys().cloned().any(|x| &x == subtitle) { - error!("Movie {} has no {} subtitles", movie.title, subtitle); - return Ok(None); - } - streams.hls_streaming_data(Some(subtitle.clone())).await? - } else { - streams.hls_streaming_data(None).await? - }; - - streaming_data.sort_by(|a, b| a.resolution.width.cmp(&b.resolution.width).reverse()); - let stream = { - match download.resolution.height { - u64::MAX => streaming_data.into_iter().next().unwrap(), - u64::MIN => streaming_data.into_iter().last().unwrap(), - _ => { - if let Some(streaming_data) = streaming_data.into_iter().find(|v| { - download.resolution.height == u64::MAX - || v.resolution.height == download.resolution.height - }) { - streaming_data - } else { - bail!( - "Resolution ({}x{}) is not available for movie {}", - download.resolution.width, - download.resolution.height, - movie.title - ) - } - } - } - }; - - Ok(Some(Format::new_from_movie(&movie, stream))) -} - -fn some_vec_or_none(v: Vec) -> Option> { - if v.is_empty() { - None - } else { - Some(v) - } -} diff --git a/crunchy-cli-core/src/cli/log.rs b/crunchy-cli-core/src/cli/log.rs deleted file mode 100644 index 7147c47..0000000 --- a/crunchy-cli-core/src/cli/log.rs +++ /dev/null @@ -1,138 +0,0 @@ -use indicatif::{ProgressBar, ProgressStyle}; -use log::{ - set_boxed_logger, set_max_level, Level, LevelFilter, Log, Metadata, Record, SetLoggerError, -}; -use std::io::{stdout, Write}; -use std::sync::Mutex; -use std::thread; -use std::time::Duration; - -#[allow(clippy::type_complexity)] -pub struct CliLogger { - all: bool, - level: LevelFilter, - progress: Mutex>, -} - -impl Log for CliLogger { - fn enabled(&self, metadata: &Metadata) -> bool { - metadata.level() <= self.level - } - - fn log(&self, record: &Record) { - if !self.enabled(record.metadata()) - || (record.target() != "progress" - && record.target() != "progress_end" - && (!self.all && !record.target().starts_with("crunchy_cli"))) - { - return; - } - - if self.level >= LevelFilter::Debug { - self.extended(record); - return; - } - - match record.target() { - "progress" => self.progress(record, false), - "progress_end" => self.progress(record, true), - _ => { - if self.progress.lock().unwrap().is_some() { - self.progress(record, false) - } else if record.level() > Level::Warn { - self.normal(record) - } else { - self.error(record) - } - } - } - } - - fn flush(&self) { - let _ = stdout().flush(); - } -} - -impl CliLogger { - pub fn new(all: bool, level: LevelFilter) -> Self { - Self { - all, - level, - progress: Mutex::new(None), - } - } - - pub fn init(all: bool, level: LevelFilter) -> Result<(), SetLoggerError> { - set_max_level(level); - set_boxed_logger(Box::new(CliLogger::new(all, level))) - } - - fn extended(&self, record: &Record) { - println!( - "[{}] {} {} ({}) {}", - chrono::Utc::now().format("%Y-%m-%d %H:%M:%S"), - record.level(), - // replace the 'progress' prefix if this function is invoked via 'progress!' - record - .target() - .replacen("crunchy_cli_core", "crunchy_cli", 1) - .replacen("progress_end", "crunchy_cli", 1) - .replacen("progress", "crunchy_cli", 1), - format!("{:?}", thread::current().id()) - .replace("ThreadId(", "") - .replace(')', ""), - record.args() - ) - } - - fn normal(&self, record: &Record) { - println!(":: {}", record.args()) - } - - fn error(&self, record: &Record) { - eprintln!(":: {}", record.args()) - } - - fn progress(&self, record: &Record, stop: bool) { - let mut progress = self.progress.lock().unwrap(); - - let msg = format!("{}", record.args()); - if stop && progress.is_some() { - if msg.is_empty() { - progress.take().unwrap().finish() - } else { - progress.take().unwrap().finish_with_message(msg) - } - } else if let Some(p) = &*progress { - p.println(format!(":: → {}", msg)) - } else { - #[cfg(not(windows))] - let finish_str = "✔"; - #[cfg(windows)] - // windows does not support all unicode characters by default in their consoles, so - // we're using this (square root?) symbol instead. microsoft. - let finish_str = "√"; - - let pb = ProgressBar::new_spinner(); - pb.set_style( - ProgressStyle::with_template(":: {spinner} {msg}") - .unwrap() - .tick_strings(&["—", "\\", "|", "/", finish_str]), - ); - pb.enable_steady_tick(Duration::from_millis(200)); - pb.set_message(msg); - *progress = Some(pb) - } - } -} - -macro_rules! tab_info { - ($($arg:tt)+) => { - if log::max_level() == log::LevelFilter::Debug { - info!($($arg)+) - } else { - info!("\t{}", format!($($arg)+)) - } - } -} -pub(crate) use tab_info; diff --git a/crunchy-cli-core/src/cli/mod.rs b/crunchy-cli-core/src/cli/mod.rs deleted file mode 100644 index c28f8e0..0000000 --- a/crunchy-cli-core/src/cli/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub mod archive; -pub mod download; -pub mod log; -pub mod login; -mod utils; diff --git a/crunchy-cli-core/src/cli/utils.rs b/crunchy-cli-core/src/cli/utils.rs deleted file mode 100644 index 753cb72..0000000 --- a/crunchy-cli-core/src/cli/utils.rs +++ /dev/null @@ -1,693 +0,0 @@ -use crate::utils::context::Context; -use anyhow::{bail, Result}; -use crunchyroll_rs::media::{Resolution, VariantData, VariantSegment}; -use crunchyroll_rs::{Locale, Media, Season}; -use indicatif::{ProgressBar, ProgressFinish, ProgressStyle}; -use lazy_static::lazy_static; -use log::{debug, LevelFilter}; -use regex::Regex; -use std::borrow::{Borrow, BorrowMut}; -use std::collections::BTreeMap; -use std::env; -use std::io::{BufRead, Write}; -use std::str::FromStr; -use std::sync::{mpsc, Arc, Mutex}; -use std::time::Duration; -use tokio::task::JoinSet; - -pub fn find_resolution( - mut streaming_data: Vec, - resolution: &Resolution, -) -> Option { - streaming_data.sort_by(|a, b| a.resolution.width.cmp(&b.resolution.width).reverse()); - match resolution.height { - u64::MAX => Some(streaming_data.into_iter().next().unwrap()), - u64::MIN => Some(streaming_data.into_iter().last().unwrap()), - _ => streaming_data - .into_iter() - .find(|v| resolution.height == u64::MAX || v.resolution.height == resolution.height), - } -} - -pub async fn download_segments( - ctx: &Context, - writer: &mut impl Write, - message: Option, - variant_data: VariantData, -) -> Result<()> { - let segments = variant_data.segments().await?; - let total_segments = segments.len(); - - let client = Arc::new(ctx.crunchy.client()); - let count = Arc::new(Mutex::new(0)); - - let progress = if log::max_level() == LevelFilter::Info { - let estimated_file_size = (variant_data.bandwidth / 8) - * segments - .iter() - .map(|s| s.length.unwrap_or_default().as_secs()) - .sum::(); - - let progress = ProgressBar::new(estimated_file_size) - .with_style( - ProgressStyle::with_template( - ":: {msg}{bytes:>10} {bytes_per_sec:>12} [{wide_bar}] {percent:>3}%", - ) - .unwrap() - .progress_chars("##-"), - ) - .with_message(message.map(|m| m + " ").unwrap_or_default()) - .with_finish(ProgressFinish::Abandon); - Some(progress) - } else { - None - }; - - let cpus = num_cpus::get(); - let mut segs: Vec> = Vec::with_capacity(cpus); - for _ in 0..cpus { - segs.push(vec![]) - } - for (i, segment) in segments.clone().into_iter().enumerate() { - segs[i - ((i / cpus) * cpus)].push(segment); - } - - let (sender, receiver) = mpsc::channel(); - - let mut join_set: JoinSet> = JoinSet::new(); - for num in 0..cpus { - let thread_client = client.clone(); - let thread_sender = sender.clone(); - let thread_segments = segs.remove(0); - let thread_count = count.clone(); - join_set.spawn(async move { - let after_download_sender = thread_sender.clone(); - - // the download process is encapsulated in its own function. this is done to easily - // catch errors which get returned with `...?` and `bail!(...)` and that the thread - // itself can report that an error has occured - let download = || async move { - for (i, segment) in thread_segments.into_iter().enumerate() { - let mut retry_count = 0; - let mut buf = loop { - let response = thread_client - .get(&segment.url) - .timeout(Duration::from_secs(60)) - .send() - .await?; - - match response.bytes().await { - Ok(b) => break b.to_vec(), - Err(e) => { - if e.is_body() { - if retry_count == 5 { - bail!("Max retry count reached ({}), multiple errors occured while receiving segment {}: {}", retry_count, num + (i * cpus), e) - } - debug!("Failed to download segment {} ({}). Retrying, {} out of 5 retries left", num + (i * cpus), e, 5 - retry_count) - } else { - bail!("{}", e) - } - } - } - - retry_count += 1; - }; - - buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec(); - - let mut c = thread_count.lock().unwrap(); - debug!( - "Downloaded and decrypted segment [{}/{} {:.2}%] {}", - num + (i * cpus), - total_segments, - ((*c + 1) as f64 / total_segments as f64) * 100f64, - segment.url - ); - - thread_sender.send((num as i32 + (i * cpus) as i32, buf))?; - - *c += 1; - } - Ok(()) - }; - - - let result = download().await; - if result.is_err() { - after_download_sender.send((-1 as i32, vec![]))?; - } - - result - }); - } - // drop the sender already here so it does not outlive all (download) threads which are the only - // real consumers of it - drop(sender); - - // this is the main loop which writes the data. it uses a BTreeMap as a buffer as the write - // happens synchronized. the download consist of multiple segments. the map keys are representing - // the segment number and the values the corresponding bytes - let mut data_pos = 0; - let mut buf: BTreeMap> = BTreeMap::new(); - for (pos, bytes) in receiver.iter() { - // if the position is lower than 0, an error occured in the sending download thread - if pos < 0 { - break; - } - - if let Some(p) = &progress { - let progress_len = p.length().unwrap(); - let estimated_segment_len = (variant_data.bandwidth / 8) - * segments - .get(pos as usize) - .unwrap() - .length - .unwrap_or_default() - .as_secs(); - let bytes_len = bytes.len() as u64; - - p.set_length(progress_len - estimated_segment_len + bytes_len); - p.inc(bytes_len) - } - - // check if the currently sent bytes are the next in the buffer. if so, write them directly - // to the target without first adding them to the buffer. - // if not, add them to the buffer - if data_pos == pos { - writer.write_all(bytes.borrow())?; - data_pos += 1; - } else { - buf.insert(pos, bytes); - } - // check if the buffer contains the next segment(s) - while let Some(b) = buf.remove(&data_pos) { - writer.write_all(b.borrow())?; - data_pos += 1; - } - } - - // if any error has occured while downloading it gets returned here - while let Some(joined) = join_set.join_next().await { - joined?? - } - - // write the remaining buffer, if existent - while let Some(b) = buf.remove(&data_pos) { - writer.write_all(b.borrow())?; - data_pos += 1; - } - - if !buf.is_empty() { - bail!( - "Download buffer is not empty. Remaining segments: {}", - buf.into_keys() - .map(|k| k.to_string()) - .collect::>() - .join(", ") - ) - } - - Ok(()) -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum FFmpegPreset { - Predefined(FFmpegCodec, Option, FFmpegQuality), - Custom(Option, Option), -} - -lazy_static! { - static ref PREDEFINED_PRESET: Regex = Regex::new(r"^\w+(-\w+)*?$").unwrap(); -} - -macro_rules! FFmpegEnum { - (enum $name:ident { $($field:ident),* }) => { - #[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] - pub enum $name { - $( - $field - ),*, - } - - impl $name { - fn all() -> Vec<$name> { - vec![ - $( - $name::$field - ),*, - ] - } - } - - impl ToString for $name { - fn to_string(&self) -> String { - match self { - $( - &$name::$field => stringify!($field).to_string().to_lowercase() - ),* - } - } - } - - impl FromStr for $name { - type Err = anyhow::Error; - - fn from_str(s: &str) -> std::result::Result { - match s { - $( - stringify!($field) => Ok($name::$field) - ),*, - _ => bail!("{} is not a valid {}", s, stringify!($name).to_lowercase()) - } - } - } - } -} - -FFmpegEnum! { - enum FFmpegCodec { - H264, - H265, - Av1 - } -} - -FFmpegEnum! { - enum FFmpegHwAccel { - Nvidia - } -} - -FFmpegEnum! { - enum FFmpegQuality { - Lossless, - Normal, - Low - } -} - -impl FFmpegPreset { - pub(crate) fn available_matches( - ) -> Vec<(FFmpegCodec, Option, Option)> { - let codecs = vec![ - ( - FFmpegCodec::H264, - FFmpegHwAccel::all(), - FFmpegQuality::all(), - ), - ( - FFmpegCodec::H265, - FFmpegHwAccel::all(), - FFmpegQuality::all(), - ), - (FFmpegCodec::Av1, vec![], FFmpegQuality::all()), - ]; - - let mut return_values = vec![]; - - for (codec, hwaccels, qualities) in codecs { - return_values.push((codec.clone(), None, None)); - for hwaccel in hwaccels.clone() { - return_values.push((codec.clone(), Some(hwaccel), None)); - } - for quality in qualities.clone() { - return_values.push((codec.clone(), None, Some(quality))) - } - for hwaccel in hwaccels { - for quality in qualities.clone() { - return_values.push((codec.clone(), Some(hwaccel.clone()), Some(quality))) - } - } - } - - return_values - } - - pub(crate) fn available_matches_human_readable() -> Vec { - let mut return_values = vec![]; - - for (codec, hwaccel, quality) in FFmpegPreset::available_matches() { - let mut description_details = vec![]; - if let Some(h) = &hwaccel { - description_details.push(format!("{} hardware acceleration", h.to_string())) - } - if let Some(q) = &quality { - description_details.push(format!("{} video quality/compression", q.to_string())) - } - - let description = if description_details.len() == 0 { - format!( - "{} encoded with default video quality/compression", - codec.to_string() - ) - } else if description_details.len() == 1 { - format!( - "{} encoded with {}", - codec.to_string(), - description_details[0] - ) - } else { - let first = description_details.remove(0); - let last = description_details.remove(description_details.len() - 1); - let mid = if !description_details.is_empty() { - format!(", {} ", description_details.join(", ")) - } else { - "".to_string() - }; - - format!( - "{} encoded with {}{} and {}", - codec.to_string(), - first, - mid, - last - ) - }; - - return_values.push(format!( - "{} ({})", - vec![ - Some(codec.to_string()), - hwaccel.map(|h| h.to_string()), - quality.map(|q| q.to_string()) - ] - .into_iter() - .flatten() - .collect::>() - .join("-"), - description - )) - } - return_values - } - - pub(crate) fn parse(s: &str) -> Result { - let env_ffmpeg_input_args = env::var("FFMPEG_INPUT_ARGS").ok(); - let env_ffmpeg_output_args = env::var("FFMPEG_OUTPUT_ARGS").ok(); - - if env_ffmpeg_input_args.is_some() || env_ffmpeg_output_args.is_some() { - if let Some(input) = &env_ffmpeg_input_args { - if shlex::split(input).is_none() { - return Err(format!("Failed to parse custom ffmpeg input '{}' (`FFMPEG_INPUT_ARGS` env variable)", input)); - } - } - if let Some(output) = &env_ffmpeg_output_args { - if shlex::split(output).is_none() { - return Err(format!("Failed to parse custom ffmpeg output '{}' (`FFMPEG_INPUT_ARGS` env variable)", output)); - } - } - - return Ok(FFmpegPreset::Custom( - env_ffmpeg_input_args, - env_ffmpeg_output_args, - )); - } else if !PREDEFINED_PRESET.is_match(s) { - return Ok(FFmpegPreset::Custom(None, Some(s.to_string()))); - } - - let mut codec: Option = None; - let mut hwaccel: Option = None; - let mut quality: Option = None; - for token in s.split('-') { - if let Some(c) = FFmpegCodec::all() - .into_iter() - .find(|p| p.to_string() == token.to_lowercase()) - { - if let Some(cc) = codec { - return Err(format!( - "cannot use multiple codecs (found {} and {})", - cc.to_string(), - c.to_string() - )); - } - codec = Some(c) - } else if let Some(h) = FFmpegHwAccel::all() - .into_iter() - .find(|p| p.to_string() == token.to_lowercase()) - { - if let Some(hh) = hwaccel { - return Err(format!( - "cannot use multiple hardware accelerations (found {} and {})", - hh.to_string(), - h.to_string() - )); - } - hwaccel = Some(h) - } else if let Some(q) = FFmpegQuality::all() - .into_iter() - .find(|p| p.to_string() == token.to_lowercase()) - { - if let Some(qq) = quality { - return Err(format!( - "cannot use multiple ffmpeg preset qualities (found {} and {})", - qq.to_string(), - q.to_string() - )); - } - quality = Some(q) - } else { - return Err(format!( - "'{}' is not a valid ffmpeg preset (unknown token '{}'", - s, token - )); - } - } - - if let Some(c) = codec { - if !FFmpegPreset::available_matches().contains(&( - c.clone(), - hwaccel.clone(), - quality.clone(), - )) { - return Err(format!("ffmpeg preset is not supported")); - } - Ok(FFmpegPreset::Predefined( - c, - hwaccel, - quality.unwrap_or(FFmpegQuality::Normal), - )) - } else { - Err(format!("cannot use ffmpeg preset with without a codec")) - } - } - - pub(crate) fn to_input_output_args(self) -> (Vec, Vec) { - match self { - FFmpegPreset::Custom(input, output) => ( - input.map_or(vec![], |i| shlex::split(&i).unwrap_or_default()), - output.map_or(vec![], |o| shlex::split(&o).unwrap_or_default()), - ), - FFmpegPreset::Predefined(codec, hwaccel_opt, quality) => { - let mut input = vec![]; - let mut output = vec![]; - - match codec { - FFmpegCodec::H264 => { - if let Some(hwaccel) = hwaccel_opt { - match hwaccel { - FFmpegHwAccel::Nvidia => { - input.extend(["-hwaccel", "cuvid", "-c:v", "h264_cuvid"]); - output.extend(["-c:v", "h264_nvenc", "-c:a", "copy"]) - } - } - } else { - output.extend(["-c:v", "libx264", "-c:a", "copy"]) - } - - match quality { - FFmpegQuality::Lossless => output.extend(["-crf", "18"]), - FFmpegQuality::Normal => (), - FFmpegQuality::Low => output.extend(["-crf", "35"]), - } - } - FFmpegCodec::H265 => { - if let Some(hwaccel) = hwaccel_opt { - match hwaccel { - FFmpegHwAccel::Nvidia => { - input.extend(["-hwaccel", "cuvid", "-c:v", "h264_cuvid"]); - output.extend(["-c:v", "hevc_nvenc", "-c:a", "copy"]) - } - } - } else { - output.extend(["-c:v", "libx265", "-c:a", "copy"]) - } - - match quality { - FFmpegQuality::Lossless => output.extend(["-crf", "20"]), - FFmpegQuality::Normal => (), - FFmpegQuality::Low => output.extend(["-crf", "35"]), - } - } - FFmpegCodec::Av1 => { - output.extend(["-c:v", "libsvtav1", "-c:a", "copy"]); - - match quality { - FFmpegQuality::Lossless => output.extend(["-crf", "22"]), - FFmpegQuality::Normal => (), - FFmpegQuality::Low => output.extend(["-crf", "35"]), - } - } - } - - ( - input - .into_iter() - .map(|s| s.to_string()) - .collect::>(), - output - .into_iter() - .map(|s| s.to_string()) - .collect::>(), - ) - } - } - } -} - -lazy_static! { - static ref DUPLICATED_SEASONS_MULTILANG_REGEX: Regex = Regex::new(r"(-arabic|-castilian|-english|-english-in|-french|-german|-hindi|-italian|-portuguese|-russian|-spanish)$").unwrap(); -} - -pub(crate) fn find_multiple_seasons_with_same_number(seasons: &Vec>) -> Vec { - let mut seasons_map: BTreeMap = BTreeMap::new(); - for season in seasons { - if let Some(s) = seasons_map.get_mut(&season.metadata.season_number) { - *s += 1; - } else { - seasons_map.insert(season.metadata.season_number, 1); - } - } - - seasons_map - .into_iter() - .filter_map(|(k, v)| { - if v > 1 { - // check if the different seasons are actual the same but with different dub languages - let mut multilang_season_vec: Vec = seasons - .iter() - .map(|s| { - DUPLICATED_SEASONS_MULTILANG_REGEX - .replace(s.slug_title.trim_end_matches("-dub"), "") - .to_string() - }) - .collect(); - multilang_season_vec.dedup(); - - if multilang_season_vec.len() > 1 { - return Some(k); - } - } - None - }) - .collect() -} - -/// Check if [`Locale::Custom("all")`] is in the provided locale list and return [`Locale::all`] if -/// so. If not, just return the provided locale list. -pub(crate) fn all_locale_in_locales(locales: Vec) -> Vec { - if locales - .iter() - .find(|l| l.to_string().to_lowercase().trim() == "all") - .is_some() - { - Locale::all() - } else { - locales - } -} - -pub(crate) fn interactive_season_choosing(seasons: Vec>) -> Vec> { - let input_regex = - Regex::new(r"((?P\d+)|(?P\d+)-(?P\d+)?)(\s|$)").unwrap(); - - let mut seasons_map: BTreeMap>> = BTreeMap::new(); - for season in seasons { - if let Some(s) = seasons_map.get_mut(&season.metadata.season_number) { - s.push(season); - } else { - seasons_map.insert(season.metadata.season_number, vec![season]); - } - } - - for (num, season_vec) in seasons_map.iter_mut() { - if season_vec.len() == 1 { - continue; - } - - // check if the different seasons are actual the same but with different dub languages - let mut multilang_season_vec: Vec = season_vec - .iter() - .map(|s| { - DUPLICATED_SEASONS_MULTILANG_REGEX - .replace(s.slug_title.trim_end_matches("-dub"), "") - .to_string() - }) - .collect(); - multilang_season_vec.dedup(); - - if multilang_season_vec.len() == 1 { - continue; - } - - println!(":: Found multiple seasons for season number {}", num); - println!(":: Select the number of the seasons you want to download (eg \"1 2 4\", \"1-3\", \"1-3 5\"):"); - for (i, season) in season_vec.iter().enumerate() { - println!(":: \t{}. {}", i + 1, season.title) - } - let mut stdout = std::io::stdout(); - let _ = write!(stdout, ":: => "); - let _ = stdout.flush(); - let mut user_input = String::new(); - std::io::stdin() - .lock() - .read_line(&mut user_input) - .expect("cannot open stdin"); - - let mut nums = vec![]; - for capture in input_regex.captures_iter(&user_input) { - if let Some(single) = capture.name("single") { - nums.push(single.as_str().parse::().unwrap() - 1); - } else { - let range_from = capture.name("range_from"); - let range_to = capture.name("range_to"); - - // input is '-' which means use all seasons - if range_from.is_none() && range_to.is_none() { - nums = vec![]; - break; - } - let from = range_from - .map(|f| f.as_str().parse::().unwrap() - 1) - .unwrap_or(usize::MIN); - let to = range_from - .map(|f| f.as_str().parse::().unwrap() - 1) - .unwrap_or(usize::MAX); - - nums.extend( - season_vec - .iter() - .enumerate() - .filter_map(|(i, _)| if i >= from && i <= to { Some(i) } else { None }) - .collect::>(), - ) - } - } - nums.dedup(); - - if !nums.is_empty() { - let mut remove_count = 0; - for i in 0..season_vec.len() - 1 { - if !nums.contains(&i) { - season_vec.remove(i - remove_count); - remove_count += 1 - } - } - } - } - - seasons_map - .into_values() - .into_iter() - .flatten() - .collect::>>() -} diff --git a/crunchy-cli-core/src/download/command.rs b/crunchy-cli-core/src/download/command.rs new file mode 100644 index 0000000..25db55b --- /dev/null +++ b/crunchy-cli-core/src/download/command.rs @@ -0,0 +1,151 @@ +use crate::download::filter::DownloadFilter; +use crate::utils::context::Context; +use crate::utils::ffmpeg::FFmpegPreset; +use crate::utils::filter::Filter; +use crate::utils::format::formats_visual_output; +use crate::utils::log::progress; +use crate::utils::os::{free_file, has_ffmpeg}; +use crate::utils::parse::parse_url; +use crate::Execute; +use anyhow::bail; +use anyhow::Result; +use crunchyroll_rs::media::Resolution; +use crunchyroll_rs::Locale; +use log::{debug, warn}; +use std::path::Path; + +#[derive(Clone, Debug, clap::Parser)] +#[clap(about = "Download a video")] +#[command(arg_required_else_help(true))] +pub struct Download { + #[arg(help = format!("Audio language. Can only be used if the provided url(s) point to a series. \ + Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] + #[arg(long_help = format!("Audio language. Can only be used if the provided url(s) point to a series. \ + Available languages are:\n{}", Locale::all().into_iter().map(|l| format!("{:<6} → {}", l.to_string(), l.to_human_readable())).collect::>().join("\n ")))] + #[arg(short, long, default_value_t = crate::utils::locale::system_locale())] + pub(crate) audio: Locale, + #[arg(help = format!("Subtitle language. Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] + #[arg(long_help = format!("Subtitle language. If set, the subtitle will be burned into the video and cannot be disabled. \ + Available languages are: {}", Locale::all().into_iter().map(|l| l.to_string()).collect::>().join(", ")))] + #[arg(short, long)] + pub(crate) subtitle: Option, + + #[arg(help = "Name of the output file")] + #[arg(long_help = "Name of the output file.\ + If you use one of the following pattern they will get replaced:\n \ + {title} → Title of the video\n \ + {series_name} → Name of the series\n \ + {season_name} → Name of the season\n \ + {audio} → Audio language of the video\n \ + {resolution} → Resolution of the video\n \ + {season_number} → Number of the season\n \ + {episode_number} → Number of the episode\n \ + {relative_episode_number} → Number of the episode relative to its season\ + {series_id} → ID of the series\n \ + {season_id} → ID of the season\n \ + {episode_id} → ID of the episode")] + #[arg(short, long, default_value = "{title}.mp4")] + pub(crate) output: String, + + #[arg(help = "Video resolution")] + #[arg(long_help = "The video resolution.\ + Can either be specified via the pixels (e.g. 1920x1080), the abbreviation for pixels (e.g. 1080p) or 'common-use' words (e.g. best). \ + Specifying the exact pixels is not recommended, use one of the other options instead. \ + Crunchyroll let you choose the quality with pixel abbreviation on their clients, so you might be already familiar with the available options. \ + The available common-use words are 'best' (choose the best resolution available) and 'worst' (worst resolution available)")] + #[arg(short, long, default_value = "best")] + #[arg(value_parser = crate::utils::clap::clap_parse_resolution)] + pub(crate) resolution: Resolution, + + #[arg(help = format!("Presets for video converting. Can be used multiple times. \ + Available presets: \n {}", FFmpegPreset::available_matches_human_readable().join("\n ")))] + #[arg(long_help = format!("Presets for video converting. Can be used multiple times. \ + Generally used to minify the file size with keeping (nearly) the same quality. \ + It is recommended to only use this if you download videos with high resolutions since low resolution videos tend to result in a larger file with any of the provided presets. \ + Available presets: \n {}", FFmpegPreset::available_matches_human_readable().join("\n ")))] + #[arg(long)] + #[arg(value_parser = FFmpegPreset::parse)] + pub(crate) ffmpeg_preset: Option, + + #[arg(help = "Skip files which are already existing")] + #[arg(long, default_value_t = false)] + pub(crate) skip_existing: bool, + + #[arg(help = "Url(s) to Crunchyroll episodes or series")] + pub(crate) urls: Vec, +} + +#[async_trait::async_trait(?Send)] +impl Execute for Download { + fn pre_check(&mut self) -> Result<()> { + if !has_ffmpeg() { + bail!("FFmpeg is needed to run this command") + } else if Path::new(&self.output) + .extension() + .unwrap_or_default() + .is_empty() + && self.output != "-" + { + bail!("No file extension found. Please specify a file extension (via `-o`) for the output file") + } + + if self.subtitle.is_some() { + if let Some(ext) = Path::new(&self.output).extension() { + if ext.to_string_lossy() != "mp4" { + warn!("Detected a non mp4 output container. Adding subtitles may take a while") + } + } + } + + Ok(()) + } + + async fn execute(self, ctx: Context) -> Result<()> { + let mut parsed_urls = vec![]; + + for (i, url) in self.urls.clone().into_iter().enumerate() { + let progress_handler = progress!("Parsing url {}", i + 1); + match parse_url(&ctx.crunchy, url.clone(), true).await { + Ok((media_collection, url_filter)) => { + progress_handler.stop(format!("Parsed url {}", i + 1)); + parsed_urls.push((media_collection, url_filter)) + } + Err(e) => bail!("url {} could not be parsed: {}", url, e), + }; + } + + for (i, (media_collection, url_filter)) in parsed_urls.into_iter().enumerate() { + let progress_handler = progress!("Fetching series details"); + let download_formats = DownloadFilter::new(url_filter, self.clone()) + .visit(media_collection) + .await?; + + if download_formats.is_empty() { + progress_handler.stop(format!("Skipping url {} (no matching videos found)", i + 1)); + continue; + } + progress_handler.stop(format!("Loaded series information for url {}", i + 1)); + + formats_visual_output(download_formats.iter().map(|(_, f)| f).collect()); + + for (downloader, format) in download_formats { + let formatted_path = format.format_path((&self.output).into(), true); + let (path, changed) = free_file(formatted_path.clone()); + + if changed && self.skip_existing { + debug!( + "Skipping already existing file '{}'", + formatted_path.to_string_lossy() + ); + continue; + } + + format.visual_output(&path); + + downloader.download(&ctx, &path).await? + } + } + + Ok(()) + } +} diff --git a/crunchy-cli-core/src/download/filter.rs b/crunchy-cli-core/src/download/filter.rs new file mode 100644 index 0000000..81b8fa9 --- /dev/null +++ b/crunchy-cli-core/src/download/filter.rs @@ -0,0 +1,349 @@ +use crate::download::Download; +use crate::utils::download::{DownloadBuilder, DownloadFormat, Downloader}; +use crate::utils::filter::Filter; +use crate::utils::format::{Format, SingleFormat}; +use crate::utils::parse::UrlFilter; +use crate::utils::video::variant_data_from_stream; +use anyhow::{bail, Result}; +use crunchyroll_rs::media::{Subtitle, VariantData}; +use crunchyroll_rs::{Concert, Episode, Movie, MovieListing, MusicVideo, Season, Series}; +use log::{error, warn}; +use std::collections::HashMap; + +pub(crate) struct FilterResult { + format: SingleFormat, + video: VariantData, + audio: VariantData, + subtitle: Option, +} + +pub(crate) struct DownloadFilter { + url_filter: UrlFilter, + download: Download, + season_episode_count: HashMap>, + season_subtitles_missing: Vec, +} + +impl DownloadFilter { + pub(crate) fn new(url_filter: UrlFilter, download: Download) -> Self { + Self { + url_filter, + download, + season_episode_count: HashMap::new(), + season_subtitles_missing: vec![], + } + } +} + +#[async_trait::async_trait] +impl Filter for DownloadFilter { + type T = FilterResult; + type Output = (Downloader, Format); + + async fn visit_series(&mut self, series: Series) -> Result> { + // `series.audio_locales` isn't always populated b/c of crunchyrolls api. so check if the + // audio is matching only if the field is populated + if !series.audio_locales.is_empty() { + if !series.audio_locales.contains(&self.download.audio) { + error!( + "Series {} is not available with {} audio", + series.title, self.download.audio + ); + return Ok(vec![]); + } + } + + let seasons = series.seasons().await?; + + Ok(seasons) + } + + async fn visit_season(&mut self, mut season: Season) -> Result> { + if !self.url_filter.is_season_valid(season.season_number) { + return Ok(vec![]); + } + + if !season + .audio_locales + .iter() + .any(|l| l == &self.download.audio) + { + if season + .available_versions() + .await? + .iter() + .any(|l| l == &self.download.audio) + { + season = season + .version(vec![self.download.audio.clone()]) + .await? + .remove(0) + } else { + error!( + "Season {} - '{}' is not available with {} audio", + season.season_number, + season.title, + self.download.audio.clone(), + ); + return Ok(vec![]); + } + } + + let mut episodes = season.episodes().await?; + + if Format::has_relative_episodes_fmt(&self.download.output) { + for episode in episodes.iter() { + self.season_episode_count + .entry(episode.season_number) + .or_insert(vec![]) + .push(episode.id.clone()) + } + } + + episodes.retain(|e| { + self.url_filter + .is_episode_valid(e.episode_number, season.season_number) + }); + + Ok(episodes) + } + + async fn visit_episode(&mut self, mut episode: Episode) -> Result> { + if !self + .url_filter + .is_episode_valid(episode.episode_number, episode.season_number) + { + return Ok(None); + } + + // check if the audio locale is correct. + // should only be incorrect if the console input was a episode url. otherwise + // `DownloadFilter::visit_season` returns the correct episodes with matching audio + if episode.audio_locale != self.download.audio { + // check if any other version (same episode, other language) of this episode is available + // with the requested audio. if not, return an error + if !episode + .available_versions() + .await? + .contains(&self.download.audio) + { + bail!( + "Episode {} ({}) of {} season {} is not available with {} audio", + episode.episode_number, + episode.title, + episode.series_title, + episode.season_number, + self.download.audio + ) + } + // overwrite the current episode with the other version episode + episode = episode + .version(vec![self.download.audio.clone()]) + .await? + .remove(0) + } + + // check if the subtitles are supported + if let Some(subtitle_locale) = &self.download.subtitle { + if !episode.subtitle_locales.contains(subtitle_locale) { + // if the episode doesn't have the requested subtitles, print a error. to print this + // error only once per season, it's checked if an error got printed before by looking + // up if the season id is present in `self.season_subtitles_missing`. if not, print + // the error and add the season id to `self.season_subtitles_missing`. if it is + // present, skip the error printing + if !self + .season_subtitles_missing + .contains(&episode.season_number) + { + self.season_subtitles_missing.push(episode.season_number); + error!( + "{} season {} is not available with {} subtitles", + episode.series_title, episode.season_number, subtitle_locale + ); + } + return Ok(None); + } + } + + // get the correct video stream + let stream = episode.streams().await?; + let (video, audio) = if let Some((video, audio)) = + variant_data_from_stream(&stream, &self.download.resolution).await? + { + (video, audio) + } else { + bail!( + "Resolution ({}) is not available for episode {} ({}) of {} season {}", + self.download.resolution, + episode.episode_number, + episode.title, + episode.series_title, + episode.season_number, + ) + }; + + // it is assumed that the subtitle, if requested, exists b/c the subtitle check above must + // be passed to reach this condition. + // the check isn't done in this if block to reduce unnecessary fetching of the stream + let subtitle = if let Some(subtitle_locale) = &self.download.subtitle { + stream.subtitles.get(subtitle_locale).map(|s| s.clone()) + } else { + None + }; + + // get the relative episode number. only done if the output string has the pattern to include + // the relative episode number as this requires some extra fetching + let relative_episode_number = if Format::has_relative_episodes_fmt(&self.download.output) { + if self + .season_episode_count + .get(&episode.season_number) + .is_none() + { + let season_episodes = episode.season().await?.episodes().await?; + self.season_episode_count.insert( + episode.season_number, + season_episodes.into_iter().map(|e| e.id).collect(), + ); + } + let relative_episode_number = self + .season_episode_count + .get(&episode.season_number) + .unwrap() + .iter() + .position(|id| id == &episode.id); + if relative_episode_number.is_none() { + warn!( + "Failed to get relative episode number for episode {} ({}) of {} season {}", + episode.episode_number, + episode.title, + episode.series_title, + episode.season_number, + ) + } + relative_episode_number + } else { + None + }; + + Ok(Some(FilterResult { + format: SingleFormat::new_from_episode( + &episode, + &video, + subtitle.clone().map_or(vec![], |s| vec![s.locale]), + relative_episode_number.map(|n| n as u32), + ), + video, + audio, + subtitle, + })) + } + + async fn visit_movie_listing(&mut self, movie_listing: MovieListing) -> Result> { + Ok(movie_listing.movies().await?) + } + + async fn visit_movie(&mut self, movie: Movie) -> Result> { + let stream = movie.streams().await?; + let (video, audio) = if let Some((video, audio)) = + variant_data_from_stream(&stream, &self.download.resolution).await? + { + (video, audio) + } else { + bail!( + "Resolution ({}) of movie '{}' is not available", + self.download.resolution, + movie.title + ) + }; + let subtitle = if let Some(subtitle_locale) = &self.download.subtitle { + let Some(subtitle) = stream.subtitles.get(subtitle_locale) else { + error!( + "Movie '{}' has no {} subtitles", + movie.title, + subtitle_locale + ); + return Ok(None) + }; + Some(subtitle.clone()) + } else { + None + }; + + Ok(Some(FilterResult { + format: SingleFormat::new_from_movie( + &movie, + &video, + subtitle.clone().map_or(vec![], |s| vec![s.locale]), + ), + video, + audio, + subtitle, + })) + } + + async fn visit_music_video(&mut self, music_video: MusicVideo) -> Result> { + let stream = music_video.streams().await?; + let (video, audio) = if let Some((video, audio)) = + variant_data_from_stream(&stream, &self.download.resolution).await? + { + (video, audio) + } else { + bail!( + "Resolution ({}) of music video {} is not available", + self.download.resolution, + music_video.title + ) + }; + + Ok(Some(FilterResult { + format: SingleFormat::new_from_music_video(&music_video, &video), + video, + audio, + subtitle: None, + })) + } + + async fn visit_concert(&mut self, concert: Concert) -> Result> { + let stream = concert.streams().await?; + let (video, audio) = if let Some((video, audio)) = + variant_data_from_stream(&stream, &self.download.resolution).await? + { + (video, audio) + } else { + bail!( + "Resolution ({}) of music video {} is not available", + self.download.resolution, + concert.title + ) + }; + + Ok(Some(FilterResult { + format: SingleFormat::new_from_concert(&concert, &video), + video, + audio, + subtitle: None, + })) + } + + async fn finish(self, mut input: Vec) -> Result> { + let mut result = vec![]; + input.sort_by(|a, b| { + a.format + .sequence_number + .total_cmp(&b.format.sequence_number) + }); + for data in input { + let mut downloader = DownloadBuilder::new() + .default_subtitle(self.download.subtitle.clone()) + .build(); + downloader.add_format(DownloadFormat { + video: (data.video, data.format.audio.clone()), + audios: vec![(data.audio, data.format.audio.clone())], + subtitles: data.subtitle.map_or(vec![], |s| vec![s]), + }); + result.push((downloader, Format::from_single_formats(vec![data.format]))) + } + + Ok(result) + } +} diff --git a/crunchy-cli-core/src/download/mod.rs b/crunchy-cli-core/src/download/mod.rs new file mode 100644 index 0000000..696872e --- /dev/null +++ b/crunchy-cli-core/src/download/mod.rs @@ -0,0 +1,4 @@ +mod command; +mod filter; + +pub use command::Download; diff --git a/crunchy-cli-core/src/lib.rs b/crunchy-cli-core/src/lib.rs index ab3f33c..5e0e113 100644 --- a/crunchy-cli-core/src/lib.rs +++ b/crunchy-cli-core/src/lib.rs @@ -1,7 +1,6 @@ -use crate::cli::log::CliLogger; use crate::utils::context::Context; use crate::utils::locale::system_locale; -use crate::utils::log::progress; +use crate::utils::log::{progress, CliLogger}; use anyhow::bail; use anyhow::Result; use clap::{Parser, Subcommand}; @@ -9,10 +8,14 @@ use crunchyroll_rs::{Crunchyroll, Locale}; use log::{debug, error, warn, LevelFilter}; use std::{env, fs}; -mod cli; +mod archive; +mod download; +mod login; mod utils; -pub use cli::{archive::Archive, download::Download, login::Login}; +pub use archive::Archive; +pub use download::Download; +pub use login::Login; #[async_trait::async_trait(?Send)] trait Execute { @@ -222,9 +225,14 @@ async fn crunchyroll_session(cli: &Cli) -> Result { lang }; - let builder = Crunchyroll::builder() + let mut builder = Crunchyroll::builder() .locale(locale) - .stabilization_locales(true); + .stabilization_locales(true) + .stabilization_season_number(true); + + if let Command::Download(download) = &cli.command { + builder = builder.preferred_audio_locale(download.audio.clone()) + } let login_methods_count = cli.login_method.credentials.is_some() as u8 + cli.login_method.etp_rt.is_some() as u8 @@ -232,7 +240,7 @@ async fn crunchyroll_session(cli: &Cli) -> Result { let progress_handler = progress!("Logging in"); if login_methods_count == 0 { - if let Some(login_file_path) = cli::login::login_file_path() { + if let Some(login_file_path) = login::login_file_path() { if login_file_path.exists() { let session = fs::read_to_string(login_file_path)?; if let Some((token_type, token)) = session.split_once(':') { diff --git a/crunchy-cli-core/src/cli/login.rs b/crunchy-cli-core/src/login/command.rs similarity index 94% rename from crunchy-cli-core/src/cli/login.rs rename to crunchy-cli-core/src/login/command.rs index c41b1b7..f164a0b 100644 --- a/crunchy-cli-core/src/cli/login.rs +++ b/crunchy-cli-core/src/login/command.rs @@ -18,6 +18,8 @@ pub struct Login { impl Execute for Login { async fn execute(self, ctx: Context) -> Result<()> { if let Some(login_file_path) = login_file_path() { + fs::create_dir_all(login_file_path.parent().unwrap())?; + match ctx.crunchy.session_token().await { SessionToken::RefreshToken(refresh_token) => Ok(fs::write( login_file_path, diff --git a/crunchy-cli-core/src/login/mod.rs b/crunchy-cli-core/src/login/mod.rs new file mode 100644 index 0000000..9025e68 --- /dev/null +++ b/crunchy-cli-core/src/login/mod.rs @@ -0,0 +1,4 @@ +mod command; + +pub use command::login_file_path; +pub use command::Login; diff --git a/crunchy-cli-core/src/utils/download.rs b/crunchy-cli-core/src/utils/download.rs new file mode 100644 index 0000000..ce0c443 --- /dev/null +++ b/crunchy-cli-core/src/utils/download.rs @@ -0,0 +1,657 @@ +use crate::utils::context::Context; +use crate::utils::ffmpeg::FFmpegPreset; +use crate::utils::log::progress; +use crate::utils::os::tempfile; +use anyhow::{bail, Result}; +use chrono::NaiveTime; +use crunchyroll_rs::media::{Subtitle, VariantData, VariantSegment}; +use crunchyroll_rs::Locale; +use indicatif::{ProgressBar, ProgressFinish, ProgressStyle}; +use log::{debug, LevelFilter}; +use regex::Regex; +use std::borrow::Borrow; +use std::borrow::BorrowMut; +use std::collections::BTreeMap; +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::sync::{mpsc, Arc, Mutex}; +use std::time::Duration; +use tempfile::TempPath; +use tokio::task::JoinSet; + +#[derive(Clone, Debug)] +pub enum MergeBehavior { + Video, + Audio, + Auto, +} + +impl MergeBehavior { + pub fn parse(s: &str) -> Result { + Ok(match s.to_lowercase().as_str() { + "video" => MergeBehavior::Video, + "audio" => MergeBehavior::Audio, + "auto" => MergeBehavior::Auto, + _ => return Err(format!("'{}' is not a valid merge behavior", s)), + }) + } +} + +#[derive(derive_setters::Setters)] +pub struct DownloadBuilder { + ffmpeg_preset: FFmpegPreset, + default_subtitle: Option, + output_format: Option, + audio_sort: Option>, + subtitle_sort: Option>, +} + +impl DownloadBuilder { + pub fn new() -> DownloadBuilder { + Self { + ffmpeg_preset: FFmpegPreset::default(), + default_subtitle: None, + output_format: None, + audio_sort: None, + subtitle_sort: None, + } + } + + pub fn build(self) -> Downloader { + Downloader { + ffmpeg_preset: self.ffmpeg_preset, + default_subtitle: self.default_subtitle, + output_format: self.output_format, + audio_sort: self.audio_sort, + subtitle_sort: self.subtitle_sort, + + formats: vec![], + } + } +} + +struct FFmpegMeta { + path: TempPath, + language: Locale, + title: String, +} + +pub struct DownloadFormat { + pub video: (VariantData, Locale), + pub audios: Vec<(VariantData, Locale)>, + pub subtitles: Vec, +} + +pub struct Downloader { + ffmpeg_preset: FFmpegPreset, + default_subtitle: Option, + output_format: Option, + audio_sort: Option>, + subtitle_sort: Option>, + + formats: Vec, +} + +impl Downloader { + pub fn add_format(&mut self, format: DownloadFormat) { + self.formats.push(format); + } + + pub async fn download(mut self, ctx: &Context, dst: &Path) -> Result<()> { + if let Some(audio_sort_locales) = &self.audio_sort { + self.formats.sort_by(|a, b| { + audio_sort_locales + .iter() + .position(|l| l == &a.video.1) + .cmp(&audio_sort_locales.iter().position(|l| l == &b.video.1)) + }); + } + for format in self.formats.iter_mut() { + if let Some(audio_sort_locales) = &self.audio_sort { + format.audios.sort_by(|(_, a), (_, b)| { + audio_sort_locales + .iter() + .position(|l| l == a) + .cmp(&audio_sort_locales.iter().position(|l| l == b)) + }) + } + if let Some(subtitle_sort) = &self.subtitle_sort { + format.subtitles.sort_by(|a, b| { + subtitle_sort + .iter() + .position(|l| l == &a.locale) + .cmp(&subtitle_sort.iter().position(|l| l == &b.locale)) + }) + } + } + + let mut videos = vec![]; + let mut audios = vec![]; + let mut subtitles = vec![]; + + for (i, format) in self.formats.iter().enumerate() { + let fmt_space = format + .audios + .iter() + .map(|(_, locale)| format!("Downloading {} audio", locale).len()) + .max() + .unwrap(); + + let video_path = self + .download_video( + ctx, + &format.video.0, + format!("{:<1$}", format!("Downloading video #{}", i + 1), fmt_space), + ) + .await?; + for (variant_data, locale) in format.audios.iter() { + let audio_path = self + .download_audio( + ctx, + variant_data, + format!("{:<1$}", format!("Downloading {} audio", locale), fmt_space), + ) + .await?; + audios.push(FFmpegMeta { + path: audio_path, + language: locale.clone(), + title: if i == 0 { + locale.to_human_readable() + } else { + format!("{} [Video: #{}]", locale.to_human_readable(), i + 1) + }, + }) + } + let len = get_video_length(&video_path)?; + for subtitle in format.subtitles.iter() { + let subtitle_path = self.download_subtitle(subtitle.clone(), len).await?; + subtitles.push(FFmpegMeta { + path: subtitle_path, + language: subtitle.locale.clone(), + title: if i == 0 { + subtitle.locale.to_human_readable() + } else { + format!( + "{} [Video: #{}]", + subtitle.locale.to_human_readable(), + i + 1 + ) + }, + }) + } + videos.push(FFmpegMeta { + path: video_path, + language: format.video.1.clone(), + title: if self.formats.len() == 1 { + "Default".to_string() + } else { + format!("#{}", i + 1) + }, + }); + } + + let mut input = vec![]; + let mut maps = vec![]; + let mut metadata = vec![]; + + for (i, meta) in videos.iter().enumerate() { + input.extend(["-i".to_string(), meta.path.to_string_lossy().to_string()]); + maps.extend(["-map".to_string(), i.to_string()]); + metadata.extend([ + format!("-metadata:s:v:{}", i), + format!("title={}", meta.title), + ]); + // the empty language metadata is created to avoid that metadata from the original track + // is copied + metadata.extend([format!("-metadata:s:v:{}", i), format!("language=")]) + } + for (i, meta) in audios.iter().enumerate() { + input.extend(["-i".to_string(), meta.path.to_string_lossy().to_string()]); + maps.extend(["-map".to_string(), (i + videos.len()).to_string()]); + metadata.extend([ + format!("-metadata:s:a:{}", i), + format!("language={}", meta.language), + ]); + metadata.extend([ + format!("-metadata:s:a:{}", i), + format!("title={}", meta.title), + ]); + } + for (i, meta) in subtitles.iter().enumerate() { + input.extend(["-i".to_string(), meta.path.to_string_lossy().to_string()]); + maps.extend([ + "-map".to_string(), + (i + videos.len() + audios.len()).to_string(), + ]); + metadata.extend([ + format!("-metadata:s:s:{}", i), + format!("language={}", meta.language), + ]); + metadata.extend([ + format!("-metadata:s:s:{}", i), + format!("title={}", meta.title), + ]); + } + + let (input_presets, mut output_presets) = self.ffmpeg_preset.into_input_output_args(); + + let mut command_args = vec!["-y".to_string(), "-hide_banner".to_string()]; + command_args.extend(input_presets); + command_args.extend(input); + command_args.extend(maps); + command_args.extend(metadata); + + // set default subtitle + if let Some(default_subtitle) = self.default_subtitle { + if let Some(position) = subtitles + .iter() + .position(|m| m.language == default_subtitle) + { + match dst.extension().unwrap_or_default().to_str().unwrap() { + "mp4" => output_presets.extend([ + "-movflags".to_string(), + "faststart".to_string(), + "-c:s".to_string(), + "mov_text".to_string(), + format!("-disposition:s:s:{}", position), + "forced".to_string(), + ]), + "mkv" => output_presets.extend([ + format!("-disposition:s:s:{}", position), + "forced".to_string(), + ]), + _ => { + // remove '-c:v copy' and '-c:a copy' from output presets as its causes issues with + // burning subs into the video + let mut last = String::new(); + let mut remove_count = 0; + for (i, s) in output_presets.clone().iter().enumerate() { + if (last == "-c:v" || last == "-c:a") && s == "copy" { + // remove last + output_presets.remove(i - remove_count - 1); + remove_count += 1; + output_presets.remove(i - remove_count); + remove_count += 1; + } + last = s.clone(); + } + + output_presets.extend([ + "-vf".to_string(), + format!( + "subtitles={}", + subtitles.get(position).unwrap().path.to_str().unwrap() + ), + ]) + } + } + } + + if let Some(position) = subtitles + .iter() + .position(|meta| meta.language == default_subtitle) + { + command_args.extend([ + format!("-disposition:s:s:{}", position), + "forced".to_string(), + ]) + } + } + + command_args.extend(output_presets); + if let Some(output_format) = self.output_format { + command_args.extend(["-f".to_string(), output_format]); + } + command_args.push(dst.to_str().unwrap().to_string()); + + debug!("ffmpeg {}", command_args.join(" ")); + + // create parent directory if it does not exist + if let Some(parent) = dst.parent() { + if !parent.exists() { + std::fs::create_dir_all(parent)? + } + } + + let progress_handler = progress!("Generating output file"); + + let ffmpeg = Command::new("ffmpeg") + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .args(command_args) + .output()?; + if !ffmpeg.status.success() { + bail!("{}", String::from_utf8_lossy(ffmpeg.stderr.as_slice())) + } + + progress_handler.stop("Output file generated"); + + Ok(()) + } + + async fn download_video( + &self, + ctx: &Context, + variant_data: &VariantData, + message: String, + ) -> Result { + let tempfile = tempfile(".mp4")?; + let (mut file, path) = tempfile.into_parts(); + + download_segments(ctx, &mut file, Some(message), variant_data).await?; + + Ok(path) + } + + async fn download_audio( + &self, + ctx: &Context, + variant_data: &VariantData, + message: String, + ) -> Result { + let tempfile = tempfile(".m4a")?; + let (mut file, path) = tempfile.into_parts(); + + download_segments(ctx, &mut file, Some(message), variant_data).await?; + + Ok(path) + } + + async fn download_subtitle( + &self, + subtitle: Subtitle, + max_length: NaiveTime, + ) -> Result { + let tempfile = tempfile(".ass")?; + let (mut file, path) = tempfile.into_parts(); + + let mut buf = vec![]; + subtitle.write_to(&mut buf).await?; + fix_subtitle_look_and_feel(&mut buf); + fix_subtitle_length(&mut buf, max_length); + + file.write_all(buf.as_slice())?; + + Ok(path) + } +} + +pub async fn download_segments( + ctx: &Context, + writer: &mut impl Write, + message: Option, + variant_data: &VariantData, +) -> Result<()> { + let segments = variant_data.segments().await?; + let total_segments = segments.len(); + + let client = Arc::new(ctx.crunchy.client()); + let count = Arc::new(Mutex::new(0)); + + let progress = if log::max_level() == LevelFilter::Info { + let estimated_file_size = + (variant_data.bandwidth / 8) * segments.iter().map(|s| s.length.as_secs()).sum::(); + + let progress = ProgressBar::new(estimated_file_size) + .with_style( + ProgressStyle::with_template( + ":: {msg}{bytes:>10} {bytes_per_sec:>12} [{wide_bar}] {percent:>3}%", + ) + .unwrap() + .progress_chars("##-"), + ) + .with_message(message.map(|m| m + " ").unwrap_or_default()) + .with_finish(ProgressFinish::Abandon); + Some(progress) + } else { + None + }; + + let cpus = num_cpus::get(); + let mut segs: Vec> = Vec::with_capacity(cpus); + for _ in 0..cpus { + segs.push(vec![]) + } + for (i, segment) in segments.clone().into_iter().enumerate() { + segs[i - ((i / cpus) * cpus)].push(segment); + } + + let (sender, receiver) = mpsc::channel(); + + let mut join_set: JoinSet> = JoinSet::new(); + for num in 0..cpus { + let thread_client = client.clone(); + let thread_sender = sender.clone(); + let thread_segments = segs.remove(0); + let thread_count = count.clone(); + join_set.spawn(async move { + let after_download_sender = thread_sender.clone(); + + // the download process is encapsulated in its own function. this is done to easily + // catch errors which get returned with `...?` and `bail!(...)` and that the thread + // itself can report that an error has occurred + let download = || async move { + for (i, segment) in thread_segments.into_iter().enumerate() { + let mut retry_count = 0; + let mut buf = loop { + let response = thread_client + .get(&segment.url) + .timeout(Duration::from_secs(60)) + .send() + .await?; + + match response.bytes().await { + Ok(b) => break b.to_vec(), + Err(e) => { + if e.is_body() { + if retry_count == 5 { + bail!("Max retry count reached ({}), multiple errors occurred while receiving segment {}: {}", retry_count, num + (i * cpus), e) + } + debug!("Failed to download segment {} ({}). Retrying, {} out of 5 retries left", num + (i * cpus), e, 5 - retry_count) + } else { + bail!("{}", e) + } + } + } + + retry_count += 1; + }; + + buf = VariantSegment::decrypt(buf.borrow_mut(), segment.key)?.to_vec(); + + let mut c = thread_count.lock().unwrap(); + debug!( + "Downloaded and decrypted segment [{}/{} {:.2}%] {}", + num + (i * cpus) + 1, + total_segments, + ((*c + 1) as f64 / total_segments as f64) * 100f64, + segment.url + ); + + thread_sender.send((num as i32 + (i * cpus) as i32, buf))?; + + *c += 1; + } + Ok(()) + }; + + + let result = download().await; + if result.is_err() { + after_download_sender.send((-1 as i32, vec![]))?; + } + + result + }); + } + // drop the sender already here so it does not outlive all download threads which are the only + // real consumers of it + drop(sender); + + // this is the main loop which writes the data. it uses a BTreeMap as a buffer as the write + // happens synchronized. the download consist of multiple segments. the map keys are representing + // the segment number and the values the corresponding bytes + let mut data_pos = 0; + let mut buf: BTreeMap> = BTreeMap::new(); + for (pos, bytes) in receiver.iter() { + // if the position is lower than 0, an error occurred in the sending download thread + if pos < 0 { + break; + } + + if let Some(p) = &progress { + let progress_len = p.length().unwrap(); + let estimated_segment_len = + (variant_data.bandwidth / 8) * segments.get(pos as usize).unwrap().length.as_secs(); + let bytes_len = bytes.len() as u64; + + p.set_length(progress_len - estimated_segment_len + bytes_len); + p.inc(bytes_len) + } + + // check if the currently sent bytes are the next in the buffer. if so, write them directly + // to the target without first adding them to the buffer. + // if not, add them to the buffer + if data_pos == pos { + writer.write_all(bytes.borrow())?; + data_pos += 1; + } else { + buf.insert(pos, bytes); + } + // check if the buffer contains the next segment(s) + while let Some(b) = buf.remove(&data_pos) { + writer.write_all(b.borrow())?; + data_pos += 1; + } + } + + // if any error has occurred while downloading it gets returned here + while let Some(joined) = join_set.join_next().await { + joined?? + } + + // write the remaining buffer, if existent + while let Some(b) = buf.remove(&data_pos) { + writer.write_all(b.borrow())?; + data_pos += 1; + } + + if !buf.is_empty() { + bail!( + "Download buffer is not empty. Remaining segments: {}", + buf.into_keys() + .map(|k| k.to_string()) + .collect::>() + .join(", ") + ) + } + + Ok(()) +} + +/// Add `ScaledBorderAndShadows: yes` to subtitles; without it they look very messy on some video +/// players. See [crunchy-labs/crunchy-cli#66](https://github.com/crunchy-labs/crunchy-cli/issues/66) +/// for more information. +fn fix_subtitle_look_and_feel(raw: &mut Vec) { + let mut script_info = false; + let mut new = String::new(); + + for line in String::from_utf8_lossy(raw.as_slice()).split('\n') { + if line.trim().starts_with('[') && script_info { + new.push_str("ScaledBorderAndShadow: yes\n"); + script_info = false + } else if line.trim() == "[Script Info]" { + script_info = true + } + new.push_str(line); + new.push('\n') + } + + *raw = new.into_bytes() +} + +/// Get the length of a video. This is required because sometimes subtitles have an unnecessary entry +/// long after the actual video ends with artificially extends the video length on some video players. +/// To prevent this, the video length must be hard set. See +/// [crunchy-labs/crunchy-cli#32](https://github.com/crunchy-labs/crunchy-cli/issues/32) for more +/// information. +pub fn get_video_length(path: &Path) -> Result { + let video_length = Regex::new(r"Duration:\s(?P