From 87f57bacfca53393b93fede7d437ac90f5e75397 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 17 Jan 2022 09:16:26 +0100 Subject: [PATCH 01/82] Renamed MatchVideo to ParseVideoURL --- crunchyroll.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/crunchyroll.go b/crunchyroll.go index e47182a..6eccde5 100644 --- a/crunchyroll.go +++ b/crunchyroll.go @@ -283,7 +283,7 @@ func (c *Crunchyroll) Search(query string, limit uint) (s []*Series, m []*Movie, // FindVideo finds a Video (Season or Movie) by a crunchyroll link // e.g. https://www.crunchyroll.com/darling-in-the-franxx func (c *Crunchyroll) FindVideo(seriesUrl string) (Video, error) { - if series, ok := MatchVideo(seriesUrl); ok { + if series, ok := ParseVideoURL(seriesUrl); ok { s, m, err := c.Search(series, 1) if err != nil { return nil, err @@ -331,8 +331,16 @@ func (c *Crunchyroll) FindEpisode(url string) ([]*Episode, error) { return nil, errors.New("invalid url") } -// MatchVideo tries to extract the crunchyroll series / movie name out of the given url -func MatchVideo(url string) (seriesName string, ok bool) { +// MatchEpisode tries to extract the crunchyroll series name and title out of the given url +// +// Deprecated: Use ParseEpisodeURL instead +func MatchEpisode(url string) (seriesName, title string, ok bool) { + seriesName, title, _, _, ok = ParseEpisodeURL(url) + return +} + +// ParseVideoURL tries to extract the crunchyroll series / movie name out of the given url +func ParseVideoURL(url string) (seriesName string, ok bool) { pattern := regexp.MustCompile(`(?m)^https?://(www\.)?crunchyroll\.com(/\w{2}(-\w{2})?)?/(?P[^/]+)/?$`) if urlMatch := pattern.FindAllStringSubmatch(url, -1); len(urlMatch) != 0 { groups := regexGroups(urlMatch, pattern.SubexpNames()...) @@ -345,14 +353,6 @@ func MatchVideo(url string) (seriesName string, ok bool) { return } -// MatchEpisode tries to extract the crunchyroll series name and title out of the given url -// -// Deprecated: Use ParseEpisodeURL instead -func MatchEpisode(url string) (seriesName, title string, ok bool) { - seriesName, title, _, _, ok = ParseEpisodeURL(url) - return -} - // ParseEpisodeURL tries to extract the crunchyroll series name, title, episode number and web id out of the given crunchyroll url // Note that the episode number can be misleading. For example if an episode has the episode number 23.5 (slime isekai) // the episode number will be 235 From 6b2d7c6e275c56f89664a3a22f81e39f8c939cff Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 17 Jan 2022 13:44:48 +0100 Subject: [PATCH 02/82] Removed FindVideo & FindEpisode; added FindVideoByName & FindEpisodeByName --- crunchyroll.go | 80 +++++++++++++++++++++++--------------------------- 1 file changed, 37 insertions(+), 43 deletions(-) diff --git a/crunchyroll.go b/crunchyroll.go index 6eccde5..457df0d 100644 --- a/crunchyroll.go +++ b/crunchyroll.go @@ -280,55 +280,49 @@ func (c *Crunchyroll) Search(query string, limit uint) (s []*Series, m []*Movie, return s, m, nil } -// FindVideo finds a Video (Season or Movie) by a crunchyroll link -// e.g. https://www.crunchyroll.com/darling-in-the-franxx -func (c *Crunchyroll) FindVideo(seriesUrl string) (Video, error) { - if series, ok := ParseVideoURL(seriesUrl); ok { - s, m, err := c.Search(series, 1) - if err != nil { - return nil, err - } - - if len(s) > 0 { - return s[0], nil - } else if len(m) > 0 { - return m[0], nil - } - return nil, errors.New("no series or movie could be found") +// FindVideoByName finds a Video (Season or Movie) by its name. +// Use this in combination with ParseVideoURL and hand over the corresponding results +// to this function. +func (c *Crunchyroll) FindVideoByName(seriesName string) (Video, error) { + s, m, err := c.Search(seriesName, 1) + if err != nil { + return nil, err } - return nil, errors.New("invalid url") + if len(s) > 0 { + return s[0], nil + } else if len(m) > 0 { + return m[0], nil + } + return nil, errors.New("no series or movie could be found") } -// FindEpisode finds an episode by its crunchyroll link -// e.g. https://www.crunchyroll.com/darling-in-the-franxx/episode-1-alone-and-lonesome-759575 -func (c *Crunchyroll) FindEpisode(url string) ([]*Episode, error) { - if series, title, _, _, ok := ParseEpisodeURL(url); ok { - video, err := c.FindVideo(fmt.Sprintf("https://www.crunchyroll.com/%s", series)) - if err != nil { - return nil, err - } - seasons, err := video.(*Series).Seasons() - if err != nil { - return nil, err - } - - var matchingEpisodes []*Episode - for _, season := range seasons { - episodes, err := season.Episodes() - if err != nil { - return nil, err - } - for _, episode := range episodes { - if episode.SlugTitle == title { - matchingEpisodes = append(matchingEpisodes, episode) - } - } - } - return matchingEpisodes, nil +// FindEpisodeByName finds an episode by its crunchyroll series name and episode title. +// Use this in combination with ParseEpisodeURL and hand over the corresponding results +// to this function. +func (c *Crunchyroll) FindEpisodeByName(seriesName, episodeTitle string) ([]*Episode, error) { + video, err := c.FindVideoByName(seriesName) + if err != nil { + return nil, err + } + seasons, err := video.(*Series).Seasons() + if err != nil { + return nil, err } - return nil, errors.New("invalid url") + var matchingEpisodes []*Episode + for _, season := range seasons { + episodes, err := season.Episodes() + if err != nil { + return nil, err + } + for _, episode := range episodes { + if episode.SlugTitle == episodeTitle { + matchingEpisodes = append(matchingEpisodes, episode) + } + } + } + return matchingEpisodes, nil } // MatchEpisode tries to extract the crunchyroll series name and title out of the given url From 08dd79ff478efd0e79e5b1e09f5aed27aa142c1e Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 17 Jan 2022 15:22:18 +0100 Subject: [PATCH 03/82] Removed structure utils api --- utils/structure.go | 693 --------------------------------------------- 1 file changed, 693 deletions(-) delete mode 100644 utils/structure.go diff --git a/utils/structure.go b/utils/structure.go deleted file mode 100644 index 127a379..0000000 --- a/utils/structure.go +++ /dev/null @@ -1,693 +0,0 @@ -package utils - -import ( - "errors" - "github.com/ByteDream/crunchyroll-go" - "sync" -) - -// StructureError is the error type which is thrown whenever a structure fails -// to receive information (formats, episodes, ...) from the api endpoint -type StructureError struct { - error -} - -func IsStructureError(err error) (ok bool) { - if err != nil { - _, ok = err.(*StructureError) - } - return -} - -// FormatStructure is the basic structure which every other structure implements. -// With it, and all other structures the api usage can be simplified magnificent -type FormatStructure struct { - // initState is true if every format, stream, ... in the structure tree is initialized - initState bool - - // getFunc specified the function which will be called if crunchyroll.Format is empty / not initialized yet. - // It returns the formats itself, the parent streams (might be nil) and an error if one occurs - getFunc func() ([]*crunchyroll.Format, []*crunchyroll.Stream, error) - // formats holds all formats which were given - formats []*crunchyroll.Format - // parents holds all parents which were given - parents []*crunchyroll.Stream -} - -func newFormatStructure(parentStructure *StreamStructure) *FormatStructure { - return &FormatStructure{ - getFunc: func() (formats []*crunchyroll.Format, parents []*crunchyroll.Stream, err error) { - streams, err := parentStructure.Streams() - if err != nil { - return - } - - var wg sync.WaitGroup - var lock sync.Mutex - - for _, stream := range streams { - wg.Add(1) - stream := stream - go func() { - defer wg.Done() - f, err := stream.Formats() - if err != nil { - errors.As(err, &StructureError{}) - return - } - lock.Lock() - defer lock.Unlock() - for _, format := range f { - formats = append(formats, format) - parents = append(parents, stream) - } - }() - } - wg.Wait() - return - }, - } -} - -// NewFormatStructure returns a new FormatStructure, based on the given formats -func NewFormatStructure(formats []*crunchyroll.Format) *FormatStructure { - return &FormatStructure{ - getFunc: func() ([]*crunchyroll.Format, []*crunchyroll.Stream, error) { - return formats, nil, nil - }, - } -} - -// Formats returns all stored formats -func (fs *FormatStructure) Formats() ([]*crunchyroll.Format, error) { - var err error - if fs.formats == nil { - if fs.formats, fs.parents, err = fs.getFunc(); err != nil { - return nil, err - } - fs.initState = true - } - return fs.formats, nil -} - -// FormatParent returns the parent stream of a format (if present). -// If the format or parent is not stored, an error will be returned -func (fs *FormatStructure) FormatParent(format *crunchyroll.Format) (*crunchyroll.Stream, error) { - formats, err := fs.Formats() - if err != nil { - return nil, err - } - - if fs.parents == nil { - return nil, errors.New("no parents are given") - } - - for i, f := range formats { - if f == format { - return fs.parents[i], nil - } - } - return nil, errors.New("given format could not be found") -} - -// InitAll recursive requests all given information. -// All functions of FormatStructure or other structs in this file which are executed after this have a much lesser chance to return any error, -// so the error return value of these functions can be pretty safely ignored. -// This function should only be called if you need to the access to any function of FormatStructure which returns a crunchyroll.Format (or an array of it). -// Re-calling this method can lead to heavy problems (believe me, it caused a simple bug and i've tried to fix it for several hours). -// Check FormatStructure.InitAllState if you can call this method without causing bugs -func (fs *FormatStructure) InitAll() error { - var err error - if fs.formats, fs.parents, err = fs.getFunc(); err != nil { - return err - } - fs.initState = true - return nil -} - -// InitAllState returns FormatStructure.InitAll or FormatStructure.Formats was called. -// If so, all errors which are returned by functions of structs in this file can be safely ignored -func (fs *FormatStructure) InitAllState() bool { - return fs.initState -} - -// AvailableLocales returns all available audio, subtitle and hardsub locales for all formats. -// If includeEmpty is given, locales with no value are included too -func (fs *FormatStructure) AvailableLocales(includeEmpty bool) (audioLocales []crunchyroll.LOCALE, subtitleLocales []crunchyroll.LOCALE, hardsubLocales []crunchyroll.LOCALE, err error) { - var formats []*crunchyroll.Format - if formats, err = fs.Formats(); err != nil { - return - } - - audioMap := map[crunchyroll.LOCALE]interface{}{} - subtitleMap := map[crunchyroll.LOCALE]interface{}{} - hardsubMap := map[crunchyroll.LOCALE]interface{}{} - for _, format := range formats { - // audio locale should always have a valid locale - if includeEmpty || !includeEmpty && format.AudioLocale != "" { - audioMap[format.AudioLocale] = nil - } - if format.Subtitles != nil { - for _, subtitle := range format.Subtitles { - if subtitle.Locale == "" && !includeEmpty { - continue - } - subtitleMap[subtitle.Locale] = nil - } - } - if includeEmpty || !includeEmpty && format.Hardsub != "" { - hardsubMap[format.Hardsub] = nil - } - } - - for k := range audioMap { - audioLocales = append(audioLocales, k) - } - for k := range subtitleMap { - subtitleLocales = append(subtitleLocales, k) - } - for k := range hardsubMap { - hardsubLocales = append(hardsubLocales, k) - } - return -} - -// FilterFormatsByAudio returns all formats which have the given locale as their audio locale -func (fs *FormatStructure) FilterFormatsByAudio(locale crunchyroll.LOCALE) (f []*crunchyroll.Format, err error) { - var formats []*crunchyroll.Format - if formats, err = fs.Formats(); err != nil { - return nil, err - } - for _, format := range formats { - if format.AudioLocale == locale { - f = append(f, format) - } - } - return -} - -// FilterFormatsBySubtitle returns all formats which have the given locale as their subtitle locale. -// Hardsub indicates if the subtitle should be shown on the video itself -func (fs *FormatStructure) FilterFormatsBySubtitle(locale crunchyroll.LOCALE, hardsub bool) (f []*crunchyroll.Format, err error) { - var formats []*crunchyroll.Format - if formats, err = fs.Formats(); err != nil { - return nil, err - } - for _, format := range formats { - if hardsub && format.Hardsub == locale { - f = append(f, format) - } else if !hardsub && format.Hardsub == "" { - f = append(f, format) - } - } - return -} - -// FilterFormatsByLocales returns all formats which have the given locales as their property. -// Hardsub is the same as in FormatStructure.FilterFormatsBySubtitle -func (fs *FormatStructure) FilterFormatsByLocales(audioLocale, subtitleLocale crunchyroll.LOCALE, hardsub bool) ([]*crunchyroll.Format, error) { - var f []*crunchyroll.Format - - formats, err := fs.Formats() - if err != nil { - return nil, err - } - for _, format := range formats { - if format.AudioLocale == audioLocale { - if hardsub && format.Hardsub == subtitleLocale { - f = append(f, format) - } else if !hardsub && format.Hardsub == "" { - f = append(f, format) - } - } - } - if len(f) == 0 { - return nil, errors.New("could not find any matching format") - } - return f, nil -} - -// OrderFormatsByID loops through all stored formats and returns a 2d slice -// where a row represents an id and the column all formats which have this id -func (fs *FormatStructure) OrderFormatsByID() ([][]*crunchyroll.Format, error) { - formats, err := fs.Formats() - if err != nil { - return nil, err - } - - formatsMap := map[string][]*crunchyroll.Format{} - for _, format := range formats { - if _, ok := formatsMap[format.ID]; !ok { - formatsMap[format.ID] = make([]*crunchyroll.Format, 0) - } - formatsMap[format.ID] = append(formatsMap[format.ID], format) - } - - var orderedFormats [][]*crunchyroll.Format - for _, v := range formatsMap { - var f []*crunchyroll.Format - for _, format := range v { - f = append(f, format) - } - orderedFormats = append(orderedFormats, f) - } - return orderedFormats, nil -} - -// StreamStructure fields are nearly same as FormatStructure -type StreamStructure struct { - *FormatStructure - - getFunc func() ([]*crunchyroll.Stream, []crunchyroll.Video, error) - streams []*crunchyroll.Stream - parents []crunchyroll.Video -} - -func newStreamStructure(structure VideoStructure) *StreamStructure { - var getFunc func() (streams []*crunchyroll.Stream, parents []crunchyroll.Video, err error) - switch structure.(type) { - case *EpisodeStructure: - episodeStructure := structure.(*EpisodeStructure) - getFunc = func() (streams []*crunchyroll.Stream, parents []crunchyroll.Video, err error) { - episodes, err := episodeStructure.Episodes() - if err != nil { - return - } - - var wg sync.WaitGroup - var lock sync.Mutex - - for _, episode := range episodes { - wg.Add(1) - episode := episode - go func() { - defer wg.Done() - s, err := episode.Streams() - if err != nil { - errors.As(err, &StructureError{}) - return - } - lock.Lock() - defer lock.Unlock() - for _, stream := range s { - streams = append(streams, stream) - parents = append(parents, episode) - } - }() - } - wg.Wait() - return - } - case *MovieListingStructure: - movieListingStructure := structure.(*MovieListingStructure) - getFunc = func() (streams []*crunchyroll.Stream, parents []crunchyroll.Video, err error) { - movieListings, err := movieListingStructure.MovieListings() - if err != nil { - return - } - - var wg sync.WaitGroup - var lock sync.Mutex - - for _, movieListing := range movieListings { - wg.Add(1) - movieListing := movieListing - go func() { - defer wg.Done() - s, err := movieListing.Streams() - if err != nil { - errors.As(err, &StructureError{}) - return - } - lock.Lock() - defer lock.Unlock() - for _, stream := range s { - streams = append(streams, stream) - parents = append(parents, movieListing) - } - }() - } - wg.Wait() - return - } - } - - ss := &StreamStructure{ - getFunc: getFunc, - } - ss.FormatStructure = newFormatStructure(ss) - return ss -} - -// NewStreamStructure returns a new StreamStructure, based on the given formats -func NewStreamStructure(streams []*crunchyroll.Stream) *StreamStructure { - ss := &StreamStructure{ - getFunc: func() ([]*crunchyroll.Stream, []crunchyroll.Video, error) { - return streams, nil, nil - }, - } - ss.FormatStructure = newFormatStructure(ss) - return ss -} - -// Streams returns all stored streams -func (ss *StreamStructure) Streams() ([]*crunchyroll.Stream, error) { - if ss.streams == nil { - var err error - if ss.streams, ss.parents, err = ss.getFunc(); err != nil { - return nil, err - } - } - return ss.streams, nil -} - -// StreamParent returns the parent video (type crunchyroll.Series or crunchyroll.Movie) of a stream (if present). -// If the stream or parent is not stored, an error will be returned -func (ss *StreamStructure) StreamParent(stream *crunchyroll.Stream) (crunchyroll.Video, error) { - streams, err := ss.Streams() - if err != nil { - return nil, err - } - - if ss.parents == nil { - return nil, errors.New("no parents are given") - } - - for i, s := range streams { - if s == stream { - return ss.parents[i], nil - } - } - return nil, errors.New("given stream could not be found") -} - -// VideoStructure is an interface which is implemented by EpisodeStructure and MovieListingStructure -type VideoStructure interface{} - -// EpisodeStructure fields are nearly same as FormatStructure -type EpisodeStructure struct { - VideoStructure - *StreamStructure - - getFunc func() ([]*crunchyroll.Episode, []*crunchyroll.Season, error) - episodes []*crunchyroll.Episode - parents []*crunchyroll.Season -} - -func newEpisodeStructure(structure *SeasonStructure) *EpisodeStructure { - es := &EpisodeStructure{ - getFunc: func() (episodes []*crunchyroll.Episode, parents []*crunchyroll.Season, err error) { - seasons, err := structure.Seasons() - if err != nil { - return - } - - var wg sync.WaitGroup - var lock sync.Mutex - - for _, season := range seasons { - wg.Add(1) - season := season - go func() { - defer wg.Done() - e, err := season.Episodes() - if err != nil { - errors.As(err, &StructureError{}) - return - } - lock.Lock() - defer lock.Unlock() - for _, episode := range e { - episodes = append(episodes, episode) - parents = append(parents, season) - } - }() - } - wg.Wait() - return - }, - } - es.StreamStructure = newStreamStructure(es) - return es -} - -// NewEpisodeStructure returns a new EpisodeStructure, based on the given formats -func NewEpisodeStructure(episodes []*crunchyroll.Episode) *EpisodeStructure { - es := &EpisodeStructure{ - getFunc: func() ([]*crunchyroll.Episode, []*crunchyroll.Season, error) { - return episodes, nil, nil - }, - } - es.StreamStructure = newStreamStructure(es) - return es -} - -// Episodes returns all stored episodes -func (es *EpisodeStructure) Episodes() ([]*crunchyroll.Episode, error) { - if es.episodes == nil { - var err error - if es.episodes, es.parents, err = es.getFunc(); err != nil { - return nil, err - } - } - return es.episodes, nil -} - -// EpisodeParent returns the parent season of a stream (if present). -// If the stream or parent is not stored, an error will be returned -func (es *EpisodeStructure) EpisodeParent(episode *crunchyroll.Episode) (*crunchyroll.Season, error) { - episodes, err := es.Episodes() - if err != nil { - return nil, err - } - - if es.parents == nil { - return nil, errors.New("no parents are given") - } - - for i, e := range episodes { - if e == episode { - return es.parents[i], nil - } - } - return nil, errors.New("given episode could not be found") -} - -// GetEpisodeByFormat returns the episode to which the given format belongs to. -// If the format or the parent is not stored, an error will be returned -func (es *EpisodeStructure) GetEpisodeByFormat(format *crunchyroll.Format) (*crunchyroll.Episode, error) { - if !es.initState { - if err := es.InitAll(); err != nil { - return nil, err - } - } - - formatParent, err := es.FormatParent(format) - if err != nil { - return nil, err - } - streamParent, err := es.StreamParent(formatParent) - if err != nil { - return nil, err - } - episode, ok := streamParent.(*crunchyroll.Episode) - if !ok { - return nil, errors.New("could not find parent episode") - } - return episode, nil -} - -// GetEpisodeByURL returns an episode by its url -func (es *EpisodeStructure) GetEpisodeByURL(url string) (*crunchyroll.Episode, error) { - _, title, episodeNumber, _, ok := crunchyroll.ParseEpisodeURL(url) - if !ok { - if episodeid, ok := crunchyroll.ParseBetaEpisodeURL(url); ok { - episodes, err := es.Episodes() - if err != nil { - return nil, err - } - - for _, episode := range episodes { - if episode.ID == episodeid { - return episode, nil - } - } - } - - return nil, errors.New("invalid url") - } - - episodes, err := es.Episodes() - if err != nil { - return nil, err - } - - for _, episode := range episodes { - if episode.SlugTitle == title { - return episode, nil - } - } - - for _, episode := range episodes { - if episode.EpisodeNumber == episodeNumber { - return episode, nil - } - } - return nil, errors.New("no episode could be found") -} - -// OrderEpisodeByID orders episodes by their ids -func (es *EpisodeStructure) OrderEpisodeByID() ([][]*crunchyroll.Episode, error) { - episodes, err := es.Episodes() - if err != nil { - return nil, err - } - - episodesMap := map[string][]*crunchyroll.Episode{} - for _, episode := range episodes { - if _, ok := episodesMap[episode.ID]; !ok { - episodesMap[episode.ID] = make([]*crunchyroll.Episode, 0) - } - episodesMap[episode.ID] = append(episodesMap[episode.ID], episode) - } - - var orderedEpisodes [][]*crunchyroll.Episode - for _, v := range episodesMap { - orderedEpisodes = append(orderedEpisodes, v) - } - return orderedEpisodes, nil -} - -// OrderFormatsByEpisodeNumber orders episodes by their episode number. -// Episode number 1 is on position 1 in the slice, number 2 on position 2, and so on. -// This was made intentionally because there is a chance that episodes with the episode number 0 are existing -// and position 0 in the slice is reserved for them. -// Therefore, if the first episode number is, for example, 20, the first 19 array entries will be nil -func (es *EpisodeStructure) OrderFormatsByEpisodeNumber() ([][]*crunchyroll.Format, error) { - formats, err := es.Formats() - if err != nil { - return nil, err - } - - formatsMap := map[int][]*crunchyroll.Format{} - for _, format := range formats { - stream, err := es.FormatParent(format) - if err != nil { - return nil, err - } - video, err := es.StreamParent(stream) - if err != nil { - return nil, err - } - - episode, ok := video.(*crunchyroll.Episode) - if !ok { - continue - } - if _, ok := formatsMap[episode.EpisodeNumber]; !ok { - formatsMap[episode.EpisodeNumber] = make([]*crunchyroll.Format, 0) - } - formatsMap[episode.EpisodeNumber] = append(formatsMap[episode.EpisodeNumber], format) - } - - var highest int - for key := range formatsMap { - if key > highest { - highest = key - } - } - - var orderedFormats [][]*crunchyroll.Format - for i := 0; i < highest+1; i++ { - if formats, ok := formatsMap[i]; ok { - orderedFormats = append(orderedFormats, formats) - } else { - // simply adds nil in case that no episode with number i exists - orderedFormats = append(orderedFormats, nil) - } - } - return orderedFormats, nil -} - -// SeasonStructure fields are nearly same as FormatStructure -type SeasonStructure struct { - *EpisodeStructure - - getFunc func() ([]*crunchyroll.Season, error) - seasons []*crunchyroll.Season -} - -// NewSeasonStructure returns a new SeasonStructure, based on the given formats -func NewSeasonStructure(seasons []*crunchyroll.Season) *SeasonStructure { - ss := &SeasonStructure{ - seasons: seasons, - } - ss.EpisodeStructure = newEpisodeStructure(ss) - return ss -} - -// Seasons returns all stored seasons -func (ss *SeasonStructure) Seasons() ([]*crunchyroll.Season, error) { - if ss.seasons == nil { - var err error - if ss.seasons, err = ss.getFunc(); err != nil { - return nil, err - } - } - return ss.seasons, nil -} - -// MovieListingStructure fields are nearly same as FormatStructure -type MovieListingStructure struct { - VideoStructure - *StreamStructure - - getFunc func() ([]*crunchyroll.MovieListing, error) - movieListings []*crunchyroll.MovieListing -} - -// NewMovieListingStructure returns a new MovieListingStructure, based on the given formats -func NewMovieListingStructure(movieListings []*crunchyroll.MovieListing) *MovieListingStructure { - ml := &MovieListingStructure{ - getFunc: func() ([]*crunchyroll.MovieListing, error) { - return movieListings, nil - }, - } - ml.StreamStructure = newStreamStructure(ml) - return ml -} - -// MovieListings returns all stored movie listings -func (ml *MovieListingStructure) MovieListings() ([]*crunchyroll.MovieListing, error) { - if ml.movieListings == nil { - var err error - if ml.movieListings, err = ml.getFunc(); err != nil { - return nil, err - } - } - return ml.movieListings, nil -} - -// GetMovieListingByFormat returns the movie listing to which the given format belongs to. -// If the format or the parent is not stored, an error will be returned -func (ml *MovieListingStructure) GetMovieListingByFormat(format *crunchyroll.Format) (*crunchyroll.MovieListing, error) { - if !ml.initState { - if err := ml.InitAll(); err != nil { - return nil, err - } - } - - formatParent, err := ml.FormatParent(format) - if err != nil { - return nil, err - } - streamParent, err := ml.StreamParent(formatParent) - if err != nil { - return nil, err - } - movieListing, ok := streamParent.(*crunchyroll.MovieListing) - if !ok { - return nil, errors.New("could not find parent movie listing") - } - return movieListing, nil -} From d1235fea1af060a2127847b1d8f0993850e4e5bc Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 17 Jan 2022 15:53:28 +0100 Subject: [PATCH 04/82] Added walk utils api --- utils/walk.go | 210 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 utils/walk.go diff --git a/utils/walk.go b/utils/walk.go new file mode 100644 index 0000000..4eb9ff3 --- /dev/null +++ b/utils/walk.go @@ -0,0 +1,210 @@ +package utils + +import ( + "fmt" + "github.com/ByteDream/crunchyroll-go" +) + +// EpisodeWalker is an easy-to-use struct which walks down given urls +// and triggers functions where you can perform further actions with +// the delivered objects +type EpisodeWalker struct { + // The Crunchyroll instance to perform all actions on + Crunchyroll *crunchyroll.Crunchyroll + + // If CheckDuplicates is true, duplicated urls, seasons and episodes + // are filtered out and the values given in OnSeason for example is + // always unique + CheckDuplicates bool + + // OnUrl gets called when an url is parsed. + // The error is generally only nil when the url is invalid + OnUrl func(url string, err error) error + // OnSeason gets called when a season was parsed + OnSeason func(season *crunchyroll.Season, err error) error + // OnEpisode gets called when a season was parsed + OnEpisode func(episode *crunchyroll.Episode, err error) error +} + +// WalkURLs walks through all urls. +// Urls to seasons and episodes are support, normal as well as beta urls +func (ew EpisodeWalker) WalkURLs(urls []string) error { + var episodeIds, seasonIds, seriesNames []string + var episodeNames map[string][]string + + for _, url := range urls { + if episodeId, ok := crunchyroll.ParseBetaEpisodeURL(url); ok && !(ew.CheckDuplicates && sliceContains(episodeIds, episodeId)) { + episodeIds = append(episodeIds, episodeId) + } else if seasonId, ok := crunchyroll.ParseBetaSeriesURL(url); ok && !(ew.CheckDuplicates && sliceContains(seasonIds, seasonId)) { + seasonIds = append(seasonIds, seasonId) + } else if seriesName, title, _, _, ok := crunchyroll.ParseEpisodeURL(url); ok { + if eps, ok := episodeNames[seriesName]; ok { + if !ew.CheckDuplicates || !sliceContains(eps, title) { + eps = append(eps, title) + } + } else { + episodeNames[seriesName] = []string{title} + } + } else if seriesName, ok := crunchyroll.ParseVideoURL(url); ok && !(ew.CheckDuplicates && sliceContains(seriesNames, seriesName)) { + seriesNames = append(seriesNames, seriesName) + } else { + err := fmt.Errorf("invalid url %s", url) + if ew.OnUrl != nil { + if err = ew.OnUrl(url, err); err != nil { + return err + } + continue + } else { + return err + } + } + + if ew.OnUrl != nil { + if err := ew.OnUrl(url, nil); err != nil { + return err + } + } + } + + for _, name := range seriesNames { + video, err := ew.Crunchyroll.FindVideoByName(name) + if err != nil { + return err + } + // in all cases i've ever tested video was a series - even + // if it was listed as a movie on the crunchyroll website. + // i just hope no error occurs here :) + seasons, err := video.(*crunchyroll.Series).Seasons() + if err != nil { + return err + } + for _, season := range seasons { + if ew.CheckDuplicates { + if sliceContains(seasonIds, season.ID) { + continue + } + seasonIds = append(seasonIds, season.ID) + } + if ew.OnSeason != nil { + if err := ew.OnSeason(season, nil); err != nil { + return err + } + } + } + } + + if err := ew.walkEpisodeIds(episodeIds); err != nil { + return err + } else if err := ew.walkSeasonIds(seasonIds); err != nil { + return err + } else if err := ew.walkEpisodeNames(episodeNames); err != nil { + return err + } + return nil +} + +func (ew EpisodeWalker) walkEpisodeIds(episodeIds []string) error { + var episodeIdsCheck []string + + for _, id := range episodeIds { + if ew.CheckDuplicates { + if sliceContains(episodeIdsCheck, id) { + continue + } + episodeIdsCheck = append(episodeIdsCheck, id) + } + + episode, err := crunchyroll.EpisodeFromID(ew.Crunchyroll, id) + if ew.OnEpisode != nil { + if err = ew.OnEpisode(episode, err); err != nil { + return err + } + } + } + return nil +} + +func (ew EpisodeWalker) walkSeasonIds(seasonIds []string) error { + var episodeIdsCheck []string + + for _, id := range seasonIds { + season, err := crunchyroll.SeasonFromID(ew.Crunchyroll, id) + if ew.OnSeason != nil { + if err = ew.OnSeason(season, err); err != nil { + return err + } + } else if err != nil { + return err + } + eps, err := season.Episodes() + if err != nil { + return err + } + for _, ep := range eps { + if ew.CheckDuplicates { + if sliceContains(episodeIdsCheck, ep.ID) { + continue + } + episodeIdsCheck = append(episodeIdsCheck, ep.ID) + } + + if ew.OnEpisode != nil { + if err = ew.OnEpisode(ep, nil); err != nil { + return err + } + } + } + } + return nil +} + +func (ew EpisodeWalker) walkEpisodeNames(episodeNames map[string][]string) error { + var episodeIdsCheck []string + + for seriesName, episodeName := range episodeNames { + video, err := ew.Crunchyroll.FindVideoByName(seriesName) + if err != nil { + return err + } + series := video.(*crunchyroll.Series) + seasons, err := series.Seasons() + if err != nil { + return err + } + for _, season := range seasons { + eps, err := season.Episodes() + if err != nil { + return err + } + for _, ep := range eps { + for _, name := range episodeName { + if ep.SlugTitle == name { + if ew.OnEpisode != nil { + if ew.CheckDuplicates { + if sliceContains(episodeIdsCheck, ep.ID) { + continue + } + episodeIdsCheck = append(episodeIdsCheck, ep.ID) + } + + if err = ew.OnEpisode(ep, nil); err != nil { + return err + } + } + break + } + } + } + } + } + return nil +} + +func sliceContains(slice []string, elem string) bool { + for _, s := range slice { + if elem == s { + return true + } + } + return false +} From 30e86631865df6f3989add87a2d6ee378637f1ce Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 17 Jan 2022 16:20:05 +0100 Subject: [PATCH 05/82] Added new api fields --- episode.go | 15 +++++++++++---- season.go | 44 +++++++++++++++++++++++++++----------------- 2 files changed, 38 insertions(+), 21 deletions(-) diff --git a/episode.go b/episode.go index 247967f..712f8cb 100644 --- a/episode.go +++ b/episode.go @@ -12,10 +12,17 @@ type Episode struct { siteCache map[string]interface{} - ID string `json:"id"` - SeriesID string `json:"series_id"` - SeriesTitle string `json:"series_title"` - SeasonNumber int `json:"season_number"` + ID string `json:"id"` + ChannelID string `json:"channel_id"` + + SeriesID string `json:"series_id"` + SeriesTitle string `json:"series_title"` + SeriesSlugTitle string `json:"series_slug_title"` + + SeasonID string `json:"season_id"` + SeasonTitle string `json:"season_title"` + SeasonSlugTitle string `json:"season_slug_title"` + SeasonNumber int `json:"season_number"` Episode string `json:"episode"` EpisodeNumber int `json:"episode_number"` diff --git a/season.go b/season.go index 0ec0802..9021af7 100644 --- a/season.go +++ b/season.go @@ -9,24 +9,34 @@ import ( type Season struct { crunchy *Crunchyroll - ID string `json:"id"` - Title string `json:"title"` - SlugTitle string `json:"slug_title"` - SeriesID string `json:"series_id"` - SeasonNumber int `json:"season_number"` - IsComplete bool `json:"is_complete"` - Description string `json:"description"` - Keywords []string `json:"keywords"` - SeasonTags []string `json:"season_tags"` - IsMature bool `json:"is_mature"` - MatureBlocked bool `json:"mature_blocked"` - IsSubbed bool `json:"is_subbed"` - IsDubbed bool `json:"is_dubbed"` - IsSimulcast bool `json:"is_simulcast"` - SeoTitle string `json:"seo_title"` - SeoDescription string `json:"seo_description"` + ID string `json:"id"` + ChannelID string `json:"channel_id"` - Language LOCALE + Title string `json:"title"` + SlugTitle string `json:"slug_title"` + + SeriesID string `json:"series_id"` + SeasonNumber int `json:"season_number"` + + IsComplete bool `json:"is_complete"` + + Description string `json:"description"` + Keywords []string `json:"keywords"` + SeasonTags []string `json:"season_tags"` + IsMature bool `json:"is_mature"` + MatureBlocked bool `json:"mature_blocked"` + IsSubbed bool `json:"is_subbed"` + IsDubbed bool `json:"is_dubbed"` + IsSimulcast bool `json:"is_simulcast"` + + SeoTitle string `json:"seo_title"` + SeoDescription string `json:"seo_description"` + + AvailabilityNotes string `json:"availability_notes"` + + // the locales are always empty, idk why this may change in the future + AudioLocales []LOCALE + SubtitleLocales []LOCALE } // SeasonFromID returns a season by its api id From effd160dc3895ad4bccf6ad3f4d2dc37fb223009 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 17 Jan 2022 19:40:33 +0100 Subject: [PATCH 06/82] Moved logger to separate file --- cmd/crunchyroll-go/cmd/logger.go | 199 +++++++++++++++++++++++++++++++ cmd/crunchyroll-go/cmd/utils.go | 190 ----------------------------- 2 files changed, 199 insertions(+), 190 deletions(-) create mode 100644 cmd/crunchyroll-go/cmd/logger.go diff --git a/cmd/crunchyroll-go/cmd/logger.go b/cmd/crunchyroll-go/cmd/logger.go new file mode 100644 index 0000000..1079dc6 --- /dev/null +++ b/cmd/crunchyroll-go/cmd/logger.go @@ -0,0 +1,199 @@ +package cmd + +import ( + "fmt" + "io" + "log" + "os" + "runtime" + "strings" + "sync" + "time" +) + +type progress struct { + status bool + message string +} + +type logger struct { + DebugLog *log.Logger + InfoLog *log.Logger + ErrLog *log.Logger + + devView bool + + progressWG sync.Mutex + progress chan progress +} + +func newLogger(debug, info, err bool, color bool) *logger { + debugLog, infoLog, errLog := log.New(io.Discard, "=> ", 0), log.New(io.Discard, "=> ", 0), log.New(io.Discard, "=> ", 0) + + debugColor, infoColor, errColor := "", "", "" + if color && runtime.GOOS != "windows" { + debugColor, infoColor, errColor = "\033[95m", "\033[96m", "\033[31m" + } + + if debug { + debugLog.SetOutput(&loggerWriter{original: os.Stdout, color: debugColor}) + } + if info { + infoLog.SetOutput(&loggerWriter{original: os.Stdout, color: infoColor}) + } + if err { + errLog.SetOutput(&loggerWriter{original: os.Stdout, color: errColor}) + } + + if debug { + debugLog = log.New(debugLog.Writer(), "[debug] ", 0) + infoLog = log.New(infoLog.Writer(), "[info] ", 0) + errLog = log.New(errLog.Writer(), "[err] ", 0) + } + + return &logger{ + DebugLog: debugLog, + InfoLog: infoLog, + ErrLog: errLog, + + devView: debug, + } +} + +func (l *logger) Empty() { + if !l.devView && l.InfoLog.Writer() != io.Discard { + fmt.Println() + } +} + +func (l *logger) StartProgress(message string) { + if l.devView { + l.InfoLog.Println(message) + return + } + l.progress = make(chan progress) + + go func() { + states := []string{"-", "\\", "|", "/"} + for i := 0; ; i++ { + l.progressWG.Lock() + select { + case p := <-l.progress: + // clearing the last line + fmt.Printf("\r%s\r", strings.Repeat(" ", len(l.InfoLog.Prefix())+len(message)+2)) + if p.status { + successTag := "✔" + if runtime.GOOS == "windows" { + successTag = "~" + } + l.InfoLog.Printf("%s %s", successTag, p.message) + } else { + errorTag := "✘" + if runtime.GOOS == "windows" { + errorTag = "!" + } + l.ErrLog.Printf("%s %s", errorTag, p.message) + } + l.progress = nil + l.progressWG.Unlock() + return + default: + if i%10 == 0 { + fmt.Printf("\r%s%s %s", l.InfoLog.Prefix(), states[i/10%4], message) + } + time.Sleep(35 * time.Millisecond) + l.progressWG.Unlock() + } + } + }() +} + +func (l *logger) StartProgressf(message string, a ...interface{}) { + l.StartProgress(fmt.Sprintf(message, a...)) +} + +func (l *logger) EndProgress(successful bool, message string) { + if l.devView { + if successful { + l.InfoLog.Print(message) + } else { + l.ErrLog.Print(message) + } + return + } else if l.progress != nil { + l.progress <- progress{ + status: successful, + message: message, + } + } +} + +func (l *logger) EndProgressf(successful bool, message string, a ...interface{}) { + l.EndProgress(successful, fmt.Sprintf(message, a...)) +} + +func (l *logger) Debugln(v ...interface{}) { + l.print(0, v...) +} + +func (l *logger) Debugf(message string, a ...interface{}) { + l.print(0, fmt.Sprintf(message, a...)) +} + +func (l *logger) Infoln(v ...interface{}) { + l.print(1, v...) +} + +func (l *logger) Infof(message string, a ...interface{}) { + l.print(1, fmt.Sprintf(message, a...)) +} + +func (l *logger) Errln(v ...interface{}) { + l.print(2, v...) +} + +func (l *logger) Errf(message string, a ...interface{}) { + l.print(2, fmt.Sprintf(message, a...)) +} + +func (l *logger) Fatalln(v ...interface{}) { + l.print(2, v...) + os.Exit(1) +} + +func (l *logger) Fatalf(message string, a ...interface{}) { + l.print(2, fmt.Sprintf(message, a...)) + os.Exit(1) +} + +func (l *logger) print(level int, v ...interface{}) { + if l.progress != nil { + l.progressWG.Lock() + defer l.progressWG.Unlock() + fmt.Print("\r") + } + + switch level { + case 0: + l.DebugLog.Print(v...) + case 1: + l.InfoLog.Print(v...) + case 2: + l.ErrLog.Print(v...) + } +} + +type loggerWriter struct { + io.Writer + + original io.Writer + color string +} + +func (lw *loggerWriter) Write(p []byte) (n int, err error) { + if lw.color != "" { + p = append([]byte(lw.color), p...) + p = append(p, []byte("\033[0m")...) + } + return lw.original.Write(p) +} diff --git a/cmd/crunchyroll-go/cmd/utils.go b/cmd/crunchyroll-go/cmd/utils.go index 5f799ca..3c357f9 100644 --- a/cmd/crunchyroll-go/cmd/utils.go +++ b/cmd/crunchyroll-go/cmd/utils.go @@ -4,9 +4,7 @@ import ( "fmt" "github.com/ByteDream/crunchyroll-go" "github.com/ByteDream/crunchyroll-go/utils" - "io" "io/ioutil" - "log" "net/http" "net/url" "os" @@ -16,199 +14,11 @@ import ( "runtime" "strconv" "strings" - "sync" "time" ) var sessionIDPath = filepath.Join(os.TempDir(), ".crunchy") -type progress struct { - status bool - message string -} - -type logger struct { - DebugLog *log.Logger - InfoLog *log.Logger - ErrLog *log.Logger - - devView bool - - progressWG sync.Mutex - progress chan progress -} - -func newLogger(debug, info, err bool, color bool) *logger { - debugLog, infoLog, errLog := log.New(io.Discard, "=> ", 0), log.New(io.Discard, "=> ", 0), log.New(io.Discard, "=> ", 0) - - debugColor, infoColor, errColor := "", "", "" - if color && runtime.GOOS != "windows" { - debugColor, infoColor, errColor = "\033[95m", "\033[96m", "\033[31m" - } - - if debug { - debugLog.SetOutput(&loggerWriter{original: os.Stdout, color: debugColor}) - } - if info { - infoLog.SetOutput(&loggerWriter{original: os.Stdout, color: infoColor}) - } - if err { - errLog.SetOutput(&loggerWriter{original: os.Stdout, color: errColor}) - } - - if debug { - debugLog = log.New(debugLog.Writer(), "[debug] ", 0) - infoLog = log.New(infoLog.Writer(), "[info] ", 0) - errLog = log.New(errLog.Writer(), "[err] ", 0) - } - - return &logger{ - DebugLog: debugLog, - InfoLog: infoLog, - ErrLog: errLog, - - devView: debug, - } -} - -func (l *logger) Empty() { - if !l.devView && l.InfoLog.Writer() != io.Discard { - fmt.Println() - } -} - -func (l *logger) StartProgress(message string) { - if l.devView { - l.InfoLog.Println(message) - return - } - l.progress = make(chan progress) - - go func() { - states := []string{"-", "\\", "|", "/"} - for i := 0; ; i++ { - l.progressWG.Lock() - select { - case p := <-l.progress: - // clearing the last line - fmt.Printf("\r%s\r", strings.Repeat(" ", len(l.InfoLog.Prefix())+len(message)+2)) - if p.status { - successTag := "✔" - if runtime.GOOS == "windows" { - successTag = "~" - } - l.InfoLog.Printf("%s %s", successTag, p.message) - } else { - errorTag := "✘" - if runtime.GOOS == "windows" { - errorTag = "!" - } - l.ErrLog.Printf("%s %s", errorTag, p.message) - } - l.progress = nil - l.progressWG.Unlock() - return - default: - if i%10 == 0 { - fmt.Printf("\r%s%s %s", l.InfoLog.Prefix(), states[i/10%4], message) - } - time.Sleep(35 * time.Millisecond) - l.progressWG.Unlock() - } - } - }() -} - -func (l *logger) StartProgressf(message string, a ...interface{}) { - l.StartProgress(fmt.Sprintf(message, a...)) -} - -func (l *logger) EndProgress(successful bool, message string) { - if l.devView { - if successful { - l.InfoLog.Print(message) - } else { - l.ErrLog.Print(message) - } - return - } else if l.progress != nil { - l.progress <- progress{ - status: successful, - message: message, - } - } -} - -func (l *logger) EndProgressf(successful bool, message string, a ...interface{}) { - l.EndProgress(successful, fmt.Sprintf(message, a...)) -} - -func (l *logger) Debugln(v ...interface{}) { - l.print(0, v...) -} - -func (l *logger) Debugf(message string, a ...interface{}) { - l.print(0, fmt.Sprintf(message, a...)) -} - -func (l *logger) Infoln(v ...interface{}) { - l.print(1, v...) -} - -func (l *logger) Infof(message string, a ...interface{}) { - l.print(1, fmt.Sprintf(message, a...)) -} - -func (l *logger) Errln(v ...interface{}) { - l.print(2, v...) -} - -func (l *logger) Errf(message string, a ...interface{}) { - l.print(2, fmt.Sprintf(message, a...)) -} - -func (l *logger) Fatalln(v ...interface{}) { - l.print(2, v...) - os.Exit(1) -} - -func (l *logger) Fatalf(message string, a ...interface{}) { - l.print(2, fmt.Sprintf(message, a...)) - os.Exit(1) -} - -func (l *logger) print(level int, v ...interface{}) { - if l.progress != nil { - l.progressWG.Lock() - defer l.progressWG.Unlock() - fmt.Print("\r") - } - - switch level { - case 0: - l.DebugLog.Print(v...) - case 1: - l.InfoLog.Print(v...) - case 2: - l.ErrLog.Print(v...) - } -} - -type loggerWriter struct { - io.Writer - - original io.Writer - color string -} - -func (lw *loggerWriter) Write(p []byte) (n int, err error) { - if lw.color != "" { - p = append([]byte(lw.color), p...) - p = append(p, []byte("\033[0m")...) - } - return lw.original.Write(p) -} - // systemLocale receives the system locale // https://stackoverflow.com/questions/51829386/golang-get-system-language/51831590#51831590 func systemLocale() crunchyroll.LOCALE { From 674b0e0de3b5482722a08be72b83bf42e39741ef Mon Sep 17 00:00:00 2001 From: bytedream Date: Tue, 18 Jan 2022 23:02:01 +0100 Subject: [PATCH 07/82] Added development notice --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 66eb204..b19ab6f 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +**This branch is under highly development, so it may contain errors which are making compiling not possible** + # crunchyroll-go A [Go](https://golang.org) library & cli for the undocumented [crunchyroll](https://www.crunchyroll.com) api. From 21619fe33301d07d21262e29f24d5c2fc5c62450 Mon Sep 17 00:00:00 2001 From: bytedream Date: Tue, 18 Jan 2022 23:46:07 +0100 Subject: [PATCH 08/82] Removed deprecated functions --- cmd/crunchyroll-go/cmd/login.go | 6 +++--- crunchyroll.go | 8 -------- format.go | 10 ---------- 3 files changed, 3 insertions(+), 21 deletions(-) diff --git a/cmd/crunchyroll-go/cmd/login.go b/cmd/crunchyroll-go/cmd/login.go index fb4b85c..c2f07e0 100644 --- a/cmd/crunchyroll-go/cmd/login.go +++ b/cmd/crunchyroll-go/cmd/login.go @@ -30,7 +30,7 @@ func init() { } func loginCredentials(email, password string) error { - out.Debugln("Logging in via credentials") + out.Debug("Logging in via credentials") session, err := crunchyroll.LoginWithCredentials(email, password, locale, client) if err != nil { return err @@ -40,11 +40,11 @@ func loginCredentials(email, password string) error { func loginSessionID(sessionID string, alreadyChecked bool) error { if !alreadyChecked { - out.Debugln("Logging in via session id") + out.Debug("Logging in via session id") if _, err := crunchyroll.LoginWithSessionID(sessionID, locale, client); err != nil { return err } } - out.Infoln("Due to security reasons, you have to login again on the next reboot") + out.Info("Due to security reasons, you have to login again on the next reboot") return ioutil.WriteFile(sessionIDPath, []byte(sessionID), 0777) } diff --git a/crunchyroll.go b/crunchyroll.go index 457df0d..3d5adf4 100644 --- a/crunchyroll.go +++ b/crunchyroll.go @@ -325,14 +325,6 @@ func (c *Crunchyroll) FindEpisodeByName(seriesName, episodeTitle string) ([]*Epi return matchingEpisodes, nil } -// MatchEpisode tries to extract the crunchyroll series name and title out of the given url -// -// Deprecated: Use ParseEpisodeURL instead -func MatchEpisode(url string) (seriesName, title string, ok bool) { - seriesName, title, _, _, ok = ParseEpisodeURL(url) - return -} - // ParseVideoURL tries to extract the crunchyroll series / movie name out of the given url func ParseVideoURL(url string) (seriesName string, ok bool) { pattern := regexp.MustCompile(`(?m)^https?://(www\.)?crunchyroll\.com(/\w{2}(-\w{2})?)?/(?P[^/]+)/?$`) diff --git a/format.go b/format.go index 7fbdb58..fb388e1 100644 --- a/format.go +++ b/format.go @@ -37,16 +37,6 @@ type Format struct { Subtitles []*Subtitle } -// Download calls DownloadGoroutines with 4 goroutines. -// See DownloadGoroutines for more details -// -// Deprecated: Use DownloadGoroutines instead -func (f *Format) Download(output *os.File, onSegmentDownload func(segment *m3u8.MediaSegment, current, total int, file *os.File, err error) error) error { - return f.DownloadGoroutines(output, 4, func(segment *m3u8.MediaSegment, current, total int, file *os.File) error { - return onSegmentDownload(segment, current, total, file, nil) - }) -} - // DownloadGoroutines downloads the format to the given output file (as .ts file). // See Format.DownloadSegments for more information func (f *Format) DownloadGoroutines(output *os.File, goroutines int, onSegmentDownload func(segment *m3u8.MediaSegment, current, total int, file *os.File) error) error { From 0e8738a3044468041dc2bbd1a837c6a90914eb7a Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 6 Feb 2022 13:25:38 +0100 Subject: [PATCH 09/82] Fixed nil map error --- utils/walk.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/walk.go b/utils/walk.go index 4eb9ff3..f5e96d3 100644 --- a/utils/walk.go +++ b/utils/walk.go @@ -30,7 +30,7 @@ type EpisodeWalker struct { // Urls to seasons and episodes are support, normal as well as beta urls func (ew EpisodeWalker) WalkURLs(urls []string) error { var episodeIds, seasonIds, seriesNames []string - var episodeNames map[string][]string + episodeNames := make(map[string][]string) for _, url := range urls { if episodeId, ok := crunchyroll.ParseBetaEpisodeURL(url); ok && !(ew.CheckDuplicates && sliceContains(episodeIds, episodeId)) { From a5d9696c9cc50ea0be0c84085939f69f572f9da3 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 7 Feb 2022 14:07:35 +0100 Subject: [PATCH 10/82] Added caching --- crunchyroll.go | 18 ++++++++++++++++++ episode.go | 16 +++++++++++++++- season.go | 9 +++++++++ stream.go | 10 ++++++++++ video.go | 20 ++++++++++++++++++++ 5 files changed, 72 insertions(+), 1 deletion(-) diff --git a/crunchyroll.go b/crunchyroll.go index 3d5adf4..8c402d6 100644 --- a/crunchyroll.go +++ b/crunchyroll.go @@ -51,6 +51,9 @@ type Crunchyroll struct { ExternalID string MaturityRating string } + + // If cache is true, internal caching is enabled + cache bool } // LoginWithCredentials logs in via crunchyroll email and password @@ -86,6 +89,7 @@ func LoginWithSessionID(sessionID string, locale LOCALE, client *http.Client) (* Client: client, Locale: locale, SessionID: sessionID, + cache: true, } var endpoint string var err error @@ -231,6 +235,20 @@ func (c *Crunchyroll) request(endpoint string) (*http.Response, error) { return resp, err } +// IsCaching returns if data gets cached or not. +// See SetCaching for more information +func (c *Crunchyroll) IsCaching() bool { + return c.cache +} + +// SetCaching enables or disables internal caching of requests made. +// Caching is enabled by default. +// If it is disabled the already cached data still gets called. +// The best way to prevent this is to create a complete new Crunchyroll struct +func (c *Crunchyroll) SetCaching(caching bool) { + c.cache = caching +} + // Search searches a query and returns all found series and movies within the given limit func (c *Crunchyroll) Search(query string, limit uint) (s []*Series, m []*Movie, err error) { searchEndpoint := fmt.Sprintf("https://beta-api.crunchyroll.com/content/v1/search?q=%s&n=%d&type=&locale=%s", diff --git a/episode.go b/episode.go index 712f8cb..36a8c34 100644 --- a/episode.go +++ b/episode.go @@ -10,6 +10,8 @@ import ( type Episode struct { crunchy *Crunchyroll + children []*Stream + siteCache map[string]interface{} ID string `json:"id"` @@ -134,7 +136,11 @@ func (e *Episode) AudioLocale() (LOCALE, error) { // Streams returns all streams which are available for the episode func (e *Episode) Streams() ([]*Stream, error) { - return fromVideoStreams(e.crunchy, fmt.Sprintf("https://beta-api.crunchyroll.com/cms/v2/%s/%s/%s/videos/%s/streams?locale=%s&Signature=%s&Policy=%s&Key-Pair-Id=%s", + if e.children != nil { + return e.children, nil + } + + streams, err := fromVideoStreams(e.crunchy, fmt.Sprintf("https://beta-api.crunchyroll.com/cms/v2/%s/%s/%s/videos/%s/streams?locale=%s&Signature=%s&Policy=%s&Key-Pair-Id=%s", e.crunchy.Config.CountryCode, e.crunchy.Config.MaturityRating, e.crunchy.Config.Channel, @@ -143,4 +149,12 @@ func (e *Episode) Streams() ([]*Stream, error) { e.crunchy.Config.Signature, e.crunchy.Config.Policy, e.crunchy.Config.KeyPairID)) + if err != nil { + return nil, err + } + + if e.crunchy.cache { + e.children = streams + } + return streams, nil } diff --git a/season.go b/season.go index 9021af7..16bc6d6 100644 --- a/season.go +++ b/season.go @@ -9,6 +9,8 @@ import ( type Season struct { crunchy *Crunchyroll + children []*Episode + ID string `json:"id"` ChannelID string `json:"channel_id"` @@ -69,6 +71,10 @@ func SeasonFromID(crunchy *Crunchyroll, id string) (*Season, error) { // Episodes returns all episodes which are available for the season func (s *Season) Episodes() (episodes []*Episode, err error) { + if s.children != nil { + return s.children, nil + } + resp, err := s.crunchy.request(fmt.Sprintf("https://beta-api.crunchyroll.com/cms/v2/%s/%s/%s/episodes?season_id=%s&locale=%s&Signature=%s&Policy=%s&Key-Pair-Id=%s", s.crunchy.Config.CountryCode, s.crunchy.Config.MaturityRating, @@ -101,5 +107,8 @@ func (s *Season) Episodes() (episodes []*Episode, err error) { episodes = append(episodes, episode) } + if s.crunchy.cache { + s.children = episodes + } return } diff --git a/stream.go b/stream.go index 59ab3c9..5fe0f96 100644 --- a/stream.go +++ b/stream.go @@ -11,6 +11,8 @@ import ( type Stream struct { crunchy *Crunchyroll + children []*Format + HardsubLocale LOCALE AudioLocale LOCALE Subtitles []*Subtitle @@ -35,6 +37,10 @@ func StreamsFromID(crunchy *Crunchyroll, id string) ([]*Stream, error) { // Formats returns all formats which are available for the stream func (s *Stream) Formats() ([]*Format, error) { + if s.children != nil { + return s.children, nil + } + resp, err := s.crunchy.Client.Get(s.streamURL) if err != nil { return nil, err @@ -57,6 +63,10 @@ func (s *Stream) Formats() ([]*Format, error) { Subtitles: s.Subtitles, }) } + + if s.crunchy.cache { + s.children = formats + } return formats, nil } diff --git a/video.go b/video.go index c39a4d0..1b83c90 100644 --- a/video.go +++ b/video.go @@ -38,6 +38,8 @@ type Movie struct { crunchy *Crunchyroll + children []*MovieListing + // not generated when calling MovieFromID MovieListingMetadata struct { AvailabilityNotes string `json:"availability_notes"` @@ -95,6 +97,10 @@ func MovieFromID(crunchy *Crunchyroll, id string) (*Movie, error) { // Beside the normal movie, sometimes movie previews are returned too, but you can try to get the actual movie // by sorting the returning MovieListing slice with the utils.MovieListingByDuration interface func (m *Movie) MovieListing() (movieListings []*MovieListing, err error) { + if m.children != nil { + return m.children, nil + } + resp, err := m.crunchy.request(fmt.Sprintf("https://beta-api.crunchyroll.com/cms/v2/%s/%s/%s/movies?movie_listing_id=%s&locale=%s&Signature=%s&Policy=%s&Key-Pair-Id=%s", m.crunchy.Config.CountryCode, m.crunchy.Config.MaturityRating, @@ -120,6 +126,10 @@ func (m *Movie) MovieListing() (movieListings []*MovieListing, err error) { } movieListings = append(movieListings, movieListing) } + + if m.crunchy.cache { + m.children = movieListings + } return movieListings, nil } @@ -129,6 +139,8 @@ type Series struct { crunchy *Crunchyroll + children []*Season + PromoDescription string `json:"promo_description"` PromoTitle string `json:"promo_title"` @@ -179,6 +191,10 @@ func SeriesFromID(crunchy *Crunchyroll, id string) (*Series, error) { // Seasons returns all seasons of a series func (s *Series) Seasons() (seasons []*Season, err error) { + if s.children != nil { + return s.children, nil + } + resp, err := s.crunchy.request(fmt.Sprintf("https://beta-api.crunchyroll.com/cms/v2/%s/%s/%s/seasons?series_id=%s&locale=%s&Signature=%s&Policy=%s&Key-Pair-Id=%s", s.crunchy.Config.CountryCode, s.crunchy.Config.MaturityRating, @@ -204,5 +220,9 @@ func (s *Series) Seasons() (seasons []*Season, err error) { } seasons = append(seasons, season) } + + if s.crunchy.cache { + s.children = seasons + } return } From 7dd74e793acce3f8e77cf370ea6bb6269ba00158 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 7 Feb 2022 14:09:22 +0100 Subject: [PATCH 11/82] Removed site cache --- episode.go | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/episode.go b/episode.go index 36a8c34..5a34366 100644 --- a/episode.go +++ b/episode.go @@ -12,8 +12,6 @@ type Episode struct { children []*Stream - siteCache map[string]interface{} - ID string `json:"id"` ChannelID string `json:"channel_id"` @@ -110,28 +108,13 @@ func EpisodeFromID(crunchy *Crunchyroll, id string) (*Episode, error) { // AudioLocale returns the audio locale of the episode. // Every episode in a season (should) have the same audio locale, -// so if you want to get the audio locale of a season, just call this method on the first episode of the season. -// Otherwise, if you call this function on every episode it will cause a massive delay and redundant network -// overload since it calls an api endpoint every time +// so if you want to get the audio locale of a season, just call this method on the first episode of the season func (e *Episode) AudioLocale() (LOCALE, error) { - resp, err := e.crunchy.request(fmt.Sprintf("https://beta-api.crunchyroll.com/cms/v2/%s/%s/%s/videos/%s/streams?locale=%s&Signature=%s&Policy=%s&Key-Pair-Id=%s", - e.crunchy.Config.CountryCode, - e.crunchy.Config.MaturityRating, - e.crunchy.Config.Channel, - e.StreamID, - e.crunchy.Locale, - e.crunchy.Config.Signature, - e.crunchy.Config.Policy, - e.crunchy.Config.KeyPairID)) + streams, err := e.Streams() if err != nil { return "", err } - defer resp.Body.Close() - var jsonBody map[string]interface{} - json.NewDecoder(resp.Body).Decode(&jsonBody) - e.siteCache = jsonBody - - return LOCALE(jsonBody["audio_locale"].(string)), nil + return streams[0].AudioLocale, nil } // Streams returns all streams which are available for the episode From 9c9a6f497fac675f578194476203ae56ec7298ed Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 7 Feb 2022 18:51:16 +0100 Subject: [PATCH 12/82] Added new download options --- downloader.go | 186 ++++++++++++++++++++++++++++++++++++++++++ format.go | 218 ++++++++++++++------------------------------------ 2 files changed, 247 insertions(+), 157 deletions(-) create mode 100644 downloader.go diff --git a/downloader.go b/downloader.go new file mode 100644 index 0000000..882904d --- /dev/null +++ b/downloader.go @@ -0,0 +1,186 @@ +package crunchyroll + +import ( + "crypto/aes" + "crypto/cipher" + "fmt" + "github.com/grafov/m3u8" + "io/ioutil" + "math" + "net/http" + "os" + "path/filepath" + "sync" + "sync/atomic" + "time" +) + +type Downloader struct { + // Filename is the filename of the output file + Filename string + // TempDir is the directory where the temporary files should be stored + TempDir string + // If IgnoreExisting is true, existing Filename's and TempDir's may be + // overwritten or deleted + IgnoreExisting bool + // If DeleteTempAfter is true, the temp directory gets deleted afterwards + DeleteTempAfter bool + + // Goroutines is the number of goroutines to download segments with + Goroutines int + + // A method to call when a segment was downloaded + OnSegmentDownload func(segment *m3u8.MediaSegment, current, total int, file *os.File) error + + // If FFmpeg is true, ffmpeg will used to merge and convert files + FFmpeg bool +} + +func NewDownloader(filename string, goroutines int, onSegmentDownload func(segment *m3u8.MediaSegment, current, total int, file *os.File) error) Downloader { + tmp, _ := os.MkdirTemp("", "crunchy_") + + return Downloader{ + Filename: filename, + TempDir: tmp, + DeleteTempAfter: true, + Goroutines: goroutines, + OnSegmentDownload: onSegmentDownload, + } +} + +// download downloads every mpeg transport stream segment to a given directory (more information below). +// After every segment download onSegmentDownload will be called with: +// the downloaded segment, the current position, the total size of segments to download, the file where the segment content was written to an error (if occurred). +// The filename is always .ts +// +// Short explanation: +// The actual crunchyroll video is split up in multiple segments (or video files) which have to be downloaded and merged after to generate a single video file. +// And this function just downloads each of this segment into the given directory. +// See https://en.wikipedia.org/wiki/MPEG_transport_stream for more information +func download(format *Format, tempDir string, goroutines int, onSegmentDownload func(segment *m3u8.MediaSegment, current, total int, file *os.File) error) error { + resp, err := format.crunchy.Client.Get(format.Video.URI) + if err != nil { + return err + } + defer resp.Body.Close() + // reads the m3u8 file + playlist, _, err := m3u8.DecodeFrom(resp.Body, true) + if err != nil { + return err + } + // extracts the segments from the playlist + var segments []*m3u8.MediaSegment + for _, segment := range playlist.(*m3u8.MediaPlaylist).Segments { + // some segments are nil, so they have to be filtered out + if segment != nil { + segments = append(segments, segment) + } + } + + var wg sync.WaitGroup + chunkSize := int(math.Ceil(float64(len(segments)) / float64(goroutines))) + + // when a onSegmentDownload call returns an error, this channel will be set to true and stop all goroutines + quit := make(chan bool) + + // receives the decrypt block and iv from the first segment. + // in my tests, only the first segment has specified this data, so the decryption data from this first segments will be used in every other segment too + block, iv, err := getCrypt(format, segments[0]) + if err != nil { + return err + } + + var total int32 + for i := 0; i < len(segments); i += chunkSize { + wg.Add(1) + end := i + chunkSize + if end > len(segments) { + end = len(segments) + } + i := i + + go func() { + for j, segment := range segments[i:end] { + select { + case <-quit: + break + default: + var file *os.File + k := 1 + for ; k < 4; k++ { + file, err = downloadSegment(format, segment, filepath.Join(tempDir, fmt.Sprintf("%d.ts", i+j)), block, iv) + if err == nil { + break + } + // sleep if an error occurs. very useful because sometimes the connection times out + time.Sleep(5 * time.Duration(k) * time.Second) + } + if k == 4 { + quit <- true + return + } + if onSegmentDownload != nil { + if err = onSegmentDownload(segment, int(atomic.AddInt32(&total, 1)), len(segments), file); err != nil { + quit <- true + file.Close() + return + } + } + file.Close() + } + } + wg.Done() + }() + } + wg.Wait() + + select { + case <-quit: + return err + default: + return nil + } +} + +// getCrypt extracts the key and iv of a m3u8 segment and converts it into a cipher.Block block and a iv byte sequence +func getCrypt(format *Format, segment *m3u8.MediaSegment) (block cipher.Block, iv []byte, err error) { + var resp *http.Response + + resp, err = format.crunchy.Client.Get(segment.Key.URI) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + key, err := ioutil.ReadAll(resp.Body) + + block, err = aes.NewCipher(key) + if err != nil { + return nil, nil, err + } + iv = []byte(segment.Key.IV) + if len(iv) == 0 { + iv = key + } + + return block, iv, nil +} + +// downloadSegment downloads a segment, decrypts it and names it after the given index +func downloadSegment(format *Format, segment *m3u8.MediaSegment, filename string, block cipher.Block, iv []byte) (*os.File, error) { + // every segment is aes-128 encrypted and has to be decrypted when downloaded + content, err := decryptSegment(format.crunchy.Client, segment, block, iv) + if err != nil { + return nil, err + } + + file, err := os.Create(filename) + if err != nil { + return nil, err + } + defer file.Close() + if _, err = file.Write(content); err != nil { + return nil, err + } + + return file, nil +} diff --git a/format.go b/format.go index fb388e1..49a67ba 100644 --- a/format.go +++ b/format.go @@ -2,29 +2,24 @@ package crunchyroll import ( "bufio" - "crypto/aes" - "crypto/cipher" "fmt" "github.com/grafov/m3u8" "io/ioutil" - "math" - "net/http" "os" + "os/exec" "path/filepath" "sort" "strconv" "strings" - "sync" - "sync/atomic" - "time" ) +type FormatType string + const ( EPISODE FormatType = "episodes" MOVIE = "movies" ) -type FormatType string type Format struct { crunchy *Crunchyroll @@ -37,166 +32,50 @@ type Format struct { Subtitles []*Subtitle } -// DownloadGoroutines downloads the format to the given output file (as .ts file). -// See Format.DownloadSegments for more information -func (f *Format) DownloadGoroutines(output *os.File, goroutines int, onSegmentDownload func(segment *m3u8.MediaSegment, current, total int, file *os.File) error) error { - downloadDir, err := os.MkdirTemp("", "crunchy_") - if err != nil { - return err +func (f *Format) Download(downloader Downloader) error { + if _, err := os.Stat(downloader.Filename); err == nil && !downloader.IgnoreExisting { + return fmt.Errorf("file %s already exists", downloader.Filename) } - defer os.RemoveAll(downloadDir) - - if err := f.DownloadSegments(downloadDir, goroutines, onSegmentDownload); err != nil { - return err - } - - return f.mergeSegments(downloadDir, output) -} - -// DownloadSegments downloads every mpeg transport stream segment to a given directory (more information below). -// After every segment download onSegmentDownload will be called with: -// the downloaded segment, the current position, the total size of segments to download, the file where the segment content was written to an error (if occurred). -// The filename is always .ts -// -// Short explanation: -// The actual crunchyroll video is split up in multiple segments (or video files) which have to be downloaded and merged after to generate a single video file. -// And this function just downloads each of this segment into the given directory. -// See https://en.wikipedia.org/wiki/MPEG_transport_stream for more information -func (f *Format) DownloadSegments(outputDir string, goroutines int, onSegmentDownload func(segment *m3u8.MediaSegment, current, total int, file *os.File) error) error { - resp, err := f.crunchy.Client.Get(f.Video.URI) - if err != nil { - return err - } - defer resp.Body.Close() - // reads the m3u8 file - playlist, _, err := m3u8.DecodeFrom(resp.Body, true) - if err != nil { - return err - } - // extracts the segments from the playlist - var segments []*m3u8.MediaSegment - for _, segment := range playlist.(*m3u8.MediaPlaylist).Segments { - // some segments are nil, so they have to be filtered out - if segment != nil { - segments = append(segments, segment) + if _, err := os.Stat(downloader.TempDir); err == nil && !downloader.IgnoreExisting { + content, err := os.ReadDir(downloader.TempDir) + if err != nil { + return err } - } - - var wg sync.WaitGroup - chunkSize := int(math.Ceil(float64(len(segments)) / float64(goroutines))) - - // when a onSegmentDownload call returns an error, this channel will be set to true and stop all goroutines - quit := make(chan bool) - - // receives the decrypt block and iv from the first segment. - // in my tests, only the first segment has specified this data, so the decryption data from this first segments will be used in every other segment too - block, iv, err := f.getCrypt(segments[0]) - if err != nil { - return err - } - - var total int32 - for i := 0; i < len(segments); i += chunkSize { - wg.Add(1) - end := i + chunkSize - if end > len(segments) { - end = len(segments) + if len(content) > 0 { + return fmt.Errorf("directory %s is not empty", downloader.Filename) } - i := i - - go func() { - for j, segment := range segments[i:end] { - select { - case <-quit: - break - default: - var file *os.File - k := 1 - for ; k < 4; k++ { - file, err = f.downloadSegment(segment, filepath.Join(outputDir, fmt.Sprintf("%d.ts", i+j)), block, iv) - if err == nil { - break - } - // sleep if an error occurs. very useful because sometimes the connection times out - time.Sleep(5 * time.Duration(k) * time.Second) - } - if k == 4 { - quit <- true - return - } - if onSegmentDownload != nil { - if err = onSegmentDownload(segment, int(atomic.AddInt32(&total, 1)), len(segments), file); err != nil { - quit <- true - file.Close() - return - } - } - file.Close() - } - } - wg.Done() - }() - } - wg.Wait() - - select { - case <-quit: + } else if err != nil && os.IsNotExist(err) { + if err := os.Mkdir(downloader.TempDir, 0755); err != nil { + return err + } + } else { return err - default: - return nil + } + + if err := download(f, downloader.TempDir, downloader.Goroutines, downloader.OnSegmentDownload); err != nil { + return err + } + + if downloader.FFmpeg { + return mergeSegmentsFFmpeg(downloader.TempDir, downloader.Filename) + } else { + return mergeSegments(downloader.TempDir, downloader.Filename) } } -// getCrypt extracts the key and iv of a m3u8 segment and converts it into a cipher.Block block and a iv byte sequence -func (f *Format) getCrypt(segment *m3u8.MediaSegment) (block cipher.Block, iv []byte, err error) { - var resp *http.Response - - resp, err = f.crunchy.Client.Get(segment.Key.URI) +// mergeSegments reads every file in tempDir and writes their content to the outputFile. +// The given output file gets created or overwritten if already existing +func mergeSegments(tempDir string, outputFile string) error { + dir, err := os.ReadDir(tempDir) if err != nil { - return nil, nil, err + return err } - defer resp.Body.Close() - key, err := ioutil.ReadAll(resp.Body) - - block, err = aes.NewCipher(key) + file, err := os.OpenFile(outputFile, os.O_CREATE|os.O_WRONLY, 0755) if err != nil { - return nil, nil, err - } - iv = []byte(segment.Key.IV) - if len(iv) == 0 { - iv = key - } - - return block, iv, nil -} - -// downloadSegment downloads a segment, decrypts it and names it after the given index -func (f *Format) downloadSegment(segment *m3u8.MediaSegment, filename string, block cipher.Block, iv []byte) (*os.File, error) { - // every segment is aes-128 encrypted and has to be decrypted when downloaded - content, err := decryptSegment(f.crunchy.Client, segment, block, iv) - if err != nil { - return nil, err - } - - file, err := os.Create(filename) - if err != nil { - return nil, err + return err } defer file.Close() - if _, err = file.Write(content); err != nil { - return nil, err - } - - return file, nil -} - -// mergeSegments reads every file in tempPath and writes their content to output -func (f *Format) mergeSegments(tempPath string, output *os.File) error { - dir, err := os.ReadDir(tempPath) - if err != nil { - return err - } - writer := bufio.NewWriter(output) + writer := bufio.NewWriter(file) defer writer.Flush() // sort the directory files after their numeric names @@ -213,7 +92,7 @@ func (f *Format) mergeSegments(tempPath string, output *os.File) error { }) for _, file := range dir { - bodyAsBytes, err := ioutil.ReadFile(filepath.Join(tempPath, file.Name())) + bodyAsBytes, err := ioutil.ReadFile(filepath.Join(tempDir, file.Name())) if err != nil { return err } @@ -223,3 +102,28 @@ func (f *Format) mergeSegments(tempPath string, output *os.File) error { } return nil } + +// mergeSegmentsFFmpeg reads every file in tempDir and merges their content to the outputFile +// with ffmpeg (https://ffmpeg.org/). +// The given output file gets created or overwritten if already existing +func mergeSegmentsFFmpeg(tempDir string, outputFile string) error { + dir, err := os.ReadDir(tempDir) + if err != nil { + return err + } + f, err := os.CreateTemp("", "*.txt") + if err != nil { + return err + } + defer os.Remove(f.Name()) + for i := 0; i < len(dir); i++ { + fmt.Fprintf(f, "file '%s.ts'\n", filepath.Join(tempDir, strconv.Itoa(i))) + } + cmd := exec.Command("ffmpeg", + "-f", "concat", + "-safe", "0", + "-i", f.Name(), + "-c", "copy", + outputFile) + return cmd.Run() +} From af2824f2168057e8df9e7ed467131764cf66f2d8 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 7 Feb 2022 18:51:28 +0100 Subject: [PATCH 13/82] Removed walk utility --- utils/walk.go | 210 -------------------------------------------------- 1 file changed, 210 deletions(-) delete mode 100644 utils/walk.go diff --git a/utils/walk.go b/utils/walk.go deleted file mode 100644 index f5e96d3..0000000 --- a/utils/walk.go +++ /dev/null @@ -1,210 +0,0 @@ -package utils - -import ( - "fmt" - "github.com/ByteDream/crunchyroll-go" -) - -// EpisodeWalker is an easy-to-use struct which walks down given urls -// and triggers functions where you can perform further actions with -// the delivered objects -type EpisodeWalker struct { - // The Crunchyroll instance to perform all actions on - Crunchyroll *crunchyroll.Crunchyroll - - // If CheckDuplicates is true, duplicated urls, seasons and episodes - // are filtered out and the values given in OnSeason for example is - // always unique - CheckDuplicates bool - - // OnUrl gets called when an url is parsed. - // The error is generally only nil when the url is invalid - OnUrl func(url string, err error) error - // OnSeason gets called when a season was parsed - OnSeason func(season *crunchyroll.Season, err error) error - // OnEpisode gets called when a season was parsed - OnEpisode func(episode *crunchyroll.Episode, err error) error -} - -// WalkURLs walks through all urls. -// Urls to seasons and episodes are support, normal as well as beta urls -func (ew EpisodeWalker) WalkURLs(urls []string) error { - var episodeIds, seasonIds, seriesNames []string - episodeNames := make(map[string][]string) - - for _, url := range urls { - if episodeId, ok := crunchyroll.ParseBetaEpisodeURL(url); ok && !(ew.CheckDuplicates && sliceContains(episodeIds, episodeId)) { - episodeIds = append(episodeIds, episodeId) - } else if seasonId, ok := crunchyroll.ParseBetaSeriesURL(url); ok && !(ew.CheckDuplicates && sliceContains(seasonIds, seasonId)) { - seasonIds = append(seasonIds, seasonId) - } else if seriesName, title, _, _, ok := crunchyroll.ParseEpisodeURL(url); ok { - if eps, ok := episodeNames[seriesName]; ok { - if !ew.CheckDuplicates || !sliceContains(eps, title) { - eps = append(eps, title) - } - } else { - episodeNames[seriesName] = []string{title} - } - } else if seriesName, ok := crunchyroll.ParseVideoURL(url); ok && !(ew.CheckDuplicates && sliceContains(seriesNames, seriesName)) { - seriesNames = append(seriesNames, seriesName) - } else { - err := fmt.Errorf("invalid url %s", url) - if ew.OnUrl != nil { - if err = ew.OnUrl(url, err); err != nil { - return err - } - continue - } else { - return err - } - } - - if ew.OnUrl != nil { - if err := ew.OnUrl(url, nil); err != nil { - return err - } - } - } - - for _, name := range seriesNames { - video, err := ew.Crunchyroll.FindVideoByName(name) - if err != nil { - return err - } - // in all cases i've ever tested video was a series - even - // if it was listed as a movie on the crunchyroll website. - // i just hope no error occurs here :) - seasons, err := video.(*crunchyroll.Series).Seasons() - if err != nil { - return err - } - for _, season := range seasons { - if ew.CheckDuplicates { - if sliceContains(seasonIds, season.ID) { - continue - } - seasonIds = append(seasonIds, season.ID) - } - if ew.OnSeason != nil { - if err := ew.OnSeason(season, nil); err != nil { - return err - } - } - } - } - - if err := ew.walkEpisodeIds(episodeIds); err != nil { - return err - } else if err := ew.walkSeasonIds(seasonIds); err != nil { - return err - } else if err := ew.walkEpisodeNames(episodeNames); err != nil { - return err - } - return nil -} - -func (ew EpisodeWalker) walkEpisodeIds(episodeIds []string) error { - var episodeIdsCheck []string - - for _, id := range episodeIds { - if ew.CheckDuplicates { - if sliceContains(episodeIdsCheck, id) { - continue - } - episodeIdsCheck = append(episodeIdsCheck, id) - } - - episode, err := crunchyroll.EpisodeFromID(ew.Crunchyroll, id) - if ew.OnEpisode != nil { - if err = ew.OnEpisode(episode, err); err != nil { - return err - } - } - } - return nil -} - -func (ew EpisodeWalker) walkSeasonIds(seasonIds []string) error { - var episodeIdsCheck []string - - for _, id := range seasonIds { - season, err := crunchyroll.SeasonFromID(ew.Crunchyroll, id) - if ew.OnSeason != nil { - if err = ew.OnSeason(season, err); err != nil { - return err - } - } else if err != nil { - return err - } - eps, err := season.Episodes() - if err != nil { - return err - } - for _, ep := range eps { - if ew.CheckDuplicates { - if sliceContains(episodeIdsCheck, ep.ID) { - continue - } - episodeIdsCheck = append(episodeIdsCheck, ep.ID) - } - - if ew.OnEpisode != nil { - if err = ew.OnEpisode(ep, nil); err != nil { - return err - } - } - } - } - return nil -} - -func (ew EpisodeWalker) walkEpisodeNames(episodeNames map[string][]string) error { - var episodeIdsCheck []string - - for seriesName, episodeName := range episodeNames { - video, err := ew.Crunchyroll.FindVideoByName(seriesName) - if err != nil { - return err - } - series := video.(*crunchyroll.Series) - seasons, err := series.Seasons() - if err != nil { - return err - } - for _, season := range seasons { - eps, err := season.Episodes() - if err != nil { - return err - } - for _, ep := range eps { - for _, name := range episodeName { - if ep.SlugTitle == name { - if ew.OnEpisode != nil { - if ew.CheckDuplicates { - if sliceContains(episodeIdsCheck, ep.ID) { - continue - } - episodeIdsCheck = append(episodeIdsCheck, ep.ID) - } - - if err = ew.OnEpisode(ep, nil); err != nil { - return err - } - } - break - } - } - } - } - } - return nil -} - -func sliceContains(slice []string, elem string) bool { - for _, s := range slice { - if elem == s { - return true - } - } - return false -} From 0cbcad2c403eac53b98ffc150abf6b8cfd1a4b58 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 7 Feb 2022 19:17:00 +0100 Subject: [PATCH 14/82] Added utility to work with urls --- utils/url.go | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 utils/url.go diff --git a/utils/url.go b/utils/url.go new file mode 100644 index 0000000..5181358 --- /dev/null +++ b/utils/url.go @@ -0,0 +1,82 @@ +package utils + +import ( + "fmt" + "github.com/ByteDream/crunchyroll-go" +) + +// ExtractEpisodesFromUrl extracts all episodes from an url. +// If audio is not empty, the episodes gets filtered after the given locale +func ExtractEpisodesFromUrl(crunchy *crunchyroll.Crunchyroll, url string, audio crunchyroll.LOCALE) ([]*crunchyroll.Episode, error) { + series, episodes, err := ParseUrl(crunchy, url) + if err != nil { + return nil, err + } + + var eps []*crunchyroll.Episode + + if series != nil { + seasons, err := series.Seasons() + if err != nil { + return nil, err + } + for _, season := range seasons { + if audio != "" { + if audioLocale, err := season.AudioLocale(); err != nil { + return nil, err + } else if audioLocale != audio { + continue + } + } + e, err := season.Episodes() + if err != nil { + return nil, err + } + eps = append(eps, e...) + } + } else if episodes != nil { + for _, episode := range episodes { + if audio == "" { + eps = append(eps, episode) + } else if audioLocale, err := episode.AudioLocale(); err != nil { + return nil, err + } else if audioLocale == audio { + eps = append(eps, episode) + } + } + } + + return eps, nil +} + +// ParseUrl parses the given url into a series or episode. +// The returning episode is a slice because non-beta urls have the same episode with different languages +func ParseUrl(crunchy *crunchyroll.Crunchyroll, url string) (*crunchyroll.Series, []*crunchyroll.Episode, error) { + if seriesId, ok := crunchyroll.ParseBetaSeriesURL(url); ok { + series, err := crunchyroll.SeriesFromID(crunchy, seriesId) + if err != nil { + return nil, nil, err + } + return series, nil, nil + } else if episodeId, ok := crunchyroll.ParseBetaEpisodeURL(url); ok { + episode, err := crunchyroll.EpisodeFromID(crunchy, episodeId) + if err != nil { + return nil, nil, err + } + return nil, []*crunchyroll.Episode{episode}, nil + } else if seriesName, ok := crunchyroll.ParseVideoURL(url); ok { + video, err := crunchy.FindVideoByName(seriesName) + if err != nil { + return nil, nil, err + } + return video.(*crunchyroll.Series), nil, nil + } else if seriesName, title, _, _, ok := crunchyroll.ParseEpisodeURL(url); ok { + episodes, err := crunchy.FindEpisodeByName(seriesName, title) + if err != nil { + return nil, nil, err + } + return nil, episodes, nil + } else { + return nil, nil, fmt.Errorf("invalid url %s", url) + } +} From 92fe5782d1592ca520689765e5eeda6dada35dcc Mon Sep 17 00:00:00 2001 From: bytedream Date: Thu, 10 Feb 2022 23:08:28 +0100 Subject: [PATCH 15/82] Added function to get specific format by resolution and subtitle --- episode.go | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/episode.go b/episode.go index 5a34366..84756d0 100644 --- a/episode.go +++ b/episode.go @@ -117,6 +117,44 @@ func (e *Episode) AudioLocale() (LOCALE, error) { return streams[0].AudioLocale, nil } +func (e *Episode) GetFormat(resolution string, subtitle LOCALE, hardsub bool) (*Format, error) { + streams, err := e.Streams() + if err != nil { + return nil, err + } + var foundStream *Stream + for _, stream := range streams { + if hardsub && stream.HardsubLocale == subtitle || stream.HardsubLocale == "" && subtitle == "" { + foundStream = stream + break + } else if !hardsub { + for _, subtitle := range stream.Subtitles { + if subtitle.Locale == subtitle.Locale { + foundStream = stream + break + } + } + if foundStream != nil { + break + } + } + } + + if foundStream == nil { + return nil, fmt.Errorf("no matching stream found") + } + formats, err := foundStream.Formats() + if err != nil { + return nil, err + } + for _, format := range formats { + if format.Video.Resolution == resolution { + return format, nil + } + } + return nil, fmt.Errorf("no matching resolution found") +} + // Streams returns all streams which are available for the episode func (e *Episode) Streams() ([]*Stream, error) { if e.children != nil { From 1c3de9ea935bb79021809f7f2f0c6559457ff5e0 Mon Sep 17 00:00:00 2001 From: bytedream Date: Thu, 10 Feb 2022 23:09:11 +0100 Subject: [PATCH 16/82] Added function to get audio locale --- season.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/season.go b/season.go index 16bc6d6..438236d 100644 --- a/season.go +++ b/season.go @@ -69,6 +69,14 @@ func SeasonFromID(crunchy *Crunchyroll, id string) (*Season, error) { return season, nil } +func (s *Season) AudioLocale() (LOCALE, error) { + episodes, err := s.Episodes() + if err != nil { + return "", err + } + return episodes[0].AudioLocale() +} + // Episodes returns all episodes which are available for the season func (s *Season) Episodes() (episodes []*Episode, err error) { if s.children != nil { From c557486089d9f510a8c5ed19b244e5367563e1db Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 13 Feb 2022 13:15:21 +0100 Subject: [PATCH 17/82] Added 'worst' and 'best' resolution filter to GetFormat(...) --- episode.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/episode.go b/episode.go index 84756d0..6392054 100644 --- a/episode.go +++ b/episode.go @@ -4,6 +4,8 @@ import ( "encoding/json" "fmt" "regexp" + "strconv" + "strings" "time" ) @@ -147,11 +149,33 @@ func (e *Episode) GetFormat(resolution string, subtitle LOCALE, hardsub bool) (* if err != nil { return nil, err } + var res *Format for _, format := range formats { + if resolution == "worst" || resolution == "best" { + curSplitRes := strings.SplitN(format.Video.Resolution, "x", 1) + curResX, _ := strconv.Atoi(curSplitRes[0]) + curResY, _ := strconv.Atoi(curSplitRes[1]) + + resSplitRes := strings.SplitN(res.Video.Resolution, "x", 1) + resResX, _ := strconv.Atoi(resSplitRes[0]) + resResY, _ := strconv.Atoi(resSplitRes[1]) + + if resolution == "worst" && curResX+curResY < resResX+resResY { + res = format + } else if resolution == "best" && curResX+curResY > resResX+resResY { + res = format + } + } + if format.Video.Resolution == resolution { return format, nil } } + + if res != nil { + return res, nil + } + return nil, fmt.Errorf("no matching resolution found") } From 0a4b9ec96e2a803b69521caa0b06e2b895b1f5ad Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 13 Feb 2022 15:01:05 +0100 Subject: [PATCH 18/82] Added context to downloader and moved method from utils to downloader --- downloader.go | 60 +++++++++++++++++++++++++++++++++++++++++++++------ utils.go | 32 --------------------------- 2 files changed, 53 insertions(+), 39 deletions(-) diff --git a/downloader.go b/downloader.go index 882904d..8e4d1a1 100644 --- a/downloader.go +++ b/downloader.go @@ -1,6 +1,7 @@ package crunchyroll import ( + "context" "crypto/aes" "crypto/cipher" "fmt" @@ -23,9 +24,14 @@ type Downloader struct { // If IgnoreExisting is true, existing Filename's and TempDir's may be // overwritten or deleted IgnoreExisting bool - // If DeleteTempAfter is true, the temp directory gets deleted afterwards + // If DeleteTempAfter is true, the temp directory gets deleted afterwards. + // Note that in case of a hard signal exit (os.Interrupt, ...) the directory + // will NOT be deleted. In such situations try to catch the signal and + // cancel Context DeleteTempAfter bool + Context context.Context + // Goroutines is the number of goroutines to download segments with Goroutines int @@ -43,6 +49,7 @@ func NewDownloader(filename string, goroutines int, onSegmentDownload func(segme Filename: filename, TempDir: tmp, DeleteTempAfter: true, + Context: context.Background(), Goroutines: goroutines, OnSegmentDownload: onSegmentDownload, } @@ -57,7 +64,7 @@ func NewDownloader(filename string, goroutines int, onSegmentDownload func(segme // The actual crunchyroll video is split up in multiple segments (or video files) which have to be downloaded and merged after to generate a single video file. // And this function just downloads each of this segment into the given directory. // See https://en.wikipedia.org/wiki/MPEG_transport_stream for more information -func download(format *Format, tempDir string, goroutines int, onSegmentDownload func(segment *m3u8.MediaSegment, current, total int, file *os.File) error) error { +func download(context context.Context, format *Format, tempDir string, goroutines int, onSegmentDownload func(segment *m3u8.MediaSegment, current, total int, file *os.File) error) error { resp, err := format.crunchy.Client.Get(format.Video.URI) if err != nil { return err @@ -100,15 +107,19 @@ func download(format *Format, tempDir string, goroutines int, onSegmentDownload i := i go func() { + defer wg.Done() + for j, segment := range segments[i:end] { select { + case <-context.Done(): + return case <-quit: - break + return default: var file *os.File k := 1 for ; k < 4; k++ { - file, err = downloadSegment(format, segment, filepath.Join(tempDir, fmt.Sprintf("%d.ts", i+j)), block, iv) + file, err = downloadSegment(context, format, segment, filepath.Join(tempDir, fmt.Sprintf("%d.ts", i+j)), block, iv) if err == nil { break } @@ -129,12 +140,13 @@ func download(format *Format, tempDir string, goroutines int, onSegmentDownload file.Close() } } - wg.Done() }() } wg.Wait() select { + case <-context.Done(): + return context.Err() case <-quit: return err default: @@ -166,9 +178,9 @@ func getCrypt(format *Format, segment *m3u8.MediaSegment) (block cipher.Block, i } // downloadSegment downloads a segment, decrypts it and names it after the given index -func downloadSegment(format *Format, segment *m3u8.MediaSegment, filename string, block cipher.Block, iv []byte) (*os.File, error) { +func downloadSegment(context context.Context, format *Format, segment *m3u8.MediaSegment, filename string, block cipher.Block, iv []byte) (*os.File, error) { // every segment is aes-128 encrypted and has to be decrypted when downloaded - content, err := decryptSegment(format.crunchy.Client, segment, block, iv) + content, err := decryptSegment(context, format.crunchy.Client, segment, block, iv) if err != nil { return nil, err } @@ -184,3 +196,37 @@ func downloadSegment(format *Format, segment *m3u8.MediaSegment, filename string return file, nil } + +// https://github.com/oopsguy/m3u8/blob/4150e93ec8f4f8718875a02973f5d792648ecb97/tool/crypt.go#L25 +func decryptSegment(context context.Context, client *http.Client, segment *m3u8.MediaSegment, block cipher.Block, iv []byte) ([]byte, error) { + req, err := http.NewRequest(http.MethodGet, segment.URI, nil) + if err != nil { + return nil, err + } + req.WithContext(context) + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + raw, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + blockMode := cipher.NewCBCDecrypter(block, iv[:block.BlockSize()]) + decrypted := make([]byte, len(raw)) + blockMode.CryptBlocks(decrypted, raw) + raw = pkcs5UnPadding(decrypted) + + return raw, nil +} + +// https://github.com/oopsguy/m3u8/blob/4150e93ec8f4f8718875a02973f5d792648ecb97/tool/crypt.go#L47 +func pkcs5UnPadding(origData []byte) []byte { + length := len(origData) + unPadding := int(origData[length-1]) + return origData[:(length - unPadding)] +} diff --git a/utils.go b/utils.go index 5983a60..a3d4191 100644 --- a/utils.go +++ b/utils.go @@ -1,11 +1,7 @@ package crunchyroll import ( - "crypto/cipher" "encoding/json" - "github.com/grafov/m3u8" - "io/ioutil" - "net/http" ) func decodeMapToStruct(m interface{}, s interface{}) error { @@ -16,34 +12,6 @@ func decodeMapToStruct(m interface{}, s interface{}) error { return json.Unmarshal(jsonBody, s) } -// https://github.com/oopsguy/m3u8/blob/4150e93ec8f4f8718875a02973f5d792648ecb97/tool/crypt.go#L25 -func decryptSegment(client *http.Client, segment *m3u8.MediaSegment, block cipher.Block, iv []byte) ([]byte, error) { - resp, err := client.Get(segment.URI) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - raw, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - blockMode := cipher.NewCBCDecrypter(block, iv[:block.BlockSize()]) - decrypted := make([]byte, len(raw)) - blockMode.CryptBlocks(decrypted, raw) - raw = pkcs5UnPadding(decrypted) - - return raw, nil -} - -// https://github.com/oopsguy/m3u8/blob/4150e93ec8f4f8718875a02973f5d792648ecb97/tool/crypt.go#L47 -func pkcs5UnPadding(origData []byte) []byte { - length := len(origData) - unPadding := int(origData[length-1]) - return origData[:(length - unPadding)] -} - func regexGroups(parsed [][]string, subexpNames ...string) map[string]string { groups := map[string]string{} for _, match := range parsed { From d5fc8824cf63af97eadc2d25191e4a892c9f2f4c Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 13 Feb 2022 15:03:53 +0100 Subject: [PATCH 19/82] Implemented downloader context --- format.go | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/format.go b/format.go index 49a67ba..f3d548f 100644 --- a/format.go +++ b/format.go @@ -2,6 +2,7 @@ package crunchyroll import ( "bufio" + "context" "fmt" "github.com/grafov/m3u8" "io/ioutil" @@ -52,20 +53,23 @@ func (f *Format) Download(downloader Downloader) error { return err } - if err := download(f, downloader.TempDir, downloader.Goroutines, downloader.OnSegmentDownload); err != nil { + if downloader.DeleteTempAfter { + defer os.RemoveAll(downloader.TempDir) + } + if err := download(downloader.Context, f, downloader.TempDir, downloader.Goroutines, downloader.OnSegmentDownload); err != nil { return err } if downloader.FFmpeg { - return mergeSegmentsFFmpeg(downloader.TempDir, downloader.Filename) + return mergeSegmentsFFmpeg(downloader.Context, downloader.TempDir, downloader.Filename) } else { - return mergeSegments(downloader.TempDir, downloader.Filename) + return mergeSegments(downloader.Context, downloader.TempDir, downloader.Filename) } } // mergeSegments reads every file in tempDir and writes their content to the outputFile. // The given output file gets created or overwritten if already existing -func mergeSegments(tempDir string, outputFile string) error { +func mergeSegments(context context.Context, tempDir string, outputFile string) error { dir, err := os.ReadDir(tempDir) if err != nil { return err @@ -92,6 +96,12 @@ func mergeSegments(tempDir string, outputFile string) error { }) for _, file := range dir { + select { + case <-context.Done(): + return context.Err() + default: + } + bodyAsBytes, err := ioutil.ReadFile(filepath.Join(tempDir, file.Name())) if err != nil { return err @@ -106,7 +116,7 @@ func mergeSegments(tempDir string, outputFile string) error { // mergeSegmentsFFmpeg reads every file in tempDir and merges their content to the outputFile // with ffmpeg (https://ffmpeg.org/). // The given output file gets created or overwritten if already existing -func mergeSegmentsFFmpeg(tempDir string, outputFile string) error { +func mergeSegmentsFFmpeg(context context.Context, tempDir string, outputFile string) error { dir, err := os.ReadDir(tempDir) if err != nil { return err @@ -125,5 +135,19 @@ func mergeSegmentsFFmpeg(tempDir string, outputFile string) error { "-i", f.Name(), "-c", "copy", outputFile) - return cmd.Run() + if err := cmd.Start(); err != nil { + return err + } + + cmdChan := make(chan error) + go func() { + cmdChan <- cmd.Wait() + }() + + select { + case err := <-cmdChan: + return err + case <-context.Done(): + return context.Err() + } } From 8544a49cab2646d80c1288cec7e05a78f13a9147 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 13 Feb 2022 16:02:32 +0100 Subject: [PATCH 20/82] Updated context usage and description --- downloader.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/downloader.go b/downloader.go index 8e4d1a1..7471a82 100644 --- a/downloader.go +++ b/downloader.go @@ -30,6 +30,12 @@ type Downloader struct { // cancel Context DeleteTempAfter bool + // Context to control the download process with. + // There is a tiny delay when canceling the context and the actual stop of the + // process. So it is not recommend stopping the program immediately after calling + // the cancel function. It's better when canceling it and then exit the program + // when Format.Download throws an error. See the signal handling section in + // cmd/crunchyroll-go/cmd/download.go for an example Context context.Context // Goroutines is the number of goroutines to download segments with @@ -42,14 +48,16 @@ type Downloader struct { FFmpeg bool } -func NewDownloader(filename string, goroutines int, onSegmentDownload func(segment *m3u8.MediaSegment, current, total int, file *os.File) error) Downloader { +// NewDownloader creates a downloader with default settings which should +// fit the most needs +func NewDownloader(context context.Context, filename string, goroutines int, onSegmentDownload func(segment *m3u8.MediaSegment, current, total int, file *os.File) error) Downloader { tmp, _ := os.MkdirTemp("", "crunchy_") return Downloader{ Filename: filename, TempDir: tmp, DeleteTempAfter: true, - Context: context.Background(), + Context: context, Goroutines: goroutines, OnSegmentDownload: onSegmentDownload, } @@ -65,7 +73,13 @@ func NewDownloader(filename string, goroutines int, onSegmentDownload func(segme // And this function just downloads each of this segment into the given directory. // See https://en.wikipedia.org/wiki/MPEG_transport_stream for more information func download(context context.Context, format *Format, tempDir string, goroutines int, onSegmentDownload func(segment *m3u8.MediaSegment, current, total int, file *os.File) error) error { - resp, err := format.crunchy.Client.Get(format.Video.URI) + req, err := http.NewRequest(http.MethodGet, format.Video.URI, nil) + if err != nil { + return err + } + req.WithContext(context) + + resp, err := format.crunchy.Client.Do(req) if err != nil { return err } From 1b0124e385986a375423ceeeb227cea133ab9a04 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 13 Feb 2022 20:31:50 +0100 Subject: [PATCH 21/82] Fixed array length and nil panic --- episode.go | 9 +++++++-- utils/sort.go | 51 +++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/episode.go b/episode.go index 6392054..07b8616 100644 --- a/episode.go +++ b/episode.go @@ -152,11 +152,16 @@ func (e *Episode) GetFormat(resolution string, subtitle LOCALE, hardsub bool) (* var res *Format for _, format := range formats { if resolution == "worst" || resolution == "best" { - curSplitRes := strings.SplitN(format.Video.Resolution, "x", 1) + if res == nil { + res = format + continue + } + + curSplitRes := strings.SplitN(format.Video.Resolution, "x", 2) curResX, _ := strconv.Atoi(curSplitRes[0]) curResY, _ := strconv.Atoi(curSplitRes[1]) - resSplitRes := strings.SplitN(res.Video.Resolution, "x", 1) + resSplitRes := strings.SplitN(res.Video.Resolution, "x", 2) resResX, _ := strconv.Atoi(resSplitRes[0]) resResY, _ := strconv.Atoi(resSplitRes[1]) diff --git a/utils/sort.go b/utils/sort.go index 628af64..bf9fb77 100644 --- a/utils/sort.go +++ b/utils/sort.go @@ -2,10 +2,45 @@ package utils import ( "github.com/ByteDream/crunchyroll-go" + "sort" "strconv" "strings" ) +// SortEpisodesBySeason sorts the given episodes by their seasons +func SortEpisodesBySeason(episodes []*crunchyroll.Episode) [][]*crunchyroll.Episode { + sortMap := map[string]map[int][]*crunchyroll.Episode{} + + for _, episode := range episodes { + if _, ok := sortMap[episode.SeriesID]; !ok { + sortMap[episode.SeriesID] = map[int][]*crunchyroll.Episode{} + } + if _, ok := sortMap[episode.SeriesID][episode.SeasonNumber]; !ok { + sortMap[episode.SeriesID][episode.SeasonNumber] = make([]*crunchyroll.Episode, 0) + } + sortMap[episode.SeriesID][episode.SeasonNumber] = append(sortMap[episode.SeriesID][episode.SeasonNumber], episode) + } + + var eps [][]*crunchyroll.Episode + for _, series := range sortMap { + keys := make([]int, len(series)) + for seriesNumber := range series { + keys = append(keys, seriesNumber) + } + sort.Ints(keys) + + for _, key := range keys { + es := series[key] + if len(es) > 0 { + sort.Sort(EpisodesByNumber(es)) + eps = append(eps, es) + } + } + } + + return eps +} + // MovieListingsByDuration sorts movie listings by their duration type MovieListingsByDuration []*crunchyroll.MovieListing @@ -32,6 +67,18 @@ func (ebd EpisodesByDuration) Less(i, j int) bool { return ebd[i].DurationMS < ebd[j].DurationMS } +type EpisodesByNumber []*crunchyroll.Episode + +func (ebn EpisodesByNumber) Len() int { + return len(ebn) +} +func (ebn EpisodesByNumber) Swap(i, j int) { + ebn[i], ebn[j] = ebn[j], ebn[i] +} +func (ebn EpisodesByNumber) Less(i, j int) bool { + return ebn[i].EpisodeNumber < ebn[j].EpisodeNumber +} + // FormatsByResolution sorts formats after their resolution type FormatsByResolution []*crunchyroll.Format @@ -42,11 +89,11 @@ func (fbr FormatsByResolution) Swap(i, j int) { fbr[i], fbr[j] = fbr[j], fbr[i] } func (fbr FormatsByResolution) Less(i, j int) bool { - iSplitRes := strings.Split(fbr[i].Video.Resolution, "x") + iSplitRes := strings.SplitN(fbr[i].Video.Resolution, "x", 2) iResX, _ := strconv.Atoi(iSplitRes[0]) iResY, _ := strconv.Atoi(iSplitRes[1]) - jSplitRes := strings.Split(fbr[j].Video.Resolution, "x") + jSplitRes := strings.SplitN(fbr[j].Video.Resolution, "x", 2) jResX, _ := strconv.Atoi(jSplitRes[0]) jResY, _ := strconv.Atoi(jSplitRes[1]) From a6858556b55036f133a0d5b4a7c85933a1ab4236 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 13 Feb 2022 20:36:41 +0100 Subject: [PATCH 22/82] Removed subtitle by locale --- utils/locale.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/utils/locale.go b/utils/locale.go index 708d82f..3d00648 100644 --- a/utils/locale.go +++ b/utils/locale.go @@ -54,17 +54,3 @@ func LocaleLanguage(locale crunchyroll.LOCALE) string { return "" } } - -// SubtitleByLocale returns the subtitle of a crunchyroll.Format by its locale. -// Check the second ok return value if the format has this subtitle -func SubtitleByLocale(format *crunchyroll.Format, locale crunchyroll.LOCALE) (subtitle *crunchyroll.Subtitle, ok bool) { - if format.Subtitles == nil { - return - } - for _, sub := range format.Subtitles { - if sub.Locale == locale { - return sub, true - } - } - return -} From 3382eca3bdc4ad0ce2219caa4690db90f31c1b12 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 13 Feb 2022 23:09:13 +0100 Subject: [PATCH 23/82] Removed redesign notice --- cmd/crunchyroll-go/main.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/crunchyroll-go/main.go b/cmd/crunchyroll-go/main.go index 0b13e39..efc6a1e 100644 --- a/cmd/crunchyroll-go/main.go +++ b/cmd/crunchyroll-go/main.go @@ -1,7 +1,5 @@ package main -// the cli will be redesigned soon - import ( "github.com/ByteDream/crunchyroll-go/cmd/crunchyroll-go/cmd" ) From 77c0ae64b4eb5452c3d4de4845a7fcc2e93b3457 Mon Sep 17 00:00:00 2001 From: bytedream Date: Thu, 17 Feb 2022 17:10:30 +0100 Subject: [PATCH 24/82] Added docs to SortEpisodesBySeason --- utils/sort.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/sort.go b/utils/sort.go index bf9fb77..760d19c 100644 --- a/utils/sort.go +++ b/utils/sort.go @@ -7,7 +7,8 @@ import ( "strings" ) -// SortEpisodesBySeason sorts the given episodes by their seasons +// SortEpisodesBySeason sorts the given episodes by their seasons. +// Note that the same episodes just with different audio locales will cause problems func SortEpisodesBySeason(episodes []*crunchyroll.Episode) [][]*crunchyroll.Episode { sortMap := map[string]map[int][]*crunchyroll.Episode{} From 22525973c7af879077d501fe6c2d24fa94cc0956 Mon Sep 17 00:00:00 2001 From: bytedream Date: Thu, 17 Feb 2022 18:27:14 +0100 Subject: [PATCH 25/82] Renamed Download to Save and changed parameter to io.Writer --- subtitle.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/subtitle.go b/subtitle.go index 6214031..06ec9aa 100644 --- a/subtitle.go +++ b/subtitle.go @@ -2,7 +2,6 @@ package crunchyroll import ( "io" - "os" ) type Subtitle struct { @@ -13,13 +12,13 @@ type Subtitle struct { Format string `json:"format"` } -func (s Subtitle) Download(file *os.File) error { +func (s Subtitle) Save(writer io.Writer) error { resp, err := s.crunchy.Client.Get(s.URL) if err != nil { return err } defer resp.Body.Close() - _, err = io.Copy(file, resp.Body) + _, err = io.Copy(writer, resp.Body) return err } From e7b5a7bf301b155596a3c60441d9a645cdce7efd Mon Sep 17 00:00:00 2001 From: bytedream Date: Thu, 17 Feb 2022 20:23:17 +0100 Subject: [PATCH 26/82] Updated download ffmpeg behavior --- downloader.go | 23 ++++++++++++++++++++--- format.go | 22 ++++++++++++++++------ 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/downloader.go b/downloader.go index 7471a82..9cca3c4 100644 --- a/downloader.go +++ b/downloader.go @@ -43,9 +43,15 @@ type Downloader struct { // A method to call when a segment was downloaded OnSegmentDownload func(segment *m3u8.MediaSegment, current, total int, file *os.File) error + // If LockOnSegmentDownload is true, only one OnSegmentDownload function can be called at + // once. Normally (because of the use of goroutines while downloading) multiple could get + // called simultaneously + LockOnSegmentDownload bool - // If FFmpeg is true, ffmpeg will used to merge and convert files - FFmpeg bool + // If FFmpegOpts is not nil, ffmpeg will be used to merge and convert files. + // The given opts will be used as ffmpeg parameters while merging. + // Some opts are already used, see mergeSegmentsFFmpeg in format.go for more details + FFmpegOpts []string } // NewDownloader creates a downloader with default settings which should @@ -72,7 +78,7 @@ func NewDownloader(context context.Context, filename string, goroutines int, onS // The actual crunchyroll video is split up in multiple segments (or video files) which have to be downloaded and merged after to generate a single video file. // And this function just downloads each of this segment into the given directory. // See https://en.wikipedia.org/wiki/MPEG_transport_stream for more information -func download(context context.Context, format *Format, tempDir string, goroutines int, onSegmentDownload func(segment *m3u8.MediaSegment, current, total int, file *os.File) error) error { +func download(context context.Context, format *Format, tempDir string, goroutines int, lockOnSegmentDownload bool, onSegmentDownload func(segment *m3u8.MediaSegment, current, total int, file *os.File) error) error { req, err := http.NewRequest(http.MethodGet, format.Video.URI, nil) if err != nil { return err @@ -99,6 +105,7 @@ func download(context context.Context, format *Format, tempDir string, goroutine } var wg sync.WaitGroup + var lock sync.Mutex chunkSize := int(math.Ceil(float64(len(segments)) / float64(goroutines))) // when a onSegmentDownload call returns an error, this channel will be set to true and stop all goroutines @@ -145,11 +152,21 @@ func download(context context.Context, format *Format, tempDir string, goroutine return } if onSegmentDownload != nil { + if lockOnSegmentDownload { + lock.Lock() + } + if err = onSegmentDownload(segment, int(atomic.AddInt32(&total, 1)), len(segments), file); err != nil { quit <- true + if lockOnSegmentDownload { + lock.Unlock() + } file.Close() return } + if lockOnSegmentDownload { + lock.Unlock() + } } file.Close() } diff --git a/format.go b/format.go index f3d548f..b1b21a4 100644 --- a/format.go +++ b/format.go @@ -56,12 +56,12 @@ func (f *Format) Download(downloader Downloader) error { if downloader.DeleteTempAfter { defer os.RemoveAll(downloader.TempDir) } - if err := download(downloader.Context, f, downloader.TempDir, downloader.Goroutines, downloader.OnSegmentDownload); err != nil { + if err := download(downloader.Context, f, downloader.TempDir, downloader.Goroutines, downloader.LockOnSegmentDownload, downloader.OnSegmentDownload); err != nil { return err } - if downloader.FFmpeg { - return mergeSegmentsFFmpeg(downloader.Context, downloader.TempDir, downloader.Filename) + if downloader.FFmpegOpts != nil { + return mergeSegmentsFFmpeg(downloader.Context, downloader.TempDir, downloader.Filename, downloader.FFmpegOpts) } else { return mergeSegments(downloader.Context, downloader.TempDir, downloader.Filename) } @@ -116,7 +116,7 @@ func mergeSegments(context context.Context, tempDir string, outputFile string) e // mergeSegmentsFFmpeg reads every file in tempDir and merges their content to the outputFile // with ffmpeg (https://ffmpeg.org/). // The given output file gets created or overwritten if already existing -func mergeSegmentsFFmpeg(context context.Context, tempDir string, outputFile string) error { +func mergeSegmentsFFmpeg(context context.Context, tempDir string, outputFile string, opts []string) error { dir, err := os.ReadDir(tempDir) if err != nil { return err @@ -129,12 +129,21 @@ func mergeSegmentsFFmpeg(context context.Context, tempDir string, outputFile str for i := 0; i < len(dir); i++ { fmt.Fprintf(f, "file '%s.ts'\n", filepath.Join(tempDir, strconv.Itoa(i))) } - cmd := exec.Command("ffmpeg", + + // predefined options ... custom options ... predefined output filename + command := []string{ "-f", "concat", "-safe", "0", "-i", f.Name(), "-c", "copy", - outputFile) + } + if opts != nil { + command = append(command, opts...) + } + command = append(command, outputFile) + + cmd := exec.Command("ffmpeg", + command...) if err := cmd.Start(); err != nil { return err } @@ -148,6 +157,7 @@ func mergeSegmentsFFmpeg(context context.Context, tempDir string, outputFile str case err := <-cmdChan: return err case <-context.Done(): + cmd.Process.Kill() return context.Err() } } From 18ce16b3bd1c61c5f0b0034caac915e7787eed2d Mon Sep 17 00:00:00 2001 From: bytedream Date: Fri, 18 Feb 2022 14:51:08 +0100 Subject: [PATCH 27/82] How tf did this work before --- utils/sort.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/sort.go b/utils/sort.go index 760d19c..1f9092a 100644 --- a/utils/sort.go +++ b/utils/sort.go @@ -24,7 +24,7 @@ func SortEpisodesBySeason(episodes []*crunchyroll.Episode) [][]*crunchyroll.Epis var eps [][]*crunchyroll.Episode for _, series := range sortMap { - keys := make([]int, len(series)) + var keys []int for seriesNumber := range series { keys = append(keys, seriesNumber) } From 3dfc69e2d98ac960c493cfd7e9441db3398e89b5 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 20 Feb 2022 13:16:58 +0100 Subject: [PATCH 28/82] Fixed wrong return and added stderr error output when merging with ffmpeg --- format.go | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/format.go b/format.go index b1b21a4..516fb71 100644 --- a/format.go +++ b/format.go @@ -2,6 +2,7 @@ package crunchyroll import ( "bufio" + "bytes" "context" "fmt" "github.com/grafov/m3u8" @@ -37,7 +38,14 @@ func (f *Format) Download(downloader Downloader) error { if _, err := os.Stat(downloader.Filename); err == nil && !downloader.IgnoreExisting { return fmt.Errorf("file %s already exists", downloader.Filename) } - if _, err := os.Stat(downloader.TempDir); err == nil && !downloader.IgnoreExisting { + if _, err := os.Stat(downloader.TempDir); err != nil { + if os.IsNotExist(err) { + err = os.Mkdir(downloader.TempDir, 0755) + } + if err != nil { + return err + } + } else if !downloader.IgnoreExisting { content, err := os.ReadDir(downloader.TempDir) if err != nil { return err @@ -45,12 +53,6 @@ func (f *Format) Download(downloader Downloader) error { if len(content) > 0 { return fmt.Errorf("directory %s is not empty", downloader.Filename) } - } else if err != nil && os.IsNotExist(err) { - if err := os.Mkdir(downloader.TempDir, 0755); err != nil { - return err - } - } else { - return err } if downloader.DeleteTempAfter { @@ -142,8 +144,11 @@ func mergeSegmentsFFmpeg(context context.Context, tempDir string, outputFile str } command = append(command, outputFile) + var errBuf bytes.Buffer cmd := exec.Command("ffmpeg", command...) + cmd.Stderr = &errBuf + if err := cmd.Start(); err != nil { return err } @@ -154,8 +159,8 @@ func mergeSegmentsFFmpeg(context context.Context, tempDir string, outputFile str }() select { - case err := <-cmdChan: - return err + case <-cmdChan: + return fmt.Errorf(errBuf.String()) case <-context.Done(): cmd.Process.Kill() return context.Err() From 685dbc622e5c35b17d0323de168bd686e97e533c Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 21 Feb 2022 15:05:45 +0100 Subject: [PATCH 29/82] More ffmpeg merge fixes --- format.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/format.go b/format.go index 516fb71..104f1e9 100644 --- a/format.go +++ b/format.go @@ -123,17 +123,19 @@ func mergeSegmentsFFmpeg(context context.Context, tempDir string, outputFile str if err != nil { return err } - f, err := os.CreateTemp("", "*.txt") + f, err := os.Create(filepath.Join(tempDir, "list.txt")) if err != nil { return err } - defer os.Remove(f.Name()) - for i := 0; i < len(dir); i++ { + // -1 is the list.txt file + for i := 0; i < len(dir)-1; i++ { fmt.Fprintf(f, "file '%s.ts'\n", filepath.Join(tempDir, strconv.Itoa(i))) } + f.Close() // predefined options ... custom options ... predefined output filename command := []string{ + "-y", "-f", "concat", "-safe", "0", "-i", f.Name(), @@ -153,14 +155,21 @@ func mergeSegmentsFFmpeg(context context.Context, tempDir string, outputFile str return err } - cmdChan := make(chan error) + cmdChan := make(chan error, 1) go func() { cmdChan <- cmd.Wait() }() select { - case <-cmdChan: - return fmt.Errorf(errBuf.String()) + case err = <-cmdChan: + if err != nil { + if errBuf.Len() > 0 { + return fmt.Errorf(errBuf.String()) + } else { + return err + } + } + return nil case <-context.Done(): cmd.Process.Kill() return context.Err() From 758b9b59c82808f56c46595ba8aeac87ae0d9865 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 21 Feb 2022 15:19:26 +0100 Subject: [PATCH 30/82] Added InitVideo() function to initialize Format.Video.Chunklist --- format.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/format.go b/format.go index 104f1e9..9dfaf3e 100644 --- a/format.go +++ b/format.go @@ -34,6 +34,28 @@ type Format struct { Subtitles []*Subtitle } +// InitVideo initializes the Format.Video completely. +// The Format.Video.Chunklist pointer is, by default, nil because an additional +// request must be made to receive its content. The request is not made when +// initializing a Format struct because it would probably cause an intense overhead +// since Format.Video.Chunklist is only used sometimes +func (f *Format) InitVideo() error { + if f.Video.Chunklist == nil { + resp, err := f.crunchy.Client.Get(f.Video.URI) + if err != nil { + return err + } + defer resp.Body.Close() + + playlist, _, err := m3u8.DecodeFrom(resp.Body, true) + if err != nil { + return err + } + f.Video.Chunklist = playlist.(*m3u8.MediaPlaylist) + } + return nil +} + func (f *Format) Download(downloader Downloader) error { if _, err := os.Stat(downloader.Filename); err == nil && !downloader.IgnoreExisting { return fmt.Errorf("file %s already exists", downloader.Filename) From 86368bf98555b105416eb818d80369fdac1a4ad4 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 21 Feb 2022 15:36:11 +0100 Subject: [PATCH 31/82] Implemented Format.InitVideo() --- downloader.go | 37 ++++++++----------------------------- 1 file changed, 8 insertions(+), 29 deletions(-) diff --git a/downloader.go b/downloader.go index 9cca3c4..828f684 100644 --- a/downloader.go +++ b/downloader.go @@ -79,58 +79,37 @@ func NewDownloader(context context.Context, filename string, goroutines int, onS // And this function just downloads each of this segment into the given directory. // See https://en.wikipedia.org/wiki/MPEG_transport_stream for more information func download(context context.Context, format *Format, tempDir string, goroutines int, lockOnSegmentDownload bool, onSegmentDownload func(segment *m3u8.MediaSegment, current, total int, file *os.File) error) error { - req, err := http.NewRequest(http.MethodGet, format.Video.URI, nil) - if err != nil { + if err := format.InitVideo(); err != nil { return err } - req.WithContext(context) - - resp, err := format.crunchy.Client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - // reads the m3u8 file - playlist, _, err := m3u8.DecodeFrom(resp.Body, true) - if err != nil { - return err - } - // extracts the segments from the playlist - var segments []*m3u8.MediaSegment - for _, segment := range playlist.(*m3u8.MediaPlaylist).Segments { - // some segments are nil, so they have to be filtered out - if segment != nil { - segments = append(segments, segment) - } - } var wg sync.WaitGroup var lock sync.Mutex - chunkSize := int(math.Ceil(float64(len(segments)) / float64(goroutines))) + chunkSize := int(math.Ceil(float64(format.Video.Chunklist.Count()) / float64(goroutines))) // when a onSegmentDownload call returns an error, this channel will be set to true and stop all goroutines quit := make(chan bool) // receives the decrypt block and iv from the first segment. // in my tests, only the first segment has specified this data, so the decryption data from this first segments will be used in every other segment too - block, iv, err := getCrypt(format, segments[0]) + block, iv, err := getCrypt(format, format.Video.Chunklist.Segments[0]) if err != nil { return err } var total int32 - for i := 0; i < len(segments); i += chunkSize { + for i := 0; i < int(format.Video.Chunklist.Count()); i += chunkSize { wg.Add(1) end := i + chunkSize - if end > len(segments) { - end = len(segments) + if end > int(format.Video.Chunklist.Count()) { + end = int(format.Video.Chunklist.Count()) } i := i go func() { defer wg.Done() - for j, segment := range segments[i:end] { + for j, segment := range format.Video.Chunklist.Segments[i:end] { select { case <-context.Done(): return @@ -156,7 +135,7 @@ func download(context context.Context, format *Format, tempDir string, goroutine lock.Lock() } - if err = onSegmentDownload(segment, int(atomic.AddInt32(&total, 1)), len(segments), file); err != nil { + if err = onSegmentDownload(segment, int(atomic.AddInt32(&total, 1)), int(format.Video.Chunklist.Count()), file); err != nil { quit <- true if lockOnSegmentDownload { lock.Unlock() From 3285d458bef1db71ba5cc0cf3615b4933ce22418 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 21 Feb 2022 16:59:57 +0100 Subject: [PATCH 32/82] Fixed always true subtitle locale comparison in GetFormat(...) --- episode.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/episode.go b/episode.go index 07b8616..12d21b9 100644 --- a/episode.go +++ b/episode.go @@ -130,8 +130,8 @@ func (e *Episode) GetFormat(resolution string, subtitle LOCALE, hardsub bool) (* foundStream = stream break } else if !hardsub { - for _, subtitle := range stream.Subtitles { - if subtitle.Locale == subtitle.Locale { + for _, streamSubtitle := range stream.Subtitles { + if streamSubtitle.Locale == subtitle { foundStream = stream break } From bd19b3408802ff1b3799a5df7b9caf9526b627e2 Mon Sep 17 00:00:00 2001 From: bytedream Date: Wed, 23 Feb 2022 22:32:20 +0100 Subject: [PATCH 33/82] Added sort function for subtitles by their locale --- utils/sort.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/utils/sort.go b/utils/sort.go index 1f9092a..16ce41a 100644 --- a/utils/sort.go +++ b/utils/sort.go @@ -100,3 +100,15 @@ func (fbr FormatsByResolution) Less(i, j int) bool { return iResX+iResY < jResX+jResY } + +type SubtitlesByLocale []*crunchyroll.Subtitle + +func (sbl SubtitlesByLocale) Len() int { + return len(sbl) +} +func (sbl SubtitlesByLocale) Swap(i, j int) { + sbl[i], sbl[j] = sbl[j], sbl[i] +} +func (sbl SubtitlesByLocale) Less(i, j int) bool { + return sbl[i].Locale < sbl[j].Locale +} From 777738a1a1473f35255eff5f552d954c07c184a8 Mon Sep 17 00:00:00 2001 From: bytedream Date: Thu, 24 Feb 2022 21:22:58 +0100 Subject: [PATCH 34/82] Changed spanish latin america and arabic locale & added portuguese --- crunchyroll.go | 5 +++-- utils/locale.go | 7 +++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/crunchyroll.go b/crunchyroll.go index 8c402d6..baa2947 100644 --- a/crunchyroll.go +++ b/crunchyroll.go @@ -18,14 +18,15 @@ type LOCALE string const ( JP LOCALE = "ja-JP" US = "en-US" - LA = "es-LA" + LA = "es-419" ES = "es-ES" FR = "fr-FR" + PT = "pt-PT" BR = "pt-BR" IT = "it-IT" DE = "de-DE" RU = "ru-RU" - ME = "ar-ME" + AR = "ar-SA" ) type Crunchyroll struct { diff --git a/utils/locale.go b/utils/locale.go index 3d00648..8d78912 100644 --- a/utils/locale.go +++ b/utils/locale.go @@ -10,11 +10,12 @@ var AllLocales = []crunchyroll.LOCALE{ crunchyroll.LA, crunchyroll.ES, crunchyroll.FR, + crunchyroll.PT, crunchyroll.BR, crunchyroll.IT, crunchyroll.DE, crunchyroll.RU, - crunchyroll.ME, + crunchyroll.AR, } // ValidateLocale validates if the given locale actually exist @@ -40,6 +41,8 @@ func LocaleLanguage(locale crunchyroll.LOCALE) string { return "Spanish (Spain)" case crunchyroll.FR: return "French" + case crunchyroll.PT: + return "Portuguese (Europe)" case crunchyroll.BR: return "Portuguese (Brazil)" case crunchyroll.IT: @@ -48,7 +51,7 @@ func LocaleLanguage(locale crunchyroll.LOCALE) string { return "German" case crunchyroll.RU: return "Russian" - case crunchyroll.ME: + case crunchyroll.AR: return "Arabic" default: return "" From b4ba50d5a4f3ff24511560a6a5b789901b6562bb Mon Sep 17 00:00:00 2001 From: bytedream Date: Thu, 24 Feb 2022 21:24:11 +0100 Subject: [PATCH 35/82] Added xz dependency --- go.mod | 1 + go.sum | 2 ++ 2 files changed, 3 insertions(+) diff --git a/go.mod b/go.mod index ee64a5c..a69da4f 100644 --- a/go.mod +++ b/go.mod @@ -5,4 +5,5 @@ go 1.16 require ( github.com/grafov/m3u8 v0.11.1 github.com/spf13/cobra v1.2.1 + github.com/ulikunitz/xz v0.5.6 ) diff --git a/go.sum b/go.sum index d4baaf3..fe25b3a 100644 --- a/go.sum +++ b/go.sum @@ -218,6 +218,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8= +github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= From ceec69ef659b89b0c539e2ded0e338f3fa825bd0 Mon Sep 17 00:00:00 2001 From: bytedream Date: Thu, 24 Feb 2022 22:09:55 +0100 Subject: [PATCH 36/82] Better audio filtering --- utils/url.go | 53 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/utils/url.go b/utils/url.go index 5181358..9bdec77 100644 --- a/utils/url.go +++ b/utils/url.go @@ -7,7 +7,7 @@ import ( // ExtractEpisodesFromUrl extracts all episodes from an url. // If audio is not empty, the episodes gets filtered after the given locale -func ExtractEpisodesFromUrl(crunchy *crunchyroll.Crunchyroll, url string, audio crunchyroll.LOCALE) ([]*crunchyroll.Episode, error) { +func ExtractEpisodesFromUrl(crunchy *crunchyroll.Crunchyroll, url string, audio ...crunchyroll.LOCALE) ([]*crunchyroll.Episode, error) { series, episodes, err := ParseUrl(crunchy, url) if err != nil { return nil, err @@ -21,10 +21,20 @@ func ExtractEpisodesFromUrl(crunchy *crunchyroll.Crunchyroll, url string, audio return nil, err } for _, season := range seasons { - if audio != "" { - if audioLocale, err := season.AudioLocale(); err != nil { + if audio != nil { + locale, err := season.AudioLocale() + if err != nil { return nil, err - } else if audioLocale != audio { + } + + var found bool + for _, l := range audio { + if locale == l { + found = true + break + } + } + if !found { continue } } @@ -35,15 +45,34 @@ func ExtractEpisodesFromUrl(crunchy *crunchyroll.Crunchyroll, url string, audio eps = append(eps, e...) } } else if episodes != nil { - for _, episode := range episodes { - if audio == "" { - eps = append(eps, episode) - } else if audioLocale, err := episode.AudioLocale(); err != nil { - return nil, err - } else if audioLocale == audio { - eps = append(eps, episode) - } + if audio == nil { + return episodes, nil } + + for _, episode := range episodes { + locale, err := episode.AudioLocale() + if err != nil { + return nil, err + } + if audio != nil { + var found bool + for _, l := range audio { + if locale == l { + found = true + break + } + } + if !found { + continue + } + } + + eps = append(eps, episode) + } + } + + if len(eps) == 0 { + return nil, fmt.Errorf("could not find any matching episode") } return eps, nil From 79d55a5d3bcb7661e1139969aec56a82371597b5 Mon Sep 17 00:00:00 2001 From: bytedream Date: Thu, 24 Feb 2022 23:11:04 +0100 Subject: [PATCH 37/82] Tests are overrated --- crunchyroll_test.go | 115 -------------------------------------------- 1 file changed, 115 deletions(-) delete mode 100644 crunchyroll_test.go diff --git a/crunchyroll_test.go b/crunchyroll_test.go deleted file mode 100644 index 7009fed..0000000 --- a/crunchyroll_test.go +++ /dev/null @@ -1,115 +0,0 @@ -package crunchyroll - -import ( - "github.com/grafov/m3u8" - "net/http" - "os" - "testing" -) - -var ( - email = os.Getenv("EMAIL") - password = os.Getenv("PASSWORD") - sessionID = os.Getenv("SESSION_ID") - - crunchy *Crunchyroll - season *Season - episode *Episode - stream *Stream -) - -func TestLogin(t *testing.T) { - var err error - if email != "" && password != "" { - crunchy, err = LoginWithCredentials(email, password, DE, http.DefaultClient) - if err != nil { - t.Error(err) - } - t.Logf("Logged in with email and password\nAuth: %s %s\nSession id: %s", - crunchy.Config.TokenType, crunchy.Config.AccessToken, crunchy.SessionID) - } else if sessionID != "" { - crunchy, err = LoginWithSessionID(sessionID, DE, http.DefaultClient) - if err != nil { - t.Error(err) - } - t.Logf("Logged in with session id\nAuth: %s %s\nSession id: %s", - crunchy.Config.TokenType, crunchy.Config.AccessToken, crunchy.SessionID) - } else { - t.Skipf("email and / or password and session id environtment variables are not set, skipping login. All following test may fail also") - } -} - -func TestCrunchy_Search(t *testing.T) { - series, movies, err := crunchy.Search("movie", 20) - if err != nil { - t.Error(err) - } - t.Logf("Found %d series and %d movie(s) for search query `movie`", len(series), len(movies)) -} - -func TestSeries_Seasons(t *testing.T) { - video, err := crunchy.FindVideo("https://www.crunchyroll.com/darling-in-the-franxx") - if err != nil { - t.Error(err) - } - series := video.(*Series) - seasons, err := series.Seasons() - if err != nil { - t.Error(err) - } - if len(seasons) > 0 { - season = seasons[4] - } else { - t.Logf("%s has no seasons, some future test will fail", series.Title) - } - t.Logf("Found %d seasons for series %s", len(seasons), series.Title) -} - -func TestCrunchyroll_FindEpisode(t *testing.T) { - episodes, err := crunchy.FindEpisode("https://www.crunchyroll.com/darling-in-the-franxx/episode-1-alone-and-lonesome-759575") - if err != nil { - t.Error(err) - } - t.Logf("Found %d episodes for episode %s", len(episodes), "https://www.crunchyroll.com/darling-in-the-franxx/episode-1-alone-and-lonesome-759575") -} - -func TestSeason_Episodes(t *testing.T) { - episodes, err := season.Episodes() - if err != nil { - t.Error(err) - } - if len(episodes) > 0 { - episode = episodes[0] - } else { - t.Logf("%s has no episodes, some future test will fail", season.Title) - } - t.Logf("Found %d episodes for season %s", len(episodes), season.Title) -} - -func TestEpisode_Streams(t *testing.T) { - streams, err := episode.Streams() - if err != nil { - t.Error(err) - } - if len(streams) > 0 { - stream = streams[0] - } else { - t.Logf("%s has no streams, some future test will fail", season.Title) - } - t.Logf("Found %d streams for episode %s", len(streams), season.Title) -} - -func TestFormat_Download(t *testing.T) { - formats, err := stream.Formats() - if err != nil { - t.Error(err) - } - file, err := os.Create("test") - if err != nil { - t.Error(err) - } - formats[0].DownloadGoroutines(file, 4, func(segment *m3u8.MediaSegment, current, total int, file *os.File) error { - t.Logf("Downloaded %.2f%% (%d/%d)", float32(current)/float32(total)*100, current, total) - return nil - }) -} From 9fbb3266aaa7e8603a6564a8d32fb5bb19919ab5 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 28 Feb 2022 14:36:11 +0100 Subject: [PATCH 38/82] Removed xz dependency --- go.mod | 8 +- go.sum | 270 ++++++++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 237 insertions(+), 41 deletions(-) diff --git a/go.mod b/go.mod index a69da4f..f5c3ea4 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,10 @@ go 1.16 require ( github.com/grafov/m3u8 v0.11.1 - github.com/spf13/cobra v1.2.1 - github.com/ulikunitz/xz v0.5.6 + github.com/spf13/cobra v1.3.0 +) + +require ( + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect ) diff --git a/go.sum b/go.sum index fe25b3a..a1bf13c 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,15 @@ cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmW cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -26,7 +35,7 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -39,23 +48,46 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -64,19 +96,32 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -85,6 +130,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -102,6 +148,7 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -115,10 +162,12 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -130,87 +179,139 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/grafov/m3u8 v0.11.1 h1:igZ7EBIB2IAsPPazKwRKdbhxcoBKO3lO1UY57PZDeNA= github.com/grafov/m3u8 v0.11.1/go.mod h1:nqzOkfBiZJENr52zTVd/Dcl03yzphIMbJqkXGu+u080= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= +github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= +github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= -github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0= +github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= +github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -218,16 +319,17 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8= -github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= +github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -235,16 +337,20 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -280,10 +386,11 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -291,9 +398,11 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -316,6 +425,9 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -327,7 +439,12 @@ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -341,21 +458,30 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -374,12 +500,27 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -388,6 +529,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -397,7 +540,6 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -405,9 +547,9 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -441,7 +583,11 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -467,7 +613,17 @@ google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34q google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -515,7 +671,29 @@ google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -535,7 +713,15 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -548,12 +734,18 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From ef4b24f068a140dd28f5d0c8cd90c4841d64e989 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 28 Feb 2022 16:38:56 +0100 Subject: [PATCH 39/82] Removed color flag and changed error output format --- cmd/crunchyroll-go/cmd/root.go | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/cmd/crunchyroll-go/cmd/root.go b/cmd/crunchyroll-go/cmd/root.go index 6f8a4e6..db2cba6 100644 --- a/cmd/crunchyroll-go/cmd/root.go +++ b/cmd/crunchyroll-go/cmd/root.go @@ -5,20 +5,17 @@ import ( "github.com/spf13/cobra" "net/http" "os" - "runtime" "runtime/debug" ) var ( client *http.Client - locale crunchyroll.LOCALE crunchy *crunchyroll.Crunchyroll - out = newLogger(false, true, true, colorFlag) + out = newLogger(false, true, true) quietFlag bool verboseFlag bool proxyFlag string - colorFlag bool ) var rootCmd = &cobra.Command{ @@ -26,9 +23,9 @@ var rootCmd = &cobra.Command{ Short: "Download crunchyroll videos with ease", PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) { if verboseFlag { - out = newLogger(true, true, true, colorFlag) + out = newLogger(true, true, true) } else if quietFlag { - out = newLogger(false, false, false, false) + out = newLogger(false, false, false) } out.DebugLog.Printf("Executing `%s` command with %d arg(s)\n", cmd.Name(), len(args)) @@ -42,23 +39,21 @@ func init() { rootCmd.PersistentFlags().BoolVarP(&quietFlag, "quiet", "q", false, "Disable all output") rootCmd.PersistentFlags().BoolVarP(&verboseFlag, "verbose", "v", false, "Adds debug messages to the normal output") rootCmd.PersistentFlags().StringVarP(&proxyFlag, "proxy", "p", "", "Proxy to use") - rootCmd.PersistentFlags().BoolVar(&colorFlag, "color", false, "Colored output. Only available on not windows systems") } func Execute() { rootCmd.CompletionOptions.DisableDefaultCmd = true defer func() { if r := recover(); r != nil { - out.Errln(r) - // change color to red - if colorFlag && runtime.GOOS != "windows" { - out.ErrLog.SetOutput(&loggerWriter{original: out.ErrLog.Writer(), color: "\033[31m"}) + if out.IsDev() { + out.Err(string(debug.Stack())) + } else { + out.Err("Unexpected error: %v", r) } - out.Debugln(string(debug.Stack())) - os.Exit(2) + os.Exit(1) } }() - if err := rootCmd.Execute(); err != nil { - out.Fatalln(err) + if rootCmd.Execute() != nil { + os.Exit(1) } } From e785cb580b895435c1c363cd579041d517bb51ea Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 28 Feb 2022 16:41:12 +0100 Subject: [PATCH 40/82] Fixed system locale receiving and added download progress, download (format) information, filename generator, url episodes extractor --- cmd/crunchyroll-go/cmd/utils.go | 220 ++++++++++++++++++++++++++++---- 1 file changed, 195 insertions(+), 25 deletions(-) diff --git a/cmd/crunchyroll-go/cmd/utils.go b/cmd/crunchyroll-go/cmd/utils.go index 3c357f9..809e982 100644 --- a/cmd/crunchyroll-go/cmd/utils.go +++ b/cmd/crunchyroll-go/cmd/utils.go @@ -11,37 +11,48 @@ import ( "os/exec" "path" "path/filepath" + "reflect" "runtime" "strconv" "strings" + "sync" "time" ) var sessionIDPath = filepath.Join(os.TempDir(), ".crunchy") +var ( + invalidWindowsChars = []string{"<", ">", ":", "\"", "/", "|", "\\", "?", "*"} + invalidLinuxChars = []string{"/"} +) + // systemLocale receives the system locale // https://stackoverflow.com/questions/51829386/golang-get-system-language/51831590#51831590 func systemLocale() crunchyroll.LOCALE { if runtime.GOOS != "windows" { if lang, ok := os.LookupEnv("LANG"); ok { - return localeToLOCALE(strings.ReplaceAll(strings.Split(lang, ".")[0], "_", "-")) + prefix := strings.Split(lang, "_")[0] + suffix := strings.Split(strings.Split(lang, ".")[0], "_")[1] + l := crunchyroll.LOCALE(fmt.Sprintf("%s-%s", prefix, suffix)) + if !utils.ValidateLocale(l) { + out.Err("%s is not a supported locale, using %s as fallback", l, crunchyroll.US) + l = crunchyroll.US + } + return l } } else { cmd := exec.Command("powershell", "Get-Culture | select -exp Name") - if output, err := cmd.Output(); err != nil { - return localeToLOCALE(strings.Trim(string(output), "\r\n")) + if output, err := cmd.Output(); err == nil { + l := crunchyroll.LOCALE(strings.Trim(string(output), "\r\n")) + if !utils.ValidateLocale(l) { + out.Err("%s is not a supported locale, using %s as fallback", l, crunchyroll.US) + l = crunchyroll.US + } + return l } } - return localeToLOCALE("en-US") -} - -func localeToLOCALE(locale string) crunchyroll.LOCALE { - if l := crunchyroll.LOCALE(locale); utils.ValidateLocale(l) { - return l - } else { - out.Errf("%s is not a supported locale, using %s as fallback\n", locale, crunchyroll.US) - return crunchyroll.US - } + out.Err("Failed to get locale, using %s", crunchyroll.US) + return crunchyroll.US } func allLocalesAsStrings() (locales []string) { @@ -55,7 +66,7 @@ func createOrDefaultClient(proxy string) (*http.Client, error) { if proxy == "" { return http.DefaultClient, nil } else { - out.Infof("Using custom proxy %s\n", proxy) + out.Info("Using custom proxy %s", proxy) proxyURL, err := url.Parse(proxy) if err != nil { return nil, err @@ -86,7 +97,8 @@ func freeFileName(filename string) (string, bool) { func loadSessionID() (string, error) { if _, stat := os.Stat(sessionIDPath); os.IsNotExist(stat) { - out.Fatalf("To use this command, login first. Type `%s login -h` to get help\n", os.Args[0]) + out.Err("To use this command, login first. Type `%s login -h` to get help", os.Args[0]) + os.Exit(1) } body, err := ioutil.ReadFile(sessionIDPath) if err != nil { @@ -96,35 +108,34 @@ func loadSessionID() (string, error) { } func loadCrunchy() { - out.StartProgress("Logging in") + out.SetProgress("Logging in") sessionID, err := loadSessionID() if err == nil { - if crunchy, err = crunchyroll.LoginWithSessionID(sessionID, locale, client); err != nil { - out.EndProgress(false, err.Error()) + if crunchy, err = crunchyroll.LoginWithSessionID(sessionID, systemLocale(), client); err != nil { + out.StopProgress(err.Error()) os.Exit(1) } } else { - out.EndProgress(false, err.Error()) + out.StopProgress(err.Error()) os.Exit(1) } - out.EndProgress(true, "Logged in") - out.Debugf("Logged in with session id %s\n", sessionID) + out.StopProgress("Logged in") + out.Debug("Logged in with session id %s", sessionID) } func hasFFmpeg() bool { - cmd := exec.Command("ffmpeg", "-h") - return cmd.Run() == nil + return exec.Command("ffmpeg", "-h").Run() == nil } func terminalWidth() int { if runtime.GOOS != "windows" { cmd := exec.Command("stty", "size") cmd.Stdin = os.Stdin - out, err := cmd.Output() + res, err := cmd.Output() if err != nil { return 60 } - width, err := strconv.Atoi(strings.Split(strings.ReplaceAll(string(out), "\n", ""), " ")[1]) + width, err := strconv.Atoi(strings.Split(strings.ReplaceAll(string(res), "\n", ""), " ")[1]) if err != nil { return 60 } @@ -132,3 +143,162 @@ func terminalWidth() int { } return 60 } + +func generateFilename(name, directory string) string { + if runtime.GOOS != "windows" { + for _, char := range invalidLinuxChars { + strings.ReplaceAll(name, char, "") + } + out.Debug("Replaced invalid characters (not windows)") + } else { + for _, char := range invalidWindowsChars { + strings.ReplaceAll(name, char, "") + } + out.Debug("Replaced invalid characters (windows)") + } + + if directory != "" { + name = filepath.Join(directory, name) + } + + filename, changed := freeFileName(name) + if changed { + out.Info("File %s already exists, changing name to %s", name, filename) + } + + return filename +} + +func extractEpisodes(url string, locales ...crunchyroll.LOCALE) [][]*crunchyroll.Episode { + final := make([][]*crunchyroll.Episode, len(locales)) + episodes, err := utils.ExtractEpisodesFromUrl(crunchy, url, locales...) + if err != nil { + out.Err("Failed to get episodes: %v", err) + os.Exit(1) + } + + // fetch all episodes and sort them by their locale + var wg sync.WaitGroup + for _, episode := range episodes { + episode := episode + wg.Add(1) + go func() { + defer wg.Done() + audioLocale, err := episode.AudioLocale() + if err != nil { + out.Err("Failed to get audio locale: %v", err) + os.Exit(1) + } + + for i, locale := range locales { + if locale == audioLocale { + final[i] = append(final[i], episode) + } + } + }() + } + wg.Wait() + + return final +} + +type FormatInformation struct { + // the format to download + format *crunchyroll.Format + + // additional formats which are only used by archive.go + additionalFormats []*crunchyroll.Format + + Title string `json:"title"` + SeriesName string `json:"series_name"` + SeasonNumber int `json:"season_number"` + EpisodeNumber int `json:"episode_number"` + Resolution string `json:"resolution"` + FPS float64 `json:"fps"` + Audio crunchyroll.LOCALE `json:"audio"` + Subtitle crunchyroll.LOCALE `json:"subtitle"` +} + +func (fi FormatInformation) Format(source string) string { + fields := reflect.TypeOf(fi) + values := reflect.ValueOf(fi) + + for i := 0; i < fields.NumField(); i++ { + var valueAsString string + switch value := values.Field(i); value.Kind() { + case reflect.String: + valueAsString = value.String() + case reflect.Int: + valueAsString = fmt.Sprintf("%02d", value.Int()) + case reflect.Float64: + valueAsString = fmt.Sprintf("%.2f", value.Float()) + case reflect.Bool: + valueAsString = fields.Field(i).Tag.Get("json") + if !value.Bool() { + valueAsString = "no " + valueAsString + } + } + source = strings.ReplaceAll(source, "{"+fields.Field(i).Tag.Get("json")+"}", valueAsString) + } + + return source +} + +type DownloadProgress struct { + Prefix string + Message string + + Total int + Current int + + Dev bool + Quiet bool + + lock sync.Mutex +} + +func (dp *DownloadProgress) Update() { + dp.update("", false) +} + +func (dp *DownloadProgress) UpdateMessage(msg string, permanent bool) { + dp.update(msg, permanent) +} + +func (dp *DownloadProgress) update(msg string, permanent bool) { + if dp.Quiet { + return + } + + if dp.Current >= dp.Total { + return + } + + dp.lock.Lock() + defer dp.lock.Unlock() + dp.Current++ + + if msg == "" { + msg = dp.Message + } + if permanent { + dp.Message = msg + } + + if dp.Dev { + fmt.Printf("%s%s\n", dp.Prefix, msg) + return + } + + percentage := float32(dp.Current) / float32(dp.Total) * 100 + progressWidth := float32(terminalWidth() - (12 + len(dp.Prefix) + len(dp.Message)) - (len(fmt.Sprint(dp.Total)))*2) + + repeatCount := int(percentage / (float32(100) / progressWidth)) + // it can be lower than zero when the terminal is very tiny + if repeatCount < 0 { + repeatCount = 0 + } + progressPercentage := (strings.Repeat("=", repeatCount) + ">")[1:] + + fmt.Printf("\r%s%s [%-"+fmt.Sprint(progressWidth)+"s]%4d%% %8d/%d", dp.Prefix, msg, progressPercentage, int(percentage), dp.Current, dp.Total) +} From 252762f410925e92205a61b387b677df4de82874 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 28 Feb 2022 17:26:03 +0100 Subject: [PATCH 41/82] Fixed logger to properly delete current line before overwriting --- cmd/crunchyroll-go/cmd/logger.go | 213 +++++++++++++------------------ 1 file changed, 89 insertions(+), 124 deletions(-) diff --git a/cmd/crunchyroll-go/cmd/logger.go b/cmd/crunchyroll-go/cmd/logger.go index 1079dc6..d9c7248 100644 --- a/cmd/crunchyroll-go/cmd/logger.go +++ b/cmd/crunchyroll-go/cmd/logger.go @@ -1,19 +1,18 @@ package cmd import ( + "context" "fmt" "io" "log" "os" - "runtime" "strings" - "sync" "time" ) type progress struct { - status bool message string + stop bool } type logger struct { @@ -23,26 +22,21 @@ type logger struct { devView bool - progressWG sync.Mutex - progress chan progress + progress chan progress + done chan interface{} } -func newLogger(debug, info, err bool, color bool) *logger { - debugLog, infoLog, errLog := log.New(io.Discard, "=> ", 0), log.New(io.Discard, "=> ", 0), log.New(io.Discard, "=> ", 0) - - debugColor, infoColor, errColor := "", "", "" - if color && runtime.GOOS != "windows" { - debugColor, infoColor, errColor = "\033[95m", "\033[96m", "\033[31m" - } +func newLogger(debug, info, err bool) *logger { + debugLog, infoLog, errLog := log.New(io.Discard, "➞ ", 0), log.New(io.Discard, "➞ ", 0), log.New(io.Discard, "➞ ", 0) if debug { - debugLog.SetOutput(&loggerWriter{original: os.Stdout, color: debugColor}) + debugLog.SetOutput(os.Stdout) } if info { - infoLog.SetOutput(&loggerWriter{original: os.Stdout, color: infoColor}) + infoLog.SetOutput(os.Stdout) } if err { - errLog.SetOutput(&loggerWriter{original: os.Stdout, color: errColor}) + errLog.SetOutput(os.Stderr) } if debug { @@ -60,140 +54,111 @@ func newLogger(debug, info, err bool, color bool) *logger { } } +func (l *logger) IsDev() bool { + return l.devView +} + +func (l *logger) IsQuiet() bool { + return l.DebugLog.Writer() == io.Discard && l.InfoLog.Writer() == io.Discard && l.ErrLog.Writer() == io.Discard +} + +func (l *logger) Debug(format string, v ...interface{}) { + l.DebugLog.Printf(format, v...) +} + +func (l *logger) Info(format string, v ...interface{}) { + l.InfoLog.Printf(format, v...) +} + +func (l *logger) Err(format string, v ...interface{}) { + l.ErrLog.Printf(format, v...) +} + func (l *logger) Empty() { - if !l.devView && l.InfoLog.Writer() != io.Discard { - fmt.Println() + if l.InfoLog.Writer() != io.Discard { + fmt.Println("") } } -func (l *logger) StartProgress(message string) { - if l.devView { - l.InfoLog.Println(message) +func (l *logger) SetProgress(format string, v ...interface{}) { + if out.InfoLog.Writer() == io.Discard { return } + + message := fmt.Sprintf(format, v...) + + if l.progress != nil { + l.progress <- progress{ + message: message, + stop: false, + } + return + } + l.progress = make(chan progress) + l.done = make(chan interface{}) go func() { states := []string{"-", "\\", "|", "/"} + var count int + for i := 0; ; i++ { - l.progressWG.Lock() + ctx, cancel := context.WithTimeout(context.Background(), 35*time.Millisecond) select { case p := <-l.progress: - // clearing the last line - fmt.Printf("\r%s\r", strings.Repeat(" ", len(l.InfoLog.Prefix())+len(message)+2)) - if p.status { - successTag := "✔" - if runtime.GOOS == "windows" { - successTag = "~" + cancel() + + if p.stop { + if !l.devView { + fmt.Printf("\r" + strings.Repeat(" ", 2+len(message))) + fmt.Printf("\r➞ %s\n", p.message) + } else { + l.Debug(message) } - l.InfoLog.Printf("%s %s", successTag, p.message) + + l.progress = nil + + if count > 0 { + fmt.Printf("↳ %s\n", p.message) + } + + l.done <- nil + return } else { - errorTag := "✘" - if runtime.GOOS == "windows" { - errorTag = "!" + if !l.devView { + fmt.Printf("\r↓ %s\n", message) + } else { + l.Debug(message) } - l.ErrLog.Printf("%s %s", errorTag, p.message) + + l.progress = make(chan progress) + count++ + + if !l.devView { + fmt.Printf("\r" + strings.Repeat(" ", 2+len(message))) + fmt.Printf("\r➞ %s\n", p.message) + } else { + l.Debug(p.message) + } + message = p.message } - l.progress = nil - l.progressWG.Unlock() - return - default: - if i%10 == 0 { - fmt.Printf("\r%s%s %s", l.InfoLog.Prefix(), states[i/10%4], message) + case <-ctx.Done(): + if !l.devView && i%10 == 0 { + fmt.Printf("\r%s %s", states[i/10%4], message) } - time.Sleep(35 * time.Millisecond) - l.progressWG.Unlock() } } }() } -func (l *logger) StartProgressf(message string, a ...interface{}) { - l.StartProgress(fmt.Sprintf(message, a...)) -} - -func (l *logger) EndProgress(successful bool, message string) { - if l.devView { - if successful { - l.InfoLog.Print(message) - } else { - l.ErrLog.Print(message) - } +func (l *logger) StopProgress(format string, v ...interface{}) { + if out.InfoLog.Writer() == io.Discard { return - } else if l.progress != nil { - l.progress <- progress{ - status: successful, - message: message, - } - } -} - -func (l *logger) EndProgressf(successful bool, message string, a ...interface{}) { - l.EndProgress(successful, fmt.Sprintf(message, a...)) -} - -func (l *logger) Debugln(v ...interface{}) { - l.print(0, v...) -} - -func (l *logger) Debugf(message string, a ...interface{}) { - l.print(0, fmt.Sprintf(message, a...)) -} - -func (l *logger) Infoln(v ...interface{}) { - l.print(1, v...) -} - -func (l *logger) Infof(message string, a ...interface{}) { - l.print(1, fmt.Sprintf(message, a...)) -} - -func (l *logger) Errln(v ...interface{}) { - l.print(2, v...) -} - -func (l *logger) Errf(message string, a ...interface{}) { - l.print(2, fmt.Sprintf(message, a...)) -} - -func (l *logger) Fatalln(v ...interface{}) { - l.print(2, v...) - os.Exit(1) -} - -func (l *logger) Fatalf(message string, a ...interface{}) { - l.print(2, fmt.Sprintf(message, a...)) - os.Exit(1) -} - -func (l *logger) print(level int, v ...interface{}) { - if l.progress != nil { - l.progressWG.Lock() - defer l.progressWG.Unlock() - fmt.Print("\r") } - switch level { - case 0: - l.DebugLog.Print(v...) - case 1: - l.InfoLog.Print(v...) - case 2: - l.ErrLog.Print(v...) + l.progress <- progress{ + message: fmt.Sprintf(format, v...), + stop: true, } -} - -type loggerWriter struct { - io.Writer - - original io.Writer - color string -} - -func (lw *loggerWriter) Write(p []byte) (n int, err error) { - if lw.color != "" { - p = append([]byte(lw.color), p...) - p = append(p, []byte("\033[0m")...) - } - return lw.original.Write(p) + <-l.done } From 733f4a97ead8d18a4104d07daa25dbe87bc7e1dd Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 28 Feb 2022 17:42:18 +0100 Subject: [PATCH 42/82] Added support for storing login credential permanently --- cmd/crunchyroll-go/cmd/login.go | 64 +++++++++++++++++++++++---------- cmd/crunchyroll-go/cmd/utils.go | 57 ++++++++++++++++++----------- 2 files changed, 81 insertions(+), 40 deletions(-) diff --git a/cmd/crunchyroll-go/cmd/login.go b/cmd/crunchyroll-go/cmd/login.go index c2f07e0..f15bc5b 100644 --- a/cmd/crunchyroll-go/cmd/login.go +++ b/cmd/crunchyroll-go/cmd/login.go @@ -1,13 +1,20 @@ package cmd import ( + "fmt" "github.com/ByteDream/crunchyroll-go" "github.com/spf13/cobra" "io/ioutil" + "os" + "os/user" + "path/filepath" + "runtime" ) var ( - sessionIDFlag bool + loginSessionIDFlag bool + + loginPersistentFlag bool ) var loginCmd = &cobra.Command{ @@ -15,36 +22,55 @@ var loginCmd = &cobra.Command{ Short: "Login to crunchyroll", Args: cobra.RangeArgs(1, 2), - RunE: func(cmd *cobra.Command, args []string) error { - if sessionIDFlag { - return loginSessionID(args[0], false) + Run: func(cmd *cobra.Command, args []string) { + if loginSessionIDFlag { + loginSessionID(args[0]) } else { - return loginCredentials(args[0], args[1]) + loginCredentials(args[0], args[1]) } }, } func init() { + loginCmd.Flags().BoolVar(&loginSessionIDFlag, "session-id", false, "Use a session id to login instead of username and password") + + loginCmd.Flags().BoolVar(&loginPersistentFlag, "persistent", false, "If the given credential should be stored persistent") + rootCmd.AddCommand(loginCmd) - loginCmd.Flags().BoolVar(&sessionIDFlag, "session-id", false, "session id") } -func loginCredentials(email, password string) error { +func loginCredentials(user, password string) error { out.Debug("Logging in via credentials") - session, err := crunchyroll.LoginWithCredentials(email, password, locale, client) - if err != nil { - return err + if _, err := crunchyroll.LoginWithCredentials(user, password, systemLocale(), client); err != nil { + out.Err(err.Error()) + os.Exit(1) } - return loginSessionID(session.SessionID, true) + + return ioutil.WriteFile(loginStorePath(), []byte(fmt.Sprintf("%s\n%s", user, password)), 0600) } -func loginSessionID(sessionID string, alreadyChecked bool) error { - if !alreadyChecked { - out.Debug("Logging in via session id") - if _, err := crunchyroll.LoginWithSessionID(sessionID, locale, client); err != nil { - return err - } +func loginSessionID(sessionID string) error { + out.Debug("Logging in via session id") + if _, err := crunchyroll.LoginWithSessionID(sessionID, systemLocale(), client); err != nil { + out.Err(err.Error()) + os.Exit(1) } - out.Info("Due to security reasons, you have to login again on the next reboot") - return ioutil.WriteFile(sessionIDPath, []byte(sessionID), 0777) + + return ioutil.WriteFile(loginStorePath(), []byte(sessionID), 0600) +} + +func loginStorePath() string { + path := filepath.Join(os.TempDir(), ".crunchy") + if loginPersistentFlag { + if runtime.GOOS != "windows" { + usr, _ := user.Current() + path = filepath.Join(usr.HomeDir, ".config/crunchyroll-go") + } + + out.Info("The login information will be stored permanently UNENCRYPTED on your drive (%s)", path) + } else { + out.Info("Due to security reasons, you have to login again on the next reboot") + } + + return path } diff --git a/cmd/crunchyroll-go/cmd/utils.go b/cmd/crunchyroll-go/cmd/utils.go index 809e982..4a9e748 100644 --- a/cmd/crunchyroll-go/cmd/utils.go +++ b/cmd/crunchyroll-go/cmd/utils.go @@ -4,11 +4,11 @@ import ( "fmt" "github.com/ByteDream/crunchyroll-go" "github.com/ByteDream/crunchyroll-go/utils" - "io/ioutil" "net/http" "net/url" "os" "os/exec" + "os/user" "path" "path/filepath" "reflect" @@ -19,8 +19,6 @@ import ( "time" ) -var sessionIDPath = filepath.Join(os.TempDir(), ".crunchy") - var ( invalidWindowsChars = []string{"<", ">", ":", "\"", "/", "|", "\\", "?", "*"} invalidLinuxChars = []string{"/"} @@ -95,32 +93,49 @@ func freeFileName(filename string) (string, bool) { return filename, j != 0 } -func loadSessionID() (string, error) { - if _, stat := os.Stat(sessionIDPath); os.IsNotExist(stat) { - out.Err("To use this command, login first. Type `%s login -h` to get help", os.Args[0]) - os.Exit(1) - } - body, err := ioutil.ReadFile(sessionIDPath) - if err != nil { - return "", err - } - return strings.ReplaceAll(string(body), "\n", ""), nil -} - func loadCrunchy() { out.SetProgress("Logging in") - sessionID, err := loadSessionID() - if err == nil { - if crunchy, err = crunchyroll.LoginWithSessionID(sessionID, systemLocale(), client); err != nil { + + files := []string{filepath.Join(os.TempDir(), ".crunchy")} + + if runtime.GOOS != "windows" { + usr, _ := user.Current() + files = append(files, filepath.Join(usr.HomeDir, ".config/crunchyroll-go")) + } + + var body []byte + var err error + for _, file := range files { + if _, err = os.Stat(file); os.IsNotExist(err) { + continue + } + body, err = os.ReadFile(file) + break + } + if body == nil { + out.Err("To use this command, login first. Type `%s login -h` to get help", os.Args[0]) + os.Exit(1) + } else if err != nil { + out.Err("Failed to read login information: %v", err) + os.Exit(1) + } + + split := strings.SplitN(string(body), "\n", 2) + if len(split) == 1 || split[2] == "" { + if crunchy, err = crunchyroll.LoginWithSessionID(split[0], systemLocale(), client); err != nil { out.StopProgress(err.Error()) os.Exit(1) } + out.Debug("Logged in with session id %s. BLANK THIS LINE OUT IF YOU'RE ASKED TO POST THE DEBUG OUTPUT SOMEWHERE", split[0]) } else { - out.StopProgress(err.Error()) - os.Exit(1) + if crunchy, err = crunchyroll.LoginWithCredentials(split[0], split[1], systemLocale(), client); err != nil { + out.StopProgress(err.Error()) + os.Exit(1) + } + out.Debug("Logged in with username '%s' and password '%s'. BLANK THIS LINE OUT IF YOU'RE ASKED TO POST THE DEBUG OUTPUT SOMEWHERE", split[0], split[1]) } + out.StopProgress("Logged in") - out.Debug("Logged in with session id %s", sessionID) } func hasFFmpeg() bool { From 647eb075e592807d6b58411a757c81d657b01b94 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 28 Feb 2022 17:42:47 +0100 Subject: [PATCH 43/82] Renamed parameter in LoginWithCredentials --- crunchyroll.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crunchyroll.go b/crunchyroll.go index baa2947..8c456a4 100644 --- a/crunchyroll.go +++ b/crunchyroll.go @@ -57,8 +57,8 @@ type Crunchyroll struct { cache bool } -// LoginWithCredentials logs in via crunchyroll email and password -func LoginWithCredentials(email string, password string, locale LOCALE, client *http.Client) (*Crunchyroll, error) { +// LoginWithCredentials logs in via crunchyroll username or email and password +func LoginWithCredentials(user string, password string, locale LOCALE, client *http.Client) (*Crunchyroll, error) { sessionIDEndpoint := fmt.Sprintf("https://api.crunchyroll.com/start_session.0.json?version=1.0&access_token=%s&device_type=%s&device_id=%s", "LNDJgOit5yaRIWN", "com.crunchyroll.windows.desktop", "Az2srGnChW65fuxYz2Xxl1GcZQgtGgI") sessResp, err := client.Get(sessionIDEndpoint) @@ -76,7 +76,7 @@ func LoginWithCredentials(email string, password string, locale LOCALE, client * loginEndpoint := "https://api.crunchyroll.com/login.0.json" authValues := url.Values{} authValues.Set("session_id", sessionID) - authValues.Set("account", email) + authValues.Set("account", user) authValues.Set("password", password) client.Post(loginEndpoint, "application/x-www-form-urlencoded", bytes.NewBufferString(authValues.Encode())) From 5219f1841709902f411e2fa4dbcacb3c2dd94098 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 28 Feb 2022 17:56:11 +0100 Subject: [PATCH 44/82] Fixed array index out of bounds error when logging in --- cmd/crunchyroll-go/cmd/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/crunchyroll-go/cmd/utils.go b/cmd/crunchyroll-go/cmd/utils.go index 4a9e748..508e2e8 100644 --- a/cmd/crunchyroll-go/cmd/utils.go +++ b/cmd/crunchyroll-go/cmd/utils.go @@ -121,7 +121,7 @@ func loadCrunchy() { } split := strings.SplitN(string(body), "\n", 2) - if len(split) == 1 || split[2] == "" { + if len(split) == 1 || split[1] == "" { if crunchy, err = crunchyroll.LoginWithSessionID(split[0], systemLocale(), client); err != nil { out.StopProgress(err.Error()) os.Exit(1) From d6347a6b6ba3b7500888d8b00483f5bfb76493df Mon Sep 17 00:00:00 2001 From: bytedream Date: Wed, 2 Mar 2022 18:56:28 +0100 Subject: [PATCH 45/82] Moved url parsing methods from utils to standard library --- utils/url.go => url.go | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) rename utils/url.go => url.go (62%) diff --git a/utils/url.go b/url.go similarity index 62% rename from utils/url.go rename to url.go index 9bdec77..5abf370 100644 --- a/utils/url.go +++ b/url.go @@ -1,19 +1,18 @@ -package utils +package crunchyroll import ( "fmt" - "github.com/ByteDream/crunchyroll-go" ) // ExtractEpisodesFromUrl extracts all episodes from an url. // If audio is not empty, the episodes gets filtered after the given locale -func ExtractEpisodesFromUrl(crunchy *crunchyroll.Crunchyroll, url string, audio ...crunchyroll.LOCALE) ([]*crunchyroll.Episode, error) { - series, episodes, err := ParseUrl(crunchy, url) +func (c *Crunchyroll) ExtractEpisodesFromUrl(url string, audio ...LOCALE) ([]*Episode, error) { + series, episodes, err := c.ParseUrl(url) if err != nil { return nil, err } - var eps []*crunchyroll.Episode + var eps []*Episode if series != nil { seasons, err := series.Seasons() @@ -80,27 +79,27 @@ func ExtractEpisodesFromUrl(crunchy *crunchyroll.Crunchyroll, url string, audio // ParseUrl parses the given url into a series or episode. // The returning episode is a slice because non-beta urls have the same episode with different languages -func ParseUrl(crunchy *crunchyroll.Crunchyroll, url string) (*crunchyroll.Series, []*crunchyroll.Episode, error) { - if seriesId, ok := crunchyroll.ParseBetaSeriesURL(url); ok { - series, err := crunchyroll.SeriesFromID(crunchy, seriesId) +func (c *Crunchyroll) ParseUrl(url string) (*Series, []*Episode, error) { + if seriesId, ok := ParseBetaSeriesURL(url); ok { + series, err := SeriesFromID(c, seriesId) if err != nil { return nil, nil, err } return series, nil, nil - } else if episodeId, ok := crunchyroll.ParseBetaEpisodeURL(url); ok { - episode, err := crunchyroll.EpisodeFromID(crunchy, episodeId) + } else if episodeId, ok := ParseBetaEpisodeURL(url); ok { + episode, err := EpisodeFromID(c, episodeId) if err != nil { return nil, nil, err } - return nil, []*crunchyroll.Episode{episode}, nil - } else if seriesName, ok := crunchyroll.ParseVideoURL(url); ok { - video, err := crunchy.FindVideoByName(seriesName) + return nil, []*Episode{episode}, nil + } else if seriesName, ok := ParseVideoURL(url); ok { + video, err := c.FindVideoByName(seriesName) if err != nil { return nil, nil, err } - return video.(*crunchyroll.Series), nil, nil - } else if seriesName, title, _, _, ok := crunchyroll.ParseEpisodeURL(url); ok { - episodes, err := crunchy.FindEpisodeByName(seriesName, title) + return video.(*Series), nil, nil + } else if seriesName, title, _, _, ok := ParseEpisodeURL(url); ok { + episodes, err := c.FindEpisodeByName(seriesName, title) if err != nil { return nil, nil, err } From 6fb6b3c03ce4fc29022ccc40138a9193a2561416 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 6 Mar 2022 01:01:25 +0100 Subject: [PATCH 46/82] Moved all download related stuff to downloader.go and fixed some errors in it --- downloader.go | 295 ++++++++++++++++++++++++++++++++++++++------------ format.go | 152 +------------------------- 2 files changed, 230 insertions(+), 217 deletions(-) diff --git a/downloader.go b/downloader.go index 828f684..d0bdf7c 100644 --- a/downloader.go +++ b/downloader.go @@ -1,29 +1,53 @@ package crunchyroll import ( + "bytes" "context" "crypto/aes" "crypto/cipher" "fmt" "github.com/grafov/m3u8" + "io" "io/ioutil" "math" "net/http" "os" + "os/exec" "path/filepath" + "regexp" "sync" "sync/atomic" "time" ) +var ffmpegInfoPattern = regexp.MustCompile(`Output #0, (.+),`) + +// NewDownloader creates a downloader with default settings which should +// fit the most needs +func NewDownloader(context context.Context, writer io.Writer, goroutines int, onSegmentDownload func(segment *m3u8.MediaSegment, current, total int, file *os.File) error) Downloader { + tmp, _ := os.MkdirTemp("", "crunchy_") + + return Downloader{ + Writer: writer, + TempDir: tmp, + DeleteTempAfter: true, + Context: context, + Goroutines: goroutines, + OnSegmentDownload: onSegmentDownload, + } +} + +// Downloader is used to download Format's type Downloader struct { - // Filename is the filename of the output file - Filename string - // TempDir is the directory where the temporary files should be stored + // The output is all written to Writer + Writer io.Writer + + // TempDir is the directory where the temporary segment files should be stored. + // The files will be placed directly into the root of the directory. + // If empty a random temporary directory on the system's default tempdir + // will be created. + // If the directory does not exist, it will be created TempDir string - // If IgnoreExisting is true, existing Filename's and TempDir's may be - // overwritten or deleted - IgnoreExisting bool // If DeleteTempAfter is true, the temp directory gets deleted afterwards. // Note that in case of a hard signal exit (os.Interrupt, ...) the directory // will NOT be deleted. In such situations try to catch the signal and @@ -41,7 +65,10 @@ type Downloader struct { // Goroutines is the number of goroutines to download segments with Goroutines int - // A method to call when a segment was downloaded + // A method to call when a segment was downloaded. + // Note that the segments are downloaded asynchronously (depending on the count of + // Goroutines) and the function gets called asynchronously too, so for example it is + // first called on segment 1, then segment 254, then segment 3 and so on OnSegmentDownload func(segment *m3u8.MediaSegment, current, total int, file *os.File) error // If LockOnSegmentDownload is true, only one OnSegmentDownload function can be called at // once. Normally (because of the use of goroutines while downloading) multiple could get @@ -50,51 +77,177 @@ type Downloader struct { // If FFmpegOpts is not nil, ffmpeg will be used to merge and convert files. // The given opts will be used as ffmpeg parameters while merging. - // Some opts are already used, see mergeSegmentsFFmpeg in format.go for more details + // + // If Writer is *os.File and -f (which sets the output format) is not specified, the output + // format will be retrieved by its file ending. If this is not the case and -f is not given, + // the output format will be mpegts / mpeg transport stream. + // Execute 'ffmpeg -muxers' to see all available output formats. FFmpegOpts []string } -// NewDownloader creates a downloader with default settings which should -// fit the most needs -func NewDownloader(context context.Context, filename string, goroutines int, onSegmentDownload func(segment *m3u8.MediaSegment, current, total int, file *os.File) error) Downloader { - tmp, _ := os.MkdirTemp("", "crunchy_") - - return Downloader{ - Filename: filename, - TempDir: tmp, - DeleteTempAfter: true, - Context: context, - Goroutines: goroutines, - OnSegmentDownload: onSegmentDownload, - } -} - -// download downloads every mpeg transport stream segment to a given directory (more information below). -// After every segment download onSegmentDownload will be called with: -// the downloaded segment, the current position, the total size of segments to download, the file where the segment content was written to an error (if occurred). -// The filename is always .ts -// -// Short explanation: -// The actual crunchyroll video is split up in multiple segments (or video files) which have to be downloaded and merged after to generate a single video file. -// And this function just downloads each of this segment into the given directory. -// See https://en.wikipedia.org/wiki/MPEG_transport_stream for more information -func download(context context.Context, format *Format, tempDir string, goroutines int, lockOnSegmentDownload bool, onSegmentDownload func(segment *m3u8.MediaSegment, current, total int, file *os.File) error) error { +// download's the given format +func (d Downloader) download(format *Format) error { if err := format.InitVideo(); err != nil { return err } + if _, err := os.Stat(d.TempDir); os.IsNotExist(err) { + if err := os.Mkdir(d.TempDir, 0700); err != nil { + return err + } + } + if d.DeleteTempAfter { + defer os.RemoveAll(d.TempDir) + } + + files, err := d.downloadSegments(format) + if err != nil { + return err + } + if d.FFmpegOpts == nil { + return d.mergeSegments(files) + } else { + return d.mergeSegmentsFFmpeg(files) + } +} + +// mergeSegments reads every file in tempDir and writes their content to Downloader.Writer. +// The given output file gets created or overwritten if already existing +func (d Downloader) mergeSegments(files []string) error { + for _, file := range files { + select { + case <-d.Context.Done(): + return d.Context.Err() + default: + f, err := os.Open(file) + if err != nil { + return err + } + if _, err = io.Copy(d.Writer, f); err != nil { + f.Close() + return err + } + f.Close() + } + } + return nil +} + +// mergeSegmentsFFmpeg reads every file in tempDir and merges their content to the outputFile +// with ffmpeg (https://ffmpeg.org/). +// The given output file gets created or overwritten if already existing +func (d Downloader) mergeSegmentsFFmpeg(files []string) error { + list, err := os.Create(filepath.Join(d.TempDir, "list.txt")) + if err != nil { + return err + } + + for _, file := range files { + if _, err = fmt.Fprintf(list, "file '%s'\n", file); err != nil { + list.Close() + return err + } + } + list.Close() + + // predefined options ... custom options ... predefined output filename + command := []string{ + "-y", + "-f", "concat", + "-safe", "0", + "-i", list.Name(), + "-c", "copy", + } + if d.FFmpegOpts != nil { + command = append(command, d.FFmpegOpts...) + var found bool + for _, opts := range d.FFmpegOpts { + if opts == "-f" { + found = true + break + } + } + if !found { + if file, ok := d.Writer.(*os.File); ok { + var outBuf bytes.Buffer + infoCmd := exec.Command("ffmpeg", file.Name()) + infoCmd.Stderr = &outBuf + + if infoCmd.Run(); err != nil { + return err + } + if parsed := ffmpegInfoPattern.FindStringSubmatch(outBuf.String()); parsed != nil { + command = append(command, "-f", parsed[1]) + } + } else { + command = append(command, "-f", "mpegts") + } + } + } + command = append(command, "pipe:1") + + var errBuf bytes.Buffer + cmd := exec.Command("ffmpeg", + command...) + cmd.Stderr = &errBuf + // io.Copy may be better but this uses less code so ¯\_(ツ)_/¯ + cmd.Stdout = d.Writer + + if err = cmd.Start(); err != nil { + return err + } + + cmdChan := make(chan error, 1) + go func() { + cmdChan <- cmd.Wait() + }() + + select { + case err = <-cmdChan: + if err != nil { + if errBuf.Len() > 0 { + return fmt.Errorf(errBuf.String()) + } else { + return err + } + } + return nil + case <-d.Context.Done(): + cmd.Process.Kill() + return d.Context.Err() + } +} + +// downloadSegments downloads every mpeg transport stream segment to a given +// directory (more information below). +// After every segment download onSegmentDownload will be called with: +// the downloaded segment, the current position, the total size of segments to download, +// the file where the segment content was written to an error (if occurred). +// The filename is always .ts +// +// Short explanation: +// The actual crunchyroll video is split up in multiple segments (or video files) which +// have to be downloaded and merged after to generate a single video file. +// And this function just downloads each of this segment into the given directory. +// See https://en.wikipedia.org/wiki/MPEG_transport_stream for more information +func (d Downloader) downloadSegments(format *Format) ([]string, error) { + if err := format.InitVideo(); err != nil { + return nil, err + } + var wg sync.WaitGroup var lock sync.Mutex - chunkSize := int(math.Ceil(float64(format.Video.Chunklist.Count()) / float64(goroutines))) + chunkSize := int(math.Ceil(float64(format.Video.Chunklist.Count()) / float64(d.Goroutines))) - // when a onSegmentDownload call returns an error, this channel will be set to true and stop all goroutines - quit := make(chan bool) + // when a onSegmentDownload call returns an error, this context will be set cancelled and stop all goroutines + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() // receives the decrypt block and iv from the first segment. // in my tests, only the first segment has specified this data, so the decryption data from this first segments will be used in every other segment too block, iv, err := getCrypt(format, format.Video.Chunklist.Segments[0]) if err != nil { - return err + return nil, err } var total int32 @@ -111,39 +264,42 @@ func download(context context.Context, format *Format, tempDir string, goroutine for j, segment := range format.Video.Chunklist.Segments[i:end] { select { - case <-context.Done(): - return - case <-quit: + case <-d.Context.Done(): + case <-ctx.Done(): return default: var file *os.File - k := 1 - for ; k < 4; k++ { - file, err = downloadSegment(context, format, segment, filepath.Join(tempDir, fmt.Sprintf("%d.ts", i+j)), block, iv) + for k := 0; k < 3; k++ { + filename := filepath.Join(d.TempDir, fmt.Sprintf("%d.ts", i+j)) + file, err = d.downloadSegment(format, segment, filename, block, iv) if err == nil { break } - // sleep if an error occurs. very useful because sometimes the connection times out - time.Sleep(5 * time.Duration(k) * time.Second) + if k == 2 { + cancel() + return + } + select { + case <-d.Context.Done(): + case <-ctx.Done(): + return + case <-time.After(5 * time.Duration(k) * time.Second): + // sleep if an error occurs. very useful because sometimes the connection times out + } } - if k == 4 { - quit <- true - return - } - if onSegmentDownload != nil { - if lockOnSegmentDownload { + if d.OnSegmentDownload != nil { + if d.LockOnSegmentDownload { lock.Lock() } - if err = onSegmentDownload(segment, int(atomic.AddInt32(&total, 1)), int(format.Video.Chunklist.Count()), file); err != nil { - quit <- true - if lockOnSegmentDownload { + if err = d.OnSegmentDownload(segment, int(atomic.AddInt32(&total, 1)), int(format.Video.Chunklist.Count()), file); err != nil { + if d.LockOnSegmentDownload { lock.Unlock() } file.Close() return } - if lockOnSegmentDownload { + if d.LockOnSegmentDownload { lock.Unlock() } } @@ -155,16 +311,21 @@ func download(context context.Context, format *Format, tempDir string, goroutine wg.Wait() select { - case <-context.Done(): - return context.Err() - case <-quit: - return err + case <-d.Context.Done(): + return nil, d.Context.Err() + case <-ctx.Done(): + return nil, err default: - return nil + var files []string + for i := 0; i < int(total); i++ { + files = append(files, filepath.Join(d.TempDir, fmt.Sprintf("%d.ts", i))) + } + + return files, nil } } -// getCrypt extracts the key and iv of a m3u8 segment and converts it into a cipher.Block block and a iv byte sequence +// getCrypt extracts the key and iv of a m3u8 segment and converts it into a cipher.Block and an iv byte sequence func getCrypt(format *Format, segment *m3u8.MediaSegment) (block cipher.Block, iv []byte, err error) { var resp *http.Response @@ -188,9 +349,9 @@ func getCrypt(format *Format, segment *m3u8.MediaSegment) (block cipher.Block, i } // downloadSegment downloads a segment, decrypts it and names it after the given index -func downloadSegment(context context.Context, format *Format, segment *m3u8.MediaSegment, filename string, block cipher.Block, iv []byte) (*os.File, error) { +func (d Downloader) downloadSegment(format *Format, segment *m3u8.MediaSegment, filename string, block cipher.Block, iv []byte) (*os.File, error) { // every segment is aes-128 encrypted and has to be decrypted when downloaded - content, err := decryptSegment(context, format.crunchy.Client, segment, block, iv) + content, err := d.decryptSegment(format.crunchy.Client, segment, block, iv) if err != nil { return nil, err } @@ -208,12 +369,12 @@ func downloadSegment(context context.Context, format *Format, segment *m3u8.Medi } // https://github.com/oopsguy/m3u8/blob/4150e93ec8f4f8718875a02973f5d792648ecb97/tool/crypt.go#L25 -func decryptSegment(context context.Context, client *http.Client, segment *m3u8.MediaSegment, block cipher.Block, iv []byte) ([]byte, error) { +func (d Downloader) decryptSegment(client *http.Client, segment *m3u8.MediaSegment, block cipher.Block, iv []byte) ([]byte, error) { req, err := http.NewRequest(http.MethodGet, segment.URI, nil) if err != nil { return nil, err } - req.WithContext(context) + req.WithContext(d.Context) resp, err := client.Do(req) if err != nil { @@ -229,13 +390,13 @@ func decryptSegment(context context.Context, client *http.Client, segment *m3u8. blockMode := cipher.NewCBCDecrypter(block, iv[:block.BlockSize()]) decrypted := make([]byte, len(raw)) blockMode.CryptBlocks(decrypted, raw) - raw = pkcs5UnPadding(decrypted) + raw = d.pkcs5UnPadding(decrypted) return raw, nil } // https://github.com/oopsguy/m3u8/blob/4150e93ec8f4f8718875a02973f5d792648ecb97/tool/crypt.go#L47 -func pkcs5UnPadding(origData []byte) []byte { +func (d Downloader) pkcs5UnPadding(origData []byte) []byte { length := len(origData) unPadding := int(origData[length-1]) return origData[:(length - unPadding)] diff --git a/format.go b/format.go index 9dfaf3e..ec94c16 100644 --- a/format.go +++ b/format.go @@ -1,18 +1,7 @@ package crunchyroll import ( - "bufio" - "bytes" - "context" - "fmt" "github.com/grafov/m3u8" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "sort" - "strconv" - "strings" ) type FormatType string @@ -56,144 +45,7 @@ func (f *Format) InitVideo() error { return nil } +// Download downloads the Format with the via Downloader specified options func (f *Format) Download(downloader Downloader) error { - if _, err := os.Stat(downloader.Filename); err == nil && !downloader.IgnoreExisting { - return fmt.Errorf("file %s already exists", downloader.Filename) - } - if _, err := os.Stat(downloader.TempDir); err != nil { - if os.IsNotExist(err) { - err = os.Mkdir(downloader.TempDir, 0755) - } - if err != nil { - return err - } - } else if !downloader.IgnoreExisting { - content, err := os.ReadDir(downloader.TempDir) - if err != nil { - return err - } - if len(content) > 0 { - return fmt.Errorf("directory %s is not empty", downloader.Filename) - } - } - - if downloader.DeleteTempAfter { - defer os.RemoveAll(downloader.TempDir) - } - if err := download(downloader.Context, f, downloader.TempDir, downloader.Goroutines, downloader.LockOnSegmentDownload, downloader.OnSegmentDownload); err != nil { - return err - } - - if downloader.FFmpegOpts != nil { - return mergeSegmentsFFmpeg(downloader.Context, downloader.TempDir, downloader.Filename, downloader.FFmpegOpts) - } else { - return mergeSegments(downloader.Context, downloader.TempDir, downloader.Filename) - } -} - -// mergeSegments reads every file in tempDir and writes their content to the outputFile. -// The given output file gets created or overwritten if already existing -func mergeSegments(context context.Context, tempDir string, outputFile string) error { - dir, err := os.ReadDir(tempDir) - if err != nil { - return err - } - file, err := os.OpenFile(outputFile, os.O_CREATE|os.O_WRONLY, 0755) - if err != nil { - return err - } - defer file.Close() - writer := bufio.NewWriter(file) - defer writer.Flush() - - // sort the directory files after their numeric names - sort.Slice(dir, func(i, j int) bool { - iNum, err := strconv.Atoi(strings.Split(dir[i].Name(), ".")[0]) - if err != nil { - return false - } - jNum, err := strconv.Atoi(strings.Split(dir[j].Name(), ".")[0]) - if err != nil { - return false - } - return iNum < jNum - }) - - for _, file := range dir { - select { - case <-context.Done(): - return context.Err() - default: - } - - bodyAsBytes, err := ioutil.ReadFile(filepath.Join(tempDir, file.Name())) - if err != nil { - return err - } - if _, err = writer.Write(bodyAsBytes); err != nil { - return err - } - } - return nil -} - -// mergeSegmentsFFmpeg reads every file in tempDir and merges their content to the outputFile -// with ffmpeg (https://ffmpeg.org/). -// The given output file gets created or overwritten if already existing -func mergeSegmentsFFmpeg(context context.Context, tempDir string, outputFile string, opts []string) error { - dir, err := os.ReadDir(tempDir) - if err != nil { - return err - } - f, err := os.Create(filepath.Join(tempDir, "list.txt")) - if err != nil { - return err - } - // -1 is the list.txt file - for i := 0; i < len(dir)-1; i++ { - fmt.Fprintf(f, "file '%s.ts'\n", filepath.Join(tempDir, strconv.Itoa(i))) - } - f.Close() - - // predefined options ... custom options ... predefined output filename - command := []string{ - "-y", - "-f", "concat", - "-safe", "0", - "-i", f.Name(), - "-c", "copy", - } - if opts != nil { - command = append(command, opts...) - } - command = append(command, outputFile) - - var errBuf bytes.Buffer - cmd := exec.Command("ffmpeg", - command...) - cmd.Stderr = &errBuf - - if err := cmd.Start(); err != nil { - return err - } - - cmdChan := make(chan error, 1) - go func() { - cmdChan <- cmd.Wait() - }() - - select { - case err = <-cmdChan: - if err != nil { - if errBuf.Len() > 0 { - return fmt.Errorf(errBuf.String()) - } else { - return err - } - } - return nil - case <-context.Done(): - cmd.Process.Kill() - return context.Err() - } + return downloader.download(f) } From 5f5ec3858529821c0cf72a66d372274da3bd6091 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 6 Mar 2022 01:04:27 +0100 Subject: [PATCH 47/82] Added function to sort episode by audio --- utils/sort.go | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/utils/sort.go b/utils/sort.go index 16ce41a..90e05c7 100644 --- a/utils/sort.go +++ b/utils/sort.go @@ -5,6 +5,7 @@ import ( "sort" "strconv" "strings" + "sync" ) // SortEpisodesBySeason sorts the given episodes by their seasons. @@ -42,6 +43,44 @@ func SortEpisodesBySeason(episodes []*crunchyroll.Episode) [][]*crunchyroll.Epis return eps } +// SortEpisodesByAudio sort the given episodes by their audio locale +func SortEpisodesByAudio(episodes []*crunchyroll.Episode) (map[crunchyroll.LOCALE][]*crunchyroll.Episode, error) { + eps := map[crunchyroll.LOCALE][]*crunchyroll.Episode{} + + errChan := make(chan error) + + var wg sync.WaitGroup + var lock sync.Mutex + for _, episode := range episodes { + episode := episode + wg.Add(1) + go func() { + defer wg.Done() + audioLocale, err := episode.AudioLocale() + if err != nil { + errChan <- err + return + } + lock.Lock() + defer lock.Unlock() + + if _, ok := eps[audioLocale]; !ok { + eps[audioLocale] = make([]*crunchyroll.Episode, 0) + } + eps[audioLocale] = append(eps[audioLocale], episode) + }() + } + go func() { + wg.Wait() + errChan <- nil + }() + + if err := <-errChan; err != nil { + return nil, err + } + return eps, nil +} + // MovieListingsByDuration sorts movie listings by their duration type MovieListingsByDuration []*crunchyroll.MovieListing From e855c16a2f87d837afcb06203ba334f18edf1951 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 6 Mar 2022 01:08:12 +0100 Subject: [PATCH 48/82] New progress bar calculate method and using utils implementation for audio sorting --- cmd/crunchyroll-go/cmd/utils.go | 44 ++++++++++++++------------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/cmd/crunchyroll-go/cmd/utils.go b/cmd/crunchyroll-go/cmd/utils.go index 508e2e8..839611b 100644 --- a/cmd/crunchyroll-go/cmd/utils.go +++ b/cmd/crunchyroll-go/cmd/utils.go @@ -186,33 +186,20 @@ func generateFilename(name, directory string) string { func extractEpisodes(url string, locales ...crunchyroll.LOCALE) [][]*crunchyroll.Episode { final := make([][]*crunchyroll.Episode, len(locales)) - episodes, err := utils.ExtractEpisodesFromUrl(crunchy, url, locales...) + episodes, err := crunchy.ExtractEpisodesFromUrl(url, locales...) if err != nil { out.Err("Failed to get episodes: %v", err) os.Exit(1) } - // fetch all episodes and sort them by their locale - var wg sync.WaitGroup - for _, episode := range episodes { - episode := episode - wg.Add(1) - go func() { - defer wg.Done() - audioLocale, err := episode.AudioLocale() - if err != nil { - out.Err("Failed to get audio locale: %v", err) - os.Exit(1) - } - - for i, locale := range locales { - if locale == audioLocale { - final[i] = append(final[i], episode) - } - } - }() + localeSorted, err := utils.SortEpisodesByAudio(episodes) + if err != nil { + out.Err("Failed to get audio locale: %v", err) + os.Exit(1) + } + for i, locale := range locales { + final[i] = append(final[i], localeSorted[locale]...) } - wg.Wait() return final } @@ -306,14 +293,21 @@ func (dp *DownloadProgress) update(msg string, permanent bool) { } percentage := float32(dp.Current) / float32(dp.Total) * 100 - progressWidth := float32(terminalWidth() - (12 + len(dp.Prefix) + len(dp.Message)) - (len(fmt.Sprint(dp.Total)))*2) - repeatCount := int(percentage / (float32(100) / progressWidth)) + pre := fmt.Sprintf("%s%s [", dp.Prefix, msg) + post := fmt.Sprintf("]%4d%% %8d/%d", int(percentage), dp.Current, dp.Total) + + // i don't really know why +2 is needed here but without it the Printf below would not print to the line end + progressWidth := terminalWidth() - len(pre) - len(post) + 2 + repeatCount := int(percentage / float32(100) * float32(progressWidth)) // it can be lower than zero when the terminal is very tiny if repeatCount < 0 { repeatCount = 0 } - progressPercentage := (strings.Repeat("=", repeatCount) + ">")[1:] + progressPercentage := strings.Repeat("=", repeatCount) + if dp.Current != dp.Total { + progressPercentage += ">" + } - fmt.Printf("\r%s%s [%-"+fmt.Sprint(progressWidth)+"s]%4d%% %8d/%d", dp.Prefix, msg, progressPercentage, int(percentage), dp.Current, dp.Total) + fmt.Printf("\r%s%-"+fmt.Sprint(progressWidth)+"s%s", pre, progressPercentage, post) } From b9b9654d2cee86c39b58927ff547d7012b2471e2 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 6 Mar 2022 01:09:39 +0100 Subject: [PATCH 49/82] Renamed persistent credentials file and restricted login message for windows users --- cmd/crunchyroll-go/cmd/login.go | 4 ++-- cmd/crunchyroll-go/cmd/utils.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/crunchyroll-go/cmd/login.go b/cmd/crunchyroll-go/cmd/login.go index f15bc5b..fa55020 100644 --- a/cmd/crunchyroll-go/cmd/login.go +++ b/cmd/crunchyroll-go/cmd/login.go @@ -64,11 +64,11 @@ func loginStorePath() string { if loginPersistentFlag { if runtime.GOOS != "windows" { usr, _ := user.Current() - path = filepath.Join(usr.HomeDir, ".config/crunchyroll-go") + path = filepath.Join(usr.HomeDir, ".config/crunchy") } out.Info("The login information will be stored permanently UNENCRYPTED on your drive (%s)", path) - } else { + } else if runtime.GOOS != "windows" { out.Info("Due to security reasons, you have to login again on the next reboot") } diff --git a/cmd/crunchyroll-go/cmd/utils.go b/cmd/crunchyroll-go/cmd/utils.go index 839611b..09b4a2f 100644 --- a/cmd/crunchyroll-go/cmd/utils.go +++ b/cmd/crunchyroll-go/cmd/utils.go @@ -100,7 +100,7 @@ func loadCrunchy() { if runtime.GOOS != "windows" { usr, _ := user.Current() - files = append(files, filepath.Join(usr.HomeDir, ".config/crunchyroll-go")) + files = append(files, filepath.Join(usr.HomeDir, ".config/crunchy")) } var body []byte From d42206b8f92fc86c0cde1deb15c2dc59458405a5 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 6 Mar 2022 01:10:16 +0100 Subject: [PATCH 50/82] Added documentation for GetFormat --- episode.go | 1 + 1 file changed, 1 insertion(+) diff --git a/episode.go b/episode.go index 12d21b9..6ab330b 100644 --- a/episode.go +++ b/episode.go @@ -119,6 +119,7 @@ func (e *Episode) AudioLocale() (LOCALE, error) { return streams[0].AudioLocale, nil } +// GetFormat returns the format which matches the given resolution and subtitle locale func (e *Episode) GetFormat(resolution string, subtitle LOCALE, hardsub bool) (*Format, error) { streams, err := e.Streams() if err != nil { From b1e5718c8d7428f3a9db04f491c96ef0473dee33 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 6 Mar 2022 01:10:38 +0100 Subject: [PATCH 51/82] Added exit logging function --- cmd/crunchyroll-go/cmd/logger.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/crunchyroll-go/cmd/logger.go b/cmd/crunchyroll-go/cmd/logger.go index d9c7248..cf15f57 100644 --- a/cmd/crunchyroll-go/cmd/logger.go +++ b/cmd/crunchyroll-go/cmd/logger.go @@ -74,6 +74,10 @@ func (l *logger) Err(format string, v ...interface{}) { l.ErrLog.Printf(format, v...) } +func (l *logger) Exit(format string, v ...interface{}) { + fmt.Fprintln(l.ErrLog.Writer(), fmt.Sprintf(format, v...)) +} + func (l *logger) Empty() { if l.InfoLog.Writer() != io.Discard { fmt.Println("") From a378c93d68d6ed9e0c3647b0289016100f51c031 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 6 Mar 2022 02:41:23 +0100 Subject: [PATCH 52/82] Made some structs private --- cmd/crunchyroll-go/cmd/utils.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/crunchyroll-go/cmd/utils.go b/cmd/crunchyroll-go/cmd/utils.go index 09b4a2f..b9bc2d0 100644 --- a/cmd/crunchyroll-go/cmd/utils.go +++ b/cmd/crunchyroll-go/cmd/utils.go @@ -204,7 +204,7 @@ func extractEpisodes(url string, locales ...crunchyroll.LOCALE) [][]*crunchyroll return final } -type FormatInformation struct { +type formatInformation struct { // the format to download format *crunchyroll.Format @@ -221,7 +221,7 @@ type FormatInformation struct { Subtitle crunchyroll.LOCALE `json:"subtitle"` } -func (fi FormatInformation) Format(source string) string { +func (fi formatInformation) Format(source string) string { fields := reflect.TypeOf(fi) values := reflect.ValueOf(fi) @@ -246,7 +246,7 @@ func (fi FormatInformation) Format(source string) string { return source } -type DownloadProgress struct { +type downloadProgress struct { Prefix string Message string @@ -259,15 +259,15 @@ type DownloadProgress struct { lock sync.Mutex } -func (dp *DownloadProgress) Update() { +func (dp *downloadProgress) Update() { dp.update("", false) } -func (dp *DownloadProgress) UpdateMessage(msg string, permanent bool) { +func (dp *downloadProgress) UpdateMessage(msg string, permanent bool) { dp.update(msg, permanent) } -func (dp *DownloadProgress) update(msg string, permanent bool) { +func (dp *downloadProgress) update(msg string, permanent bool) { if dp.Quiet { return } From 600eaeeecac7ec6e353d360e684b391b3fce9181 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 6 Mar 2022 16:04:44 +0100 Subject: [PATCH 53/82] Print empty line only if dev view is not enabled --- cmd/crunchyroll-go/cmd/logger.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/crunchyroll-go/cmd/logger.go b/cmd/crunchyroll-go/cmd/logger.go index cf15f57..22e4582 100644 --- a/cmd/crunchyroll-go/cmd/logger.go +++ b/cmd/crunchyroll-go/cmd/logger.go @@ -79,7 +79,7 @@ func (l *logger) Exit(format string, v ...interface{}) { } func (l *logger) Empty() { - if l.InfoLog.Writer() != io.Discard { + if !l.devView && l.InfoLog.Writer() != io.Discard { fmt.Println("") } } From 60fed00b0e93158c939af9019454d543cbd44de2 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 14 Mar 2022 23:28:52 +0100 Subject: [PATCH 54/82] Added .tar.* file ending --- cmd/crunchyroll-go/cmd/utils.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/crunchyroll-go/cmd/utils.go b/cmd/crunchyroll-go/cmd/utils.go index b9bc2d0..208b1d5 100644 --- a/cmd/crunchyroll-go/cmd/utils.go +++ b/cmd/crunchyroll-go/cmd/utils.go @@ -9,7 +9,6 @@ import ( "os" "os/exec" "os/user" - "path" "path/filepath" "reflect" "runtime" @@ -81,8 +80,13 @@ func createOrDefaultClient(proxy string) (*http.Client, error) { } func freeFileName(filename string) (string, bool) { - ext := path.Ext(filename) + ext := filepath.Ext(filename) base := strings.TrimSuffix(filename, ext) + // checks if a .tar stands before the "actual" file ending + if extraExt := filepath.Ext(base); extraExt == ".tar" { + ext = extraExt + ext + base = strings.TrimSuffix(base, extraExt) + } j := 0 for ; ; j++ { if _, stat := os.Stat(filename); stat != nil && !os.IsExist(stat) { From b1945d672d47a83b442d0ce556766bafbca1b15a Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 14 Mar 2022 23:41:55 +0100 Subject: [PATCH 55/82] Changed filename processing when merging with ffmpeg --- downloader.go | 92 +++++++++++++++++++++++---------------------------- 1 file changed, 41 insertions(+), 51 deletions(-) diff --git a/downloader.go b/downloader.go index d0bdf7c..48460c3 100644 --- a/downloader.go +++ b/downloader.go @@ -14,14 +14,12 @@ import ( "os" "os/exec" "path/filepath" - "regexp" + "strings" "sync" "sync/atomic" "time" ) -var ffmpegInfoPattern = regexp.MustCompile(`Output #0, (.+),`) - // NewDownloader creates a downloader with default settings which should // fit the most needs func NewDownloader(context context.Context, writer io.Writer, goroutines int, onSegmentDownload func(segment *m3u8.MediaSegment, current, total int, file *os.File) error) Downloader { @@ -160,62 +158,54 @@ func (d Downloader) mergeSegmentsFFmpeg(files []string) error { } if d.FFmpegOpts != nil { command = append(command, d.FFmpegOpts...) - var found bool + } + + var tmpfile string + if _, ok := d.Writer.(*io.PipeWriter); ok { + if file, ok := d.Writer.(*os.File); ok { + tmpfile = filepath.Base(file.Name()) + } + } + if filepath.Ext(tmpfile) == "" { + // checks if the -f flag is set (overwrites the output format) + var hasF bool for _, opts := range d.FFmpegOpts { - if opts == "-f" { - found = true + if strings.TrimSpace(opts) == "-f" { + hasF = true break } } - if !found { - if file, ok := d.Writer.(*os.File); ok { - var outBuf bytes.Buffer - infoCmd := exec.Command("ffmpeg", file.Name()) - infoCmd.Stderr = &outBuf - - if infoCmd.Run(); err != nil { - return err - } - if parsed := ffmpegInfoPattern.FindStringSubmatch(outBuf.String()); parsed != nil { - command = append(command, "-f", parsed[1]) - } - } else { - command = append(command, "-f", "mpegts") - } - } - } - command = append(command, "pipe:1") - - var errBuf bytes.Buffer - cmd := exec.Command("ffmpeg", - command...) - cmd.Stderr = &errBuf - // io.Copy may be better but this uses less code so ¯\_(ツ)_/¯ - cmd.Stdout = d.Writer - - if err = cmd.Start(); err != nil { - return err - } - - cmdChan := make(chan error, 1) - go func() { - cmdChan <- cmd.Wait() - }() - - select { - case err = <-cmdChan: - if err != nil { - if errBuf.Len() > 0 { - return fmt.Errorf(errBuf.String()) - } else { + if !hasF { + command = append(command, "-f", "matroska") + f, err := os.CreateTemp(d.TempDir, "") + if err != nil { return err } + f.Close() + tmpfile = f.Name() } - return nil - case <-d.Context.Done(): - cmd.Process.Kill() - return d.Context.Err() } + command = append(command, tmpfile) + + var errBuf bytes.Buffer + cmd := exec.CommandContext(d.Context, "ffmpeg", + command...) + cmd.Stderr = &errBuf + + if err = cmd.Run(); err != nil { + if errBuf.Len() > 0 { + return fmt.Errorf(errBuf.String()) + } else { + return err + } + } + file, err := os.Open(tmpfile) + if err != nil { + return err + } + defer file.Close() + _, err = io.Copy(d.Writer, file) + return err } // downloadSegments downloads every mpeg transport stream segment to a given From 4d538dfc0c0dae0eb0e333375f0aa70bec3ffc8e Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 14 Mar 2022 23:42:28 +0100 Subject: [PATCH 56/82] Added actual error message to recover error output --- cmd/crunchyroll-go/cmd/root.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cmd/crunchyroll-go/cmd/root.go b/cmd/crunchyroll-go/cmd/root.go index db2cba6..2fe3a61 100644 --- a/cmd/crunchyroll-go/cmd/root.go +++ b/cmd/crunchyroll-go/cmd/root.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "github.com/ByteDream/crunchyroll-go" "github.com/spf13/cobra" "net/http" @@ -21,6 +22,10 @@ var ( var rootCmd = &cobra.Command{ Use: "crunchyroll", Short: "Download crunchyroll videos with ease", + + SilenceErrors: true, + SilenceUsage: true, + PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) { if verboseFlag { out = newLogger(true, true, true) @@ -46,14 +51,17 @@ func Execute() { defer func() { if r := recover(); r != nil { if out.IsDev() { - out.Err(string(debug.Stack())) + out.Err("%v: %s", r, debug.Stack()) } else { out.Err("Unexpected error: %v", r) } os.Exit(1) } }() - if rootCmd.Execute() != nil { + if err := rootCmd.Execute(); err != nil { + if err != context.Canceled { + out.Exit("An error occurred: %v", err) + } os.Exit(1) } } From aa1d72868b457e83adbe1c45557c11107577c81e Mon Sep 17 00:00:00 2001 From: bytedream Date: Tue, 15 Mar 2022 13:05:57 +0100 Subject: [PATCH 57/82] Fixed debug stop message --- cmd/crunchyroll-go/cmd/logger.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/crunchyroll-go/cmd/logger.go b/cmd/crunchyroll-go/cmd/logger.go index 22e4582..45ddd15 100644 --- a/cmd/crunchyroll-go/cmd/logger.go +++ b/cmd/crunchyroll-go/cmd/logger.go @@ -117,7 +117,7 @@ func (l *logger) SetProgress(format string, v ...interface{}) { fmt.Printf("\r" + strings.Repeat(" ", 2+len(message))) fmt.Printf("\r➞ %s\n", p.message) } else { - l.Debug(message) + l.Debug(p.message) } l.progress = nil From 3ee5f7a538b1f51c4cc051586148737890b3b64a Mon Sep 17 00:00:00 2001 From: bytedream Date: Tue, 15 Mar 2022 13:41:39 +0100 Subject: [PATCH 58/82] Added ids to ...FromID functions (how did this work before??) --- episode.go | 1 + movie_listing.go | 1 + season.go | 1 + video.go | 2 ++ 4 files changed, 5 insertions(+) diff --git a/episode.go b/episode.go index 6ab330b..d9814cb 100644 --- a/episode.go +++ b/episode.go @@ -94,6 +94,7 @@ func EpisodeFromID(crunchy *Crunchyroll, id string) (*Episode, error) { episode := &Episode{ crunchy: crunchy, + ID: id, } if err := decodeMapToStruct(jsonBody, episode); err != nil { return nil, err diff --git a/movie_listing.go b/movie_listing.go index cab4e9d..9646c18 100644 --- a/movie_listing.go +++ b/movie_listing.go @@ -56,6 +56,7 @@ func MovieListingFromID(crunchy *Crunchyroll, id string) (*MovieListing, error) movieListing := &MovieListing{ crunchy: crunchy, + ID: id, } if err = decodeMapToStruct(jsonBody, movieListing); err != nil { return nil, err diff --git a/season.go b/season.go index 438236d..c9a17da 100644 --- a/season.go +++ b/season.go @@ -61,6 +61,7 @@ func SeasonFromID(crunchy *Crunchyroll, id string) (*Season, error) { season := &Season{ crunchy: crunchy, + ID: id, } if err := decodeMapToStruct(jsonBody, season); err != nil { return nil, err diff --git a/video.go b/video.go index 1b83c90..f543df1 100644 --- a/video.go +++ b/video.go @@ -86,6 +86,7 @@ func MovieFromID(crunchy *Crunchyroll, id string) (*Movie, error) { movieListing := &Movie{ crunchy: crunchy, } + movieListing.ID = id if err = decodeMapToStruct(jsonBody, movieListing); err != nil { return nil, err } @@ -182,6 +183,7 @@ func SeriesFromID(crunchy *Crunchyroll, id string) (*Series, error) { series := &Series{ crunchy: crunchy, } + series.ID = id if err = decodeMapToStruct(jsonBody, series); err != nil { return nil, err } From 1d6b930d9a2cf82eb14c60a53a48654529d7a430 Mon Sep 17 00:00:00 2001 From: bytedream Date: Tue, 15 Mar 2022 18:35:05 +0100 Subject: [PATCH 59/82] Added pattern to filter episodes from (series) urls --- cmd/crunchyroll-go/cmd/utils.go | 77 ++++++++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 6 deletions(-) diff --git a/cmd/crunchyroll-go/cmd/utils.go b/cmd/crunchyroll-go/cmd/utils.go index 208b1d5..d244e58 100644 --- a/cmd/crunchyroll-go/cmd/utils.go +++ b/cmd/crunchyroll-go/cmd/utils.go @@ -11,6 +11,7 @@ import ( "os/user" "path/filepath" "reflect" + "regexp" "runtime" "strconv" "strings" @@ -23,6 +24,8 @@ var ( invalidLinuxChars = []string{"/"} ) +var urlFilter = regexp.MustCompile(`(S(\d+))?(E(\d+))?((-)(S(\d+))?(E(\d+))?)?(,|$)`) + // systemLocale receives the system locale // https://stackoverflow.com/questions/51829386/golang-get-system-language/51831590#51831590 func systemLocale() crunchyroll.LOCALE { @@ -188,24 +191,86 @@ func generateFilename(name, directory string) string { return filename } -func extractEpisodes(url string, locales ...crunchyroll.LOCALE) [][]*crunchyroll.Episode { +func extractEpisodes(url string, locales ...crunchyroll.LOCALE) ([][]*crunchyroll.Episode, error) { + var matches [][]string + + lastOpen := strings.LastIndex(url, "[") + if strings.HasSuffix(url, "]") && lastOpen != -1 && lastOpen < len(url) { + matches = urlFilter.FindAllStringSubmatch(url[lastOpen+1:len(url)-1], -1) + + var all string + for _, match := range matches { + all += match[0] + } + if all != url[lastOpen+1:len(url)-1] { + return nil, fmt.Errorf("invalid episode filter") + } + url = url[:lastOpen] + } + final := make([][]*crunchyroll.Episode, len(locales)) episodes, err := crunchy.ExtractEpisodesFromUrl(url, locales...) if err != nil { - out.Err("Failed to get episodes: %v", err) - os.Exit(1) + return nil, fmt.Errorf("failed to get episodes: %v", err) + } + + if len(episodes) == 0 { + return nil, fmt.Errorf("no episodes found") + } + + if matches != nil { + for _, match := range matches { + fromSeason, fromEpisode, toSeason, toEpisode := -1, -1, -1, -1 + if match[2] != "" { + fromSeason, _ = strconv.Atoi(match[2]) + } + if match[4] != "" { + fromEpisode, _ = strconv.Atoi(match[4]) + } + if match[8] != "" { + toSeason, _ = strconv.Atoi(match[8]) + } + if match[10] != "" { + toEpisode, _ = strconv.Atoi(match[10]) + } + + if match[6] != "-" { + toSeason = fromSeason + toEpisode = fromEpisode + } + + tmpEps := make([]*crunchyroll.Episode, 0) + for _, episode := range episodes { + if fromSeason != -1 && episode.SeasonNumber < fromSeason { + continue + } else if toSeason != -1 && episode.SeasonNumber > toSeason { + continue + } else if fromEpisode != -1 && episode.EpisodeNumber < fromEpisode { + continue + } else if toEpisode != -1 && episode.EpisodeNumber > toEpisode { + continue + } else { + tmpEps = append(tmpEps, episode) + } + } + + if len(tmpEps) == 0 { + return nil, fmt.Errorf("no episodes are matching the given filter") + } + + episodes = tmpEps + } } localeSorted, err := utils.SortEpisodesByAudio(episodes) if err != nil { - out.Err("Failed to get audio locale: %v", err) - os.Exit(1) + return nil, fmt.Errorf("failed to get audio locale: %v", err) } for i, locale := range locales { final[i] = append(final[i], localeSorted[locale]...) } - return final + return final, nil } type formatInformation struct { From 52a8f81356468be1233047decd9617e4b99bbcbe Mon Sep 17 00:00:00 2001 From: bytedream Date: Tue, 15 Mar 2022 22:02:06 +0100 Subject: [PATCH 60/82] Added wiki notice --- cmd/crunchyroll-go/cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/crunchyroll-go/cmd/root.go b/cmd/crunchyroll-go/cmd/root.go index 2fe3a61..3442597 100644 --- a/cmd/crunchyroll-go/cmd/root.go +++ b/cmd/crunchyroll-go/cmd/root.go @@ -21,7 +21,7 @@ var ( var rootCmd = &cobra.Command{ Use: "crunchyroll", - Short: "Download crunchyroll videos with ease", + Short: "Download crunchyroll videos with ease. See the wiki for details about the cli and library: https://github.com/ByteDream/crunchyroll-go/wiki", SilenceErrors: true, SilenceUsage: true, From 16fcf08f3474e7bacd16091a30d3b72807850223 Mon Sep 17 00:00:00 2001 From: bytedream Date: Tue, 15 Mar 2022 22:13:28 +0100 Subject: [PATCH 61/82] Replaced ioutil package with io --- cmd/crunchyroll-go/cmd/login.go | 5 ++--- crunchyroll.go | 8 ++++---- downloader.go | 5 ++--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/cmd/crunchyroll-go/cmd/login.go b/cmd/crunchyroll-go/cmd/login.go index fa55020..3bd4004 100644 --- a/cmd/crunchyroll-go/cmd/login.go +++ b/cmd/crunchyroll-go/cmd/login.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/ByteDream/crunchyroll-go" "github.com/spf13/cobra" - "io/ioutil" "os" "os/user" "path/filepath" @@ -46,7 +45,7 @@ func loginCredentials(user, password string) error { os.Exit(1) } - return ioutil.WriteFile(loginStorePath(), []byte(fmt.Sprintf("%s\n%s", user, password)), 0600) + return os.WriteFile(loginStorePath(), []byte(fmt.Sprintf("%s\n%s", user, password)), 0600) } func loginSessionID(sessionID string) error { @@ -56,7 +55,7 @@ func loginSessionID(sessionID string) error { os.Exit(1) } - return ioutil.WriteFile(loginStorePath(), []byte(sessionID), 0600) + return os.WriteFile(loginStorePath(), []byte(sessionID), 0600) } func loginStorePath() string { diff --git a/crunchyroll.go b/crunchyroll.go index 8c456a4..f59fb30 100644 --- a/crunchyroll.go +++ b/crunchyroll.go @@ -5,7 +5,7 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "net/http" "net/url" "regexp" @@ -68,7 +68,7 @@ func LoginWithCredentials(user string, password string, locale LOCALE, client *h defer sessResp.Body.Close() var data map[string]interface{} - body, _ := ioutil.ReadAll(sessResp.Body) + body, _ := io.ReadAll(sessResp.Body) json.Unmarshal(body, &data) sessionID := data["data"].(map[string]interface{})["session_id"].(string) @@ -211,7 +211,7 @@ func (c *Crunchyroll) request(endpoint string) (*http.Response, error) { resp, err := c.Client.Do(req) if err == nil { - bodyAsBytes, _ := ioutil.ReadAll(resp.Body) + bodyAsBytes, _ := io.ReadAll(resp.Body) defer resp.Body.Close() if resp.StatusCode == http.StatusUnauthorized { return nil, &AccessError{ @@ -231,7 +231,7 @@ func (c *Crunchyroll) request(endpoint string) (*http.Response, error) { } } } - resp.Body = ioutil.NopCloser(bytes.NewBuffer(bodyAsBytes)) + resp.Body = io.NopCloser(bytes.NewBuffer(bodyAsBytes)) } return resp, err } diff --git a/downloader.go b/downloader.go index 48460c3..8c0278b 100644 --- a/downloader.go +++ b/downloader.go @@ -8,7 +8,6 @@ import ( "fmt" "github.com/grafov/m3u8" "io" - "io/ioutil" "math" "net/http" "os" @@ -324,7 +323,7 @@ func getCrypt(format *Format, segment *m3u8.MediaSegment) (block cipher.Block, i return nil, nil, err } defer resp.Body.Close() - key, err := ioutil.ReadAll(resp.Body) + key, err := io.ReadAll(resp.Body) block, err = aes.NewCipher(key) if err != nil { @@ -372,7 +371,7 @@ func (d Downloader) decryptSegment(client *http.Client, segment *m3u8.MediaSegme } defer resp.Body.Close() - raw, err := ioutil.ReadAll(resp.Body) + raw, err := io.ReadAll(resp.Body) if err != nil { return nil, err } From 73591cd75c35ccb3419d94cf91c96c0c9c9c8238 Mon Sep 17 00:00:00 2001 From: bytedream Date: Tue, 15 Mar 2022 22:50:48 +0100 Subject: [PATCH 62/82] Fixed episode filter not working properly --- cmd/crunchyroll-go/cmd/utils.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/crunchyroll-go/cmd/utils.go b/cmd/crunchyroll-go/cmd/utils.go index d244e58..9a61280 100644 --- a/cmd/crunchyroll-go/cmd/utils.go +++ b/cmd/crunchyroll-go/cmd/utils.go @@ -241,13 +241,13 @@ func extractEpisodes(url string, locales ...crunchyroll.LOCALE) ([][]*crunchyrol tmpEps := make([]*crunchyroll.Episode, 0) for _, episode := range episodes { - if fromSeason != -1 && episode.SeasonNumber < fromSeason { + if fromSeason != -1 && (episode.SeasonNumber < fromSeason || (fromEpisode != -1 && episode.EpisodeNumber < fromEpisode)) { continue - } else if toSeason != -1 && episode.SeasonNumber > toSeason { + } else if fromSeason == -1 && fromEpisode != -1 && fromEpisode < episode.EpisodeNumber { continue - } else if fromEpisode != -1 && episode.EpisodeNumber < fromEpisode { + } else if toSeason != -1 && (episode.SeasonNumber > toSeason || (toEpisode != -1 && episode.EpisodeNumber > toEpisode)) { continue - } else if toEpisode != -1 && episode.EpisodeNumber > toEpisode { + } else if toSeason == -1 && toEpisode != -1 && episode.EpisodeNumber > toEpisode { continue } else { tmpEps = append(tmpEps, episode) From c822beca53bb54c2a28eb9c9a4770c8f75f69564 Mon Sep 17 00:00:00 2001 From: bytedream Date: Wed, 16 Mar 2022 17:25:35 +0100 Subject: [PATCH 63/82] Fixed episode filter episode number checking not working properly --- cmd/crunchyroll-go/cmd/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/crunchyroll-go/cmd/utils.go b/cmd/crunchyroll-go/cmd/utils.go index 9a61280..d1d010d 100644 --- a/cmd/crunchyroll-go/cmd/utils.go +++ b/cmd/crunchyroll-go/cmd/utils.go @@ -243,7 +243,7 @@ func extractEpisodes(url string, locales ...crunchyroll.LOCALE) ([][]*crunchyrol for _, episode := range episodes { if fromSeason != -1 && (episode.SeasonNumber < fromSeason || (fromEpisode != -1 && episode.EpisodeNumber < fromEpisode)) { continue - } else if fromSeason == -1 && fromEpisode != -1 && fromEpisode < episode.EpisodeNumber { + } else if fromSeason == -1 && fromEpisode != -1 && episode.EpisodeNumber < fromEpisode { continue } else if toSeason != -1 && (episode.SeasonNumber > toSeason || (toEpisode != -1 && episode.EpisodeNumber > toEpisode)) { continue From cddb6ce03352bbfec5a6cfd15a3dcd39109ad0b0 Mon Sep 17 00:00:00 2001 From: bytedream Date: Wed, 16 Mar 2022 20:48:40 +0100 Subject: [PATCH 64/82] Fixed not working invalid character replacing in file name --- cmd/crunchyroll-go/cmd/utils.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/crunchyroll-go/cmd/utils.go b/cmd/crunchyroll-go/cmd/utils.go index d1d010d..f5ed9c1 100644 --- a/cmd/crunchyroll-go/cmd/utils.go +++ b/cmd/crunchyroll-go/cmd/utils.go @@ -20,8 +20,9 @@ import ( ) var ( - invalidWindowsChars = []string{"<", ">", ":", "\"", "/", "|", "\\", "?", "*"} - invalidLinuxChars = []string{"/"} + // ahh i love windows :))) + invalidWindowsChars = []string{"<", ">", ":", "\"", "/", "|", "\\", "?", "*"} + invalidNotWindowsChars = []string{"/"} ) var urlFilter = regexp.MustCompile(`(S(\d+))?(E(\d+))?((-)(S(\d+))?(E(\d+))?)?(,|$)`) @@ -168,13 +169,13 @@ func terminalWidth() int { func generateFilename(name, directory string) string { if runtime.GOOS != "windows" { - for _, char := range invalidLinuxChars { - strings.ReplaceAll(name, char, "") + for _, char := range invalidNotWindowsChars { + name = strings.ReplaceAll(name, char, "") } out.Debug("Replaced invalid characters (not windows)") } else { for _, char := range invalidWindowsChars { - strings.ReplaceAll(name, char, "") + name = strings.ReplaceAll(name, char, "") } out.Debug("Replaced invalid characters (windows)") } From 8525b913b80641f7a29448782694e8eff9a8e7ba Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 20 Mar 2022 13:07:09 +0100 Subject: [PATCH 65/82] More FFmpeg optimizations when writer is a file --- downloader.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/downloader.go b/downloader.go index 8c0278b..d455f89 100644 --- a/downloader.go +++ b/downloader.go @@ -160,9 +160,9 @@ func (d Downloader) mergeSegmentsFFmpeg(files []string) error { } var tmpfile string - if _, ok := d.Writer.(*io.PipeWriter); ok { + if _, ok := d.Writer.(*io.PipeWriter); !ok { if file, ok := d.Writer.(*os.File); ok { - tmpfile = filepath.Base(file.Name()) + tmpfile = file.Name() } } if filepath.Ext(tmpfile) == "" { @@ -198,12 +198,14 @@ func (d Downloader) mergeSegmentsFFmpeg(files []string) error { return err } } - file, err := os.Open(tmpfile) - if err != nil { - return err + if f, ok := d.Writer.(*os.File); !ok || f.Name() != tmpfile { + file, err := os.Open(tmpfile) + if err != nil { + return err + } + defer file.Close() + _, err = io.Copy(d.Writer, file) } - defer file.Close() - _, err = io.Copy(d.Writer, file) return err } From 668c6737506dbfbe1a7713de5889f263b1ca4f9a Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 20 Mar 2022 16:14:12 +0100 Subject: [PATCH 66/82] Added season name for output formatting --- cmd/crunchyroll-go/cmd/utils.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/crunchyroll-go/cmd/utils.go b/cmd/crunchyroll-go/cmd/utils.go index f5ed9c1..186f152 100644 --- a/cmd/crunchyroll-go/cmd/utils.go +++ b/cmd/crunchyroll-go/cmd/utils.go @@ -283,6 +283,7 @@ type formatInformation struct { Title string `json:"title"` SeriesName string `json:"series_name"` + SeasonName string `json:"season_name"` SeasonNumber int `json:"season_number"` EpisodeNumber int `json:"episode_number"` Resolution string `json:"resolution"` From dafa2f4d70ac832435d53e657ce852e1fe5a9498 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 20 Mar 2022 17:08:39 +0100 Subject: [PATCH 67/82] Just some style changes --- cmd/crunchyroll-go/cmd/login.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cmd/crunchyroll-go/cmd/login.go b/cmd/crunchyroll-go/cmd/login.go index 3bd4004..38ae68e 100644 --- a/cmd/crunchyroll-go/cmd/login.go +++ b/cmd/crunchyroll-go/cmd/login.go @@ -11,9 +11,9 @@ import ( ) var ( - loginSessionIDFlag bool - loginPersistentFlag bool + + loginSessionIDFlag bool ) var loginCmd = &cobra.Command{ @@ -31,9 +31,15 @@ var loginCmd = &cobra.Command{ } func init() { - loginCmd.Flags().BoolVar(&loginSessionIDFlag, "session-id", false, "Use a session id to login instead of username and password") + loginCmd.Flags().BoolVar(&loginPersistentFlag, + "persistent", + false, + "If the given credential should be stored persistent") - loginCmd.Flags().BoolVar(&loginPersistentFlag, "persistent", false, "If the given credential should be stored persistent") + loginCmd.Flags().BoolVar(&loginSessionIDFlag, + "session-id", + false, + "Use a session id to login instead of username and password") rootCmd.AddCommand(loginCmd) } From e9fdd8fd437df2a1e7c66335fb0ba9a16cc2215c Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 20 Mar 2022 17:15:44 +0100 Subject: [PATCH 68/82] Added ability to format directory names too --- cmd/crunchyroll-go/cmd/utils.go | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/cmd/crunchyroll-go/cmd/utils.go b/cmd/crunchyroll-go/cmd/utils.go index 186f152..37f642c 100644 --- a/cmd/crunchyroll-go/cmd/utils.go +++ b/cmd/crunchyroll-go/cmd/utils.go @@ -167,23 +167,7 @@ func terminalWidth() int { return 60 } -func generateFilename(name, directory string) string { - if runtime.GOOS != "windows" { - for _, char := range invalidNotWindowsChars { - name = strings.ReplaceAll(name, char, "") - } - out.Debug("Replaced invalid characters (not windows)") - } else { - for _, char := range invalidWindowsChars { - name = strings.ReplaceAll(name, char, "") - } - out.Debug("Replaced invalid characters (windows)") - } - - if directory != "" { - name = filepath.Join(directory, name) - } - +func generateFilename(name string) string { filename, changed := freeFileName(name) if changed { out.Info("File %s already exists, changing name to %s", name, filename) @@ -292,7 +276,7 @@ type formatInformation struct { Subtitle crunchyroll.LOCALE `json:"subtitle"` } -func (fi formatInformation) Format(source string) string { +func (fi formatInformation) Format(source string, removeInvalidChars bool) string { fields := reflect.TypeOf(fi) values := reflect.ValueOf(fi) @@ -311,6 +295,19 @@ func (fi formatInformation) Format(source string) string { valueAsString = "no " + valueAsString } } + + if removeInvalidChars { + if runtime.GOOS != "windows" { + for _, char := range invalidNotWindowsChars { + valueAsString = strings.ReplaceAll(valueAsString, char, "") + } + } else { + for _, char := range invalidWindowsChars { + valueAsString = strings.ReplaceAll(valueAsString, char, "") + } + } + } + source = strings.ReplaceAll(source, "{"+fields.Field(i).Tag.Get("json")+"}", valueAsString) } From 31e03c1e089646d04e080a162d86cc8dc0260324 Mon Sep 17 00:00:00 2001 From: bytedream Date: Sun, 20 Mar 2022 17:34:13 +0100 Subject: [PATCH 69/82] Dependency updates --- go.mod | 7 +- go.sum | 754 +-------------------------------------------------------- 2 files changed, 3 insertions(+), 758 deletions(-) diff --git a/go.mod b/go.mod index f5c3ea4..d3e701e 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,5 @@ go 1.16 require ( github.com/grafov/m3u8 v0.11.1 - github.com/spf13/cobra v1.3.0 -) - -require ( - github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/cobra v1.4.0 ) diff --git a/go.sum b/go.sum index a1bf13c..34693ca 100644 --- a/go.sum +++ b/go.sum @@ -1,762 +1,12 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= -github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/grafov/m3u8 v0.11.1 h1:igZ7EBIB2IAsPPazKwRKdbhxcoBKO3lO1UY57PZDeNA= github.com/grafov/m3u8 v0.11.1/go.mod h1:nqzOkfBiZJENr52zTVd/Dcl03yzphIMbJqkXGu+u080= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= -github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= -github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= -github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= -github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= -github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= -github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= -github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= -github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0= -github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= +github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= -github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= -github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From 98f5da3bf367a570a582b03f821c48d870e17e5b Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 21 Mar 2022 10:11:33 +0100 Subject: [PATCH 70/82] Reverted and extended generateFilename function --- cmd/crunchyroll-go/cmd/utils.go | 40 ++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/cmd/crunchyroll-go/cmd/utils.go b/cmd/crunchyroll-go/cmd/utils.go index 37f642c..032d066 100644 --- a/cmd/crunchyroll-go/cmd/utils.go +++ b/cmd/crunchyroll-go/cmd/utils.go @@ -21,7 +21,7 @@ import ( var ( // ahh i love windows :))) - invalidWindowsChars = []string{"<", ">", ":", "\"", "/", "|", "\\", "?", "*"} + invalidWindowsChars = []string{"\\", "<", ">", ":", "\"", "/", "|", "?", "*"} invalidNotWindowsChars = []string{"/"} ) @@ -167,7 +167,29 @@ func terminalWidth() int { return 60 } -func generateFilename(name string) string { +func generateFilename(name, directory string) string { + if runtime.GOOS != "windows" { + for _, char := range invalidNotWindowsChars { + name = strings.ReplaceAll(name, char, "") + } + out.Debug("Replaced invalid characters (not windows)") + } else { + for _, char := range invalidWindowsChars { + name = strings.ReplaceAll(name, char, "") + } + // this needs only to be done on windows lol :) + if directory != "" { + for _, char := range invalidWindowsChars[1:] { + directory = strings.ReplaceAll(directory, char, "") + } + } + out.Debug("Replaced invalid characters (windows)") + } + + if directory != "" { + name = filepath.Join(directory, name) + } + filename, changed := freeFileName(name) if changed { out.Info("File %s already exists, changing name to %s", name, filename) @@ -276,7 +298,7 @@ type formatInformation struct { Subtitle crunchyroll.LOCALE `json:"subtitle"` } -func (fi formatInformation) Format(source string, removeInvalidChars bool) string { +func (fi formatInformation) Format(source string) string { fields := reflect.TypeOf(fi) values := reflect.ValueOf(fi) @@ -296,18 +318,6 @@ func (fi formatInformation) Format(source string, removeInvalidChars bool) strin } } - if removeInvalidChars { - if runtime.GOOS != "windows" { - for _, char := range invalidNotWindowsChars { - valueAsString = strings.ReplaceAll(valueAsString, char, "") - } - } else { - for _, char := range invalidWindowsChars { - valueAsString = strings.ReplaceAll(valueAsString, char, "") - } - } - } - source = strings.ReplaceAll(source, "{"+fields.Field(i).Tag.Get("json")+"}", valueAsString) } From 588d402c631d0f62a91a4fe628d4778a6fe347c5 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 21 Mar 2022 14:30:04 +0100 Subject: [PATCH 71/82] Just some style corrections --- downloader.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/downloader.go b/downloader.go index d455f89..48c0295 100644 --- a/downloader.go +++ b/downloader.go @@ -89,7 +89,7 @@ func (d Downloader) download(format *Format) error { } if _, err := os.Stat(d.TempDir); os.IsNotExist(err) { - if err := os.Mkdir(d.TempDir, 0700); err != nil { + if err = os.Mkdir(d.TempDir, 0700); err != nil { return err } } @@ -267,12 +267,14 @@ func (d Downloader) downloadSegments(format *Format) ([]string, error) { break } if k == 2 { + file.Close() cancel() return } select { case <-d.Context.Done(): case <-ctx.Done(): + file.Close() return case <-time.After(5 * time.Duration(k) * time.Second): // sleep if an error occurs. very useful because sometimes the connection times out From a6c14cb6c9e62a5518eb07daac5bce8fdbb6cb8e Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 21 Mar 2022 16:40:00 +0100 Subject: [PATCH 72/82] Reworked logger progress --- cmd/crunchyroll-go/cmd/logger.go | 78 ++++++++++++++++---------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/cmd/crunchyroll-go/cmd/logger.go b/cmd/crunchyroll-go/cmd/logger.go index 45ddd15..750ffc9 100644 --- a/cmd/crunchyroll-go/cmd/logger.go +++ b/cmd/crunchyroll-go/cmd/logger.go @@ -1,12 +1,12 @@ package cmd import ( - "context" "fmt" "io" "log" "os" "strings" + "sync" "time" ) @@ -24,6 +24,7 @@ type logger struct { progress chan progress done chan interface{} + lock sync.Mutex } func newLogger(debug, info, err bool) *logger { @@ -87,69 +88,66 @@ func (l *logger) Empty() { func (l *logger) SetProgress(format string, v ...interface{}) { if out.InfoLog.Writer() == io.Discard { return - } - - message := fmt.Sprintf(format, v...) - - if l.progress != nil { - l.progress <- progress{ - message: message, - stop: false, - } + } else if l.devView { + l.Debug(format, v...) return } - l.progress = make(chan progress) - l.done = make(chan interface{}) + initialMessage := fmt.Sprintf(format, v...) + + p := progress{ + message: initialMessage, + } + + l.lock.Lock() + if l.done != nil { + l.progress <- p + return + } else { + l.progress = make(chan progress, 1) + l.progress <- p + l.done = make(chan interface{}) + } go func() { states := []string{"-", "\\", "|", "/"} + var count int for i := 0; ; i++ { - ctx, cancel := context.WithTimeout(context.Background(), 35*time.Millisecond) select { case p := <-l.progress: - cancel() - if p.stop { - if !l.devView { - fmt.Printf("\r" + strings.Repeat(" ", 2+len(message))) - fmt.Printf("\r➞ %s\n", p.message) + fmt.Printf("\r" + strings.Repeat(" ", 2+len(initialMessage))) + if count > 1 { + fmt.Printf("\r↳ %s\n", p.message) } else { - l.Debug(p.message) + fmt.Printf("\r➞ %s\n", p.message) } + if l.done != nil { + l.done <- nil + } l.progress = nil - if count > 0 { - fmt.Printf("↳ %s\n", p.message) - } - - l.done <- nil + l.lock.Unlock() return } else { - if !l.devView { - fmt.Printf("\r↓ %s\n", message) - } else { - l.Debug(message) + if count > 0 { + fmt.Printf("\r↓ %s\n", p.message) } + l.progress = make(chan progress, 1) - l.progress = make(chan progress) count++ - if !l.devView { - fmt.Printf("\r" + strings.Repeat(" ", 2+len(message))) - fmt.Printf("\r➞ %s\n", p.message) - } else { - l.Debug(p.message) - } - message = p.message + fmt.Printf("\r%s %s", states[i/10%4], initialMessage) + l.lock.Unlock() } - case <-ctx.Done(): - if !l.devView && i%10 == 0 { - fmt.Printf("\r%s %s", states[i/10%4], message) + default: + if i%10 == 0 { + fmt.Printf("\r%s %s", states[i/10%4], initialMessage) } + time.Sleep(35 * time.Millisecond) } } }() @@ -160,9 +158,11 @@ func (l *logger) StopProgress(format string, v ...interface{}) { return } + l.lock.Lock() l.progress <- progress{ message: fmt.Sprintf(format, v...), stop: true, } <-l.done + l.done = nil } From ac7904668f212cfadb58fb98248648c8aa775bc5 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 21 Mar 2022 16:55:27 +0100 Subject: [PATCH 73/82] Added context to the crunchyroll struct --- crunchyroll.go | 4 ++++ downloader.go | 3 +-- subtitle.go | 8 +++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/crunchyroll.go b/crunchyroll.go index f59fb30..82cd118 100644 --- a/crunchyroll.go +++ b/crunchyroll.go @@ -2,6 +2,7 @@ package crunchyroll import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -32,6 +33,8 @@ const ( type Crunchyroll struct { // Client is the http.Client to perform all requests over Client *http.Client + // Context can be used to stop requests with Client and is context.Background by default + Context context.Context // Locale specifies in which language all results should be returned / requested Locale LOCALE // SessionID is the crunchyroll session id which was used for authentication @@ -88,6 +91,7 @@ func LoginWithCredentials(user string, password string, locale LOCALE, client *h func LoginWithSessionID(sessionID string, locale LOCALE, client *http.Client) (*Crunchyroll, error) { crunchy := &Crunchyroll{ Client: client, + Context: context.Background(), Locale: locale, SessionID: sessionID, cache: true, diff --git a/downloader.go b/downloader.go index 48c0295..be625b3 100644 --- a/downloader.go +++ b/downloader.go @@ -363,11 +363,10 @@ func (d Downloader) downloadSegment(format *Format, segment *m3u8.MediaSegment, // https://github.com/oopsguy/m3u8/blob/4150e93ec8f4f8718875a02973f5d792648ecb97/tool/crypt.go#L25 func (d Downloader) decryptSegment(client *http.Client, segment *m3u8.MediaSegment, block cipher.Block, iv []byte) ([]byte, error) { - req, err := http.NewRequest(http.MethodGet, segment.URI, nil) + req, err := http.NewRequestWithContext(d.Context, http.MethodGet, segment.URI, nil) if err != nil { return nil, err } - req.WithContext(d.Context) resp, err := client.Do(req) if err != nil { diff --git a/subtitle.go b/subtitle.go index 06ec9aa..6a33e14 100644 --- a/subtitle.go +++ b/subtitle.go @@ -2,6 +2,7 @@ package crunchyroll import ( "io" + "net/http" ) type Subtitle struct { @@ -13,7 +14,12 @@ type Subtitle struct { } func (s Subtitle) Save(writer io.Writer) error { - resp, err := s.crunchy.Client.Get(s.URL) + req, err := http.NewRequestWithContext(s.crunchy.Context, http.MethodGet, s.URL, nil) + if err != nil { + return err + } + + resp, err := s.crunchy.Client.Do(req) if err != nil { return err } From f6f51799d774c2f0089ef9c64412d22be6de6c99 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 21 Mar 2022 17:11:01 +0100 Subject: [PATCH 74/82] Extended checking if error is context cancelling --- cmd/crunchyroll-go/cmd/root.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/crunchyroll-go/cmd/root.go b/cmd/crunchyroll-go/cmd/root.go index 3442597..25c8270 100644 --- a/cmd/crunchyroll-go/cmd/root.go +++ b/cmd/crunchyroll-go/cmd/root.go @@ -7,6 +7,7 @@ import ( "net/http" "os" "runtime/debug" + "strings" ) var ( @@ -59,7 +60,7 @@ func Execute() { } }() if err := rootCmd.Execute(); err != nil { - if err != context.Canceled { + if !strings.HasSuffix(err.Error(), context.Canceled.Error()) { out.Exit("An error occurred: %v", err) } os.Exit(1) From d090a5a1daa781be9aac83950a8318267ccd5317 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 21 Mar 2022 19:36:19 +0100 Subject: [PATCH 75/82] Windows things --- cmd/crunchyroll-go/cmd/logger.go | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/cmd/crunchyroll-go/cmd/logger.go b/cmd/crunchyroll-go/cmd/logger.go index 750ffc9..8c549f6 100644 --- a/cmd/crunchyroll-go/cmd/logger.go +++ b/cmd/crunchyroll-go/cmd/logger.go @@ -5,11 +5,31 @@ import ( "io" "log" "os" + "runtime" "strings" "sync" "time" ) +var prefix, progressDown, progressDownFinish string + +func initPrefixBecauseWindowsSucksBallsHard() { + // dear windows user, please change to a good OS, linux in the best case. + // MICROSHIT DOES NOT GET IT DONE TO SHOW THE SYMBOLS IN THE ELSE CLAUSE + // CORRECTLY. NOT IN THE CMD NOR POWERSHELL. WHY TF, IT IS ONE OF THE MOST + // PROFITABLE COMPANIES ON THIS PLANET AND CANNOT SHOW A PROPER UTF-8 SYMBOL + // IN THEIR OWN PRODUCT WHICH GETS USED MILLION TIMES A DAY + if runtime.GOOS == "windows" { + prefix = "=>" + progressDown = "|" + progressDownFinish = "->" + } else { + prefix = "➞" + progressDown = "↓" + progressDownFinish = "↳" + } +} + type progress struct { message string stop bool @@ -28,7 +48,9 @@ type logger struct { } func newLogger(debug, info, err bool) *logger { - debugLog, infoLog, errLog := log.New(io.Discard, "➞ ", 0), log.New(io.Discard, "➞ ", 0), log.New(io.Discard, "➞ ", 0) + initPrefixBecauseWindowsSucksBallsHard() + + debugLog, infoLog, errLog := log.New(io.Discard, prefix+" ", 0), log.New(io.Discard, prefix+" ", 0), log.New(io.Discard, prefix+" ", 0) if debug { debugLog.SetOutput(os.Stdout) @@ -118,11 +140,11 @@ func (l *logger) SetProgress(format string, v ...interface{}) { select { case p := <-l.progress: if p.stop { - fmt.Printf("\r" + strings.Repeat(" ", 2+len(initialMessage))) + fmt.Printf("\r" + strings.Repeat(" ", len(prefix)+len(initialMessage))) if count > 1 { - fmt.Printf("\r↳ %s\n", p.message) + fmt.Printf("\r%s %s\n", progressDownFinish, p.message) } else { - fmt.Printf("\r➞ %s\n", p.message) + fmt.Printf("\r%s %s\n", prefix, p.message) } if l.done != nil { @@ -134,7 +156,7 @@ func (l *logger) SetProgress(format string, v ...interface{}) { return } else { if count > 0 { - fmt.Printf("\r↓ %s\n", p.message) + fmt.Printf("\r%s %s\n", progressDown, p.message) } l.progress = make(chan progress, 1) From ba8054b611fd1140530888c9c59b3d5c2c280fa1 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 21 Mar 2022 19:36:39 +0100 Subject: [PATCH 76/82] Optimized filename generation --- cmd/crunchyroll-go/cmd/utils.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/cmd/crunchyroll-go/cmd/utils.go b/cmd/crunchyroll-go/cmd/utils.go index 032d066..7af5b3e 100644 --- a/cmd/crunchyroll-go/cmd/utils.go +++ b/cmd/crunchyroll-go/cmd/utils.go @@ -177,20 +177,10 @@ func generateFilename(name, directory string) string { for _, char := range invalidWindowsChars { name = strings.ReplaceAll(name, char, "") } - // this needs only to be done on windows lol :) - if directory != "" { - for _, char := range invalidWindowsChars[1:] { - directory = strings.ReplaceAll(directory, char, "") - } - } out.Debug("Replaced invalid characters (windows)") } - if directory != "" { - name = filepath.Join(directory, name) - } - - filename, changed := freeFileName(name) + filename, changed := freeFileName(filepath.Join(directory, name)) if changed { out.Info("File %s already exists, changing name to %s", name, filename) } @@ -318,6 +308,18 @@ func (fi formatInformation) Format(source string) string { } } + if runtime.GOOS != "windows" { + for _, char := range invalidNotWindowsChars { + valueAsString = strings.ReplaceAll(valueAsString, char, "") + } + out.Debug("Replaced invalid characters (not windows)") + } else { + for _, char := range invalidWindowsChars { + valueAsString = strings.ReplaceAll(valueAsString, char, "") + } + out.Debug("Replaced invalid characters (windows)") + } + source = strings.ReplaceAll(source, "{"+fields.Field(i).Tag.Get("json")+"}", valueAsString) } From 83c9ae4927877a1cbdf71f142a0fcd5365930979 Mon Sep 17 00:00:00 2001 From: bytedream Date: Mon, 21 Mar 2022 20:45:38 +0100 Subject: [PATCH 77/82] Made verbose output optional when receiving system locale --- cmd/crunchyroll-go/cmd/login.go | 4 ++-- cmd/crunchyroll-go/cmd/utils.go | 18 ++++++++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/cmd/crunchyroll-go/cmd/login.go b/cmd/crunchyroll-go/cmd/login.go index 38ae68e..1303516 100644 --- a/cmd/crunchyroll-go/cmd/login.go +++ b/cmd/crunchyroll-go/cmd/login.go @@ -46,7 +46,7 @@ func init() { func loginCredentials(user, password string) error { out.Debug("Logging in via credentials") - if _, err := crunchyroll.LoginWithCredentials(user, password, systemLocale(), client); err != nil { + if _, err := crunchyroll.LoginWithCredentials(user, password, systemLocale(false), client); err != nil { out.Err(err.Error()) os.Exit(1) } @@ -56,7 +56,7 @@ func loginCredentials(user, password string) error { func loginSessionID(sessionID string) error { out.Debug("Logging in via session id") - if _, err := crunchyroll.LoginWithSessionID(sessionID, systemLocale(), client); err != nil { + if _, err := crunchyroll.LoginWithSessionID(sessionID, systemLocale(false), client); err != nil { out.Err(err.Error()) os.Exit(1) } diff --git a/cmd/crunchyroll-go/cmd/utils.go b/cmd/crunchyroll-go/cmd/utils.go index 7af5b3e..7008cf7 100644 --- a/cmd/crunchyroll-go/cmd/utils.go +++ b/cmd/crunchyroll-go/cmd/utils.go @@ -29,14 +29,16 @@ var urlFilter = regexp.MustCompile(`(S(\d+))?(E(\d+))?((-)(S(\d+))?(E(\d+))?)?(, // systemLocale receives the system locale // https://stackoverflow.com/questions/51829386/golang-get-system-language/51831590#51831590 -func systemLocale() crunchyroll.LOCALE { +func systemLocale(verbose bool) crunchyroll.LOCALE { if runtime.GOOS != "windows" { if lang, ok := os.LookupEnv("LANG"); ok { prefix := strings.Split(lang, "_")[0] suffix := strings.Split(strings.Split(lang, ".")[0], "_")[1] l := crunchyroll.LOCALE(fmt.Sprintf("%s-%s", prefix, suffix)) if !utils.ValidateLocale(l) { - out.Err("%s is not a supported locale, using %s as fallback", l, crunchyroll.US) + if verbose { + out.Err("%s is not a supported locale, using %s as fallback", l, crunchyroll.US) + } l = crunchyroll.US } return l @@ -46,13 +48,17 @@ func systemLocale() crunchyroll.LOCALE { if output, err := cmd.Output(); err == nil { l := crunchyroll.LOCALE(strings.Trim(string(output), "\r\n")) if !utils.ValidateLocale(l) { - out.Err("%s is not a supported locale, using %s as fallback", l, crunchyroll.US) + if verbose { + out.Err("%s is not a supported locale, using %s as fallback", l, crunchyroll.US) + } l = crunchyroll.US } return l } } - out.Err("Failed to get locale, using %s", crunchyroll.US) + if verbose { + out.Err("Failed to get locale, using %s", crunchyroll.US) + } return crunchyroll.US } @@ -130,13 +136,13 @@ func loadCrunchy() { split := strings.SplitN(string(body), "\n", 2) if len(split) == 1 || split[1] == "" { - if crunchy, err = crunchyroll.LoginWithSessionID(split[0], systemLocale(), client); err != nil { + if crunchy, err = crunchyroll.LoginWithSessionID(split[0], systemLocale(true), client); err != nil { out.StopProgress(err.Error()) os.Exit(1) } out.Debug("Logged in with session id %s. BLANK THIS LINE OUT IF YOU'RE ASKED TO POST THE DEBUG OUTPUT SOMEWHERE", split[0]) } else { - if crunchy, err = crunchyroll.LoginWithCredentials(split[0], split[1], systemLocale(), client); err != nil { + if crunchy, err = crunchyroll.LoginWithCredentials(split[0], split[1], systemLocale(true), client); err != nil { out.StopProgress(err.Error()) os.Exit(1) } From c1b3cd7b0a4f0c7fd9484d863a28054926f136b4 Mon Sep 17 00:00:00 2001 From: bytedream Date: Tue, 22 Mar 2022 13:10:42 +0100 Subject: [PATCH 78/82] Fixed logger softlock --- cmd/crunchyroll-go/cmd/logger.go | 3 +++ cmd/crunchyroll-go/cmd/utils.go | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/crunchyroll-go/cmd/logger.go b/cmd/crunchyroll-go/cmd/logger.go index 8c549f6..83bc214 100644 --- a/cmd/crunchyroll-go/cmd/logger.go +++ b/cmd/crunchyroll-go/cmd/logger.go @@ -178,6 +178,9 @@ func (l *logger) SetProgress(format string, v ...interface{}) { func (l *logger) StopProgress(format string, v ...interface{}) { if out.InfoLog.Writer() == io.Discard { return + } else if l.devView { + l.Debug(format, v...) + return } l.lock.Lock() diff --git a/cmd/crunchyroll-go/cmd/utils.go b/cmd/crunchyroll-go/cmd/utils.go index 7008cf7..398b457 100644 --- a/cmd/crunchyroll-go/cmd/utils.go +++ b/cmd/crunchyroll-go/cmd/utils.go @@ -127,10 +127,10 @@ func loadCrunchy() { break } if body == nil { - out.Err("To use this command, login first. Type `%s login -h` to get help", os.Args[0]) + out.StopProgress("To use this command, login first. Type `%s login -h` to get help", os.Args[0]) os.Exit(1) } else if err != nil { - out.Err("Failed to read login information: %v", err) + out.StopProgress("Failed to read login information: %v", err) os.Exit(1) } From 893be7dd5306c594ba0ec9cb82b0b212959eff66 Mon Sep 17 00:00:00 2001 From: bytedream Date: Tue, 22 Mar 2022 19:22:42 +0100 Subject: [PATCH 79/82] Made free filename silent by default and changed renaming behavior --- cmd/crunchyroll-go/cmd/utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/crunchyroll-go/cmd/utils.go b/cmd/crunchyroll-go/cmd/utils.go index 398b457..bf70564 100644 --- a/cmd/crunchyroll-go/cmd/utils.go +++ b/cmd/crunchyroll-go/cmd/utils.go @@ -102,7 +102,7 @@ func freeFileName(filename string) (string, bool) { if _, stat := os.Stat(filename); stat != nil && !os.IsExist(stat) { break } - filename = fmt.Sprintf("%s (%d)%s", base, j, ext) + filename = fmt.Sprintf("%s (%d)%s", base, j+1, ext) } return filename, j != 0 } @@ -188,7 +188,7 @@ func generateFilename(name, directory string) string { filename, changed := freeFileName(filepath.Join(directory, name)) if changed { - out.Info("File %s already exists, changing name to %s", name, filename) + out.Debug("File `%s` already exists, changing name to `%s`", filepath.Base(name), filepath.Base(filename)) } return filename From f65cef84ca65917720ff3aff6f59df7eba7dd0c1 Mon Sep 17 00:00:00 2001 From: bytedream Date: Tue, 22 Mar 2022 19:39:33 +0100 Subject: [PATCH 80/82] Rewrote complete download command --- cmd/crunchyroll-go/cmd/download.go | 974 ++++++++--------------------- 1 file changed, 266 insertions(+), 708 deletions(-) diff --git a/cmd/crunchyroll-go/cmd/download.go b/cmd/crunchyroll-go/cmd/download.go index 0465f78..a3d78a8 100644 --- a/cmd/crunchyroll-go/cmd/download.go +++ b/cmd/crunchyroll-go/cmd/download.go @@ -1,40 +1,31 @@ package cmd import ( - "encoding/json" + "context" "fmt" "github.com/ByteDream/crunchyroll-go" "github.com/ByteDream/crunchyroll-go/utils" "github.com/grafov/m3u8" "github.com/spf13/cobra" "os" - "os/exec" "os/signal" - "path" "path/filepath" - "reflect" - "regexp" "runtime" "sort" "strconv" "strings" - "syscall" ) var ( - audioFlag string - subtitleFlag string - noHardsubFlag bool - onlySubFlag bool + downloadAudioFlag string + downloadSubtitleFlag string - directoryFlag string - outputFlag string + downloadDirectoryFlag string + downloadOutputFlag string - resolutionFlag string + downloadResolutionFlag string - alternativeProgressFlag bool - - goroutinesFlag int + downloadGoroutinesFlag int ) var getCmd = &cobra.Command{ @@ -42,721 +33,288 @@ var getCmd = &cobra.Command{ Short: "Download a video", Args: cobra.MinimumNArgs(1), - Run: func(cmd *cobra.Command, args []string) { + PreRunE: func(cmd *cobra.Command, args []string) error { + out.Debug("Validating arguments") + + if filepath.Ext(downloadOutputFlag) != ".ts" { + if !hasFFmpeg() { + return fmt.Errorf("the file ending for the output file (%s) is not `.ts`. "+ + "Install ffmpeg (https://ffmpeg.org/download.html) to use other media file endings (e.g. `.mp4`)", downloadOutputFlag) + } else { + out.Debug("Custom file ending '%s' (ffmpeg is installed)", filepath.Ext(downloadOutputFlag)) + } + } + + if !utils.ValidateLocale(crunchyroll.LOCALE(downloadAudioFlag)) { + return fmt.Errorf("%s is not a valid audio locale. Choose from: %s", downloadAudioFlag, strings.Join(allLocalesAsStrings(), ", ")) + } else if downloadSubtitleFlag != "" && !utils.ValidateLocale(crunchyroll.LOCALE(downloadSubtitleFlag)) { + return fmt.Errorf("%s is not a valid subtitle locale. Choose from: %s", downloadSubtitleFlag, strings.Join(allLocalesAsStrings(), ", ")) + } + out.Debug("Locales: audio: %s / subtitle: %s", downloadAudioFlag, downloadSubtitleFlag) + + switch downloadResolutionFlag { + case "1080p", "720p", "480p", "360p", "240p": + intRes, _ := strconv.ParseFloat(strings.TrimSuffix(downloadResolutionFlag, "p"), 84) + downloadResolutionFlag = fmt.Sprintf("%dx%s", int(intRes*(16/9)), strings.TrimSuffix(downloadResolutionFlag, "p")) + case "1920x1080", "1280x720", "640x480", "480x360", "428x240", "best", "worst": + default: + return fmt.Errorf("'%s' is not a valid resolution", downloadResolutionFlag) + } + out.Debug("Using resolution '%s'", downloadResolutionFlag) + + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { loadCrunchy() - sig := make(chan os.Signal) - signal.Notify(sig, os.Interrupt, syscall.SIGTERM) - go func() { - <-sig - if cleanupPath != "" { - os.RemoveAll(cleanupPath) - } - os.Exit(1) - }() - - download(args) + return download(args) }, } -var ( - invalidWindowsChars = []string{"<", ">", ":", "\"", "/", "|", "\\", "?", "*"} - invalidLinuxChars = []string{"/"} -) - -var cleanupPath string - func init() { - rootCmd.AddCommand(getCmd) - getCmd.Flags().StringVarP(&audioFlag, "audio", "a", "", "The locale of the audio. Available locales: "+strings.Join(allLocalesAsStrings(), ", ")) - getCmd.Flags().StringVarP(&subtitleFlag, "subtitle", "s", "", "The locale of the subtitle. Available locales: "+strings.Join(allLocalesAsStrings(), ", ")) - getCmd.Flags().BoolVar(&noHardsubFlag, "no-hardsub", false, "Same as '-s', but the subtitles are not stored in the video itself, but in a separate file") - getCmd.Flags().BoolVar(&onlySubFlag, "only-sub", false, "Downloads only the subtitles without the corresponding video") + getCmd.Flags().StringVarP(&downloadAudioFlag, "audio", + "a", + string(systemLocale(false)), + "The locale of the audio. Available locales: "+strings.Join(allLocalesAsStrings(), ", ")) + getCmd.Flags().StringVarP(&downloadSubtitleFlag, + "subtitle", + "s", + "", + "The locale of the subtitle. Available locales: "+strings.Join(allLocalesAsStrings(), ", ")) cwd, _ := os.Getwd() - getCmd.Flags().StringVarP(&directoryFlag, "directory", "d", cwd, "The directory to download the file to") - getCmd.Flags().StringVarP(&outputFlag, "output", "o", "{title}.ts", "Name of the output file\n"+ - "If you use the following things in the name, the will get replaced\n"+ - "\t{title} » Title of the video\n"+ - "\t{series_name} » Name of the series\n"+ - "\t{season_title} » Title of the season\n"+ - "\t{season_number} » Number of the season\n"+ - "\t{episode_number} » Number of the episode\n"+ - "\t{resolution} » Resolution of the video\n"+ - "\t{fps} » Frame Rate of the video\n"+ - "\t{audio} » Audio locale of the video\n"+ - "\t{subtitle} » Subtitle locale of the video\n") + getCmd.Flags().StringVarP(&downloadDirectoryFlag, + "directory", + "d", + cwd, + "The directory to download the file(s) into") + getCmd.Flags().StringVarP(&downloadOutputFlag, + "output", + "o", + "{title}.ts", + "Name of the output file. "+ + "If you use the following things in the name, the will get replaced:\n"+ + "\t{title} » Title of the video\n"+ + "\t{series_name} » Name of the series\n"+ + "\t{season_name} » Name of the season\n"+ + "\t{season_number} » Number of the season\n"+ + "\t{episode_number} » Number of the episode\n"+ + "\t{resolution} » Resolution of the video\n"+ + "\t{fps} » Frame Rate of the video\n"+ + "\t{audio} » Audio locale of the video\n"+ + "\t{subtitle} » Subtitle locale of the video") - getCmd.Flags().StringVarP(&resolutionFlag, "resolution", "r", "best", "The video resolution. Can either be specified via the pixels, the abbreviation for pixels, or 'common-use' words\n"+ - "\tAvailable pixels: 1920x1080, 1280x720, 640x480, 480x360, 426x240\n"+ - "\tAvailable abbreviations: 1080p, 720p, 480p, 360p, 240p\n"+ - "\tAvailable common-use words: best (best available resolution), worst (worst available resolution)\n") + getCmd.Flags().StringVarP(&downloadResolutionFlag, + "resolution", + "r", + "best", + "The video resolution. Can either be specified via the pixels, the abbreviation for pixels, or 'common-use' words\n"+ + "\tAvailable pixels: 1920x1080, 1280x720, 640x480, 480x360, 428x240\n"+ + "\tAvailable abbreviations: 1080p, 720p, 480p, 360p, 240p\n"+ + "\tAvailable common-use words: best (best available resolution), worst (worst available resolution)") - getCmd.Flags().BoolVar(&alternativeProgressFlag, "alternative-progress", false, "Shows an alternative, not so user-friendly progress instead of the progress bar") + getCmd.Flags().IntVarP(&downloadGoroutinesFlag, + "goroutines", + "g", + runtime.NumCPU(), + "Sets how many parallel segment downloads should be used") - // TODO: Rename this to something understandable (for "normal" users) - getCmd.Flags().IntVarP(&goroutinesFlag, "goroutines", "g", 4, "Sets how many parallel segment downloads should be used") + rootCmd.AddCommand(getCmd) } -type episodeInformation struct { - Format *crunchyroll.Format - Title string - URL string - SeriesTitle string - SeasonNum int - EpisodeNum int - AllSubtitles []*crunchyroll.Subtitle -} - -type information struct { - Title string `json:"title"` - SeriesName string `json:"series_name"` - SeasonNumber int `json:"season_number"` - EpisodeNumber int `json:"episode_number"` - OriginalURL string `json:"original_url"` - DownloadURL string `json:"download_url"` - Resolution string `json:"resolution"` - FPS float64 `json:"fps"` - Audio crunchyroll.LOCALE `json:"audio"` - Subtitle crunchyroll.LOCALE `json:"subtitle"` - Hardsub bool `json:"hardsub"` -} - -func download(urls []string) { - switch path.Ext(outputFlag) { - case ".ts": - // checks if only subtitles should be downloaded and if so, if the output flag has the default value - if onlySubFlag && outputFlag == "{title}.ts" { - outputFlag = "{title}.ass" - } - break - case ".ass": - if !onlySubFlag { - break - } - fallthrough - default: - if !hasFFmpeg() { - out.Fatalf("The file ending for the output file (%s) is not `.ts`. "+ - "Install ffmpeg (https://ffmpeg.org/download.html) use other media file endings (e.g. `.mp4`)\n", outputFlag) - } - } - allEpisodes, total, successes := parseURLs(urls) - out.Infof("%d of %d episodes could be parsed\n", successes, total) - - out.Empty() - if len(allEpisodes) == 0 { - out.Fatalf("Nothing to download, aborting\n") - } - if onlySubFlag { - out.Infof("Downloads (only subtitles):") - } else { - out.Infof("Downloads:") - } - for i, episode := range allEpisodes { - video := episode.Format.Video - if onlySubFlag && subtitleFlag == "" { - out.Infof("\t%d. %s » %spx, %.2f FPS (%s S%02dE%02d)\n", - i+1, episode.Title, video.Resolution, video.FrameRate, episode.SeriesTitle, episode.SeasonNum, episode.EpisodeNum) - } else { - out.Infof("\t%d. %s » %spx, %.2f FPS, %s audio (%s S%02dE%02d)\n", - i+1, episode.Title, video.Resolution, video.FrameRate, utils.LocaleLanguage(episode.Format.AudioLocale), episode.SeriesTitle, episode.SeasonNum, episode.EpisodeNum) - } - } - out.Empty() - - if fileInfo, stat := os.Stat(directoryFlag); os.IsNotExist(stat) { - if err := os.MkdirAll(directoryFlag, 0777); err != nil { - out.Fatalf("Failed to create directory which was given from the `-d`/`--directory` flag: %s\n", err) - } - } else if !fileInfo.IsDir() { - out.Fatalf("%s (given from the `-d`/`--directory` flag) is not a directory\n", directoryFlag) - } - - var success int - for _, episode := range allEpisodes { - var subtitle crunchyroll.LOCALE - if subtitleFlag != "" { - subtitle = localeToLOCALE(subtitleFlag) - } - info := information{ - Title: episode.Title, - SeriesName: episode.SeriesTitle, - SeasonNumber: episode.SeasonNum, - EpisodeNumber: episode.EpisodeNum, - OriginalURL: episode.URL, - DownloadURL: episode.Format.Video.URI, - Resolution: episode.Format.Video.Resolution, - FPS: episode.Format.Video.FrameRate, - Audio: episode.Format.AudioLocale, - Subtitle: subtitle, - } - - if verboseFlag { - fmtOptionsBytes, err := json.Marshal(info) - if err != nil { - fmtOptionsBytes = make([]byte, 0) - } - out.Debugf("Information (json): %s\n", string(fmtOptionsBytes)) - } - - filename := outputFlag - - fields := reflect.TypeOf(info) - values := reflect.ValueOf(info) - for i := 0; i < fields.NumField(); i++ { - field := fields.Field(i) - value := values.Field(i) - - var valueAsString string - switch value.Kind() { - case reflect.String: - valueAsString = value.String() - case reflect.Int: - valueAsString = strconv.Itoa(int(value.Int())) - if len(valueAsString) == 1 { - valueAsString = "0" + valueAsString - } - case reflect.Float64: - valueAsString = strconv.FormatFloat(value.Float(), 'f', 2, 64) - case reflect.Bool: - if value.Bool() { - valueAsString = field.Tag.Get("json") - } else { - valueAsString = fmt.Sprintf("no %s", field.Tag.Get("json")) - } - } - - filename = strings.ReplaceAll(filename, "{"+field.Tag.Get("json")+"}", valueAsString) - } - - invalidChars := invalidLinuxChars - if runtime.GOOS == "windows" { - invalidChars = invalidWindowsChars - } - - // replaces all the invalid characters - for _, char := range invalidChars { - filename = strings.ReplaceAll(filename, char, "") - } - - if onlySubFlag { - var found bool - if subtitleFlag == "" { - for _, formatSubtitle := range episode.AllSubtitles { - ext := path.Ext(filename) - base := strings.TrimSuffix(filename, ext) - - originalSubtitleFilename := fmt.Sprintf("%s_%s%s", base, formatSubtitle.Locale, ext) - subtitleFilename, changed := freeFileName(originalSubtitleFilename) - if changed { - out.Infof("The file %s already exist, renaming the download file to %s", originalSubtitleFilename, subtitleFilename) - } - file, err := os.Create(subtitleFilename) - if err != nil { - out.Errf("Failed to open subtitle file for locale %s: %v", formatSubtitle.Locale, err) - continue - } - if err = formatSubtitle.Download(file); err != nil { - out.Errf("Error while downloading %s subtitles: %s", formatSubtitle.Locale, err) - continue - } - found = true - } - } else { - for _, formatSubtitle := range episode.Format.Subtitles { - if formatSubtitle.Locale == subtitle { - file, err := os.Create(filename) - if err != nil { - out.Errf("Failed to open file %s: %v", filename, err) - break - } - if err = formatSubtitle.Download(file); err != nil { - out.Errf("Error while downloading subtitles: %v", err) - break - } - found = true - break - } - } - } - if found { - out.Infof("Downloaded subtitles for %s", episode.Title) - success++ - } - } else { - if downloadFormat(episode.Format, episode.AllSubtitles, filename, info) { - success++ - } - out.Empty() - } - } - - if onlySubFlag { - out.Infof("Downloaded all %d out of %d video subtitles\n", success, len(allEpisodes)) - } else { - out.Infof("Downloaded %d out of %d videos\n", success, len(allEpisodes)) - } -} - -func parseURLs(urls []string) (allEpisodes []episodeInformation, total, successes int) { - videoDupes := map[string]utils.VideoStructure{} - - betaUrl := regexp.MustCompile(`(?m)^https?://(www\.)?beta\.crunchyroll\.com*`) - +func download(urls []string) error { for i, url := range urls { - out.StartProgressf("Parsing url %d", i+1) - - var localTotal, localSuccesses int - - var err error - var video utils.VideoStructure - var episode *crunchyroll.Episode - if betaUrl.MatchString(url) { - if episodeid, ok := crunchyroll.ParseBetaEpisodeURL(url); ok { - episode, err = crunchyroll.EpisodeFromID(crunchy, episodeid) - } else if seriesid, ok := crunchyroll.ParseBetaSeriesURL(url); ok { - var vid crunchyroll.Video - vid, err = crunchyroll.SeriesFromID(crunchy, seriesid) - - switch vid.(type) { - case *crunchyroll.Series: - seasons, err := video.(*crunchyroll.Series).Seasons() - if err != nil { - out.EndProgressf(false, "Failed to get seasons for url %s: %s\n", url, err) - continue - } - video = utils.NewSeasonStructure(seasons).EpisodeStructure - if err := video.(*utils.EpisodeStructure).InitAll(); err != nil { - out.EndProgressf(false, "Failed to initialize series for url %s\n", url) - continue - } - case *crunchyroll.Movie: - movieListings, err := video.(*crunchyroll.Movie).MovieListing() - if err != nil { - out.EndProgressf(false, "Failed to get movie listing for url %s\n", url) - continue - } - video = utils.NewMovieListingStructure(movieListings) - if err := video.(*utils.MovieListingStructure).InitAll(); err != nil { - out.EndProgressf(false, "Failed to initialize movie for url %s\n", url) - continue - } - } - } - } else { - var seriesName string - var ok bool - if seriesName, _, _, _, ok = crunchyroll.ParseEpisodeURL(url); !ok { - seriesName, ok = crunchyroll.MatchVideo(url) - } - - if ok { - dupe, ok := videoDupes[seriesName] - if !ok { - var vid crunchyroll.Video - vid, err = crunchy.FindVideo(fmt.Sprintf("https://www.crunchyroll.com/%s", seriesName)) - - switch vid.(type) { - case *crunchyroll.Series: - seasons, err := vid.(*crunchyroll.Series).Seasons() - if err != nil { - out.EndProgressf(false, "Failed to get seasons for url %s: %s\n", url, err) - continue - } - dupe = utils.NewSeasonStructure(seasons).EpisodeStructure - if err := dupe.(*utils.EpisodeStructure).InitAll(); err != nil { - out.EndProgressf(false, "Failed to initialize series for url %s\n", url) - continue - } - case *crunchyroll.Movie: - movieListings, err := vid.(*crunchyroll.Movie).MovieListing() - if err != nil { - out.EndProgressf(false, "Failed to get movie listing for url %s\n", url) - continue - } - dupe = utils.NewMovieListingStructure(movieListings) - if err := dupe.(*utils.MovieListingStructure).InitAll(); err != nil { - out.EndProgressf(false, "Failed to initialize movie for url %s\n", url) - continue - } - } - } - video = dupe - } else { - err = fmt.Errorf("") - } - } - + out.SetProgress("Parsing url %d", i+1) + episodes, err := downloadExtractEpisodes(url) if err != nil { - out.EndProgressf(false, "URL %d seems to be invalid\n", i+1) - } else if episode != nil { - epstruct := utils.NewEpisodeStructure([]*crunchyroll.Episode{episode}) + out.StopProgress("Failed to parse url %d", i+1) + return err + } + out.StopProgress("Parsed url %d", i+1) - if err = epstruct.InitAll(); err != nil { - out.EndProgressf(false, "Could not init url %d, skipping\n", i+1) - } else if ep := parseEpisodes(epstruct, url); ep.Format != nil { - allEpisodes = append(allEpisodes, ep) - localSuccesses++ - } else { - out.EndProgressf(false, "Could not parse url %d, skipping\n", i+1) + for _, season := range episodes { + out.Info("%s Season %d", season[0].SeriesName, season[0].SeasonNumber) + + for j, info := range season { + out.Info("\t%d. %s » %spx, %.2f FPS (S%02dE%02d)", + j+1, + info.Title, + info.Resolution, + info.FPS, + info.SeasonNumber, + info.EpisodeNumber) } - localTotal++ - } else if video != nil { - if _, ok := crunchyroll.MatchVideo(url); ok { - out.Debugf("Parsed url %d as video\n", i+1) - var parsed []episodeInformation - parsed, localTotal, localSuccesses = parseVideo(video, url) - allEpisodes = append(allEpisodes, parsed...) - } else if _, _, _, _, ok = crunchyroll.ParseEpisodeURL(url); ok { - out.Debugf("Parsed url %d as episode\n", i+1) - if episode := parseEpisodes(video.(*utils.EpisodeStructure), url); episode.Format != nil { - allEpisodes = append(allEpisodes, episode) - localSuccesses++ - } else { - out.EndProgressf(false, "Could not parse url %d, skipping\n", i+1) + } + out.Empty() + + for _, season := range episodes { + for _, info := range season { + dir := info.Format(downloadDirectoryFlag) + if _, err = os.Stat(dir); os.IsNotExist(err) { + if err = os.MkdirAll(dir, 0777); err != nil { + return fmt.Errorf("error while creating directory: %v", err) + } } - localTotal++ - } else { - out.EndProgressf(false, "Could not parse url %d, skipping\n", i+1) - continue - } - } else { - out.EndProgressf(false, "URL %d seems to be invalid\n", i+1) - } - - out.EndProgressf(true, "Parsed url %d with %d successes and %d fails\n", i+1, localSuccesses, localTotal-localSuccesses) - - total += localTotal - successes += localSuccesses - } - return -} - -func parseVideo(videoStructure utils.VideoStructure, url string) (episodeInformations []episodeInformation, total, successes int) { - var orderedFormats [][]*crunchyroll.Format - - switch videoStructure.(type) { - case *utils.EpisodeStructure: - orderedFormats, _ = videoStructure.(*utils.EpisodeStructure).OrderFormatsByEpisodeNumber() - case *utils.MovieListingStructure: - unorderedFormats, _ := videoStructure.(*utils.MovieListingStructure).Formats() - orderedFormats = append(orderedFormats, unorderedFormats) - } - - out.Debugf("Found %d different episodes\n", len(orderedFormats)) - - for _, formats := range orderedFormats { - if formats == nil { - continue - } - total++ - - var title string - switch videoStructure.(type) { - case *utils.EpisodeStructure: - episode, _ := videoStructure.(*utils.EpisodeStructure).GetEpisodeByFormat(formats[0]) - title = episode.Title - case *utils.MovieListingStructure: - movieListing, _ := videoStructure.(*utils.MovieListingStructure).GetMovieListingByFormat(formats[0]) - title = movieListing.Title - } - - if format := findFormat(formats, title); format != nil { - info := episodeInformation{Format: format, URL: url} - switch videoStructure.(type) { - case *utils.EpisodeStructure: - episode, _ := videoStructure.(*utils.EpisodeStructure).GetEpisodeByFormat(format) - info.Title = episode.Title - info.SeriesTitle = episode.SeriesTitle - info.SeasonNum = episode.SeasonNumber - info.EpisodeNum = episode.EpisodeNumber - case *utils.MovieListingStructure: - movieListing, _ := videoStructure.(*utils.MovieListingStructure).GetMovieListingByFormat(format) - info.Title = movieListing.Title - info.SeriesTitle = movieListing.Title - info.SeasonNum, info.EpisodeNum = 1, 1 - } - - for _, audioFormat := range formats { - if audioFormat.AudioLocale == crunchyroll.JP { - info.AllSubtitles = audioFormat.Subtitles - break - } - } - - episodeInformations = append(episodeInformations, info) - out.Debugf("Successful parsed %s\n", title) - } - successes++ - } - - return -} - -func parseEpisodes(episodeStructure *utils.EpisodeStructure, url string) episodeInformation { - episode, _ := episodeStructure.GetEpisodeByURL(url) - ordered, _ := episodeStructure.OrderFormatsByEpisodeNumber() - - var subtitles []*crunchyroll.Subtitle - formats := ordered[episode.EpisodeNumber] - for _, format := range formats { - if format.AudioLocale == crunchyroll.JP { - subtitles = format.Subtitles - break - } - } - - out.Debugf("Found %d formats\n", len(formats)) - if format := findFormat(formats, episode.Title); format != nil { - episode, _ = episodeStructure.GetEpisodeByFormat(format) - out.Debugf("Found matching episode %s\n", episode.Title) - return episodeInformation{ - Format: format, - AllSubtitles: subtitles, - Title: episode.Title, - URL: url, - SeriesTitle: episode.SeriesTitle, - SeasonNum: episode.SeasonNumber, - EpisodeNum: episode.EpisodeNumber, - } - } - return episodeInformation{} -} - -func findFormat(formats []*crunchyroll.Format, name string) (format *crunchyroll.Format) { - formatStructure := utils.NewFormatStructure(formats) - - // if the only sub flag is given the japanese format gets returned because it has all subtitles available - if onlySubFlag { - jpFormat, _ := formatStructure.FilterFormatsByAudio(crunchyroll.JP) - return jpFormat[0] - } - - var audioLocale, subtitleLocale crunchyroll.LOCALE - - if audioFlag != "" { - audioLocale = localeToLOCALE(audioFlag) - } else { - audioLocale = systemLocale() - } - if subtitleFlag != "" { - subtitleLocale = localeToLOCALE(subtitleFlag) - } - - formats, _ = formatStructure.FilterFormatsByLocales(audioLocale, subtitleLocale, !noHardsubFlag) - if formats == nil { - if audioFlag == "" { - out.Errf("Failed to find episode with '%s' audio and '%s' subtitles, tying with %s audio\n", audioLocale, subtitleLocale, strings.ToLower(utils.LocaleLanguage(crunchyroll.JP))) - audioLocale = crunchyroll.JP - formats, _ = formatStructure.FilterFormatsByLocales(audioLocale, subtitleLocale, !noHardsubFlag) - } - if formats == nil && subtitleFlag == "" { - out.Errf("Failed to find episode with '%s' audio and '%s' subtitles, tying with %s subtitle\n", audioLocale, subtitleLocale, strings.ToLower(utils.LocaleLanguage(systemLocale()))) - subtitleLocale = systemLocale() - formats, _ = formatStructure.FilterFormatsByLocales(audioLocale, subtitleLocale, !noHardsubFlag) - } - if formats == nil { - out.Errf("Could not find matching video with '%s' audio and '%s' subtitles for %s. Try to change the '--audio' and / or '--subtitle' flag\n", audioLocale, subtitleLocale, name) - return nil - } - } - if resolutionFlag == "best" || resolutionFlag == "" { - sort.Sort(sort.Reverse(utils.FormatsByResolution(formats))) - format = formats[0] - } else if resolutionFlag == "worst" { - sort.Sort(utils.FormatsByResolution(formats)) - format = formats[0] - } else if strings.HasSuffix(resolutionFlag, "p") { - for _, f := range formats { - if strings.Split(f.Video.Resolution, "x")[1] == strings.TrimSuffix(resolutionFlag, "p") { - format = f - break - } - } - } else if strings.Contains(resolutionFlag, "x") { - for _, f := range formats { - if f.Video.Resolution == resolutionFlag { - format = f - break - } - } - } - if format == nil { - out.Errf("Failed to get video with resolution '%s'\n", resolutionFlag) - } - - subtitleFlag = string(subtitleLocale) - return -} - -func downloadFormat(format *crunchyroll.Format, subtitles []*crunchyroll.Subtitle, outFile string, info information) bool { - oldOutFile := outFile - outFile, changed := freeFileName(outFile) - ext := path.Ext(outFile) - out.Debugf("Download filename: %s\n", outFile) - if changed { - out.Errf("The file %s already exist, renaming the download file to %s\n", oldOutFile, outFile) - } - if ext != ".ts" { - if !hasFFmpeg() { - out.Fatalf("The file ending for the output file (%s) is not `.ts`. "+ - "Install ffmpeg (https://ffmpeg.org/download.html) use other media file endings (e.g. `.mp4`)\n", outFile) - } - out.Debugf("File will be converted via ffmpeg") - } - var subtitleFilename string - if noHardsubFlag { - subtitle, ok := utils.SubtitleByLocale(format, info.Subtitle) - if !ok { - out.Errf("Failed to get %s subtitles\n", info.Subtitle) - return false - } - subtitleFilename, _ = freeFileName(fmt.Sprintf("%s.%s", strings.TrimSuffix(outFile, ext), subtitle.Format)) - out.Debugf("Subtitles will be saved as '%s'\n", subtitleFilename) - } - - out.Infof("Downloading '%s' (%s) as '%s'\n", info.Title, info.OriginalURL, outFile) - out.Infof("Series: %s\n", info.SeriesName) - out.Infof("Season & Episode: S%02dE%02d", info.SeasonNumber, info.EpisodeNumber) - out.Infof("Audio: %s\n", info.Audio) - out.Infof("Subtitle: %s\n", info.Subtitle) - out.Infof("Hardsub: %v\n", format.Hardsub != "") - out.Infof("Resolution: %s\n", info.Resolution) - out.Infof("FPS: %.2f\n", info.FPS) - - var err error - if ext == ".ts" { - var file *os.File - file, err = os.Create(outFile) - defer file.Close() - if err != nil { - out.Errf("Could not create file '%s' to download episode '%s' (%s): %s, skipping\n", outFile, info.Title, info.OriginalURL, err) - return false - } - - err = format.DownloadGoroutines(file, goroutinesFlag, downloadProgress) - // newline to avoid weird output - fmt.Println() - } else { - var tempDir string - tempDir, err = os.MkdirTemp("", "crunchy_") - if err != nil { - out.Errln("Failed to create temp download dir. Skipping") - return false - } - - var segmentCount int - err = format.DownloadSegments(tempDir, goroutinesFlag, func(segment *m3u8.MediaSegment, current, total int, file *os.File) error { - segmentCount++ - return downloadProgress(segment, current, total, file) - }) - // newline to avoid weird output - fmt.Println() - - f, _ := os.CreateTemp("", "*.txt") - for i := 0; i < segmentCount; i++ { - fmt.Fprintf(f, "file '%s.ts'\n", filepath.Join(tempDir, strconv.Itoa(i))) - } - defer os.Remove(f.Name()) - f.Close() - - args := []string{ - "-f", "concat", - "-safe", "0", - "-i", f.Name(), - } - if ext == ".mkv" && subtitleFlag == "" { - // this saves all subtitles into a mkv file. see https://github.com/ByteDream/crunchyroll-go/issues/5 for some details - - ffmpegInput := make([]string, 0) - ffmpegMap := []string{"-map", "0"} - ffmpegMetadata := make([]string, 0) - for i, subtitle := range subtitles { - subtitleFilepath := filepath.Join(cleanupPath, fmt.Sprintf("%s.%s", subtitle.Locale, subtitle.Format)) - - var file *os.File - file, err = os.Create(subtitleFilepath) + file, err := os.Create(generateFilename(info.Format(downloadOutputFlag), dir)) if err != nil { - out.Errf("Could not create file to download %s subtitles to: %v", subtitle.Locale, err) - continue + return fmt.Errorf("failed to create output file: %v", err) } - if err = subtitle.Download(file); err != nil { - out.Errf("Failed to download subtitles: %s", err) - continue + + if err = downloadInfo(info, file); err != nil { + file.Close() + os.Remove(file.Name()) + return err } - ffmpegInput = append(ffmpegInput, "-i", subtitleFilepath) - ffmpegMap = append(ffmpegMap, "-map", strconv.Itoa(i+1)) - ffmpegMetadata = append(ffmpegMetadata, fmt.Sprintf("-metadata:s:s:%d", i), fmt.Sprintf("language=%s", strings.Split(string(subtitle.Locale), "-")[0])) + file.Close() } - - args = append(args, ffmpegInput...) - args = append(args, ffmpegMap...) - args = append(args, ffmpegMetadata...) - } - args = append(args, "-c", "copy", outFile) - - cmd := exec.Command("ffmpeg", args...) - err = cmd.Run() - } - os.RemoveAll(cleanupPath) - cleanupPath = "" - - if err != nil { - out.Errf("Failed to download video, skipping: %v", err) - } else { - if info.Subtitle == "" { - out.Infof("Downloaded '%s' as '%s' with %s audio locale\n", info.Title, outFile, strings.ToLower(utils.LocaleLanguage(info.Audio))) - } else { - out.Infof("Downloaded '%s' as '%s' with %s audio locale and %s subtitle locale\n", info.Title, outFile, strings.ToLower(utils.LocaleLanguage(info.Audio)), strings.ToLower(utils.LocaleLanguage(info.Subtitle))) - if subtitleFilename != "" { - file, err := os.Create(subtitleFilename) - if err != nil { - out.Errf("Failed to download subtitles: %s\n", err) - return false - } else { - subtitle, ok := utils.SubtitleByLocale(format, info.Subtitle) - if !ok { - out.Errf("Failed to get %s subtitles\n", info.Subtitle) - return false - } - if err := subtitle.Download(file); err != nil { - out.Errf("Failed to download subtitles: %s\n", err) - return false - } - out.Infof("Downloaded '%s' subtitles to '%s'\n", info.Subtitle, subtitleFilename) - } - } - } - } - - return true -} - -func downloadProgress(segment *m3u8.MediaSegment, current, total int, file *os.File) error { - if cleanupPath == "" { - cleanupPath = path.Dir(file.Name()) - } - - if !quietFlag { - percentage := float32(current) / float32(total) * 100 - if alternativeProgressFlag { - out.Infof("Downloading %d/%d (%.2f%%) » %s", current, total, percentage, segment.URI) - } else { - progressWidth := float32(terminalWidth() - (14 + len(out.InfoLog.Prefix())) - (len(fmt.Sprint(total)))*2) - - repeatCount := int(percentage / (float32(100) / progressWidth)) - // it can be lower than zero when the terminal is very tiny - if repeatCount < 0 { - repeatCount = 0 - } - - // alternative: - // progressPercentage := strings.Repeat("█", repeatCount) - progressPercentage := (strings.Repeat("=", repeatCount) + ">")[1:] - - fmt.Printf("\r%s[%-"+fmt.Sprint(progressWidth)+"s]%4d%% %8d/%d", out.InfoLog.Prefix(), progressPercentage, int(percentage), current, total) } } return nil } + +func downloadInfo(info formatInformation, file *os.File) error { + out.Debug("Entering season %d, episode %d", info.SeasonNumber, info.EpisodeNumber) + + if err := info.format.InitVideo(); err != nil { + return fmt.Errorf("error while initializing the video: %v", err) + } + + downloadProgress := &downloadProgress{ + Prefix: out.InfoLog.Prefix(), + Message: "Downloading video", + // number of segments a video has +2 is for merging and the success message + Total: int(info.format.Video.Chunklist.Count()) + 2, + Dev: out.IsDev(), + Quiet: out.IsQuiet(), + } + if out.IsDev() { + downloadProgress.Prefix = out.DebugLog.Prefix() + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + downloader := crunchyroll.NewDownloader(ctx, file, downloadGoroutinesFlag, func(segment *m3u8.MediaSegment, current, total int, file *os.File) error { + // check if the context was cancelled. + // must be done in to not print any progress messages if ctrl+c was pressed + if ctx.Err() != nil { + return nil + } + + if out.IsDev() { + downloadProgress.UpdateMessage(fmt.Sprintf("Downloading %d/%d (%.2f%%) » %s", current, total, float32(current)/float32(total)*100, segment.URI), false) + } else { + downloadProgress.Update() + } + + if current == total { + downloadProgress.UpdateMessage("Merging segments", false) + } + return nil + }) + + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt) + go func() { + select { + case <-sig: + signal.Stop(sig) + out.Exit("Exiting... (may take a few seconds)") + out.Exit("To force exit press ctrl+c (again)") + cancel() + // os.Exit(1) is not called because an immediate exit after the cancel function does not let + // the download process enough time to stop gratefully. A result of this is that the temporary + // directory where the segments are downloaded to will not be deleted + case <-ctx.Done(): + // this is just here to end the goroutine and prevent it from running forever without a reason + } + }() + out.Debug("Set up signal catcher") + + out.Info("Downloading episode `%s` to `%s`", info.Title, filepath.Base(file.Name())) + out.Info("\tEpisode: S%02dE%02d", info.SeasonNumber, info.EpisodeNumber) + out.Info("\tAudio: %s", info.Audio) + out.Info("\tSubtitle: %s", info.Subtitle) + out.Info("\tResolution: %spx", info.Resolution) + out.Info("\tFPS: %.2f", info.FPS) + if err := info.format.Download(downloader); err != nil { + return fmt.Errorf("error while downloading: %v", err) + } + + downloadProgress.UpdateMessage("Download finished", false) + + signal.Stop(sig) + out.Debug("Stopped signal catcher") + + out.Empty() + out.Empty() + + return nil +} + +func downloadExtractEpisodes(url string) ([][]formatInformation, error) { + episodes, err := extractEpisodes(url, crunchyroll.JP, crunchyroll.LOCALE(downloadAudioFlag)) + if err != nil { + return nil, err + } + japanese := episodes[0] + custom := episodes[1] + + sort.Sort(utils.EpisodesByNumber(japanese)) + sort.Sort(utils.EpisodesByNumber(custom)) + + var errMessages []string + + var final []*crunchyroll.Episode + if len(japanese) == 0 || len(japanese) == len(custom) { + final = custom + } else { + for _, jp := range japanese { + before := len(final) + for _, episode := range custom { + if jp.SeasonNumber == episode.SeasonNumber && jp.EpisodeNumber == episode.EpisodeNumber { + final = append(final, episode) + } + } + if before == len(final) { + errMessages = append(errMessages, fmt.Sprintf("%s has no %s audio, using %s as fallback", jp.Title, crunchyroll.LOCALE(downloadAudioFlag), crunchyroll.JP)) + final = append(final, jp) + } + } + } + + if len(errMessages) > 10 { + for _, msg := range errMessages[:10] { + out.SetProgress(msg) + } + out.SetProgress("... and %d more", len(errMessages)-10) + } else { + for _, msg := range errMessages { + out.SetProgress(msg) + } + } + + var infoFormat [][]formatInformation + for _, season := range utils.SortEpisodesBySeason(final) { + tmpFormatInformation := make([]formatInformation, 0) + for _, episode := range season { + format, err := episode.GetFormat(downloadResolutionFlag, crunchyroll.LOCALE(downloadSubtitleFlag), true) + if err != nil { + return nil, fmt.Errorf("error while receiving format for %s: %v", episode.Title, err) + } + tmpFormatInformation = append(tmpFormatInformation, formatInformation{ + format: format, + + Title: episode.Title, + SeriesName: episode.SeriesTitle, + SeasonName: episode.SeasonTitle, + SeasonNumber: episode.SeasonNumber, + EpisodeNumber: episode.EpisodeNumber, + Resolution: format.Video.Resolution, + FPS: format.Video.FrameRate, + Audio: format.AudioLocale, + }) + } + infoFormat = append(infoFormat, tmpFormatInformation) + } + return infoFormat, nil +} From b2393530391e933f17f89f77d0ec9f001116c418 Mon Sep 17 00:00:00 2001 From: bytedream Date: Tue, 22 Mar 2022 19:39:43 +0100 Subject: [PATCH 81/82] Added new archive command --- cmd/crunchyroll-go/cmd/archive.go | 798 ++++++++++++++++++++++++++++++ 1 file changed, 798 insertions(+) create mode 100644 cmd/crunchyroll-go/cmd/archive.go diff --git a/cmd/crunchyroll-go/cmd/archive.go b/cmd/crunchyroll-go/cmd/archive.go new file mode 100644 index 0000000..17072eb --- /dev/null +++ b/cmd/crunchyroll-go/cmd/archive.go @@ -0,0 +1,798 @@ +package cmd + +import ( + "archive/tar" + "archive/zip" + "bufio" + "bytes" + "compress/gzip" + "context" + "fmt" + "github.com/ByteDream/crunchyroll-go" + "github.com/ByteDream/crunchyroll-go/utils" + "github.com/grafov/m3u8" + "github.com/spf13/cobra" + "io" + "os" + "os/exec" + "os/signal" + "path/filepath" + "regexp" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +var ( + archiveLanguagesFlag []string + + archiveDirectoryFlag string + archiveOutputFlag string + + archiveMergeFlag string + + archiveCompressFlag string + + archiveResolutionFlag string + + archiveGoroutinesFlag int +) + +var archiveCmd = &cobra.Command{ + Use: "archive", + Short: "Stores the given videos with all subtitles and multiple audios in a .mkv file", + Args: cobra.MinimumNArgs(1), + + PreRunE: func(cmd *cobra.Command, args []string) error { + out.Debug("Validating arguments") + + if !hasFFmpeg() { + return fmt.Errorf("ffmpeg is needed to run this command correctly") + } + out.Debug("FFmpeg detected") + + if filepath.Ext(archiveOutputFlag) != ".mkv" { + return fmt.Errorf("currently only matroska / .mkv files are supported") + } + + for _, locale := range archiveLanguagesFlag { + if !utils.ValidateLocale(crunchyroll.LOCALE(locale)) { + // if locale is 'all', match all known locales + if locale == "all" { + archiveLanguagesFlag = allLocalesAsStrings() + break + } + return fmt.Errorf("%s is not a valid locale. Choose from: %s", locale, strings.Join(allLocalesAsStrings(), ", ")) + } + } + out.Debug("Using following audio locales: %s", strings.Join(archiveLanguagesFlag, ", ")) + + var found bool + for _, mode := range []string{"auto", "audio", "video"} { + if archiveMergeFlag == mode { + out.Debug("Using %s merge behavior", archiveMergeFlag) + found = true + break + } + } + if !found { + return fmt.Errorf("'%s' is no valid merge flag. Use 'auto', 'audio' or 'video'", archiveMergeFlag) + } + + if archiveCompressFlag != "" { + found = false + for _, algo := range []string{".tar", ".tar.gz", ".tgz", ".zip"} { + if strings.HasSuffix(archiveCompressFlag, algo) { + out.Debug("Using %s compression", algo) + found = true + break + } + } + if !found { + return fmt.Errorf("'%s' is no valid compress algorithm. Valid algorithms / file endings are '.tar', '.tar.gz', '.zip'", + archiveCompressFlag) + } + } + + switch archiveResolutionFlag { + case "1080p", "720p", "480p", "360p", "240p": + intRes, _ := strconv.ParseFloat(strings.TrimSuffix(archiveResolutionFlag, "p"), 84) + archiveResolutionFlag = fmt.Sprintf("%dx%s", int(intRes*(16/9)), strings.TrimSuffix(archiveResolutionFlag, "p")) + case "1920x1080", "1280x720", "640x480", "480x360", "428x240", "best", "worst": + default: + return fmt.Errorf("'%s' is not a valid resolution", archiveResolutionFlag) + } + out.Debug("Using resolution '%s'", archiveResolutionFlag) + + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + loadCrunchy() + + return archive(args) + }, +} + +func init() { + archiveCmd.Flags().StringSliceVarP(&archiveLanguagesFlag, + "language", + "l", + []string{string(systemLocale(false)), string(crunchyroll.JP)}, + "Audio locale which should be downloaded. Can be used multiple times") + + cwd, _ := os.Getwd() + archiveCmd.Flags().StringVarP(&archiveDirectoryFlag, + "directory", + "d", + cwd, + "The directory to store the files into") + archiveCmd.Flags().StringVarP(&archiveOutputFlag, + "output", + "o", + "{title}.mkv", + "Name of the output file. If you use the following things in the name, the will get replaced:\n"+ + "\t{title} » Title of the video\n"+ + "\t{series_name} » Name of the series\n"+ + "\t{season_name} » Name of the season\n"+ + "\t{season_number} » Number of the season\n"+ + "\t{episode_number} » Number of the episode\n"+ + "\t{resolution} » Resolution of the video\n"+ + "\t{fps} » Frame Rate of the video\n"+ + "\t{audio} » Audio locale of the video\n"+ + "\t{subtitle} » Subtitle locale of the video") + + archiveCmd.Flags().StringVarP(&archiveMergeFlag, + "merge", + "m", + "auto", + "Sets the behavior of the stream merging. Valid behaviors are 'auto', 'audio', 'video'") + + archiveCmd.Flags().StringVarP(&archiveCompressFlag, + "compress", + "c", + "", + "If is set, all output will be compresses into an archive (every url generates a new one). "+ + "This flag sets the name of the compressed output file. The file ending specifies the compression algorithm. "+ + "The following algorithms are supported: gzip, tar, zip") + + archiveCmd.Flags().StringVarP(&archiveResolutionFlag, + "resolution", + "r", + "best", + "The video resolution. Can either be specified via the pixels, the abbreviation for pixels, or 'common-use' words\n"+ + "\tAvailable pixels: 1920x1080, 1280x720, 640x480, 480x360, 428x240\n"+ + "\tAvailable abbreviations: 1080p, 720p, 480p, 360p, 240p\n"+ + "\tAvailable common-use words: best (best available resolution), worst (worst available resolution)") + + archiveCmd.Flags().IntVarP(&archiveGoroutinesFlag, + "goroutines", + "g", + runtime.NumCPU(), + "Number of parallel segment downloads") + + rootCmd.AddCommand(archiveCmd) +} + +func archive(urls []string) error { + for i, url := range urls { + out.SetProgress("Parsing url %d", i+1) + episodes, err := archiveExtractEpisodes(url) + if err != nil { + out.StopProgress("Failed to parse url %d", i+1) + return err + } + out.StopProgress("Parsed url %d", i+1) + + var compressFile *os.File + var c compress + + if archiveCompressFlag != "" { + compressFile, err = os.Create(generateFilename(archiveCompressFlag, "")) + if err != nil { + return fmt.Errorf("failed to create archive file: %v", err) + } + if strings.HasSuffix(archiveCompressFlag, ".tar") { + c = newTarCompress(compressFile) + } else if strings.HasSuffix(archiveCompressFlag, ".tar.gz") || strings.HasSuffix(archiveCompressFlag, ".tgz") { + c = newGzipCompress(compressFile) + } else if strings.HasSuffix(archiveCompressFlag, ".zip") { + c = newZipCompress(compressFile) + } + } + + for _, season := range episodes { + out.Info("%s Season %d", season[0].SeriesName, season[0].SeasonNumber) + + for j, info := range season { + out.Info("\t%d. %s » %spx, %.2f FPS (S%02dE%02d)", + j+1, + info.Title, + info.Resolution, + info.FPS, + info.SeasonNumber, + info.EpisodeNumber) + } + } + out.Empty() + + for _, season := range episodes { + for _, info := range season { + var filename string + var writeCloser io.WriteCloser + if c != nil { + filename = info.Format(archiveOutputFlag) + writeCloser, err = c.NewFile(info) + if err != nil { + return fmt.Errorf("failed to pre generate new archive file: %v", err) + } + } else { + dir := info.Format(downloadDirectoryFlag) + if _, err = os.Stat(dir); os.IsNotExist(err) { + if err = os.MkdirAll(dir, 0777); err != nil { + return fmt.Errorf("error while creating directory: %v", err) + } + } + filename = generateFilename(info.Format(archiveOutputFlag), dir) + writeCloser, err = os.Create(filename) + if err != nil { + return fmt.Errorf("failed to create new file: %v", err) + } + } + + if err = archiveInfo(info, writeCloser, filename); err != nil { + writeCloser.Close() + if f, ok := writeCloser.(*os.File); ok { + os.Remove(f.Name()) + } else { + c.Close() + compressFile.Close() + os.RemoveAll(compressFile.Name()) + } + return err + } + + writeCloser.Close() + } + } + if c != nil { + c.Close() + } + if compressFile != nil { + compressFile.Close() + } + } + return nil +} + +func archiveInfo(info formatInformation, writeCloser io.WriteCloser, filename string) error { + out.Debug("Entering season %d, episode %d with %d additional formats", info.SeasonNumber, info.EpisodeNumber, len(info.additionalFormats)) + + downloadProgress, err := createArchiveProgress(info) + if err != nil { + return fmt.Errorf("error while setting up downloader: %v", err) + } + + rootFile, err := os.CreateTemp("", fmt.Sprintf("%s_*.ts", strings.TrimSuffix(filepath.Base(filename), filepath.Ext(filename)))) + if err != nil { + return fmt.Errorf("failed to create temp file: %v", err) + } + defer os.Remove(rootFile.Name()) + defer rootFile.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + downloader := crunchyroll.NewDownloader(ctx, rootFile, downloadGoroutinesFlag, func(segment *m3u8.MediaSegment, current, total int, file *os.File) error { + // check if the context was cancelled. + // must be done in to not print any progress messages if ctrl+c was pressed + if ctx.Err() != nil { + return nil + } + + if out.IsDev() { + downloadProgress.UpdateMessage(fmt.Sprintf("Downloading %d/%d (%.2f%%) » %s", current, total, float32(current)/float32(total)*100, segment.URI), false) + } else { + downloadProgress.Update() + } + + if current == total { + downloadProgress.UpdateMessage("Merging segments", false) + } + return nil + }) + + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt) + go func() { + select { + case <-sig: + signal.Stop(sig) + out.Exit("Exiting... (may take a few seconds)") + out.Exit("To force exit press ctrl+c (again)") + cancel() + // os.Exit(1) is not called since an immediate exit after the cancel function does not let + // the download process enough time to stop gratefully. A result of this is that the temporary + // directory where the segments are downloaded to will not be deleted + case <-ctx.Done(): + // this is just here to end the goroutine and prevent it from running forever without a reason + } + }() + out.Debug("Set up signal catcher") + + var additionalDownloaderOpts []string + var mergeMessage string + switch archiveMergeFlag { + case "auto": + additionalDownloaderOpts = []string{"-vn"} + for _, format := range info.additionalFormats { + if format.Video.Bandwidth != info.format.Video.Bandwidth { + // revoke the changed FFmpegOpts above + additionalDownloaderOpts = []string{} + break + } + } + if len(additionalDownloaderOpts) > 0 { + mergeMessage = "merging audio for additional formats" + } else { + mergeMessage = "merging video for additional formats" + } + case "audio": + additionalDownloaderOpts = []string{"-vn"} + mergeMessage = "merging audio for additional formats" + case "video": + mergeMessage = "merging video for additional formats" + } + + out.Info("Downloading episode `%s` to `%s` (%s)", info.Title, filepath.Base(filename), mergeMessage) + out.Info("\tEpisode: S%02dE%02d", info.SeasonNumber, info.EpisodeNumber) + out.Info("\tAudio: %s", info.Audio) + out.Info("\tSubtitle: %s", info.Subtitle) + out.Info("\tResolution: %spx", info.Resolution) + out.Info("\tFPS: %.2f", info.FPS) + + var videoFiles, audioFiles, subtitleFiles []string + defer func() { + for _, f := range append(append(videoFiles, audioFiles...), subtitleFiles...) { + os.RemoveAll(f) + } + }() + + var f []string + if f, err = archiveDownloadVideos(downloader, filepath.Base(filename), true, info.format); err != nil { + if err != ctx.Err() { + return fmt.Errorf("error while downloading: %v", err) + } + return err + } + videoFiles = append(videoFiles, f[0]) + + if len(additionalDownloaderOpts) == 0 { + var videos []string + downloader.FFmpegOpts = additionalDownloaderOpts + if videos, err = archiveDownloadVideos(downloader, filepath.Base(filename), true, info.additionalFormats...); err != nil { + return fmt.Errorf("error while downloading additional videos: %v", err) + } + downloader.FFmpegOpts = []string{} + videoFiles = append(videoFiles, videos...) + } else { + var audios []string + if audios, err = archiveDownloadVideos(downloader, filepath.Base(filename), false, info.additionalFormats...); err != nil { + return fmt.Errorf("error while downloading additional videos: %v", err) + } + audioFiles = append(audioFiles, audios...) + } + + sort.Sort(utils.SubtitlesByLocale(info.format.Subtitles)) + if len(archiveLanguagesFlag) > 0 && archiveLanguagesFlag[0] != "all" { + for j, language := range archiveLanguagesFlag { + locale := crunchyroll.LOCALE(language) + for k, subtitle := range info.format.Subtitles { + if subtitle.Locale == locale { + info.format.Subtitles = append(info.format.Subtitles[:k], info.format.Subtitles[k+1:]...) + info.format.Subtitles = append(info.format.Subtitles[:j], append([]*crunchyroll.Subtitle{subtitle}, info.format.Subtitles[j:]...)...) + break + } + } + } + } + + var subtitles []string + if subtitles, err = archiveDownloadSubtitles(filepath.Base(filename), info.format.Subtitles...); err != nil { + return fmt.Errorf("error while downloading subtitles: %v", err) + } + subtitleFiles = append(subtitleFiles, subtitles...) + + if err = archiveFFmpeg(ctx, writeCloser, videoFiles, audioFiles, subtitleFiles); err != nil { + return fmt.Errorf("failed to merge files: %v", err) + } + + downloadProgress.UpdateMessage("Download finished", false) + + signal.Stop(sig) + out.Debug("Stopped signal catcher") + + out.Empty() + out.Empty() + + return nil +} + +func createArchiveProgress(info formatInformation) (*downloadProgress, error) { + var progressCount int + if err := info.format.InitVideo(); err != nil { + return nil, fmt.Errorf("error while initializing a video: %v", err) + } + // + number of segments a video has +1 is for merging + progressCount += int(info.format.Video.Chunklist.Count()) + 1 + for _, f := range info.additionalFormats { + if f == info.format { + continue + } + + if err := f.InitVideo(); err != nil { + return nil, err + } + // + number of segments a video has +1 is for merging + progressCount += int(f.Video.Chunklist.Count()) + 1 + } + + downloadProgress := &downloadProgress{ + Prefix: out.InfoLog.Prefix(), + Message: "Downloading video", + // number of segments a video +1 is for the success message + Total: progressCount + 1, + Dev: out.IsDev(), + Quiet: out.IsQuiet(), + } + if out.IsDev() { + downloadProgress.Prefix = out.DebugLog.Prefix() + } + + return downloadProgress, nil +} + +func archiveDownloadVideos(downloader crunchyroll.Downloader, filename string, video bool, formats ...*crunchyroll.Format) ([]string, error) { + var files []string + + for _, format := range formats { + var name string + if video { + name = fmt.Sprintf("%s_%s_video_*.ts", filename, format.AudioLocale) + } else { + name = fmt.Sprintf("%s_%s_audio_*.aac", filename, format.AudioLocale) + } + + f, err := os.CreateTemp("", name) + if err != nil { + return nil, err + } + files = append(files, f.Name()) + + downloader.Writer = f + if err = format.Download(downloader); err != nil { + f.Close() + for _, file := range files { + os.Remove(file) + } + return nil, err + } + f.Close() + + out.Debug("Downloaded '%s' video", format.AudioLocale) + } + + return files, nil +} + +func archiveDownloadSubtitles(filename string, subtitles ...*crunchyroll.Subtitle) ([]string, error) { + var files []string + + for _, subtitle := range subtitles { + f, err := os.CreateTemp("", fmt.Sprintf("%s_%s_subtitle_*.ass", filename, subtitle.Locale)) + if err != nil { + return nil, err + } + files = append(files, f.Name()) + + if err := subtitle.Save(f); err != nil { + f.Close() + for _, file := range files { + os.Remove(file) + } + return nil, err + } + f.Close() + + out.Debug("Downloaded '%s' subtitles", subtitle.Locale) + } + + return files, nil +} + +func archiveFFmpeg(ctx context.Context, dst io.Writer, videoFiles, audioFiles, subtitleFiles []string) error { + var input, maps, metadata []string + re := regexp.MustCompile(`(?m)_([a-z]{2}-([A-Z]{2}|[0-9]{3}))_(video|audio|subtitle)`) + + for i, video := range videoFiles { + input = append(input, "-i", video) + maps = append(maps, "-map", strconv.Itoa(i)) + locale := crunchyroll.LOCALE(re.FindStringSubmatch(video)[1]) + metadata = append(metadata, fmt.Sprintf("-metadata:s:v:%d", i), fmt.Sprintf("language=%s", utils.LocaleLanguage(locale))) + metadata = append(metadata, fmt.Sprintf("-metadata:s:a:%d", i), fmt.Sprintf("language=%s", utils.LocaleLanguage(locale))) + } + + for i, audio := range audioFiles { + input = append(input, "-i", audio) + maps = append(maps, "-map", strconv.Itoa(i+len(videoFiles))) + locale := crunchyroll.LOCALE(re.FindStringSubmatch(audio)[1]) + metadata = append(metadata, fmt.Sprintf("-metadata:s:a:%d", i), fmt.Sprintf("language=%s", utils.LocaleLanguage(locale))) + } + + for i, subtitle := range subtitleFiles { + input = append(input, "-i", subtitle) + maps = append(maps, "-map", strconv.Itoa(i+len(videoFiles)+len(audioFiles))) + locale := crunchyroll.LOCALE(re.FindStringSubmatch(subtitle)[1]) + metadata = append(metadata, fmt.Sprintf("-metadata:s:s:%d", i), fmt.Sprintf("title=%s", utils.LocaleLanguage(locale))) + } + + commandOptions := []string{"-y"} + commandOptions = append(commandOptions, input...) + commandOptions = append(commandOptions, maps...) + commandOptions = append(commandOptions, metadata...) + // we have to create a temporary file here because it must be seekable + // for ffmpeg. + // ffmpeg could write to dst too, but this would require to re-encode + // the audio which results in much higher time and resource consumption + // (0-1 second with the temp file, ~20 seconds with re-encoding on my system) + file, err := os.CreateTemp("", "") + if err != nil { + return err + } + file.Close() + defer os.Remove(file.Name()) + + commandOptions = append(commandOptions, "-c", "copy", "-f", "matroska", file.Name()) + + // just a little nicer debug output to copy and paste the ffmpeg for debug reasons + if out.IsDev() { + var debugOptions []string + + for _, option := range commandOptions { + if strings.HasPrefix(option, "title=") { + debugOptions = append(debugOptions, "title=\""+strings.TrimPrefix(option, "title=")+"\"") + } else if strings.HasPrefix(option, "language=") { + debugOptions = append(debugOptions, "language=\""+strings.TrimPrefix(option, "language=")+"\"") + } else if strings.Contains(option, " ") { + debugOptions = append(debugOptions, "\""+option+"\"") + } else { + debugOptions = append(debugOptions, option) + } + } + out.Debug("FFmpeg merge command: ffmpeg %s", strings.Join(debugOptions, " ")) + } + + var errBuf bytes.Buffer + cmd := exec.CommandContext(ctx, "ffmpeg", commandOptions...) + cmd.Stderr = &errBuf + if err = cmd.Run(); err != nil { + return fmt.Errorf(errBuf.String()) + } + + file, err = os.Open(file.Name()) + if err != nil { + return err + } + defer file.Close() + + _, err = bufio.NewWriter(dst).ReadFrom(file) + return err +} + +func archiveExtractEpisodes(url string) ([][]formatInformation, error) { + var hasJapanese bool + languagesAsLocale := []crunchyroll.LOCALE{crunchyroll.JP} + for _, language := range archiveLanguagesFlag { + locale := crunchyroll.LOCALE(language) + if locale == crunchyroll.JP { + hasJapanese = true + } else { + languagesAsLocale = append(languagesAsLocale, locale) + } + } + + episodes, err := extractEpisodes(url, languagesAsLocale...) + if err != nil { + return nil, err + } + + if !hasJapanese && len(episodes[1:]) == 0 { + return nil, fmt.Errorf("no episodes found") + } + + for i, eps := range episodes { + if len(eps) == 0 { + out.SetProgress("%s has no matching episodes", languagesAsLocale[i]) + } else if len(episodes[0]) > len(eps) { + out.SetProgress("%s has %d less episodes than existing in japanese (%d)", languagesAsLocale[i], len(episodes[0])-len(eps), len(episodes[0])) + } + } + + if !hasJapanese { + episodes = episodes[1:] + } + + eps := make(map[int]map[int]*formatInformation) + for _, lang := range episodes { + for _, season := range utils.SortEpisodesBySeason(lang) { + if _, ok := eps[season[0].SeasonNumber]; !ok { + eps[season[0].SeasonNumber] = map[int]*formatInformation{} + } + for _, episode := range season { + format, err := episode.GetFormat(archiveResolutionFlag, "", false) + if err != nil { + return nil, fmt.Errorf("error while receiving format for %s: %v", episode.Title, err) + } + + if _, ok := eps[episode.SeasonNumber][episode.EpisodeNumber]; !ok { + eps[episode.SeasonNumber][episode.EpisodeNumber] = &formatInformation{ + format: format, + additionalFormats: make([]*crunchyroll.Format, 0), + + Title: episode.Title, + SeriesName: episode.SeriesTitle, + SeasonName: episode.SeasonTitle, + SeasonNumber: episode.SeasonNumber, + EpisodeNumber: episode.EpisodeNumber, + Resolution: format.Video.Resolution, + FPS: format.Video.FrameRate, + Audio: format.AudioLocale, + } + } else { + eps[episode.SeasonNumber][episode.EpisodeNumber].additionalFormats = append(eps[episode.SeasonNumber][episode.EpisodeNumber].additionalFormats, format) + } + } + } + } + + var infoFormat [][]formatInformation + for _, e := range eps { + var tmpFormatInfo []formatInformation + + var keys []int + for episodeNumber := range e { + keys = append(keys, episodeNumber) + } + sort.Ints(keys) + + for _, key := range keys { + tmpFormatInfo = append(tmpFormatInfo, *e[key]) + } + + infoFormat = append(infoFormat, tmpFormatInfo) + } + + return infoFormat, nil +} + +type compress interface { + io.Closer + + NewFile(information formatInformation) (io.WriteCloser, error) +} + +func newGzipCompress(file *os.File) *tarCompress { + gw := gzip.NewWriter(file) + return &tarCompress{ + parent: gw, + dst: tar.NewWriter(gw), + } +} + +func newTarCompress(file *os.File) *tarCompress { + return &tarCompress{ + dst: tar.NewWriter(file), + } +} + +type tarCompress struct { + compress + + wg sync.WaitGroup + + parent *gzip.Writer + dst *tar.Writer +} + +func (tc *tarCompress) Close() error { + // we have to wait here in case the actual content isn't copied completely into the + // writer yet + tc.wg.Wait() + + var err, err2 error + if tc.parent != nil { + err2 = tc.parent.Close() + } + err = tc.dst.Close() + + if err != nil && err2 != nil { + // best way to show double errors at once that i've found + return fmt.Errorf("%v\n%v", err, err2) + } else if err == nil && err2 != nil { + err = err2 + } + + return err +} + +func (tc *tarCompress) NewFile(information formatInformation) (io.WriteCloser, error) { + rp, wp := io.Pipe() + go func() { + tc.wg.Add(1) + defer tc.wg.Done() + var buf bytes.Buffer + io.Copy(&buf, rp) + + header := &tar.Header{ + Name: filepath.Join(fmt.Sprintf("S%2d", information.SeasonNumber), information.Title), + ModTime: time.Now(), + Mode: 0644, + Typeflag: tar.TypeReg, + // fun fact: i did not set the size for quiet some time because i thought that it isn't + // required. well because of this i debugged this part for multiple hours because without + // proper size information only a tiny amount gets copied into the tar (or zip) writer. + // this is also the reason why the file content is completely copied into a buffer before + // writing it to the writer. i could bypass this and save some memory but this requires + // some rewriting and im nearly at the (planned) finish for version 2 so nah in the future + // maybe + Size: int64(buf.Len()), + } + tc.dst.WriteHeader(header) + io.Copy(tc.dst, &buf) + }() + return wp, nil +} + +func newZipCompress(file *os.File) *zipCompress { + return &zipCompress{ + dst: zip.NewWriter(file), + } +} + +type zipCompress struct { + compress + + wg sync.WaitGroup + + dst *zip.Writer +} + +func (zc *zipCompress) Close() error { + zc.wg.Wait() + return zc.dst.Close() +} + +func (zc *zipCompress) NewFile(information formatInformation) (io.WriteCloser, error) { + rp, wp := io.Pipe() + go func() { + zc.wg.Add(1) + defer zc.wg.Done() + + var buf bytes.Buffer + io.Copy(&buf, rp) + + header := &zip.FileHeader{ + Name: filepath.Join(fmt.Sprintf("S%2d", information.SeasonNumber), information.Title), + Modified: time.Now(), + Method: zip.Deflate, + UncompressedSize64: uint64(buf.Len()), + } + header.SetMode(0644) + + hw, _ := zc.dst.CreateHeader(header) + io.Copy(hw, &buf) + }() + + return wp, nil +} From c367636d96f923cdc074dc86df566e79024c061e Mon Sep 17 00:00:00 2001 From: bytedream Date: Tue, 22 Mar 2022 19:39:55 +0100 Subject: [PATCH 82/82] v2 update --- Makefile | 11 +- README.md | 311 +++++++++++++---------------------------------- crunchyroll-go.1 | 120 +++++++++++++----- 3 files changed, 177 insertions(+), 265 deletions(-) diff --git a/Makefile b/Makefile index 8a26748..0744c35 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -VERSION=1.2.4 +VERSION=2.0.0 BINARY_NAME=crunchy VERSION_BINARY_NAME=$(BINARY_NAME)-v$(VERSION) @@ -21,13 +21,10 @@ uninstall: rm -f $(DESTDIR)$(PREFIX)/share/man/man1/crunchyroll-go.1 rm -f $(DESTDIR)$(PREFIX)/share/licenses/crunchyroll-go/LICENSE -test: - go test -v . - release: - cd cmd/crunchyroll-go && GOOS=linux GOARCH=amd64 go build -o $(VERSION_BINARY_NAME)_linux - cd cmd/crunchyroll-go && GOOS=windows GOARCH=amd64 go build -o $(VERSION_BINARY_NAME)_windows.exe - cd cmd/crunchyroll-go && GOOS=darwin GOARCH=amd64 go build -o $(VERSION_BINARY_NAME)_darwin + cd cmd/crunchyroll-go && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o $(VERSION_BINARY_NAME)_linux + cd cmd/crunchyroll-go && CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o $(VERSION_BINARY_NAME)_windows.exe + cd cmd/crunchyroll-go && CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -o $(VERSION_BINARY_NAME)_darwin strip cmd/crunchyroll-go/$(VERSION_BINARY_NAME)_linux diff --git a/README.md b/README.md index b19ab6f..709c3ae 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,8 @@ -**This branch is under highly development, so it may contain errors which are making compiling not possible** +

Version 2 is out 🥳, see all the changes.

# crunchyroll-go -A [Go](https://golang.org) library & cli for the undocumented [crunchyroll](https://www.crunchyroll.com) api. - -**You surely need a crunchyroll premium account to get full (api) access.** +A [Go](https://golang.org) library & cli for the undocumented [crunchyroll](https://www.crunchyroll.com) api. To use it, you need a crunchyroll premium account to for full (api) access.

@@ -32,20 +30,22 @@ A [Go](https://golang.org) library & cli for the undocumented [crunchyroll](http • Library 📚 • - Credits 🙏 - • - Notice 🗒️ + Disclaimer ☝️ • License ⚖

-## 🖥️ CLI +# 🖥️ CLI -#### ✨ Features -- Download single videos and entire series from [crunchyroll](https://www.crunchyroll.com) +## ✨ Features -#### Get the executable -- 📥 Download the latest binaries [here](https://github.com/ByteDream/crunchyroll-go/releases/latest) or get it from below +- Download single videos and entire series from [crunchyroll](https://www.crunchyroll.com). +- Archive episode or seasons in an `.mkv` file with multiple subtitles and audios and compress them to gzip or zip files. +- Specify a range which episodes to download from an anime. + +## 💾 Get the executable + +- 📥 Download the latest binaries [here](https://github.com/ByteDream/crunchyroll-go/releases/latest) or get it from below: - [Linux (x64)](https://smartrelease.bytedream.org/github/ByteDream/crunchyroll-go/crunchy-{tag}_linux) - [Windows (x64)](https://smartrelease.bytedream.org/github/ByteDream/crunchyroll-go/crunchy-{tag}_windows.exe) - [MacOS (x64)](https://smartrelease.bytedream.org/github/ByteDream/crunchyroll-go/crunchy-{tag}_darwin) @@ -54,22 +54,26 @@ A [Go](https://golang.org) library & cli for the undocumented [crunchyroll](http $ yay -S crunchyroll-go ``` - 🛠 Build it yourself - - use `make` (requires `go` to be installed): + - use `make` (requires `go` to be installed): ``` $ git clone https://github.com/ByteDream/crunchyroll-go $ cd crunchyroll-go $ make && sudo make install ``` - - use `go`: + - use `go`: ``` $ git clone https://github.com/ByteDream/crunchyroll-go $ cd crunchyroll-go/cmd/crunchyroll-go $ go build -o crunchy ``` -### 📝 Examples +## 📝 Examples + +_Before reading_: Because of the huge functionality not all cases can be covered in the README. +Make sure to check the [wiki](https://github.com/ByteDream/crunchyroll-go/wiki/Cli), further usages and options are described there. + +### Login -#### Login Before you can do something, you have to login first. This can be performed via crunchyroll account email and password. @@ -82,11 +86,9 @@ or via session id $ crunchy login --session-id 8e9gs135defhga790dvrf2i0eris8gts ``` -#### Download +### Download -**With the cli you can download single videos or entire series.** - -By default the cli tries to download the episode with your system language as audio. +By default, the cli tries to download the episode with your system language as audio. If no streams with your system language are available, the video will be downloaded with japanese audio and hardsubbed subtitles in your system language. **If your system language is not supported, an error message will be displayed and en-US (american english) will be chosen as language.** @@ -95,7 +97,6 @@ $ crunchy download https://www.crunchyroll.com/darling-in-the-franxx/episode-1-a ``` With `-r best` the video(s) will have the best available resolution (mostly 1920x1080 / Full HD). - ``` $ crunchy download -r best https://www.crunchyroll.com/darling-in-the-franxx/episode-1-alone-and-lonesome-759575 ``` @@ -105,250 +106,106 @@ The file is by default saved as a `.ts` (mpeg transport stream) file. With the `-o` flag, you can change the name (and file ending) of the output file. So if you want to save it as, for example, `mp4` file, just name it `whatever.mp4`. **You need [ffmpeg](https://ffmpeg.org) to store the video in other file formats.** - ``` $ crunchy download -o "daaaaaaaaaaaaaaaarling.ts" https://www.crunchyroll.com/darling-in-the-franxx/episode-1-alone-and-lonesome-759575 ``` With the `--audio` flag you can specify which audio the video should have and with `--subtitle` which subtitle it should have. Type `crunchy help download` to see all available locales. - ``` $ crunchy download --audio ja-JP --subtitle de-DE https://www.crunchyroll.com/darling-in-the-franxx ``` ##### Flags -- `--audio` » forces audio of the video(s) -- `--subtitle` » forces subtitle of the video(s) -- `--no-hardsub` » forces that the subtitles are stored as a separate file and are not directly embedded into the video -- `--only-sub` » downloads only the subtitles without the corresponding video -- `-d`, `--directory` » directory to download the video(s) to -- `-o`, `--output` » name of the output file +The following flags can be (optional) passed to modify the [download](#download) process. -- `-r`, `--resolution` » the resolution of the video(s). `best` for best resolution, `worst` for worst +| Short | Extended | Description | +|-------|----------------|--------------------------------------------------------------------------------| +| `-a` | `--audio` | Forces audio of the video(s). | +| `-s` | `--subtitle` | Forces subtitle of the video(s). | +| `-d` | `--directory` | Directory to download the video(s) to. | +| `-o` | `--output` | Name of the output file. | +| `-r` | `--resolution` | The resolution of the video(s). `best` for best resolution, `worst` for worst. | +| `-g` | `--goroutines` | Sets how many parallel segment downloads should be used. | -- `--alternative-progress` » shows an alternative, not so user-friendly progress instead of the progress bar +### Archive -- `-g`, `--goroutines` » sets how many parallel segment downloads should be used +Archive works just like [download](#download). It downloads the given videos as `.mkv` files and stores all (soft) subtitles in it. +Default audio locales are japanese and your system language (if available) but you can set more or less with the `--language` flag. + +Archive a file +```shell +$ crunchy archive https://www.crunchyroll.com/darling-in-the-franxx/darling-in-the-franxx/episode-1-alone-and-lonesome-759575 +``` + +Downloads the first two episode of Darling in the FranXX and stores it compressed in a file. +```shell +$ crunchy archive -c "ditf.tar.gz" https://www.crunchyroll.com/darling-in-the-franxx/darling-in-the-franxx +``` + +##### Flags + +The following flags can be (optional) passed to modify the [archive](#archive) process. + +| Short | Extended | Description | +|-------|----------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `-l` | `--language` | Audio locale which should be downloaded. Can be used multiple times. | +| `-d` | `--directory` | Directory to download the video(s) to. | +| `-o` | `--output` | Name of the output file. | +| `-m` | `--merge` | Sets the behavior of the stream merging. Valid behaviors are 'auto', 'audio', 'video'. See the [wiki](https://github.com/ByteDream/crunchyroll-go/wiki/Cli#archive) for more information. | +| `-c` | `--compress` | If is set, all output will be compresses into an archive. This flag sets the name of the compressed output file and the file ending specifies the compression algorithm (gzip, tar, zip are supported). | +| `-r` | `--resolution` | The resolution of the video(s). `best` for best resolution, `worst` for worst. | +| `-g` | `--goroutines` | Sets how many parallel segment downloads should be used. | + +### Help -#### Help - General help - ``` + ```shell $ crunchy help ``` - Login help - ``` + ```shell $ crunchy help login ``` - Download help - ``` + ```shell $ crunchy help download ``` -#### Global flags +- Archive help + ```shell + $ crunchy help archive + ``` + +### Global flags + These flags you can use across every sub-command -- `-q`, `--quiet` » disables all output -- `-v`, `--verbose` » shows additional debug output -- `--color` » adds color to the output (works only on not windows systems) +| Flag | Description | +|------|------------------------------------------------------| +| `-q` | Disables all output. | +| `-v` | Shows additional debug output. | +| `-p` | Use a proxy to hide your ip / redirect your traffic. | -- `-p`, `--proxy` » use a proxy to hide your ip / redirect your traffic +# 📚 Library -- `-l`, `--locale` » the language to display video specific things like the title. default is your system language - -## 📚 Library Download the library via `go get` - -``` +```shell $ go get github.com/ByteDream/crunchyroll-go ``` -### 📝 Examples -```go -func main() { - // login with credentials - crunchy, err := crunchyroll.LoginWithCredentials("user@example.com", "password", crunchyroll.US, http.DefaultClient) - if err != nil { - panic(err) - } +The documentation is available on [pkg.go.dev](https://pkg.go.dev/github.com/ByteDream/crunchyroll-go). - // finds a series or movie by a crunchyroll link - video, err := crunchy.FindVideo("https://www.crunchyroll.com/darling-in-the-franxx") - if err != nil { - panic(err) - } +Examples how to use the library and some features of it are described in the [wiki](https://github.com/ByteDream/crunchyroll-go/wiki/Library). - series := video.(*crunchyroll.Series) - seasons, err := series.Seasons() - if err != nil { - panic(err) - } - fmt.Printf("Found %d seasons for series %s\n", len(seasons), series.Title) +# ☝️ Disclaimer - // search `Darling` and return 20 results - series, movies, err := crunchy.Search("Darling", 20) - if err != nil { - panic(err) - } - fmt.Printf("Found %d series and %d movies for query `Darling`\n", len(series), len(movies)) -} -``` +This tool is **ONLY** meant to be used for private purposes. +To use this tool you need crunchyroll premium anyway, so there is no reason why rip and share the episodes. -```go -func main() { - crunchy, err := crunchyroll.LoginWithSessionID("8e9gs135defhga790dvrf2i0eris8gts", crunchyroll.US, http.DefaultClient) - if err != nil { - panic(err) - } - - // returns an episode slice with all episodes which are matching the given url. - // the episodes in the returning slice differs from the underlying streams, but are all pointing to the first ditf episode - episodes, err := crunchy.FindEpisode("https://www.crunchyroll.com/darling-in-the-franxx/episode-1-alone-and-lonesome-759575") - if err != nil { - panic(err) - } - fmt.Printf("Found %d episodes\n", len(episodes)) -} -``` - -

Structure

- -Because of the apis structure, it can lead very fast much redundant code for simple tasks, like getting all episodes -with japanese audio and german subtitle. For this case and some other, the api has a utility called `Structure` in its utils. - -```go -func main() { - crunchy, err := crunchyroll.LoginWithCredentials("user@example.com", "password", crunchyroll.US, http.DefaultClient) - if err != nil { - panic(err) - } - - // search `Darling` and return 20 results (series and movies) or less - series, movies, err := crunchy.Search("Darling", 20) - if err != nil { - panic(err) - } - fmt.Printf("Found %d series and %d movies for search query `Darling`\n", len(series), len(movies)) - - seasons, err := series[0].Seasons() - if err != nil { - panic(err) - } - - // in the crunchyroll.utils package, you find some structs which can be used to simplify tasks. - // you can recursively search all underlying content - seriesStructure := utils.NewSeasonStructure(seasons) - - // this returns every format of all the above given seasons - formats, err := seriesStructure.Formats() - if err != nil { - panic(err) - } - fmt.Printf("Found %d formats\n", len(formats)) - - filteredFormats, err := seriesStructure.FilterFormatsByLocales(crunchyroll.JP, crunchyroll.DE, true) - if err != nil { - panic(err) - } - fmt.Printf("Found %d formats with japanese audio and hardsubbed german subtitles\n", len(filteredFormats)) - - // reverse sorts the formats after their resolution by calling a sort type which is also defined in the api utils - // and stores the format with the highest resolution in a variable - sort.Sort(sort.Reverse(utils.FormatsByResolution(filteredFormats))) - format := formats[0] - // get the episode from which the format is a child - episode, err := seriesStructure.FilterEpisodeByFormat(format) - if err != nil { - panic(err) - } - - file, err := os.Create(fmt.Sprintf("%s.ts", episode.Title)) - if err != nil { - panic(err) - } - - // download the format to the file - if err := format.DownloadGoroutines(file, 4, nil); err != nil { - panic(err) - } - fmt.Printf("Downloaded %s with %s resolution and %.2f fps as %s\n", episode.Title, format.Video.Resolution, format.Video.FPS, file.Name()) - - // for more useful structure function just let your IDE's autocomplete make its thing -} -``` - -As you can see in the example above, most of the `crunchyroll.utils` Structure functions are returning errors. There is -a build-in functionality with are avoiding causing the most errors and let you safely ignore them as well. -**Note that errors still can appear** - -```go -func main() { - crunchy, err := crunchyroll.LoginWithCredentials("user@example.com", "password", crunchyroll.US, http.DefaultClient) - if err != nil { - panic(err) - } - - foundEpisodes, err := crunchy.FindEpisode("https://www.crunchyroll.com/darling-in-the-franxx/episode-1-alone-and-lonesome-759575") - if err != nil { - panic(err) - } - episodeStructure := utils.NewEpisodeStructure(foundEpisodes) - - // this function recursively calls all api endpoints, receives everything and stores it in memory, - // so that after executing this, no more request to the crunchyroll server has to be made. - // note that it could cause much network load while running this method. - // - // you should check the InitAllState before, because InitAll could have been already called or - // another function has the initialization as side effect and re-initializing everything - // will change every pointer in the struct which can cause massive problems afterwards. - if !episodeStructure.InitAllState() { - if err := episodeStructure.InitAll(); err != nil { - panic(err) - } - } - - formats, _ := episodeStructure.Formats() - streams, _ := episodeStructure.Streams() - episodes, _ := episodeStructure.Episodes() - fmt.Printf("Initialized %d formats, %d streams and %d episodes\n", len(formats), len(streams), len(episodes)) -} -``` - -### Tests -You can also run test to see if the api works correctly. -Before doing this, make sure to either set your crunchyroll email and password or sessions as environment variable. -The email variable has to be named `EMAIL` and the password variable `PASSWORD`. If you want to use your session id, the variable must be named `SESSION_ID`. - -You can run the test via `make` -``` -$ make test -``` - -or via `go` directly -``` -$ go test . -``` - -# 🙏 Credits - -### [Kamyroll-Python](https://github.com/hyugogirubato/Kamyroll-Python) -- Extracted all api endpoints and the login process from this - -### [m3u8](https://github.com/oopsguy/m3u8) -- Decrypting mpeg stream files - -### All libraries -- [m3u8](https://github.com/grafov/m3u8) (not the m3u8 library from above) » mpeg stream info library -- [cobra](https://github.com/spf13/cobra) » cli library - -# 🗒️ Notice - -Sometimes the download stops without a reason on linux and does not go further. In this case the `tmpfs` / `/tmp` directory may be full. Execute `df /tmp` to see how much of the space is used. - -I would really appreciate if someone rewrites the complete cli. I'm not satisfied with it's current structure but at the moment I have no time and no desire to do it myself. +**The responsibility for what happens to the downloaded videos lies entirely with the user who downloaded them.** # ⚖ License -This project is licensed under the GNU Lesser General Public License v3.0 (LGPL-3.0) - see the [LICENSE](LICENSE) file -for more details. +This project is licensed under the GNU Lesser General Public License v3.0 (LGPL-3.0) - see the [LICENSE](LICENSE) file for more details. diff --git a/crunchyroll-go.1 b/crunchyroll-go.1 index 40c7f83..4054554 100644 --- a/crunchyroll-go.1 +++ b/crunchyroll-go.1 @@ -1,16 +1,18 @@ -.TH crunchyroll-go 1 "13 September 2021" "crunchyroll-go" "Crunchyroll Downloader" +.TH crunchyroll-go 1 "21 March 2022" "crunchyroll-go" "Crunchyroll Downloader" .SH NAME crunchyroll-go - A cli for downloading videos and entire series from crunchyroll. .SH SYNOPSIS -crunchyroll-go [\fB-h\fR] [\fB--color\fR] [\fB-p\fR \fIPROXY\fR] [\fB-q\fR] [\fB-v\fR] +crunchyroll-go [\fB-h\fR] [\fB-p\fR \fIPROXY\fR] [\fB-q\fR] [\fB-v\fR] .br crunchyroll-go help .br -crunchyroll-go login [\fB--session-id\fR \fISESSION_ID\fR] [\fIemail\fR, \fIpassword\fR] +crunchyroll-go login [\fB--persistent\fR] [\fB--session-id\fR \fISESSION_ID\fR] [\fIusername\fR, \fIpassword\fR] .br -crunchyroll-go download [\fB--alternative-progress\fR] [\fB-a\fR \fILOCALE\fR] [\fB-d\fR \fIDIRECTORY\fR] [\fB--no-hardsub\fR] [\fB-o\fR \fIOUTPUT\fR] [\fB-r\fR \fIRESOLUTION\fR] [\fB-s\fR \fILOCALE\fR] \fIURL…\fR +crunchyroll-go download [\fB-a\fR \fIAUDIO\fR] [\fB-s\fR \fISUBTITLE\fR] [\fB-d\fR \fIDIRECTORY\fR] [\fB-o\fR \fIOUTPUT\fR] [\fB-r\fR \fIRESOLUTION\fR] [\fB-g\fR \fIGOROUTINES\fR] \fIURLs...\fR +.br +crunchyroll-go archive [\fB-l\fR \fILANGUAGE\fR] [\fB-d\fR \fIDIRECTORY\fR] [\fB-o\fR \fIOUTPUT\fR] [\fB-m\fR \fIMERGE BEHAVIOR\fR] [\fB-c\fR \fICOMPRESS\fR] [\fB-r\fR \fIRESOLUTION\fR] [\fB-g\fR \fIGOROUTINES\fR] \fIURLs...\fR .SH DESCRIPTION .TP @@ -28,10 +30,6 @@ This options can be passed to every action. Shows help. .TP -\fB--color\fR -Shows the output in different colors which will make it easier to differ the output. -.TP - \fB-p, --proxy PROXY\fR Sets a proxy through which all traffic will be routed. .TP @@ -43,42 +41,40 @@ Disables all output. \fB-v, --verbose\fR Shows verbose output. -.SH LOGIN OPTIONS -This options can only be used when calling the \fIlogin\fR action. +.SH LOGIN COMMAND +This command logs in to crunchyroll and stores the session id or credentials on the drive. This needs to be done before calling other commands since they need a valid login to operate. +.TP + +\fB--persistent\fR +Stores the given credentials permanent on the drive. The *nix path for it is $HOME/.config/crunchy. +.br +NOTE: The credentials are stored in plain text and if you not use \fB--session-id\fR your credentials are used (if you not use the \fB--persistent\fR flag only a session id gets stored regardless if you login with username/password or a session id). .TP \fB--session-id SESSION_ID\fR -Login via a session id (which can be extracted from a crunchyroll browser cookie) instead of using email and password. +Login via a session id (which can be extracted from a crunchyroll browser cookie) instead of using username and password. -.SH DOWNLOAD OPTIONS -This options can only be used when calling the \fIdownload\fR action. +.SH DOWNLOAD COMMAND +A command to simply download videos. The output file is stored as a \fI.ts\fR file. \fIffmpeg\fR has to be installed if you want to change the format the videos are stored in. .TP -\fB--alternative-progress\fR -Shows an alternative, not so user-friendly progress instead of the progress bar which contains more information. +\fB-a, --audio AUDIO\fR +Forces to download videos with the given audio locale. If no video with this audio locale is available, nothing will be downloaded. Available locales are: ja-JP, en-US, es-419, es-ES, fr-FR, pt-PT, pt-BR, it-IT, de-DE, ru-RU, ar-SA. .TP -\fB-a, --audio LOCALE\fR -Forces to download videos with the given audio locale. If no video with this audio locale is available, nothing will be downloaded. Available locales are: ja-JP, en-US, es-LA, es-ES, fr-FR, pt-BR, it-IT, de-DE, ru-RU, ar-ME. +\fB-s, --subtitle SUBTITLE\fR +Forces to download the videos with subtitles in the given locale / language. If no video with this subtitle locale is available, nothing will be downloaded. Available locales are: ja-JP, en-US, es-419, es-ES, fr-FR, pt-PT, pt-BR, it-IT, de-DE, ru-RU, ar-SA. .TP \fB-d, --directory DIRECTORY\fR The directory to download all files to. .TP -\fB--no-hardsub\fR -Same as '-s', but the subtitles are not stored in the video itself, but in a separate file. -.TP - -\fB--only-sub\fR -Downloads only the subtitles without the corresponding video. -.TP - \fB-o, --output OUTPUT\fR Name of the output file. Formatting is also supported, so if the name contains one or more of the following things, they will get replaced. {title} » Title of the video. {series_name} » Name of the series. - {season_title} » Title of the season. + {season_name} » Name of the season. {season_number} » Number of the season. {episode_number} » Number of the episode. {resolution} » Resolution of the video. @@ -94,12 +90,66 @@ The video resolution. Can either be specified via the pixels (e.g. 1920x1080), t Available common-use words: best (best available resolution), worst (worst available resolution). .TP -\fB-s, --subtitle LOCALE\fR -Forces to download the videos with subtitles in the given locale / language. If no video with this subtitle locale is available, nothing will be downloaded. Available locales are: ja-JP, en-US, es-LA, es-ES, fr-FR, pt-BR, it-IT, de-DE, ru-RU, ar-ME. +\fB-g, --goroutines GOROUTINES\fR +Sets the number of parallel downloads for the segments the final video is made of. Default is the number of cores the computer has. + +.SH ARCHIVE COMMAND +This command behaves like \fBdownload\fR besides the fact that it requires \fIffmpeg\fR and stores the output only to .mkv files. +.TP + +\fB-l, --language LANGUAGE\fR +Audio locales which should be downloaded. Can be used multiple times. Available locales are: ja-JP, en-US, es-419, es-ES, fr-FR, pt-PT, pt-BR, it-IT, de-DE, ru-RU, ar-SA. +.TP + +\fB-d, --directory DIRECTORY\fR +The directory to download all files to. +.TP + +\fB-o, --output OUTPUT\fR +Name of the output file. Formatting is also supported, so if the name contains one or more of the following things, they will get replaced. + {title} » Title of the video. + {series_name} » Name of the series. + {season_name} » Name of the season. + {season_number} » Number of the season. + {episode_number} » Number of the episode. + {resolution} » Resolution of the video. + {fps} » Frame Rate of the video. + {audio} » Audio locale of the video. + {subtitle} » Subtitle locale of the video. +.TP + +\fB-m, --merge MERGE BEHAVIOR\fR +Sets the behavior of the stream merging. Valid behaviors are 'auto', 'audio', 'video'. \fB--audio\fR stores one video and only the audio of all other languages, \fBvideo\fR stores all videos of the given languages and their audio, \fBauto\fR (which is the default) only behaves like video if the length of two videos are different (and only for the two videos), else like audio. +.TP + +\fB-c, --compress COMPRESS\fR +If is set, all output will be compresses into an archive (every url generates a new one). This flag sets the name of the compressed output file. The file ending specifies the compression algorithm. The following algorithms are supported: gzip, tar, zip. +Just like \fB--output\fR the name can be formatted. But the only option available here is \fI{series_name}\fR. +.TP + +\fB-r, resolution RESOLUTION\fR +The video resolution. Can either be specified via the pixels (e.g. 1920x1080), the abbreviation for pixels (e.g. 1080p) or "common-use" words (e.g. best). + Available pixels: 1920x1080, 1280x720, 640x480, 480x360, 426x240. + Available abbreviations: 1080p, 720p, 480p, 360p, 240p. + Available common-use words: best (best available resolution), worst (worst available resolution). .TP \fB-g, --goroutines GOROUTINES\fR -Sets the number of parallel downloads for the segments the final video is made of. +Sets the number of parallel downloads for the segments the final video is made of. Default is the number of cores the computer has. + +.SH URL OPTIONS +If you want to download only specific episode of a series, you could either pass every single episode url to the downloader (which is fine for 1 - 3 episodes) or use filtering. +It works pretty simple, just put a specific pattern surrounded by square brackets at the end of the url from the anime you want to download. A season and / or episode as well as a range from where to where episodes should be downloaded can be specified. +Use the list below to get a better overview what is possible + ...[E5] - Download the fifth episode. + ...[S1] - Download the full first season. + ...[-S2] - Download all seasons up to and including season 2. + ...[S3E4-] - Download all episodes from and including season 3, episode 4. + ...[S1E4-S3] - Download all episodes from and including season 1, episode 4, until and including season 3. + +In practise, it would look like this: \fIhttps://beta.crunchyroll.com/series/12345678/example[S1E5-S3E2]\fR. + +The \fBS\fR, followed by the number indicates the season number, \fBE\fR, followed by the number indicates an episode number. It doesn't matter if \fBS\fR, \fBE\fR or both are missing. Theoretically \fB[-]\fR is a valid pattern too. Note that \fBS\fR must always stay before \fBE\fR when used. .SH EXAMPLES Login via crunchyroll account email and password. @@ -116,7 +166,15 @@ $ crunchyroll-go download -o "darling.mp4" -r 720p https://www.crunchyroll.com/d Download a episode with japanese audio and american subtitles. .br -$ crunchyroll-go download -a ja-JP -s en-US https://www.crunchyroll.com/darling-in-the-franxx/episode-1-alone-and-lonesome-759575 +$ crunchyroll-go download -a ja-JP -s en-US https://www.crunchyroll.com/darling-in-the-franxx[E3-E5] + +Stores the episode in a .mkv file. +.br +$ crunchyroll-go archive https://www.crunchyroll.com/darling-in-the-franxx/darling-in-the-franxx/episode-1-alone-and-lonesome-759575 + +Downloads the first two episode of Darling in the FranXX and stores it compressed in a file. +.br +$ crunchyroll-go archive -c "ditf.tar.gz" https://www.crunchyroll.com/darling-in-the-franxx/darling-in-the-franxx[E1-E2] .SH BUGS If you notice any bug or want an enhancement, feel free to create a new issue or pull request in the GitHub repository. @@ -127,7 +185,7 @@ ByteDream Source: https://github.com/ByteDream/crunchyroll-go .SH COPYRIGHT -Copyright (C) 2021 ByteDream +Copyright (C) 2022 ByteDream This program is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public