From 91c68d881667eeb7bf5befb2427b33c0fadc70aa Mon Sep 17 00:00:00 2001 From: Justin Hawkins Date: Sat, 9 Apr 2022 15:13:22 +0930 Subject: [PATCH] Show more information when downloading from playlists --- CHANGELOG.md | 6 ++ download/download.go | 20 ++++++ download/download_test.go | 129 ++++++++++++++++++++++++++++++++++++++ main.go | 6 +- web/layout.tmpl | 3 +- web/popup.html | 4 ++ 6 files changed, 165 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bc92a7..19e4694 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +## [v0.6.0] - 2022-04 + +- When downloading from a playlist, show the total number of videos and how many have been downloaded +- Update status when initially downloading playlist details +- Show version in web UI + ## [v0.5.5] - 2022-04-09 - Fix a bug which would erase configuration when migrating from v1 to v2 config diff --git a/download/download.go b/download/download.go index 7cc304b..43ab69e 100644 --- a/download/download.go +++ b/download/download.go @@ -28,6 +28,8 @@ type Download struct { Finished bool `json:"finished"` FinishedTS time.Time `json:"finished_ts"` Files []string `json:"files"` + PlaylistCurrent int `json:"playlist_current"` + PlaylistTotal int `json:"playlist_total"` Eta string `json:"eta"` Percent float32 `json:"percent"` Log []string `json:"log"` @@ -314,4 +316,22 @@ func (dl *Download) updateMetadata(s string) { } } } + + // [download] Downloading video 1 of 3 + playlistDetails := regexp.MustCompile(`Downloading video (\d+) of (\d+)`) + matches = playlistDetails.FindStringSubmatch(s) + if len(matches) == 3 { + total, _ := strconv.ParseInt(matches[2], 10, 32) + current, _ := strconv.ParseInt(matches[1], 10, 32) + dl.PlaylistTotal = int(total) + dl.PlaylistCurrent = int(current) + } + + // [Site] user: Downloading JSON metadata page 2 + metadataDL := regexp.MustCompile(`Downloading JSON metadata page (\d+)`) + matches = metadataDL.FindStringSubmatch(s) + if len(matches) == 2 { + dl.State = "Downloading metadata, page " + matches[1] + } + } diff --git a/download/download_test.go b/download/download_test.go index d12abe5..cd7bf70 100644 --- a/download/download_test.go +++ b/download/download_test.go @@ -1,6 +1,7 @@ package download import ( + "strings" "testing" "time" @@ -139,3 +140,131 @@ func TestQueue(t *testing.T) { } } + +func TestUpdateMetadataPlaylist(t *testing.T) { + + output := ` +start of log... +[download] Downloading playlist: niceuser +[RedGifsUser] niceuser: Downloading JSON metadata page 1 +[RedGifsUser] niceuser: Downloading JSON metadata page 2 +[RedGifsUser] niceuser: Downloading JSON metadata page 3 +[RedGifsUser] niceuser: Downloading JSON metadata page 4 +[RedGifsUser] niceuser: Downloading JSON metadata page 5 +[RedGifsUser] niceuser: Downloading JSON metadata page 6 +[info] Writing playlist metadata as JSON to: niceuser [niceuser].info.json +[RedGifsUser] playlist niceuser: Downloading 3 videos +[download] Downloading video 1 of 3 +[info] wrongpreciouschrysomelid: Downloading 1 format(s): hd +[info] Writing video metadata as JSON to: Splendid Wonderful Speaker Power Chocolate Drop [wrongpreciouschrysomelid].info.json +[download] Destination: Splendid Wonderful Speaker Power Chocolate Drop [wrongpreciouschrysomelid].mp4 +[download] 0.0% of 4.96MiB at Unknown speed ETA Unknown +[download] 0.1% of 4.96MiB at 1.76MiB/s ETA 00:02 +[download] 20.1% of 4.96MiB at 7.28MiB/s ETA 00:00 +[download] 40.3% of 4.96MiB at 10.06MiB/s ETA 00:00 +[download] 80.6% of 4.96MiB at 14.93MiB/s ETA 00:00 +[download] 100% of 4.96MiB at 17.33MiB/s ETA 00:00 +[download] 100% of 4.96MiB in 00:00 +[download] Downloading video 2 of 3 +[info] silentnaughtyborzoi: Downloading 1 format(s): hd +[info] Writing video metadata as JSON to: Splendid Printer Tray Computer Outdoor Window Wonderful [silentnaughtyborzoi].info.json +[download] Destination: Splendid Printer Tray Computer Outdoor Window Wonderful [silentnaughtyborzoi].mp4 +[download] 0.0% of 5.81MiB at 896.03KiB/s ETA 00:06 +[download] 0.1% of 5.81MiB at 1.28MiB/s ETA 00:04 +[download] 0.1% of 5.81MiB at 1.59MiB/s ETA 00:03 +[download] 34.4% of 5.81MiB at 9.90MiB/s ETA 00:00 +[download] 68.8% of 5.81MiB at 12.49MiB/s ETA 00:00 +[download] 100% of 5.81MiB at 15.77MiB/s ETA 00:00 +[download] 100% of 5.81MiB in 00:00 +[download] Downloading video 3 of 3 +[info] mammothremarkablewhooper: Downloading 1 format(s): hd +[info] Writing video metadata as JSON to: Porthole Splendid Close Up Gun Gunshot Window Wonderful [mammothremarkablewhooper].info.json +[download] Destination: Porthole Splendid Close Up Gun Gunshot Window Wonderful [mammothremarkablewhooper].mp4 +[download] 0.0% of 2.89MiB at Unknown speed ETA Unknown +[download] 0.1% of 2.89MiB at 1.77MiB/s ETA 00:01 +[download] 0.2% of 2.89MiB at 2.26MiB/s ETA 00:01 +[download] 34.5% of 2.89MiB at 8.23MiB/s ETA 00:00 +[download] 69.1% of 2.89MiB at 11.63MiB/s ETA 00:00 +[download] 100% of 2.89MiB at 14.25MiB/s ETA 00:00 +[download] 100% of 2.89MiB in 00:00 +[info] Writing updated playlist metadata as JSON to: niceuser [niceuser].info.json +[download] Finished downloading playlist: niceuser +` + newD := Download{} + + lines := strings.Split(output, "\n") + for _, l := range lines { + // t.Log(l) + newD.updateMetadata(l) + } + + if len(newD.Files) != 3 { + t.Errorf("%d files, not 3", len(newD.Files)) + } else { + if newD.Files[0] != "Splendid Wonderful Speaker Power Chocolate Drop [wrongpreciouschrysomelid].mp4" { + t.Error("Wrong 1st file") + } + if newD.Files[1] != "Splendid Printer Tray Computer Outdoor Window Wonderful [silentnaughtyborzoi].mp4" { + t.Error("Wrong 2nd file") + } + if newD.Files[2] != "Porthole Splendid Close Up Gun Gunshot Window Wonderful [mammothremarkablewhooper].mp4" { + t.Error("Wrong 3rd file") + } + } + + if newD.PlaylistTotal != 3 { + t.Errorf("playlist has total %d should be 3", newD.PlaylistTotal) + } + +} + +func TestUpdateMetadataSingle(t *testing.T) { + + output := ` +[youtube] 2WoDQBhJCVQ: Downloading webpage +[youtube] 2WoDQBhJCVQ: Downloading android player API JSON +[info] 2WoDQBhJCVQ: Downloading 1 format(s): 137+140 +[info] Writing video metadata as JSON to: The Greatest Shot In Television [2WoDQBhJCVQ].info.json +[download] Destination: The Greatest Shot In Television [2WoDQBhJCVQ].f137.mp4 +[download] 0.0% of 12.82MiB at 510.94KiB/s ETA 00:26 +[download] 0.0% of 12.82MiB at 966.50KiB/s ETA 00:13 +[download] 0.1% of 12.82MiB at 1.54MiB/s ETA 00:08 +[download] 0.1% of 12.82MiB at 2.75MiB/s ETA 00:04 +[download] 0.2% of 12.82MiB at 1.30MiB/s ETA 00:09 +[download] 77.5% of 12.82MiB at 2.54MiB/s ETA 00:01 +[download] 79.4% of 12.82MiB at 3.89MiB/s ETA 00:00 +[download] 83.3% of 12.82MiB at 6.44MiB/s ETA 00:00 +[download] 91.1% of 12.82MiB at 10.28MiB/s ETA 00:00 +[download] 100% of 12.82MiB at 12.77MiB/s ETA 00:00 +[download] 100% of 12.82MiB in 00:01 +[download] Destination: The Greatest Shot In Television [2WoDQBhJCVQ].f140.m4a +[download] 0.1% of 1.10MiB at 286.46KiB/s ETA 00:03 +[download] 0.3% of 1.10MiB at 716.49KiB/s ETA 00:01 +[download] 0.6% of 1.10MiB at 1.42MiB/s ETA 00:00 +[download] 91.0% of 1.10MiB at 6.67MiB/s ETA 00:00 +[download] 100% of 1.10MiB at 7.06MiB/s ETA 00:00 +[download] 100% of 1.10MiB in 00:00 +[Merger] Merging formats into "The Greatest Shot In Television [2WoDQBhJCVQ].mp4" +Deleting original file The Greatest Shot In Television [2WoDQBhJCVQ].f137.mp4 (pass -k to keep) +Deleting original file The Greatest Shot In Television [2WoDQBhJCVQ].f140.m4a (pass -k to keep) +` + newD := Download{} + + lines := strings.Split(output, "\n") + for _, l := range lines { + // t.Log(l) + newD.updateMetadata(l) + } + + if len(newD.Files) != 1 { + t.Errorf("%d files, not 1", len(newD.Files)) + } else { + if newD.Files[0] != "The Greatest Shot In Television [2WoDQBhJCVQ].mp4" { + t.Error("Wrong 1st file") + } + } + if newD.PlaylistTotal != 0 { + t.Error("playlist detected but should not be") + } + +} diff --git a/main.go b/main.go index 53414bd..d2595ef 100644 --- a/main.go +++ b/main.go @@ -23,7 +23,7 @@ var downloads download.Downloads var downloadId = 0 var configService *config.ConfigService -var versionInfo = version.Info{CurrentVersion: "v0.5.5"} +var versionInfo = version.Info{CurrentVersion: "v0.6.0"} //go:embed web var webFS embed.FS @@ -133,12 +133,14 @@ func homeHandler(w http.ResponseWriter, r *http.Request) { Downloads []*download.Download BookmarkletURL template.URL Config *config.Config + Version version.Info } info := Info{ Downloads: downloads, BookmarkletURL: template.URL(bookmarkletURL), Config: configService.Config, + Version: versionInfo, } err = t.ExecuteTemplate(w, "layout", info) @@ -350,7 +352,7 @@ func fetchHandler(w http.ResponseWriter, r *http.Request) { panic(err) } - templateData := map[string]interface{}{"dl": newDownload, "config": configService.Config, "canStop": download.CanStopDownload} + templateData := map[string]interface{}{"Version": versionInfo, "dl": newDownload, "config": configService.Config, "canStop": download.CanStopDownload} err = t.ExecuteTemplate(w, "layout", templateData) if err != nil { diff --git a/web/layout.tmpl b/web/layout.tmpl index dee3ecd..89e1ae7 100644 --- a/web/layout.tmpl +++ b/web/layout.tmpl @@ -71,7 +71,8 @@ {{ template "content" . }} {{ template "js" . }} diff --git a/web/popup.html b/web/popup.html index dfcfbe4..c1bd91d 100644 --- a/web/popup.html +++ b/web/popup.html @@ -16,6 +16,7 @@ current filename state + playlist progress progress ETA @@ -37,6 +38,7 @@ history.replaceState(null, '', ['/fetch/{{ .dl.Id }}']) return { eta: '', percent: 0.0, state: '??', filename: '', finished: false, log :'', + playlist_current: 0, playlist_total: 0, profile_chosen: null, watch_profile() { this.$watch('profile_chosen', value => this.profile_chosen(value)) @@ -73,6 +75,8 @@ this.eta = info.eta; this.percent = info.percent + "%"; this.state = info.state; + this.playlist_current = info.playlist_current; + this.playlist_total = info.playlist_total; if (this.state != 'choose profile') { this.profile_chosen = true; }