diff --git a/CHANGELOG.md b/CHANGELOG.md index 49e968c..cc294e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +## [v0.5.3] - 2021-11-21 + +- Add config option to limit number of simultaneous downloads +- Remove old download entries from the index after they are complete + ## [v0.5.2] - 2021-10-26 - Provide link to re-display the popup window from the index @@ -12,7 +17,7 @@ All notable changes to this project will be documented in this file. ## [v0.5.1] - 2021-10-25 - Add note about adblockers potentially blocking the popup -- Make it possible to refresh the popup window without initating a new download +- Make it possible to refresh the popup window without initiating a new download ## [v0.5.0] - 2021-10-01 diff --git a/config/config.go b/config/config.go index e8c18fe..199cdf0 100644 --- a/config/config.go +++ b/config/config.go @@ -12,10 +12,10 @@ import ( ) type Server struct { - Port int `yaml:"port" json:"port"` - Address string `yaml:"address" json:"address"` - DownloadPath string `yaml:"download_path" json:"download_path"` - MaximumActiveDownloadsPerDomain int `yaml:"maximum_active_downloads_per_domain" json:"maximum_active_downloads_per_domain"` + Port int `yaml:"port" json:"port"` + Address string `yaml:"address" json:"address"` + DownloadPath string `yaml:"download_path" json:"download_path"` + MaximumActiveDownloads int `yaml:"maximum_active_downloads_per_domain" json:"maximum_active_downloads_per_domain"` } type DownloadProfile struct { @@ -36,6 +36,12 @@ type Config struct { DownloadProfiles []DownloadProfile `yaml:"profiles" json:"profiles"` } +func TestConfig() *Config { + config := DefaultConfig() + config.DownloadProfiles = []DownloadProfile{{Name: "test profile", Command: "sleep", Args: []string{"5"}}} + return config +} + func DefaultConfig() *Config { defaultConfig := Config{} stdProfile := DownloadProfile{Name: "standard video", Command: "youtube-dl", Args: []string{ @@ -61,7 +67,7 @@ func DefaultConfig() *Config { defaultConfig.UI.PopupWidth = 500 defaultConfig.UI.PopupHeight = 500 - defaultConfig.Server.MaximumActiveDownloadsPerDomain = 2 + defaultConfig.Server.MaximumActiveDownloads = 2 defaultConfig.ConfigVersion = 2 @@ -107,7 +113,7 @@ func (c *Config) UpdateFromJSON(j []byte) error { return fmt.Errorf("path '%s' is not a directory", newConfig.Server.DownloadPath) } - if newConfig.Server.MaximumActiveDownloadsPerDomain < 0 { + if newConfig.Server.MaximumActiveDownloads < 0 { return fmt.Errorf("maximum active downloads can not be < 0") } @@ -200,7 +206,7 @@ func LoadConfig() (*Config, error) { // do migrations configMigrated := false if c.ConfigVersion == 1 { - c.Server.MaximumActiveDownloadsPerDomain = 2 + c.Server.MaximumActiveDownloads = 2 c.ConfigVersion = 2 configMigrated = true log.Print("migrated config from version 1 => 2") diff --git a/download/download.go b/download/download.go index d03d17d..06db447 100644 --- a/download/download.go +++ b/download/download.go @@ -8,6 +8,7 @@ import ( "strconv" "strings" "sync" + "time" "github.com/tardisx/gropple/config" ) @@ -21,6 +22,7 @@ type Download struct { State string `json:"state"` DownloadProfile config.DownloadProfile `json:"download_profile"` Finished bool `json:"finished"` + FinishedTS time.Time `json:"finished_ts"` Files []string `json:"files"` Eta string `json:"eta"` Percent float32 `json:"percent"` @@ -28,12 +30,66 @@ type Download struct { Config *config.Config } +type Downloads []*Download + +// StartQueued starts any downloads that have been queued, we would not exceed +// maxRunning. If maxRunning is 0, there is no limit. +func (dls Downloads) StartQueued(maxRunning int) { + active := 0 + queued := 0 + + for _, dl := range dls { + if dl.State == "downloading" { + active++ + } + if dl.State == "queued" { + queued++ + } + } + + // there is room, so start one + if queued > 0 && (active < maxRunning || maxRunning == 0) { + for _, dl := range dls { + if dl.State == "queued" { + dl.State = "downloading" + go func() { dl.Begin() }() + return + } + } + } + +} + +// Cleanup removes old downloads from the list. Hardcoded to remove them one hour +// completion. +func (dls Downloads) Cleanup() Downloads { + newDLs := Downloads{} + for _, dl := range dls { + if dl.Finished && time.Since(dl.FinishedTS) > time.Duration(time.Hour) { + // do nothing + } else { + newDLs = append(newDLs, dl) + } + } + return newDLs +} + +// Queue queues a download +func (dl *Download) Queue() { + dl.State = "queued" +} + // Begin starts a download, by starting the command specified in the DownloadProfile. // It blocks until the download is complete. func (dl *Download) Begin() { + dl.State = "downloading" cmdSlice := []string{} cmdSlice = append(cmdSlice, dl.DownloadProfile.Args...) - cmdSlice = append(cmdSlice, dl.Url) + + // only add the url if it's not empty. This helps us with testing + if dl.Url != "" { + cmdSlice = append(cmdSlice, dl.Url) + } cmd := exec.Command(dl.DownloadProfile.Command, cmdSlice...) cmd.Dir = dl.Config.Server.DownloadPath @@ -42,6 +98,7 @@ func (dl *Download) Begin() { if err != nil { dl.State = "failed" dl.Finished = true + dl.FinishedTS = time.Now() dl.Log = append(dl.Log, fmt.Sprintf("error setting up stdout pipe: %v", err)) return } @@ -50,6 +107,7 @@ func (dl *Download) Begin() { if err != nil { dl.State = "failed" dl.Finished = true + dl.FinishedTS = time.Now() dl.Log = append(dl.Log, fmt.Sprintf("error setting up stderr pipe: %v", err)) return } @@ -58,7 +116,8 @@ func (dl *Download) Begin() { if err != nil { dl.State = "failed" dl.Finished = true - dl.Log = append(dl.Log, fmt.Sprintf("error starting youtube-dl: %v", err)) + dl.FinishedTS = time.Now() + dl.Log = append(dl.Log, fmt.Sprintf("error starting command '%s': %v", dl.DownloadProfile.Command, err)) return } dl.Pid = cmd.Process.Pid @@ -81,6 +140,7 @@ func (dl *Download) Begin() { dl.State = "complete" dl.Finished = true + dl.FinishedTS = time.Now() dl.ExitCode = cmd.ProcessState.ExitCode() if dl.ExitCode != 0 { diff --git a/download/download_test.go b/download/download_test.go index 1e75e44..39a9e0f 100644 --- a/download/download_test.go +++ b/download/download_test.go @@ -1,6 +1,10 @@ package download -import "testing" +import ( + "testing" + + "github.com/tardisx/gropple/config" +) func TestUpdateMetadata(t *testing.T) { newD := Download{} @@ -60,3 +64,36 @@ func TestUpdateMetadata(t *testing.T) { // [download] 100.0% of 4.64MiB at 10.12MiB/s ETA 00:00 // [download] 100% of 4.64MiB in 00:00 // [ffmpeg] Merging formats into "Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.mp4" + +// This test is a bit broken, because StartQueued immediately starts the queued +// download, it +func TestQueue(t *testing.T) { + conf := config.TestConfig() + + new1 := Download{Id: 1, State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf} + new2 := Download{Id: 2, State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf} + new3 := Download{Id: 3, State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf} + new4 := Download{Id: 4, State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf} + + dls := Downloads{&new1, &new2, &new3, &new4} + dls.StartQueued(1) + if dls[0].State == "queued" { + t.Error("#1 was not started") + } + if dls[1].State != "queued" { + t.Error("#2 is not queued") + } + + // this should start no more, as one is still going + dls.StartQueued(1) + if dls[1].State != "queued" { + t.Error("#2 was started when it should not be") + } + + dls.StartQueued(2) + if dls[1].State == "queued" { + t.Error("#2 was not started but it should be") + + } + +} diff --git a/main.go b/main.go index b3a3748..4852fdd 100644 --- a/main.go +++ b/main.go @@ -19,7 +19,7 @@ import ( "github.com/tardisx/gropple/version" ) -var downloads []*download.Download +var downloads download.Downloads var downloadId = 0 var conf *config.Config @@ -82,6 +82,16 @@ func main() { } }() + // start downloading queued downloads when slots available, and clean up + // old entries + go func() { + for { + downloads.StartQueued(conf.Server.MaximumActiveDownloads) + downloads = downloads.Cleanup() + time.Sleep(time.Second) + } + }() + log.Printf("starting gropple %s - https://github.com/tardisx/gropple", versionInfo.CurrentVersion) log.Printf("go to %s for details on installing the bookmarklet and to check status", conf.Server.Address) log.Fatal(srv.ListenAndServe()) @@ -220,7 +230,7 @@ func fetchInfoOneRESTHandler(w http.ResponseWriter, r *http.Request) { // set the profile thisDownload.DownloadProfile = *profile - go func() { thisDownload.Begin() }() + thisDownload.Queue() succRes := successResponse{Success: true, Message: "download started"} succResB, _ := json.Marshal(succRes) w.Write(succResB)