Implement download queue (default size 2) and cleanup old entries after a while

CHANGELOG.md

@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+## [v0.5.3] - 2021-11-21
+
+- Add config option to limit number of simultaneous downloads
+- Remove old download entries from the index after they are complete
+
 ## [v0.5.2] - 2021-10-26
 
 - Provide link to re-display the popup window from the index
@@ -12,7 +17,7 @@ All notable changes to this project will be documented in this file.
 ## [v0.5.1] - 2021-10-25
 
 - Add note about adblockers potentially blocking the popup
-- Make it possible to refresh the popup window without initating a new download
+- Make it possible to refresh the popup window without initiating a new download
 
 ## [v0.5.0] - 2021-10-01
 

config/config.go

@@ -12,9 +12,10 @@ import (
 )
 
 type Server struct {
-	Port         int    `yaml:"port" json:"port"`
-	Address      string `yaml:"address" json:"address"`
-	DownloadPath string `yaml:"download_path" json:"download_path"`
+	Port                   int    `yaml:"port" json:"port"`
+	Address                string `yaml:"address" json:"address"`
+	DownloadPath           string `yaml:"download_path" json:"download_path"`
+	MaximumActiveDownloads int    `yaml:"maximum_active_downloads_per_domain" json:"maximum_active_downloads_per_domain"`
 }
 
 type DownloadProfile struct {
@@ -35,6 +36,12 @@ type Config struct {
 	DownloadProfiles []DownloadProfile `yaml:"profiles" json:"profiles"`
 }
 
+func TestConfig() *Config {
+	config := DefaultConfig()
+	config.DownloadProfiles = []DownloadProfile{{Name: "test profile", Command: "sleep", Args: []string{"5"}}}
+	return config
+}
+
 func DefaultConfig() *Config {
 	defaultConfig := Config{}
 	stdProfile := DownloadProfile{Name: "standard video", Command: "youtube-dl", Args: []string{
@@ -60,7 +67,9 @@ func DefaultConfig() *Config {
 	defaultConfig.UI.PopupWidth = 500
 	defaultConfig.UI.PopupHeight = 500
 
-	defaultConfig.ConfigVersion = 1
+	defaultConfig.Server.MaximumActiveDownloads = 2
+
+	defaultConfig.ConfigVersion = 2
 
 	return &defaultConfig
 }
@@ -104,6 +113,10 @@ func (c *Config) UpdateFromJSON(j []byte) error {
 		return fmt.Errorf("path '%s' is not a directory", newConfig.Server.DownloadPath)
 	}
 
+	if newConfig.Server.MaximumActiveDownloads < 0 {
+		return fmt.Errorf("maximum active downloads can not be < 0")
+	}
+
 	// check profile name uniqueness
 	for i, p1 := range newConfig.DownloadProfiles {
 		for j, p2 := range newConfig.DownloadProfiles {
@@ -189,6 +202,21 @@ func LoadConfig() (*Config, error) {
 		log.Printf("Could not parse YAML config '%s': %v", path, err)
 		return nil, err
 	}
+
+	// do migrations
+	configMigrated := false
+	if c.ConfigVersion == 1 {
+		c.Server.MaximumActiveDownloads = 2
+		c.ConfigVersion = 2
+		configMigrated = true
+		log.Print("migrated config from version 1 => 2")
+	}
+
+	if configMigrated {
+		log.Print("Writing new config after version migration")
+		c.WriteConfig()
+	}
+
 	return &c, nil
 }
 

download/download.go

@@ -8,6 +8,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/tardisx/gropple/config"
 )
@@ -21,6 +22,7 @@ type Download struct {
 	State           string                 `json:"state"`
 	DownloadProfile config.DownloadProfile `json:"download_profile"`
 	Finished        bool                   `json:"finished"`
+	FinishedTS      time.Time              `json:"finished_ts"`
 	Files           []string               `json:"files"`
 	Eta             string                 `json:"eta"`
 	Percent         float32                `json:"percent"`
@@ -28,12 +30,66 @@ type Download struct {
 	Config          *config.Config
 }
 
+type Downloads []*Download
+
+// StartQueued starts any downloads that have been queued, we would not exceed
+// maxRunning. If maxRunning is 0, there is no limit.
+func (dls Downloads) StartQueued(maxRunning int) {
+	active := 0
+	queued := 0
+
+	for _, dl := range dls {
+		if dl.State == "downloading" {
+			active++
+		}
+		if dl.State == "queued" {
+			queued++
+		}
+	}
+
+	// there is room, so start one
+	if queued > 0 && (active < maxRunning || maxRunning == 0) {
+		for _, dl := range dls {
+			if dl.State == "queued" {
+				dl.State = "downloading"
+				go func() { dl.Begin() }()
+				return
+			}
+		}
+	}
+
+}
+
+// Cleanup removes old downloads from the list. Hardcoded to remove them one hour
+// completion.
+func (dls Downloads) Cleanup() Downloads {
+	newDLs := Downloads{}
+	for _, dl := range dls {
+		if dl.Finished && time.Since(dl.FinishedTS) > time.Duration(time.Hour) {
+			// do nothing
+		} else {
+			newDLs = append(newDLs, dl)
+		}
+	}
+	return newDLs
+}
+
+// Queue queues a download
+func (dl *Download) Queue() {
+	dl.State = "queued"
+}
+
 // Begin starts a download, by starting the command specified in the DownloadProfile.
 // It blocks until the download is complete.
 func (dl *Download) Begin() {
+	dl.State = "downloading"
 	cmdSlice := []string{}
 	cmdSlice = append(cmdSlice, dl.DownloadProfile.Args...)
-	cmdSlice = append(cmdSlice, dl.Url)
+
+	// only add the url if it's not empty. This helps us with testing
+	if dl.Url != "" {
+		cmdSlice = append(cmdSlice, dl.Url)
+	}
 
 	cmd := exec.Command(dl.DownloadProfile.Command, cmdSlice...)
 	cmd.Dir = dl.Config.Server.DownloadPath
@@ -42,6 +98,7 @@ func (dl *Download) Begin() {
 	if err != nil {
 		dl.State = "failed"
 		dl.Finished = true
+		dl.FinishedTS = time.Now()
 		dl.Log = append(dl.Log, fmt.Sprintf("error setting up stdout pipe: %v", err))
 		return
 	}
@@ -50,6 +107,7 @@ func (dl *Download) Begin() {
 	if err != nil {
 		dl.State = "failed"
 		dl.Finished = true
+		dl.FinishedTS = time.Now()
 		dl.Log = append(dl.Log, fmt.Sprintf("error setting up stderr pipe: %v", err))
 		return
 	}
@@ -58,7 +116,8 @@ func (dl *Download) Begin() {
 	if err != nil {
 		dl.State = "failed"
 		dl.Finished = true
-		dl.Log = append(dl.Log, fmt.Sprintf("error starting youtube-dl: %v", err))
+		dl.FinishedTS = time.Now()
+		dl.Log = append(dl.Log, fmt.Sprintf("error starting command '%s': %v", dl.DownloadProfile.Command, err))
 		return
 	}
 	dl.Pid = cmd.Process.Pid
@@ -81,6 +140,7 @@ func (dl *Download) Begin() {
 
 	dl.State = "complete"
 	dl.Finished = true
+	dl.FinishedTS = time.Now()
 	dl.ExitCode = cmd.ProcessState.ExitCode()
 
 	if dl.ExitCode != 0 {

download/download_test.go

@@ -1,6 +1,10 @@
 package download
 
-import "testing"
+import (
+	"testing"
+
+	"github.com/tardisx/gropple/config"
+)
 
 func TestUpdateMetadata(t *testing.T) {
 	newD := Download{}
@@ -60,3 +64,36 @@ func TestUpdateMetadata(t *testing.T) {
 // [download] 100.0% of 4.64MiB at 10.12MiB/s ETA 00:00
 // [download] 100% of 4.64MiB in 00:00
 // [ffmpeg] Merging formats into "Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.mp4"
+
+// This test is a bit broken, because StartQueued immediately starts the queued
+// download, it
+func TestQueue(t *testing.T) {
+	conf := config.TestConfig()
+
+	new1 := Download{Id: 1, State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
+	new2 := Download{Id: 2, State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
+	new3 := Download{Id: 3, State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
+	new4 := Download{Id: 4, State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
+
+	dls := Downloads{&new1, &new2, &new3, &new4}
+	dls.StartQueued(1)
+	if dls[0].State == "queued" {
+		t.Error("#1 was not started")
+	}
+	if dls[1].State != "queued" {
+		t.Error("#2 is not queued")
+	}
+
+	// this should start no more, as one is still going
+	dls.StartQueued(1)
+	if dls[1].State != "queued" {
+		t.Error("#2 was started when it should not be")
+	}
+
+	dls.StartQueued(2)
+	if dls[1].State == "queued" {
+		t.Error("#2 was not started but it should be")
+
+	}
+
+}

main.go

@@ -19,11 +19,11 @@ import (
 	"github.com/tardisx/gropple/version"
 )
 
-var downloads []*download.Download
+var downloads download.Downloads
 var downloadId = 0
 var conf *config.Config
 
-var versionInfo = version.Info{CurrentVersion: "v0.5.2"}
+var versionInfo = version.Info{CurrentVersion: "v0.5.3"}
 
 //go:embed web
 var webFS embed.FS
@@ -82,6 +82,16 @@ func main() {
 		}
 	}()
 
+	// start downloading queued downloads when slots available, and clean up
+	// old entries
+	go func() {
+		for {
+			downloads.StartQueued(conf.Server.MaximumActiveDownloads)
+			downloads = downloads.Cleanup()
+			time.Sleep(time.Second)
+		}
+	}()
+
 	log.Printf("starting gropple %s - https://github.com/tardisx/gropple", versionInfo.CurrentVersion)
 	log.Printf("go to %s for details on installing the bookmarklet and to check status", conf.Server.Address)
 	log.Fatal(srv.ListenAndServe())
@@ -220,7 +230,7 @@ func fetchInfoOneRESTHandler(w http.ResponseWriter, r *http.Request) {
 				// set the profile
 				thisDownload.DownloadProfile = *profile
 
-				go func() { thisDownload.Begin() }()
+				thisDownload.Queue()
 				succRes := successResponse{Success: true, Message: "download started"}
 				succResB, _ := json.Marshal(succRes)
 				w.Write(succResB)

web/config.html

@@ -33,6 +33,10 @@
                     <input type="text" id="config-server-downloadpath" placeholder="path" class="input-long" x-model="config.server.download_path" />
                     <span class="pure-form-message">The path on the server to download files to.</span>
 
+                    <label for="config-server-max-downloads">Maximum active downloads</label>
+                    <input type="text" id="config-server-max-downloads" placeholder="2" class="input-long" x-model.number="config.server.maximum_active_downloads_per_domain" />
+                    <span class="pure-form-message">How many downloads can be simultaneously active. Use '0' for no limit.</span>
+
                     <legend>UI</legend>
 
                     <p>Note that changes to the popup dimensions will require you to recreate your bookmarklet.</p>