Implement download queue (default size 2) and cleanup old entries after a while

CHANGELOG.md

@@ -4,10 +4,20 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+## [v0.5.3] - 2021-11-21
+
+- Add config option to limit number of simultaneous downloads
+- Remove old download entries from the index after they are complete
+
+## [v0.5.2] - 2021-10-26
+
+- Provide link to re-display the popup window from the index
+- Visual improvements
+
 ## [v0.5.1] - 2021-10-25
 
 - Add note about adblockers potentially blocking the popup
-- Make it possible to refresh the popup window without initating a new download
+- Make it possible to refresh the popup window without initiating a new download
 
 ## [v0.5.0] - 2021-10-01
 

config/config.go

@@ -12,9 +12,10 @@ import (
 )
 
 type Server struct {
-	Port         int    `yaml:"port" json:"port"`
+	Port                   int    `yaml:"port" json:"port"`
-	Address      string `yaml:"address" json:"address"`
+	Address                string `yaml:"address" json:"address"`
-	DownloadPath string `yaml:"download_path" json:"download_path"`
+	DownloadPath           string `yaml:"download_path" json:"download_path"`
+	MaximumActiveDownloads int    `yaml:"maximum_active_downloads_per_domain" json:"maximum_active_downloads_per_domain"`
 }
 
 type DownloadProfile struct {
@@ -35,6 +36,12 @@ type Config struct {
 	DownloadProfiles []DownloadProfile `yaml:"profiles" json:"profiles"`
 }
 
+func TestConfig() *Config {
+	config := DefaultConfig()
+	config.DownloadProfiles = []DownloadProfile{{Name: "test profile", Command: "sleep", Args: []string{"5"}}}
+	return config
+}
+
 func DefaultConfig() *Config {
 	defaultConfig := Config{}
 	stdProfile := DownloadProfile{Name: "standard video", Command: "youtube-dl", Args: []string{
@@ -60,7 +67,9 @@ func DefaultConfig() *Config {
 	defaultConfig.UI.PopupWidth = 500
 	defaultConfig.UI.PopupHeight = 500
 
-	defaultConfig.ConfigVersion = 1
+	defaultConfig.Server.MaximumActiveDownloads = 2
+
+	defaultConfig.ConfigVersion = 2
 
 	return &defaultConfig
 }
@@ -104,6 +113,10 @@ func (c *Config) UpdateFromJSON(j []byte) error {
 		return fmt.Errorf("path '%s' is not a directory", newConfig.Server.DownloadPath)
 	}
 
+	if newConfig.Server.MaximumActiveDownloads < 0 {
+		return fmt.Errorf("maximum active downloads can not be < 0")
+	}
+
 	// check profile name uniqueness
 	for i, p1 := range newConfig.DownloadProfiles {
 		for j, p2 := range newConfig.DownloadProfiles {
@@ -189,6 +202,21 @@ func LoadConfig() (*Config, error) {
 		log.Printf("Could not parse YAML config '%s': %v", path, err)
 		return nil, err
 	}
+
+	// do migrations
+	configMigrated := false
+	if c.ConfigVersion == 1 {
+		c.Server.MaximumActiveDownloads = 2
+		c.ConfigVersion = 2
+		configMigrated = true
+		log.Print("migrated config from version 1 => 2")
+	}
+
+	if configMigrated {
+		log.Print("Writing new config after version migration")
+		c.WriteConfig()
+	}
+
 	return &c, nil
 }
 

download/download.go

@@ -8,6 +8,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/tardisx/gropple/config"
 )
@@ -15,11 +16,13 @@ import (
 type Download struct {
 	Id              int                    `json:"id"`
 	Url             string                 `json:"url"`
+	PopupUrl        string                 `json:"popup_url"`
 	Pid             int                    `json:"pid"`
 	ExitCode        int                    `json:"exit_code"`
 	State           string                 `json:"state"`
 	DownloadProfile config.DownloadProfile `json:"download_profile"`
 	Finished        bool                   `json:"finished"`
+	FinishedTS      time.Time              `json:"finished_ts"`
 	Files           []string               `json:"files"`
 	Eta             string                 `json:"eta"`
 	Percent         float32                `json:"percent"`
@@ -27,12 +30,66 @@ type Download struct {
 	Config          *config.Config
 }
 
+type Downloads []*Download
+
+// StartQueued starts any downloads that have been queued, we would not exceed
+// maxRunning. If maxRunning is 0, there is no limit.
+func (dls Downloads) StartQueued(maxRunning int) {
+	active := 0
+	queued := 0
+
+	for _, dl := range dls {
+		if dl.State == "downloading" {
+			active++
+		}
+		if dl.State == "queued" {
+			queued++
+		}
+	}
+
+	// there is room, so start one
+	if queued > 0 && (active < maxRunning || maxRunning == 0) {
+		for _, dl := range dls {
+			if dl.State == "queued" {
+				dl.State = "downloading"
+				go func() { dl.Begin() }()
+				return
+			}
+		}
+	}
+
+}
+
+// Cleanup removes old downloads from the list. Hardcoded to remove them one hour
+// completion.
+func (dls Downloads) Cleanup() Downloads {
+	newDLs := Downloads{}
+	for _, dl := range dls {
+		if dl.Finished && time.Since(dl.FinishedTS) > time.Duration(time.Hour) {
+			// do nothing
+		} else {
+			newDLs = append(newDLs, dl)
+		}
+	}
+	return newDLs
+}
+
+// Queue queues a download
+func (dl *Download) Queue() {
+	dl.State = "queued"
+}
+
 // Begin starts a download, by starting the command specified in the DownloadProfile.
 // It blocks until the download is complete.
 func (dl *Download) Begin() {
+	dl.State = "downloading"
 	cmdSlice := []string{}
 	cmdSlice = append(cmdSlice, dl.DownloadProfile.Args...)
-	cmdSlice = append(cmdSlice, dl.Url)
+
+	// only add the url if it's not empty. This helps us with testing
+	if dl.Url != "" {
+		cmdSlice = append(cmdSlice, dl.Url)
+	}
 
 	cmd := exec.Command(dl.DownloadProfile.Command, cmdSlice...)
 	cmd.Dir = dl.Config.Server.DownloadPath
@@ -41,6 +98,7 @@ func (dl *Download) Begin() {
 	if err != nil {
 		dl.State = "failed"
 		dl.Finished = true
+		dl.FinishedTS = time.Now()
 		dl.Log = append(dl.Log, fmt.Sprintf("error setting up stdout pipe: %v", err))
 		return
 	}
@@ -49,6 +107,7 @@ func (dl *Download) Begin() {
 	if err != nil {
 		dl.State = "failed"
 		dl.Finished = true
+		dl.FinishedTS = time.Now()
 		dl.Log = append(dl.Log, fmt.Sprintf("error setting up stderr pipe: %v", err))
 		return
 	}
@@ -57,7 +116,8 @@ func (dl *Download) Begin() {
 	if err != nil {
 		dl.State = "failed"
 		dl.Finished = true
-		dl.Log = append(dl.Log, fmt.Sprintf("error starting youtube-dl: %v", err))
+		dl.FinishedTS = time.Now()
+		dl.Log = append(dl.Log, fmt.Sprintf("error starting command '%s': %v", dl.DownloadProfile.Command, err))
 		return
 	}
 	dl.Pid = cmd.Process.Pid
@@ -80,6 +140,7 @@ func (dl *Download) Begin() {
 
 	dl.State = "complete"
 	dl.Finished = true
+	dl.FinishedTS = time.Now()
 	dl.ExitCode = cmd.ProcessState.ExitCode()
 
 	if dl.ExitCode != 0 {

download/download_test.go

@@ -1,6 +1,10 @@
 package download
 
-import "testing"
+import (
+	"testing"
+
+	"github.com/tardisx/gropple/config"
+)
 
 func TestUpdateMetadata(t *testing.T) {
 	newD := Download{}
@@ -60,3 +64,36 @@ func TestUpdateMetadata(t *testing.T) {
 // [download] 100.0% of 4.64MiB at 10.12MiB/s ETA 00:00
 // [download] 100% of 4.64MiB in 00:00
 // [ffmpeg] Merging formats into "Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.mp4"
+
+// This test is a bit broken, because StartQueued immediately starts the queued
+// download, it
+func TestQueue(t *testing.T) {
+	conf := config.TestConfig()
+
+	new1 := Download{Id: 1, State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
+	new2 := Download{Id: 2, State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
+	new3 := Download{Id: 3, State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
+	new4 := Download{Id: 4, State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
+
+	dls := Downloads{&new1, &new2, &new3, &new4}
+	dls.StartQueued(1)
+	if dls[0].State == "queued" {
+		t.Error("#1 was not started")
+	}
+	if dls[1].State != "queued" {
+		t.Error("#2 is not queued")
+	}
+
+	// this should start no more, as one is still going
+	dls.StartQueued(1)
+	if dls[1].State != "queued" {
+		t.Error("#2 was started when it should not be")
+	}
+
+	dls.StartQueued(2)
+	if dls[1].State == "queued" {
+		t.Error("#2 was not started but it should be")
+
+	}
+
+}

main.go

@@ -19,11 +19,11 @@ import (
 	"github.com/tardisx/gropple/version"
 )
 
-var downloads []*download.Download
+var downloads download.Downloads
 var downloadId = 0
 var conf *config.Config
 
-var versionInfo = version.Info{CurrentVersion: "v0.5.1"}
+var versionInfo = version.Info{CurrentVersion: "v0.5.3"}
 
 //go:embed web
 var webFS embed.FS
@@ -82,6 +82,16 @@ func main() {
 		}
 	}()
 
+	// start downloading queued downloads when slots available, and clean up
+	// old entries
+	go func() {
+		for {
+			downloads.StartQueued(conf.Server.MaximumActiveDownloads)
+			downloads = downloads.Cleanup()
+			time.Sleep(time.Second)
+		}
+	}()
+
 	log.Printf("starting gropple %s - https://github.com/tardisx/gropple", versionInfo.CurrentVersion)
 	log.Printf("go to %s for details on installing the bookmarklet and to check status", conf.Server.Address)
 	log.Fatal(srv.ListenAndServe())
@@ -111,11 +121,13 @@ func homeHandler(w http.ResponseWriter, r *http.Request) {
 	type Info struct {
 		Downloads      []*download.Download
 		BookmarkletURL template.URL
+		Config         *config.Config
 	}
 
 	info := Info{
 		Downloads:      downloads,
 		BookmarkletURL: template.URL(bookmarkletURL),
+		Config:         conf,
 	}
 
 	err = t.ExecuteTemplate(w, "layout", info)
@@ -218,7 +230,7 @@ func fetchInfoOneRESTHandler(w http.ResponseWriter, r *http.Request) {
 				// set the profile
 				thisDownload.DownloadProfile = *profile
 
-				go func() { thisDownload.Begin() }()
+				thisDownload.Queue()
 				succRes := successResponse{Success: true, Message: "download started"}
 				succResB, _ := json.Marshal(succRes)
 				w.Write(succResB)
@@ -292,6 +304,7 @@ func fetchHandler(w http.ResponseWriter, r *http.Request) {
 
 			Id:       downloadId,
 			Url:      url[0],
+			PopupUrl: fmt.Sprintf("/fetch/%d", downloadId),
 			State:    "choose profile",
 			Finished: false,
 			Eta:      "?",

web/config.html

@@ -33,6 +33,10 @@
                     <input type="text" id="config-server-downloadpath" placeholder="path" class="input-long" x-model="config.server.download_path" />
                     <span class="pure-form-message">The path on the server to download files to.</span>
 
+                    <label for="config-server-max-downloads">Maximum active downloads</label>
+                    <input type="text" id="config-server-max-downloads" placeholder="2" class="input-long" x-model.number="config.server.maximum_active_downloads_per_domain" />
+                    <span class="pure-form-message">How many downloads can be simultaneously active. Use '0' for no limit.</span>
+
                     <legend>UI</legend>
 
                     <p>Note that changes to the popup dimensions will require you to recreate your bookmarklet.</p>

web/index.html

@@ -2,7 +2,6 @@
 
 {{ template "menu.tmpl" . }}
 
-
 <div x-data="index()" x-init="fetch_data(); fetch_version()">
 
     <p x-cloak x-show="version && version.upgrade_available">
@@ -25,7 +24,7 @@
     <table class="pure-table">
         <thead>
             <tr>
-                <th>id</th><th>filename</th><th>url</th><th>state</th><th>percent</th><th>eta</th><th>finished</th>
+		    <th>id</th><th>filename</th><th>url</th><th>show</th><th>state</th><th>percent</th><th>eta</th><th>finished</th>
             </tr>
         </thead>
         <tbody>
@@ -33,11 +32,12 @@
                 <tr>
                     <td x-text="item.id"></td>
                     <td x-text="item.files"></td>
-                    <td><a x-bind:href="item.url">link</a></td>
+                    <td><a class="int-link" x-bind:href="item.url">&#x2197;</a></td>
-                    <td x-text="item.state"></td>
+                    <td><a class="int-link" @click="show_popup(item)" href="#">&#x1F4C4;</a></td>
+                    <td :class="'state-'+item.state" x-text="item.state"></td>
                     <td x-text="item.percent"></td>
                     <td x-text="item.eta"></td>
-                    <td x-text="item.finished"></td>
+                    <td x-text="item.finished ? '&#x2714;' : '-'"></td>
                 </tr>
     
             </template>
@@ -78,6 +78,9 @@
                     setTimeout(() => { this.fetch_data() }, 1000);
                 })
             },
+            show_popup(item) {
+			    window.open(item.popup_url, item.id, "width={{ .Config.UI.PopupWidth }},height={{ .Config.UI.PopupHeight }}");
+	    },
         }
     }
 </script>

web/layout.tmpl

@@ -14,36 +14,48 @@
           overflow:auto;
         }
         footer {
-            padding-top: 50px;
+          padding-top: 50px;
-            font-size: 30%;
+          font-size: 30%;
+        }
+        .int-link {
+          text-decoration: none;
+          hover { color: red; }
+        }
+        .state-failed {
+          color: red;
+        }
+        .state-downloading {
+          color: blue;
+        }
+        .state-complete {
+          color: green;
         }
         .gropple-config {
-            font-size: 80%;
+          font-size: 80%;
         }
         .gropple-config input.input-long {
-            width: 27em;
+          width: 27em;
         }
         .gropple-config button {
-                    border-radius: 12px;
+          border-radius: 12px;
         }
         .gropple-config button.button-del {
-            background: rgb(202, 60, 60);
+          background: rgb(202, 60, 60);
         }
         .gropple-config button.button-add {
-            background: rgb(60, 200, 60);
+          background: rgb(60, 200, 60);
         }
         .gropple-config .pure-form-message {
-            padding-top: .5em;
+          padding-top: .5em;
-            padding-bottom: 1.5em;
+          padding-bottom: 1.5em;
         }
         .error {
-            color: red;
+          color: red;
         }
         .success {
-            color: green;
+          color: green;
         }
 
-
         [x-cloak] { display: none !important; }
 
     </style>
@@ -57,4 +69,4 @@
 </body>
     {{ template "js" . }}
 </html>
-{{ end }}
+{{ end }}