Implement download queue (default size 2) and cleanup old entries after a while

CHANGELOG.md

@@ -4,6 +4,21 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+## [v0.5.3] - 2021-11-21
+
+- Add config option to limit number of simultaneous downloads
+- Remove old download entries from the index after they are complete
+
+## [v0.5.2] - 2021-10-26
+
+- Provide link to re-display the popup window from the index
+- Visual improvements
+
+## [v0.5.1] - 2021-10-25
+
+- Add note about adblockers potentially blocking the popup
+- Make it possible to refresh the popup window without initiating a new download
+
 ## [v0.5.0] - 2021-10-01
 
 - No more command line options, all configuration is now app-managed

config/config.go

@@ -12,9 +12,10 @@ import (
 )
 
 type Server struct {
-	Port         int    `yaml:"port" json:"port"`
+	Port                   int    `yaml:"port" json:"port"`
-	Address      string `yaml:"address" json:"address"`
+	Address                string `yaml:"address" json:"address"`
-	DownloadPath string `yaml:"download_path" json:"download_path"`
+	DownloadPath           string `yaml:"download_path" json:"download_path"`
+	MaximumActiveDownloads int    `yaml:"maximum_active_downloads_per_domain" json:"maximum_active_downloads_per_domain"`
 }
 
 type DownloadProfile struct {
@@ -35,6 +36,12 @@ type Config struct {
 	DownloadProfiles []DownloadProfile `yaml:"profiles" json:"profiles"`
 }
 
+func TestConfig() *Config {
+	config := DefaultConfig()
+	config.DownloadProfiles = []DownloadProfile{{Name: "test profile", Command: "sleep", Args: []string{"5"}}}
+	return config
+}
+
 func DefaultConfig() *Config {
 	defaultConfig := Config{}
 	stdProfile := DownloadProfile{Name: "standard video", Command: "youtube-dl", Args: []string{
@@ -60,7 +67,9 @@ func DefaultConfig() *Config {
 	defaultConfig.UI.PopupWidth = 500
 	defaultConfig.UI.PopupHeight = 500
 
-	defaultConfig.ConfigVersion = 1
+	defaultConfig.Server.MaximumActiveDownloads = 2
+
+	defaultConfig.ConfigVersion = 2
 
 	return &defaultConfig
 }
@@ -104,6 +113,10 @@ func (c *Config) UpdateFromJSON(j []byte) error {
 		return fmt.Errorf("path '%s' is not a directory", newConfig.Server.DownloadPath)
 	}
 
+	if newConfig.Server.MaximumActiveDownloads < 0 {
+		return fmt.Errorf("maximum active downloads can not be < 0")
+	}
+
 	// check profile name uniqueness
 	for i, p1 := range newConfig.DownloadProfiles {
 		for j, p2 := range newConfig.DownloadProfiles {
@@ -189,6 +202,21 @@ func LoadConfig() (*Config, error) {
 		log.Printf("Could not parse YAML config '%s': %v", path, err)
 		return nil, err
 	}
+
+	// do migrations
+	configMigrated := false
+	if c.ConfigVersion == 1 {
+		c.Server.MaximumActiveDownloads = 2
+		c.ConfigVersion = 2
+		configMigrated = true
+		log.Print("migrated config from version 1 => 2")
+	}
+
+	if configMigrated {
+		log.Print("Writing new config after version migration")
+		c.WriteConfig()
+	}
+
 	return &c, nil
 }
 

download/download.go

@@ -8,6 +8,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/tardisx/gropple/config"
 )
@@ -15,11 +16,13 @@ import (
 type Download struct {
 	Id              int                    `json:"id"`
 	Url             string                 `json:"url"`
+	PopupUrl        string                 `json:"popup_url"`
 	Pid             int                    `json:"pid"`
 	ExitCode        int                    `json:"exit_code"`
 	State           string                 `json:"state"`
 	DownloadProfile config.DownloadProfile `json:"download_profile"`
 	Finished        bool                   `json:"finished"`
+	FinishedTS      time.Time              `json:"finished_ts"`
 	Files           []string               `json:"files"`
 	Eta             string                 `json:"eta"`
 	Percent         float32                `json:"percent"`
@@ -27,12 +30,66 @@ type Download struct {
 	Config          *config.Config
 }
 
+type Downloads []*Download
+
+// StartQueued starts any downloads that have been queued, we would not exceed
+// maxRunning. If maxRunning is 0, there is no limit.
+func (dls Downloads) StartQueued(maxRunning int) {
+	active := 0
+	queued := 0
+
+	for _, dl := range dls {
+		if dl.State == "downloading" {
+			active++
+		}
+		if dl.State == "queued" {
+			queued++
+		}
+	}
+
+	// there is room, so start one
+	if queued > 0 && (active < maxRunning || maxRunning == 0) {
+		for _, dl := range dls {
+			if dl.State == "queued" {
+				dl.State = "downloading"
+				go func() { dl.Begin() }()
+				return
+			}
+		}
+	}
+
+}
+
+// Cleanup removes old downloads from the list. Hardcoded to remove them one hour
+// completion.
+func (dls Downloads) Cleanup() Downloads {
+	newDLs := Downloads{}
+	for _, dl := range dls {
+		if dl.Finished && time.Since(dl.FinishedTS) > time.Duration(time.Hour) {
+			// do nothing
+		} else {
+			newDLs = append(newDLs, dl)
+		}
+	}
+	return newDLs
+}
+
+// Queue queues a download
+func (dl *Download) Queue() {
+	dl.State = "queued"
+}
+
 // Begin starts a download, by starting the command specified in the DownloadProfile.
 // It blocks until the download is complete.
 func (dl *Download) Begin() {
+	dl.State = "downloading"
 	cmdSlice := []string{}
 	cmdSlice = append(cmdSlice, dl.DownloadProfile.Args...)
-	cmdSlice = append(cmdSlice, dl.Url)
+
+	// only add the url if it's not empty. This helps us with testing
+	if dl.Url != "" {
+		cmdSlice = append(cmdSlice, dl.Url)
+	}
 
 	cmd := exec.Command(dl.DownloadProfile.Command, cmdSlice...)
 	cmd.Dir = dl.Config.Server.DownloadPath
@@ -41,6 +98,7 @@ func (dl *Download) Begin() {
 	if err != nil {
 		dl.State = "failed"
 		dl.Finished = true
+		dl.FinishedTS = time.Now()
 		dl.Log = append(dl.Log, fmt.Sprintf("error setting up stdout pipe: %v", err))
 		return
 	}
@@ -49,6 +107,7 @@ func (dl *Download) Begin() {
 	if err != nil {
 		dl.State = "failed"
 		dl.Finished = true
+		dl.FinishedTS = time.Now()
 		dl.Log = append(dl.Log, fmt.Sprintf("error setting up stderr pipe: %v", err))
 		return
 	}
@@ -57,7 +116,8 @@ func (dl *Download) Begin() {
 	if err != nil {
 		dl.State = "failed"
 		dl.Finished = true
-		dl.Log = append(dl.Log, fmt.Sprintf("error starting youtube-dl: %v", err))
+		dl.FinishedTS = time.Now()
+		dl.Log = append(dl.Log, fmt.Sprintf("error starting command '%s': %v", dl.DownloadProfile.Command, err))
 		return
 	}
 	dl.Pid = cmd.Process.Pid
@@ -80,6 +140,7 @@ func (dl *Download) Begin() {
 
 	dl.State = "complete"
 	dl.Finished = true
+	dl.FinishedTS = time.Now()
 	dl.ExitCode = cmd.ProcessState.ExitCode()
 
 	if dl.ExitCode != 0 {
@@ -119,7 +180,7 @@ func (dl *Download) updateDownload(r io.Reader) {
 func (dl *Download) updateMetadata(s string) {
 
 	// [download]  49.7% of ~15.72MiB at  5.83MiB/s ETA 00:07
-	etaRE := regexp.MustCompile(`download.+ETA +(\d\d:\d\d)`)
+	etaRE := regexp.MustCompile(`download.+ETA +(\d\d:\d\d(?::\d\d)?)$`)
 	matches := etaRE.FindStringSubmatch(s)
 	if len(matches) == 2 {
 		dl.Eta = matches[1]

download/download_test.go

@@ -0,0 +1,99 @@
+package download
+
+import (
+	"testing"
+
+	"github.com/tardisx/gropple/config"
+)
+
+func TestUpdateMetadata(t *testing.T) {
+	newD := Download{}
+
+	// first time we spot a filename
+	newD.updateMetadata("[download] Destination: Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.f401.mp4")
+	if len(newD.Files) != 1 || newD.Files[0] != "Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.f401.mp4" {
+		t.Fatalf("incorrect Files:%v", newD.Files)
+	}
+
+	// eta's might be xx:xx:xx or xx:xx
+	newD.updateMetadata("[download]   0.0% of 504.09MiB at 135.71KiB/s ETA 01:03:36")
+	if newD.Eta != "01:03:36" {
+		t.Fatalf("bad long eta in dl\n%v", newD)
+	}
+	newD.updateMetadata("[download]   0.0% of 504.09MiB at 397.98KiB/s ETA 21:38")
+	if newD.Eta != "21:38" {
+		t.Fatalf("bad short eta in dl\n%v", newD)
+	}
+
+	// added a new file, now we are tracking two
+	newD.updateMetadata("[download] Destination: Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.f140.m4a")
+	if len(newD.Files) != 2 || newD.Files[1] != "Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.f140.m4a" {
+		t.Fatalf("incorrect Files:%v", newD.Files)
+	}
+
+	// merging
+	newD.updateMetadata("[ffmpeg] Merging formats into \"Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.mp4\"")
+	if len(newD.Files) != 3 || newD.Files[2] != "Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.mp4" {
+		t.Fatalf("did not find merged filename")
+		t.Fatalf("%v", newD.Files)
+	}
+
+	// deletes
+	// TODO. Not sure why I don't always see the "Deleting original file" messages after merge -
+	// maybe a youtube-dl fork thing?
+
+}
+
+// [youtube] wi7Agv1M6PY: Downloading webpage
+// [info] Writing video description metadata as JSON to: Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.info.json
+// [download] Destination: Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.f401.mp4
+// [download]   0.0% of 504.09MiB at 135.71KiB/s ETA 01:03:36
+// [download]   0.0% of 504.09MiB at 397.98KiB/s ETA 21:38
+// [download]   0.0% of 504.09MiB at 918.97KiB/s ETA 09:22
+// [download]   0.0% of 504.09MiB at  1.90MiB/s ETA 04:25
+// ..
+// [download]  99.6% of 504.09MiB at  8.91MiB/s ETA 00:00
+// [download] 100.0% of 504.09MiB at  9.54MiB/s ETA 00:00
+// [download] 100% of 504.09MiB in 01:00
+// [download] Destination: Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.f140.m4a
+// [download]   0.0% of 4.64MiB at 155.26KiB/s ETA 00:30
+// [download]   0.1% of 4.64MiB at 457.64KiB/s ETA 00:10
+// [download]   0.1% of 4.64MiB at  1.03MiB/s ETA 00:04
+// ..
+// [download]  86.2% of 4.64MiB at 10.09MiB/s ETA 00:00
+// [download] 100.0% of 4.64MiB at 10.12MiB/s ETA 00:00
+// [download] 100% of 4.64MiB in 00:00
+// [ffmpeg] Merging formats into "Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.mp4"
+
+// This test is a bit broken, because StartQueued immediately starts the queued
+// download, it
+func TestQueue(t *testing.T) {
+	conf := config.TestConfig()
+
+	new1 := Download{Id: 1, State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
+	new2 := Download{Id: 2, State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
+	new3 := Download{Id: 3, State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
+	new4 := Download{Id: 4, State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
+
+	dls := Downloads{&new1, &new2, &new3, &new4}
+	dls.StartQueued(1)
+	if dls[0].State == "queued" {
+		t.Error("#1 was not started")
+	}
+	if dls[1].State != "queued" {
+		t.Error("#2 is not queued")
+	}
+
+	// this should start no more, as one is still going
+	dls.StartQueued(1)
+	if dls[1].State != "queued" {
+		t.Error("#2 was started when it should not be")
+	}
+
+	dls.StartQueued(2)
+	if dls[1].State == "queued" {
+		t.Error("#2 was not started but it should be")
+
+	}
+
+}

main.go

@@ -19,11 +19,11 @@ import (
 	"github.com/tardisx/gropple/version"
 )
 
-var downloads []*download.Download
+var downloads download.Downloads
 var downloadId = 0
 var conf *config.Config
 
-var versionInfo = version.Info{CurrentVersion: "v0.5.0"}
+var versionInfo = version.Info{CurrentVersion: "v0.5.3"}
 
 //go:embed web
 var webFS embed.FS
@@ -55,6 +55,7 @@ func main() {
 	r.HandleFunc("/", homeHandler)
 	r.HandleFunc("/config", configHandler)
 	r.HandleFunc("/fetch", fetchHandler)
+	r.HandleFunc("/fetch/{id}", fetchHandler)
 
 	// info for the list
 	r.HandleFunc("/rest/fetch", fetchInfoRESTHandler)
@@ -81,6 +82,16 @@ func main() {
 		}
 	}()
 
+	// start downloading queued downloads when slots available, and clean up
+	// old entries
+	go func() {
+		for {
+			downloads.StartQueued(conf.Server.MaximumActiveDownloads)
+			downloads = downloads.Cleanup()
+			time.Sleep(time.Second)
+		}
+	}()
+
 	log.Printf("starting gropple %s - https://github.com/tardisx/gropple", versionInfo.CurrentVersion)
 	log.Printf("go to %s for details on installing the bookmarklet and to check status", conf.Server.Address)
 	log.Fatal(srv.ListenAndServe())
@@ -110,11 +121,13 @@ func homeHandler(w http.ResponseWriter, r *http.Request) {
 	type Info struct {
 		Downloads      []*download.Download
 		BookmarkletURL template.URL
+		Config         *config.Config
 	}
 
 	info := Info{
 		Downloads:      downloads,
 		BookmarkletURL: template.URL(bookmarkletURL),
+		Config:         conf,
 	}
 
 	err = t.ExecuteTemplate(w, "layout", info)
@@ -217,7 +230,7 @@ func fetchInfoOneRESTHandler(w http.ResponseWriter, r *http.Request) {
 				// set the profile
 				thisDownload.DownloadProfile = *profile
 
-				go func() { thisDownload.Begin() }()
+				thisDownload.Queue()
 				succRes := successResponse{Success: true, Message: "download started"}
 				succResB, _ := json.Marshal(succRes)
 				w.Write(succResB)
@@ -241,6 +254,32 @@ func fetchInfoRESTHandler(w http.ResponseWriter, r *http.Request) {
 
 func fetchHandler(w http.ResponseWriter, r *http.Request) {
 
+	// if they refreshed the popup, just load the existing object, don't
+	// create a new one
+	vars := mux.Vars(r)
+	idString := vars["id"]
+
+	idInt, err := strconv.ParseInt(idString, 10, 32)
+	if err == nil && idInt > 0 {
+		for _, dl := range downloads {
+			if dl.Id == int(idInt) {
+				t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/popup.html")
+				if err != nil {
+					panic(err)
+				}
+
+				templateData := map[string]interface{}{"dl": dl, "config": conf}
+
+				err = t.ExecuteTemplate(w, "layout", templateData)
+				if err != nil {
+					panic(err)
+				}
+				return
+			}
+		}
+
+	}
+
 	query := r.URL.Query()
 	url, present := query["url"]
 
@@ -265,6 +304,7 @@ func fetchHandler(w http.ResponseWriter, r *http.Request) {
 
 			Id:       downloadId,
 			Url:      url[0],
+			PopupUrl: fmt.Sprintf("/fetch/%d", downloadId),
 			State:    "choose profile",
 			Finished: false,
 			Eta:      "?",

web/config.html

@@ -33,6 +33,10 @@
                     <input type="text" id="config-server-downloadpath" placeholder="path" class="input-long" x-model="config.server.download_path" />
                     <span class="pure-form-message">The path on the server to download files to.</span>
 
+                    <label for="config-server-max-downloads">Maximum active downloads</label>
+                    <input type="text" id="config-server-max-downloads" placeholder="2" class="input-long" x-model.number="config.server.maximum_active_downloads_per_domain" />
+                    <span class="pure-form-message">How many downloads can be simultaneously active. Use '0' for no limit.</span>
+
                     <legend>UI</legend>
 
                     <p>Note that changes to the popup dimensions will require you to recreate your bookmarklet.</p>

web/index.html

@@ -2,7 +2,6 @@
 
 {{ template "menu.tmpl" . }}
 
-
 <div x-data="index()" x-init="fetch_data(); fetch_version()">
 
     <p x-cloak x-show="version && version.upgrade_available">
@@ -17,12 +16,15 @@
             Drag this bookmarklet: <a href="{{ .BookmarkletURL }}">Gropple</a> to your bookmark bar, and click it
             on any page you want to grab the video from.
         </p>
+	<p>
+	    Please note that some adblockers may prevent the bookmarklet from opening the popup window.
+	</p>
     </div>
 
     <table class="pure-table">
         <thead>
             <tr>
-                <th>id</th><th>filename</th><th>url</th><th>state</th><th>percent</th><th>eta</th><th>finished</th>
+		    <th>id</th><th>filename</th><th>url</th><th>show</th><th>state</th><th>percent</th><th>eta</th><th>finished</th>
             </tr>
         </thead>
         <tbody>
@@ -30,11 +32,12 @@
                 <tr>
                     <td x-text="item.id"></td>
                     <td x-text="item.files"></td>
-                    <td><a x-bind:href="item.url">link</a></td>
+                    <td><a class="int-link" x-bind:href="item.url">&#x2197;</a></td>
-                    <td x-text="item.state"></td>
+                    <td><a class="int-link" @click="show_popup(item)" href="#">&#x1F4C4;</a></td>
+                    <td :class="'state-'+item.state" x-text="item.state"></td>
                     <td x-text="item.percent"></td>
                     <td x-text="item.eta"></td>
-                    <td x-text="item.finished"></td>
+                    <td x-text="item.finished ? '&#x2714;' : '-'"></td>
                 </tr>
     
             </template>
@@ -75,7 +78,10 @@
                     setTimeout(() => { this.fetch_data() }, 1000);
                 })
             },
+            show_popup(item) {
+			    window.open(item.popup_url, item.id, "width={{ .Config.UI.PopupWidth }},height={{ .Config.UI.PopupHeight }}");
+	    },
         }
     }
 </script>
-{{ end }}
+{{ end }}

web/layout.tmpl

@@ -14,36 +14,48 @@
           overflow:auto;
         }
         footer {
-            padding-top: 50px;
+          padding-top: 50px;
-            font-size: 30%;
+          font-size: 30%;
+        }
+        .int-link {
+          text-decoration: none;
+          hover { color: red; }
+        }
+        .state-failed {
+          color: red;
+        }
+        .state-downloading {
+          color: blue;
+        }
+        .state-complete {
+          color: green;
         }
         .gropple-config {
-            font-size: 80%;
+          font-size: 80%;
         }
         .gropple-config input.input-long {
-            width: 27em;
+          width: 27em;
         }
         .gropple-config button {
-                    border-radius: 12px;
+          border-radius: 12px;
         }
         .gropple-config button.button-del {
-            background: rgb(202, 60, 60);
+          background: rgb(202, 60, 60);
         }
         .gropple-config button.button-add {
-            background: rgb(60, 200, 60);
+          background: rgb(60, 200, 60);
         }
         .gropple-config .pure-form-message {
-            padding-top: .5em;
+          padding-top: .5em;
-            padding-bottom: 1.5em;
+          padding-bottom: 1.5em;
         }
         .error {
-            color: red;
+          color: red;
         }
         .success {
-            color: green;
+          color: green;
         }
 
-
         [x-cloak] { display: none !important; }
 
     </style>
@@ -57,4 +69,4 @@
 </body>
     {{ template "js" . }}
 </html>
-{{ end }}
+{{ end }}

web/popup.html

@@ -30,11 +30,11 @@
 {{ define "js" }}
 <script>
     function popup() {
+        history.replaceState(null, '', ['/fetch/{{ .dl.Id }}'])
         return { 
             eta: '', percent: 0.0, state: '??', filename: '', finished: false, log :'',
             profile_chosen: null,
             watch_profile() {
-                console.log('will wtch profile');
                 this.$watch('profile_chosen', value => this.profile_chosen(value))
             },
             update_profile(name) {
@@ -57,6 +57,9 @@
                     this.eta = info.eta;
                     this.percent = info.percent + "%";
                     this.state = info.state;
+                    if (this.state != 'choose profile') {
+                        this.profile_chosen = true;
+                    }
                     this.finished = info.finished;
                     if (info.files && info.files.length > 0) {
                         this.filename = info.files[info.files.length - 1];