Update changelog, skip docker for now until it is ready

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: tardisx

.vscode/settings.json

@@ -1,9 +1,10 @@
 {
     "cSpell.words": [
         "Cleanup",
+        "gropple",
+        "succ",
         "tmpl",
-        "vars",
-        "gropple"
+        "vars"
     ],
     "cSpell.language": "en-GB"
 }

CHANGELOG.md

@@ -4,6 +4,24 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+## [v0.6.0] - 2023-03-15
+
+- Configurable destinations for downloads
+  - Multiple destination directories can be configured
+  - When queueing a download, an alternate destination can be selected
+- When downloading from a playlist, show the total number of videos and how many have been downloaded
+- Show version in web UI
+- Improve index page (show URL of queued downloads instead of nothing)
+- Fixes and improvements to capturing output info and showing it in the UI
+- Show all log output in the popup
+- Fixes to handling of queued downloads
+- Fix portable mode to look in binary directory, not current directory
+- Automatically cleanup download list, removing old entries automatically
+
+## [v0.5.5] - 2022-04-09
+
+- Fix a bug which would erase configuration when migrating from v1 to v2 config
+
 ## [v0.5.4] - 2022-04-07
 
 - Check the chosen command exists when configuring a profile

Dockerfile

@@ -0,0 +1,47 @@
+# Start from golang base image
+FROM golang:alpine as builder
+
+# Install git. (alpine image does not have git in it)
+RUN apk update && apk add --no-cache git curl
+
+# Set current working directory
+WORKDIR /app
+
+RUN curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /app/yt-dlp
+RUN chmod a+x /app/yt-dlp
+
+# Note here: To avoid downloading dependencies every time we
+# build image. Here, we are caching all the dependencies by
+# first copying go.mod and go.sum files and downloading them,
+# to be used every time we build the image if the dependencies
+# are not changed.
+
+# Copy go mod and sum files
+COPY go.mod ./
+COPY go.sum ./
+
+# Download all dependencies.
+RUN go mod download
+
+# Now, copy the source code
+COPY . .
+
+# Note here: CGO_ENABLED is disabled for cross system compilation
+# It is also a common best practise.
+
+# Build the application.
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ./bin/gropple .
+
+# Finally our multi-stage to build a small image
+# Start a new stage from scratch
+FROM golang:alpine
+
+# Copy the Pre-built binary file
+COPY --from=builder /app/bin/gropple .
+COPY --from=builder /app/yt-dlp /bin/
+
+# Install things we need to support yt-dlp
+RUN apk update && apk add --no-cache python3 ffmpeg
+
+# Run executable
+CMD ["./gropple", "--config-path", "/config/gropple.json"]

README.md

@@ -16,9 +16,10 @@ A frontend to youtube-dl (or compatible forks, like yt-dlp) to download videos w
 
 ## Binaries
 
-Binaries are available at https://github.com/tardisx/gropple/releases
+Binaries are available at <https://github.com/tardisx/gropple/releases>
 
-Gropple will automatically check for available updates and prompt you to upgrade.
+Gropple will automatically check for available updates and prompt you to
+upgrade.
 
 ## Running
 
@@ -32,66 +33,75 @@ interface. The address will be printed after startup:
 
 ## Using
 
-Bring up `http://localhost:6283` (or your configured address) in your browser. You 
-should see a link to the bookmarklet at the top of the screen, and the list of
-downloads (currently empty).
+Bring up `http://localhost:6283` (or your configured address) in your browser.
+You should see a link to the bookmarklet at the top of the screen, and the list
+of downloads (currently empty).
 
-Drag the bookmarklet to your favourites bar, or otherwise bookmark it as you 
-see fit. Any kind of browser bookmark should work. The bookmarklet contains
-embedded javascript to pass the URL of whatever page you are currently on back
-to gropple.
+Drag the bookmarklet to your favourites bar, or otherwise bookmark it as you see
+fit. Any kind of browser bookmark should work. The bookmarklet contains embedded
+javascript to pass the URL of whatever page you are currently on back to
+gropple.
 
-So, whenever you are on a page with a video you would like to download just 
-click the bookmarklet.
+Whenever you are on a page with a video you would like to download just click
+the bookmarklet.
 
-A popup window will appear. Choose a download profile and the download will start.
-The status will be shown in the window, updating in real time.
+A popup window will appear. Choose a download profile and the download will
+start. The status will be shown in the window, updating in real time.
 
 You may close this window at any time without stopping the download, the status
 of all downloads is available on the index page.
 
 ## Configuration
 
-Click the "config" link on the index page to configure gropple. The default options 
-are fine if you are running on your local machine. If you are running it remotely
-you will need to set the "server address" to ensure the bookmarklet has the correct
-URL in it.
+Click the "config" link on the index page to configure gropple. The default
+options are fine if you are running on your local machine. If you are running it
+remotely you will need to set the "server address" to ensure the bookmarklet has
+the correct URL in it.
 
 ### Configuring Downloaders
 
-Gropple's default configuration uses the original youtube-dl and has two profiles set
-up, one for downloading video, the other for downloading audio (mp3).
+Gropple's default configuration uses `yt-dlp` and has two profiles set up, one
+for downloading video, the other for downloading audio (mp3).
 
-Note that gropple does not include any downloaders, you have to install them separately.
+Note that gropple does not include any downloaders, you have to install them
+separately.
 
-If you would like to use a youtube-dl fork (like [yt-dlp](https://github.com/yt-dlp/yt-dlp))
-or change the options, you can do so on the right hand side. Create as many profiles as you 
-wish, whenever you start a download you can choose the appropriate profile.
+If you would like to use a youtube-dl compatible fork or change the options you
+can do so on the right hand side. Create as many profiles as you wish, whenever
+you start a download you can choose the appropriate profile.
 
-Note that the command arguments must each be specified separately - see the default configuration
-for an example.
+Note that the command arguments must each be specified separately - see the
+default configuration for an example.
 
-While gropple will use your `PATH` to find the executable, you can also specify a full path
-instead. Note that any tools that the downloader calls itself (for instance, ffmpeg) will 
-probably need to be available on your path.
+While gropple will use your `PATH` to find the executable, you can also specify
+a full path instead. Note that any tools that the downloader calls itself (for
+instance, `ffmpeg`) will need to be available on your path.
+
+### Alternate destinations
+
+Gropple supports adding additional optional destinations. By default, all
+downloads will be stored in the main download path specified in the config. You
+can also add one or more destinations, and you can choose one of these
+destinations when queueing a new download, or while it is still downloading from
+the popup.
+
+The file will be moved after downloading is complete.
 
 ## Portable mode
 
-If you'd like to use gropple from a USB stick or similar, copy the config file from
-it's default location (shown when you start gropple) to the same location as the binary, and rename it to `gropple.yml`. 
-
-If that file is present in the same directory as the binary, it will be used instead.
+If you'd like to use gropple from a USB stick or similar, copy the config file
+from its default location (shown when you start gropple) to the same location as
+the binary, and rename it to `gropple.yml`.
 
 ## Problems
 
-Most download problems are probably diagnosable via the log - check in the popup window and scroll
-the log down to the bottom. The most common problem is that youtube-dl cannot be found, or its
-dependency (like ffmpeg) cannot be found on your path. 
+Many download problems are diagnosable via the log - check in the popup window
+and scroll the log down to the bottom. The most common problem is that `yt-dlp`
+cannot be found, or its dependency (like `ffmpeg`) cannot be found on your path.
 
 For other problems, please file an issue on github.
 
 ## TODO
 
-Many things. Please raise an issue after checking the [currently open issues](https://github.com/tardisx/gropple/issues).
-
-
+Many things. Please raise an issue after checking the [currently open
+issues](https://github.com/tardisx/gropple/issues).

build_release.pl

@@ -8,7 +8,7 @@ open my $fh, "<", "main.go" || die $!;
 my $version;
 while (<$fh>) {
   # CurrentVersion: "v0.04"
-  $version = $1 if /CurrentVersion:\s*"(v[\d\.]+)"/;
+  $version = $1 if /CurrentVersion:\s*"(v.*?)"/;
 }
 close $fh;
 
@@ -38,3 +38,12 @@ foreach my $type (keys %build) {
   system "go", "build", "-o", "release/$type/" . $build{$type}->{filename};
   system "zip", "-j", "dist/gropple-$type-$version.zip", ( glob "release/$type/*" );
 }
+
+# now docker
+exit 0;
+$ENV{VERSION}="$version";
+system "docker-compose", "-f", "docker-compose.build.yml", "build";
+system "docker", "tag", "tardisx/gropple:$version", "tardisx/gropple:latest";
+system "docker", "push", "tardisx/gropple:$version";
+system "docker", "push", "tardisx/gropple:latest";
+

config/config.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 
 	"gopkg.in/yaml.v2"
@@ -19,21 +20,31 @@ type Server struct {
 	MaximumActiveDownloads int    `yaml:"maximum_active_downloads_per_domain" json:"maximum_active_downloads_per_domain"`
 }
 
+// DownloadProfile holds the details for executing a downloader
 type DownloadProfile struct {
 	Name    string   `yaml:"name" json:"name"`
 	Command string   `yaml:"command" json:"command"`
 	Args    []string `yaml:"args" json:"args"`
 }
 
+// UI holds the configuration for the user interface
 type UI struct {
 	PopupWidth  int `yaml:"popup_width" json:"popup_width"`
 	PopupHeight int `yaml:"popup_height" json:"popup_height"`
 }
 
+// Destination is the path for a place that a download can be moved to
+type Destination struct {
+	Name string `yaml:"name" json:"name"` // Name for this location
+	Path string `yaml:"path" json:"path"` // Path on disk
+}
+
+// Config is the top level of the user configuration
 type Config struct {
 	ConfigVersion    int               `yaml:"config_version" json:"config_version"`
 	Server           Server            `yaml:"server" json:"server"`
 	UI               UI                `yaml:"ui" json:"ui"`
+	Destinations     []Destination     `yaml:"destinations" json:"destinations"`
 	DownloadProfiles []DownloadProfile `yaml:"profiles" json:"profiles"`
 }
 
@@ -46,18 +57,19 @@ type ConfigService struct {
 
 func (cs *ConfigService) LoadTestConfig() {
 	cs.LoadDefaultConfig()
+	cs.Config.Server.DownloadPath = "/tmp"
 	cs.Config.DownloadProfiles = []DownloadProfile{{Name: "test profile", Command: "sleep", Args: []string{"5"}}}
 }
 
 func (cs *ConfigService) LoadDefaultConfig() {
 	defaultConfig := Config{}
-	stdProfile := DownloadProfile{Name: "standard video", Command: "youtube-dl", Args: []string{
+	stdProfile := DownloadProfile{Name: "standard video", Command: "yt-dlp", Args: []string{
 		"--newline",
 		"--write-info-json",
 		"-f",
 		"bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best",
 	}}
-	mp3Profile := DownloadProfile{Name: "standard mp3", Command: "youtube-dl", Args: []string{
+	mp3Profile := DownloadProfile{Name: "standard mp3", Command: "yt-dlp", Args: []string{
 		"--newline",
 		"--write-info-json",
 		"--extract-audio",
@@ -69,18 +81,19 @@ func (cs *ConfigService) LoadDefaultConfig() {
 
 	defaultConfig.Server.Port = 6123
 	defaultConfig.Server.Address = "http://localhost:6123"
-	defaultConfig.Server.DownloadPath = "./"
+	defaultConfig.Server.DownloadPath = "/downloads"
 
 	defaultConfig.UI.PopupWidth = 500
 	defaultConfig.UI.PopupHeight = 500
 
 	defaultConfig.Server.MaximumActiveDownloads = 2
 
-	defaultConfig.ConfigVersion = 2
+	defaultConfig.Destinations = make([]Destination, 0)
+
+	defaultConfig.ConfigVersion = 3
 
 	cs.Config = &defaultConfig
 
-	return
 }
 
 func (c *Config) ProfileCalled(name string) *DownloadProfile {
@@ -92,6 +105,15 @@ func (c *Config) ProfileCalled(name string) *DownloadProfile {
 	return nil
 }
 
+func (c *Config) DestinationCalled(name string) *Destination {
+	for _, p := range c.Destinations {
+		if p.Name == name {
+			return &p
+		}
+	}
+	return nil
+}
+
 func (c *Config) UpdateFromJSON(j []byte) error {
 	newConfig := Config{}
 	err := json.Unmarshal(j, &newConfig)
@@ -163,18 +185,41 @@ func (c *Config) UpdateFromJSON(j []byte) error {
 		}
 	}
 
+	// check destinations
+	for _, dest := range newConfig.Destinations {
+		s, err := os.Stat(dest.Path)
+		if err != nil {
+			return fmt.Errorf("destination '%s' (%s) is bad: %s", dest.Name, dest.Path, err)
+		}
+		if !s.IsDir() {
+			return fmt.Errorf("destination '%s' (%s) is not a directory", dest.Name, dest.Path)
+		}
+	}
+
 	*c = newConfig
 	return nil
 }
 
 // DetermineConfigDir determines where the config is (or should be) stored.
 func (cs *ConfigService) DetermineConfigDir() {
-	// check current directory first, for a file called gropple.yml
-	_, err := os.Stat("gropple.yml")
+	// check binary path first, for a file called gropple.yml
+	binaryPath := os.Args[0]
+	binaryDir := filepath.Dir(binaryPath)
+	potentialConfigPath := filepath.Join(binaryDir, "gropple.yml")
+
+	_, err := os.Stat(potentialConfigPath)
 	if err == nil {
-		// exists in current directory, use that.
-		cs.ConfigPath = "gropple.yml"
+		// exists in binary directory, use that
+		// fully qualify, just for clarity in the log
+		config, err := filepath.Abs(potentialConfigPath)
+		if err == nil {
+			log.Printf("found portable config in %s", config)
+			cs.ConfigPath = config
 			return
+		} else {
+			log.Printf("got error when trying to convert config to absolute path: %s", err)
+			log.Print("falling back to using UserConfigDir")
+		}
 	}
 
 	// otherwise fall back to using the UserConfigDir
@@ -224,6 +269,8 @@ func (cs *ConfigService) LoadConfig() error {
 		return fmt.Errorf("Could not read config '%s': %v", path, err)
 	}
 	c := Config{}
+	cs.Config = &c
+
 	err = yaml.Unmarshal(b, &c)
 	if err != nil {
 		return fmt.Errorf("Could not parse YAML config '%s': %v", path, err)
@@ -236,6 +283,14 @@ func (cs *ConfigService) LoadConfig() error {
 		c.ConfigVersion = 2
 		configMigrated = true
 		log.Print("migrated config from version 1 => 2")
+
+	}
+
+	if c.ConfigVersion == 2 {
+		c.Destinations = make([]Destination, 0)
+		c.ConfigVersion = 3
+		configMigrated = true
+		log.Print("migrated config from version 2 => 3")
 	}
 
 	if configMigrated {
@@ -243,8 +298,6 @@ func (cs *ConfigService) LoadConfig() error {
 		cs.WriteConfig()
 	}
 
-	cs.Config = &c
-
 	return nil
 }
 
@@ -265,6 +318,9 @@ func (cs *ConfigService) WriteConfig() {
 	}
 	defer file.Close()
 
-	file.Write(s)
+	_, err = file.Write(s)
+	if err != nil {
+		log.Fatalf("could not write config file %s: %s", path, err)
+	}
 	file.Close()
 }

config/config_test.go

@@ -0,0 +1,60 @@
+package config
+
+import (
+	"os"
+	"testing"
+)
+
+func TestMigrationV1toV3(t *testing.T) {
+	v2Config := `config_version: 1
+server:
+  port: 6123
+  address: http://localhost:6123
+  download_path: ./
+ui:
+  popup_width: 500
+  popup_height: 500
+profiles:
+- name: standard video
+  command: youtube-dl
+  args:
+  - --newline
+  - --write-info-json
+  - -f
+  - bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best
+- name: standard mp3
+  command: youtube-dl
+  args:
+  - --newline
+  - --write-info-json
+  - --extract-audio
+  - --audio-format
+  - mp3
+`
+	cs := configServiceFromString(v2Config)
+	err := cs.LoadConfig()
+	if err != nil {
+		t.Errorf("got error when loading config: %s", err)
+	}
+	if cs.Config.ConfigVersion != 3 {
+		t.Errorf("did not migrate version (it is '%d')", cs.Config.ConfigVersion)
+	}
+	if cs.Config.Server.MaximumActiveDownloads != 2 {
+		t.Error("did not add MaximumActiveDownloads")
+	}
+	if len(cs.Config.Destinations) != 0 {
+		t.Error("incorrect number of destinations added")
+	}
+	os.Remove(cs.ConfigPath)
+}
+
+func configServiceFromString(configString string) *ConfigService {
+	tmpFile, _ := os.CreateTemp("", "gropple_test_*.yml")
+	tmpFile.Write([]byte(configString))
+	tmpFile.Close()
+	cs := ConfigService{
+		Config:     &Config{},
+		ConfigPath: tmpFile.Name(),
+	}
+	return &cs
+}

docker-compose.build.yml

@@ -0,0 +1,6 @@
+version: "3.9"
+    
+services:
+  gropple:
+    build: .
+    image: tardisx/gropple:$VERSION

docker-compose.yml

@@ -0,0 +1,12 @@
+version: "3.9"
+    
+services:
+  gropple:
+    build: .
+    image: tardisx/gropple:latest
+    volumes:
+      - ./gropple-config-dir:/config
+      - ./downloads:/downloads/
+    restart: always
+    ports:
+      - "6123:6123"

download/download.go

@@ -1,12 +1,14 @@
 package download
 
 import (
+	"encoding/json"
 	"fmt"
 	"io"
 	"log"
 	"net/url"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
@@ -23,118 +25,241 @@ type Download struct {
 	PopupUrl        string                 `json:"popup_url"`
 	Process         *os.Process            `json:"-"`
 	ExitCode        int                    `json:"exit_code"`
-	State           string                 `json:"state"`
+	State           State                  `json:"state"`
 	DownloadProfile config.DownloadProfile `json:"download_profile"`
+	Destination     *config.Destination    `json:"destination"`
 	Finished        bool                   `json:"finished"`
 	FinishedTS      time.Time              `json:"finished_ts"`
 	Files           []string               `json:"files"`
+	PlaylistCurrent int                    `json:"playlist_current"`
+	PlaylistTotal   int                    `json:"playlist_total"`
 	Eta             string                 `json:"eta"`
 	Percent         float32                `json:"percent"`
 	Log             []string               `json:"log"`
 	Config          *config.Config
-	mutex           sync.Mutex
+	Lock            sync.Mutex
 }
 
-type Downloads []*Download
+// The Manager holds and is responsible for all Download objects.
+type Manager struct {
+	Downloads    []*Download
+	MaxPerDomain int
+	Lock         sync.Mutex
+}
+
+func (m *Manager) String() string {
+	m.Lock.Lock()
+	defer m.Lock.Unlock()
+	out := fmt.Sprintf("Max per domain: %d, downloads: %d\n", m.MaxPerDomain, len(m.Downloads))
+
+	for _, dl := range m.Downloads {
+		out = out + fmt.Sprintf("%3d: (%10s) %30s\n", dl.Id, dl.State, dl.Url)
+	}
+
+	return out
+
+}
+
+type State string
+
+const (
+	STATE_PREPARING            State = "Preparing to start"
+	STATE_CHOOSE_PROFILE       State = "Choose Profile"
+	STATE_QUEUED               State = "Queued"
+	STATE_DOWNLOADING          State = "Downloading"
+	STATE_DOWNLOADING_METADATA State = "Downloading metadata"
+	STATE_FAILED               State = "Failed"
+	STATE_COMPLETE             State = "Complete"
+	STATE_MOVED                State = "Moved"
+)
 
 var CanStopDownload = false
 
 var downloadId int32 = 0
 
-// StartQueued starts any downloads that have been queued, we would not exceed
+func (m *Manager) ManageQueue() {
+	for {
+		m.Lock.Lock()
+
+		m.startQueued(m.MaxPerDomain)
+		m.moveToDest()
+		m.cleanup()
+		m.Lock.Unlock()
+
+		time.Sleep(time.Second)
+	}
+}
+
+func (m *Manager) DownloadsAsJSON() ([]byte, error) {
+
+	m.Lock.Lock()
+	defer m.Lock.Unlock()
+	for _, dl := range m.Downloads {
+		dl.Lock.Lock()
+		defer dl.Lock.Unlock()
+	}
+	b, err := json.Marshal(m.Downloads)
+	return b, err
+}
+
+func (m *Manager) moveToDest() {
+
+	// move any downloads that are complete and have a dest
+	for _, dl := range m.Downloads {
+
+		dl.Lock.Lock()
+		if dl.Destination != nil && dl.State == STATE_COMPLETE {
+			dl.State = STATE_MOVED
+			for _, fn := range dl.Files {
+				src := filepath.Join(dl.Config.Server.DownloadPath, fn)
+				dst := filepath.Join(dl.Destination.Path, fn)
+				err := os.Rename(src, dst)
+				if err != nil {
+					log.Printf("%s", err)
+					dl.Log = append(dl.Log, fmt.Sprintf("Could not move %s to %s - %s", fn, dl.Destination.Path, err))
+					break
+				} else {
+					dl.Log = append(dl.Log, fmt.Sprintf("Moved %s to %s", fn, dl.Destination.Path))
+
+				}
+			}
+		}
+		dl.Lock.Unlock()
+	}
+}
+
+// startQueued starts any downloads that have been queued, we would not exceed
 // maxRunning. If maxRunning is 0, there is no limit.
-func (dls Downloads) StartQueued(maxRunning int) {
+func (m *Manager) startQueued(maxRunning int) {
+
 	active := make(map[string]int)
 
-	for _, dl := range dls {
+	for _, dl := range m.Downloads {
+		dl.Lock.Lock()
 
-		dl.mutex.Lock()
-
-		if dl.State == "downloading" {
+		if dl.State == STATE_DOWNLOADING || dl.State == STATE_PREPARING {
 			active[dl.domain()]++
 		}
-		dl.mutex.Unlock()
+		dl.Lock.Unlock()
 
 	}
 
-	for _, dl := range dls {
+	for _, dl := range m.Downloads {
 
-		dl.mutex.Lock()
+		dl.Lock.Lock()
 
-		if dl.State == "queued" && (maxRunning == 0 || active[dl.domain()] < maxRunning) {
-			dl.State = "downloading"
+		if dl.State == STATE_QUEUED && (maxRunning == 0 || active[dl.domain()] < maxRunning) {
+			dl.State = STATE_PREPARING
 			active[dl.domain()]++
 			log.Printf("Starting download for id:%d (%s)", dl.Id, dl.Url)
-			dl.mutex.Unlock()
-			go func() { dl.Begin() }()
+
+			dl.Lock.Unlock()
+
+			go func(sdl *Download) {
+				sdl.Begin()
+			}(dl)
 		} else {
-			dl.mutex.Unlock()
+			dl.Lock.Unlock()
 		}
+
 	}
 
 }
 
-// Cleanup removes old downloads from the list. Hardcoded to remove them one hour
-// completion.
-func (dls Downloads) Cleanup() Downloads {
-	newDLs := Downloads{}
-	for _, dl := range dls {
-
-		dl.mutex.Lock()
-
+// cleanup removes old downloads from the list. Hardcoded to remove them one hour
+// completion. Expects the Manager to be locked.
+func (m *Manager) cleanup() {
+	newDLs := []*Download{}
+	for _, dl := range m.Downloads {
+		dl.Lock.Lock()
 		if dl.Finished && time.Since(dl.FinishedTS) > time.Duration(time.Hour) {
 			// do nothing
 		} else {
 			newDLs = append(newDLs, dl)
 		}
-		dl.mutex.Unlock()
+		dl.Lock.Unlock()
 
 	}
-	return newDLs
+	m.Downloads = newDLs
+}
+
+// GetDlById returns one of the downloads in our current list.
+func (m *Manager) GetDlById(id int) (*Download, error) {
+	m.Lock.Lock()
+	defer m.Lock.Unlock()
+	for _, dl := range m.Downloads {
+		if dl.Id == id {
+			return dl, nil
+		}
+	}
+	return nil, fmt.Errorf("no download with id %d", id)
 }
 
 // Queue queues a download
-func (dl *Download) Queue() {
+func (m *Manager) Queue(dl *Download) {
+	dl.Lock.Lock()
+	defer dl.Lock.Unlock()
+	dl.State = STATE_QUEUED
+}
 
-	dl.mutex.Lock()
-	defer dl.mutex.Unlock()
+func (m *Manager) ChangeDestination(dl *Download, dest *config.Destination) {
+	dl.Lock.Lock()
+	// we can only change destination is certain cases...
+	if dl.State != STATE_FAILED && dl.State != STATE_MOVED {
+		dl.Destination = dest
+	}
 
-	dl.State = "queued"
+	dl.Lock.Unlock()
 
 }
 
-func NewDownload(conf *config.Config, url string) *Download {
+func NewDownload(url string, conf *config.Config) *Download {
 	atomic.AddInt32(&downloadId, 1)
 	dl := Download{
-		Config: conf,
-
 		Id:       int(downloadId),
 		Url:      url,
 		PopupUrl: fmt.Sprintf("/fetch/%d", int(downloadId)),
-		State:    "choose profile",
-		Finished: false,
-		Eta:      "?",
-		Percent:  0.0,
-		Log:      make([]string, 0, 1000),
+		State:    STATE_CHOOSE_PROFILE,
+		Files:    []string{},
+		Log:      []string{},
+		Config:   conf,
+		Lock:     sync.Mutex{},
 	}
 	return &dl
 }
 
+func (m *Manager) AddDownload(dl *Download) {
+	m.Lock.Lock()
+	defer m.Lock.Unlock()
+	m.Downloads = append(m.Downloads, dl)
+}
+
+// func (dl *Download) AppendLog(text string) {
+// 	dl.Lock.Lock()
+// 	defer dl.Lock.Unlock()
+// 	dl.Log = append(dl.Log, text)
+// }
+
+// Stop the download.
 func (dl *Download) Stop() {
 	if !CanStopDownload {
 		log.Print("attempted to stop download on a platform that it is not currently supported on - please report this as a bug")
 		os.Exit(1)
 	}
+
 	log.Printf("stopping the download")
-	dl.mutex.Lock()
+	dl.Lock.Lock()
+	defer dl.Lock.Unlock()
 	dl.Log = append(dl.Log, "aborted by user")
-	defer dl.mutex.Unlock()
-	dl.Process.Kill()
+	err := dl.Process.Kill()
+	if err != nil {
+		log.Printf("could not send kill to process: %s", err)
+	}
 }
 
+// domain returns a domain for this Download. Download should be locked.
 func (dl *Download) domain() string {
 
-	// note that we expect to already have the mutex locked by the caller
 	url, err := url.Parse(dl.Url)
 	if err != nil {
 		log.Printf("Unknown domain for url: %s", dl.Url)
@@ -148,10 +273,9 @@ func (dl *Download) domain() string {
 // Begin starts a download, by starting the command specified in the DownloadProfile.
 // It blocks until the download is complete.
 func (dl *Download) Begin() {
+	dl.Lock.Lock()
 
-	dl.mutex.Lock()
-
-	dl.State = "downloading"
+	dl.State = STATE_DOWNLOADING
 	cmdSlice := []string{}
 	cmdSlice = append(cmdSlice, dl.DownloadProfile.Args...)
 
@@ -165,38 +289,45 @@ func (dl *Download) Begin() {
 
 	stdout, err := cmd.StdoutPipe()
 	if err != nil {
-		dl.State = "failed"
+		dl.State = STATE_FAILED
 		dl.Finished = true
 		dl.FinishedTS = time.Now()
 		dl.Log = append(dl.Log, fmt.Sprintf("error setting up stdout pipe: %v", err))
+		dl.Lock.Unlock()
+
 		return
 	}
 
 	stderr, err := cmd.StderrPipe()
 	if err != nil {
-		dl.State = "failed"
+		dl.State = STATE_FAILED
 		dl.Finished = true
 		dl.FinishedTS = time.Now()
 		dl.Log = append(dl.Log, fmt.Sprintf("error setting up stderr pipe: %v", err))
+		dl.Lock.Unlock()
+
 		return
 	}
 
 	log.Printf("Executing command: %v", cmd)
 	err = cmd.Start()
 	if err != nil {
-		dl.State = "failed"
+		dl.State = STATE_FAILED
 		dl.Finished = true
 		dl.FinishedTS = time.Now()
 		dl.Log = append(dl.Log, fmt.Sprintf("error starting command '%s': %v", dl.DownloadProfile.Command, err))
+		dl.Lock.Unlock()
+
 		return
 	}
 	dl.Process = cmd.Process
 
 	var wg sync.WaitGroup
 
-	dl.mutex.Unlock()
-
 	wg.Add(2)
+
+	dl.Lock.Unlock()
+
 	go func() {
 		defer wg.Done()
 		dl.updateDownload(stdout)
@@ -208,24 +339,39 @@ func (dl *Download) Begin() {
 	}()
 
 	wg.Wait()
-	cmd.Wait()
 
-	dl.mutex.Lock()
-	log.Printf("Process finished for id: %d (%v)", dl.Id, cmd)
+	err = cmd.Wait()
+	dl.Lock.Lock()
 
-	dl.State = "complete"
+	if err != nil {
+		log.Printf("process failed for id: %d: %s", dl.Id, err)
+
+		dl.State = STATE_FAILED
+		dl.Finished = true
+		dl.FinishedTS = time.Now()
+		dl.ExitCode = cmd.ProcessState.ExitCode()
+
+	} else {
+
+		log.Printf("process finished for id: %d (%v)", dl.Id, cmd)
+
+		dl.State = STATE_COMPLETE
 		dl.Finished = true
 		dl.FinishedTS = time.Now()
 		dl.ExitCode = cmd.ProcessState.ExitCode()
 
 		if dl.ExitCode != 0 {
-		dl.State = "failed"
+			dl.State = STATE_FAILED
 		}
-	dl.mutex.Unlock()
+	}
+	dl.Lock.Unlock()
 
 }
 
+// updateDownload updates the download based on data from the reader. Expects the
+// Download to be unlocked.
 func (dl *Download) updateDownload(r io.Reader) {
+
 	// XXX not sure if we might get a partial line?
 	buf := make([]byte, 1024)
 	for {
@@ -240,15 +386,13 @@ func (dl *Download) updateDownload(r io.Reader) {
 					continue
 				}
 
-				dl.mutex.Lock()
-
 				// append the raw log
+				dl.Lock.Lock()
 				dl.Log = append(dl.Log, l)
-
-				dl.mutex.Unlock()
-
 				// look for the percent and eta and other metadata
 				dl.updateMetadata(l)
+				dl.Lock.Unlock()
+
 			}
 		}
 		if err != nil {
@@ -257,18 +401,16 @@ func (dl *Download) updateDownload(r io.Reader) {
 	}
 }
 
+// updateMetadata parses some metadata and updates the Download. Download must be locked.
 func (dl *Download) updateMetadata(s string) {
 
-	dl.mutex.Lock()
-
-	defer dl.mutex.Unlock()
-
 	// [download]  49.7% of ~15.72MiB at  5.83MiB/s ETA 00:07
-	etaRE := regexp.MustCompile(`download.+ETA +(\d\d:\d\d(?::\d\d)?)$`)
+	// [download]  99.3% of ~1.42GiB at 320.87KiB/s ETA 00:07 (frag 212/214)
+	etaRE := regexp.MustCompile(`download.+ETA +(\d\d:\d\d(?::\d\d)?)`)
 	matches := etaRE.FindStringSubmatch(s)
 	if len(matches) == 2 {
 		dl.Eta = matches[1]
-		dl.State = "downloading"
+		dl.State = STATE_DOWNLOADING
 
 	}
 
@@ -314,4 +456,29 @@ func (dl *Download) updateMetadata(s string) {
 			}
 		}
 	}
+
+	// [download] Downloading video 1 of 3
+	playlistDetails := regexp.MustCompile(`Downloading video (\d+) of (\d+)`)
+	matches = playlistDetails.FindStringSubmatch(s)
+	if len(matches) == 3 {
+		total, _ := strconv.ParseInt(matches[2], 10, 32)
+		current, _ := strconv.ParseInt(matches[1], 10, 32)
+		dl.PlaylistTotal = int(total)
+		dl.PlaylistCurrent = int(current)
+	}
+
+	// [Site] user: Downloading JSON metadata page 2
+	metadataDL := regexp.MustCompile(`Downloading JSON metadata page (\d+)`)
+	matches = metadataDL.FindStringSubmatch(s)
+	if len(matches) == 2 {
+		dl.State = STATE_DOWNLOADING_METADATA
+	}
+
+	// [FixupM3u8] Fixing MPEG-TS in MP4 container of "file [-168849776_456239489].mp4"
+	metadataFixup := regexp.MustCompile(`Fixing MPEG-TS in MP4 container`)
+	matches = metadataFixup.FindStringSubmatch(s)
+	if len(matches) == 1 {
+		dl.State = "Fixing MPEG-TS in MP4"
+	}
+
 }

download/download_notestdata.go

@@ -0,0 +1,8 @@
+//go:build !testdata
+
+package download
+
+import "github.com/tardisx/gropple/config"
+
+func (m *Manager) AddStressTestData(c *config.ConfigService) {
+}

download/download_posix.go

@@ -0,0 +1,5 @@
+package download
+
+func init() {
+	CanStopDownload = true
+}

download/download_test.go

@@ -1,6 +1,8 @@
 package download
 
 import (
+	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -39,6 +41,18 @@ func TestUpdateMetadata(t *testing.T) {
 		t.Fatalf("%v", newD.Files)
 	}
 
+	// different download
+	newD.updateMetadata("[download]  99.3% of ~1.42GiB at 320.87KiB/s ETA 00:07 (frag 212/214)")
+	if newD.Eta != "00:07" {
+		t.Fatalf("bad short eta in dl with frag\n%v", newD)
+	}
+
+	// [FixupM3u8] Fixing MPEG-TS in MP4 container of "file [-168849776_456239489].mp4"
+	newD.updateMetadata("[FixupM3u8] Fixing MPEG-TS in MP4 container of \"file [-168849776_456239489].mp4")
+	if newD.State != "Fixing MPEG-TS in MP4" {
+		t.Fatalf("did not see fixup state - state is %s", newD.State)
+	}
+
 	// deletes
 	// TODO. Not sure why I don't always see the "Deleting original file" messages after merge -
 	// maybe a youtube-dl fork thing?
@@ -71,71 +85,278 @@ func TestQueue(t *testing.T) {
 	cs.LoadTestConfig()
 	conf := cs.Config
 
-	new1 := Download{Id: 1, Url: "http://sub.example.org/foo1", State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
-	new2 := Download{Id: 2, Url: "http://sub.example.org/foo2", State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
-	new3 := Download{Id: 3, Url: "http://sub.example.org/foo3", State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
-	new4 := Download{Id: 4, Url: "http://example.org/", State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
+	new1 := NewDownload("http://sub.example.org/foo1", conf)
+	new2 := NewDownload("http://sub.example.org/foo2", conf)
+	new3 := NewDownload("http://sub.example.org/foo3", conf)
+	new4 := NewDownload("http://example.org/", conf)
 
-	dls := Downloads{&new1, &new2, &new3, &new4}
-	dls.StartQueued(1)
+	// pretend the user chose a profile for each
+	new1.DownloadProfile = *conf.ProfileCalled("test profile")
+	new2.DownloadProfile = *conf.ProfileCalled("test profile")
+	new3.DownloadProfile = *conf.ProfileCalled("test profile")
+	new4.DownloadProfile = *conf.ProfileCalled("test profile")
+	new1.State = STATE_QUEUED
+	new2.State = STATE_QUEUED
+	new3.State = STATE_QUEUED
+	new4.State = STATE_QUEUED
+
+	q := Manager{
+		Downloads:    []*Download{},
+		MaxPerDomain: 2,
+		Lock:         sync.Mutex{},
+	}
+
+	q.AddDownload(new1)
+	q.AddDownload(new2)
+	q.AddDownload(new3)
+	q.AddDownload(new4)
+
+	q.startQueued(1)
+
+	// two should start, one from each of the two domains
 	time.Sleep(time.Millisecond * 100)
-	if dls[0].State == "queued" {
-		t.Error("#1 was not started")
+	if q.Downloads[0].State != STATE_DOWNLOADING {
+		t.Errorf("#1 was not downloading - %s instead ", q.Downloads[0].State)
+		t.Log(q.String())
 	}
-	if dls[1].State != "queued" {
-		t.Error("#2 is not queued")
+	if q.Downloads[1].State != STATE_QUEUED {
+		t.Errorf("#2 is not queued - %s instead", q.Downloads[1].State)
+		t.Log(q.String())
 	}
-	if dls[3].State == "queued" {
-		t.Error("#4 is not started")
+	if q.Downloads[2].State != STATE_QUEUED {
+		t.Errorf("#3 is not queued - %s instead", q.Downloads[2].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[3].State != STATE_DOWNLOADING {
+		t.Errorf("#4 is not downloading - %s instead", q.Downloads[3].State)
+		t.Log(q.String())
 	}
 
 	// this should start no more, as one is still going
-	dls.StartQueued(1)
+	q.startQueued(1)
 	time.Sleep(time.Millisecond * 100)
-	if dls[1].State != "queued" {
-		t.Error("#2 was started when it should not be")
+	if q.Downloads[0].State != STATE_DOWNLOADING {
+		t.Errorf("#1 was not downloading - %s instead ", q.Downloads[0].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[1].State != STATE_QUEUED {
+		t.Errorf("#2 is not queued - %s instead", q.Downloads[1].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[2].State != STATE_QUEUED {
+		t.Errorf("#3 is not queued - %s instead", q.Downloads[2].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[3].State != STATE_DOWNLOADING {
+		t.Errorf("#4 is not downloading - %s instead", q.Downloads[3].State)
+		t.Log(q.String())
 	}
 
-	dls.StartQueued(2)
-	time.Sleep(time.Millisecond * 100)
-	if dls[1].State == "queued" {
-		t.Error("#2 was not started but it should be")
-
+	// wait until the two finish, check
+	time.Sleep(time.Second * 5.0)
+	if q.Downloads[0].State != STATE_COMPLETE {
+		t.Errorf("#1 was not complete - %s instead ", q.Downloads[0].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[1].State != STATE_QUEUED {
+		t.Errorf("#2 is not queued - %s instead", q.Downloads[1].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[2].State != STATE_QUEUED {
+		t.Errorf("#3 is not queued - %s instead", q.Downloads[2].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[3].State != STATE_COMPLETE {
+		t.Errorf("#4 is not complete - %s instead", q.Downloads[3].State)
+		t.Log(q.String())
 	}
 
-	dls.StartQueued(2)
+	// this should start one more, as one is still going
+	q.startQueued(1)
 	time.Sleep(time.Millisecond * 100)
-	if dls[3].State == "queued" {
-		t.Error("#4 was not started but it should be")
+	if q.Downloads[0].State != STATE_COMPLETE {
+		t.Errorf("#1 was not complete - %s instead ", q.Downloads[0].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[1].State != STATE_DOWNLOADING {
+		t.Errorf("#2 is not downloading - %s instead", q.Downloads[1].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[2].State != STATE_QUEUED {
+		t.Errorf("#3 is not queued - %s instead", q.Downloads[2].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[3].State != STATE_COMPLETE {
+		t.Errorf("#4 is not complete - %s instead", q.Downloads[3].State)
+		t.Log(q.String())
 	}
 
-	// reset them all
-	dls[0].State = "queued"
-	dls[1].State = "queued"
-	dls[2].State = "queued"
-	dls[3].State = "queued"
-
-	dls.StartQueued(0)
+	// this should start no more, as one is still going
+	q.startQueued(1)
 	time.Sleep(time.Millisecond * 100)
-
-	// they should all be going
-	if dls[0].State == "queued" || dls[1].State == "queued" || dls[2].State == "queued" || dls[3].State == "queued" {
-		t.Error("none should be queued")
+	if q.Downloads[0].State != STATE_COMPLETE {
+		t.Errorf("#1 was not complete - %s instead ", q.Downloads[0].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[1].State != STATE_DOWNLOADING {
+		t.Errorf("#2 is not downloading - %s instead", q.Downloads[1].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[2].State != STATE_QUEUED {
+		t.Errorf("#3 is not queued - %s instead", q.Downloads[2].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[3].State != STATE_COMPLETE {
+		t.Errorf("#4 is not complete - %s instead", q.Downloads[3].State)
+		t.Log(q.String())
 	}
 
-	// reset them all
-	dls[0].State = "queued"
-	dls[1].State = "queued"
-	dls[2].State = "queued"
-	dls[3].State = "queued"
-
-	dls.StartQueued(2)
+	// but if we allow two per domain, the other queued one will start
+	q.startQueued(2)
 	time.Sleep(time.Millisecond * 100)
-
-	// first two should be running, third not (same domain) and 4th running (different domain)
-	if dls[0].State == "queued" || dls[1].State == "queued" || dls[2].State != "queued" || dls[3].State == "queued" {
-		t.Error("incorrect queued")
-
+	if q.Downloads[0].State != STATE_COMPLETE {
+		t.Errorf("#1 was not complete - %s instead ", q.Downloads[0].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[1].State != STATE_DOWNLOADING {
+		t.Errorf("#2 is not downloading - %s instead", q.Downloads[1].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[2].State != STATE_DOWNLOADING {
+		t.Errorf("#3 is not downloading - %s instead", q.Downloads[2].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[3].State != STATE_COMPLETE {
+		t.Errorf("#4 is not complete - %s instead", q.Downloads[3].State)
+		t.Log(q.String())
+	}
+
+}
+
+func TestUpdateMetadataPlaylist(t *testing.T) {
+
+	output := `
+start of log...
+[download] Downloading playlist: niceuser
+[RedGifsUser] niceuser: Downloading JSON metadata page 1
+[RedGifsUser] niceuser: Downloading JSON metadata page 2
+[RedGifsUser] niceuser: Downloading JSON metadata page 3
+[RedGifsUser] niceuser: Downloading JSON metadata page 4
+[RedGifsUser] niceuser: Downloading JSON metadata page 5
+[RedGifsUser] niceuser: Downloading JSON metadata page 6
+[info] Writing playlist metadata as JSON to: niceuser [niceuser].info.json
+[RedGifsUser] playlist niceuser: Downloading 3 videos
+[download] Downloading video 1 of 3
+[info] wrongpreciouschrysomelid: Downloading 1 format(s): hd
+[info] Writing video metadata as JSON to: Splendid Wonderful Speaker Power Chocolate Drop [wrongpreciouschrysomelid].info.json
+[download] Destination: Splendid Wonderful Speaker Power Chocolate Drop [wrongpreciouschrysomelid].mp4
+[download]   0.0% of 4.96MiB at Unknown speed ETA Unknown
+[download]   0.1% of 4.96MiB at  1.76MiB/s ETA 00:02
+[download]  20.1% of 4.96MiB at  7.28MiB/s ETA 00:00
+[download]  40.3% of 4.96MiB at 10.06MiB/s ETA 00:00
+[download]  80.6% of 4.96MiB at 14.93MiB/s ETA 00:00
+[download] 100% of 4.96MiB at 17.33MiB/s ETA 00:00
+[download] 100% of 4.96MiB in 00:00
+[download] Downloading video 2 of 3
+[info] silentnaughtyborzoi: Downloading 1 format(s): hd
+[info] Writing video metadata as JSON to: Splendid Printer Tray Computer Outdoor Window Wonderful [silentnaughtyborzoi].info.json
+[download] Destination: Splendid Printer Tray Computer Outdoor Window Wonderful [silentnaughtyborzoi].mp4
+[download]   0.0% of 5.81MiB at 896.03KiB/s ETA 00:06
+[download]   0.1% of 5.81MiB at  1.28MiB/s ETA 00:04
+[download]   0.1% of 5.81MiB at  1.59MiB/s ETA 00:03
+[download]  34.4% of 5.81MiB at  9.90MiB/s ETA 00:00
+[download]  68.8% of 5.81MiB at 12.49MiB/s ETA 00:00
+[download] 100% of 5.81MiB at 15.77MiB/s ETA 00:00
+[download] 100% of 5.81MiB in 00:00
+[download] Downloading video 3 of 3
+[info] mammothremarkablewhooper: Downloading 1 format(s): hd
+[info] Writing video metadata as JSON to: Porthole Splendid Close Up Gun Gunshot Window Wonderful [mammothremarkablewhooper].info.json
+[download] Destination: Porthole Splendid Close Up Gun Gunshot Window Wonderful [mammothremarkablewhooper].mp4
+[download]   0.0% of 2.89MiB at Unknown speed ETA Unknown
+[download]   0.1% of 2.89MiB at  1.77MiB/s ETA 00:01
+[download]   0.2% of 2.89MiB at  2.26MiB/s ETA 00:01
+[download]  34.5% of 2.89MiB at  8.23MiB/s ETA 00:00
+[download]  69.1% of 2.89MiB at 11.63MiB/s ETA 00:00
+[download] 100% of 2.89MiB at 14.25MiB/s ETA 00:00
+[download] 100% of 2.89MiB in 00:00
+[info] Writing updated playlist metadata as JSON to: niceuser [niceuser].info.json
+[download] Finished downloading playlist: niceuser
+`
+	newD := Download{}
+
+	lines := strings.Split(output, "\n")
+	for _, l := range lines {
+		// t.Log(l)
+		newD.updateMetadata(l)
+	}
+
+	if len(newD.Files) != 3 {
+		t.Errorf("%d files, not 3", len(newD.Files))
+	} else {
+		if newD.Files[0] != "Splendid Wonderful Speaker Power Chocolate Drop [wrongpreciouschrysomelid].mp4" {
+			t.Error("Wrong 1st file")
+		}
+		if newD.Files[1] != "Splendid Printer Tray Computer Outdoor Window Wonderful [silentnaughtyborzoi].mp4" {
+			t.Error("Wrong 2nd file")
+		}
+		if newD.Files[2] != "Porthole Splendid Close Up Gun Gunshot Window Wonderful [mammothremarkablewhooper].mp4" {
+			t.Error("Wrong 3rd file")
+		}
+	}
+
+	if newD.PlaylistTotal != 3 {
+		t.Errorf("playlist has total %d should be 3", newD.PlaylistTotal)
+	}
+
+}
+
+func TestUpdateMetadataSingle(t *testing.T) {
+
+	output := `
+[youtube] 2WoDQBhJCVQ: Downloading webpage
+[youtube] 2WoDQBhJCVQ: Downloading android player API JSON
+[info] 2WoDQBhJCVQ: Downloading 1 format(s): 137+140
+[info] Writing video metadata as JSON to: The Greatest Shot In Television [2WoDQBhJCVQ].info.json
+[download] Destination: The Greatest Shot In Television [2WoDQBhJCVQ].f137.mp4
+[download]   0.0% of 12.82MiB at 510.94KiB/s ETA 00:26
+[download]   0.0% of 12.82MiB at 966.50KiB/s ETA 00:13
+[download]   0.1% of 12.82MiB at  1.54MiB/s ETA 00:08
+[download]   0.1% of 12.82MiB at  2.75MiB/s ETA 00:04
+[download]   0.2% of 12.82MiB at  1.30MiB/s ETA 00:09
+[download]  77.5% of 12.82MiB at  2.54MiB/s ETA 00:01
+[download]  79.4% of 12.82MiB at  3.89MiB/s ETA 00:00
+[download]  83.3% of 12.82MiB at  6.44MiB/s ETA 00:00
+[download]  91.1% of 12.82MiB at 10.28MiB/s ETA 00:00
+[download] 100% of 12.82MiB at 12.77MiB/s ETA 00:00
+[download] 100% of 12.82MiB in 00:01
+[download] Destination: The Greatest Shot In Television [2WoDQBhJCVQ].f140.m4a
+[download]   0.1% of 1.10MiB at 286.46KiB/s ETA 00:03
+[download]   0.3% of 1.10MiB at 716.49KiB/s ETA 00:01
+[download]   0.6% of 1.10MiB at  1.42MiB/s ETA 00:00
+[download]  91.0% of 1.10MiB at  6.67MiB/s ETA 00:00
+[download] 100% of 1.10MiB at  7.06MiB/s ETA 00:00
+[download] 100% of 1.10MiB in 00:00
+[Merger] Merging formats into "The Greatest Shot In Television [2WoDQBhJCVQ].mp4"
+Deleting original file The Greatest Shot In Television [2WoDQBhJCVQ].f137.mp4 (pass -k to keep)
+Deleting original file The Greatest Shot In Television [2WoDQBhJCVQ].f140.m4a (pass -k to keep)
+`
+	newD := Download{}
+
+	lines := strings.Split(output, "\n")
+	for _, l := range lines {
+		// t.Log(l)
+		newD.updateMetadata(l)
+	}
+
+	if len(newD.Files) != 1 {
+		t.Errorf("%d files, not 1", len(newD.Files))
+	} else {
+		if newD.Files[0] != "The Greatest Shot In Television [2WoDQBhJCVQ].mp4" {
+			t.Error("Wrong 1st file")
+		}
+	}
+	if newD.PlaylistTotal != 0 {
+		t.Error("playlist detected but should not be")
 	}
 
 }

download/download_testdata.go

@@ -0,0 +1,28 @@
+//go:build testdata
+
+package download
+
+import "github.com/tardisx/gropple/config"
+
+func (m *Manager) AddStressTestData(c *config.ConfigService) {
+
+	urls := []string{
+		"https://www.youtube.com/watch?v=qG_rRkuGBW8",
+		"https://www.youtube.com/watch?v=ZUzhZpQAU40",
+		"https://www.youtube.com/watch?v=kVxM3eRWGak",
+		"https://www.youtube.com/watch?v=pl-y9869y0w",
+		"https://vimeo.com/783453809",
+		"https://www.youtube.com/watch?v=Uw4NEPE4l3A",
+		"https://www.youtube.com/watch?v=2RF0lcTuuYE",
+		"https://www.youtube.com/watch?v=lymwNQY0dus",
+		"https://www.youtube.com/watch?v=NTc-I4Z_duc",
+		"https://www.youtube.com/watch?v=wNSm1TJ84Ac",
+		"https://vimeo.com/786570322",
+	}
+	for _, u := range urls {
+		d := NewDownload(u, c.Config)
+		d.DownloadProfile = *c.Config.ProfileCalled("standard video")
+		m.AddDownload(d)
+		m.Queue(d)
+	}
+}

go.mod

@@ -1,9 +1,9 @@
 module github.com/tardisx/gropple
 
-go 1.16
+go 1.20
 
 require (
 	github.com/gorilla/mux v1.8.0
-	golang.org/x/mod v0.5.1
+	golang.org/x/mod v0.9.0
 	gopkg.in/yaml.v2 v2.4.0
 )

go.sum

@@ -1,18 +1,7 @@
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
-golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs=
+golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=

main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"embed"
 	"encoding/json"
+	"flag"
 	"fmt"
 	"html/template"
 	"io"
@@ -19,11 +20,12 @@ import (
 	"github.com/tardisx/gropple/version"
 )
 
-var downloads download.Downloads
-var downloadId = 0
+var dm *download.Manager
 var configService *config.ConfigService
 
-var versionInfo = version.Info{CurrentVersion: "v0.5.4"}
+var versionInfo = version.Manager{
+	VersionInfo: version.Info{CurrentVersion: "v0.6.0-alpha.4"},
+}
 
 //go:embed web
 var webFS embed.FS
@@ -39,10 +41,19 @@ type errorResponse struct {
 }
 
 func main() {
-	log.Printf("Starting gropple %s - https://github.com/tardisx/gropple", versionInfo.CurrentVersion)
+	log.Printf("Starting gropple %s - https://github.com/tardisx/gropple", versionInfo.GetInfo().CurrentVersion)
+
+	var configPath string
+	flag.StringVar(&configPath, "config-path", "", "path to config file")
+	flag.Parse()
 
 	configService = &config.ConfigService{}
+	if configPath != "" {
+		configService.ConfigPath = configPath
+	} else {
 		configService.DetermineConfigDir()
+	}
+
 	exists, err := configService.ConfigFileExists()
 	if err != nil {
 		log.Fatal(err)
@@ -58,9 +69,11 @@ func main() {
 			log.Fatal(err)
 		}
 		log.Printf("Configuration loaded from %s", configService.ConfigPath)
-
 	}
 
+	// create the download manager
+	dm = &download.Manager{MaxPerDomain: configService.Config.Server.MaximumActiveDownloads}
+
 	r := mux.NewRouter()
 	r.HandleFunc("/", homeHandler)
 	r.HandleFunc("/static/{filename}", staticHandler)
@@ -88,20 +101,18 @@ func main() {
 	// check for a new version every 4 hours
 	go func() {
 		for {
-			versionInfo.UpdateGitHubVersion()
+			err := versionInfo.UpdateGitHubVersion()
+			if err != nil {
+				log.Printf("could not get version info: %s", err)
+			}
 			time.Sleep(time.Hour * 4)
 		}
 	}()
 
 	// start downloading queued downloads when slots available, and clean up
 	// old entries
-	go func() {
-		for {
-			downloads.StartQueued(configService.Config.Server.MaximumActiveDownloads)
-			downloads = downloads.Cleanup()
-			time.Sleep(time.Second)
-		}
-	}()
+	go dm.ManageQueue()
+	dm.AddStressTestData(configService)
 
 	log.Printf("Visit %s for details on installing the bookmarklet and to check status", configService.Config.Server.Address)
 	log.Fatal(srv.ListenAndServe())
@@ -110,9 +121,12 @@ func main() {
 
 // versionRESTHandler returns the version information, if we have up-to-date info from github
 func versionRESTHandler(w http.ResponseWriter, r *http.Request) {
-	if versionInfo.GithubVersionFetched {
-		b, _ := json.Marshal(versionInfo)
-		w.Write(b)
+	if versionInfo.GetInfo().GithubVersionFetched {
+		b, _ := json.Marshal(versionInfo.GetInfo())
+		_, err := w.Write(b)
+		if err != nil {
+			log.Printf("could not write to client: %s", err)
+		}
 	} else {
 		w.WriteHeader(400)
 	}
@@ -130,17 +144,21 @@ func homeHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	type Info struct {
-		Downloads      []*download.Download
+		Manager        *download.Manager
 		BookmarkletURL template.URL
 		Config         *config.Config
+		Version        version.Info
 	}
 
 	info := Info{
-		Downloads:      downloads,
+		Manager:        dm,
 		BookmarkletURL: template.URL(bookmarkletURL),
 		Config:         configService.Config,
+		Version:        versionInfo.GetInfo(),
 	}
 
+	dm.Lock.Lock()
+	defer dm.Lock.Unlock()
 	err = t.ExecuteTemplate(w, "layout", info)
 	if err != nil {
 		panic(err)
@@ -160,7 +178,10 @@ func staticHandler(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 		w.WriteHeader(http.StatusOK)
-		io.Copy(w, f)
+		_, err = io.Copy(w, f)
+		if err != nil {
+			log.Printf("could not write to client: %s", err)
+		}
 		return
 	}
 	w.WriteHeader(http.StatusNotFound)
@@ -196,16 +217,21 @@ func configRESTHandler(w http.ResponseWriter, r *http.Request) {
 			errorRes := errorResponse{Success: false, Error: err.Error()}
 			errorResB, _ := json.Marshal(errorRes)
 			w.WriteHeader(400)
-			w.Write(errorResB)
+			_, err = w.Write(errorResB)
+			if err != nil {
+				log.Printf("could not write to client: %s", err)
+			}
 			return
 		}
 		configService.WriteConfig()
 	}
 	b, _ := json.Marshal(configService.Config)
-	w.Write(b)
+	_, err := w.Write(b)
+	if err != nil {
+		log.Printf("could not write config to client: %s", err)
+	}
 }
 
-//
 func fetchInfoOneRESTHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	idString := vars["id"]
@@ -216,23 +242,21 @@ func fetchInfoOneRESTHandler(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 
-		// find the download
-		var thisDownload *download.Download
-		for _, dl := range downloads {
-			if dl.Id == id {
-				thisDownload = dl
-			}
-		}
-		if thisDownload == nil {
+		thisDownload, err := dm.GetDlById(id)
+		if err != nil {
 			http.NotFound(w, r)
 			return
 		}
+		if thisDownload == nil {
+			panic("should not happen")
+		}
 
 		if r.Method == "POST" {
 
 			type updateRequest struct {
 				Action      string `json:"action"`
 				Profile     string `json:"profile"`
+				Destination string `json:"destination"`
 			}
 
 			thisReq := updateRequest{}
@@ -247,7 +271,10 @@ func fetchInfoOneRESTHandler(w http.ResponseWriter, r *http.Request) {
 				errorRes := errorResponse{Success: false, Error: err.Error()}
 				errorResB, _ := json.Marshal(errorRes)
 				w.WriteHeader(400)
-				w.Write(errorResB)
+				_, err = w.Write(errorResB)
+				if err != nil {
+					log.Printf("could not write to client: %s", err)
+				}
 				return
 			}
 
@@ -258,27 +285,62 @@ func fetchInfoOneRESTHandler(w http.ResponseWriter, r *http.Request) {
 					panic("bad profile name?")
 				}
 				// set the profile
+				thisDownload.Lock.Lock()
 				thisDownload.DownloadProfile = *profile
+				thisDownload.Lock.Unlock()
+
+				dm.Queue(thisDownload)
 
-				thisDownload.Queue()
 				succRes := successResponse{Success: true, Message: "download started"}
 				succResB, _ := json.Marshal(succRes)
-				w.Write(succResB)
+				_, err = w.Write(succResB)
+				if err != nil {
+					log.Printf("could not write to client: %s", err)
+				}
+				return
+			}
+
+			if thisReq.Action == "change_destination" {
+
+				// nil means (probably) that they chose "don't move" - which is fine,
+				// and maps to nil on the Download (the default state).
+				destination := configService.Config.DestinationCalled(thisReq.Destination)
+				dm.ChangeDestination(thisDownload, destination)
+
+				//				log.Printf("%#v", thisDownload)
+
+				succRes := successResponse{Success: true, Message: "destination changed"}
+				succResB, _ := json.Marshal(succRes)
+				_, err = w.Write(succResB)
+				if err != nil {
+					log.Printf("could not write to client: %s", err)
+				}
 				return
 			}
 
 			if thisReq.Action == "stop" {
+
 				thisDownload.Stop()
 				succRes := successResponse{Success: true, Message: "download stopped"}
 				succResB, _ := json.Marshal(succRes)
-				w.Write(succResB)
+				_, err = w.Write(succResB)
+				if err != nil {
+					log.Printf("could not write to client: %s", err)
+				}
 				return
 			}
 		}
 
 		// just a get, return the object
+		thisDownload.Lock.Lock()
+		defer thisDownload.Lock.Unlock()
+
 		b, _ := json.Marshal(thisDownload)
-		w.Write(b)
+
+		_, err = w.Write(b)
+		if err != nil {
+			log.Printf("could not write to client: %s", err)
+		}
 		return
 	} else {
 		http.NotFound(w, r)
@@ -286,8 +348,15 @@ func fetchInfoOneRESTHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func fetchInfoRESTHandler(w http.ResponseWriter, r *http.Request) {
-	b, _ := json.Marshal(downloads)
-	w.Write(b)
+
+	b, err := dm.DownloadsAsJSON()
+	if err != nil {
+		panic(err)
+	}
+	_, err = w.Write(b)
+	if err != nil {
+		log.Printf("could not write to client: %s", err)
+	}
 }
 
 func fetchHandler(w http.ResponseWriter, r *http.Request) {
@@ -298,9 +367,17 @@ func fetchHandler(w http.ResponseWriter, r *http.Request) {
 	idString := vars["id"]
 
 	idInt, err := strconv.ParseInt(idString, 10, 32)
+
+	// existing, load it up
 	if err == nil && idInt > 0 {
-		for _, dl := range downloads {
-			if dl.Id == int(idInt) {
+
+		dl, err := dm.GetDlById(int(idInt))
+		if err != nil {
+			log.Printf("not found")
+			w.WriteHeader(404)
+			return
+		}
+
 		t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/popup.html")
 		if err != nil {
 			panic(err)
@@ -314,9 +391,6 @@ func fetchHandler(w http.ResponseWriter, r *http.Request) {
 		}
 		return
 	}
-		}
-
-	}
 
 	query := r.URL.Query()
 	url, present := query["url"]
@@ -327,6 +401,7 @@ func fetchHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	} else {
 
+		log.Printf("popup for %s", url)
 		// check the URL for a sudden but inevitable betrayal
 		if strings.Contains(url[0], configService.Config.Server.Address) {
 			w.WriteHeader(400)
@@ -334,23 +409,19 @@ func fetchHandler(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 
-		// create the record
-		newDownload := download.NewDownload(configService.Config, url[0])
-		downloads = append(downloads, newDownload)
-		// XXX atomic ^^
-
-		newDownload.Log = append(newDownload.Log, "start of log...")
-
-		// go func() {
-		// 	newDownload.Begin()
-		// }()
+		// create the new download
+		newDL := download.NewDownload(url[0], configService.Config)
+		dm.AddDownload(newDL)
 
 		t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/popup.html")
 		if err != nil {
 			panic(err)
 		}
 
-		templateData := map[string]interface{}{"dl": newDownload, "config": configService.Config, "canStop": download.CanStopDownload}
+		newDL.Lock.Lock()
+		defer newDL.Lock.Unlock()
+
+		templateData := map[string]interface{}{"Version": versionInfo.GetInfo(), "dl": newDL, "config": configService.Config, "canStop": download.CanStopDownload}
 
 		err = t.ExecuteTemplate(w, "layout", templateData)
 		if err != nil {

version/version.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"sync"
 
 	"golang.org/x/mod/semver"
 )
@@ -19,8 +20,23 @@ type Info struct {
 	GithubVersionFetched bool   `json:"-"`
 }
 
-func (i *Info) UpdateGitHubVersion() error {
-	i.GithubVersionFetched = false
+type Manager struct {
+	VersionInfo Info
+	lock        sync.Mutex
+}
+
+func (m *Manager) GetInfo() Info {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+
+	return m.VersionInfo
+}
+
+func (m *Manager) UpdateGitHubVersion() error {
+	m.lock.Lock()
+	m.VersionInfo.GithubVersionFetched = false
+	m.lock.Unlock()
+
 	versionUrl := "https://api.github.com/repos/tardisx/gropple/releases"
 	resp, err := http.Get(versionUrl)
 	if err != nil {
@@ -51,27 +67,30 @@ func (i *Info) UpdateGitHubVersion() error {
 		return errors.New("no releases found")
 	}
 
-	i.GithubVersion = releases[0].Name
+	m.lock.Lock()
+
+	defer m.lock.Unlock()
+	m.VersionInfo.GithubVersion = releases[0].Name
+	m.VersionInfo.GithubVersionFetched = true
+	m.VersionInfo.UpgradeAvailable = m.canUpgrade()
 
-	i.GithubVersionFetched = true
-	i.UpgradeAvailable = i.canUpgrade()
 	return nil
 }
 
-func (i *Info) canUpgrade() bool {
-	if !i.GithubVersionFetched {
+func (m *Manager) canUpgrade() bool {
+	if !m.VersionInfo.GithubVersionFetched {
 		return false
 	}
 
-	if !semver.IsValid(i.CurrentVersion) {
-		log.Printf("current version %s is invalid", i.CurrentVersion)
+	if !semver.IsValid(m.VersionInfo.CurrentVersion) {
+		log.Printf("current version %s is invalid", m.VersionInfo.CurrentVersion)
 	}
 
-	if !semver.IsValid(i.GithubVersion) {
-		log.Printf("github version %s is invalid", i.GithubVersion)
+	if !semver.IsValid(m.VersionInfo.GithubVersion) {
+		log.Printf("github version %s is invalid", m.VersionInfo.GithubVersion)
 	}
 
-	if semver.Compare(i.CurrentVersion, i.GithubVersion) == -1 {
+	if semver.Compare(m.VersionInfo.CurrentVersion, m.VersionInfo.GithubVersion) == -1 {
 		return true
 	}
 	return false

web/config.html

@@ -7,11 +7,17 @@
     <p class="error"  x-show="error_message"  x-transition.duration.500ms x-text="error_message"></p>
     <p class="success" x-show="success_message" x-transition.duration.500ms x-text="success_message"></p>
 
-    <p>Note: changes are not saved until the "Save Config" button is pressed at the bottom of the page.</p>
+    <p>Note: changes are not saved until the "Save Config" button is pressed.</p>
+
+    <div class="pure-g">
+        <div class="pure-u-1">
+            <button class="pure-button pure-button-primary" @click="save_config();" href="#">Save Config</button>
+        </div>
+    </div>
 
     <div class="pure-g">
 
-        <div class="pure-u-md-1-2 pure-u-1 l-box">
+        <div class="pure-u-lg-1-3 pure-u-1 l-box">
    
             <form class="pure-form pure-form-stacked gropple-config">
                 <fieldset>
@@ -31,7 +37,7 @@
 
                     <label for="config-server-downloadpath">Download path</label>
                     <input type="text" id="config-server-downloadpath" placeholder="path" class="input-long" x-model="config.server.download_path" />
-                    <span class="pure-form-message">The path on the server to download files to.</span>
+                    <span class="pure-form-message">The default path on the server to download files to.</span>
 
                     <label for="config-server-max-downloads">Maximum active downloads per domain</label>
                     <input type="text" id="config-server-max-downloads" placeholder="2" class="input-long" x-model.number="config.server.maximum_active_downloads_per_domain" />
@@ -53,7 +59,7 @@
             </form>
 
         </div>
-        <div class="pure-u-md-1-2 pure-u-1 l-box">
+        <div class="pure-u-lg-1-3 pure-u-1 l-box">
             <form class="pure-form gropple-config">
                 <fieldset>
 
@@ -103,6 +109,39 @@
                 </fieldset>
             </form>
         </div>
+
+        <div class="pure-u-lg-1-3 pure-u-1 l-box">
+            <form class="pure-form gropple-config">
+                <fieldset>
+                    <legend>Destinations</legend>
+                    <p>You can specify custom destinations (directories) here. Downloads can be
+                        moved to one of these directories after completion from the index page, 
+                        if you do not want them to be left in the download path above.</p>
+                    </p>
+                    <template x-for="(dest, i) in config.destinations">
+                    <div>
+                        <label x-bind:for="'config-destinations-'+i+'-name'">Name of destination <span x-text="i+1"></span>
+                        </label>                                
+                        
+                        <input type="text" x-bind:id="'config-destinations-'+i+'-name'" class="input-long" placeholder="name" x-model="dest.name" />
+
+                        <span class="pure-form-message">The name of this destination. For your information only.</span>
+
+                        <label x-bind:for="'config-destinations-'+i+'-command'">Path</label>
+                        <input type="text" x-bind:id="'config-destinations-'+i+'-command'" class="input-long" placeholder="name" x-model="dest.path" />
+                        <span class="pure-form-message">Path to move completed downloads to.</span>
+
+                        <button class="pure-button button-del" href="#" @click.prevent="config.destinations.splice(i, 1);">delete destination</button>
+
+                        <hr>
+
+                    </div>
+                </template>
+
+                <button class="pure-button button-add" href="#" @click.prevent="config.destinations.push({name: 'new destination', path: '/tmp'});">add destination</button>
+
+            </fieldset>
+        </form>
     </div>
     <div class="pure-g">
         <div class="pure-u-1">
@@ -119,7 +158,7 @@
 <script>
     function config() {
         return { 
-            config: { server : {}, ui : {}, profiles: [] }, 
+            config: { server : {}, ui : {}, profiles: [], destinations: []}, 
             error_message: '',
             success_message: '',
 

web/index.html

@@ -31,7 +31,18 @@
             <template x-for="item in items">
                 <tr>
                     <td x-text="item.id"></td>
-                    <td x-text="item.files"></td>
+                    <td>
+                        <span x-show="item.files && item.files.length > 0">
+                            <ul>
+                                <template x-for="file in item.files">
+                                    <li x-text="file"></li>
+                                </template>
+                            </ul>
+                        </span>
+                        <span x-show="! item.files || item.files.length == 0"
+                              x-text="item.url">
+                        </span>
+                    </td>
                     <td><a class="int-link" x-bind:href="item.url">&#x2197;</a></td>
                     <td><a class="int-link" @click="show_popup(item)" href="#">&#x1F4C4;</a></td>
                     <td :class="'state-'+item.state" x-text="item.state"></td>
@@ -43,8 +54,6 @@
             </template>
               
 
-            {{ range $k, $v := .Downloads }}
-            {{ end }}
         </tbody>  
     </table>
 </div>
@@ -54,7 +63,7 @@
 <script>
     function index() {
         return { 
-            items: [], version: {},
+            items: [], version: {}, popups: {},
             fetch_version() {
                 fetch('/rest/version')
                 .then(response => response.json())
@@ -79,7 +88,9 @@
                 })
             },
             show_popup(item) {
-			    window.open(item.popup_url, item.id, "width={{ .Config.UI.PopupWidth }},height={{ .Config.UI.PopupHeight }}");
+                // allegedly you can use the reference to pop the window to the front on subsequent
+                // clicks, but I can't seem to find a reliable way to do so.
+                this.popups[item.id] = window.open(item.popup_url, item.id, "width={{ .Config.UI.PopupWidth }},height={{ .Config.UI.PopupHeight }}");
             },
 	    }
     }

web/layout.tmpl

@@ -33,6 +33,9 @@
         .state-downloading {
           color: blue;
         }
+        .state-moved {
+          color: green;
+        }
         .state-complete {
           color: green;
         }
@@ -71,7 +74,8 @@
 
     {{ template "content" . }}
     <footer>
-    Homepage: <a href="https://github.com/tardisx/gropple">https://github.com/tardisx/gropple</a>
+    Homepage: <a href="https://github.com/tardisx/gropple">https://github.com/tardisx/gropple</a><br>
+    Version: {{ .Version.CurrentVersion }}
     </footer>
 </body>
     {{ template "js" . }}

web/popup.html

@@ -15,18 +15,30 @@
                 </td>
             </tr>
             <tr><th>current filename</th><td x-text="filename"></td></tr>
+            <tr>
+                <th>destination</th>
+                <td>
+                    <select  x-on:change="update_destination()" class="pure-input-1-2" x-model="destination_chosen">
+                        <option value="-">leave in {{ .config.Server.DownloadPath }}</option>
+                    {{ range $i := .config.Destinations }}
+                        <option>{{ $i.Name }}</option>
+                    {{ end }}
+                    </select>
+                </td>
+            </tr>
             <tr><th>state</th><td x-text="state"></td></tr>
+            <tr x-show="playlist_total > 0"><th>playlist progress</th><td x-text="playlist_current + '/' + playlist_total"></td></tr>
             <tr><th>progress</th><td x-text="percent"></td></tr>
             <tr><th>ETA</th><td x-text="eta"></td></tr>
             
         </table>
         <p>You can close this window and your download will continue. Check the <a href="/" target="_gropple_status">Status page</a> to see all downloads in progress.</p>
         {{ if .canStop }}
-        <button x-show="state=='downloading'" class="pure-button" @click="stop()">stop</button>
+        <button x-show="state=='Downloading'" class="pure-button" @click="stop()">stop</button>
         {{ end }}
         <div>
             <h4>Logs</h4>
-            <pre x-text="log">
+            <pre x-text="log" style="height: auto;">
             </pre>
         </div>
     </div>
@@ -37,7 +49,9 @@
         history.replaceState(null, '', ['/fetch/{{ .dl.Id }}'])
         return { 
             eta: '', percent: 0.0, state: '??', filename: '', finished: false, log :'',
+            playlist_current: 0, playlist_total: 0,
             profile_chosen: null,
+            destination_chosen: null,
             watch_profile() {
                 this.$watch('profile_chosen', value => this.profile_chosen(value))
             },
@@ -54,6 +68,18 @@
                     console.log(info)
                 })
             },
+            update_destination(name) {
+                let op = {
+                   method: 'POST',
+                   body: JSON.stringify({action: 'change_destination', destination: this.destination_chosen}),
+                   headers: { 'Content-Type': 'application/json' }
+                };
+                fetch('/rest/fetch/{{ .dl.Id }}', op)
+                .then(response => response.json())
+                .then(info => {
+                    console.log(info)
+                })
+            },
             stop() {
                 let op = {
                    method: 'POST',
@@ -73,7 +99,13 @@
                     this.eta = info.eta;
                     this.percent = info.percent + "%";
                     this.state = info.state;
-                    if (this.state != 'choose profile') {
+                    this.playlist_current = info.playlist_current;
+                    this.playlist_total = info.playlist_total;
+                    this.destination_chosen = null;
+                    if (info.destination) {
+                        this.destination_chosen = info.destination.name;
+                    }
+                    if (this.state != 'Choose Profile') {
                         this.profile_chosen = true;
                     }
                     this.finished = info.finished;