Update changelog

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: tardisx

.vscode/settings.json

@@ -0,0 +1,10 @@
+{
+    "cSpell.words": [
+        "Cleanup",
+        "gropple",
+        "succ",
+        "tmpl",
+        "vars"
+    ],
+    "cSpell.language": "en-GB"
+}

CHANGELOG.md

@@ -4,6 +4,30 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+## [v0.6.0] - 2023-03-09
+
+- Configurable destinations for downloads
+- When downloading from a playlist, show the total number of videos and how many have been downloaded
+- Show version in web UI
+- Improve index page (show URL of queued downloads instead of nothing)
+- Add docker support
+- Fixes and improvements to capturing output info and showing it in the UI
+- Fixes to handling of queued downloads
+- Fix portable mode to look in binary directory, not current directory
+
+## [v0.5.5] - 2022-04-09
+
+- Fix a bug which would erase configuration when migrating from v1 to v2 config
+
+## [v0.5.4] - 2022-04-07
+
+- Check the chosen command exists when configuring a profile
+- Improve documentation
+- Add a stop button in the popup to abort a download (Linux/Mac only)
+- Move included JS to local app instead of accessing from a CDN
+- Make the simultaneous download limit apply to each unique domain
+- Support "portable" mode, reading gropple.yml from the current directory, if present
+
 ## [v0.5.3] - 2021-11-21
 
 - Add config option to limit number of simultaneous downloads

Dockerfile

@@ -0,0 +1,47 @@
+# Start from golang base image
+FROM golang:alpine as builder
+
+# Install git. (alpine image does not have git in it)
+RUN apk update && apk add --no-cache git curl
+
+# Set current working directory
+WORKDIR /app
+
+RUN curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /app/yt-dlp
+RUN chmod a+x /app/yt-dlp
+
+# Note here: To avoid downloading dependencies every time we
+# build image. Here, we are caching all the dependencies by
+# first copying go.mod and go.sum files and downloading them,
+# to be used every time we build the image if the dependencies
+# are not changed.
+
+# Copy go mod and sum files
+COPY go.mod ./
+COPY go.sum ./
+
+# Download all dependencies.
+RUN go mod download
+
+# Now, copy the source code
+COPY . .
+
+# Note here: CGO_ENABLED is disabled for cross system compilation
+# It is also a common best practise.
+
+# Build the application.
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ./bin/gropple .
+
+# Finally our multi-stage to build a small image
+# Start a new stage from scratch
+FROM golang:alpine
+
+# Copy the Pre-built binary file
+COPY --from=builder /app/bin/gropple .
+COPY --from=builder /app/yt-dlp /bin/
+
+# Install things we need to support yt-dlp
+RUN apk update && apk add --no-cache python3 ffmpeg
+
+# Run executable
+CMD ["./gropple", "--config-path", "/config/gropple.json"]

README.md

@@ -1,6 +1,6 @@
 # gropple
 
-A frontend to youtube-dl (and forks) to download videos with a single click, straight from your web browser.
+A frontend to youtube-dl (or compatible forks, like yt-dlp) to download videos with a single click, straight from your web browser.
 
 ![Screencast](/screencast.gif)
 
@@ -75,6 +75,13 @@ While gropple will use your `PATH` to find the executable, you can also specify
 instead. Note that any tools that the downloader calls itself (for instance, ffmpeg) will 
 probably need to be available on your path.
 
+## Portable mode
+
+If you'd like to use gropple from a USB stick or similar, copy the config file from
+it's default location (shown when you start gropple) to the same location as the binary, and rename it to `gropple.yml`. 
+
+If that file is present in the same directory as the binary, it will be used instead.
+
 ## Problems
 
 Most download problems are probably diagnosable via the log - check in the popup window and scroll

build_release.pl

@@ -8,7 +8,7 @@ open my $fh, "<", "main.go" || die $!;
 my $version;
 while (<$fh>) {
   # CurrentVersion: "v0.04"
-  $version = $1 if /CurrentVersion:\s*"(v[\d\.]+)"/;
+  $version = $1 if /CurrentVersion:\s*"(v.*?)"/;
 }
 close $fh;
 
@@ -38,3 +38,11 @@ foreach my $type (keys %build) {
   system "go", "build", "-o", "release/$type/" . $build{$type}->{filename};
   system "zip", "-j", "dist/gropple-$type-$version.zip", ( glob "release/$type/*" );
 }
+
+# now docker
+$ENV{VERSION}="$version";
+system "docker-compose", "-f", "docker-compose.build.yml", "build";
+system "docker", "tag", "tardisx/gropple:$version", "tardisx/gropple:latest";
+system "docker", "push", "tardisx/gropple:$version";
+system "docker", "push", "tardisx/gropple:latest";
+

config/config.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"os/exec"
+	"path/filepath"
 	"strings"
 
 	"gopkg.in/yaml.v2"
@@ -18,39 +20,55 @@ type Server struct {
 	MaximumActiveDownloads int    `yaml:"maximum_active_downloads_per_domain" json:"maximum_active_downloads_per_domain"`
 }
 
+// DownloadProfile holds the details for executing a downloader
 type DownloadProfile struct {
 	Name    string   `yaml:"name" json:"name"`
 	Command string   `yaml:"command" json:"command"`
 	Args    []string `yaml:"args" json:"args"`
 }
 
+// UI holds the configuration for the user interface
 type UI struct {
 	PopupWidth  int `yaml:"popup_width" json:"popup_width"`
 	PopupHeight int `yaml:"popup_height" json:"popup_height"`
 }
 
+// Destination is the path for a place that a download can be moved to
+type Destination struct {
+	Name string `yaml:"name" json:"name"` // Name for this location
+	Path string `yaml:"path" json:"path"` // Path on disk
+}
+
+// Config is the top level of the user configuration
 type Config struct {
 	ConfigVersion    int               `yaml:"config_version" json:"config_version"`
 	Server           Server            `yaml:"server" json:"server"`
 	UI               UI                `yaml:"ui" json:"ui"`
+	Destinations     []Destination     `yaml:"destinations" json:"destinations"`
 	DownloadProfiles []DownloadProfile `yaml:"profiles" json:"profiles"`
 }
 
-func TestConfig() *Config {
-	config := DefaultConfig()
-	config.DownloadProfiles = []DownloadProfile{{Name: "test profile", Command: "sleep", Args: []string{"5"}}}
-	return config
+// ConfigService is a struct to handle configuration requests, allowing for the
+// location that config files are loaded to be customised.
+type ConfigService struct {
+	Config     *Config
+	ConfigPath string
 }
 
-func DefaultConfig() *Config {
+func (cs *ConfigService) LoadTestConfig() {
+	cs.LoadDefaultConfig()
+	cs.Config.DownloadProfiles = []DownloadProfile{{Name: "test profile", Command: "sleep", Args: []string{"5"}}}
+}
+
+func (cs *ConfigService) LoadDefaultConfig() {
 	defaultConfig := Config{}
-	stdProfile := DownloadProfile{Name: "standard video", Command: "youtube-dl", Args: []string{
+	stdProfile := DownloadProfile{Name: "standard video", Command: "yt-dlp", Args: []string{
 		"--newline",
 		"--write-info-json",
 		"-f",
 		"bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best",
 	}}
-	mp3Profile := DownloadProfile{Name: "standard mp3", Command: "youtube-dl", Args: []string{
+	mp3Profile := DownloadProfile{Name: "standard mp3", Command: "yt-dlp", Args: []string{
 		"--newline",
 		"--write-info-json",
 		"--extract-audio",
@@ -62,16 +80,20 @@ func DefaultConfig() *Config {
 
 	defaultConfig.Server.Port = 6123
 	defaultConfig.Server.Address = "http://localhost:6123"
-	defaultConfig.Server.DownloadPath = "./"
+	defaultConfig.Server.DownloadPath = "/downloads"
 
 	defaultConfig.UI.PopupWidth = 500
 	defaultConfig.UI.PopupHeight = 500
 
 	defaultConfig.Server.MaximumActiveDownloads = 2
 
-	defaultConfig.ConfigVersion = 2
+	defaultConfig.Destinations = make([]Destination, 0)
 
-	return &defaultConfig
+	defaultConfig.ConfigVersion = 3
+
+	cs.Config = &defaultConfig
+
+	return
 }
 
 func (c *Config) ProfileCalled(name string) *DownloadProfile {
@@ -83,6 +105,15 @@ func (c *Config) ProfileCalled(name string) *DownloadProfile {
 	return nil
 }
 
+func (c *Config) DestinationCalled(name string) *Destination {
+	for _, p := range c.Destinations {
+		if p.Name == name {
+			return &p
+		}
+	}
+	return nil
+}
+
 func (c *Config) UpdateFromJSON(j []byte) error {
 	newConfig := Config{}
 	err := json.Unmarshal(j, &newConfig)
@@ -147,13 +178,51 @@ func (c *Config) UpdateFromJSON(j []byte) error {
 			}
 		}
 
+		// check the command exists
+		_, err := exec.LookPath(newConfig.DownloadProfiles[i].Command)
+		if err != nil {
+			return fmt.Errorf("Could not find %s on the path", newConfig.DownloadProfiles[i].Command)
+		}
+	}
+
+	// check destinations
+	for _, dest := range newConfig.Destinations {
+		s, err := os.Stat(dest.Path)
+		if err != nil {
+			return fmt.Errorf("destination '%s' (%s) is bad: %s", dest.Name, dest.Path, err)
+		}
+		if !s.IsDir() {
+			return fmt.Errorf("destination '%s' (%s) is not a directory", dest.Name, dest.Path)
+		}
 	}
 
 	*c = newConfig
 	return nil
 }
 
-func configPath() string {
+// DetermineConfigDir determines where the config is (or should be) stored.
+func (cs *ConfigService) DetermineConfigDir() {
+	// check binary path first, for a file called gropple.yml
+	binaryPath := os.Args[0]
+	binaryDir := filepath.Dir(binaryPath)
+	potentialConfigPath := filepath.Join(binaryDir, "gropple.yml")
+
+	_, err := os.Stat(potentialConfigPath)
+	if err == nil {
+		// exists in binary directory, use that
+		// fully qualify, just for clarity in the log
+		config, err := filepath.Abs(potentialConfigPath)
+		if err == nil {
+			log.Printf("found portable config in %s", config)
+			cs.ConfigPath = config
+			return
+		} else {
+			log.Printf("got error when trying to convert config to absolute path: %s", err)
+			log.Print("falling back to using UserConfigDir")
+		}
+	}
+
+	// otherwise fall back to using the UserConfigDir
 	dir, err := os.UserConfigDir()
 	if err != nil {
 		log.Fatalf("cannot find a directory to store config: %v", err)
@@ -171,36 +240,40 @@ func configPath() string {
 	}
 
 	fullFilename := fullPath + string(os.PathSeparator) + "config.yml"
-	return fullFilename
+	cs.ConfigPath = fullFilename
 }
 
-func ConfigFileExists() bool {
-	info, err := os.Stat(configPath())
+// ConfigFileExists checks if the config file already exists, and also checks
+// if there is an error accessing it
+func (cs *ConfigService) ConfigFileExists() (bool, error) {
+	path := cs.ConfigPath
+	info, err := os.Stat(path)
 	if os.IsNotExist(err) {
-		return false
+		return false, nil
 	}
 	if err != nil {
-		log.Fatal(err)
+		return false, fmt.Errorf("could not check if '%s' exists: %s", path, err)
 	}
 	if info.Size() == 0 {
-		log.Print("config file is 0 bytes?")
-		return false
+		return false, errors.New("config file is 0 bytes")
 	}
-	return true
+	return true, nil
 }
 
-func LoadConfig() (*Config, error) {
-	path := configPath()
+// LoadConfig loads the configuration from disk, migrating and updating it to the
+// latest version if needed.
+func (cs *ConfigService) LoadConfig() error {
+	path := cs.ConfigPath
 	b, err := os.ReadFile(path)
 	if err != nil {
-		log.Printf("Could not read config '%s': %v", path, err)
-		return nil, err
+		return fmt.Errorf("Could not read config '%s': %v", path, err)
 	}
 	c := Config{}
+	cs.Config = &c
+
 	err = yaml.Unmarshal(b, &c)
 	if err != nil {
-		log.Printf("Could not parse YAML config '%s': %v", path, err)
-		return nil, err
+		return fmt.Errorf("Could not parse YAML config '%s': %v", path, err)
 	}
 
 	// do migrations
@@ -210,34 +283,41 @@ func LoadConfig() (*Config, error) {
 		c.ConfigVersion = 2
 		configMigrated = true
 		log.Print("migrated config from version 1 => 2")
+
+	}
+
+	if c.ConfigVersion == 2 {
+		c.Destinations = make([]Destination, 0)
+		c.ConfigVersion = 3
+		configMigrated = true
+		log.Print("migrated config from version 2 => 3")
 	}
 
 	if configMigrated {
 		log.Print("Writing new config after version migration")
-		c.WriteConfig()
+		cs.WriteConfig()
 	}
 
-	return &c, nil
+	return nil
 }
 
-func (c *Config) WriteConfig() {
-	s, err := yaml.Marshal(c)
+// WriteConfig writes the in-memory config to disk.
+func (cs *ConfigService) WriteConfig() {
+	s, err := yaml.Marshal(cs.Config)
 	if err != nil {
 		panic(err)
 	}
 
-	path := configPath()
+	path := cs.ConfigPath
 	file, err := os.Create(
 		path,
 	)
 
 	if err != nil {
-		log.Fatalf("Could not open config file")
+		log.Fatalf("Could not open config file %s: %s", path, err)
 	}
 	defer file.Close()
 
 	file.Write(s)
 	file.Close()
-
-	log.Printf("Wrote configuration out to %s", path)
 }

config/config_test.go

@@ -0,0 +1,60 @@
+package config
+
+import (
+	"os"
+	"testing"
+)
+
+func TestMigrationV1toV3(t *testing.T) {
+	v2Config := `config_version: 1
+server:
+  port: 6123
+  address: http://localhost:6123
+  download_path: ./
+ui:
+  popup_width: 500
+  popup_height: 500
+profiles:
+- name: standard video
+  command: youtube-dl
+  args:
+  - --newline
+  - --write-info-json
+  - -f
+  - bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best
+- name: standard mp3
+  command: youtube-dl
+  args:
+  - --newline
+  - --write-info-json
+  - --extract-audio
+  - --audio-format
+  - mp3
+`
+	cs := configServiceFromString(v2Config)
+	err := cs.LoadConfig()
+	if err != nil {
+		t.Errorf("got error when loading config: %s", err)
+	}
+	if cs.Config.ConfigVersion != 3 {
+		t.Errorf("did not migrate version (it is '%d')", cs.Config.ConfigVersion)
+	}
+	if cs.Config.Server.MaximumActiveDownloads != 2 {
+		t.Error("did not add MaximumActiveDownloads")
+	}
+	if len(cs.Config.Destinations) != 0 {
+		t.Error("incorrect number of destinations added")
+	}
+	os.Remove(cs.ConfigPath)
+}
+
+func configServiceFromString(configString string) *ConfigService {
+	tmpFile, _ := os.CreateTemp("", "gropple_test_*.yml")
+	tmpFile.Write([]byte(configString))
+	tmpFile.Close()
+	cs := ConfigService{
+		Config:     &Config{},
+		ConfigPath: tmpFile.Name(),
+	}
+	return &cs
+}

docker-compose.build.yml

@@ -0,0 +1,6 @@
+version: "3.9"
+    
+services:
+  gropple:
+    build: .
+    image: tardisx/gropple:$VERSION

docker-compose.yml

@@ -0,0 +1,12 @@
+version: "3.9"
+    
+services:
+  gropple:
+    build: .
+    image: tardisx/gropple:latest
+    volumes:
+      - ./gropple-config-dir:/config
+      - ./downloads:/downloads/
+    restart: always
+    ports:
+      - "6123:6123"

download/download.go

@@ -3,11 +3,15 @@ package download
 import (
 	"fmt"
 	"io"
+	"log"
+	"net/url"
+	"os"
 	"os/exec"
 	"regexp"
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/tardisx/gropple/config"
@@ -17,77 +21,185 @@ type Download struct {
 	Id              int                    `json:"id"`
 	Url             string                 `json:"url"`
 	PopupUrl        string                 `json:"popup_url"`
-	Pid             int                    `json:"pid"`
+	Process         *os.Process            `json:"-"`
 	ExitCode        int                    `json:"exit_code"`
 	State           string                 `json:"state"`
 	DownloadProfile config.DownloadProfile `json:"download_profile"`
+	Destination     *config.Destination    `json:"destination"`
 	Finished        bool                   `json:"finished"`
 	FinishedTS      time.Time              `json:"finished_ts"`
 	Files           []string               `json:"files"`
+	PlaylistCurrent int                    `json:"playlist_current"`
+	PlaylistTotal   int                    `json:"playlist_total"`
 	Eta             string                 `json:"eta"`
 	Percent         float32                `json:"percent"`
 	Log             []string               `json:"log"`
 	Config          *config.Config
+	Lock            sync.Mutex
 }
 
-type Downloads []*Download
+// The Manager holds and is responsible for all Download objects.
+type Manager struct {
+	Downloads    []*Download
+	MaxPerDomain int
+	Lock         sync.Mutex
+}
 
-// StartQueued starts any downloads that have been queued, we would not exceed
+var CanStopDownload = false
+
+var downloadId int32 = 0
+
+func (m *Manager) ManageQueue() {
+	for {
+		m.Lock.Lock()
+
+		m.startQueued(m.MaxPerDomain)
+		// m.cleanup()
+		m.Lock.Unlock()
+
+		time.Sleep(time.Second)
+	}
+}
+
+// startQueued starts any downloads that have been queued, we would not exceed
 // maxRunning. If maxRunning is 0, there is no limit.
-func (dls Downloads) StartQueued(maxRunning int) {
-	active := 0
-	queued := 0
+func (m *Manager) startQueued(maxRunning int) {
 
-	for _, dl := range dls {
-		if dl.State == "downloading" {
-			active++
-		}
-		if dl.State == "queued" {
-			queued++
+	active := make(map[string]int)
+
+	for _, dl := range m.Downloads {
+		dl.Lock.Lock()
+
+		if dl.State == "downloading" || dl.State == "preparing to start" {
+			active[dl.domain()]++
 		}
+		dl.Lock.Unlock()
+
 	}
 
-	// there is room, so start one
-	if queued > 0 && (active < maxRunning || maxRunning == 0) {
-		for _, dl := range dls {
-			if dl.State == "queued" {
-				dl.State = "downloading"
-				go func() { dl.Begin() }()
-				return
-			}
+	for _, dl := range m.Downloads {
+
+		dl.Lock.Lock()
+
+		if dl.State == "queued" && (maxRunning == 0 || active[dl.domain()] < maxRunning) {
+			dl.State = "preparing to start"
+			active[dl.domain()]++
+			log.Printf("Starting download for id:%d (%s)", dl.Id, dl.Url)
+
+			dl.Lock.Unlock()
+
+			go func(sdl *Download) {
+				sdl.Begin()
+			}(dl)
+		} else {
+			dl.Lock.Unlock()
 		}
+
 	}
 
 }
 
-// Cleanup removes old downloads from the list. Hardcoded to remove them one hour
+// cleanup removes old downloads from the list. Hardcoded to remove them one hour
 // completion.
-func (dls Downloads) Cleanup() Downloads {
-	newDLs := Downloads{}
-	for _, dl := range dls {
+func (m *Manager) XXXcleanup() {
+	newDLs := []*Download{}
+	for _, dl := range m.Downloads {
+
 		if dl.Finished && time.Since(dl.FinishedTS) > time.Duration(time.Hour) {
 			// do nothing
 		} else {
 			newDLs = append(newDLs, dl)
 		}
+
 	}
-	return newDLs
+	m.Downloads = newDLs
+}
+
+// GetDlById returns one of the downloads in our current list.
+func (m *Manager) GetDlById(id int) (*Download, error) {
+	m.Lock.Lock()
+	defer m.Lock.Unlock()
+	for _, dl := range m.Downloads {
+		if dl.Id == id {
+			return dl, nil
+		}
+	}
+	return nil, fmt.Errorf("no download with id %d", id)
 }
 
 // Queue queues a download
-func (dl *Download) Queue() {
+func (m *Manager) Queue(dl *Download) {
+	dl.Lock.Lock()
+	defer dl.Lock.Unlock()
 	dl.State = "queued"
 }
 
+func NewDownload(url string, conf *config.Config) *Download {
+	atomic.AddInt32(&downloadId, 1)
+	dl := Download{
+		Id:       int(downloadId),
+		Url:      url,
+		PopupUrl: fmt.Sprintf("/fetch/%d", int(downloadId)),
+		State:    "choose profile",
+		Files:    []string{},
+		Log:      []string{},
+		Config:   conf,
+		Lock:     sync.Mutex{},
+	}
+	return &dl
+}
+
+func (m *Manager) AddDownload(dl *Download) {
+	m.Lock.Lock()
+	defer m.Lock.Unlock()
+	m.Downloads = append(m.Downloads, dl)
+	return
+}
+
+// func (dl *Download) AppendLog(text string) {
+// 	dl.Lock.Lock()
+// 	defer dl.Lock.Unlock()
+// 	dl.Log = append(dl.Log, text)
+// }
+
+// Stop the download.
+func (dl *Download) Stop() {
+	if !CanStopDownload {
+		log.Print("attempted to stop download on a platform that it is not currently supported on - please report this as a bug")
+		os.Exit(1)
+	}
+
+	log.Printf("stopping the download")
+	dl.Lock.Lock()
+	defer dl.Lock.Unlock()
+	dl.Log = append(dl.Log, "aborted by user")
+	dl.Process.Kill()
+}
+
+// domain returns a domain for this Download. Download should be locked.
+func (dl *Download) domain() string {
+
+	url, err := url.Parse(dl.Url)
+	if err != nil {
+		log.Printf("Unknown domain for url: %s", dl.Url)
+		return "unknown"
+	}
+
+	return url.Hostname()
+
+}
+
 // Begin starts a download, by starting the command specified in the DownloadProfile.
 // It blocks until the download is complete.
 func (dl *Download) Begin() {
+	dl.Lock.Lock()
+
 	dl.State = "downloading"
 	cmdSlice := []string{}
 	cmdSlice = append(cmdSlice, dl.DownloadProfile.Args...)
 
-	// only add the url if it's not empty. This helps us with testing
-	if dl.Url != "" {
+	// only add the url if it's not empty or an example URL. This helps us with testing
+	if !(dl.Url == "" || strings.Contains(dl.domain(), "example.org")) {
 		cmdSlice = append(cmdSlice, dl.Url)
 	}
 
@@ -100,6 +212,8 @@ func (dl *Download) Begin() {
 		dl.Finished = true
 		dl.FinishedTS = time.Now()
 		dl.Log = append(dl.Log, fmt.Sprintf("error setting up stdout pipe: %v", err))
+		dl.Lock.Unlock()
+
 		return
 	}
 
@@ -109,22 +223,30 @@ func (dl *Download) Begin() {
 		dl.Finished = true
 		dl.FinishedTS = time.Now()
 		dl.Log = append(dl.Log, fmt.Sprintf("error setting up stderr pipe: %v", err))
+		dl.Lock.Unlock()
+
 		return
 	}
 
+	log.Printf("Executing command: %v", cmd)
 	err = cmd.Start()
 	if err != nil {
 		dl.State = "failed"
 		dl.Finished = true
 		dl.FinishedTS = time.Now()
 		dl.Log = append(dl.Log, fmt.Sprintf("error starting command '%s': %v", dl.DownloadProfile.Command, err))
+		dl.Lock.Unlock()
+
 		return
 	}
-	dl.Pid = cmd.Process.Pid
+	dl.Process = cmd.Process
 
 	var wg sync.WaitGroup
 
 	wg.Add(2)
+
+	dl.Lock.Unlock()
+
 	go func() {
 		defer wg.Done()
 		dl.updateDownload(stdout)
@@ -138,6 +260,10 @@ func (dl *Download) Begin() {
 	wg.Wait()
 	cmd.Wait()
 
+	dl.Lock.Lock()
+
+	log.Printf("Process finished for id: %d (%v)", dl.Id, cmd)
+
 	dl.State = "complete"
 	dl.Finished = true
 	dl.FinishedTS = time.Now()
@@ -146,10 +272,14 @@ func (dl *Download) Begin() {
 	if dl.ExitCode != 0 {
 		dl.State = "failed"
 	}
+	dl.Lock.Unlock()
 
 }
 
+// updateDownload updates the download based on data from the reader. Expects the
+// Download to be unlocked.
 func (dl *Download) updateDownload(r io.Reader) {
+
 	// XXX not sure if we might get a partial line?
 	buf := make([]byte, 1024)
 	for {
@@ -165,10 +295,12 @@ func (dl *Download) updateDownload(r io.Reader) {
 				}
 
 				// append the raw log
+				dl.Lock.Lock()
 				dl.Log = append(dl.Log, l)
-
 				// look for the percent and eta and other metadata
 				dl.updateMetadata(l)
+				dl.Lock.Unlock()
+
 			}
 		}
 		if err != nil {
@@ -177,10 +309,12 @@ func (dl *Download) updateDownload(r io.Reader) {
 	}
 }
 
+// updateMetadata parses some metadata and updates the Download. Download must be locked.
 func (dl *Download) updateMetadata(s string) {
 
 	// [download]  49.7% of ~15.72MiB at  5.83MiB/s ETA 00:07
-	etaRE := regexp.MustCompile(`download.+ETA +(\d\d:\d\d(?::\d\d)?)$`)
+	// [download]  99.3% of ~1.42GiB at 320.87KiB/s ETA 00:07 (frag 212/214)
+	etaRE := regexp.MustCompile(`download.+ETA +(\d\d:\d\d(?::\d\d)?)`)
 	matches := etaRE.FindStringSubmatch(s)
 	if len(matches) == 2 {
 		dl.Eta = matches[1]
@@ -230,4 +364,29 @@ func (dl *Download) updateMetadata(s string) {
 			}
 		}
 	}
+
+	// [download] Downloading video 1 of 3
+	playlistDetails := regexp.MustCompile(`Downloading video (\d+) of (\d+)`)
+	matches = playlistDetails.FindStringSubmatch(s)
+	if len(matches) == 3 {
+		total, _ := strconv.ParseInt(matches[2], 10, 32)
+		current, _ := strconv.ParseInt(matches[1], 10, 32)
+		dl.PlaylistTotal = int(total)
+		dl.PlaylistCurrent = int(current)
+	}
+
+	// [Site] user: Downloading JSON metadata page 2
+	metadataDL := regexp.MustCompile(`Downloading JSON metadata page (\d+)`)
+	matches = metadataDL.FindStringSubmatch(s)
+	if len(matches) == 2 {
+		dl.State = "Downloading metadata, page " + matches[1]
+	}
+
+	// [FixupM3u8] Fixing MPEG-TS in MP4 container of "file [-168849776_456239489].mp4"
+	metadataFixup := regexp.MustCompile(`Fixing MPEG-TS in MP4 container`)
+	matches = metadataFixup.FindStringSubmatch(s)
+	if len(matches) == 1 {
+		dl.State = "Fixing MPEG-TS in MP4"
+	}
+
 }

download/download_posix.go

@@ -0,0 +1,5 @@
+package download
+
+func init() {
+	CanStopDownload = true
+}

download/download_test.go

@@ -1,9 +1,8 @@
 package download
 
 import (
+	"strings"
 	"testing"
-
-	"github.com/tardisx/gropple/config"
 )
 
 func TestUpdateMetadata(t *testing.T) {
@@ -38,6 +37,18 @@ func TestUpdateMetadata(t *testing.T) {
 		t.Fatalf("%v", newD.Files)
 	}
 
+	// different download
+	newD.updateMetadata("[download]  99.3% of ~1.42GiB at 320.87KiB/s ETA 00:07 (frag 212/214)")
+	if newD.Eta != "00:07" {
+		t.Fatalf("bad short eta in dl with frag\n%v", newD)
+	}
+
+	// [FixupM3u8] Fixing MPEG-TS in MP4 container of "file [-168849776_456239489].mp4"
+	newD.updateMetadata("[FixupM3u8] Fixing MPEG-TS in MP4 container of \"file [-168849776_456239489].mp4")
+	if newD.State != "Fixing MPEG-TS in MP4" {
+		t.Fatalf("did not see fixup state - state is %s", newD.State)
+	}
+
 	// deletes
 	// TODO. Not sure why I don't always see the "Deleting original file" messages after merge -
 	// maybe a youtube-dl fork thing?
@@ -65,35 +76,204 @@ func TestUpdateMetadata(t *testing.T) {
 // [download] 100% of 4.64MiB in 00:00
 // [ffmpeg] Merging formats into "Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.mp4"
 
-// This test is a bit broken, because StartQueued immediately starts the queued
-// download, it
-func TestQueue(t *testing.T) {
-	conf := config.TestConfig()
+// func TestQueue(t *testing.T) {
+// 	cs := config.ConfigService{}
+// 	cs.LoadTestConfig()
+// 	conf := cs.Config
 
-	new1 := Download{Id: 1, State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
-	new2 := Download{Id: 2, State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
-	new3 := Download{Id: 3, State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
-	new4 := Download{Id: 4, State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
+// 	new1 := Download{Id: 1, Url: "http://sub.example.org/foo1", State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
+// 	new2 := Download{Id: 2, Url: "http://sub.example.org/foo2", State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
+// 	new3 := Download{Id: 3, Url: "http://sub.example.org/foo3", State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
+// 	new4 := Download{Id: 4, Url: "http://example.org/", State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
 
-	dls := Downloads{&new1, &new2, &new3, &new4}
-	dls.StartQueued(1)
-	if dls[0].State == "queued" {
-		t.Error("#1 was not started")
-	}
-	if dls[1].State != "queued" {
-		t.Error("#2 is not queued")
+// 	dls := Downloads{&new1, &new2, &new3, &new4}
+// 	dls.StartQueued(1)
+// 	time.Sleep(time.Millisecond * 100)
+// 	if dls[0].State == "queued" {
+// 		t.Error("#1 was not started")
+// 	}
+// 	if dls[1].State != "queued" {
+// 		t.Error("#2 is not queued")
+// 	}
+// 	if dls[3].State == "queued" {
+// 		t.Error("#4 is not started")
+// 	}
+
+// 	// this should start no more, as one is still going
+// 	dls.StartQueued(1)
+// 	time.Sleep(time.Millisecond * 100)
+// 	if dls[1].State != "queued" {
+// 		t.Error("#2 was started when it should not be")
+// 	}
+
+// 	dls.StartQueued(2)
+// 	time.Sleep(time.Millisecond * 100)
+// 	if dls[1].State == "queued" {
+// 		t.Error("#2 was not started but it should be")
+
+// 	}
+
+// 	dls.StartQueued(2)
+// 	time.Sleep(time.Millisecond * 100)
+// 	if dls[3].State == "queued" {
+// 		t.Error("#4 was not started but it should be")
+// 	}
+
+// 	// reset them all
+// 	dls[0].State = "queued"
+// 	dls[1].State = "queued"
+// 	dls[2].State = "queued"
+// 	dls[3].State = "queued"
+
+// 	dls.StartQueued(0)
+// 	time.Sleep(time.Millisecond * 100)
+
+// 	// they should all be going
+// 	if dls[0].State == "queued" || dls[1].State == "queued" || dls[2].State == "queued" || dls[3].State == "queued" {
+// 		t.Error("none should be queued")
+// 	}
+
+// 	// reset them all
+// 	dls[0].State = "queued"
+// 	dls[1].State = "queued"
+// 	dls[2].State = "queued"
+// 	dls[3].State = "queued"
+
+// 	dls.StartQueued(2)
+// 	time.Sleep(time.Millisecond * 100)
+
+// 	// first two should be running, third not (same domain) and 4th running (different domain)
+// 	if dls[0].State == "queued" || dls[1].State == "queued" || dls[2].State != "queued" || dls[3].State == "queued" {
+// 		t.Error("incorrect queued")
+
+// 	}
+
+// }
+
+func TestUpdateMetadataPlaylist(t *testing.T) {
+
+	output := `
+start of log...
+[download] Downloading playlist: niceuser
+[RedGifsUser] niceuser: Downloading JSON metadata page 1
+[RedGifsUser] niceuser: Downloading JSON metadata page 2
+[RedGifsUser] niceuser: Downloading JSON metadata page 3
+[RedGifsUser] niceuser: Downloading JSON metadata page 4
+[RedGifsUser] niceuser: Downloading JSON metadata page 5
+[RedGifsUser] niceuser: Downloading JSON metadata page 6
+[info] Writing playlist metadata as JSON to: niceuser [niceuser].info.json
+[RedGifsUser] playlist niceuser: Downloading 3 videos
+[download] Downloading video 1 of 3
+[info] wrongpreciouschrysomelid: Downloading 1 format(s): hd
+[info] Writing video metadata as JSON to: Splendid Wonderful Speaker Power Chocolate Drop [wrongpreciouschrysomelid].info.json
+[download] Destination: Splendid Wonderful Speaker Power Chocolate Drop [wrongpreciouschrysomelid].mp4
+[download]   0.0% of 4.96MiB at Unknown speed ETA Unknown
+[download]   0.1% of 4.96MiB at  1.76MiB/s ETA 00:02
+[download]  20.1% of 4.96MiB at  7.28MiB/s ETA 00:00
+[download]  40.3% of 4.96MiB at 10.06MiB/s ETA 00:00
+[download]  80.6% of 4.96MiB at 14.93MiB/s ETA 00:00
+[download] 100% of 4.96MiB at 17.33MiB/s ETA 00:00
+[download] 100% of 4.96MiB in 00:00
+[download] Downloading video 2 of 3
+[info] silentnaughtyborzoi: Downloading 1 format(s): hd
+[info] Writing video metadata as JSON to: Splendid Printer Tray Computer Outdoor Window Wonderful [silentnaughtyborzoi].info.json
+[download] Destination: Splendid Printer Tray Computer Outdoor Window Wonderful [silentnaughtyborzoi].mp4
+[download]   0.0% of 5.81MiB at 896.03KiB/s ETA 00:06
+[download]   0.1% of 5.81MiB at  1.28MiB/s ETA 00:04
+[download]   0.1% of 5.81MiB at  1.59MiB/s ETA 00:03
+[download]  34.4% of 5.81MiB at  9.90MiB/s ETA 00:00
+[download]  68.8% of 5.81MiB at 12.49MiB/s ETA 00:00
+[download] 100% of 5.81MiB at 15.77MiB/s ETA 00:00
+[download] 100% of 5.81MiB in 00:00
+[download] Downloading video 3 of 3
+[info] mammothremarkablewhooper: Downloading 1 format(s): hd
+[info] Writing video metadata as JSON to: Porthole Splendid Close Up Gun Gunshot Window Wonderful [mammothremarkablewhooper].info.json
+[download] Destination: Porthole Splendid Close Up Gun Gunshot Window Wonderful [mammothremarkablewhooper].mp4
+[download]   0.0% of 2.89MiB at Unknown speed ETA Unknown
+[download]   0.1% of 2.89MiB at  1.77MiB/s ETA 00:01
+[download]   0.2% of 2.89MiB at  2.26MiB/s ETA 00:01
+[download]  34.5% of 2.89MiB at  8.23MiB/s ETA 00:00
+[download]  69.1% of 2.89MiB at 11.63MiB/s ETA 00:00
+[download] 100% of 2.89MiB at 14.25MiB/s ETA 00:00
+[download] 100% of 2.89MiB in 00:00
+[info] Writing updated playlist metadata as JSON to: niceuser [niceuser].info.json
+[download] Finished downloading playlist: niceuser
+`
+	newD := Download{}
+
+	lines := strings.Split(output, "\n")
+	for _, l := range lines {
+		// t.Log(l)
+		newD.updateMetadata(l)
 	}
 
-	// this should start no more, as one is still going
-	dls.StartQueued(1)
-	if dls[1].State != "queued" {
-		t.Error("#2 was started when it should not be")
+	if len(newD.Files) != 3 {
+		t.Errorf("%d files, not 3", len(newD.Files))
+	} else {
+		if newD.Files[0] != "Splendid Wonderful Speaker Power Chocolate Drop [wrongpreciouschrysomelid].mp4" {
+			t.Error("Wrong 1st file")
+		}
+		if newD.Files[1] != "Splendid Printer Tray Computer Outdoor Window Wonderful [silentnaughtyborzoi].mp4" {
+			t.Error("Wrong 2nd file")
+		}
+		if newD.Files[2] != "Porthole Splendid Close Up Gun Gunshot Window Wonderful [mammothremarkablewhooper].mp4" {
+			t.Error("Wrong 3rd file")
+		}
 	}
 
-	dls.StartQueued(2)
-	if dls[1].State == "queued" {
-		t.Error("#2 was not started but it should be")
-
+	if newD.PlaylistTotal != 3 {
+		t.Errorf("playlist has total %d should be 3", newD.PlaylistTotal)
+	}
+
+}
+
+func TestUpdateMetadataSingle(t *testing.T) {
+
+	output := `
+[youtube] 2WoDQBhJCVQ: Downloading webpage
+[youtube] 2WoDQBhJCVQ: Downloading android player API JSON
+[info] 2WoDQBhJCVQ: Downloading 1 format(s): 137+140
+[info] Writing video metadata as JSON to: The Greatest Shot In Television [2WoDQBhJCVQ].info.json
+[download] Destination: The Greatest Shot In Television [2WoDQBhJCVQ].f137.mp4
+[download]   0.0% of 12.82MiB at 510.94KiB/s ETA 00:26
+[download]   0.0% of 12.82MiB at 966.50KiB/s ETA 00:13
+[download]   0.1% of 12.82MiB at  1.54MiB/s ETA 00:08
+[download]   0.1% of 12.82MiB at  2.75MiB/s ETA 00:04
+[download]   0.2% of 12.82MiB at  1.30MiB/s ETA 00:09
+[download]  77.5% of 12.82MiB at  2.54MiB/s ETA 00:01
+[download]  79.4% of 12.82MiB at  3.89MiB/s ETA 00:00
+[download]  83.3% of 12.82MiB at  6.44MiB/s ETA 00:00
+[download]  91.1% of 12.82MiB at 10.28MiB/s ETA 00:00
+[download] 100% of 12.82MiB at 12.77MiB/s ETA 00:00
+[download] 100% of 12.82MiB in 00:01
+[download] Destination: The Greatest Shot In Television [2WoDQBhJCVQ].f140.m4a
+[download]   0.1% of 1.10MiB at 286.46KiB/s ETA 00:03
+[download]   0.3% of 1.10MiB at 716.49KiB/s ETA 00:01
+[download]   0.6% of 1.10MiB at  1.42MiB/s ETA 00:00
+[download]  91.0% of 1.10MiB at  6.67MiB/s ETA 00:00
+[download] 100% of 1.10MiB at  7.06MiB/s ETA 00:00
+[download] 100% of 1.10MiB in 00:00
+[Merger] Merging formats into "The Greatest Shot In Television [2WoDQBhJCVQ].mp4"
+Deleting original file The Greatest Shot In Television [2WoDQBhJCVQ].f137.mp4 (pass -k to keep)
+Deleting original file The Greatest Shot In Television [2WoDQBhJCVQ].f140.m4a (pass -k to keep)
+`
+	newD := Download{}
+
+	lines := strings.Split(output, "\n")
+	for _, l := range lines {
+		// t.Log(l)
+		newD.updateMetadata(l)
+	}
+
+	if len(newD.Files) != 1 {
+		t.Errorf("%d files, not 1", len(newD.Files))
+	} else {
+		if newD.Files[0] != "The Greatest Shot In Television [2WoDQBhJCVQ].mp4" {
+			t.Error("Wrong 1st file")
+		}
+	}
+	if newD.PlaylistTotal != 0 {
+		t.Error("playlist detected but should not be")
 	}
 
 }

go.mod

@@ -1,6 +1,6 @@
 module github.com/tardisx/gropple
 
-go 1.16
+go 1.20
 
 require (
 	github.com/gorilla/mux v1.8.0

go.sum

@@ -1,18 +1,7 @@
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
 golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=

main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"embed"
 	"encoding/json"
+	"flag"
 	"fmt"
 	"html/template"
 	"io"
@@ -19,11 +20,13 @@ import (
 	"github.com/tardisx/gropple/version"
 )
 
-var downloads download.Downloads
+var dm *download.Manager
 var downloadId = 0
-var conf *config.Config
+var configService *config.ConfigService
 
-var versionInfo = version.Info{CurrentVersion: "v0.5.3"}
+var versionInfo = version.Manager{
+	VersionInfo: version.Info{CurrentVersion: "v0.6.0-alpha.1"},
+}
 
 //go:embed web
 var webFS embed.FS
@@ -39,20 +42,42 @@ type errorResponse struct {
 }
 
 func main() {
-	if !config.ConfigFileExists() {
-		log.Print("No config file - creating default config")
-		conf = config.DefaultConfig()
-		conf.WriteConfig()
+	log.Printf("Starting gropple %s - https://github.com/tardisx/gropple", versionInfo.GetInfo().CurrentVersion)
+
+	var configPath string
+	flag.StringVar(&configPath, "config-path", "", "path to config file")
+	flag.Parse()
+
+	configService = &config.ConfigService{}
+	if configPath != "" {
+		configService.ConfigPath = configPath
 	} else {
-		loadedConfig, err := config.LoadConfig()
+		configService.DetermineConfigDir()
+	}
+
+	exists, err := configService.ConfigFileExists()
+	if err != nil {
+		log.Fatal(err)
+	}
+	if !exists {
+		log.Print("No config file - creating default config")
+		configService.LoadDefaultConfig()
+		configService.WriteConfig()
+		log.Printf("Configuration written to %s", configService.ConfigPath)
+	} else {
+		err := configService.LoadConfig()
 		if err != nil {
 			log.Fatal(err)
 		}
-		conf = loadedConfig
+		log.Printf("Configuration loaded from %s", configService.ConfigPath)
 	}
 
+	// create the download manager
+	dm = &download.Manager{MaxPerDomain: configService.Config.Server.MaximumActiveDownloads}
+
 	r := mux.NewRouter()
 	r.HandleFunc("/", homeHandler)
+	r.HandleFunc("/static/{filename}", staticHandler)
 	r.HandleFunc("/config", configHandler)
 	r.HandleFunc("/fetch", fetchHandler)
 	r.HandleFunc("/fetch/{id}", fetchHandler)
@@ -68,7 +93,7 @@ func main() {
 
 	srv := &http.Server{
 		Handler: r,
-		Addr:    fmt.Sprintf(":%d", conf.Server.Port),
+		Addr:    fmt.Sprintf(":%d", configService.Config.Server.Port),
 		// Good practice: enforce timeouts for servers you create!
 		WriteTimeout: 5 * time.Second,
 		ReadTimeout:  5 * time.Second,
@@ -84,23 +109,36 @@ func main() {
 
 	// start downloading queued downloads when slots available, and clean up
 	// old entries
-	go func() {
-		for {
-			downloads.StartQueued(conf.Server.MaximumActiveDownloads)
-			downloads = downloads.Cleanup()
-			time.Sleep(time.Second)
-		}
-	}()
+	go dm.ManageQueue()
 
-	log.Printf("starting gropple %s - https://github.com/tardisx/gropple", versionInfo.CurrentVersion)
-	log.Printf("go to %s for details on installing the bookmarklet and to check status", conf.Server.Address)
+	urls := []string{
+		"https://www.youtube.com/watch?v=qG_rRkuGBW8",
+		"https://www.youtube.com/watch?v=ZUzhZpQAU40",
+		// "https://www.youtube.com/watch?v=kVxM3eRWGak",
+		// "https://www.youtube.com/watch?v=pl-y9869y0w",
+		// "https://www.youtube.com/watch?v=Uw4NEPE4l3A",
+		// "https://www.youtube.com/watch?v=6tIsT57_nS0",
+		// "https://www.youtube.com/watch?v=2RF0lcTuuYE",
+		// "https://www.youtube.com/watch?v=lymwNQY0dus",
+		// "https://www.youtube.com/watch?v=NTc-I4Z_duc",
+		// "https://www.youtube.com/watch?v=wNSm1TJ84Ac",
+	}
+	for _, u := range urls {
+		d := download.NewDownload(u, configService.Config)
+		d.DownloadProfile = *configService.Config.ProfileCalled("standard video")
+		dm.AddDownload(d)
+		dm.Queue(d)
+	}
+
+	log.Printf("Visit %s for details on installing the bookmarklet and to check status", configService.Config.Server.Address)
 	log.Fatal(srv.ListenAndServe())
+
 }
 
 // versionRESTHandler returns the version information, if we have up-to-date info from github
 func versionRESTHandler(w http.ResponseWriter, r *http.Request) {
-	if versionInfo.GithubVersionFetched {
-		b, _ := json.Marshal(versionInfo)
+	if versionInfo.GetInfo().GithubVersionFetched {
+		b, _ := json.Marshal(versionInfo.GetInfo())
 		w.Write(b)
 	} else {
 		w.WriteHeader(400)
@@ -111,7 +149,7 @@ func versionRESTHandler(w http.ResponseWriter, r *http.Request) {
 func homeHandler(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusOK)
 
-	bookmarkletURL := fmt.Sprintf("javascript:(function(f,s,n,o){window.open(f+encodeURIComponent(s),n,o)}('%s/fetch?url=',window.location,'yourform','width=%d,height=%d'));", conf.Server.Address, conf.UI.PopupWidth, conf.UI.PopupHeight)
+	bookmarkletURL := fmt.Sprintf("javascript:(function(f,s,n,o){window.open(f+encodeURIComponent(s),n,o)}('%s/fetch?url=',window.location,'yourform','width=%d,height=%d'));", configService.Config.Server.Address, configService.Config.UI.PopupWidth, configService.Config.UI.PopupHeight)
 
 	t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/menu.tmpl", "web/index.html")
 	if err != nil {
@@ -119,23 +157,46 @@ func homeHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	type Info struct {
-		Downloads      []*download.Download
+		Manager        *download.Manager
 		BookmarkletURL template.URL
 		Config         *config.Config
+		Version        version.Info
 	}
 
 	info := Info{
-		Downloads:      downloads,
+		Manager:        dm,
 		BookmarkletURL: template.URL(bookmarkletURL),
-		Config:         conf,
+		Config:         configService.Config,
+		Version:        versionInfo.GetInfo(),
 	}
 
+	dm.Lock.Lock()
+	defer dm.Lock.Unlock()
 	err = t.ExecuteTemplate(w, "layout", info)
 	if err != nil {
 		panic(err)
 	}
 }
 
+// staticHandler handles requests for static files
+func staticHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+
+	filename := vars["filename"]
+	if strings.Index(filename, ".js") == len(filename)-3 {
+		f, err := webFS.Open("web/" + filename)
+		if err != nil {
+			log.Printf("error accessing %s - %v", filename, err)
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+		io.Copy(w, f)
+		return
+	}
+	w.WriteHeader(http.StatusNotFound)
+}
+
 // configHandler returns the configuration page
 func configHandler(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusOK)
@@ -160,7 +221,7 @@ func configRESTHandler(w http.ResponseWriter, r *http.Request) {
 		if err != nil {
 			panic(err)
 		}
-		err = conf.UpdateFromJSON(b)
+		err = configService.Config.UpdateFromJSON(b)
 
 		if err != nil {
 			errorRes := errorResponse{Success: false, Error: err.Error()}
@@ -169,13 +230,12 @@ func configRESTHandler(w http.ResponseWriter, r *http.Request) {
 			w.Write(errorResB)
 			return
 		}
-		conf.WriteConfig()
+		configService.WriteConfig()
 	}
-	b, _ := json.Marshal(conf)
+	b, _ := json.Marshal(configService.Config)
 	w.Write(b)
 }
 
-//
 func fetchInfoOneRESTHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	idString := vars["id"]
@@ -186,23 +246,21 @@ func fetchInfoOneRESTHandler(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 
-		// find the download
-		var thisDownload *download.Download
-		for _, dl := range downloads {
-			if dl.Id == id {
-				thisDownload = dl
-			}
-		}
-		if thisDownload == nil {
+		thisDownload, err := dm.GetDlById(id)
+		if err != nil {
 			http.NotFound(w, r)
 			return
 		}
+		if thisDownload == nil {
+			panic("should not happen")
+		}
 
 		if r.Method == "POST" {
 
 			type updateRequest struct {
-				Action  string `json:"action"`
-				Profile string `json:"profile"`
+				Action      string `json:"action"`
+				Profile     string `json:"profile"`
+				Destination string `json:"destination"`
 			}
 
 			thisReq := updateRequest{}
@@ -223,19 +281,49 @@ func fetchInfoOneRESTHandler(w http.ResponseWriter, r *http.Request) {
 
 			if thisReq.Action == "start" {
 				// find the profile they asked for
-				profile := conf.ProfileCalled(thisReq.Profile)
+				profile := configService.Config.ProfileCalled(thisReq.Profile)
 				if profile == nil {
 					panic("bad profile name?")
 				}
 				// set the profile
+				thisDownload.Lock.Lock()
 				thisDownload.DownloadProfile = *profile
+				thisDownload.Lock.Unlock()
+
+				dm.Queue(thisDownload)
 
-				thisDownload.Queue()
 				succRes := successResponse{Success: true, Message: "download started"}
 				succResB, _ := json.Marshal(succRes)
 				w.Write(succResB)
 				return
 			}
+
+			if thisReq.Action == "change_destination" {
+
+				// nil means (probably) that they chose "don't move" - which is fine,
+				// and maps to nil on the Download (the default state).
+				destination := configService.Config.DestinationCalled(thisReq.Destination)
+
+				thisDownload.Lock.Lock()
+				thisDownload.Destination = destination
+				thisDownload.Lock.Unlock()
+
+				log.Printf("%#v", thisDownload)
+
+				succRes := successResponse{Success: true, Message: "destination changed"}
+				succResB, _ := json.Marshal(succRes)
+				w.Write(succResB)
+				return
+			}
+
+			if thisReq.Action == "stop" {
+
+				thisDownload.Stop()
+				succRes := successResponse{Success: true, Message: "download stopped"}
+				succResB, _ := json.Marshal(succRes)
+				w.Write(succResB)
+				return
+			}
 		}
 
 		// just a get, return the object
@@ -248,7 +336,10 @@ func fetchInfoOneRESTHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func fetchInfoRESTHandler(w http.ResponseWriter, r *http.Request) {
-	b, _ := json.Marshal(downloads)
+
+	dm.Lock.Lock()
+	defer dm.Lock.Unlock()
+	b, _ := json.Marshal(dm.Downloads)
 	w.Write(b)
 }
 
@@ -260,24 +351,29 @@ func fetchHandler(w http.ResponseWriter, r *http.Request) {
 	idString := vars["id"]
 
 	idInt, err := strconv.ParseInt(idString, 10, 32)
+
+	// existing, load it up
 	if err == nil && idInt > 0 {
-		for _, dl := range downloads {
-			if dl.Id == int(idInt) {
-				t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/popup.html")
-				if err != nil {
-					panic(err)
-				}
 
-				templateData := map[string]interface{}{"dl": dl, "config": conf}
-
-				err = t.ExecuteTemplate(w, "layout", templateData)
-				if err != nil {
-					panic(err)
-				}
-				return
-			}
+		dl, err := dm.GetDlById(int(idInt))
+		if err != nil {
+			log.Printf("not found")
+			w.WriteHeader(404)
+			return
 		}
 
+		t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/popup.html")
+		if err != nil {
+			panic(err)
+		}
+
+		templateData := map[string]interface{}{"dl": dl, "config": configService.Config, "canStop": download.CanStopDownload}
+
+		err = t.ExecuteTemplate(w, "layout", templateData)
+		if err != nil {
+			panic(err)
+		}
+		return
 	}
 
 	query := r.URL.Query()
@@ -289,47 +385,36 @@ func fetchHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	} else {
 
+		log.Printf("popup for %s", url)
 		// check the URL for a sudden but inevitable betrayal
-		if strings.Contains(url[0], conf.Server.Address) {
+		if strings.Contains(url[0], configService.Config.Server.Address) {
 			w.WriteHeader(400)
-			fmt.Fprint(w, "you musn't gropple your gropple :-)")
+			fmt.Fprint(w, "you mustn't gropple your gropple :-)")
 			return
 		}
 
-		// create the record
-		// XXX should be atomic!
-		downloadId++
-		newDownload := download.Download{
-			Config: conf,
-
-			Id:       downloadId,
-			Url:      url[0],
-			PopupUrl: fmt.Sprintf("/fetch/%d", downloadId),
-			State:    "choose profile",
-			Finished: false,
-			Eta:      "?",
-			Percent:  0.0,
-			Log:      make([]string, 0, 1000),
-		}
-		downloads = append(downloads, &newDownload)
-		// XXX atomic ^^
-
-		newDownload.Log = append(newDownload.Log, "start of log...")
-
-		// go func() {
-		// 	newDownload.Begin()
-		// }()
+		// create the new download
+		log.Print("creating")
+		newDL := download.NewDownload(url[0], configService.Config)
+		log.Print("adding")
+		dm.AddDownload(newDL)
+		log.Print("done")
 
 		t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/popup.html")
 		if err != nil {
 			panic(err)
 		}
 
-		templateData := map[string]interface{}{"dl": newDownload, "config": conf}
+		log.Print("lock dl")
+		newDL.Lock.Lock()
+		defer newDL.Lock.Unlock()
+
+		templateData := map[string]interface{}{"Version": versionInfo.GetInfo(), "dl": newDL, "config": configService.Config, "canStop": download.CanStopDownload}
 
 		err = t.ExecuteTemplate(w, "layout", templateData)
 		if err != nil {
 			panic(err)
 		}
+		log.Print("unlock dl because rendered")
 	}
 }

version/version.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"sync"
 
 	"golang.org/x/mod/semver"
 )
@@ -19,8 +20,24 @@ type Info struct {
 	GithubVersionFetched bool   `json:"-"`
 }
 
-func (i *Info) UpdateGitHubVersion() error {
-	i.GithubVersionFetched = false
+type Manager struct {
+	VersionInfo Info
+	lock        sync.Mutex
+}
+
+func (m *Manager) GetInfo() Info {
+	// log.Print("getting info... b4 lock")
+	m.lock.Lock()
+	defer m.lock.Unlock()
+
+	return m.VersionInfo
+}
+
+func (m *Manager) UpdateGitHubVersion() error {
+	m.lock.Lock()
+	m.VersionInfo.GithubVersionFetched = false
+	m.lock.Unlock()
+
 	versionUrl := "https://api.github.com/repos/tardisx/gropple/releases"
 	resp, err := http.Get(versionUrl)
 	if err != nil {
@@ -51,29 +68,30 @@ func (i *Info) UpdateGitHubVersion() error {
 		return errors.New("no releases found")
 	}
 
-	i.GithubVersion = releases[0].Name
+	m.lock.Lock()
+
+	defer m.lock.Unlock()
+	m.VersionInfo.GithubVersion = releases[0].Name
+	m.VersionInfo.GithubVersionFetched = true
+	m.VersionInfo.UpgradeAvailable = m.canUpgrade()
 
-	i.GithubVersionFetched = true
-	i.UpgradeAvailable = i.canUpgrade()
 	return nil
 }
 
-func (i *Info) canUpgrade() bool {
-	if !i.GithubVersionFetched {
+func (m *Manager) canUpgrade() bool {
+	if !m.VersionInfo.GithubVersionFetched {
 		return false
 	}
 
-	log.Printf("We are %s, github is %s", i.CurrentVersion, i.GithubVersion)
-
-	if !semver.IsValid(i.CurrentVersion) {
-		log.Printf("current version %s is invalid", i.CurrentVersion)
+	if !semver.IsValid(m.VersionInfo.CurrentVersion) {
+		log.Printf("current version %s is invalid", m.VersionInfo.CurrentVersion)
 	}
 
-	if !semver.IsValid(i.GithubVersion) {
-		log.Printf("github version %s is invalid", i.GithubVersion)
+	if !semver.IsValid(m.VersionInfo.GithubVersion) {
+		log.Printf("github version %s is invalid", m.VersionInfo.GithubVersion)
 	}
 
-	if semver.Compare(i.CurrentVersion, i.GithubVersion) == -1 {
+	if semver.Compare(m.VersionInfo.CurrentVersion, m.VersionInfo.GithubVersion) == -1 {
 		return true
 	}
 	return false

web/config.html

@@ -7,11 +7,17 @@
     <p class="error"  x-show="error_message"  x-transition.duration.500ms x-text="error_message"></p>
     <p class="success" x-show="success_message" x-transition.duration.500ms x-text="success_message"></p>
 
-    <p>Note: changes are not saved until the "Save Config" button is pressed at the bottom of the page.</p>
+    <p>Note: changes are not saved until the "Save Config" button is pressed.</p>
+
+    <div class="pure-g">
+        <div class="pure-u-1">
+            <button class="pure-button pure-button-primary" @click="save_config();" href="#">Save Config</button>
+        </div>
+    </div>
 
     <div class="pure-g">
 
-        <div class="pure-u-md-1-2 pure-u-1">
+        <div class="pure-u-lg-1-3 pure-u-1 l-box">
    
             <form class="pure-form pure-form-stacked gropple-config">
                 <fieldset>
@@ -31,11 +37,11 @@
 
                     <label for="config-server-downloadpath">Download path</label>
                     <input type="text" id="config-server-downloadpath" placeholder="path" class="input-long" x-model="config.server.download_path" />
-                    <span class="pure-form-message">The path on the server to download files to.</span>
+                    <span class="pure-form-message">The default path on the server to download files to.</span>
 
-                    <label for="config-server-max-downloads">Maximum active downloads</label>
+                    <label for="config-server-max-downloads">Maximum active downloads per domain</label>
                     <input type="text" id="config-server-max-downloads" placeholder="2" class="input-long" x-model.number="config.server.maximum_active_downloads_per_domain" />
-                    <span class="pure-form-message">How many downloads can be simultaneously active. Use '0' for no limit.</span>
+                    <span class="pure-form-message">How many downloads can be simultaneously active. Use '0' for no limit. This limit is applied per domain that you download from.</span>
 
                     <legend>UI</legend>
 
@@ -53,7 +59,7 @@
             </form>
 
         </div>
-        <div class="pure-u-md-1-2 pure-u-1">
+        <div class="pure-u-lg-1-3 pure-u-1 l-box">
             <form class="pure-form gropple-config">
                 <fieldset>
 
@@ -91,6 +97,7 @@
                             </template>
 
                             <button class="pure-button button-add" href="#" @click.prevent="profile.args.push('');">add arg</button>
+                            <span class="pure-form-message">Arguments for the command. Note that the shell is not used, so there is no need to quote or escape arguments, including those with spaces.</span>
 
                             <hr>
 
@@ -102,6 +109,39 @@
                 </fieldset>
             </form>
         </div>
+
+        <div class="pure-u-lg-1-3 pure-u-1 l-box">
+            <form class="pure-form gropple-config">
+                <fieldset>
+                    <legend>Destinations</legend>
+                    <p>You can specify custom destinations (directories) here. Downloads can be
+                        moved to one of these directories after completion from the index page, 
+                        if you do not want them to be left in the download path above.</p>
+                    </p>
+                    <template x-for="(dest, i) in config.destinations">
+                    <div>
+                        <label x-bind:for="'config-destinations-'+i+'-name'">Name of destination <span x-text="i+1"></span>
+                        </label>                                
+                        
+                        <input type="text" x-bind:id="'config-destinations-'+i+'-name'" class="input-long" placeholder="name" x-model="dest.name" />
+
+                        <span class="pure-form-message">The name of this destination. For your information only.</span>
+
+                        <label x-bind:for="'config-destinations-'+i+'-command'">Path</label>
+                        <input type="text" x-bind:id="'config-destinations-'+i+'-command'" class="input-long" placeholder="name" x-model="dest.path" />
+                        <span class="pure-form-message">Path to move completed downloads to.</span>
+
+                        <button class="pure-button button-del" href="#" @click.prevent="config.destinations.splice(i, 1);">delete destination</button>
+
+                        <hr>
+
+                    </div>
+                </template>
+
+                <button class="pure-button button-add" href="#" @click.prevent="config.destinations.push({name: 'new destination', path: '/tmp'});">add destination</button>
+
+            </fieldset>
+        </form>
     </div>
     <div class="pure-g">
         <div class="pure-u-1">
@@ -118,7 +158,7 @@
 <script>
     function config() {
         return { 
-            config: { server : {}, ui : {}, profiles: [] }, 
+            config: { server : {}, ui : {}, profiles: [], destinations: []}, 
             error_message: '',
             success_message: '',
 

web/index.html

@@ -31,7 +31,18 @@
             <template x-for="item in items">
                 <tr>
                     <td x-text="item.id"></td>
-                    <td x-text="item.files"></td>
+                    <td>
+                        <span x-show="item.files && item.files.length > 0">
+                            <ul>
+                                <template x-for="file in item.files">
+                                    <li x-text="file"></li>
+                                </template>
+                            </ul>
+                        </span>
+                        <span x-show="! item.files || item.files.length == 0"
+                              x-text="item.url">
+                        </span>
+                    </td>
                     <td><a class="int-link" x-bind:href="item.url">&#x2197;</a></td>
                     <td><a class="int-link" @click="show_popup(item)" href="#">&#x1F4C4;</a></td>
                     <td :class="'state-'+item.state" x-text="item.state"></td>
@@ -43,8 +54,6 @@
             </template>
               
 
-            {{ range $k, $v := .Downloads }}
-            {{ end }}
         </tbody>  
     </table>
 </div>
@@ -54,7 +63,7 @@
 <script>
     function index() {
         return { 
-            items: [], version: {},
+            items: [], version: {}, popups: {},
             fetch_version() {
                 fetch('/rest/version')
                 .then(response => response.json())
@@ -79,9 +88,11 @@
                 })
             },
             show_popup(item) {
-			    window.open(item.popup_url, item.id, "width={{ .Config.UI.PopupWidth }},height={{ .Config.UI.PopupHeight }}");
-	    },
-        }
+                // allegedly you can use the reference to pop the window to the front on subsequent
+                // clicks, but I can't seem to find a reliable way to do so.
+                this.popups[item.id] = window.open(item.popup_url, item.id, "width={{ .Config.UI.PopupWidth }},height={{ .Config.UI.PopupHeight }}");
+            },
+	    }
     }
 </script>
 {{ end }}

web/layout.tmpl

@@ -3,11 +3,17 @@
 <head>
     <meta charset="utf-8">
     <title>gropple</title>
-    <script src="//unpkg.com/alpinejs" defer></script>
+    <script src="/static/alpine.min.js" defer></script>
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <link rel="stylesheet" href="https://unpkg.com/purecss@2.0.6/build/pure-min.css" integrity="sha384-Uu6IeWbM+gzNVXJcM9XV3SohHtmWE+3VGi496jvgX1jyvDTXfdK+rfZc8C1Aehk5" crossorigin="anonymous">
     <link rel="stylesheet" href="https://unpkg.com/purecss@2.0.6/build/grids-responsive-min.css">
     <style>
+        .pure-g > div {
+          box-sizing: border-box;
+        }
+        .l-box {
+          padding: 2em;
+         }
         pre {
           font-size: 60%;
           height: 100px;
@@ -51,6 +57,7 @@
         }
         .error {
           color: red;
+          font-size: 150%;
         }
         .success {
           color: green;
@@ -64,7 +71,8 @@
 
     {{ template "content" . }}
     <footer>
-    Homepage: <a href="https://github.com/tardisx/gropple">https://github.com/tardisx/gropple</a>
+    Homepage: <a href="https://github.com/tardisx/gropple">https://github.com/tardisx/gropple</a><br>
+    Version: {{ .Version.CurrentVersion }}
     </footer>
 </body>
     {{ template "js" . }}

web/popup.html

@@ -15,11 +15,27 @@
                 </td>
             </tr>
             <tr><th>current filename</th><td x-text="filename"></td></tr>
+            <tr>
+                <th>destination</th>
+                <td>
+                    <select  x-on:change="update_destination()" class="pure-input-1-2" x-model="destination_chosen">
+                        <option value="-">leave in {{ .config.Server.DownloadPath }}</option>
+                    {{ range $i := .config.Destinations }}
+                        <option>{{ $i.Name }}</option>
+                    {{ end }}
+                    </select>
+                </td>
+            </tr>
             <tr><th>state</th><td x-text="state"></td></tr>
+            <tr x-show="playlist_total > 0"><th>playlist progress</th><td x-text="playlist_current + '/' + playlist_total"></td></tr>
             <tr><th>progress</th><td x-text="percent"></td></tr>
             <tr><th>ETA</th><td x-text="eta"></td></tr>
+            
         </table>
         <p>You can close this window and your download will continue. Check the <a href="/" target="_gropple_status">Status page</a> to see all downloads in progress.</p>
+        {{ if .canStop }}
+        <button x-show="state=='downloading'" class="pure-button" @click="stop()">stop</button>
+        {{ end }}
         <div>
             <h4>Logs</h4>
             <pre x-text="log">
@@ -33,7 +49,9 @@
         history.replaceState(null, '', ['/fetch/{{ .dl.Id }}'])
         return { 
             eta: '', percent: 0.0, state: '??', filename: '', finished: false, log :'',
+            playlist_current: 0, playlist_total: 0,
             profile_chosen: null,
+            destination_chosen: null,
             watch_profile() {
                 this.$watch('profile_chosen', value => this.profile_chosen(value))
             },
@@ -50,6 +68,30 @@
                     console.log(info)
                 })
             },
+            update_destination(name) {
+                let op = {
+                   method: 'POST',
+                   body: JSON.stringify({action: 'change_destination', destination: this.destination_chosen}),
+                   headers: { 'Content-Type': 'application/json' }
+                };
+                fetch('/rest/fetch/{{ .dl.Id }}', op)
+                .then(response => response.json())
+                .then(info => {
+                    console.log(info)
+                })
+            },
+            stop() {
+                let op = {
+                   method: 'POST',
+                   body: JSON.stringify({action: 'stop'}),
+                   headers: { 'Content-Type': 'application/json' }
+                };
+                fetch('/rest/fetch/{{ .dl.Id }}', op)
+                .then(response => response.json())
+                .then(info => {
+                    console.log(info)
+                })
+            },
             fetch_data() {
                 fetch('/rest/fetch/{{ .dl.Id }}')
                 .then(response => response.json())
@@ -57,6 +99,8 @@
                     this.eta = info.eta;
                     this.percent = info.percent + "%";
                     this.state = info.state;
+                    this.playlist_current = info.playlist_current;
+                    this.playlist_total = info.playlist_total;
                     if (this.state != 'choose profile') {
                         this.profile_chosen = true;
                     }