Be specific about version

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: tardisx

.github/workflows/go.yml

@@ -16,7 +16,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.17
+        go-version: 1.22
 
     - name: Build
       run: go build -v ./...

.gitignore

@@ -1,3 +1,6 @@
 gropple
-dist
 release
+dist
+.env
+dist/
+.DS_Store

.goreleaser.yaml

@@ -0,0 +1,53 @@
+version: 2
+
+before:
+  hooks:
+    - go mod tidy
+    - go test ./...
+
+builds:
+  - env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - windows
+      - darwin
+
+archives:
+  - formats: [tar.gz]
+    # this name template makes the OS and Arch compatible with the results of `uname`.
+    name_template: >-
+      {{ .ProjectName }}_
+      {{- title .Os }}_
+      {{- if eq .Arch "amd64" }}x86_64
+      {{- else if eq .Arch "386" }}i386
+      {{- else }}{{ .Arch }}{{ end }}
+      {{- if .Arm }}v{{ .Arm }}{{ end }}
+    # use zip for windows archives
+    format_overrides:
+      - goos: windows
+        formats: [zip]
+
+changelog:
+  disable: true
+
+dockers:
+  - image_templates:
+      - "tardisx/gropple:{{ .Tag }}-amd64"
+    use: buildx
+    build_flag_templates:
+      - "--pull"
+      - "--platform=linux/amd64"
+  - image_templates:
+      - "tardisx/gropple:{{ .Tag }}-arm64"
+    use: buildx
+    build_flag_templates:
+      - "--pull"
+      - "--platform=linux/arm64"
+    goarch: arm64
+
+docker_manifests:
+  - name_template: "tardisx/gropple:{{ .Tag }}"
+    image_templates:
+      - "tardisx/gropple:{{ .Tag }}-amd64"
+      - "tardisx/gropple:{{ .Tag }}-arm64"

.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Launch Package",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${fileDirname}"
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,12 @@
+{
+    "cSpell.words": [
+        "Cleanup",
+        "frag",
+        "gropple",
+        "succ",
+        "tmpl",
+        "vars",
+        "youtube"
+    ],
+    "cSpell.language": "en-GB"
+}

CHANGELOG.md

@@ -2,12 +2,83 @@
 
 All notable changes to this project will be documented in this file.
 
-## [Unreleased]
+## [v1.1.4] - 2025-04-25
+
+- Minor refactorings, upgrade dependencies
+- Migrate to goreleaser v2 configuration
+- Generate arm64 docker builds
+
+## [v1.1.3] - 2024-03-17
+
+- Code cleanups, better error checking
+
+## [v1.1.2] - 2024-03-16
+
+- Fix a crash for a certain pattern of log line
+
+## [v1.1.1] - 2023-12-08
+
+- Fix bug where a brand-new config was created with an out-of-date version
+- Fix for portable mode and using executable in the current working directory
+
+## [v1.1.0] - 2023-11-25
+
+- Add feature to bulk add URL's for downloading
+
+## [v1.0.1] - 2023-11-24
+
+- Fix crash on migrating a config that had > 1 destinations
+
+## [v1.0.0] - 2023-11-23
+
+- Don't start downloads until "start download" is pressed
+- Add "download option" for more per-download customisability, especially for destinations
+- Removed "destinations" as that is now possible more flexibly with download options.
+  - Existing configurations using destinations are automatically migrated to an appropriate `yt-dlp -o ...` download options
+- Gropple now available via docker
+- Clean up web interface display on index page, especially when a playlist with many files is downloading
+
+## [v0.6.0] - 2023-03-15
+
+- Configurable destinations for downloads
+  - Multiple destination directories can be configured
+  - When queueing a download, an alternate destination can be selected
+- When downloading from a playlist, show the total number of videos and how many have been downloaded
+- Show version in web UI
+- Improve index page (show URL of queued downloads instead of nothing)
+- Fixes and improvements to capturing output info and showing it in the UI
+- Show all log output in the popup
+- Fixes to handling of queued downloads
+- Fix portable mode to look in binary directory, not current directory
+- Automatically cleanup download list, removing old entries automatically
+
+## [v0.5.5] - 2022-04-09
+
+- Fix a bug which would erase configuration when migrating from v1 to v2 config
+
+## [v0.5.4] - 2022-04-07
+
+- Check the chosen command exists when configuring a profile
+- Improve documentation
+- Add a stop button in the popup to abort a download (Linux/Mac only)
+- Move included JS to local app instead of accessing from a CDN
+- Make the simultaneous download limit apply to each unique domain
+- Support "portable" mode, reading gropple.yml from the current directory, if present
+
+## [v0.5.3] - 2021-11-21
+
+- Add config option to limit number of simultaneous downloads
+- Remove old download entries from the index after they are complete
+
+## [v0.5.2] - 2021-10-26
+
+- Provide link to re-display the popup window from the index
+- Visual improvements
 
 ## [v0.5.1] - 2021-10-25
 
 - Add note about adblockers potentially blocking the popup
-- Make it possible to refresh the popup window without initating a new download
+- Make it possible to refresh the popup window without initiating a new download
 
 ## [v0.5.0] - 2021-10-01
 

Dockerfile

@@ -0,0 +1,9 @@
+FROM ubuntu:noble
+COPY gropple /
+
+RUN apt update &&  apt install -y curl python3 ffmpeg
+RUN curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/bin/yt-dlp
+RUN chmod a+x /usr/bin/yt-dlp
+
+# Run executable
+CMD ["/gropple", "--config-path", "/config/gropple.json"]

README.md

@@ -1,90 +1,196 @@
 # gropple
 
-A frontend to youtube-dl (and forks) to download videos with a single click, straight from your web browser.
+A frontend to youtube-dl (or compatible forks, like yt-dlp) to download videos
+with a single click, straight from your web browser.
 
 ![Screencast](/screencast.gif)
 
-## Pre-requisites
+## Installing
 
-* some familiarity with the command line
+### From Source
-* youtube-dl (plus any of its required dependencies, like ffmpeg)
-* golang compiler (only if you'd like to build from source)
-
-## Build
 
     go build
 
-## Binaries
+### Standalone Binaries
 
-Binaries are available at https://github.com/tardisx/gropple/releases
+Binaries are available at <https://github.com/tardisx/gropple/releases> for most
-
+platforms.
-Gropple will automatically check for available updates and prompt you to upgrade.
 
 ## Running
 
-    ./gropple
+### From Binaries
+
+     ./gropple
 
 There are no command line arguments. All configuration is done via the web
 interface. The address will be printed after startup:
 
-    2021/09/30 23:53:00 starting gropple v0.5.0 - https://github.com/tardisx/gropple
+    2023/11/22 22:42:06 Starting gropple v1.0.0 - https://github.com/tardisx/gropple
-    2021/09/30 23:53:00 go to http://localhost:6123 for details on installing the bookmarklet and to check status
+    2023/11/22 22:42:07 Configuration loaded from /Users/username/path/config.yml
+    2023/11/22 22:42:07 Visit http://localhost:6123 for details on installing the bookmarklet and to check status
+
+### Docker
+
+Copy the `docker-compose.yml` to a directory somewhere.
+
+Edit the two `volume` entries to point to local paths where you would like to
+store the config file, and the downloads (the path on the left hand side of the
+colon).
+
+Run `docker-compose up -d` to start the program.
+
+Note that the docker images include `yt-dlp` and `ffmpeg` and are thus
+completely self-contained.
+
+Run `docker-compose logs` to see the output of the program, if you are having
+problems.
 
 ## Using
 
-Bring up `http://localhost:6283` (or your configured address) in your browser. You 
+Bring up `http://localhost:6283` (or the appropriate host if you are running it
-should see a link to the bookmarklet at the top of the screen, and the list of
+on a different machine) in your browser. You should see a link to the
-downloads (currently empty).
+bookmarklet at the top of the screen, and the list of downloads (currently
+empty).
 
-Drag the bookmarklet to your favourites bar, or otherwise bookmark it as you 
+Drag the bookmarklet to your favourites bar, or otherwise bookmark it as you see
-see fit. Any kind of browser bookmark should work. The bookmarklet contains
+fit. Any kind of browser bookmark should work. The bookmarklet contains embedded
-embedded javascript to pass the URL of whatever page you are currently on back
+javascript to pass the URL of whatever page you are currently on back to
-to gropple.
+gropple.
 
-So, whenever you are on a page with a video you would like to download just 
+Whenever you are on a page with a video you would like to download just click
-click the bookmarklet.
+the bookmarklet.
 
-A popup window will appear. Choose a download profile and the download will start.
+A popup window will appear. Choose a download profile and the download will
-The status will be shown in the window, updating in real time.
+start. The status will be shown in the window, updating in real time.
 
-You may close this window at any time without stopping the download, the status 
+There is also an optional "download option" you can choose. These are discussed
-of all downloads is available on the index page.
+below.
+
+You may close this window at any time without stopping the download, the status
+of all downloads is available on the index page. Clicking on the id number will
+show the popup again.
 
 ## Configuration
 
-Click the "config" link on the index page to configure gropple. The default options 
+Click the "config" link on the index page to configure gropple.
-are fine if you are running on your local machine. If you are running it remotely
-you will need to set the "server address" to ensure the bookmarklet has the correct
-URL in it.
 
-### Configuring Downloaders
+The options in each part are dicussed below.
 
-Gropple's default configuration uses the original youtube-dl and has two profiles set
+### Server
-up, one for downloading video, the other for downloading audio (mp3).
 
-Note that gropple does not include any downloaders, you have to install them separately.
+#### Port and Server Address
 
-If you would like to use a youtube-dl fork (like [yt-dlp](https://github.com/yt-dlp/yt-dlp))
+You can configure the port number here if you do not want the default of `6123`.
-or change the options, you can do so on the right hand side. Create as many profiles as you 
-wish, whenever you start a download you can choose the appropriate profile.
 
-Note that the command arguments must each be specified separately - see the default configuration
+If you are running it on a machine other than `localhost` you will need to set
-for an example.
+the "server address" to ensure the bookmarklet has the correct URL in it.
+Similarly, if you are running it behind a reverse proxy, the address here must
+match what you would type in the browser so that the bookmarklet will work
+correctly.
 
-While gropple will use your `PATH` to find the executable, you can also specify a full path
+#### Download path
-instead. Note that any tools that the downloader calls itself (for instance, ffmpeg) will 
+
-probably need to be available on your path.
+The download path specifies where downloads will end up, *if* no specific `-o`
+options are passed to `yt-dlp`.
+
+#### Maximum active downloads per domain
+
+Gropple will limit the number of downloads per domain to this number. Increasing
+this will likely result in failed downloads when server rate limiters notice
+you.
+
+#### UI popup size
+
+Changes the size of the popup window.
+
+### Download Profiles
+
+Gropple's default configuration uses `yt-dlp` and has two profiles set up, one
+for downloading video, the other for downloading audio (mp3).
+
+Each download profile consists of a name (for your reference), a command to run,
+and a number of arguments.
+
+Note that gropple does not include any downloaders, you have to install them
+separately (unless using the docker image).
+
+If you would like to use a youtube-dl compatible fork or change the options you
+can do so here. Create as many profiles as you wish, whenever you start a
+download you can choose the appropriate profile.
+
+Note that the command arguments must each be specified separately - see the
+default configuration. For example, if you have a single argument like
+`--audio-format mp3`, it will be parsed by the `yt-dlp` as a single, long
+unknown argument, and will fail. This needs to be configured as two arguments,
+`--audio-format` and `mp3`.
+
+While gropple will use your `PATH` to find the executable, you can also specify
+a full path instead. Note that any tools that the downloader calls itself (for
+instance, `ffmpeg`) will need to be available on your path.
+
+### Download Options
+
+There are also an arbitrary amount of Download Options you can configure. Each
+one specifies one or more extra arguments to add to the downloader command line.
+The most common use for this is to have customised download paths. For instance,
+sometimes you might want to bundle all files into a single directory, other
+times you might want to separate files by download playlist URL or similar.
+
+Most of this is done directly through appropriate options for `yt-dlp`, see the
+[output template
+documentation](https://github.com/yt-dlp/yt-dlp#output-template).
+
+However, gropple offers two extra substitutions:
+
+  * `%GROPPLE_HOST%`
+  * `%GROPPLE_PATH%`
+
+These will be replaced with the hostname, and path of the download,
+respectively.
+
+So, a playlist URL `https://www.youtube.com/@UsernameHere`
+
+With a download option setup like this:
+
+    * Name of Option: Split by Host and Path
+    * Arguments:
+      * -o
+      * /Downloads/%GROPPLE_HOST%/%GROPPLE_PATH%/%(title)s [%(id)s].%(ext)s
+
+Will result in downloads going into the path
+`/Downloads/www.youtube.com/@UsernameHere/...`.
+
+Note that this also means that `yt-dlp` can resume partially downloaded files, and
+also automatically 'backfill', downloading only files that have not been
+downloaded yet from that playlist.
+
+## Downloading a list of URL's in bulk
+
+From main index page you can click the "Bulk" link in the menu to bring up the
+bulk queue page.
+
+In all respects this acts the same as the usual bookmarklet, but it has a
+textbox for pasting many URLs at once. All downloads will be queued immediately.
+
+## Portable mode
+
+If you'd like to use gropple from a USB stick or similar, copy the config file
+from its default location (shown when you start gropple) to the same location as
+the binary, and rename it to `gropple.yml`.
 
 ## Problems
 
-Most download problems are probably diagnosable via the log - check in the popup window and scroll
+Many download problems are diagnosable via the log - check in the popup window
-the log down to the bottom. The most common problem is that youtube-dl cannot be found, or its
+and scroll the log down to the bottom. The most common problem is that `yt-dlp`
-dependency (like ffmpeg) cannot be found on your path. 
+cannot be found, or its dependency (like `ffmpeg`) cannot be found on your path.
+
+Gropple only calls external tools like `yt-dlp` to do the downloading. If you
+are having problems downloading from a site, make sure that `yt-dlp` is updated
+to the latest version (`yd-dlp -U`).
 
 For other problems, please file an issue on github.
 
 ## TODO
 
-Many things. Please raise an issue after checking the [currently open issues](https://github.com/tardisx/gropple/issues).
+Many things. Please raise an issue after checking the [currently open
-
+issues](https://github.com/tardisx/gropple/issues).
-

build_release.pl

@@ -1,40 +0,0 @@
-#!/usr/bin/env perl
-
-use strict;
-use warnings;
-
-open my $fh, "<", "main.go" || die $!;
-
-my $version;
-while (<$fh>) {
-  # CurrentVersion: "v0.04"
-  $version = $1 if /CurrentVersion:\s*"(v[\d\.]+)"/;
-}
-close $fh;
-
-die "no version?" unless defined $version;
-
-# quit if tests fail
-system("go test ./...") && die "not building release with failing tests";
-
-# so lazy
-system "rm", "-rf", "release", "dist";
-system "mkdir", "release";
-system "mkdir", "dist";
-
-my %build = (
-  win   => { env => { GOOS => 'windows', GOARCH => 'amd64' }, filename => 'gropple.exe' },
-  linux => { env => { GOOS => 'linux',   GOARCH => 'amd64' }, filename => 'gropple' },
-  mac   => { env => { GOOS => 'darwin',  GOARCH => 'amd64' }, filename => 'gropple' },
-); 
-
-foreach my $type (keys %build) {
-  mkdir "release/$type";
-}
-
-foreach my $type (keys %build) {
-  local $ENV{GOOS}   = $build{$type}->{env}->{GOOS};
-  local $ENV{GOARCH} = $build{$type}->{env}->{GOARCH};
-  system "go", "build", "-o", "release/$type/" . $build{$type}->{filename};
-  system "zip", "-j", "dist/gropple-$type-$version.zip", ( glob "release/$type/*" );
-}

config/config.go

@@ -6,44 +6,77 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"os/exec"
+	"path/filepath"
 	"strings"
 
 	"gopkg.in/yaml.v2"
 )
 
 type Server struct {
-	Port         int    `yaml:"port" json:"port"`
+	Port                   int    `yaml:"port" json:"port"`
-	Address      string `yaml:"address" json:"address"`
+	Address                string `yaml:"address" json:"address"`
-	DownloadPath string `yaml:"download_path" json:"download_path"`
+	DownloadPath           string `yaml:"download_path" json:"download_path"`
+	MaximumActiveDownloads int    `yaml:"maximum_active_downloads_per_domain" json:"maximum_active_downloads_per_domain"`
 }
 
+// DownloadProfile holds the details for executing a downloader
 type DownloadProfile struct {
 	Name    string   `yaml:"name" json:"name"`
 	Command string   `yaml:"command" json:"command"`
 	Args    []string `yaml:"args" json:"args"`
 }
 
+// DownloadOption contains configuration for extra arguments to pass to the download command
+type DownloadOption struct {
+	Name string   `yaml:"name" json:"name"`
+	Args []string `yaml:"args" json:"args"`
+}
+
+// UI holds the configuration for the user interface
 type UI struct {
 	PopupWidth  int `yaml:"popup_width" json:"popup_width"`
 	PopupHeight int `yaml:"popup_height" json:"popup_height"`
 }
 
+// Destination is the path for a place that a download can be moved to
+type Destination struct {
+	Name string `yaml:"name" json:"name"` // Name for this location
+	Path string `yaml:"path" json:"path"` // Path on disk
+}
+
+// Config is the top level of the user configuration
 type Config struct {
 	ConfigVersion    int               `yaml:"config_version" json:"config_version"`
 	Server           Server            `yaml:"server" json:"server"`
 	UI               UI                `yaml:"ui" json:"ui"`
+	Destinations     []Destination     `yaml:"destinations" json:"destinations"` // no longer in use, see DownloadOptions
 	DownloadProfiles []DownloadProfile `yaml:"profiles" json:"profiles"`
+	DownloadOptions  []DownloadOption  `yaml:"download_options" json:"download_options"`
 }
 
-func DefaultConfig() *Config {
+// ConfigService is a struct to handle configuration requests, allowing for the
+// location that config files are loaded to be customised.
+type ConfigService struct {
+	Config     *Config
+	ConfigPath string
+}
+
+func (cs *ConfigService) LoadTestConfig() {
+	cs.LoadDefaultConfig()
+	cs.Config.Server.DownloadPath = "/tmp"
+	cs.Config.DownloadProfiles = []DownloadProfile{{Name: "test profile", Command: "/bin/sleep", Args: []string{"5"}}}
+}
+
+func (cs *ConfigService) LoadDefaultConfig() {
 	defaultConfig := Config{}
-	stdProfile := DownloadProfile{Name: "standard video", Command: "youtube-dl", Args: []string{
+	stdProfile := DownloadProfile{Name: "standard video", Command: "yt-dlp", Args: []string{
 		"--newline",
 		"--write-info-json",
 		"-f",
 		"bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best",
 	}}
-	mp3Profile := DownloadProfile{Name: "standard mp3", Command: "youtube-dl", Args: []string{
+	mp3Profile := DownloadProfile{Name: "standard mp3", Command: "yt-dlp", Args: []string{
 		"--newline",
 		"--write-info-json",
 		"--extract-audio",
@@ -55,16 +88,23 @@ func DefaultConfig() *Config {
 
 	defaultConfig.Server.Port = 6123
 	defaultConfig.Server.Address = "http://localhost:6123"
-	defaultConfig.Server.DownloadPath = "./"
+	defaultConfig.Server.DownloadPath = "/downloads"
 
 	defaultConfig.UI.PopupWidth = 500
 	defaultConfig.UI.PopupHeight = 500
 
-	defaultConfig.ConfigVersion = 1
+	defaultConfig.Server.MaximumActiveDownloads = 2
+
+	defaultConfig.Destinations = nil
+	defaultConfig.DownloadOptions = make([]DownloadOption, 0)
+
+	defaultConfig.ConfigVersion = 4
+
+	cs.Config = &defaultConfig
 
-	return &defaultConfig
 }
 
+// ProfileCalled returns the corresponding DownloadProfile, or nil if it does not exist
 func (c *Config) ProfileCalled(name string) *DownloadProfile {
 	for _, p := range c.DownloadProfiles {
 		if p.Name == name {
@@ -74,6 +114,16 @@ func (c *Config) ProfileCalled(name string) *DownloadProfile {
 	return nil
 }
 
+// DownloadOptionCalled returns the corresponding DownloadOption, or nil if it does not exist
+func (c *Config) DownloadOptionCalled(name string) *DownloadOption {
+	for _, o := range c.DownloadOptions {
+		if o.Name == name {
+			return &o
+		}
+	}
+	return nil
+}
+
 func (c *Config) UpdateFromJSON(j []byte) error {
 	newConfig := Config{}
 	err := json.Unmarshal(j, &newConfig)
@@ -104,6 +154,10 @@ func (c *Config) UpdateFromJSON(j []byte) error {
 		return fmt.Errorf("path '%s' is not a directory", newConfig.Server.DownloadPath)
 	}
 
+	if newConfig.Server.MaximumActiveDownloads < 0 {
+		return fmt.Errorf("maximum active downloads can not be < 0")
+	}
+
 	// check profile name uniqueness
 	for i, p1 := range newConfig.DownloadProfiles {
 		for j, p2 := range newConfig.DownloadProfiles {
@@ -134,13 +188,41 @@ func (c *Config) UpdateFromJSON(j []byte) error {
 			}
 		}
 
+		// check the command exists
+
+		_, err := AbsPathToExecutable(newConfig.DownloadProfiles[i].Command)
+		if err != nil {
+			return fmt.Errorf("problem with command '%s': %s", newConfig.DownloadProfiles[i].Command, err)
+		}
 	}
 
 	*c = newConfig
 	return nil
 }
 
-func configPath() string {
+// DetermineConfigDir determines where the config is (or should be) stored.
+func (cs *ConfigService) DetermineConfigDir() {
+	// check binary path first, for a file called gropple.yml
+	binaryPath := os.Args[0]
+	binaryDir := filepath.Dir(binaryPath)
+	potentialConfigPath := filepath.Join(binaryDir, "gropple.yml")
+
+	_, err := os.Stat(potentialConfigPath)
+	if err == nil {
+		// exists in binary directory, use that
+		// fully qualify, just for clarity in the log
+		config, err := filepath.Abs(potentialConfigPath)
+		if err == nil {
+			log.Printf("found portable config in %s", config)
+			cs.ConfigPath = config
+			return
+		} else {
+			log.Printf("got error when trying to convert config to absolute path: %s", err)
+			log.Print("falling back to using UserConfigDir")
+		}
+	}
+
+	// otherwise fall back to using the UserConfigDir
 	dir, err := os.UserConfigDir()
 	if err != nil {
 		log.Fatalf("cannot find a directory to store config: %v", err)
@@ -158,58 +240,127 @@ func configPath() string {
 	}
 
 	fullFilename := fullPath + string(os.PathSeparator) + "config.yml"
-	return fullFilename
+	cs.ConfigPath = fullFilename
 }
 
-func ConfigFileExists() bool {
+// ConfigFileExists checks if the config file already exists, and also checks
-	info, err := os.Stat(configPath())
+// if there is an error accessing it
+func (cs *ConfigService) ConfigFileExists() (bool, error) {
+	path := cs.ConfigPath
+	info, err := os.Stat(path)
 	if os.IsNotExist(err) {
-		return false
+		return false, nil
 	}
 	if err != nil {
-		log.Fatal(err)
+		return false, fmt.Errorf("could not check if '%s' exists: %s", path, err)
 	}
 	if info.Size() == 0 {
-		log.Print("config file is 0 bytes?")
+		return false, errors.New("config file is 0 bytes")
-		return false
 	}
-	return true
+	return true, nil
 }
 
-func LoadConfig() (*Config, error) {
+// LoadConfig loads the configuration from disk, migrating and updating it to the
-	path := configPath()
+// latest version if needed.
+func (cs *ConfigService) LoadConfig() error {
+	path := cs.ConfigPath
 	b, err := os.ReadFile(path)
 	if err != nil {
-		log.Printf("Could not read config '%s': %v", path, err)
+		return fmt.Errorf("could not read config '%s': %v", path, err)
-		return nil, err
 	}
 	c := Config{}
+	cs.Config = &c
+
 	err = yaml.Unmarshal(b, &c)
 	if err != nil {
-		log.Printf("Could not parse YAML config '%s': %v", path, err)
+		return fmt.Errorf("could not parse YAML config '%s': %v", path, err)
-		return nil, err
 	}
-	return &c, nil
+
+	// do migrations
+	configMigrated := false
+	if c.ConfigVersion == 1 {
+		c.Server.MaximumActiveDownloads = 2
+		c.ConfigVersion = 2
+		configMigrated = true
+		log.Print("migrated config from version 1 => 2")
+
+	}
+
+	if c.ConfigVersion == 2 {
+		c.Destinations = make([]Destination, 0)
+		c.ConfigVersion = 3
+		configMigrated = true
+		log.Print("migrated config from version 2 => 3")
+	}
+
+	if c.ConfigVersion == 3 {
+		c.ConfigVersion = 4
+		for i := range c.Destinations {
+			newDownloadOption := DownloadOption{
+				Name: c.Destinations[i].Name,
+				Args: []string{"-o", fmt.Sprintf("%s/%%(title)s [%%(id)s].%%(ext)s", c.Destinations[i].Path)},
+			}
+			c.DownloadOptions = append(c.DownloadOptions, newDownloadOption)
+		}
+		c.Destinations = nil
+		configMigrated = true
+		log.Print("migrated config from version 3 => 4")
+	}
+
+	if configMigrated {
+		log.Print("Writing new config after version migration")
+		cs.WriteConfig()
+	}
+
+	return nil
 }
 
-func (c *Config) WriteConfig() {
+// WriteConfig writes the in-memory config to disk.
-	s, err := yaml.Marshal(c)
+func (cs *ConfigService) WriteConfig() {
+	s, err := yaml.Marshal(cs.Config)
 	if err != nil {
-		panic(err)
+		log.Printf("error writing config: %s", err)
+		os.Exit(1)
 	}
 
-	path := configPath()
+	path := cs.ConfigPath
 	file, err := os.Create(
 		path,
 	)
 
 	if err != nil {
-		log.Fatalf("Could not open config file")
+		log.Fatalf("Could not open config file %s: %s", path, err)
 	}
 	defer file.Close()
 
-	file.Write(s)
+	_, err = file.Write(s)
+	if err != nil {
+		log.Fatalf("could not write config file %s: %s", path, err)
+	}
 	file.Close()
-
+}
-	log.Printf("Wrote configuration out to %s", path)
+
+// AbsPathToExecutable takes a command name, which may or may not be path-qualified,
+// and returns the fully qualified path to it, or an error if could not be found, or
+// if it does not appear to be a file.
+func AbsPathToExecutable(cmd string) (string, error) {
+
+	pathCmd, err := exec.LookPath(cmd)
+	if err != nil {
+		return "", fmt.Errorf("could not LookPath '%s': %w", cmd, err)
+	}
+
+	execAbsolutePath, err := filepath.Abs(pathCmd)
+	if err != nil {
+		return "", fmt.Errorf("could not get absolute path to '%s': %w", cmd, err)
+	}
+	fi, err := os.Stat(execAbsolutePath)
+	if err != nil {
+		return "", fmt.Errorf("could not get stat '%s': %w", cmd, err)
+	}
+	if !fi.Mode().IsRegular() {
+		return "", fmt.Errorf("'%s' is not a regular file: %w", cmd, err)
+	}
+
+	return execAbsolutePath, nil
 }

config/config_test.go

@@ -0,0 +1,220 @@
+package config
+
+import (
+	"errors"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMigrationV1toV4(t *testing.T) {
+	v1Config := `config_version: 1
+server:
+  port: 6123
+  address: http://localhost:6123
+  download_path: ./
+ui:
+  popup_width: 500
+  popup_height: 500
+profiles:
+- name: standard video
+  command: youtube-dl
+  args:
+  - --newline
+  - --write-info-json
+  - -f
+  - bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best
+- name: standard mp3
+  command: youtube-dl
+  args:
+  - --newline
+  - --write-info-json
+  - --extract-audio
+  - --audio-format
+  - mp3
+`
+	cs := configServiceFromString(v1Config)
+	err := cs.LoadConfig()
+	if err != nil {
+		t.Errorf("got error when loading config: %s", err)
+	}
+	if cs.Config.ConfigVersion != 4 {
+		t.Errorf("did not migrate version (it is '%d')", cs.Config.ConfigVersion)
+	}
+	if cs.Config.Server.MaximumActiveDownloads != 2 {
+		t.Error("did not add MaximumActiveDownloads")
+	}
+	if len(cs.Config.Destinations) != 0 {
+		t.Error("incorrect number of destinations added")
+	}
+	os.Remove(cs.ConfigPath)
+}
+
+func TestMigrateV3toV4(t *testing.T) {
+	v3Config := `config_version: 3
+server:
+  port: 6123
+  address: http://localhost:6123
+  download_path: /tmp/Downloads
+  maximum_active_downloads_per_domain: 2
+ui:
+  popup_width: 900
+  popup_height: 900
+destinations:
+  - name: cool destination
+    path: /tmp/coolness
+profiles:
+  - name: standard video
+    command: yt-dlp
+    args:
+      - --newline
+      - --write-info-json
+      - -f
+      - bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best
+  - name: standard mp3
+    command: yt-dlp
+    args:
+      - --newline
+      - --write-info-json
+      - --extract-audio
+      - --audio-format
+      - mp3`
+	cs := configServiceFromString(v3Config)
+	err := cs.LoadConfig()
+	if err != nil {
+		t.Errorf("got error when loading config: %s", err)
+	}
+	if cs.Config.ConfigVersion != 4 {
+		t.Errorf("did not migrate version (it is '%d')", cs.Config.ConfigVersion)
+	}
+	if cs.Config.Server.MaximumActiveDownloads != 2 {
+		t.Error("did not add MaximumActiveDownloads")
+	}
+	if len(cs.Config.Destinations) != 0 {
+		t.Error("incorrect number of destinations from migrated file")
+	}
+	if assert.Len(t, cs.Config.DownloadOptions, 1) {
+		if assert.Len(t, cs.Config.DownloadOptions[0].Args, 2) {
+			assert.Equal(t, "-o", cs.Config.DownloadOptions[0].Args[0])
+			assert.Equal(t, "/tmp/coolness/%(title)s [%(id)s].%(ext)s", cs.Config.DownloadOptions[0].Args[1])
+		}
+	}
+	os.Remove(cs.ConfigPath)
+}
+
+func TestMigrateV3toV4CrashBug(t *testing.T) {
+	v3Config := `config_version: 3
+server:
+  port: 6123
+  address: https://superaddress.here.com
+  download_path: /home/path/gropple
+  maximum_active_downloads_per_domain: 2
+ui:
+  popup_width: 500
+  popup_height: 500
+destinations:
+- name: somegifs
+  path: /home/path/somegifs
+- name: otherstuff
+  path: /home/path/otherstuff
+profiles:
+- name: standard video
+  command: yt-dlp
+  args:
+  - --newline
+  - --write-info-json
+  - -f
+  - bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best
+  - --verbose
+  - --embed-metadata
+  - --embed-subs
+  - --embed-thumbnail
+- name: standard mp3
+  command: yt-dlp
+  args:
+  - --extract-audio
+  - --audio-format
+  - mp3
+  - --prefer-ffmpeg
+`
+	cs := configServiceFromString(v3Config)
+	err := cs.LoadConfig()
+	if err != nil {
+		t.Errorf("got error when loading config: %s", err)
+	}
+	if cs.Config.ConfigVersion != 4 {
+		t.Errorf("did not migrate version (it is '%d')", cs.Config.ConfigVersion)
+	}
+	if cs.Config.Server.MaximumActiveDownloads != 2 {
+		t.Error("did not add MaximumActiveDownloads")
+	}
+	if len(cs.Config.Destinations) != 0 {
+		t.Error("incorrect number of destinations from migrated file")
+	}
+	if assert.Len(t, cs.Config.DownloadOptions, 2) {
+		if assert.Len(t, cs.Config.DownloadOptions[0].Args, 2) {
+			assert.Equal(t, "-o", cs.Config.DownloadOptions[0].Args[0])
+			assert.Equal(t, "/home/path/somegifs/%(title)s [%(id)s].%(ext)s", cs.Config.DownloadOptions[0].Args[1])
+			assert.Equal(t, "-o", cs.Config.DownloadOptions[1].Args[0])
+			assert.Equal(t, "/home/path/otherstuff/%(title)s [%(id)s].%(ext)s", cs.Config.DownloadOptions[1].Args[1])
+		}
+	}
+	os.Remove(cs.ConfigPath)
+}
+
+func configServiceFromString(configString string) *ConfigService {
+	tmpFile, _ := os.CreateTemp("", "gropple_test_*.yml")
+	_, err1 := tmpFile.Write([]byte(configString))
+	err2 := tmpFile.Close()
+
+	if errors.Join(err1, err2) != nil {
+		panic("got unexpected error")
+	}
+
+	cs := ConfigService{
+		Config:     &Config{},
+		ConfigPath: tmpFile.Name(),
+	}
+	return &cs
+}
+
+func TestLookForExecutable(t *testing.T) {
+	cmdPath, err := exec.LookPath("sleep")
+	if err != nil {
+		t.Errorf("cannot run this test without knowing about sleep: %s", err)
+		t.FailNow()
+	}
+	cmdDir := filepath.Dir(cmdPath)
+
+	cmd := "sleep"
+	path, err := AbsPathToExecutable(cmd)
+	if assert.NoError(t, err) {
+		assert.Equal(t, cmdPath, path)
+	}
+
+	cmd = cmdPath
+	path, err = AbsPathToExecutable(cmd)
+	if assert.NoError(t, err) {
+		assert.Equal(t, cmdPath, path)
+	}
+
+	cmd = "../../../../../../../../.." + cmdPath
+	path, err = AbsPathToExecutable(cmd)
+	if assert.NoError(t, err) {
+		assert.Equal(t, cmdPath, path)
+	}
+	cmd = "./sleep"
+	_, err = AbsPathToExecutable(cmd)
+	assert.Error(t, err)
+
+	os.Chdir(cmdDir) //nolint
+	cmd = "./sleep"
+	path, err = AbsPathToExecutable(cmd)
+	if assert.NoError(t, err) {
+		assert.Equal(t, cmdPath, path)
+	}
+
+}

docker-compose.yml

@@ -0,0 +1,11 @@
+version: "3.9"
+
+services:
+  gropple:
+    image: tardisx/gropple:v1.1.4
+    volumes:
+      - /tmp/gropple-config-dir/:/config
+      - /tmp/downloads/:/downloads/
+    restart: always
+    ports:
+      - "6123:6123"

download/download.go

@@ -1,13 +1,20 @@
 package download
 
 import (
+	"encoding/json"
 	"fmt"
 	"io"
+	"log"
+	"net/url"
+	"os"
 	"os/exec"
+	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
+	"time"
 
 	"github.com/tardisx/gropple/config"
 )
@@ -15,56 +22,322 @@ import (
 type Download struct {
 	Id              int                    `json:"id"`
 	Url             string                 `json:"url"`
-	Pid             int                    `json:"pid"`
+	PopupUrl        string                 `json:"popup_url"`
+	Process         *os.Process            `json:"-"`
 	ExitCode        int                    `json:"exit_code"`
-	State           string                 `json:"state"`
+	State           State                  `json:"state"`
 	DownloadProfile config.DownloadProfile `json:"download_profile"`
+	DownloadOption  *config.DownloadOption `json:"download_option"`
 	Finished        bool                   `json:"finished"`
+	FinishedTS      time.Time              `json:"finished_ts"`
 	Files           []string               `json:"files"`
+	PlaylistCurrent int                    `json:"playlist_current"`
+	PlaylistTotal   int                    `json:"playlist_total"`
 	Eta             string                 `json:"eta"`
 	Percent         float32                `json:"percent"`
 	Log             []string               `json:"log"`
 	Config          *config.Config
+	Lock            sync.Mutex
+}
+
+// The Manager holds and is responsible for all Download objects.
+type Manager struct {
+	Downloads    []*Download
+	MaxPerDomain int
+	Lock         sync.Mutex
+}
+
+func (m *Manager) String() string {
+	m.Lock.Lock()
+	defer m.Lock.Unlock()
+	out := fmt.Sprintf("Max per domain: %d, downloads: %d\n", m.MaxPerDomain, len(m.Downloads))
+
+	for _, dl := range m.Downloads {
+		out = out + fmt.Sprintf("%3d: (%10s) %30s\n", dl.Id, dl.State, dl.Url)
+	}
+
+	return out
+
+}
+
+type State string
+
+const (
+	STATE_PREPARING            State = "Preparing to start"
+	STATE_CHOOSE_PROFILE       State = "Choose Profile"
+	STATE_QUEUED               State = "Queued"
+	STATE_DOWNLOADING          State = "Downloading"
+	STATE_DOWNLOADING_METADATA State = "Downloading metadata"
+	STATE_FAILED               State = "Failed"
+	STATE_COMPLETE             State = "Complete"
+	STATE_MOVED                State = "Moved"
+)
+
+var CanStopDownload = false
+
+var downloadId int32 = 0
+
+func (m *Manager) ManageQueue() {
+	for {
+		m.Lock.Lock()
+
+		m.startQueued(m.MaxPerDomain)
+		m.cleanup()
+		m.Lock.Unlock()
+
+		time.Sleep(time.Second)
+	}
+}
+
+func (m *Manager) DownloadsAsJSON() ([]byte, error) {
+
+	m.Lock.Lock()
+	defer m.Lock.Unlock()
+	for _, dl := range m.Downloads {
+		dl.Lock.Lock()
+		defer dl.Lock.Unlock()
+	}
+	b, err := json.Marshal(m.Downloads)
+	return b, err
+}
+
+// startQueued starts any downloads that have been queued, we would not exceed
+// maxRunning. If maxRunning is 0, there is no limit.
+func (m *Manager) startQueued(maxRunning int) {
+
+	active := make(map[string]int)
+
+	for _, dl := range m.Downloads {
+		dl.Lock.Lock()
+
+		if dl.State == STATE_DOWNLOADING || dl.State == STATE_PREPARING {
+			active[dl.domain()]++
+		}
+		dl.Lock.Unlock()
+
+	}
+
+	for _, dl := range m.Downloads {
+
+		dl.Lock.Lock()
+
+		if dl.State == STATE_QUEUED && (maxRunning == 0 || active[dl.domain()] < maxRunning) {
+			dl.State = STATE_PREPARING
+			active[dl.domain()]++
+			log.Printf("Starting download for id:%d (%s)", dl.Id, dl.Url)
+
+			dl.Lock.Unlock()
+
+			go func(sdl *Download) {
+				sdl.Begin()
+			}(dl)
+		} else {
+			dl.Lock.Unlock()
+		}
+
+	}
+
+}
+
+// cleanup removes old downloads from the list. Hardcoded to remove them one hour
+// completion. Expects the Manager to be locked.
+func (m *Manager) cleanup() {
+	newDLs := []*Download{}
+	for _, dl := range m.Downloads {
+		dl.Lock.Lock()
+		if dl.Finished && time.Since(dl.FinishedTS) > time.Duration(time.Hour) {
+			// do nothing
+		} else {
+			newDLs = append(newDLs, dl)
+		}
+		dl.Lock.Unlock()
+
+	}
+	m.Downloads = newDLs
+}
+
+// GetDlById returns one of the downloads in our current list.
+func (m *Manager) GetDlById(id int) (*Download, error) {
+	m.Lock.Lock()
+	defer m.Lock.Unlock()
+	for _, dl := range m.Downloads {
+		if dl.Id == id {
+			return dl, nil
+		}
+	}
+	return nil, fmt.Errorf("no download with id %d", id)
+}
+
+// Queue queues a download
+func (m *Manager) Queue(dl *Download) {
+	dl.Lock.Lock()
+	defer dl.Lock.Unlock()
+	dl.State = STATE_QUEUED
+}
+
+func NewDownload(url string, conf *config.Config) *Download {
+	atomic.AddInt32(&downloadId, 1)
+	dl := Download{
+		Id:       int(downloadId),
+		Url:      url,
+		PopupUrl: fmt.Sprintf("/fetch/%d", int(downloadId)),
+		State:    STATE_CHOOSE_PROFILE,
+		Files:    []string{},
+		Log:      []string{},
+		Config:   conf,
+		Lock:     sync.Mutex{},
+	}
+	return &dl
+}
+
+func (m *Manager) AddDownload(dl *Download) {
+	m.Lock.Lock()
+	defer m.Lock.Unlock()
+	m.Downloads = append(m.Downloads, dl)
+}
+
+// func (dl *Download) AppendLog(text string) {
+// 	dl.Lock.Lock()
+// 	defer dl.Lock.Unlock()
+// 	dl.Log = append(dl.Log, text)
+// }
+
+// Stop the download.
+func (dl *Download) Stop() {
+	if !CanStopDownload {
+		log.Print("attempted to stop download on a platform that it is not currently supported on - please report this as a bug")
+		os.Exit(1)
+	}
+
+	log.Printf("stopping the download")
+	dl.Lock.Lock()
+	defer dl.Lock.Unlock()
+	dl.Log = append(dl.Log, "aborted by user")
+	err := dl.Process.Kill()
+	if err != nil {
+		log.Printf("could not send kill to process: %s", err)
+	}
+}
+
+// domain returns a domain for this Download. Download should be locked.
+func (dl *Download) domain() string {
+
+	url, err := url.Parse(dl.Url)
+	if err != nil {
+		log.Printf("Unknown domain for url: %s", dl.Url)
+		return "unknown"
+	}
+
+	return url.Hostname()
+
 }
 
 // Begin starts a download, by starting the command specified in the DownloadProfile.
 // It blocks until the download is complete.
 func (dl *Download) Begin() {
-	cmdSlice := []string{}
+	dl.Lock.Lock()
-	cmdSlice = append(cmdSlice, dl.DownloadProfile.Args...)
+	u, err := url.Parse(dl.Url)
-	cmdSlice = append(cmdSlice, dl.Url)
+	if err != nil {
+		log.Printf("Bad url '%s': %s", dl.Url, err.Error())
+	}
 
-	cmd := exec.Command(dl.DownloadProfile.Command, cmdSlice...)
+	// grab the host and path for substitutions
+	host := u.Host
+	path := u.Path
+
+	// strip the leading /
+	if strings.Index(path, "/") == 0 {
+		path = path[1:]
+	}
+
+	// escape them in a way that should mean we can use them as a filepath
+	host = strings.ReplaceAll(host, string(filepath.Separator), "_")
+	host = strings.ReplaceAll(host, string(filepath.ListSeparator), "_")
+	path = strings.ReplaceAll(path, string(filepath.Separator), "_")
+	path = strings.ReplaceAll(path, string(filepath.ListSeparator), "_")
+
+	dl.State = STATE_DOWNLOADING
+	cmdSlice := []string{}
+
+	for i := range dl.DownloadProfile.Args {
+		arg := dl.DownloadProfile.Args[i]
+		arg = strings.ReplaceAll(arg, "%GROPPLE_HOST%", host)
+		arg = strings.ReplaceAll(arg, "%GROPPLE_PATH%", path)
+		cmdSlice = append(cmdSlice, arg)
+	}
+
+	// add the option, if any
+	if dl.DownloadOption != nil {
+		for i := range dl.DownloadOption.Args {
+			arg := dl.DownloadOption.Args[i]
+			arg = strings.ReplaceAll(arg, "%GROPPLE_HOST%", host)
+			arg = strings.ReplaceAll(arg, "%GROPPLE_PATH%", path)
+			cmdSlice = append(cmdSlice, arg)
+		}
+	}
+
+	// only add the url if it's not empty or an example URL. This helps us with testing
+	if dl.Url != "" && !strings.Contains(dl.domain(), "example.org") {
+		cmdSlice = append(cmdSlice, dl.Url)
+	}
+
+	cmdPath, err := config.AbsPathToExecutable(dl.DownloadProfile.Command)
+	if err != nil {
+		dl.State = STATE_FAILED
+		dl.Finished = true
+		dl.FinishedTS = time.Now()
+		dl.Log = append(dl.Log, fmt.Sprintf("error finding executable for downloader: %s", err.Error()))
+		dl.Lock.Unlock()
+		return
+	}
+
+	dl.Log = append(dl.Log, fmt.Sprintf("executing: %s (%s) with args: %s", dl.DownloadProfile.Command, cmdPath, strings.Join(cmdSlice, " ")))
+
+	cmd := exec.Command(cmdPath, cmdSlice...)
 	cmd.Dir = dl.Config.Server.DownloadPath
+	log.Printf("Executing command executable: %s) in %s", cmdPath, dl.Config.Server.DownloadPath)
 
 	stdout, err := cmd.StdoutPipe()
 	if err != nil {
-		dl.State = "failed"
+		dl.State = STATE_FAILED
 		dl.Finished = true
+		dl.FinishedTS = time.Now()
 		dl.Log = append(dl.Log, fmt.Sprintf("error setting up stdout pipe: %v", err))
+		dl.Lock.Unlock()
 		return
 	}
 
 	stderr, err := cmd.StderrPipe()
 	if err != nil {
-		dl.State = "failed"
+		dl.State = STATE_FAILED
 		dl.Finished = true
+		dl.FinishedTS = time.Now()
 		dl.Log = append(dl.Log, fmt.Sprintf("error setting up stderr pipe: %v", err))
+		dl.Lock.Unlock()
+
 		return
 	}
 
 	err = cmd.Start()
 	if err != nil {
-		dl.State = "failed"
+		log.Printf("Executing command failed: %s", err.Error())
+
+		dl.State = STATE_FAILED
 		dl.Finished = true
-		dl.Log = append(dl.Log, fmt.Sprintf("error starting youtube-dl: %v", err))
+		dl.FinishedTS = time.Now()
+		dl.Log = append(dl.Log, fmt.Sprintf("error starting command '%s': %v", dl.DownloadProfile.Command, err))
+		dl.Lock.Unlock()
+
 		return
 	}
-	dl.Pid = cmd.Process.Pid
+	dl.Process = cmd.Process
 
 	var wg sync.WaitGroup
 
 	wg.Add(2)
+
+	dl.Lock.Unlock()
+
 	go func() {
 		defer wg.Done()
 		dl.updateDownload(stdout)
@@ -76,19 +349,38 @@ func (dl *Download) Begin() {
 	}()
 
 	wg.Wait()
-	cmd.Wait()
 
-	dl.State = "complete"
+	err = cmd.Wait()
-	dl.Finished = true
+	dl.Lock.Lock()
-	dl.ExitCode = cmd.ProcessState.ExitCode()
 
-	if dl.ExitCode != 0 {
+	if err != nil {
-		dl.State = "failed"
+		log.Printf("process failed for id: %d: %s", dl.Id, err)
+
+		dl.State = STATE_FAILED
+		dl.Finished = true
+		dl.FinishedTS = time.Now()
+		dl.ExitCode = cmd.ProcessState.ExitCode()
+
+	} else {
+
+		log.Printf("process finished for id: %d (%v)", dl.Id, cmd)
+
+		dl.State = STATE_COMPLETE
+		dl.Finished = true
+		dl.FinishedTS = time.Now()
+		dl.ExitCode = cmd.ProcessState.ExitCode()
+
+		if dl.ExitCode != 0 {
+			dl.State = STATE_FAILED
+		}
 	}
-
+	dl.Lock.Unlock()
 }
 
+// updateDownload updates the download based on data from the reader. Expects the
+// Download to be unlocked.
 func (dl *Download) updateDownload(r io.Reader) {
+
 	// XXX not sure if we might get a partial line?
 	buf := make([]byte, 1024)
 	for {
@@ -104,10 +396,12 @@ func (dl *Download) updateDownload(r io.Reader) {
 				}
 
 				// append the raw log
+				dl.Lock.Lock()
 				dl.Log = append(dl.Log, l)
-
 				// look for the percent and eta and other metadata
 				dl.updateMetadata(l)
+				dl.Lock.Unlock()
+
 			}
 		}
 		if err != nil {
@@ -116,14 +410,16 @@ func (dl *Download) updateDownload(r io.Reader) {
 	}
 }
 
+// updateMetadata parses some metadata and updates the Download. Download must be locked.
 func (dl *Download) updateMetadata(s string) {
 
 	// [download]  49.7% of ~15.72MiB at  5.83MiB/s ETA 00:07
-	etaRE := regexp.MustCompile(`download.+ETA +(\d\d:\d\d(?::\d\d)?)$`)
+	// [download]  99.3% of ~1.42GiB at 320.87KiB/s ETA 00:07 (frag 212/214)
+	etaRE := regexp.MustCompile(`download.+ETA +(\d\d:\d\d(?::\d\d)?)`)
 	matches := etaRE.FindStringSubmatch(s)
 	if len(matches) == 2 {
 		dl.Eta = matches[1]
-		dl.State = "downloading"
+		dl.State = STATE_DOWNLOADING
 
 	}
 
@@ -133,8 +429,6 @@ func (dl *Download) updateMetadata(s string) {
 		p, err := strconv.ParseFloat(matches[1], 32)
 		if err == nil {
 			dl.Percent = float32(p)
-		} else {
-			panic(err)
 		}
 	}
 
@@ -169,4 +463,29 @@ func (dl *Download) updateMetadata(s string) {
 			}
 		}
 	}
+
+	// [download] Downloading video 1 of 3
+	playlistDetails := regexp.MustCompile(`Downloading video (\d+) of (\d+)`)
+	matches = playlistDetails.FindStringSubmatch(s)
+	if len(matches) == 3 {
+		total, _ := strconv.ParseInt(matches[2], 10, 32)
+		current, _ := strconv.ParseInt(matches[1], 10, 32)
+		dl.PlaylistTotal = int(total)
+		dl.PlaylistCurrent = int(current)
+	}
+
+	// [Site] user: Downloading JSON metadata page 2
+	metadataDL := regexp.MustCompile(`Downloading JSON metadata page (\d+)`)
+	matches = metadataDL.FindStringSubmatch(s)
+	if len(matches) == 2 {
+		dl.State = STATE_DOWNLOADING_METADATA
+	}
+
+	// [FixupM3u8] Fixing MPEG-TS in MP4 container of "file [-168849776_456239489].mp4"
+	metadataFixup := regexp.MustCompile(`Fixing MPEG-TS in MP4 container`)
+	matches = metadataFixup.FindStringSubmatch(s)
+	if len(matches) == 1 {
+		dl.State = "Fixing MPEG-TS in MP4"
+	}
+
 }

download/download_notestdata.go

@@ -0,0 +1,8 @@
+//go:build !testdata
+
+package download
+
+import "github.com/tardisx/gropple/config"
+
+func (m *Manager) AddStressTestData(c *config.ConfigService) {
+}

download/download_posix.go

@@ -0,0 +1,5 @@
+package download
+
+func init() {
+	CanStopDownload = true
+}

download/download_test.go

@@ -1,6 +1,13 @@
 package download
 
-import "testing"
+import (
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/tardisx/gropple/config"
+)
 
 func TestUpdateMetadata(t *testing.T) {
 	newD := Download{}
@@ -14,11 +21,11 @@ func TestUpdateMetadata(t *testing.T) {
 	// eta's might be xx:xx:xx or xx:xx
 	newD.updateMetadata("[download]   0.0% of 504.09MiB at 135.71KiB/s ETA 01:03:36")
 	if newD.Eta != "01:03:36" {
-		t.Fatalf("bad long eta in dl\n%v", newD)
+		t.Fatalf("bad long eta in dl\n%#v", newD) //nolint
 	}
 	newD.updateMetadata("[download]   0.0% of 504.09MiB at 397.98KiB/s ETA 21:38")
 	if newD.Eta != "21:38" {
-		t.Fatalf("bad short eta in dl\n%v", newD)
+		t.Fatalf("bad short eta in dl\n%#v", newD) //nolint
 	}
 
 	// added a new file, now we are tracking two
@@ -34,6 +41,18 @@ func TestUpdateMetadata(t *testing.T) {
 		t.Fatalf("%v", newD.Files)
 	}
 
+	// different download
+	newD.updateMetadata("[download]  99.3% of ~1.42GiB at 320.87KiB/s ETA 00:07 (frag 212/214)")
+	if newD.Eta != "00:07" {
+		t.Fatalf("bad short eta in dl with frag\n%v", newD) //nolint
+	}
+
+	// [FixupM3u8] Fixing MPEG-TS in MP4 container of "file [-168849776_456239489].mp4"
+	newD.updateMetadata("[FixupM3u8] Fixing MPEG-TS in MP4 container of \"file [-168849776_456239489].mp4")
+	if newD.State != "Fixing MPEG-TS in MP4" {
+		t.Fatalf("did not see fixup state - state is %s", newD.State)
+	}
+
 	// deletes
 	// TODO. Not sure why I don't always see the "Deleting original file" messages after merge -
 	// maybe a youtube-dl fork thing?
@@ -60,3 +79,284 @@ func TestUpdateMetadata(t *testing.T) {
 // [download] 100.0% of 4.64MiB at 10.12MiB/s ETA 00:00
 // [download] 100% of 4.64MiB in 00:00
 // [ffmpeg] Merging formats into "Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.mp4"
+
+func TestQueue(t *testing.T) {
+	cs := config.ConfigService{}
+	cs.LoadTestConfig()
+	conf := cs.Config
+
+	new1 := NewDownload("http://sub.example.org/foo1", conf)
+	new2 := NewDownload("http://sub.example.org/foo2", conf)
+	new3 := NewDownload("http://sub.example.org/foo3", conf)
+	new4 := NewDownload("http://example.org/", conf)
+
+	// pretend the user chose a profile for each
+	new1.DownloadProfile = *conf.ProfileCalled("test profile")
+	new2.DownloadProfile = *conf.ProfileCalled("test profile")
+	new3.DownloadProfile = *conf.ProfileCalled("test profile")
+	new4.DownloadProfile = *conf.ProfileCalled("test profile")
+	new1.State = STATE_QUEUED
+	new2.State = STATE_QUEUED
+	new3.State = STATE_QUEUED
+	new4.State = STATE_QUEUED
+
+	q := Manager{
+		Downloads:    []*Download{},
+		MaxPerDomain: 2,
+		Lock:         sync.Mutex{},
+	}
+
+	q.AddDownload(new1)
+	q.AddDownload(new2)
+	q.AddDownload(new3)
+	q.AddDownload(new4)
+
+	q.startQueued(1)
+
+	// two should start, one from each of the two domains
+	time.Sleep(time.Millisecond * 100)
+	if q.Downloads[0].State != STATE_DOWNLOADING {
+		t.Errorf("#1 was not downloading - %s instead ", q.Downloads[0].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[1].State != STATE_QUEUED {
+		t.Errorf("#2 is not queued - %s instead", q.Downloads[1].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[2].State != STATE_QUEUED {
+		t.Errorf("#3 is not queued - %s instead", q.Downloads[2].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[3].State != STATE_DOWNLOADING {
+		t.Errorf("#4 is not downloading - %s instead", q.Downloads[3].State)
+		t.Log(q.String())
+	}
+
+	// this should start no more, as one is still going
+	q.startQueued(1)
+	time.Sleep(time.Millisecond * 100)
+	if q.Downloads[0].State != STATE_DOWNLOADING {
+		t.Errorf("#1 was not downloading - %s instead ", q.Downloads[0].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[1].State != STATE_QUEUED {
+		t.Errorf("#2 is not queued - %s instead", q.Downloads[1].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[2].State != STATE_QUEUED {
+		t.Errorf("#3 is not queued - %s instead", q.Downloads[2].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[3].State != STATE_DOWNLOADING {
+		t.Errorf("#4 is not downloading - %s instead", q.Downloads[3].State)
+		t.Log(q.String())
+	}
+
+	// wait until the two finish, check
+	time.Sleep(time.Second * 5.0)
+	if q.Downloads[0].State != STATE_COMPLETE {
+		t.Errorf("#1 was not complete - %s instead ", q.Downloads[0].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[1].State != STATE_QUEUED {
+		t.Errorf("#2 is not queued - %s instead", q.Downloads[1].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[2].State != STATE_QUEUED {
+		t.Errorf("#3 is not queued - %s instead", q.Downloads[2].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[3].State != STATE_COMPLETE {
+		t.Errorf("#4 is not complete - %s instead", q.Downloads[3].State)
+		t.Log(q.String())
+	}
+
+	// this should start one more, as one is still going
+	q.startQueued(1)
+	time.Sleep(time.Millisecond * 100)
+	if q.Downloads[0].State != STATE_COMPLETE {
+		t.Errorf("#1 was not complete - %s instead ", q.Downloads[0].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[1].State != STATE_DOWNLOADING {
+		t.Errorf("#2 is not downloading - %s instead", q.Downloads[1].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[2].State != STATE_QUEUED {
+		t.Errorf("#3 is not queued - %s instead", q.Downloads[2].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[3].State != STATE_COMPLETE {
+		t.Errorf("#4 is not complete - %s instead", q.Downloads[3].State)
+		t.Log(q.String())
+	}
+
+	// this should start no more, as one is still going
+	q.startQueued(1)
+	time.Sleep(time.Millisecond * 100)
+	if q.Downloads[0].State != STATE_COMPLETE {
+		t.Errorf("#1 was not complete - %s instead ", q.Downloads[0].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[1].State != STATE_DOWNLOADING {
+		t.Errorf("#2 is not downloading - %s instead", q.Downloads[1].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[2].State != STATE_QUEUED {
+		t.Errorf("#3 is not queued - %s instead", q.Downloads[2].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[3].State != STATE_COMPLETE {
+		t.Errorf("#4 is not complete - %s instead", q.Downloads[3].State)
+		t.Log(q.String())
+	}
+
+	// but if we allow two per domain, the other queued one will start
+	q.startQueued(2)
+	time.Sleep(time.Millisecond * 100)
+	if q.Downloads[0].State != STATE_COMPLETE {
+		t.Errorf("#1 was not complete - %s instead ", q.Downloads[0].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[1].State != STATE_DOWNLOADING {
+		t.Errorf("#2 is not downloading - %s instead", q.Downloads[1].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[2].State != STATE_DOWNLOADING {
+		t.Errorf("#3 is not downloading - %s instead", q.Downloads[2].State)
+		t.Log(q.String())
+	}
+	if q.Downloads[3].State != STATE_COMPLETE {
+		t.Errorf("#4 is not complete - %s instead", q.Downloads[3].State)
+		t.Log(q.String())
+	}
+
+}
+
+func TestUpdateMetadataPlaylist(t *testing.T) {
+
+	output := `
+start of log...
+[download] Downloading playlist: nice_user
+[RedGifsUser] nice_user: Downloading JSON metadata page 1
+[RedGifsUser] nice_user: Downloading JSON metadata page 2
+[RedGifsUser] nice_user: Downloading JSON metadata page 3
+[RedGifsUser] nice_user: Downloading JSON metadata page 4
+[RedGifsUser] nice_user: Downloading JSON metadata page 5
+[RedGifsUser] nice_user: Downloading JSON metadata page 6
+[info] Writing playlist metadata as JSON to: nice_user [nice_user].info.json
+[RedGifsUser] playlist nice_user: Downloading 3 videos
+[download] Downloading video 1 of 3
+[info] wrongpreciouschrysomelid: Downloading 1 format(s): hd
+[info] Writing video metadata as JSON to: Splendid Wonderful Speaker Power Chocolate Drop [wrongpreciouschrysomelid].info.json
+[download] Destination: Splendid Wonderful Speaker Power Chocolate Drop [wrongpreciouschrysomelid].mp4
+[download]   0.0% of 4.96MiB at Unknown speed ETA Unknown
+[download]   0.1% of 4.96MiB at  1.76MiB/s ETA 00:02
+[download]  20.1% of 4.96MiB at  7.28MiB/s ETA 00:00
+[download]  40.3% of 4.96MiB at 10.06MiB/s ETA 00:00
+[download]  80.6% of 4.96MiB at 14.93MiB/s ETA 00:00
+[download] 100% of 4.96MiB at 17.33MiB/s ETA 00:00
+[download] 100% of 4.96MiB in 00:00
+[download] Downloading video 2 of 3
+[info] silentnaughtyborzoi: Downloading 1 format(s): hd
+[info] Writing video metadata as JSON to: Splendid Printer Tray Computer Outdoor Window Wonderful [silentnaughtyborzoi].info.json
+[download] Destination: Splendid Printer Tray Computer Outdoor Window Wonderful [silentnaughtyborzoi].mp4
+[download]   0.0% of 5.81MiB at 896.03KiB/s ETA 00:06
+[download]   0.1% of 5.81MiB at  1.28MiB/s ETA 00:04
+[download]   0.1% of 5.81MiB at  1.59MiB/s ETA 00:03
+[download]  34.4% of 5.81MiB at  9.90MiB/s ETA 00:00
+[download]  68.8% of 5.81MiB at 12.49MiB/s ETA 00:00
+[download] 100% of 5.81MiB at 15.77MiB/s ETA 00:00
+[download] 100% of 5.81MiB in 00:00
+[download] Downloading video 3 of 3
+[info] mammothremarkablewhooper: Downloading 1 format(s): hd
+[info] Writing video metadata as JSON to: Porthole Splendid Close Up Gun Gunshot Window Wonderful [mammothremarkablewhooper].info.json
+[download] Destination: Porthole Splendid Close Up Gun Gunshot Window Wonderful [mammothremarkablewhooper].mp4
+[download]   0.0% of 2.89MiB at Unknown speed ETA Unknown
+[download]   0.1% of 2.89MiB at  1.77MiB/s ETA 00:01
+[download]   0.2% of 2.89MiB at  2.26MiB/s ETA 00:01
+[download]  34.5% of 2.89MiB at  8.23MiB/s ETA 00:00
+[download]  69.1% of 2.89MiB at 11.63MiB/s ETA 00:00
+[download] 100% of 2.89MiB at 14.25MiB/s ETA 00:00
+[download] 100% of 2.89MiB in 00:00
+[info] Writing updated playlist metadata as JSON to: nice_user [nice_user].info.json
+[download] Finished downloading playlist: nice_user
+`
+	newD := Download{}
+
+	lines := strings.Split(output, "\n")
+	for _, l := range lines {
+		// t.Log(l)
+		newD.updateMetadata(l)
+	}
+
+	if len(newD.Files) != 3 {
+		t.Errorf("%d files, not 3", len(newD.Files))
+	} else {
+		if newD.Files[0] != "Splendid Wonderful Speaker Power Chocolate Drop [wrongpreciouschrysomelid].mp4" {
+			t.Error("Wrong 1st file")
+		}
+		if newD.Files[1] != "Splendid Printer Tray Computer Outdoor Window Wonderful [silentnaughtyborzoi].mp4" {
+			t.Error("Wrong 2nd file")
+		}
+		if newD.Files[2] != "Porthole Splendid Close Up Gun Gunshot Window Wonderful [mammothremarkablewhooper].mp4" {
+			t.Error("Wrong 3rd file")
+		}
+	}
+
+	if newD.PlaylistTotal != 3 {
+		t.Errorf("playlist has total %d should be 3", newD.PlaylistTotal)
+	}
+
+}
+
+func TestUpdateMetadataSingle(t *testing.T) {
+
+	output := `
+[youtube] 2WoDQBhJCVQ: Downloading webpage
+[youtube] 2WoDQBhJCVQ: Downloading android player API JSON
+[info] 2WoDQBhJCVQ: Downloading 1 format(s): 137+140
+[info] Writing video metadata as JSON to: The Greatest Shot In Television [2WoDQBhJCVQ].info.json
+[debug] Invoking hlsnative downloader on "https://example.org/urls/1.2.3.4%
+[download]   0.0% of 12.82MiB at 510.94KiB/s ETA 00:26
+[download]   0.0% of 12.82MiB at 966.50KiB/s ETA 00:13
+[download]   0.1% of 12.82MiB at  1.54MiB/s ETA 00:08
+[download]   0.1% of 12.82MiB at  2.75MiB/s ETA 00:04
+[download]   0.2% of 12.82MiB at  1.30MiB/s ETA 00:09
+[download]  77.5% of 12.82MiB at  2.54MiB/s ETA 00:01
+[download]  79.4% of 12.82MiB at  3.89MiB/s ETA 00:00
+[download]  83.3% of 12.82MiB at  6.44MiB/s ETA 00:00
+[download]  91.1% of 12.82MiB at 10.28MiB/s ETA 00:00
+[download] 100% of 12.82MiB at 12.77MiB/s ETA 00:00
+[download] 100% of 12.82MiB in 00:01
+[download] Destination: The Greatest Shot In Television [2WoDQBhJCVQ].f140.m4a
+[download]   0.1% of 1.10MiB at 286.46KiB/s ETA 00:03
+[download]   0.3% of 1.10MiB at 716.49KiB/s ETA 00:01
+[download]   0.6% of 1.10MiB at  1.42MiB/s ETA 00:00
+[download]  91.0% of 1.10MiB at  6.67MiB/s ETA 00:00
+[download] 100% of 1.10MiB at  7.06MiB/s ETA 00:00
+[download] 100% of 1.10MiB in 00:00
+[Merger] Merging formats into "The Greatest Shot In Television [2WoDQBhJCVQ].mp4"
+Deleting original file The Greatest Shot In Television [2WoDQBhJCVQ].f137.mp4 (pass -k to keep)
+Deleting original file The Greatest Shot In Television [2WoDQBhJCVQ].f140.m4a (pass -k to keep)
+`
+	newD := Download{}
+
+	lines := strings.Split(output, "\n")
+	for _, l := range lines {
+		// t.Log(l)
+		newD.updateMetadata(l)
+	}
+
+	if len(newD.Files) != 1 {
+		t.Errorf("%d files, not 1", len(newD.Files))
+	} else {
+		if newD.Files[0] != "The Greatest Shot In Television [2WoDQBhJCVQ].mp4" {
+			t.Error("Wrong 1st file")
+		}
+	}
+	if newD.PlaylistTotal != 0 {
+		t.Error("playlist detected but should not be")
+	}
+
+}

download/download_testdata.go

@@ -0,0 +1,41 @@
+//go:build testdata
+
+package download
+
+import "github.com/tardisx/gropple/config"
+
+func (m *Manager) AddStressTestData(c *config.ConfigService) {
+
+	urls := []string{
+		"https://www.youtube.com/watch?v=qG_rRkuGBW8",
+		"https://www.youtube.com/watch?v=ZUzhZpQAU40",
+		"https://www.youtube.com/watch?v=kVxM3eRWGak",
+		"https://www.youtube.com/watch?v=pl-y9869y0w",
+		"https://www.youtube.com/watch?v=Uw4NEPE4l3A",
+		"https://www.youtube.com/watch?v=2RF0lcTuuYE",
+		"https://www.youtube.com/watch?v=lymwNQY0dus",
+		"https://www.youtube.com/watch?v=NTc-I4Z_duc",
+		"https://www.youtube.com/watch?v=wNSm1TJ84Ac",
+		"https://www.youtube.com/watch?v=tyixMpuGEL8",
+		"https://www.youtube.com/watch?v=VnxbkH_3E_4",
+		"https://www.youtube.com/watch?v=VStscvYLYLs",
+		"https://www.youtube.com/watch?v=vYMiSz-WlEY",
+
+		"https://vimeo.com/786570322",
+		"https://vimeo.com/783453809",
+
+		"https://www.gamespot.com/videos/survival-fps-how-metro-2033-solidified-a-subgenre/2300-6408243/",
+		"https://www.gamespot.com/videos/dirt-3-right-back-where-you-started-gameplay-movie/2300-6314712/",
+		"https://www.gamespot.com/videos/the-b-list-driver-san-francisco/2300-6405593/",
+
+		"https://www.imdb.com/video/vi1914750745/?listId=ls053181649&ref_=hm_hp_i_hero-video-1_1",
+		"https://www.imdb.com/video/vi3879585561/?listId=ls053181649&ref_=vp_pl_ap_6",
+		"https://www.imdb.com/video/vi54445849/?listId=ls053181649&ref_=vp_nxt_btn",
+	}
+	for _, u := range urls {
+		d := NewDownload(u, c.Config)
+		d.DownloadProfile = *c.Config.ProfileCalled("standard video")
+		m.AddDownload(d)
+		m.Queue(d)
+	}
+}

go.mod

@@ -1,9 +1,18 @@
 module github.com/tardisx/gropple
 
-go 1.16
+go 1.23.0
+
+toolchain go1.24.1
 
 require (
-	github.com/gorilla/mux v1.8.0
+	github.com/gorilla/mux v1.8.1
-	golang.org/x/mod v0.5.1
+	github.com/stretchr/testify v1.9.0
+	golang.org/x/mod v0.24.0
 	gopkg.in/yaml.v2 v2.4.0
 )
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

go.sum

@@ -1,19 +1,16 @@
-github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
-golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -1,74 +1,67 @@
 package main
 
 import (
-	"embed"
+	"flag"
-	"encoding/json"
 	"fmt"
-	"html/template"
-	"io"
 	"log"
 	"net/http"
-	"strings"
 	"time"
 
-	"strconv"
-
-	"github.com/gorilla/mux"
 	"github.com/tardisx/gropple/config"
 	"github.com/tardisx/gropple/download"
-	"github.com/tardisx/gropple/version"
+	v "github.com/tardisx/gropple/version"
+	"github.com/tardisx/gropple/web"
 )
 
-var downloads []*download.Download
+var (
-var downloadId = 0
+	version = "dev"
-var conf *config.Config
+)
-
-var versionInfo = version.Info{CurrentVersion: "v0.5.1"}
-
-//go:embed web
-var webFS embed.FS
-
-type successResponse struct {
-	Success bool   `json:"success"`
-	Message string `json:"message"`
-}
-
-type errorResponse struct {
-	Success bool   `json:"success"`
-	Error   string `json:"error"`
-}
 
 func main() {
-	if !config.ConfigFileExists() {
+	versionInfo := &v.Manager{
-		log.Print("No config file - creating default config")
+		// version from goreleaser has no leading 'v', even if the tag does
-		conf = config.DefaultConfig()
+		VersionInfo: v.Info{CurrentVersion: "v" + version},
-		conf.WriteConfig()
+	}
+	log.Printf("Starting gropple %s - https://github.com/tardisx/gropple", versionInfo.GetInfo().CurrentVersion)
+
+	var configPath string
+	flag.StringVar(&configPath, "config-path", "", "path to config file")
+
+	flag.Parse()
+
+	configService := &config.ConfigService{}
+	if configPath != "" {
+		configService.ConfigPath = configPath
 	} else {
-		loadedConfig, err := config.LoadConfig()
+		configService.DetermineConfigDir()
+	}
+
+	exists, err := configService.ConfigFileExists()
+	if err != nil {
+		log.Fatal(err)
+	}
+	if !exists {
+		log.Print("No config file - creating default config")
+		configService.LoadDefaultConfig()
+		configService.WriteConfig()
+		log.Printf("Configuration written to %s", configService.ConfigPath)
+	} else {
+		err := configService.LoadConfig()
 		if err != nil {
 			log.Fatal(err)
 		}
-		conf = loadedConfig
+		log.Printf("Configuration loaded from %s", configService.ConfigPath)
 	}
 
-	r := mux.NewRouter()
+	// create the download manager
-	r.HandleFunc("/", homeHandler)
+	downloadManager := &download.Manager{MaxPerDomain: configService.Config.Server.MaximumActiveDownloads}
-	r.HandleFunc("/config", configHandler)
-	r.HandleFunc("/fetch", fetchHandler)
-	r.HandleFunc("/fetch/{id}", fetchHandler)
 
-	// info for the list
+	// create the web handlers
-	r.HandleFunc("/rest/fetch", fetchInfoRESTHandler)
+	r := web.CreateRoutes(configService, downloadManager, versionInfo)
-	// info for one, including update
-	r.HandleFunc("/rest/fetch/{id}", fetchInfoOneRESTHandler)
-	r.HandleFunc("/rest/version", versionRESTHandler)
-	r.HandleFunc("/rest/config", configRESTHandler)
-
-	http.Handle("/", r)
 
 	srv := &http.Server{
 		Handler: r,
-		Addr:    fmt.Sprintf(":%d", conf.Server.Port),
+		Addr:    fmt.Sprintf(":%d", configService.Config.Server.Port),
 		// Good practice: enforce timeouts for servers you create!
 		WriteTimeout: 5 * time.Second,
 		ReadTimeout:  5 * time.Second,
@@ -77,246 +70,22 @@ func main() {
 	// check for a new version every 4 hours
 	go func() {
 		for {
-			versionInfo.UpdateGitHubVersion()
+			err := versionInfo.UpdateGitHubVersion()
+			if err != nil {
+				log.Printf("could not get version info: %s", err)
+			}
 			time.Sleep(time.Hour * 4)
 		}
 	}()
 
-	log.Printf("starting gropple %s - https://github.com/tardisx/gropple", versionInfo.CurrentVersion)
+	// start downloading queued downloads when slots available, and clean up
-	log.Printf("go to %s for details on installing the bookmarklet and to check status", conf.Server.Address)
+	// old entries
+	go downloadManager.ManageQueue()
+
+	// add testdata if compiled with the '-tags testdata' flag
+	downloadManager.AddStressTestData(configService)
+
+	log.Printf("Visit %s for details on installing the bookmarklet and to check status", configService.Config.Server.Address)
 	log.Fatal(srv.ListenAndServe())
-}
+
-
-// versionRESTHandler returns the version information, if we have up-to-date info from github
-func versionRESTHandler(w http.ResponseWriter, r *http.Request) {
-	if versionInfo.GithubVersionFetched {
-		b, _ := json.Marshal(versionInfo)
-		w.Write(b)
-	} else {
-		w.WriteHeader(400)
-	}
-}
-
-// homeHandler returns the main index page
-func homeHandler(w http.ResponseWriter, r *http.Request) {
-	w.WriteHeader(http.StatusOK)
-
-	bookmarkletURL := fmt.Sprintf("javascript:(function(f,s,n,o){window.open(f+encodeURIComponent(s),n,o)}('%s/fetch?url=',window.location,'yourform','width=%d,height=%d'));", conf.Server.Address, conf.UI.PopupWidth, conf.UI.PopupHeight)
-
-	t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/menu.tmpl", "web/index.html")
-	if err != nil {
-		panic(err)
-	}
-
-	type Info struct {
-		Downloads      []*download.Download
-		BookmarkletURL template.URL
-	}
-
-	info := Info{
-		Downloads:      downloads,
-		BookmarkletURL: template.URL(bookmarkletURL),
-	}
-
-	err = t.ExecuteTemplate(w, "layout", info)
-	if err != nil {
-		panic(err)
-	}
-}
-
-// configHandler returns the configuration page
-func configHandler(w http.ResponseWriter, r *http.Request) {
-	w.WriteHeader(http.StatusOK)
-
-	t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/menu.tmpl", "web/config.html")
-	if err != nil {
-		panic(err)
-	}
-
-	err = t.ExecuteTemplate(w, "layout", nil)
-	if err != nil {
-		panic(err)
-	}
-}
-
-// configRESTHandler handles both reading and writing of the configuration
-func configRESTHandler(w http.ResponseWriter, r *http.Request) {
-
-	if r.Method == "POST" {
-		log.Printf("Updating config")
-		b, err := io.ReadAll(r.Body)
-		if err != nil {
-			panic(err)
-		}
-		err = conf.UpdateFromJSON(b)
-
-		if err != nil {
-			errorRes := errorResponse{Success: false, Error: err.Error()}
-			errorResB, _ := json.Marshal(errorRes)
-			w.WriteHeader(400)
-			w.Write(errorResB)
-			return
-		}
-		conf.WriteConfig()
-	}
-	b, _ := json.Marshal(conf)
-	w.Write(b)
-}
-
-//
-func fetchInfoOneRESTHandler(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	idString := vars["id"]
-	if idString != "" {
-		id, err := strconv.Atoi(idString)
-		if err != nil {
-			http.NotFound(w, r)
-			return
-		}
-
-		// find the download
-		var thisDownload *download.Download
-		for _, dl := range downloads {
-			if dl.Id == id {
-				thisDownload = dl
-			}
-		}
-		if thisDownload == nil {
-			http.NotFound(w, r)
-			return
-		}
-
-		if r.Method == "POST" {
-
-			type updateRequest struct {
-				Action  string `json:"action"`
-				Profile string `json:"profile"`
-			}
-
-			thisReq := updateRequest{}
-
-			b, err := io.ReadAll(r.Body)
-			if err != nil {
-				panic(err)
-			}
-
-			err = json.Unmarshal(b, &thisReq)
-			if err != nil {
-				errorRes := errorResponse{Success: false, Error: err.Error()}
-				errorResB, _ := json.Marshal(errorRes)
-				w.WriteHeader(400)
-				w.Write(errorResB)
-				return
-			}
-
-			if thisReq.Action == "start" {
-				// find the profile they asked for
-				profile := conf.ProfileCalled(thisReq.Profile)
-				if profile == nil {
-					panic("bad profile name?")
-				}
-				// set the profile
-				thisDownload.DownloadProfile = *profile
-
-				go func() { thisDownload.Begin() }()
-				succRes := successResponse{Success: true, Message: "download started"}
-				succResB, _ := json.Marshal(succRes)
-				w.Write(succResB)
-				return
-			}
-		}
-
-		// just a get, return the object
-		b, _ := json.Marshal(thisDownload)
-		w.Write(b)
-		return
-	} else {
-		http.NotFound(w, r)
-	}
-}
-
-func fetchInfoRESTHandler(w http.ResponseWriter, r *http.Request) {
-	b, _ := json.Marshal(downloads)
-	w.Write(b)
-}
-
-func fetchHandler(w http.ResponseWriter, r *http.Request) {
-
-	// if they refreshed the popup, just load the existing object, don't
-	// create a new one
-	vars := mux.Vars(r)
-	idString := vars["id"]
-
-	idInt, err := strconv.ParseInt(idString, 10, 32)
-	if err == nil && idInt > 0 {
-		for _, dl := range downloads {
-			if dl.Id == int(idInt) {
-				t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/popup.html")
-				if err != nil {
-					panic(err)
-				}
-
-				templateData := map[string]interface{}{"dl": dl, "config": conf}
-
-				err = t.ExecuteTemplate(w, "layout", templateData)
-				if err != nil {
-					panic(err)
-				}
-				return
-			}
-		}
-
-	}
-
-	query := r.URL.Query()
-	url, present := query["url"]
-
-	if !present {
-		w.WriteHeader(400)
-		fmt.Fprint(w, "No url supplied")
-		return
-	} else {
-
-		// check the URL for a sudden but inevitable betrayal
-		if strings.Contains(url[0], conf.Server.Address) {
-			w.WriteHeader(400)
-			fmt.Fprint(w, "you musn't gropple your gropple :-)")
-			return
-		}
-
-		// create the record
-		// XXX should be atomic!
-		downloadId++
-		newDownload := download.Download{
-			Config: conf,
-
-			Id:       downloadId,
-			Url:      url[0],
-			State:    "choose profile",
-			Finished: false,
-			Eta:      "?",
-			Percent:  0.0,
-			Log:      make([]string, 0, 1000),
-		}
-		downloads = append(downloads, &newDownload)
-		// XXX atomic ^^
-
-		newDownload.Log = append(newDownload.Log, "start of log...")
-
-		// go func() {
-		// 	newDownload.Begin()
-		// }()
-
-		t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/popup.html")
-		if err != nil {
-			panic(err)
-		}
-
-		templateData := map[string]interface{}{"dl": newDownload, "config": conf}
-
-		err = t.ExecuteTemplate(w, "layout", templateData)
-		if err != nil {
-			panic(err)
-		}
-	}
 }

version/version.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"sync"
 
 	"golang.org/x/mod/semver"
 )
@@ -19,8 +20,23 @@ type Info struct {
 	GithubVersionFetched bool   `json:"-"`
 }
 
-func (i *Info) UpdateGitHubVersion() error {
+type Manager struct {
-	i.GithubVersionFetched = false
+	VersionInfo Info
+	lock        sync.Mutex
+}
+
+func (m *Manager) GetInfo() Info {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+
+	return m.VersionInfo
+}
+
+func (m *Manager) UpdateGitHubVersion() error {
+	m.lock.Lock()
+	m.VersionInfo.GithubVersionFetched = false
+	m.lock.Unlock()
+
 	versionUrl := "https://api.github.com/repos/tardisx/gropple/releases"
 	resp, err := http.Get(versionUrl)
 	if err != nil {
@@ -51,29 +67,30 @@ func (i *Info) UpdateGitHubVersion() error {
 		return errors.New("no releases found")
 	}
 
-	i.GithubVersion = releases[0].Name
+	m.lock.Lock()
+
+	defer m.lock.Unlock()
+	m.VersionInfo.GithubVersion = releases[0].Name
+	m.VersionInfo.GithubVersionFetched = true
+	m.VersionInfo.UpgradeAvailable = m.canUpgrade()
 
-	i.GithubVersionFetched = true
-	i.UpgradeAvailable = i.canUpgrade()
 	return nil
 }
 
-func (i *Info) canUpgrade() bool {
+func (m *Manager) canUpgrade() bool {
-	if !i.GithubVersionFetched {
+	if !m.VersionInfo.GithubVersionFetched {
 		return false
 	}
 
-	log.Printf("We are %s, github is %s", i.CurrentVersion, i.GithubVersion)
+	if !semver.IsValid(m.VersionInfo.CurrentVersion) {
-
+		log.Printf("current version %s is invalid", m.VersionInfo.CurrentVersion)
-	if !semver.IsValid(i.CurrentVersion) {
-		log.Printf("current version %s is invalid", i.CurrentVersion)
 	}
 
-	if !semver.IsValid(i.GithubVersion) {
+	if !semver.IsValid(m.VersionInfo.GithubVersion) {
-		log.Printf("github version %s is invalid", i.GithubVersion)
+		log.Printf("github version %s is invalid", m.VersionInfo.GithubVersion)
 	}
 
-	if semver.Compare(i.CurrentVersion, i.GithubVersion) == -1 {
+	if semver.Compare(m.VersionInfo.CurrentVersion, m.VersionInfo.GithubVersion) == -1 {
 		return true
 	}
 	return false

web/data/templates/bulk.tmpl

@@ -0,0 +1,91 @@
+{{ define "content" }}
+
+{{ template "menu.tmpl" . }}
+
+<div id="layout" class="pure-g pure-u-1" x-data="bulk_create()" >
+
+<h1>Bulk upload</h1>
+
+    <p class="error"  x-show="error_message"  x-transition.duration.500ms x-text="error_message"></p>
+    <p class="success" x-show="success_message" x-transition.duration.500ms x-text="success_message"></p>
+
+    <p>Paste URLs here, one per line:</p>
+
+    <textarea x-model="urls" rows="20" cols="80">
+    </textarea>
+
+    <br><br>
+
+    <table class="pure-table" >
+        <tr>
+            <th>profile</th>
+            <td>
+                <select class="pure-input-1-2" x-model="profile_chosen">
+                    <option value="">choose a profile</option>
+                {{ range $i := .config.DownloadProfiles }}
+                    <option name="{{$i.Name}}">{{ $i.Name }}</option>
+                {{ end }}
+                </select>
+            </td>
+        </tr>
+        <tr>
+            <th>download option</th>
+            <td>
+                <select  class="pure-input-1-2" x-model="download_option_chosen">
+                    <option value="">no option</option>
+                {{ range $i := .config.DownloadOptions }}
+                    <option name="{{$i.Name}}">{{ $i.Name }}</option>
+                {{ end }}
+                </select>
+            </td>
+        </tr>
+        <tr>
+            <th>&nbsp;</th>
+            <td>
+                <button class="button-small pure-button" @click="start()">add to queue</button>
+            </td>
+        </tr>
+
+    </table>
+
+
+
+
+</div>
+
+{{ end }}
+
+{{ define "js" }}
+<script>
+    function bulk_create() {
+        return {
+            profile_chosen: "",
+            download_option_chosen: "",
+            urls: "",
+            error_message: "",
+            success_message: "",
+            start() {
+                let op = {
+                   method: 'POST',
+                   body: JSON.stringify({action: 'start', urls: this.urls, profile: this.profile_chosen, download_option: this.download_option_chosen}),
+                   headers: { 'Content-Type': 'application/json' }
+                };
+                fetch('/bulk', op)
+                .then(response => response.json())
+                .then(response => {
+                    console.log(response)
+                     if (response.error) {
+                        this.error_message = response.error;
+                        this.success_message = '';
+                        document.body.scrollTop = document.documentElement.scrollTop = 0;
+                    } else {
+                        this.error_message = '';
+                        this.success_message = response.message;
+                        this.urls = '';
+                    }
+                })
+            }
+        }
+    }
+</script>
+{{ end }}

web/data/templates/config.tmpl

@@ -7,12 +7,18 @@
     <p class="error"  x-show="error_message"  x-transition.duration.500ms x-text="error_message"></p>
     <p class="success" x-show="success_message" x-transition.duration.500ms x-text="success_message"></p>
 
-    <p>Note: changes are not saved until the "Save Config" button is pressed at the bottom of the page.</p>
+    <p>Note: changes are not saved until the "Save Config" button is pressed.</p>
+
+    <div class="pure-g">
+        <div class="pure-u-1">
+            <button class="button-small pure-button button-small pure-button-primary" @click="save_config();" href="#">Save Config</button>
+        </div>
+    </div>
 
     <div class="pure-g">
 
-        <div class="pure-u-md-1-2 pure-u-1">
+        <div class="pure-u-lg-1-3 pure-u-1 l-box">
-   
+
             <form class="pure-form pure-form-stacked gropple-config">
                 <fieldset>
 
@@ -31,7 +37,11 @@
 
                     <label for="config-server-downloadpath">Download path</label>
                     <input type="text" id="config-server-downloadpath" placeholder="path" class="input-long" x-model="config.server.download_path" />
-                    <span class="pure-form-message">The path on the server to download files to.</span>
+                    <span class="pure-form-message">The default path on the server to download files to.</span>
+
+                    <label for="config-server-max-downloads">Maximum active downloads per domain</label>
+                    <input type="text" id="config-server-max-downloads" placeholder="2" class="input-long" x-model.number="config.server.maximum_active_downloads_per_domain" />
+                    <span class="pure-form-message">How many downloads can be simultaneously active. Use '0' for no limit. This limit is applied per domain that you download from.</span>
 
                     <legend>UI</legend>
 
@@ -49,14 +59,14 @@
             </form>
 
         </div>
-        <div class="pure-u-md-1-2 pure-u-1">
+        <div class="pure-u-lg-1-3 pure-u-1 l-box">
             <form class="pure-form gropple-config">
                 <fieldset>
 
                     <legend>Download Profiles</legend>
 
-                    <p>Gropple supports multiple download profiles. Each profile specifies a different youtube-dl 
+                    <p>Gropple supports multiple download profiles. Each profile specifies a different youtube-dl
-                        compatible command, and arguments. When starting a download, you may choose which profile 
+                        compatible command, and arguments. When starting a download, you may choose which profile
                         to use. The URL will be appended to the argument list at the end.
                     </p>
 
@@ -65,16 +75,19 @@
                     <template x-for="(profile, i) in config.profiles">
                         <div>
                             <label x-bind:for="'config-profiles-'+i+'-name'">Name of profile <span x-text="i+1"></span>
-                            </label>                                
+                            </label>
-                            
+
                             <input type="text" x-bind:id="'config-profiles-'+i+'-name'" class="input-long" placeholder="name" x-model="profile.name" />
-                            <button class="pure-button button-del" href="#" @click.prevent="config.profiles.splice(i, 1);;">delete profile</button>
+                            <button class="button-small pure-button button-del" href="#" @click.prevent="config.profiles.splice(i, 1);;">delete profile</button>
 
                             <span class="pure-form-message">The name of this profile. For your information only.</span>
 
                             <label x-bind:for="'config-profiles-'+i+'-command'">Command to run</label>
                             <input type="text" x-bind:id="'config-profiles-'+i+'-command'" class="input-long" placeholder="name" x-model="profile.command" />
-                            <span class="pure-form-message">Which command to run. Your path will be searched, or you can specify the full path here.</span>
+                            <span class="pure-form-message">Which command to run. Your path will be searched, or you can specify the full path here.
+                            If you are using gropple in portable mode and store the executables with the gropple executable, use a prefix of
+                            <tt>./</tt>, for instance <tt>yt-dlp.exe</tt>.
+                            </span>
 
 
                             <label>Arguments</label>
@@ -82,26 +95,65 @@
                             <template x-for="(arg, j) in profile.args">
                                 <div>
                                     <input type="text" x-bind:id="'config-profiles-'+i+'-arg-'+j" placeholder="arg" x-model="profile.args[j]" />
-                                    <button class="pure-button button-del" href="#" @click.prevent="profile.args.splice(j, 1);;">delete arg</button>
+                                    <button class="button-small pure-button button-del" href="#" @click.prevent="profile.args.splice(j, 1);;">delete arg</button>
                                 </div>
                             </template>
 
-                            <button class="pure-button button-add" href="#" @click.prevent="profile.args.push('');">add arg</button>
+                            <button class="button-small pure-button button-add" href="#" @click.prevent="profile.args.push('');">add arg</button>
+                            <span class="pure-form-message">Arguments for the command. Note that the shell is not used, so there is no need to quote or escape arguments, including those with spaces.</span>
 
                             <hr>
 
                         </div>
                     </template>
 
-                    <button class="pure-button button-add" href="#" @click.prevent="config.profiles.push({name: 'new profile', command: 'youtube-dl', args: []});">add profile</button>
+                    <button class="button-small pure-button button-add" href="#" @click.prevent="config.profiles.push({name: 'new profile', command: 'youtube-dl', args: []});">add profile</button>
 
                 </fieldset>
             </form>
         </div>
+
+        <div class="pure-u-lg-1-3 pure-u-1 l-box">
+            <form class="pure-form gropple-config">
+                <fieldset>
+                    <legend>Download Options</legend>
+                    <p>You can specify custom download options here. These are (optionally) selectable in addition
+                    to the profile when starting a download. They append extra arguments to the downloader command.
+                    The most common use is to specify a particular <tt>-o</tt> argument to <tt>yt-dlp</tt> to allow files to be downloaded
+                    to a custom path.</p>
+                    </p>
+                    <template x-for="(download_option, i) in config.download_options">
+                    <div>
+                        <label x-bind:for="'config-download-option-'+i+'-name'">Name of option <span x-text="i+1"></span>
+                        </label>
+
+                        <input type="text" x-bind:id="'config-download-option-'+i+'-name'" class="input-long" placeholder="name" x-model="download_option.name" />
+
+                        <span class="pure-form-message">The name of this option. For your information only.</span>
+
+                            <label>Arguments</label>
+
+                            <template x-for="(arg, j) in download_option.args">
+                                <div>
+                                    <input type="text" x-bind:id="'config-download-option-'+i+'-arg-'+j" placeholder="arg" x-model="download_option.args[j]" />
+                                    <button class="button-small pure-button button-del" href="#" @click.prevent="download_option.args.splice(j, 1);;">delete arg</button>
+                                </div>
+                            </template>
+                        <button class="button-small pure-button button-del" href="#" @click.prevent="config.download_options.splice(i, 1);">delete option</button>
+
+                        <hr>
+
+                    </div>
+                </template>
+
+                <button class="button-small pure-button button-add" href="#" @click.prevent="config.download_options.push({name: 'new option', args: ['-o', 'someting']});">add option</button>
+
+            </fieldset>
+        </form>
     </div>
     <div class="pure-g">
         <div class="pure-u-1">
-            <button class="pure-button pure-button-primary" @click="save_config();" href="#">Save Config</button>
+            <button class="button-small pure-button button-small pure-button-primary" @click="save_config();" href="#">Save Config</button>
         </div>
     </div>
 
@@ -113,8 +165,8 @@
 {{ define "js" }}
 <script>
     function config() {
-        return { 
+        return {
-            config: { server : {}, ui : {}, profiles: [] }, 
+            config: { server : {}, ui : {}, profiles: [], download_options: []},
             error_message: '',
             success_message: '',
 

web/data/templates/index.tmpl

@@ -0,0 +1,114 @@
+{{ define "content" }}
+
+{{ template "menu.tmpl" . }}
+
+<div x-data="index()" x-init="fetch_data(); fetch_version()">
+
+    <p x-cloak x-show="version && version.upgrade_available">
+        <a href="https://github.com/tardisx/gropple/releases">Upgrade is available</a> -
+        you have
+        <span x-text="version.current_version"></span> and
+        <span x-text="version.github_version"></span>
+    is available.</p>
+
+    <div>
+        <p>
+            Drag this bookmarklet: <a href="{{ .BookmarkletURL }}">Gropple</a> to your bookmark bar, and click it
+            on any page you want to grab the video from.
+        </p>
+	<p>
+	    Please note that some adblockers may prevent the bookmarklet from opening the popup window.
+	</p>
+    </div>
+
+    <table class="pure-table">
+        <thead>
+            <tr>
+                <th>id</th>
+                <th>filename</th>
+                <th>url</th>
+                <th>state</th>
+                <th>percent</th>
+                <th>eta</th>
+                <th>finished</th>
+            </tr>
+        </thead>
+        <tbody>
+            <template x-for="item in items">
+                <tr>
+                    <td>
+                        <a class="int-link" @click="show_popup(item)" href="#">
+                          <span x-text="item.id">
+                        </a>
+                    </td>
+                    <td>
+                        <span x-show="item.files && item.files.length == 1">
+                            <span class="filelist" x-text="item.files[0]"></span>
+                        </span>
+                        <span x-data="{open: false}" x-show="item.files && item.files.length > 1">
+                            <span class="filelist" x-text="item.files.length + ' files...'"></span>
+                            <button class="pure-button button-small" @click="open = ! open" x-text="open ? 'hide' : 'show'"></button>
+                            <div x-show="open" x-transition>
+                                <ul class="filelist">
+                                    <template x-for="file in item.files">
+                                        <li x-text="file"></li>
+                                    </template>
+                                </ul>
+                            </div>
+                        </span>
+                        <span class="filelist" x-show="! item.files || item.files.length == 0"
+                              x-text="'fetching ' + item.url + '...'">
+                        </span>
+                    </td>
+                    <td><a class="int-link" x-bind:href="item.url">&#x1F517;</a></td>
+                    <td :class="'state-'+item.state" x-text="item.state"></td>
+                    <td x-text="item.percent"></td>
+                    <td x-text="item.eta"></td>
+                    <td x-text="item.finished ? '&#x2714;' : '-'"></td>
+                </tr>
+
+            </template>
+
+
+        </tbody>
+    </table>
+</div>
+{{ end }}
+
+{{ define "js" }}
+<script>
+    function index() {
+        return {
+            items: [], version: {}, popups: {},
+            fetch_version() {
+                fetch('/rest/version')
+                .then(response => response.json())
+                .then(info => {
+                    this.version = info;
+                    setTimeout(() => { this.fetch_version() }, 1000 * 60 );
+                })
+                .catch(error => {
+                    console.log('failed to fetch version info - will retry');
+                    setTimeout(() => { this.fetch_version() }, 1000 );
+                });
+            },
+            fetch_data() {
+                fetch('/rest/fetch')
+                .then(response => response.json())
+                .then(info => {
+                    // will be null if no downloads yet
+                    if (info) {
+                        this.items = info;
+                    }
+                    setTimeout(() => { this.fetch_data() }, 1000);
+                })
+            },
+            show_popup(item) {
+                // allegedly you can use the reference to pop the window to the front on subsequent
+                // clicks, but I can't seem to find a reliable way to do so.
+                this.popups[item.id] = window.open(item.popup_url, item.id, "width={{ .Config.UI.PopupWidth }},height={{ .Config.UI.PopupHeight }}");
+            },
+	    }
+    }
+</script>
+{{ end }}

web/data/templates/layout.tmpl

@@ -0,0 +1,98 @@
+{{ define "layout" }}
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <title>gropple</title>
+    <script src="/static/alpine.min.js" defer></script>
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link rel="preconnect" href="https://rsms.me/">
+    <link rel="stylesheet" href="https://rsms.me/inter/inter.css">
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/purecss@3.0.0/build/pure-min.css" integrity="sha384-X38yfunGUhNzHpBaEBsWLO+A0HDYOQi8ufWDkZ0k9e0eXz/tH3II7uKZ9msv++Ls" crossorigin="anonymous">
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/purecss@3.0.0/build/grids-responsive-min.css">
+    <style>
+
+:root {
+  font-family: Inter, sans-serif;
+  font-feature-settings: 'liga' 1, 'calt' 1; /* fix for Chrome */
+}
+@supports (font-variation-settings: normal) {
+  :root { font-family: InterVariable, sans-serif; }
+}
+
+        .button-small {
+            font-size: 85%;
+        }
+
+        .pure-g > div {
+          box-sizing: border-box;
+        }
+        .l-box {
+          padding: 2em;
+         }
+        pre {
+          font-size: 60%;
+          height: 100px;
+          overflow:auto;
+        }
+        .filelist {
+          font-size: 60%;
+        }
+        footer {
+          padding-top: 50px;
+          font-size: 30%;
+        }
+        .int-link {
+          text-decoration: none;
+          hover { color: red; }
+        }
+        .state-failed {
+          color: red;
+        }
+        .state-downloading {
+          color: blue;
+        }
+        .state-moved {
+          color: green;
+        }
+        .state-complete {
+          color: green;
+        }
+        .gropple-config {
+          font-size: 80%;
+        }
+        .gropple-config input.input-long {
+          width: 27em;
+        }
+        .gropple-config button.button-del {
+          background: rgb(202, 60, 60);
+        }
+        .gropple-config button.button-add {
+          background: rgb(60, 200, 60);
+        }
+        .gropple-config .pure-form-message {
+          padding-top: .5em;
+          padding-bottom: 1.5em;
+        }
+        .error {
+          color: red;
+          font-size: 120%;
+        }
+        .success {
+          color: green;
+        }
+
+        [x-cloak] { display: none !important; }
+
+    </style>
+</head>
+<body style="margin:4; padding:4">
+
+    {{ template "content" . }}
+    <footer>
+    Homepage: <a href="https://github.com/tardisx/gropple">https://github.com/tardisx/gropple</a><br>
+    Version: {{ .Version.CurrentVersion }}
+    </footer>
+</body>
+    {{ template "js" . }}
+</html>
+{{ end }}

web/data/templates/menu.tmpl

@@ -7,9 +7,11 @@
             <li class="pure-menu-item">
                 <a href="/config" class="pure-menu-link">Config</a>
             </li>
+            <li class="pure-menu-item">
+                <a href="/bulk" class="pure-menu-link">Bulk</a>
+            </li>
             <li class="pure-menu-item">
                 <a href="https://github.com/tardisx/gropple" class="pure-menu-link">Github</a>
             </li>
         </ul>
     </div>
-    

web/data/templates/popup.tmpl

@@ -2,27 +2,33 @@
     <div id="layout" class="pure-g pure-u-1" x-data="popup()" x-init="fetch_data()">
         <h2>Download started</h2>
         <p>Fetching <tt>{{ .dl.Url }}</tt></p>
+        <form class="pure-form">
         <table class="pure-table" >
             <tr>
                 <th>profile</th>
-                <td>
+                <td>{{ .dl.DownloadProfile.Name }}</td>
-                    <select x-bind:disabled="profile_chosen" x-on:change="update_profile()" class="pure-input-1-2" x-model="profile_chosen">
-                        <option value="">choose a profile to start</option>
-                    {{ range $i := .config.DownloadProfiles }}
-                        <option>{{ $i.Name }}</option>
-                    {{ end }}
-                    </select>
-                </td>
             </tr>
             <tr><th>current filename</th><td x-text="filename"></td></tr>
+            <tr>
+                <th>option</th>
+                <td>
+                  {{ if .dl.DownloadOption }} {{ .dl.DownloadOption.Name }} {{ else }} n/a {{ end }}
+                </td>
+            </tr>
             <tr><th>state</th><td x-text="state"></td></tr>
+            <tr x-show="playlist_total > 0"><th>playlist progress</th><td x-text="playlist_current + '/' + playlist_total"></td></tr>
             <tr><th>progress</th><td x-text="percent"></td></tr>
             <tr><th>ETA</th><td x-text="eta"></td></tr>
+
         </table>
         <p>You can close this window and your download will continue. Check the <a href="/" target="_gropple_status">Status page</a> to see all downloads in progress.</p>
+        {{ if .canStop }}
+        <button x-show="state=='Downloading'" class="button-small pure-button" @click="stop()">stop</button>
+        {{ end }}
+        </form>
         <div>
             <h4>Logs</h4>
-            <pre x-text="log">
+            <pre x-text="log" style="height: auto;">
             </pre>
         </div>
     </div>
@@ -31,17 +37,13 @@
 <script>
     function popup() {
         history.replaceState(null, '', ['/fetch/{{ .dl.Id }}'])
-        return { 
+        return {
             eta: '', percent: 0.0, state: '??', filename: '', finished: false, log :'',
-            profile_chosen: null,
+            playlist_current: 0, playlist_total: 0,
-            watch_profile() {
+            stop() {
-                this.$watch('profile_chosen', value => this.profile_chosen(value))
-            },
-            update_profile(name) {
-                console.log('you chose name', this.profile_chosen);
                 let op = {
                    method: 'POST',
-                   body: JSON.stringify({action: 'start', profile: this.profile_chosen}),
+                   body: JSON.stringify({action: 'stop'}),
                    headers: { 'Content-Type': 'application/json' }
                 };
                 fetch('/rest/fetch/{{ .dl.Id }}', op)
@@ -57,9 +59,8 @@
                     this.eta = info.eta;
                     this.percent = info.percent + "%";
                     this.state = info.state;
-                    if (this.state != 'choose profile') {
+                    this.playlist_current = info.playlist_current;
-                        this.profile_chosen = true;
+                    this.playlist_total = info.playlist_total;
-                    }
                     this.finished = info.finished;
                     if (info.files && info.files.length > 0) {
                         this.filename = info.files[info.files.length - 1];

web/data/templates/popup_create.tmpl

@@ -0,0 +1,74 @@
+{{ define "content" }}
+    <div id="layout" class="pure-g pure-u-1" x-data="popup_create()" >
+
+        <h2>Download create</h2>
+        <p>URL: <tt>{{ .url }}</tt></p>
+
+        <p class="error"  x-show="error_message"  x-transition.duration.500ms x-text="error_message"></p>
+
+        <table class="pure-table" >
+            <tr>
+                <th>profile</th>
+                <td>
+                    <select class="pure-input-1-2" x-model="profile_chosen">
+                        <option value="">choose a profile</option>
+                    {{ range $i := .config.DownloadProfiles }}
+                        <option name="{{$i.Name}}">{{ $i.Name }}</option>
+                    {{ end }}
+                    </select>
+                </td>
+            </tr>
+            <tr>
+                <th>download option</th>
+                <td>
+                    <select  class="pure-input-1-2" x-model="download_option_chosen">
+                        <option value="">no option</option>
+                    {{ range $i := .config.DownloadOptions }}
+                        <option name="{{$i.Name}}">{{ $i.Name }}</option>
+                    {{ end }}
+                    </select>
+                </td>
+            </tr>
+            <tr>
+                <th>&nbsp;</th>
+                <td>
+                    <button class="button-small pure-button" @click="start()">start download</button>
+                </td>
+            </tr>
+
+        </table>
+    </div>
+{{ end }}
+{{ define "js" }}
+<script>
+    function popup_create() {
+        return {
+            profile_chosen: "",
+            download_option_chosen: "",
+            error_message: "",
+            start() {
+                let op = {
+                   method: 'POST',
+                   body: JSON.stringify({action: 'start', url: '{{ .url }}', profile: this.profile_chosen, download_option: this.download_option_chosen}),
+                   headers: { 'Content-Type': 'application/json' }
+                };
+                fetch('/fetch', op)
+                .then(response => response.json())
+                .then(response => {
+                    console.log(response)
+                     if (response.error) {
+                        this.error_message = response.error;
+                        this.success_message = '';
+                        document.body.scrollTop = document.documentElement.scrollTop = 0;
+                    } else {
+                        this.error_message = '';
+                        console.log(response.location)
+                        window.location = response.location
+                    }
+                })
+            }
+        }
+    }
+</script>
+{{ end }}
+

web/index.html

@@ -1,84 +0,0 @@
-{{ define "content" }}
-
-{{ template "menu.tmpl" . }}
-
-
-<div x-data="index()" x-init="fetch_data(); fetch_version()">
-
-    <p x-cloak x-show="version && version.upgrade_available">
-        <a href="https://github.com/tardisx/gropple/releases">Upgrade is available</a> - 
-        you have 
-        <span x-text="version.current_version"></span> and 
-        <span x-text="version.github_version"></span>
-    is available.</p>
-
-    <div>
-        <p>
-            Drag this bookmarklet: <a href="{{ .BookmarkletURL }}">Gropple</a> to your bookmark bar, and click it
-            on any page you want to grab the video from.
-        </p>
-	<p>
-	    Please note that some adblockers may prevent the bookmarklet from opening the popup window.
-	</p>
-    </div>
-
-    <table class="pure-table">
-        <thead>
-            <tr>
-                <th>id</th><th>filename</th><th>url</th><th>state</th><th>percent</th><th>eta</th><th>finished</th>
-            </tr>
-        </thead>
-        <tbody>
-            <template x-for="item in items">
-                <tr>
-                    <td x-text="item.id"></td>
-                    <td x-text="item.files"></td>
-                    <td><a x-bind:href="item.url">link</a></td>
-                    <td x-text="item.state"></td>
-                    <td x-text="item.percent"></td>
-                    <td x-text="item.eta"></td>
-                    <td x-text="item.finished"></td>
-                </tr>
-    
-            </template>
-              
-
-            {{ range $k, $v := .Downloads }}
-            {{ end }}
-        </tbody>  
-    </table>
-</div>
-{{ end }}
-
-{{ define "js" }}
-<script>
-    function index() {
-        return { 
-            items: [], version: {},
-            fetch_version() {
-                fetch('/rest/version')
-                .then(response => response.json())
-                .then(info => {
-                    this.version = info;
-                    setTimeout(() => { this.fetch_version() }, 1000 * 60 );
-                })
-                .catch(error => {
-                    console.log('failed to fetch version info - will retry');
-                    setTimeout(() => { this.fetch_version() }, 1000 );
-                });
-            },
-            fetch_data() {
-                fetch('/rest/fetch')
-                .then(response => response.json())
-                .then(info => {
-                    // will be null if no downloads yet
-                    if (info) {
-                        this.items = info;
-                    }
-                    setTimeout(() => { this.fetch_data() }, 1000);
-                })
-            },
-        }
-    }
-</script>
-{{ end }}

web/layout.tmpl

@@ -1,60 +0,0 @@
-{{ define "layout" }}
-<html lang="en">
-<head>
-    <meta charset="utf-8">
-    <title>gropple</title>
-    <script src="//unpkg.com/alpinejs" defer></script>
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <link rel="stylesheet" href="https://unpkg.com/purecss@2.0.6/build/pure-min.css" integrity="sha384-Uu6IeWbM+gzNVXJcM9XV3SohHtmWE+3VGi496jvgX1jyvDTXfdK+rfZc8C1Aehk5" crossorigin="anonymous">
-    <link rel="stylesheet" href="https://unpkg.com/purecss@2.0.6/build/grids-responsive-min.css">
-    <style>
-        pre {
-          font-size: 60%;
-          height: 100px;
-          overflow:auto;
-        }
-        footer {
-            padding-top: 50px;
-            font-size: 30%;
-        }
-        .gropple-config {
-            font-size: 80%;
-        }
-        .gropple-config input.input-long {
-            width: 27em;
-        }
-        .gropple-config button {
-                    border-radius: 12px;
-        }
-        .gropple-config button.button-del {
-            background: rgb(202, 60, 60);
-        }
-        .gropple-config button.button-add {
-            background: rgb(60, 200, 60);
-        }
-        .gropple-config .pure-form-message {
-            padding-top: .5em;
-            padding-bottom: 1.5em;
-        }
-        .error {
-            color: red;
-        }
-        .success {
-            color: green;
-        }
-
-
-        [x-cloak] { display: none !important; }
-
-    </style>
-</head>
-<body style="margin:4; padding:4">
-
-    {{ template "content" . }}
-    <footer>
-    Homepage: <a href="https://github.com/tardisx/gropple">https://github.com/tardisx/gropple</a>
-    </footer>
-</body>
-    {{ template "js" . }}
-</html>
-{{ end }}

web/web.go

@@ -0,0 +1,544 @@
+package web
+
+import (
+	"embed"
+	"encoding/json"
+	"fmt"
+	"html/template"
+	"io"
+	"log"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/gorilla/mux"
+	"github.com/tardisx/gropple/config"
+	"github.com/tardisx/gropple/download"
+	"github.com/tardisx/gropple/version"
+)
+
+type successResponse struct {
+	Success bool   `json:"success"`
+	Message string `json:"message"`
+}
+
+type queuedResponse struct {
+	Success  bool   `json:"success"`
+	Location string `json:"location"`
+}
+
+type errorResponse struct {
+	Success bool   `json:"success"`
+	Error   string `json:"error"`
+}
+
+//go:embed data/**
+var webFS embed.FS
+
+func CreateRoutes(cs *config.ConfigService, dm *download.Manager, vm *version.Manager) *mux.Router {
+	r := mux.NewRouter()
+
+	// main index page
+	r.HandleFunc("/", homeHandler(cs, vm, dm))
+	// update info on the status page
+	r.HandleFunc("/rest/fetch", fetchInfoRESTHandler(dm))
+
+	// return static files
+	r.HandleFunc("/static/{filename}", staticHandler())
+
+	// return the config page
+	r.HandleFunc("/config", configHandler())
+	// handle config fetches/updates
+	r.HandleFunc("/rest/config", configRESTHandler(cs))
+
+	// create or present a download in the popup
+	r.HandleFunc("/fetch", fetchHandler(cs, vm, dm))
+	r.HandleFunc("/fetch/{id}", fetchHandler(cs, vm, dm))
+
+	// handle the bulk uploader
+	r.HandleFunc("/bulk", bulkHandler(cs, vm, dm))
+
+	// get/update info on a download
+	r.HandleFunc("/rest/fetch/{id}", fetchInfoOneRESTHandler(cs, dm))
+
+	// version information
+	r.HandleFunc("/rest/version", versionRESTHandler(vm))
+
+	http.Handle("/", r)
+	return r
+}
+
+// versionRESTHandler returns the version information, if we have up-to-date info from github
+func versionRESTHandler(vm *version.Manager) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if vm.GetInfo().GithubVersionFetched {
+			b, _ := json.Marshal(vm.GetInfo())
+			_, err := w.Write(b)
+			if err != nil {
+				log.Printf("could not write to client: %s", err)
+			}
+		} else {
+			w.WriteHeader(400)
+		}
+	}
+}
+
+// homeHandler returns the main index page
+func homeHandler(cs *config.ConfigService, vm *version.Manager, dm *download.Manager) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+
+		bookmarkletURL := fmt.Sprintf("javascript:(function(f,s,n,o){window.open(f+encodeURIComponent(s),n,o)}('%s/fetch?url=',window.location,'yourform','width=%d,height=%d'));", cs.Config.Server.Address, cs.Config.UI.PopupWidth, cs.Config.UI.PopupHeight)
+
+		t, err := template.ParseFS(webFS, "data/templates/layout.tmpl", "data/templates/menu.tmpl", "data/templates/index.tmpl")
+		if err != nil {
+			log.Printf("error: %s", err)
+			w.WriteHeader(http.StatusInternalServerError)
+			_, _ = w.Write([]byte(err.Error()))
+			return
+		}
+
+		type Info struct {
+			Manager        *download.Manager
+			BookmarkletURL template.URL
+			Config         *config.Config
+			Version        version.Info
+		}
+
+		info := Info{
+			Manager:        dm,
+			BookmarkletURL: template.URL(bookmarkletURL),
+			Config:         cs.Config,
+			Version:        vm.GetInfo(),
+		}
+
+		dm.Lock.Lock()
+		defer dm.Lock.Unlock()
+		err = t.ExecuteTemplate(w, "layout", info)
+		if err != nil {
+			log.Printf("error: %s", err)
+			w.WriteHeader(http.StatusInternalServerError)
+			_, _ = w.Write([]byte(err.Error()))
+			return
+		}
+	}
+}
+
+// staticHandler handles requests for static files
+func staticHandler() func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		vars := mux.Vars(r)
+
+		filename := vars["filename"]
+		if strings.Index(filename, ".js") == len(filename)-3 {
+			f, err := webFS.Open("data/js/" + filename)
+			if err != nil {
+				log.Printf("error accessing %s - %v", filename, err)
+				w.WriteHeader(http.StatusNotFound)
+				return
+			}
+			w.WriteHeader(http.StatusOK)
+			_, err = io.Copy(w, f)
+			if err != nil {
+				log.Printf("could not write to client: %s", err)
+			}
+			return
+		}
+		w.WriteHeader(http.StatusNotFound)
+	}
+}
+
+// configHandler returns the configuration page
+func configHandler() func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+
+		t, err := template.ParseFS(webFS, "data/templates/layout.tmpl", "data/templates/menu.tmpl", "data/templates/config.tmpl")
+		if err != nil {
+			log.Printf("error: %s", err)
+			w.WriteHeader(http.StatusInternalServerError)
+			_, _ = w.Write([]byte(err.Error()))
+			return
+		}
+
+		err = t.ExecuteTemplate(w, "layout", nil)
+		if err != nil {
+			log.Printf("error: %s", err)
+			w.WriteHeader(http.StatusInternalServerError)
+			_, _ = w.Write([]byte(err.Error()))
+			return
+		}
+	}
+}
+
+// configRESTHandler handles both reading and writing of the configuration
+func configRESTHandler(cs *config.ConfigService) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+
+		if r.Method == "POST" {
+			log.Printf("Updating config")
+			b, err := io.ReadAll(r.Body)
+			if err != nil {
+				log.Printf("error: %s", err)
+				w.WriteHeader(http.StatusInternalServerError)
+				_, _ = w.Write([]byte(err.Error()))
+				return
+			}
+			err = cs.Config.UpdateFromJSON(b)
+
+			if err != nil {
+				errorRes := errorResponse{Success: false, Error: err.Error()}
+				errorResB, _ := json.Marshal(errorRes)
+				w.WriteHeader(400)
+				_, err = w.Write(errorResB)
+				if err != nil {
+					log.Printf("could not write to client: %s", err)
+				}
+				return
+			}
+			cs.WriteConfig()
+		}
+		b, _ := json.Marshal(cs.Config)
+		_, err := w.Write(b)
+		if err != nil {
+			log.Printf("could not write config to client: %s", err)
+		}
+	}
+}
+
+func fetchInfoOneRESTHandler(cs *config.ConfigService, dm *download.Manager) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		vars := mux.Vars(r)
+		idString := vars["id"]
+		if idString != "" {
+			id, err := strconv.Atoi(idString)
+			if err != nil {
+				http.NotFound(w, r)
+				return
+			}
+
+			thisDownload, err := dm.GetDlById(id)
+			if err != nil {
+				http.NotFound(w, r)
+				return
+			}
+			if thisDownload == nil {
+				panic("should not happen")
+			}
+
+			if r.Method == "POST" {
+
+				type updateRequest struct {
+					Action string `json:"action"`
+				}
+
+				thisReq := updateRequest{}
+
+				b, err := io.ReadAll(r.Body)
+				if err != nil {
+					log.Printf("error: %s", err)
+					w.WriteHeader(http.StatusInternalServerError)
+					_, _ = w.Write([]byte(err.Error()))
+					return
+				}
+
+				err = json.Unmarshal(b, &thisReq)
+				if err != nil {
+					errorRes := errorResponse{Success: false, Error: err.Error()}
+					errorResB, _ := json.Marshal(errorRes)
+					w.WriteHeader(400)
+					_, err = w.Write(errorResB)
+					if err != nil {
+						log.Printf("could not write to client: %s", err)
+					}
+					return
+				}
+
+				if thisReq.Action == "stop" {
+
+					thisDownload.Stop()
+					succRes := successResponse{Success: true, Message: "download stopped"}
+					succResB, _ := json.Marshal(succRes)
+					_, err = w.Write(succResB)
+					if err != nil {
+						log.Printf("could not write to client: %s", err)
+					}
+					return
+				}
+			}
+
+			// just a get, return the object
+			thisDownload.Lock.Lock()
+			defer thisDownload.Lock.Unlock()
+
+			b, _ := json.Marshal(thisDownload)
+
+			_, err = w.Write(b)
+			if err != nil {
+				log.Printf("could not write to client: %s", err)
+			}
+			return
+		} else {
+			http.NotFound(w, r)
+		}
+	}
+}
+
+func fetchInfoRESTHandler(dm *download.Manager) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+
+		b, err := dm.DownloadsAsJSON()
+		if err != nil {
+			log.Printf("error: %s", err)
+			w.WriteHeader(http.StatusInternalServerError)
+			_, _ = w.Write([]byte(err.Error()))
+			return
+		}
+		_, err = w.Write(b)
+		if err != nil {
+			log.Printf("could not write to client: %s", err)
+		}
+	}
+}
+
+// fetchHandler shows the popup, either the initial form (for create) or the form when in
+// progress (to be updated by REST). It also handles the form POST for creating a new download.
+func fetchHandler(cs *config.ConfigService, vm *version.Manager, dm *download.Manager) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		log.Printf("fetchHandler")
+
+		method := r.Method
+
+		// if they refreshed the popup, just load the existing object, don't
+		// create a new one
+		vars := mux.Vars(r)
+		idString := vars["id"]
+
+		idInt, idOK := strconv.ParseInt(idString, 10, 32)
+
+		if method == "GET" && idOK == nil && idInt > 0 {
+			// existing, load it up
+			log.Printf("loading popup for id %d", idInt)
+			dl, err := dm.GetDlById(int(idInt))
+			if err != nil {
+				log.Printf("not found")
+				w.WriteHeader(404)
+				return
+			}
+
+			t, err := template.ParseFS(webFS, "data/templates/layout.tmpl", "data/templates/popup.tmpl")
+			if err != nil {
+				log.Printf("error: %s", err)
+				w.WriteHeader(http.StatusInternalServerError)
+				_, _ = w.Write([]byte(err.Error()))
+				return
+			}
+
+			templateData := map[string]interface{}{"dl": dl, "config": cs.Config, "canStop": download.CanStopDownload, "Version": vm.GetInfo()}
+
+			err = t.ExecuteTemplate(w, "layout", templateData)
+			if err != nil {
+				log.Printf("error: %s", err)
+				w.WriteHeader(http.StatusInternalServerError)
+				_, _ = w.Write([]byte(err.Error()))
+				return
+			}
+			return
+		} else if method == "POST" {
+			// creating a new one
+			type reqType struct {
+				URL                  string `json:"url"`
+				ProfileChosen        string `json:"profile"`
+				DownloadOptionChosen string `json:"download_option"`
+			}
+
+			req := reqType{}
+			err := json.NewDecoder(r.Body).Decode(&req)
+			if err != nil {
+				log.Printf("error decoding body of request: %s", err)
+				w.WriteHeader(http.StatusBadRequest)
+				_, _ = w.Write([]byte(err.Error()))
+				return
+			}
+
+			log.Printf("popup POST request: %#v", req)
+
+			if req.URL == "" {
+				w.WriteHeader(http.StatusBadRequest)
+				_ = json.NewEncoder(w).Encode(errorResponse{
+					Success: false,
+					Error:   "No URL supplied",
+				})
+
+				return
+			} else {
+				if req.ProfileChosen == "" {
+
+					w.WriteHeader(400)
+					_ = json.NewEncoder(w).Encode(errorResponse{
+						Success: false,
+						Error:   "you must choose a profile",
+					})
+					return
+				}
+
+				profile := cs.Config.ProfileCalled(req.ProfileChosen)
+				if profile == nil {
+					w.WriteHeader(400)
+					_ = json.NewEncoder(w).Encode(errorResponse{
+						Success: false,
+						Error:   fmt.Sprintf("no such profile: '%s'", req.ProfileChosen),
+					})
+					return
+				}
+
+				option := cs.Config.DownloadOptionCalled(req.DownloadOptionChosen)
+
+				// create the new download
+				newDL := download.NewDownload(req.URL, cs.Config)
+				id := newDL.Id
+				newDL.DownloadOption = option
+				newDL.DownloadProfile = *profile
+				dm.AddDownload(newDL)
+				dm.Queue(newDL)
+
+				w.WriteHeader(200)
+				_ = json.NewEncoder(w).Encode(queuedResponse{
+					Success:  true,
+					Location: fmt.Sprintf("/fetch/%d", id),
+				})
+			}
+		} else {
+			// a GET, show the popup so they can start the download
+			log.Print("loading popup for a new download")
+			query := r.URL.Query()
+			url, present := query["url"]
+
+			if !present {
+				w.WriteHeader(400)
+				_, _ = fmt.Fprint(w, "No url supplied")
+				return
+			}
+
+			t, err := template.ParseFS(webFS, "data/templates/layout.tmpl", "data/templates/popup_create.tmpl")
+			if err != nil {
+				log.Printf("error: %s", err)
+				w.WriteHeader(http.StatusInternalServerError)
+				_, _ = w.Write([]byte(err.Error()))
+				return
+			}
+			templateData := map[string]interface{}{"config": cs.Config, "url": url[0], "Version": vm.GetInfo()}
+
+			err = t.ExecuteTemplate(w, "layout", templateData)
+			if err != nil {
+				log.Printf("error: %s", err)
+				w.WriteHeader(http.StatusInternalServerError)
+				_, _ = w.Write([]byte(err.Error()))
+				return
+			}
+
+		}
+
+	}
+}
+
+func bulkHandler(cs *config.ConfigService, vm *version.Manager, dm *download.Manager) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		log.Printf("bulkHandler")
+
+		method := r.Method
+		switch method {
+		case "GET":
+
+			t, err := template.ParseFS(webFS, "data/templates/layout.tmpl", "data/templates/menu.tmpl", "data/templates/bulk.tmpl")
+			if err != nil {
+				log.Printf("error: %s", err)
+				w.WriteHeader(http.StatusInternalServerError)
+				_, _ = w.Write([]byte(err.Error()))
+				return
+			}
+			templateData := map[string]interface{}{"config": cs.Config, "Version": vm.GetInfo()}
+
+			err = t.ExecuteTemplate(w, "layout", templateData)
+			if err != nil {
+				log.Printf("error: %s", err)
+				w.WriteHeader(http.StatusInternalServerError)
+				_, _ = w.Write([]byte(err.Error()))
+				return
+			}
+
+			return
+		case "POST":
+			type reqBulkType struct {
+				URLs                 string `json:"urls"`
+				ProfileChosen        string `json:"profile"`
+				DownloadOptionChosen string `json:"download_option"`
+			}
+
+			req := reqBulkType{}
+			err := json.NewDecoder(r.Body).Decode(&req)
+			if err != nil {
+				log.Printf("error decoding request body: %s", err)
+				w.WriteHeader(http.StatusBadRequest)
+				_, _ = w.Write([]byte(err.Error()))
+				return
+			}
+
+			log.Printf("bulk POST request: %#v", req)
+
+			if req.URLs == "" {
+				w.WriteHeader(400)
+				_ = json.NewEncoder(w).Encode(errorResponse{
+					Success: false,
+					Error:   "No URLs supplied",
+				})
+				return
+			}
+
+			if req.ProfileChosen == "" {
+
+				w.WriteHeader(400)
+				_ = json.NewEncoder(w).Encode(errorResponse{
+					Success: false,
+					Error:   "you must choose a profile",
+				})
+				return
+			}
+
+			profile := cs.Config.ProfileCalled(req.ProfileChosen)
+			if profile == nil {
+				w.WriteHeader(400)
+				_ = json.NewEncoder(w).Encode(errorResponse{
+					Success: false,
+					Error:   fmt.Sprintf("no such profile: '%s'", req.ProfileChosen),
+				})
+				return
+			}
+
+			option := cs.Config.DownloadOptionCalled(req.DownloadOptionChosen)
+
+			// create the new downloads
+			urls := strings.Split(req.URLs, "\n")
+			count := 0
+			for _, thisURL := range urls {
+				thisURL = strings.TrimSpace(thisURL)
+				if thisURL != "" {
+					newDL := download.NewDownload(thisURL, cs.Config)
+					newDL.DownloadOption = option
+					newDL.DownloadProfile = *profile
+					dm.AddDownload(newDL)
+					dm.Queue(newDL)
+					log.Printf("queued %s", thisURL)
+					count++
+				}
+			}
+
+			w.WriteHeader(200)
+			_ = json.NewEncoder(w).Encode(successResponse{
+				Success: true,
+				Message: fmt.Sprintf("queued %d downloads", count),
+			})
+			return
+		}
+	}
+}