Appease linter

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: tardisx

.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "gomod" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "daily"

.vscode/settings.json

@@ -0,0 +1,10 @@
+{
+    "cSpell.words": [
+        "Cleanup",
+        "gropple",
+        "succ",
+        "tmpl",
+        "vars"
+    ],
+    "cSpell.language": "en-GB"
+}

CHANGELOG.md

@@ -0,0 +1,73 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+## [Unreleased]
+
+## [v0.6.0] - 2023-03-09
+
+- Configurable destinations for downloads
+- When downloading from a playlist, show the total number of videos and how many have been downloaded
+- Show version in web UI
+- Improve index page (show URL of queued downloads instead of nothing)
+- Add docker support
+- Fixes and improvements to capturing output info and showing it in the UI
+- Fixes to handling of queued downloads
+- Fix portable mode to look in binary directory, not current directory
+
+## [v0.5.5] - 2022-04-09
+
+- Fix a bug which would erase configuration when migrating from v1 to v2 config
+
+## [v0.5.4] - 2022-04-07
+
+- Check the chosen command exists when configuring a profile
+- Improve documentation
+- Add a stop button in the popup to abort a download (Linux/Mac only)
+- Move included JS to local app instead of accessing from a CDN
+- Make the simultaneous download limit apply to each unique domain
+- Support "portable" mode, reading gropple.yml from the current directory, if present
+
+## [v0.5.3] - 2021-11-21
+
+- Add config option to limit number of simultaneous downloads
+- Remove old download entries from the index after they are complete
+
+## [v0.5.2] - 2021-10-26
+
+- Provide link to re-display the popup window from the index
+- Visual improvements
+
+## [v0.5.1] - 2021-10-25
+
+- Add note about adblockers potentially blocking the popup
+- Make it possible to refresh the popup window without initiating a new download
+
+## [v0.5.0] - 2021-10-01
+
+- No more command line options, all configuration is now app-managed
+- Beautiful (ok, less ugly) new web interface
+- Multiple youtube-dl profiles, a profile can be chosen for each download
+- Bundled profiles include a standard video download and an mp3 download
+- Configuration via web interface, including download profile configuration
+
+## [v0.4.0] - 2021-09-26
+
+- Moved to semantic versioning
+- Automatic version check, prompts for upgrade in GUI
+- Fixed regex to properly match "merging" lines
+- Automatically refresh index page
+
+## [0.03] - 2021-09-24
+
+- Add option to change command (to use youtube-dlc or other forks) and command line arguments
+- Improve log display in popup
+- Improve documentation (slightly)
+
+## [0.02] - 2021-09-22
+
+- Fix #4 so that deleted files are removed from the results
+
+## [0.01] - 2021-09-22
+
+- Initial release

Dockerfile

@@ -0,0 +1,47 @@
+# Start from golang base image
+FROM golang:alpine as builder
+
+# Install git. (alpine image does not have git in it)
+RUN apk update && apk add --no-cache git curl
+
+# Set current working directory
+WORKDIR /app
+
+RUN curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /app/yt-dlp
+RUN chmod a+x /app/yt-dlp
+
+# Note here: To avoid downloading dependencies every time we
+# build image. Here, we are caching all the dependencies by
+# first copying go.mod and go.sum files and downloading them,
+# to be used every time we build the image if the dependencies
+# are not changed.
+
+# Copy go mod and sum files
+COPY go.mod ./
+COPY go.sum ./
+
+# Download all dependencies.
+RUN go mod download
+
+# Now, copy the source code
+COPY . .
+
+# Note here: CGO_ENABLED is disabled for cross system compilation
+# It is also a common best practise.
+
+# Build the application.
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ./bin/gropple .
+
+# Finally our multi-stage to build a small image
+# Start a new stage from scratch
+FROM golang:alpine
+
+# Copy the Pre-built binary file
+COPY --from=builder /app/bin/gropple .
+COPY --from=builder /app/yt-dlp /bin/
+
+# Install things we need to support yt-dlp
+RUN apk update && apk add --no-cache python3 ffmpeg
+
+# Run executable
+CMD ["./gropple", "--config-path", "/config/gropple.json"]

README.md

@@ -1,14 +1,14 @@
 # gropple
 
-A web service and bookmarklet to download videos with a single click.
+A frontend to youtube-dl (or compatible forks, like yt-dlp) to download videos with a single click, straight from your web browser.
 
 ![Screencast](/screencast.gif)
 
 ## Pre-requisites
 
-* a passing familiarity with the command line
+* some familiarity with the command line
 * youtube-dl (plus any of its required dependencies, like ffmpeg)
-* golang compiler (if you'd like to build from source)
+* golang compiler (only if you'd like to build from source)
 
 ## Build
 
@@ -18,62 +18,77 @@ A web service and bookmarklet to download videos with a single click.
 
 Binaries are available at https://github.com/tardisx/gropple/releases
 
+Gropple will automatically check for available updates and prompt you to upgrade.
+
 ## Running
 
-    gropple -port 6283 -address http://hostname:6283 -path /downloads
+    ./gropple
 
-With no arguments, it will listen on port 6283 and use an address of 'http://localhost:6283'.
+There are no command line arguments. All configuration is done via the web
+interface. The address will be printed after startup:
 
-The address must be specified so that the bookmarklet can refer to the correct
-host if it is not running on your local machine. You may also need to specify
-a different address if you are running it behind a proxy server or similar.
+    2021/09/30 23:53:00 starting gropple v0.5.0 - https://github.com/tardisx/gropple
+    2021/09/30 23:53:00 go to http://localhost:6123 for details on installing the bookmarklet and to check status
 
 ## Using
 
-Bring up `http://localhost:6283` (or your chosen address) in your browser. You 
+Bring up `http://localhost:6283` (or your configured address) in your browser. You 
 should see a link to the bookmarklet at the top of the screen, and the list of
 downloads (currently empty).
 
 Drag the bookmarklet to your favourites bar, or otherwise bookmark it as you 
-see fit.
+see fit. Any kind of browser bookmark should work. The bookmarklet contains
+embedded javascript to pass the URL of whatever page you are currently on back
+to gropple.
 
-Whenever you are on a page with a video you would like to download, simply 
+So, whenever you are on a page with a video you would like to download just 
 click the bookmarklet.
 
-A popup window will appear, the download will start on the your gropple server 
-and the status will be shown in the window.
+A popup window will appear. Choose a download profile and the download will start.
+The status will be shown in the window, updating in real time.
 
 You may close this window at any time without stopping the download, the status 
 of all downloads is available on the index page.
 
-## Using an alternative downloader
+## Configuration
 
-The default downloader is youtube-dl. It is possible to use a different downloader 
-via the `-dl-cmd` command line option.
+Click the "config" link on the index page to configure gropple. The default options 
+are fine if you are running on your local machine. If you are running it remotely
+you will need to set the "server address" to ensure the bookmarklet has the correct
+URL in it.
 
-While `gropple` will use your `PATH` to find the executable, you may also want 
-to specify a full path instead.o
+### Configuring Downloaders
 
-So, for instance, to use `youtube-dlc` instead of `youtube-dl` and specify the 
-full path:
+Gropple's default configuration uses the original youtube-dl and has two profiles set
+up, one for downloading video, the other for downloading audio (mp3).
 
-`gropple -dl-cmd /home/username/bin/youtube-dlc`
+Note that gropple does not include any downloaders, you have to install them separately.
 
-Note that this is only the path to the executable. If you need to change the 
-command arguments, see below.
+If you would like to use a youtube-dl fork (like [yt-dlp](https://github.com/yt-dlp/yt-dlp))
+or change the options, you can do so on the right hand side. Create as many profiles as you 
+wish, whenever you start a download you can choose the appropriate profile.
 
-## Changing the youtube-dl arguments
+Note that the command arguments must each be specified separately - see the default configuration
+for an example.
 
-The default arguments passed to `youtube-dl` are:
+While gropple will use your `PATH` to find the executable, you can also specify a full path
+instead. Note that any tools that the downloader calls itself (for instance, ffmpeg) will 
+probably need to be available on your path.
 
-* `--newline` (needed to allow gropple to properly parse the output)
-* `--write-info-json` (optional, but provides information on the download in the corresponding .json file)
-* `-f` and `bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best` (choose the type of video `youtube-dl` will download)
+## Portable mode
 
-These are customisable on the command line for `gropple`. For example, to duplicate these default options, you would 
-do:
+If you'd like to use gropple from a USB stick or similar, copy the config file from
+it's default location (shown when you start gropple) to the same location as the binary, and rename it to `gropple.yml`. 
 
-`gropple -dl-args '--newline' -dl-args '--write-info-json' -dl-args '-f' -dl-args 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best`
+If that file is present in the same directory as the binary, it will be used instead.
+
+## Problems
+
+Most download problems are probably diagnosable via the log - check in the popup window and scroll
+the log down to the bottom. The most common problem is that youtube-dl cannot be found, or its
+dependency (like ffmpeg) cannot be found on your path. 
+
+For other problems, please file an issue on github.
 
 ## TODO
 

build_release.pl

@@ -7,7 +7,8 @@ open my $fh, "<", "main.go" || die $!;
 
 my $version;
 while (<$fh>) {
-  $version = $1 if /^const\s+currentVersion.*?"(v[\d\.]+)"/;
+  # CurrentVersion: "v0.04"
+  $version = $1 if /CurrentVersion:\s*"(v.*?)"/;
 }
 close $fh;
 
@@ -37,3 +38,11 @@ foreach my $type (keys %build) {
   system "go", "build", "-o", "release/$type/" . $build{$type}->{filename};
   system "zip", "-j", "dist/gropple-$type-$version.zip", ( glob "release/$type/*" );
 }
+
+# now docker
+$ENV{VERSION}="$version";
+system "docker-compose", "-f", "docker-compose.build.yml", "build";
+system "docker", "tag", "tardisx/gropple:$version", "tardisx/gropple:latest";
+system "docker", "push", "tardisx/gropple:$version";
+system "docker", "push", "tardisx/gropple:latest";
+

config/config.go

@@ -0,0 +1,323 @@
+package config
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"gopkg.in/yaml.v2"
+)
+
+type Server struct {
+	Port                   int    `yaml:"port" json:"port"`
+	Address                string `yaml:"address" json:"address"`
+	DownloadPath           string `yaml:"download_path" json:"download_path"`
+	MaximumActiveDownloads int    `yaml:"maximum_active_downloads_per_domain" json:"maximum_active_downloads_per_domain"`
+}
+
+// DownloadProfile holds the details for executing a downloader
+type DownloadProfile struct {
+	Name    string   `yaml:"name" json:"name"`
+	Command string   `yaml:"command" json:"command"`
+	Args    []string `yaml:"args" json:"args"`
+}
+
+// UI holds the configuration for the user interface
+type UI struct {
+	PopupWidth  int `yaml:"popup_width" json:"popup_width"`
+	PopupHeight int `yaml:"popup_height" json:"popup_height"`
+}
+
+// Destination is the path for a place that a download can be moved to
+type Destination struct {
+	Name string `yaml:"name" json:"name"` // Name for this location
+	Path string `yaml:"path" json:"path"` // Path on disk
+}
+
+// Config is the top level of the user configuration
+type Config struct {
+	ConfigVersion    int               `yaml:"config_version" json:"config_version"`
+	Server           Server            `yaml:"server" json:"server"`
+	UI               UI                `yaml:"ui" json:"ui"`
+	Destinations     []Destination     `yaml:"destinations" json:"destinations"`
+	DownloadProfiles []DownloadProfile `yaml:"profiles" json:"profiles"`
+}
+
+// ConfigService is a struct to handle configuration requests, allowing for the
+// location that config files are loaded to be customised.
+type ConfigService struct {
+	Config     *Config
+	ConfigPath string
+}
+
+func (cs *ConfigService) LoadTestConfig() {
+	cs.LoadDefaultConfig()
+	cs.Config.DownloadProfiles = []DownloadProfile{{Name: "test profile", Command: "sleep", Args: []string{"5"}}}
+}
+
+func (cs *ConfigService) LoadDefaultConfig() {
+	defaultConfig := Config{}
+	stdProfile := DownloadProfile{Name: "standard video", Command: "yt-dlp", Args: []string{
+		"--newline",
+		"--write-info-json",
+		"-f",
+		"bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best",
+	}}
+	mp3Profile := DownloadProfile{Name: "standard mp3", Command: "yt-dlp", Args: []string{
+		"--newline",
+		"--write-info-json",
+		"--extract-audio",
+		"--audio-format", "mp3",
+	}}
+
+	defaultConfig.DownloadProfiles = append(defaultConfig.DownloadProfiles, stdProfile)
+	defaultConfig.DownloadProfiles = append(defaultConfig.DownloadProfiles, mp3Profile)
+
+	defaultConfig.Server.Port = 6123
+	defaultConfig.Server.Address = "http://localhost:6123"
+	defaultConfig.Server.DownloadPath = "/downloads"
+
+	defaultConfig.UI.PopupWidth = 500
+	defaultConfig.UI.PopupHeight = 500
+
+	defaultConfig.Server.MaximumActiveDownloads = 2
+
+	defaultConfig.Destinations = make([]Destination, 0)
+
+	defaultConfig.ConfigVersion = 3
+
+	cs.Config = &defaultConfig
+
+	return
+}
+
+func (c *Config) ProfileCalled(name string) *DownloadProfile {
+	for _, p := range c.DownloadProfiles {
+		if p.Name == name {
+			return &p
+		}
+	}
+	return nil
+}
+
+func (c *Config) DestinationCalled(name string) *Destination {
+	for _, p := range c.Destinations {
+		if p.Name == name {
+			return &p
+		}
+	}
+	return nil
+}
+
+func (c *Config) UpdateFromJSON(j []byte) error {
+	newConfig := Config{}
+	err := json.Unmarshal(j, &newConfig)
+	if err != nil {
+		log.Printf("Unmarshal error in config: %v", err)
+		return err
+	}
+
+	// sanity checks
+	if newConfig.UI.PopupHeight < 100 || newConfig.UI.PopupHeight > 2000 {
+		return errors.New("invalid popup height - should be 100-2000")
+	}
+	if newConfig.UI.PopupWidth < 100 || newConfig.UI.PopupWidth > 2000 {
+		return errors.New("invalid popup width - should be 100-2000")
+	}
+
+	// check listen port
+	if newConfig.Server.Port < 1 || newConfig.Server.Port > 65535 {
+		return errors.New("invalid server listen port")
+	}
+
+	// check download path
+	fi, err := os.Stat(newConfig.Server.DownloadPath)
+	if os.IsNotExist(err) {
+		return fmt.Errorf("path '%s' does not exist", newConfig.Server.DownloadPath)
+	}
+	if !fi.IsDir() {
+		return fmt.Errorf("path '%s' is not a directory", newConfig.Server.DownloadPath)
+	}
+
+	if newConfig.Server.MaximumActiveDownloads < 0 {
+		return fmt.Errorf("maximum active downloads can not be < 0")
+	}
+
+	// check profile name uniqueness
+	for i, p1 := range newConfig.DownloadProfiles {
+		for j, p2 := range newConfig.DownloadProfiles {
+			if i != j && p1.Name == p2.Name {
+				return fmt.Errorf("duplicate download profile name '%s'", p1.Name)
+			}
+		}
+	}
+
+	// remove leading/trailing spaces from args and commands and check for emptiness
+	for i := range newConfig.DownloadProfiles {
+		newConfig.DownloadProfiles[i].Name = strings.TrimSpace(newConfig.DownloadProfiles[i].Name)
+
+		if newConfig.DownloadProfiles[i].Name == "" {
+			return errors.New("profile name cannot be empty")
+		}
+
+		newConfig.DownloadProfiles[i].Command = strings.TrimSpace(newConfig.DownloadProfiles[i].Command)
+		if newConfig.DownloadProfiles[i].Command == "" {
+			return fmt.Errorf("command in profile '%s' cannot be empty", newConfig.DownloadProfiles[i].Name)
+		}
+
+		// check the args
+		for j := range newConfig.DownloadProfiles[i].Args {
+			newConfig.DownloadProfiles[i].Args[j] = strings.TrimSpace(newConfig.DownloadProfiles[i].Args[j])
+			if newConfig.DownloadProfiles[i].Args[j] == "" {
+				return fmt.Errorf("argument %d of profile '%s' is empty", j+1, newConfig.DownloadProfiles[i].Name)
+			}
+		}
+
+		// check the command exists
+		_, err := exec.LookPath(newConfig.DownloadProfiles[i].Command)
+		if err != nil {
+			return fmt.Errorf("Could not find %s on the path", newConfig.DownloadProfiles[i].Command)
+		}
+	}
+
+	// check destinations
+	for _, dest := range newConfig.Destinations {
+		s, err := os.Stat(dest.Path)
+		if err != nil {
+			return fmt.Errorf("destination '%s' (%s) is bad: %s", dest.Name, dest.Path, err)
+		}
+		if !s.IsDir() {
+			return fmt.Errorf("destination '%s' (%s) is not a directory", dest.Name, dest.Path)
+		}
+	}
+
+	*c = newConfig
+	return nil
+}
+
+// DetermineConfigDir determines where the config is (or should be) stored.
+func (cs *ConfigService) DetermineConfigDir() {
+	// check binary path first, for a file called gropple.yml
+	binaryPath := os.Args[0]
+	binaryDir := filepath.Dir(binaryPath)
+	potentialConfigPath := filepath.Join(binaryDir, "gropple.yml")
+
+	_, err := os.Stat(potentialConfigPath)
+	if err == nil {
+		// exists in binary directory, use that
+		// fully qualify, just for clarity in the log
+		config, err := filepath.Abs(potentialConfigPath)
+		if err == nil {
+			log.Printf("found portable config in %s", config)
+			cs.ConfigPath = config
+			return
+		} else {
+			log.Printf("got error when trying to convert config to absolute path: %s", err)
+			log.Print("falling back to using UserConfigDir")
+		}
+	}
+
+	// otherwise fall back to using the UserConfigDir
+	dir, err := os.UserConfigDir()
+	if err != nil {
+		log.Fatalf("cannot find a directory to store config: %v", err)
+	}
+	appDir := "gropple"
+
+	fullPath := dir + string(os.PathSeparator) + appDir
+	_, err = os.Stat(fullPath)
+
+	if os.IsNotExist(err) {
+		err := os.Mkdir(fullPath, 0777)
+		if err != nil {
+			log.Fatalf("Could not create config dir '%s': %v", fullPath, err)
+		}
+	}
+
+	fullFilename := fullPath + string(os.PathSeparator) + "config.yml"
+	cs.ConfigPath = fullFilename
+}
+
+// ConfigFileExists checks if the config file already exists, and also checks
+// if there is an error accessing it
+func (cs *ConfigService) ConfigFileExists() (bool, error) {
+	path := cs.ConfigPath
+	info, err := os.Stat(path)
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	if err != nil {
+		return false, fmt.Errorf("could not check if '%s' exists: %s", path, err)
+	}
+	if info.Size() == 0 {
+		return false, errors.New("config file is 0 bytes")
+	}
+	return true, nil
+}
+
+// LoadConfig loads the configuration from disk, migrating and updating it to the
+// latest version if needed.
+func (cs *ConfigService) LoadConfig() error {
+	path := cs.ConfigPath
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("Could not read config '%s': %v", path, err)
+	}
+	c := Config{}
+	cs.Config = &c
+
+	err = yaml.Unmarshal(b, &c)
+	if err != nil {
+		return fmt.Errorf("Could not parse YAML config '%s': %v", path, err)
+	}
+
+	// do migrations
+	configMigrated := false
+	if c.ConfigVersion == 1 {
+		c.Server.MaximumActiveDownloads = 2
+		c.ConfigVersion = 2
+		configMigrated = true
+		log.Print("migrated config from version 1 => 2")
+
+	}
+
+	if c.ConfigVersion == 2 {
+		c.Destinations = make([]Destination, 0)
+		c.ConfigVersion = 3
+		configMigrated = true
+		log.Print("migrated config from version 2 => 3")
+	}
+
+	if configMigrated {
+		log.Print("Writing new config after version migration")
+		cs.WriteConfig()
+	}
+
+	return nil
+}
+
+// WriteConfig writes the in-memory config to disk.
+func (cs *ConfigService) WriteConfig() {
+	s, err := yaml.Marshal(cs.Config)
+	if err != nil {
+		panic(err)
+	}
+
+	path := cs.ConfigPath
+	file, err := os.Create(
+		path,
+	)
+
+	if err != nil {
+		log.Fatalf("Could not open config file %s: %s", path, err)
+	}
+	defer file.Close()
+
+	file.Write(s)
+	file.Close()
+}

config/config_test.go

@@ -0,0 +1,60 @@
+package config
+
+import (
+	"os"
+	"testing"
+)
+
+func TestMigrationV1toV3(t *testing.T) {
+	v2Config := `config_version: 1
+server:
+  port: 6123
+  address: http://localhost:6123
+  download_path: ./
+ui:
+  popup_width: 500
+  popup_height: 500
+profiles:
+- name: standard video
+  command: youtube-dl
+  args:
+  - --newline
+  - --write-info-json
+  - -f
+  - bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best
+- name: standard mp3
+  command: youtube-dl
+  args:
+  - --newline
+  - --write-info-json
+  - --extract-audio
+  - --audio-format
+  - mp3
+`
+	cs := configServiceFromString(v2Config)
+	err := cs.LoadConfig()
+	if err != nil {
+		t.Errorf("got error when loading config: %s", err)
+	}
+	if cs.Config.ConfigVersion != 3 {
+		t.Errorf("did not migrate version (it is '%d')", cs.Config.ConfigVersion)
+	}
+	if cs.Config.Server.MaximumActiveDownloads != 2 {
+		t.Error("did not add MaximumActiveDownloads")
+	}
+	if len(cs.Config.Destinations) != 0 {
+		t.Error("incorrect number of destinations added")
+	}
+	os.Remove(cs.ConfigPath)
+}
+
+func configServiceFromString(configString string) *ConfigService {
+	tmpFile, _ := os.CreateTemp("", "gropple_test_*.yml")
+	tmpFile.Write([]byte(configString))
+	tmpFile.Close()
+	cs := ConfigService{
+		Config:     &Config{},
+		ConfigPath: tmpFile.Name(),
+	}
+	return &cs
+}

docker-compose.build.yml

@@ -0,0 +1,6 @@
+version: "3.9"
+    
+services:
+  gropple:
+    build: .
+    image: tardisx/gropple:$VERSION

docker-compose.yml

@@ -0,0 +1,12 @@
+version: "3.9"
+    
+services:
+  gropple:
+    build: .
+    image: tardisx/gropple:latest
+    volumes:
+      - ./gropple-config-dir:/config
+      - ./downloads:/downloads/
+    restart: always
+    ports:
+      - "6123:6123"

download/download.go

@@ -0,0 +1,391 @@
+package download
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"net/url"
+	"os"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/tardisx/gropple/config"
+)
+
+type Download struct {
+	Id              int                    `json:"id"`
+	Url             string                 `json:"url"`
+	PopupUrl        string                 `json:"popup_url"`
+	Process         *os.Process            `json:"-"`
+	ExitCode        int                    `json:"exit_code"`
+	State           string                 `json:"state"`
+	DownloadProfile config.DownloadProfile `json:"download_profile"`
+	Destination     *config.Destination    `json:"destination"`
+	Finished        bool                   `json:"finished"`
+	FinishedTS      time.Time              `json:"finished_ts"`
+	Files           []string               `json:"files"`
+	PlaylistCurrent int                    `json:"playlist_current"`
+	PlaylistTotal   int                    `json:"playlist_total"`
+	Eta             string                 `json:"eta"`
+	Percent         float32                `json:"percent"`
+	Log             []string               `json:"log"`
+	Config          *config.Config
+	Lock            sync.Mutex
+}
+
+// The Manager holds and is responsible for all Download objects.
+type Manager struct {
+	Downloads    []*Download
+	MaxPerDomain int
+	Lock         sync.Mutex
+}
+
+var CanStopDownload = false
+
+var downloadId int32 = 0
+
+func (m *Manager) ManageQueue() {
+	for {
+		m.Lock.Lock()
+
+		m.startQueued(m.MaxPerDomain)
+		// m.cleanup()
+		m.Lock.Unlock()
+
+		time.Sleep(time.Second)
+	}
+}
+
+// startQueued starts any downloads that have been queued, we would not exceed
+// maxRunning. If maxRunning is 0, there is no limit.
+func (m *Manager) startQueued(maxRunning int) {
+
+	active := make(map[string]int)
+
+	for _, dl := range m.Downloads {
+		dl.Lock.Lock()
+
+		if dl.State == "downloading" || dl.State == "preparing to start" {
+			active[dl.domain()]++
+		}
+		dl.Lock.Unlock()
+
+	}
+
+	for _, dl := range m.Downloads {
+
+		dl.Lock.Lock()
+
+		if dl.State == "queued" && (maxRunning == 0 || active[dl.domain()] < maxRunning) {
+			dl.State = "preparing to start"
+			active[dl.domain()]++
+			log.Printf("Starting download for id:%d (%s)", dl.Id, dl.Url)
+
+			dl.Lock.Unlock()
+
+			go func(sdl *Download) {
+				sdl.Begin()
+			}(dl)
+		} else {
+			dl.Lock.Unlock()
+		}
+
+	}
+
+}
+
+// cleanup removes old downloads from the list. Hardcoded to remove them one hour
+// completion.
+func (m *Manager) XXXcleanup() {
+	newDLs := []*Download{}
+	for _, dl := range m.Downloads {
+
+		if dl.Finished && time.Since(dl.FinishedTS) > time.Duration(time.Hour) {
+			// do nothing
+		} else {
+			newDLs = append(newDLs, dl)
+		}
+
+	}
+	m.Downloads = newDLs
+}
+
+// GetDlById returns one of the downloads in our current list.
+func (m *Manager) GetDlById(id int) (*Download, error) {
+	m.Lock.Lock()
+	defer m.Lock.Unlock()
+	for _, dl := range m.Downloads {
+		if dl.Id == id {
+			return dl, nil
+		}
+	}
+	return nil, fmt.Errorf("no download with id %d", id)
+}
+
+// Queue queues a download
+func (m *Manager) Queue(dl *Download) {
+	dl.Lock.Lock()
+	defer dl.Lock.Unlock()
+	dl.State = "queued"
+}
+
+func NewDownload(url string, conf *config.Config) *Download {
+	atomic.AddInt32(&downloadId, 1)
+	dl := Download{
+		Id:       int(downloadId),
+		Url:      url,
+		PopupUrl: fmt.Sprintf("/fetch/%d", int(downloadId)),
+		State:    "choose profile",
+		Files:    []string{},
+		Log:      []string{},
+		Config:   conf,
+		Lock:     sync.Mutex{},
+	}
+	return &dl
+}
+
+func (m *Manager) AddDownload(dl *Download) {
+	m.Lock.Lock()
+	defer m.Lock.Unlock()
+	m.Downloads = append(m.Downloads, dl)
+}
+
+// func (dl *Download) AppendLog(text string) {
+// 	dl.Lock.Lock()
+// 	defer dl.Lock.Unlock()
+// 	dl.Log = append(dl.Log, text)
+// }
+
+// Stop the download.
+func (dl *Download) Stop() {
+	if !CanStopDownload {
+		log.Print("attempted to stop download on a platform that it is not currently supported on - please report this as a bug")
+		os.Exit(1)
+	}
+
+	log.Printf("stopping the download")
+	dl.Lock.Lock()
+	defer dl.Lock.Unlock()
+	dl.Log = append(dl.Log, "aborted by user")
+	dl.Process.Kill()
+}
+
+// domain returns a domain for this Download. Download should be locked.
+func (dl *Download) domain() string {
+
+	url, err := url.Parse(dl.Url)
+	if err != nil {
+		log.Printf("Unknown domain for url: %s", dl.Url)
+		return "unknown"
+	}
+
+	return url.Hostname()
+
+}
+
+// Begin starts a download, by starting the command specified in the DownloadProfile.
+// It blocks until the download is complete.
+func (dl *Download) Begin() {
+	dl.Lock.Lock()
+
+	dl.State = "downloading"
+	cmdSlice := []string{}
+	cmdSlice = append(cmdSlice, dl.DownloadProfile.Args...)
+
+	// only add the url if it's not empty or an example URL. This helps us with testing
+	if !(dl.Url == "" || strings.Contains(dl.domain(), "example.org")) {
+		cmdSlice = append(cmdSlice, dl.Url)
+	}
+
+	cmd := exec.Command(dl.DownloadProfile.Command, cmdSlice...)
+	cmd.Dir = dl.Config.Server.DownloadPath
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		dl.State = "failed"
+		dl.Finished = true
+		dl.FinishedTS = time.Now()
+		dl.Log = append(dl.Log, fmt.Sprintf("error setting up stdout pipe: %v", err))
+		dl.Lock.Unlock()
+
+		return
+	}
+
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		dl.State = "failed"
+		dl.Finished = true
+		dl.FinishedTS = time.Now()
+		dl.Log = append(dl.Log, fmt.Sprintf("error setting up stderr pipe: %v", err))
+		dl.Lock.Unlock()
+
+		return
+	}
+
+	log.Printf("Executing command: %v", cmd)
+	err = cmd.Start()
+	if err != nil {
+		dl.State = "failed"
+		dl.Finished = true
+		dl.FinishedTS = time.Now()
+		dl.Log = append(dl.Log, fmt.Sprintf("error starting command '%s': %v", dl.DownloadProfile.Command, err))
+		dl.Lock.Unlock()
+
+		return
+	}
+	dl.Process = cmd.Process
+
+	var wg sync.WaitGroup
+
+	wg.Add(2)
+
+	dl.Lock.Unlock()
+
+	go func() {
+		defer wg.Done()
+		dl.updateDownload(stdout)
+	}()
+
+	go func() {
+		defer wg.Done()
+		dl.updateDownload(stderr)
+	}()
+
+	wg.Wait()
+	cmd.Wait()
+
+	dl.Lock.Lock()
+
+	log.Printf("Process finished for id: %d (%v)", dl.Id, cmd)
+
+	dl.State = "complete"
+	dl.Finished = true
+	dl.FinishedTS = time.Now()
+	dl.ExitCode = cmd.ProcessState.ExitCode()
+
+	if dl.ExitCode != 0 {
+		dl.State = "failed"
+	}
+	dl.Lock.Unlock()
+
+}
+
+// updateDownload updates the download based on data from the reader. Expects the
+// Download to be unlocked.
+func (dl *Download) updateDownload(r io.Reader) {
+
+	// XXX not sure if we might get a partial line?
+	buf := make([]byte, 1024)
+	for {
+		n, err := r.Read(buf)
+		if n > 0 {
+			s := string(buf[:n])
+			lines := strings.Split(s, "\n")
+
+			for _, l := range lines {
+
+				if l == "" {
+					continue
+				}
+
+				// append the raw log
+				dl.Lock.Lock()
+				dl.Log = append(dl.Log, l)
+				// look for the percent and eta and other metadata
+				dl.updateMetadata(l)
+				dl.Lock.Unlock()
+
+			}
+		}
+		if err != nil {
+			break
+		}
+	}
+}
+
+// updateMetadata parses some metadata and updates the Download. Download must be locked.
+func (dl *Download) updateMetadata(s string) {
+
+	// [download]  49.7% of ~15.72MiB at  5.83MiB/s ETA 00:07
+	// [download]  99.3% of ~1.42GiB at 320.87KiB/s ETA 00:07 (frag 212/214)
+	etaRE := regexp.MustCompile(`download.+ETA +(\d\d:\d\d(?::\d\d)?)`)
+	matches := etaRE.FindStringSubmatch(s)
+	if len(matches) == 2 {
+		dl.Eta = matches[1]
+		dl.State = "downloading"
+
+	}
+
+	percentRE := regexp.MustCompile(`download.+?([\d\.]+)%`)
+	matches = percentRE.FindStringSubmatch(s)
+	if len(matches) == 2 {
+		p, err := strconv.ParseFloat(matches[1], 32)
+		if err == nil {
+			dl.Percent = float32(p)
+		} else {
+			panic(err)
+		}
+	}
+
+	// This appears once per destination file
+	// [download] Destination: Filename with spaces and other punctuation here be careful!.mp4
+	filename := regexp.MustCompile(`download.+?Destination: (.+)$`)
+	matches = filename.FindStringSubmatch(s)
+	if len(matches) == 2 {
+		dl.Files = append(dl.Files, matches[1])
+	}
+
+	// This means a file has been "created" by merging others
+	// [ffmpeg] Merging formats into "Toto - Africa (Official HD Video)-FTQbiNvZqaY.mp4"
+	mergedFilename := regexp.MustCompile(`Merging formats into "(.+)"$`)
+	matches = mergedFilename.FindStringSubmatch(s)
+	if len(matches) == 2 {
+		dl.Files = append(dl.Files, matches[1])
+	}
+
+	// This means a file has been deleted
+	// Gross - this time it's unquoted and has trailing guff
+	// Deleting original file Toto - Africa (Official HD Video)-FTQbiNvZqaY.f137.mp4 (pass -k to keep)
+	// This is very fragile
+	deletedFile := regexp.MustCompile(`Deleting original file (.+) \(pass -k to keep\)$`)
+	matches = deletedFile.FindStringSubmatch(s)
+	if len(matches) == 2 {
+		// find the index
+		for i, f := range dl.Files {
+			if f == matches[1] {
+				dl.Files = append(dl.Files[:i], dl.Files[i+1:]...)
+				break
+			}
+		}
+	}
+
+	// [download] Downloading video 1 of 3
+	playlistDetails := regexp.MustCompile(`Downloading video (\d+) of (\d+)`)
+	matches = playlistDetails.FindStringSubmatch(s)
+	if len(matches) == 3 {
+		total, _ := strconv.ParseInt(matches[2], 10, 32)
+		current, _ := strconv.ParseInt(matches[1], 10, 32)
+		dl.PlaylistTotal = int(total)
+		dl.PlaylistCurrent = int(current)
+	}
+
+	// [Site] user: Downloading JSON metadata page 2
+	metadataDL := regexp.MustCompile(`Downloading JSON metadata page (\d+)`)
+	matches = metadataDL.FindStringSubmatch(s)
+	if len(matches) == 2 {
+		dl.State = "Downloading metadata, page " + matches[1]
+	}
+
+	// [FixupM3u8] Fixing MPEG-TS in MP4 container of "file [-168849776_456239489].mp4"
+	metadataFixup := regexp.MustCompile(`Fixing MPEG-TS in MP4 container`)
+	matches = metadataFixup.FindStringSubmatch(s)
+	if len(matches) == 1 {
+		dl.State = "Fixing MPEG-TS in MP4"
+	}
+
+}

download/download_posix.go

@@ -0,0 +1,5 @@
+package download
+
+func init() {
+	CanStopDownload = true
+}

download/download_test.go

@@ -0,0 +1,279 @@
+package download
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestUpdateMetadata(t *testing.T) {
+	newD := Download{}
+
+	// first time we spot a filename
+	newD.updateMetadata("[download] Destination: Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.f401.mp4")
+	if len(newD.Files) != 1 || newD.Files[0] != "Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.f401.mp4" {
+		t.Fatalf("incorrect Files:%v", newD.Files)
+	}
+
+	// eta's might be xx:xx:xx or xx:xx
+	newD.updateMetadata("[download]   0.0% of 504.09MiB at 135.71KiB/s ETA 01:03:36")
+	if newD.Eta != "01:03:36" {
+		t.Fatalf("bad long eta in dl\n%v", newD)
+	}
+	newD.updateMetadata("[download]   0.0% of 504.09MiB at 397.98KiB/s ETA 21:38")
+	if newD.Eta != "21:38" {
+		t.Fatalf("bad short eta in dl\n%v", newD)
+	}
+
+	// added a new file, now we are tracking two
+	newD.updateMetadata("[download] Destination: Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.f140.m4a")
+	if len(newD.Files) != 2 || newD.Files[1] != "Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.f140.m4a" {
+		t.Fatalf("incorrect Files:%v", newD.Files)
+	}
+
+	// merging
+	newD.updateMetadata("[ffmpeg] Merging formats into \"Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.mp4\"")
+	if len(newD.Files) != 3 || newD.Files[2] != "Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.mp4" {
+		t.Fatalf("did not find merged filename")
+		t.Fatalf("%v", newD.Files)
+	}
+
+	// different download
+	newD.updateMetadata("[download]  99.3% of ~1.42GiB at 320.87KiB/s ETA 00:07 (frag 212/214)")
+	if newD.Eta != "00:07" {
+		t.Fatalf("bad short eta in dl with frag\n%v", newD)
+	}
+
+	// [FixupM3u8] Fixing MPEG-TS in MP4 container of "file [-168849776_456239489].mp4"
+	newD.updateMetadata("[FixupM3u8] Fixing MPEG-TS in MP4 container of \"file [-168849776_456239489].mp4")
+	if newD.State != "Fixing MPEG-TS in MP4" {
+		t.Fatalf("did not see fixup state - state is %s", newD.State)
+	}
+
+	// deletes
+	// TODO. Not sure why I don't always see the "Deleting original file" messages after merge -
+	// maybe a youtube-dl fork thing?
+
+}
+
+// [youtube] wi7Agv1M6PY: Downloading webpage
+// [info] Writing video description metadata as JSON to: Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.info.json
+// [download] Destination: Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.f401.mp4
+// [download]   0.0% of 504.09MiB at 135.71KiB/s ETA 01:03:36
+// [download]   0.0% of 504.09MiB at 397.98KiB/s ETA 21:38
+// [download]   0.0% of 504.09MiB at 918.97KiB/s ETA 09:22
+// [download]   0.0% of 504.09MiB at  1.90MiB/s ETA 04:25
+// ..
+// [download]  99.6% of 504.09MiB at  8.91MiB/s ETA 00:00
+// [download] 100.0% of 504.09MiB at  9.54MiB/s ETA 00:00
+// [download] 100% of 504.09MiB in 01:00
+// [download] Destination: Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.f140.m4a
+// [download]   0.0% of 4.64MiB at 155.26KiB/s ETA 00:30
+// [download]   0.1% of 4.64MiB at 457.64KiB/s ETA 00:10
+// [download]   0.1% of 4.64MiB at  1.03MiB/s ETA 00:04
+// ..
+// [download]  86.2% of 4.64MiB at 10.09MiB/s ETA 00:00
+// [download] 100.0% of 4.64MiB at 10.12MiB/s ETA 00:00
+// [download] 100% of 4.64MiB in 00:00
+// [ffmpeg] Merging formats into "Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.mp4"
+
+// func TestQueue(t *testing.T) {
+// 	cs := config.ConfigService{}
+// 	cs.LoadTestConfig()
+// 	conf := cs.Config
+
+// 	new1 := Download{Id: 1, Url: "http://sub.example.org/foo1", State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
+// 	new2 := Download{Id: 2, Url: "http://sub.example.org/foo2", State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
+// 	new3 := Download{Id: 3, Url: "http://sub.example.org/foo3", State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
+// 	new4 := Download{Id: 4, Url: "http://example.org/", State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
+
+// 	dls := Downloads{&new1, &new2, &new3, &new4}
+// 	dls.StartQueued(1)
+// 	time.Sleep(time.Millisecond * 100)
+// 	if dls[0].State == "queued" {
+// 		t.Error("#1 was not started")
+// 	}
+// 	if dls[1].State != "queued" {
+// 		t.Error("#2 is not queued")
+// 	}
+// 	if dls[3].State == "queued" {
+// 		t.Error("#4 is not started")
+// 	}
+
+// 	// this should start no more, as one is still going
+// 	dls.StartQueued(1)
+// 	time.Sleep(time.Millisecond * 100)
+// 	if dls[1].State != "queued" {
+// 		t.Error("#2 was started when it should not be")
+// 	}
+
+// 	dls.StartQueued(2)
+// 	time.Sleep(time.Millisecond * 100)
+// 	if dls[1].State == "queued" {
+// 		t.Error("#2 was not started but it should be")
+
+// 	}
+
+// 	dls.StartQueued(2)
+// 	time.Sleep(time.Millisecond * 100)
+// 	if dls[3].State == "queued" {
+// 		t.Error("#4 was not started but it should be")
+// 	}
+
+// 	// reset them all
+// 	dls[0].State = "queued"
+// 	dls[1].State = "queued"
+// 	dls[2].State = "queued"
+// 	dls[3].State = "queued"
+
+// 	dls.StartQueued(0)
+// 	time.Sleep(time.Millisecond * 100)
+
+// 	// they should all be going
+// 	if dls[0].State == "queued" || dls[1].State == "queued" || dls[2].State == "queued" || dls[3].State == "queued" {
+// 		t.Error("none should be queued")
+// 	}
+
+// 	// reset them all
+// 	dls[0].State = "queued"
+// 	dls[1].State = "queued"
+// 	dls[2].State = "queued"
+// 	dls[3].State = "queued"
+
+// 	dls.StartQueued(2)
+// 	time.Sleep(time.Millisecond * 100)
+
+// 	// first two should be running, third not (same domain) and 4th running (different domain)
+// 	if dls[0].State == "queued" || dls[1].State == "queued" || dls[2].State != "queued" || dls[3].State == "queued" {
+// 		t.Error("incorrect queued")
+
+// 	}
+
+// }
+
+func TestUpdateMetadataPlaylist(t *testing.T) {
+
+	output := `
+start of log...
+[download] Downloading playlist: niceuser
+[RedGifsUser] niceuser: Downloading JSON metadata page 1
+[RedGifsUser] niceuser: Downloading JSON metadata page 2
+[RedGifsUser] niceuser: Downloading JSON metadata page 3
+[RedGifsUser] niceuser: Downloading JSON metadata page 4
+[RedGifsUser] niceuser: Downloading JSON metadata page 5
+[RedGifsUser] niceuser: Downloading JSON metadata page 6
+[info] Writing playlist metadata as JSON to: niceuser [niceuser].info.json
+[RedGifsUser] playlist niceuser: Downloading 3 videos
+[download] Downloading video 1 of 3
+[info] wrongpreciouschrysomelid: Downloading 1 format(s): hd
+[info] Writing video metadata as JSON to: Splendid Wonderful Speaker Power Chocolate Drop [wrongpreciouschrysomelid].info.json
+[download] Destination: Splendid Wonderful Speaker Power Chocolate Drop [wrongpreciouschrysomelid].mp4
+[download]   0.0% of 4.96MiB at Unknown speed ETA Unknown
+[download]   0.1% of 4.96MiB at  1.76MiB/s ETA 00:02
+[download]  20.1% of 4.96MiB at  7.28MiB/s ETA 00:00
+[download]  40.3% of 4.96MiB at 10.06MiB/s ETA 00:00
+[download]  80.6% of 4.96MiB at 14.93MiB/s ETA 00:00
+[download] 100% of 4.96MiB at 17.33MiB/s ETA 00:00
+[download] 100% of 4.96MiB in 00:00
+[download] Downloading video 2 of 3
+[info] silentnaughtyborzoi: Downloading 1 format(s): hd
+[info] Writing video metadata as JSON to: Splendid Printer Tray Computer Outdoor Window Wonderful [silentnaughtyborzoi].info.json
+[download] Destination: Splendid Printer Tray Computer Outdoor Window Wonderful [silentnaughtyborzoi].mp4
+[download]   0.0% of 5.81MiB at 896.03KiB/s ETA 00:06
+[download]   0.1% of 5.81MiB at  1.28MiB/s ETA 00:04
+[download]   0.1% of 5.81MiB at  1.59MiB/s ETA 00:03
+[download]  34.4% of 5.81MiB at  9.90MiB/s ETA 00:00
+[download]  68.8% of 5.81MiB at 12.49MiB/s ETA 00:00
+[download] 100% of 5.81MiB at 15.77MiB/s ETA 00:00
+[download] 100% of 5.81MiB in 00:00
+[download] Downloading video 3 of 3
+[info] mammothremarkablewhooper: Downloading 1 format(s): hd
+[info] Writing video metadata as JSON to: Porthole Splendid Close Up Gun Gunshot Window Wonderful [mammothremarkablewhooper].info.json
+[download] Destination: Porthole Splendid Close Up Gun Gunshot Window Wonderful [mammothremarkablewhooper].mp4
+[download]   0.0% of 2.89MiB at Unknown speed ETA Unknown
+[download]   0.1% of 2.89MiB at  1.77MiB/s ETA 00:01
+[download]   0.2% of 2.89MiB at  2.26MiB/s ETA 00:01
+[download]  34.5% of 2.89MiB at  8.23MiB/s ETA 00:00
+[download]  69.1% of 2.89MiB at 11.63MiB/s ETA 00:00
+[download] 100% of 2.89MiB at 14.25MiB/s ETA 00:00
+[download] 100% of 2.89MiB in 00:00
+[info] Writing updated playlist metadata as JSON to: niceuser [niceuser].info.json
+[download] Finished downloading playlist: niceuser
+`
+	newD := Download{}
+
+	lines := strings.Split(output, "\n")
+	for _, l := range lines {
+		// t.Log(l)
+		newD.updateMetadata(l)
+	}
+
+	if len(newD.Files) != 3 {
+		t.Errorf("%d files, not 3", len(newD.Files))
+	} else {
+		if newD.Files[0] != "Splendid Wonderful Speaker Power Chocolate Drop [wrongpreciouschrysomelid].mp4" {
+			t.Error("Wrong 1st file")
+		}
+		if newD.Files[1] != "Splendid Printer Tray Computer Outdoor Window Wonderful [silentnaughtyborzoi].mp4" {
+			t.Error("Wrong 2nd file")
+		}
+		if newD.Files[2] != "Porthole Splendid Close Up Gun Gunshot Window Wonderful [mammothremarkablewhooper].mp4" {
+			t.Error("Wrong 3rd file")
+		}
+	}
+
+	if newD.PlaylistTotal != 3 {
+		t.Errorf("playlist has total %d should be 3", newD.PlaylistTotal)
+	}
+
+}
+
+func TestUpdateMetadataSingle(t *testing.T) {
+
+	output := `
+[youtube] 2WoDQBhJCVQ: Downloading webpage
+[youtube] 2WoDQBhJCVQ: Downloading android player API JSON
+[info] 2WoDQBhJCVQ: Downloading 1 format(s): 137+140
+[info] Writing video metadata as JSON to: The Greatest Shot In Television [2WoDQBhJCVQ].info.json
+[download] Destination: The Greatest Shot In Television [2WoDQBhJCVQ].f137.mp4
+[download]   0.0% of 12.82MiB at 510.94KiB/s ETA 00:26
+[download]   0.0% of 12.82MiB at 966.50KiB/s ETA 00:13
+[download]   0.1% of 12.82MiB at  1.54MiB/s ETA 00:08
+[download]   0.1% of 12.82MiB at  2.75MiB/s ETA 00:04
+[download]   0.2% of 12.82MiB at  1.30MiB/s ETA 00:09
+[download]  77.5% of 12.82MiB at  2.54MiB/s ETA 00:01
+[download]  79.4% of 12.82MiB at  3.89MiB/s ETA 00:00
+[download]  83.3% of 12.82MiB at  6.44MiB/s ETA 00:00
+[download]  91.1% of 12.82MiB at 10.28MiB/s ETA 00:00
+[download] 100% of 12.82MiB at 12.77MiB/s ETA 00:00
+[download] 100% of 12.82MiB in 00:01
+[download] Destination: The Greatest Shot In Television [2WoDQBhJCVQ].f140.m4a
+[download]   0.1% of 1.10MiB at 286.46KiB/s ETA 00:03
+[download]   0.3% of 1.10MiB at 716.49KiB/s ETA 00:01
+[download]   0.6% of 1.10MiB at  1.42MiB/s ETA 00:00
+[download]  91.0% of 1.10MiB at  6.67MiB/s ETA 00:00
+[download] 100% of 1.10MiB at  7.06MiB/s ETA 00:00
+[download] 100% of 1.10MiB in 00:00
+[Merger] Merging formats into "The Greatest Shot In Television [2WoDQBhJCVQ].mp4"
+Deleting original file The Greatest Shot In Television [2WoDQBhJCVQ].f137.mp4 (pass -k to keep)
+Deleting original file The Greatest Shot In Television [2WoDQBhJCVQ].f140.m4a (pass -k to keep)
+`
+	newD := Download{}
+
+	lines := strings.Split(output, "\n")
+	for _, l := range lines {
+		// t.Log(l)
+		newD.updateMetadata(l)
+	}
+
+	if len(newD.Files) != 1 {
+		t.Errorf("%d files, not 1", len(newD.Files))
+	} else {
+		if newD.Files[0] != "The Greatest Shot In Television [2WoDQBhJCVQ].mp4" {
+			t.Error("Wrong 1st file")
+		}
+	}
+	if newD.PlaylistTotal != 0 {
+		t.Error("playlist detected but should not be")
+	}
+
+}

go.mod

@@ -1,5 +1,9 @@
 module github.com/tardisx/gropple
 
-go 1.16
+go 1.20
 
-require github.com/gorilla/mux v1.8.0
+require (
+	github.com/gorilla/mux v1.8.0
+	golang.org/x/mod v0.9.0
+	gopkg.in/yaml.v2 v2.4.0
+)

go.sum

@@ -1,2 +1,8 @@
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs=
+golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

main.go

@@ -9,124 +9,234 @@ import (
 	"io"
 	"log"
 	"net/http"
-	"os/exec"
-	"regexp"
 	"strings"
-	"sync"
 	"time"
 
 	"strconv"
 
 	"github.com/gorilla/mux"
+	"github.com/tardisx/gropple/config"
+	"github.com/tardisx/gropple/download"
+	"github.com/tardisx/gropple/version"
 )
 
-type download struct {
-	Id       int      `json:"id"`
-	Url      string   `json:"url"`
-	Pid      int      `json:"pid"`
-	ExitCode int      `json:"exit_code"`
-	State    string   `json:"state"`
-	Finished bool     `json:"finished"`
-	Files    []string `json:"files"`
-	Eta      string   `json:"eta"`
-	Percent  float32  `json:"percent"`
-	Log      []string `json:"log"`
-}
-
-var downloads []*download
+var dm *download.Manager
 var downloadId = 0
-var downloadPath = "./"
+var configService *config.ConfigService
 
-var address string
-
-var dlCmd = "youtube-dl"
-
-type args []string
-
-var dlArgs = args{}
-var defaultArgs = args{
-	"--write-info-json",
-	"-f", "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best",
-	"--newline",
+var versionInfo = version.Manager{
+	VersionInfo: version.Info{CurrentVersion: "v0.6.0-alpha.1"},
 }
 
-const currentVersion = "v0.03"
-
 //go:embed web
 var webFS embed.FS
 
-func (i *args) Set(value string) error {
-	*i = append(*i, strings.TrimSpace(value))
-	return nil
+type successResponse struct {
+	Success bool   `json:"success"`
+	Message string `json:"message"`
 }
 
-func (i *args) String() string {
-	return strings.Join(*i, ",")
+type errorResponse struct {
+	Success bool   `json:"success"`
+	Error   string `json:"error"`
 }
 
 func main() {
-	var port int
-	flag.IntVar(&port, "port", 6283, "port to listen on")
-	flag.StringVar(&address, "address", "http://localhost:6283", "address for the service")
-	flag.StringVar(&downloadPath, "path", "", "path for downloaded files - defaults to current directory")
-	flag.StringVar(&dlCmd, "dl-cmd", "youtube-dl", "downloader to use")
-	flag.Var(&dlArgs, "dl-args", "arguments to the downloader")
+	log.Printf("Starting gropple %s - https://github.com/tardisx/gropple", versionInfo.GetInfo().CurrentVersion)
 
+	var configPath string
+	flag.StringVar(&configPath, "config-path", "", "path to config file")
 	flag.Parse()
 
-	if len(dlArgs) == 0 {
-		dlArgs = defaultArgs
+	configService = &config.ConfigService{}
+	if configPath != "" {
+		configService.ConfigPath = configPath
+	} else {
+		configService.DetermineConfigDir()
 	}
 
+	exists, err := configService.ConfigFileExists()
+	if err != nil {
+		log.Fatal(err)
+	}
+	if !exists {
+		log.Print("No config file - creating default config")
+		configService.LoadDefaultConfig()
+		configService.WriteConfig()
+		log.Printf("Configuration written to %s", configService.ConfigPath)
+	} else {
+		err := configService.LoadConfig()
+		if err != nil {
+			log.Fatal(err)
+		}
+		log.Printf("Configuration loaded from %s", configService.ConfigPath)
+	}
+
+	// create the download manager
+	dm = &download.Manager{MaxPerDomain: configService.Config.Server.MaximumActiveDownloads}
+
 	r := mux.NewRouter()
-	r.HandleFunc("/", HomeHandler)
-	r.HandleFunc("/fetch", FetchHandler)
-	r.HandleFunc("/fetch/info/{id}", FetchInfoHandler)
+	r.HandleFunc("/", homeHandler)
+	r.HandleFunc("/static/{filename}", staticHandler)
+	r.HandleFunc("/config", configHandler)
+	r.HandleFunc("/fetch", fetchHandler)
+	r.HandleFunc("/fetch/{id}", fetchHandler)
+
+	// info for the list
+	r.HandleFunc("/rest/fetch", fetchInfoRESTHandler)
+	// info for one, including update
+	r.HandleFunc("/rest/fetch/{id}", fetchInfoOneRESTHandler)
+	r.HandleFunc("/rest/version", versionRESTHandler)
+	r.HandleFunc("/rest/config", configRESTHandler)
 
 	http.Handle("/", r)
 
 	srv := &http.Server{
 		Handler: r,
-		Addr:    fmt.Sprintf(":%d", port),
+		Addr:    fmt.Sprintf(":%d", configService.Config.Server.Port),
 		// Good practice: enforce timeouts for servers you create!
 		WriteTimeout: 5 * time.Second,
 		ReadTimeout:  5 * time.Second,
 	}
 
-	log.Printf("starting gropple %s - https://github.com/tardisx/gropple", currentVersion)
-	log.Printf("go to %s for details on installing the bookmarklet and to check status", address)
+	// check for a new version every 4 hours
+	go func() {
+		for {
+			versionInfo.UpdateGitHubVersion()
+			time.Sleep(time.Hour * 4)
+		}
+	}()
+
+	// start downloading queued downloads when slots available, and clean up
+	// old entries
+	go dm.ManageQueue()
+
+	urls := []string{
+		"https://www.youtube.com/watch?v=qG_rRkuGBW8",
+		"https://www.youtube.com/watch?v=ZUzhZpQAU40",
+		// "https://www.youtube.com/watch?v=kVxM3eRWGak",
+		// "https://www.youtube.com/watch?v=pl-y9869y0w",
+		// "https://www.youtube.com/watch?v=Uw4NEPE4l3A",
+		// "https://www.youtube.com/watch?v=6tIsT57_nS0",
+		// "https://www.youtube.com/watch?v=2RF0lcTuuYE",
+		// "https://www.youtube.com/watch?v=lymwNQY0dus",
+		// "https://www.youtube.com/watch?v=NTc-I4Z_duc",
+		// "https://www.youtube.com/watch?v=wNSm1TJ84Ac",
+	}
+	for _, u := range urls {
+		d := download.NewDownload(u, configService.Config)
+		d.DownloadProfile = *configService.Config.ProfileCalled("standard video")
+		dm.AddDownload(d)
+		dm.Queue(d)
+	}
+
+	log.Printf("Visit %s for details on installing the bookmarklet and to check status", configService.Config.Server.Address)
 	log.Fatal(srv.ListenAndServe())
 
 }
 
-func HomeHandler(w http.ResponseWriter, r *http.Request) {
+// versionRESTHandler returns the version information, if we have up-to-date info from github
+func versionRESTHandler(w http.ResponseWriter, r *http.Request) {
+	if versionInfo.GetInfo().GithubVersionFetched {
+		b, _ := json.Marshal(versionInfo.GetInfo())
+		w.Write(b)
+	} else {
+		w.WriteHeader(400)
+	}
+}
+
+// homeHandler returns the main index page
+func homeHandler(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusOK)
 
-	bookmarkletURL := fmt.Sprintf("javascript:(function(f,s,n,o){window.open(f+encodeURIComponent(s),n,o)}('%s/fetch?url=',window.location,'yourform','width=500,height=500'));", address)
+	bookmarkletURL := fmt.Sprintf("javascript:(function(f,s,n,o){window.open(f+encodeURIComponent(s),n,o)}('%s/fetch?url=',window.location,'yourform','width=%d,height=%d'));", configService.Config.Server.Address, configService.Config.UI.PopupWidth, configService.Config.UI.PopupHeight)
 
-	t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/index.html")
+	t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/menu.tmpl", "web/index.html")
 	if err != nil {
 		panic(err)
 	}
 
 	type Info struct {
-		Downloads      []*download
+		Manager        *download.Manager
 		BookmarkletURL template.URL
+		Config         *config.Config
+		Version        version.Info
 	}
 
 	info := Info{
-		Downloads:      downloads,
+		Manager:        dm,
 		BookmarkletURL: template.URL(bookmarkletURL),
+		Config:         configService.Config,
+		Version:        versionInfo.GetInfo(),
 	}
 
+	dm.Lock.Lock()
+	defer dm.Lock.Unlock()
 	err = t.ExecuteTemplate(w, "layout", info)
 	if err != nil {
 		panic(err)
 	}
-
 }
 
-func FetchInfoHandler(w http.ResponseWriter, r *http.Request) {
+// staticHandler handles requests for static files
+func staticHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+
+	filename := vars["filename"]
+	if strings.Index(filename, ".js") == len(filename)-3 {
+		f, err := webFS.Open("web/" + filename)
+		if err != nil {
+			log.Printf("error accessing %s - %v", filename, err)
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+		io.Copy(w, f)
+		return
+	}
+	w.WriteHeader(http.StatusNotFound)
+}
+
+// configHandler returns the configuration page
+func configHandler(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+
+	t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/menu.tmpl", "web/config.html")
+	if err != nil {
+		panic(err)
+	}
+
+	err = t.ExecuteTemplate(w, "layout", nil)
+	if err != nil {
+		panic(err)
+	}
+}
+
+// configRESTHandler handles both reading and writing of the configuration
+func configRESTHandler(w http.ResponseWriter, r *http.Request) {
+
+	if r.Method == "POST" {
+		log.Printf("Updating config")
+		b, err := io.ReadAll(r.Body)
+		if err != nil {
+			panic(err)
+		}
+		err = configService.Config.UpdateFromJSON(b)
+
+		if err != nil {
+			errorRes := errorResponse{Success: false, Error: err.Error()}
+			errorResB, _ := json.Marshal(errorRes)
+			w.WriteHeader(400)
+			w.Write(errorResB)
+			return
+		}
+		configService.WriteConfig()
+	}
+	b, _ := json.Marshal(configService.Config)
+	w.Write(b)
+}
+
+func fetchInfoOneRESTHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	idString := vars["id"]
 	if idString != "" {
@@ -136,199 +246,175 @@ func FetchInfoHandler(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 
-		for _, dl := range downloads {
-			if dl.Id == id {
-				b, _ := json.Marshal(dl)
-				w.Write(b)
+		thisDownload, err := dm.GetDlById(id)
+		if err != nil {
+			http.NotFound(w, r)
+			return
+		}
+		if thisDownload == nil {
+			panic("should not happen")
+		}
+
+		if r.Method == "POST" {
+
+			type updateRequest struct {
+				Action      string `json:"action"`
+				Profile     string `json:"profile"`
+				Destination string `json:"destination"`
+			}
+
+			thisReq := updateRequest{}
+
+			b, err := io.ReadAll(r.Body)
+			if err != nil {
+				panic(err)
+			}
+
+			err = json.Unmarshal(b, &thisReq)
+			if err != nil {
+				errorRes := errorResponse{Success: false, Error: err.Error()}
+				errorResB, _ := json.Marshal(errorRes)
+				w.WriteHeader(400)
+				w.Write(errorResB)
+				return
+			}
+
+			if thisReq.Action == "start" {
+				// find the profile they asked for
+				profile := configService.Config.ProfileCalled(thisReq.Profile)
+				if profile == nil {
+					panic("bad profile name?")
+				}
+				// set the profile
+				thisDownload.Lock.Lock()
+				thisDownload.DownloadProfile = *profile
+				thisDownload.Lock.Unlock()
+
+				dm.Queue(thisDownload)
+
+				succRes := successResponse{Success: true, Message: "download started"}
+				succResB, _ := json.Marshal(succRes)
+				w.Write(succResB)
+				return
+			}
+
+			if thisReq.Action == "change_destination" {
+
+				// nil means (probably) that they chose "don't move" - which is fine,
+				// and maps to nil on the Download (the default state).
+				destination := configService.Config.DestinationCalled(thisReq.Destination)
+
+				thisDownload.Lock.Lock()
+				thisDownload.Destination = destination
+				thisDownload.Lock.Unlock()
+
+				log.Printf("%#v", thisDownload)
+
+				succRes := successResponse{Success: true, Message: "destination changed"}
+				succResB, _ := json.Marshal(succRes)
+				w.Write(succResB)
+				return
+			}
+
+			if thisReq.Action == "stop" {
+
+				thisDownload.Stop()
+				succRes := successResponse{Success: true, Message: "download stopped"}
+				succResB, _ := json.Marshal(succRes)
+				w.Write(succResB)
 				return
 			}
 		}
+
+		// just a get, return the object
+		b, _ := json.Marshal(thisDownload)
+		w.Write(b)
+		return
 	} else {
 		http.NotFound(w, r)
 	}
 }
 
-func FetchHandler(w http.ResponseWriter, r *http.Request) {
+func fetchInfoRESTHandler(w http.ResponseWriter, r *http.Request) {
 
-	query := r.URL.Query()
-	url, present := query["url"] //filters=["color", "price", "brand"]
+	dm.Lock.Lock()
+	defer dm.Lock.Unlock()
+	b, _ := json.Marshal(dm.Downloads)
+	w.Write(b)
+}
 
-	if !present {
-		fmt.Fprint(w, "something")
-	} else {
+func fetchHandler(w http.ResponseWriter, r *http.Request) {
 
-		// create the record
-		// XXX should be atomic!
-		downloadId++
-		newDownload := download{
-			Id:       downloadId,
-			Url:      url[0],
-			State:    "starting",
-			Finished: false,
-			Eta:      "?",
-			Percent:  0.0,
-			Log:      make([]string, 0, 1000),
+	// if they refreshed the popup, just load the existing object, don't
+	// create a new one
+	vars := mux.Vars(r)
+	idString := vars["id"]
+
+	idInt, err := strconv.ParseInt(idString, 10, 32)
+
+	// existing, load it up
+	if err == nil && idInt > 0 {
+
+		dl, err := dm.GetDlById(int(idInt))
+		if err != nil {
+			log.Printf("not found")
+			w.WriteHeader(404)
+			return
 		}
-		downloads = append(downloads, &newDownload)
-		// XXX atomic ^^
 
-		newDownload.Log = append(newDownload.Log, "start of log...")
-
-		go func() {
-			queue(&newDownload)
-		}()
 		t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/popup.html")
 		if err != nil {
 			panic(err)
 		}
-		err = t.ExecuteTemplate(w, "layout", newDownload)
+
+		templateData := map[string]interface{}{"dl": dl, "config": configService.Config, "canStop": download.CanStopDownload}
+
+		err = t.ExecuteTemplate(w, "layout", templateData)
+		if err != nil {
+			panic(err)
+		}
+		return
+	}
+
+	query := r.URL.Query()
+	url, present := query["url"]
+
+	if !present {
+		w.WriteHeader(400)
+		fmt.Fprint(w, "No url supplied")
+		return
+	} else {
+
+		log.Printf("popup for %s", url)
+		// check the URL for a sudden but inevitable betrayal
+		if strings.Contains(url[0], configService.Config.Server.Address) {
+			w.WriteHeader(400)
+			fmt.Fprint(w, "you mustn't gropple your gropple :-)")
+			return
+		}
+
+		// create the new download
+		log.Print("creating")
+		newDL := download.NewDownload(url[0], configService.Config)
+		log.Print("adding")
+		dm.AddDownload(newDL)
+		log.Print("done")
+
+		t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/popup.html")
 		if err != nil {
 			panic(err)
 		}
 
-		//		fmt.Fprintf(w, "Started DL %d!", downloadId)
-	}
-}
+		log.Print("lock dl")
+		newDL.Lock.Lock()
+		defer newDL.Lock.Unlock()
 
-func queue(dl *download) {
-	cmdSlice := []string{}
-	cmdSlice = append(cmdSlice, dlArgs...)
-	cmdSlice = append(cmdSlice, dl.Url)
+		templateData := map[string]interface{}{"Version": versionInfo.GetInfo(), "dl": newDL, "config": configService.Config, "canStop": download.CanStopDownload}
 
-	cmd := exec.Command(dlCmd, cmdSlice...)
-	cmd.Dir = downloadPath
-
-	stdout, err := cmd.StdoutPipe()
-	if err != nil {
-		dl.State = "failed"
-		dl.Finished = true
-		dl.Log = append(dl.Log, fmt.Sprintf("error setting up stdout pipe: %v", err))
-		return
-	}
-
-	stderr, err := cmd.StderrPipe()
-	if err != nil {
-		dl.State = "failed"
-		dl.Finished = true
-		dl.Log = append(dl.Log, fmt.Sprintf("error setting up stderr pipe: %v", err))
-		return
-	}
-
-	err = cmd.Start()
-	if err != nil {
-		dl.State = "failed"
-		dl.Finished = true
-		dl.Log = append(dl.Log, fmt.Sprintf("error starting youtube-dl: %v", err))
-		return
-	}
-	dl.Pid = cmd.Process.Pid
-
-	var wg sync.WaitGroup
-
-	wg.Add(2)
-	go func() {
-		defer wg.Done()
-		updateDownload(stdout, dl)
-	}()
-
-	go func() {
-		defer wg.Done()
-		updateDownload(stderr, dl)
-	}()
-
-	wg.Wait()
-	cmd.Wait()
-
-	dl.State = "complete"
-	dl.Finished = true
-	dl.ExitCode = cmd.ProcessState.ExitCode()
-
-	if dl.ExitCode != 0 {
-		dl.State = "failed"
-	}
-
-}
-
-func updateDownload(r io.Reader, dl *download) {
-	// XXX not sure if we might get a partial line?
-	buf := make([]byte, 1024)
-	for {
-		n, err := r.Read(buf)
-		if n > 0 {
-			s := string(buf[:n])
-			lines := strings.Split(s, "\n")
-
-			for _, l := range lines {
-
-				if l == "" {
-					continue
-				}
-
-				// append the raw log
-				dl.Log = append(dl.Log, l)
-
-				// look for the percent and eta and other metadata
-				updateMetadata(dl, l)
-			}
-		}
+		err = t.ExecuteTemplate(w, "layout", templateData)
 		if err != nil {
-			break
-		}
-	}
-}
-
-func updateMetadata(dl *download, s string) {
-
-	// [download]  49.7% of ~15.72MiB at  5.83MiB/s ETA 00:07
-	etaRE := regexp.MustCompile(`download.+ETA +(\d\d:\d\d)`)
-	matches := etaRE.FindStringSubmatch(s)
-	if len(matches) == 2 {
-		dl.Eta = matches[1]
-		dl.State = "downloading"
-
-	}
-
-	percentRE := regexp.MustCompile(`download.+?([\d\.]+)%`)
-	matches = percentRE.FindStringSubmatch(s)
-	if len(matches) == 2 {
-		p, err := strconv.ParseFloat(matches[1], 32)
-		if err == nil {
-			dl.Percent = float32(p)
-		} else {
 			panic(err)
 		}
+		log.Print("unlock dl because rendered")
 	}
-
-	// This appears once per destination file
-	// [download] Destination: Filename with spaces and other punctuation here be careful!.mp4
-	filename := regexp.MustCompile(`download.+?Destination: (.+)$`)
-	matches = filename.FindStringSubmatch(s)
-	if len(matches) == 2 {
-		dl.Files = append(dl.Files, matches[1])
-	}
-
-	// This means a file has been "created" by merging others
-	// [ffmpeg] Merging formats into "Toto - Africa (Official HD Video)-FTQbiNvZqaY.mp4"
-	mergedFilename := regexp.MustCompile(`Merging formats into "(.+)$`)
-	matches = mergedFilename.FindStringSubmatch(s)
-	if len(matches) == 2 {
-		dl.Files = append(dl.Files, matches[1])
-	}
-
-	// This means a file has been deleted
-	// Gross - this time it's unquoted and has trailing guff
-	// Deleting original file Toto - Africa (Official HD Video)-FTQbiNvZqaY.f137.mp4 (pass -k to keep)
-	// This is very fragile
-	deletedFile := regexp.MustCompile(`Deleting original file (.+) \(pass -k to keep\)$`)
-	matches = deletedFile.FindStringSubmatch(s)
-	if len(matches) == 2 {
-		// find the index
-		for i, f := range dl.Files {
-			if f == matches[1] {
-				dl.Files = append(dl.Files[:i], dl.Files[i+1:]...)
-				break
-			}
-		}
-	}
-
 }

version/version.go

@@ -0,0 +1,98 @@
+// Package version deals with versioning of the software
+package version
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"sync"
+
+	"golang.org/x/mod/semver"
+)
+
+type Info struct {
+	CurrentVersion       string `json:"current_version"`
+	GithubVersion        string `json:"github_version"`
+	UpgradeAvailable     bool   `json:"upgrade_available"`
+	GithubVersionFetched bool   `json:"-"`
+}
+
+type Manager struct {
+	VersionInfo Info
+	lock        sync.Mutex
+}
+
+func (m *Manager) GetInfo() Info {
+	// log.Print("getting info... b4 lock")
+	m.lock.Lock()
+	defer m.lock.Unlock()
+
+	return m.VersionInfo
+}
+
+func (m *Manager) UpdateGitHubVersion() error {
+	m.lock.Lock()
+	m.VersionInfo.GithubVersionFetched = false
+	m.lock.Unlock()
+
+	versionUrl := "https://api.github.com/repos/tardisx/gropple/releases"
+	resp, err := http.Get(versionUrl)
+	if err != nil {
+		log.Printf("Error getting response: %v", err)
+		return err
+	}
+	defer resp.Body.Close()
+
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("failed to read body: %v", err)
+	}
+
+	type release struct {
+		HTMLUrl string `json:"html_url"`
+		TagName string `json:"tag_name"`
+		Name    string `json:"name"`
+	}
+
+	var releases []release
+
+	err = json.Unmarshal(b, &releases)
+	if err != nil {
+		return fmt.Errorf("failed to read unmarshal: %v", err)
+	}
+	if len(releases) == 0 {
+		log.Printf("found no releases on github?")
+		return errors.New("no releases found")
+	}
+
+	m.lock.Lock()
+
+	defer m.lock.Unlock()
+	m.VersionInfo.GithubVersion = releases[0].Name
+	m.VersionInfo.GithubVersionFetched = true
+	m.VersionInfo.UpgradeAvailable = m.canUpgrade()
+
+	return nil
+}
+
+func (m *Manager) canUpgrade() bool {
+	if !m.VersionInfo.GithubVersionFetched {
+		return false
+	}
+
+	if !semver.IsValid(m.VersionInfo.CurrentVersion) {
+		log.Printf("current version %s is invalid", m.VersionInfo.CurrentVersion)
+	}
+
+	if !semver.IsValid(m.VersionInfo.GithubVersion) {
+		log.Printf("github version %s is invalid", m.VersionInfo.GithubVersion)
+	}
+
+	if semver.Compare(m.VersionInfo.CurrentVersion, m.VersionInfo.GithubVersion) == -1 {
+		return true
+	}
+	return false
+}

web/config.html

@@ -0,0 +1,204 @@
+{{ define "content" }}
+
+{{ template "menu.tmpl" . }}
+
+<div x-data="config()" x-init="fetch_config();">
+
+    <p class="error"  x-show="error_message"  x-transition.duration.500ms x-text="error_message"></p>
+    <p class="success" x-show="success_message" x-transition.duration.500ms x-text="success_message"></p>
+
+    <p>Note: changes are not saved until the "Save Config" button is pressed.</p>
+
+    <div class="pure-g">
+        <div class="pure-u-1">
+            <button class="pure-button pure-button-primary" @click="save_config();" href="#">Save Config</button>
+        </div>
+    </div>
+
+    <div class="pure-g">
+
+        <div class="pure-u-lg-1-3 pure-u-1 l-box">
+   
+            <form class="pure-form pure-form-stacked gropple-config">
+                <fieldset>
+
+                    <legend>Server</legend>
+
+                    <label for="config-server-port">Listen Port</label>
+                    <input type="text" id="config-server-port" placeholder="port number" x-model.number="config.server.port" />
+                    <span class="pure-form-message">The port the web server will listen on.</span>
+
+                    <label for="config-server-address">Server address (URL)</label>
+                    <input type="text" id="config-server-address" class="input-long" placeholder="server address" x-model="config.server.address" />
+                    <span class="pure-form-message">
+                        The address the service will be available on. Generally it will be http://hostname:port where
+                        hostname is the host the server is running on, and port is the port you set above.
+                    </span>
+
+                    <label for="config-server-downloadpath">Download path</label>
+                    <input type="text" id="config-server-downloadpath" placeholder="path" class="input-long" x-model="config.server.download_path" />
+                    <span class="pure-form-message">The default path on the server to download files to.</span>
+
+                    <label for="config-server-max-downloads">Maximum active downloads per domain</label>
+                    <input type="text" id="config-server-max-downloads" placeholder="2" class="input-long" x-model.number="config.server.maximum_active_downloads_per_domain" />
+                    <span class="pure-form-message">How many downloads can be simultaneously active. Use '0' for no limit. This limit is applied per domain that you download from.</span>
+
+                    <legend>UI</legend>
+
+                    <p>Note that changes to the popup dimensions will require you to recreate your bookmarklet.</p>
+
+                    <label for="config-ui-popupwidth">Popup Width</label>
+                    <input type="text" id="config-ui-popupwidth" placeholder="width in pixels" x-model.number="config.ui.popup_width" />
+                    <span class="pure-form-message">The width of popup windows in pixels.</span>
+
+                    <label for="config-ui-popupheight">Popup Height</label>
+                    <input type="text" id="config-ui-popupheight" placeholder="height in pixels" x-model.number="config.ui.popup_height" />
+                    <span class="pure-form-message">The height of popup windows in pixels.</span>
+
+                </fieldset>
+            </form>
+
+        </div>
+        <div class="pure-u-lg-1-3 pure-u-1 l-box">
+            <form class="pure-form gropple-config">
+                <fieldset>
+
+                    <legend>Download Profiles</legend>
+
+                    <p>Gropple supports multiple download profiles. Each profile specifies a different youtube-dl 
+                        compatible command, and arguments. When starting a download, you may choose which profile 
+                        to use. The URL will be appended to the argument list at the end.
+                    </p>
+
+                    <hr>
+
+                    <template x-for="(profile, i) in config.profiles">
+                        <div>
+                            <label x-bind:for="'config-profiles-'+i+'-name'">Name of profile <span x-text="i+1"></span>
+                            </label>                                
+                            
+                            <input type="text" x-bind:id="'config-profiles-'+i+'-name'" class="input-long" placeholder="name" x-model="profile.name" />
+                            <button class="pure-button button-del" href="#" @click.prevent="config.profiles.splice(i, 1);;">delete profile</button>
+
+                            <span class="pure-form-message">The name of this profile. For your information only.</span>
+
+                            <label x-bind:for="'config-profiles-'+i+'-command'">Command to run</label>
+                            <input type="text" x-bind:id="'config-profiles-'+i+'-command'" class="input-long" placeholder="name" x-model="profile.command" />
+                            <span class="pure-form-message">Which command to run. Your path will be searched, or you can specify the full path here.</span>
+
+
+                            <label>Arguments</label>
+
+                            <template x-for="(arg, j) in profile.args">
+                                <div>
+                                    <input type="text" x-bind:id="'config-profiles-'+i+'-arg-'+j" placeholder="arg" x-model="profile.args[j]" />
+                                    <button class="pure-button button-del" href="#" @click.prevent="profile.args.splice(j, 1);;">delete arg</button>
+                                </div>
+                            </template>
+
+                            <button class="pure-button button-add" href="#" @click.prevent="profile.args.push('');">add arg</button>
+                            <span class="pure-form-message">Arguments for the command. Note that the shell is not used, so there is no need to quote or escape arguments, including those with spaces.</span>
+
+                            <hr>
+
+                        </div>
+                    </template>
+
+                    <button class="pure-button button-add" href="#" @click.prevent="config.profiles.push({name: 'new profile', command: 'youtube-dl', args: []});">add profile</button>
+
+                </fieldset>
+            </form>
+        </div>
+
+        <div class="pure-u-lg-1-3 pure-u-1 l-box">
+            <form class="pure-form gropple-config">
+                <fieldset>
+                    <legend>Destinations</legend>
+                    <p>You can specify custom destinations (directories) here. Downloads can be
+                        moved to one of these directories after completion from the index page, 
+                        if you do not want them to be left in the download path above.</p>
+                    </p>
+                    <template x-for="(dest, i) in config.destinations">
+                    <div>
+                        <label x-bind:for="'config-destinations-'+i+'-name'">Name of destination <span x-text="i+1"></span>
+                        </label>                                
+                        
+                        <input type="text" x-bind:id="'config-destinations-'+i+'-name'" class="input-long" placeholder="name" x-model="dest.name" />
+
+                        <span class="pure-form-message">The name of this destination. For your information only.</span>
+
+                        <label x-bind:for="'config-destinations-'+i+'-command'">Path</label>
+                        <input type="text" x-bind:id="'config-destinations-'+i+'-command'" class="input-long" placeholder="name" x-model="dest.path" />
+                        <span class="pure-form-message">Path to move completed downloads to.</span>
+
+                        <button class="pure-button button-del" href="#" @click.prevent="config.destinations.splice(i, 1);">delete destination</button>
+
+                        <hr>
+
+                    </div>
+                </template>
+
+                <button class="pure-button button-add" href="#" @click.prevent="config.destinations.push({name: 'new destination', path: '/tmp'});">add destination</button>
+
+            </fieldset>
+        </form>
+    </div>
+    <div class="pure-g">
+        <div class="pure-u-1">
+            <button class="pure-button pure-button-primary" @click="save_config();" href="#">Save Config</button>
+        </div>
+    </div>
+
+
+</div>
+
+{{ end }}
+
+{{ define "js" }}
+<script>
+    function config() {
+        return { 
+            config: { server : {}, ui : {}, profiles: [], destinations: []}, 
+            error_message: '',
+            success_message: '',
+
+            fetch_config() {
+                fetch('/rest/config')
+                .then(response => response.json())
+                .then(config => {
+                    this.config = config;
+                })
+                .catch(error => {
+                    console.log('failed to fetch config', error);
+                });
+            },
+            save_config() {
+                let op = {
+                   method: 'POST',
+                   body: JSON.stringify(this.config),
+                   headers: { 'Content-Type': 'application/json' }
+                };
+                fetch('/rest/config', op)
+                .then(response => {
+                    return response.json();
+                })
+                .then(response => {
+                    if (response.error) {
+                        this.error_message = response.error;
+                        this.success_message = '';
+                        document.body.scrollTop = document.documentElement.scrollTop = 0;
+                    } else {
+                        this.error_message = '';
+                        this.success_message = 'configuration saved';
+                        document.body.scrollTop = document.documentElement.scrollTop = 0;
+                        this.config = response;
+                    }
+                })
+                .catch(error => {
+                    console.log('exception' ,error);
+                });
+            }
+        }
+    }
+</script>
+{{ end }}

web/index.html

@@ -1,34 +1,98 @@
 {{ define "content" }}
-<div>
-<p>
-    Drag this bookmarklet: <a href="{{ .BookmarkletURL }}">Gropple</a> to your bookmark bar, and click it
-    on any page you want to grab the video from.
-</p>
-</div>
 
-<div>
+{{ template "menu.tmpl" . }}
+
+<div x-data="index()" x-init="fetch_data(); fetch_version()">
+
+    <p x-cloak x-show="version && version.upgrade_available">
+        <a href="https://github.com/tardisx/gropple/releases">Upgrade is available</a> - 
+        you have 
+        <span x-text="version.current_version"></span> and 
+        <span x-text="version.github_version"></span>
+    is available.</p>
+
+    <div>
+        <p>
+            Drag this bookmarklet: <a href="{{ .BookmarkletURL }}">Gropple</a> to your bookmark bar, and click it
+            on any page you want to grab the video from.
+        </p>
+	<p>
+	    Please note that some adblockers may prevent the bookmarklet from opening the popup window.
+	</p>
+    </div>
+
     <table class="pure-table">
         <thead>
             <tr>
-                <th>id</th><th>filename</th><th>url</th><th>state</th><th>percent</th><th>eta</th><th>finished</th>
+		    <th>id</th><th>filename</th><th>url</th><th>show</th><th>state</th><th>percent</th><th>eta</th><th>finished</th>
             </tr>
         </thead>
         <tbody>
-            {{ range $k, $v := .Downloads }}
-            <tr>
-                <td>{{ $v.Id }}</td>
-                <td>{{ range $_, $f := $v.Files }}{{ $f }}<br>{{ end }}</td>
-                <td><a href="{{ $v.Url }}">link</a></td>
-                <td>{{ $v.State }}</td>
-                <td>{{ $v.Percent }}</td>
-                <td>{{ $v.Eta }}</td>
-                <td>{{ $v.Finished }}</td>
-            </tr>
-            {{ end }}
+            <template x-for="item in items">
+                <tr>
+                    <td x-text="item.id"></td>
+                    <td>
+                        <span x-show="item.files && item.files.length > 0">
+                            <ul>
+                                <template x-for="file in item.files">
+                                    <li x-text="file"></li>
+                                </template>
+                            </ul>
+                        </span>
+                        <span x-show="! item.files || item.files.length == 0"
+                              x-text="item.url">
+                        </span>
+                    </td>
+                    <td><a class="int-link" x-bind:href="item.url">&#x2197;</a></td>
+                    <td><a class="int-link" @click="show_popup(item)" href="#">&#x1F4C4;</a></td>
+                    <td :class="'state-'+item.state" x-text="item.state"></td>
+                    <td x-text="item.percent"></td>
+                    <td x-text="item.eta"></td>
+                    <td x-text="item.finished ? '&#x2714;' : '-'"></td>
+                </tr>
+    
+            </template>
+              
+
         </tbody>  
     </table>
 </div>
 {{ end }}
 
 {{ define "js" }}
-{{ end }}
+<script>
+    function index() {
+        return { 
+            items: [], version: {}, popups: {},
+            fetch_version() {
+                fetch('/rest/version')
+                .then(response => response.json())
+                .then(info => {
+                    this.version = info;
+                    setTimeout(() => { this.fetch_version() }, 1000 * 60 );
+                })
+                .catch(error => {
+                    console.log('failed to fetch version info - will retry');
+                    setTimeout(() => { this.fetch_version() }, 1000 );
+                });
+            },
+            fetch_data() {
+                fetch('/rest/fetch')
+                .then(response => response.json())
+                .then(info => {
+                    // will be null if no downloads yet
+                    if (info) {
+                        this.items = info;
+                    }
+                    setTimeout(() => { this.fetch_data() }, 1000);
+                })
+            },
+            show_popup(item) {
+                // allegedly you can use the reference to pop the window to the front on subsequent
+                // clicks, but I can't seem to find a reliable way to do so.
+                this.popups[item.id] = window.open(item.popup_url, item.id, "width={{ .Config.UI.PopupWidth }},height={{ .Config.UI.PopupHeight }}");
+            },
+	    }
+    }
+</script>
+{{ end }}

web/layout.tmpl

@@ -3,21 +3,78 @@
 <head>
     <meta charset="utf-8">
     <title>gropple</title>
-    <script src="//unpkg.com/alpinejs" defer></script>
+    <script src="/static/alpine.min.js" defer></script>
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <link rel="stylesheet" href="https://unpkg.com/purecss@2.0.6/build/pure-min.css" integrity="sha384-Uu6IeWbM+gzNVXJcM9XV3SohHtmWE+3VGi496jvgX1jyvDTXfdK+rfZc8C1Aehk5" crossorigin="anonymous">
     <link rel="stylesheet" href="https://unpkg.com/purecss@2.0.6/build/grids-responsive-min.css">
     <style>
+        .pure-g > div {
+          box-sizing: border-box;
+        }
+        .l-box {
+          padding: 2em;
+         }
         pre {
           font-size: 60%;
           height: 100px;
           overflow:auto;
         }
+        footer {
+          padding-top: 50px;
+          font-size: 30%;
+        }
+        .int-link {
+          text-decoration: none;
+          hover { color: red; }
+        }
+        .state-failed {
+          color: red;
+        }
+        .state-downloading {
+          color: blue;
+        }
+        .state-complete {
+          color: green;
+        }
+        .gropple-config {
+          font-size: 80%;
+        }
+        .gropple-config input.input-long {
+          width: 27em;
+        }
+        .gropple-config button {
+          border-radius: 12px;
+        }
+        .gropple-config button.button-del {
+          background: rgb(202, 60, 60);
+        }
+        .gropple-config button.button-add {
+          background: rgb(60, 200, 60);
+        }
+        .gropple-config .pure-form-message {
+          padding-top: .5em;
+          padding-bottom: 1.5em;
+        }
+        .error {
+          color: red;
+          font-size: 150%;
+        }
+        .success {
+          color: green;
+        }
+
+        [x-cloak] { display: none !important; }
+
     </style>
 </head>
 <body style="margin:4; padding:4">
+
     {{ template "content" . }}
+    <footer>
+    Homepage: <a href="https://github.com/tardisx/gropple">https://github.com/tardisx/gropple</a><br>
+    Version: {{ .Version.CurrentVersion }}
+    </footer>
 </body>
     {{ template "js" . }}
 </html>
-{{ end }}
+{{ end }}

web/menu.tmpl

@@ -0,0 +1,15 @@
+    <div class="pure-menu pure-menu-horizontal" style="height: 2em;">
+        <a href="#" class="pure-menu-heading pure-menu-link">gropple</a>
+        <ul class="pure-menu-list">
+            <li class="pure-menu-item">
+                <a href="/" class="pure-menu-link">Home</a>
+            </li>
+            <li class="pure-menu-item">
+                <a href="/config" class="pure-menu-link">Config</a>
+            </li>
+            <li class="pure-menu-item">
+                <a href="https://github.com/tardisx/gropple" class="pure-menu-link">Github</a>
+            </li>
+        </ul>
+    </div>
+    

web/popup.html

@@ -1,14 +1,41 @@
 {{ define "content" }}
-    <div id="layout" class="pure-g pure-u-1" x-data="popup()"  x-init="fetch_data()">
+    <div id="layout" class="pure-g pure-u-1" x-data="popup()" x-init="fetch_data()">
         <h2>Download started</h2>
-        <p>Fetching <tt>{{ .Url }}</tt></p>
+        <p>Fetching <tt>{{ .dl.Url }}</tt></p>
         <table class="pure-table" >
+            <tr>
+                <th>profile</th>
+                <td>
+                    <select x-bind:disabled="profile_chosen" x-on:change="update_profile()" class="pure-input-1-2" x-model="profile_chosen">
+                        <option value="">choose a profile to start</option>
+                    {{ range $i := .config.DownloadProfiles }}
+                        <option>{{ $i.Name }}</option>
+                    {{ end }}
+                    </select>
+                </td>
+            </tr>
             <tr><th>current filename</th><td x-text="filename"></td></tr>
+            <tr>
+                <th>destination</th>
+                <td>
+                    <select  x-on:change="update_destination()" class="pure-input-1-2" x-model="destination_chosen">
+                        <option value="-">leave in {{ .config.Server.DownloadPath }}</option>
+                    {{ range $i := .config.Destinations }}
+                        <option>{{ $i.Name }}</option>
+                    {{ end }}
+                    </select>
+                </td>
+            </tr>
             <tr><th>state</th><td x-text="state"></td></tr>
+            <tr x-show="playlist_total > 0"><th>playlist progress</th><td x-text="playlist_current + '/' + playlist_total"></td></tr>
             <tr><th>progress</th><td x-text="percent"></td></tr>
             <tr><th>ETA</th><td x-text="eta"></td></tr>
+            
         </table>
         <p>You can close this window and your download will continue. Check the <a href="/" target="_gropple_status">Status page</a> to see all downloads in progress.</p>
+        {{ if .canStop }}
+        <button x-show="state=='downloading'" class="pure-button" @click="stop()">stop</button>
+        {{ end }}
         <div>
             <h4>Logs</h4>
             <pre x-text="log">
@@ -19,15 +46,64 @@
 {{ define "js" }}
 <script>
     function popup() {
+        history.replaceState(null, '', ['/fetch/{{ .dl.Id }}'])
         return { 
             eta: '', percent: 0.0, state: '??', filename: '', finished: false, log :'',
+            playlist_current: 0, playlist_total: 0,
+            profile_chosen: null,
+            destination_chosen: null,
+            watch_profile() {
+                this.$watch('profile_chosen', value => this.profile_chosen(value))
+            },
+            update_profile(name) {
+                console.log('you chose name', this.profile_chosen);
+                let op = {
+                   method: 'POST',
+                   body: JSON.stringify({action: 'start', profile: this.profile_chosen}),
+                   headers: { 'Content-Type': 'application/json' }
+                };
+                fetch('/rest/fetch/{{ .dl.Id }}', op)
+                .then(response => response.json())
+                .then(info => {
+                    console.log(info)
+                })
+            },
+            update_destination(name) {
+                let op = {
+                   method: 'POST',
+                   body: JSON.stringify({action: 'change_destination', destination: this.destination_chosen}),
+                   headers: { 'Content-Type': 'application/json' }
+                };
+                fetch('/rest/fetch/{{ .dl.Id }}', op)
+                .then(response => response.json())
+                .then(info => {
+                    console.log(info)
+                })
+            },
+            stop() {
+                let op = {
+                   method: 'POST',
+                   body: JSON.stringify({action: 'stop'}),
+                   headers: { 'Content-Type': 'application/json' }
+                };
+                fetch('/rest/fetch/{{ .dl.Id }}', op)
+                .then(response => response.json())
+                .then(info => {
+                    console.log(info)
+                })
+            },
             fetch_data() {
-                fetch('/fetch/info/{{ .Id }}')
+                fetch('/rest/fetch/{{ .dl.Id }}')
                 .then(response => response.json())
                 .then(info => {
                     this.eta = info.eta;
                     this.percent = info.percent + "%";
                     this.state = info.state;
+                    this.playlist_current = info.playlist_current;
+                    this.playlist_total = info.playlist_total;
+                    if (this.state != 'choose profile') {
+                        this.profile_chosen = true;
+                    }
                     this.finished = info.finished;
                     if (info.files && info.files.length > 0) {
                         this.filename = info.files[info.files.length - 1];