Bump version

.gitignore

@@ -1,3 +1,4 @@
 gropple
-dist
 release
+dist
+.env

.goreleaser.yaml

@@ -0,0 +1,49 @@
+# This is an example .goreleaser.yml file with some sensible defaults.
+# Make sure to check the documentation at https://goreleaser.com
+
+# The lines below are called `modelines`. See `:help modeline`
+# Feel free to remove those if you don't want/need to use them.
+# yaml-language-server: $schema=https://goreleaser.com/static/schema.json
+# vim: set ts=2 sw=2 tw=0 fo=cnqoj
+
+version: 1
+
+before:
+  hooks:
+    # You may remove this if you don't use go modules.
+    - go mod tidy
+
+builds:
+  - env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - windows
+      - darwin
+
+archives:
+  - format: tar.gz
+    # this name template makes the OS and Arch compatible with the results of `uname`.
+    name_template: >-
+      {{ .ProjectName }}_
+      {{- title .Os }}_
+      {{- if eq .Arch "amd64" }}x86_64
+      {{- else if eq .Arch "386" }}i386
+      {{- else }}{{ .Arch }}{{ end }}
+      {{- if .Arm }}v{{ .Arm }}{{ end }}
+    # use zip for windows archives
+    format_overrides:
+      - goos: windows
+        format: zip
+
+changelog:
+  skip: true
+
+dockers:
+  - image_templates:
+      - "tardisx/gropple:{{ .Tag }}"
+      - "tardisx/gropple:v{{ .Major }}"
+      - "tardisx/gropple:v{{ .Major }}.{{ .Minor }}"
+      - "tardisx/gropple:latest"
+
+

.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Launch Package",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${fileDirname}"
+        }
+    ]
+}

.vscode/settings.json

@@ -1,10 +1,12 @@
 {
     "cSpell.words": [
         "Cleanup",
+        "frag",
         "gropple",
         "succ",
         "tmpl",
-        "vars"
+        "vars",
+        "youtube"
     ],
     "cSpell.language": "en-GB"
 }

Dockerfile

@@ -1,47 +1,9 @@
-# Start from golang base image
+FROM ubuntu:mantic
-FROM golang:alpine as builder
+COPY gropple /
 
-# Install git. (alpine image does not have git in it)
+RUN apt update &&  apt install -y curl python3 ffmpeg
-RUN apk update && apk add --no-cache git curl
+RUN curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/bin/yt-dlp
-
+RUN chmod a+x /usr/bin/yt-dlp
-# Set current working directory
-WORKDIR /app
-
-RUN curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /app/yt-dlp
-RUN chmod a+x /app/yt-dlp
-
-# Note here: To avoid downloading dependencies every time we
-# build image. Here, we are caching all the dependencies by
-# first copying go.mod and go.sum files and downloading them,
-# to be used every time we build the image if the dependencies
-# are not changed.
-
-# Copy go mod and sum files
-COPY go.mod ./
-COPY go.sum ./
-
-# Download all dependencies.
-RUN go mod download
-
-# Now, copy the source code
-COPY . .
-
-# Note here: CGO_ENABLED is disabled for cross system compilation
-# It is also a common best practise.
-
-# Build the application.
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ./bin/gropple .
-
-# Finally our multi-stage to build a small image
-# Start a new stage from scratch
-FROM golang:alpine
-
-# Copy the Pre-built binary file
-COPY --from=builder /app/bin/gropple .
-COPY --from=builder /app/yt-dlp /bin/
-
-# Install things we need to support yt-dlp
-RUN apk update && apk add --no-cache python3 ffmpeg
 
 # Run executable
-CMD ["./gropple", "--config-path", "/config/gropple.json"]
+CMD ["/gropple", "--config-path", "/config/gropple.json"]

README.md

@@ -99,6 +99,10 @@ Many download problems are diagnosable via the log - check in the popup window
 and scroll the log down to the bottom. The most common problem is that `yt-dlp`
 cannot be found, or its dependency (like `ffmpeg`) cannot be found on your path.
 
+Gropple only calls external tools like `yt-dlp` to do the downloading. If you
+are having problems downloading from a site, make sure that `yt-dlp` is updated
+to the latest version (`yd-dlp -U`).
+
 For other problems, please file an issue on github.
 
 ## TODO

config/config.go

@@ -27,6 +27,12 @@ type DownloadProfile struct {
 	Args    []string `yaml:"args" json:"args"`
 }
 
+// DownloadOption contains configuration for extra arguments to pass to the download command
+type DownloadOption struct {
+	Name string   `yaml:"name" json:"name"`
+	Args []string `yaml:"args" json:"args"`
+}
+
 // UI holds the configuration for the user interface
 type UI struct {
 	PopupWidth  int `yaml:"popup_width" json:"popup_width"`
@@ -44,8 +50,9 @@ type Config struct {
 	ConfigVersion    int               `yaml:"config_version" json:"config_version"`
 	Server           Server            `yaml:"server" json:"server"`
 	UI               UI                `yaml:"ui" json:"ui"`
-	Destinations     []Destination     `yaml:"destinations" json:"destinations"`
+	Destinations     []Destination     `yaml:"destinations" json:"destinations"` // no longer in use, see DownloadOptions
 	DownloadProfiles []DownloadProfile `yaml:"profiles" json:"profiles"`
+	DownloadOptions  []DownloadOption  `yaml:"download_options" json:"download_options"`
 }
 
 // ConfigService is a struct to handle configuration requests, allowing for the
@@ -88,7 +95,8 @@ func (cs *ConfigService) LoadDefaultConfig() {
 
 	defaultConfig.Server.MaximumActiveDownloads = 2
 
-	defaultConfig.Destinations = make([]Destination, 0)
+	defaultConfig.Destinations = nil
+	defaultConfig.DownloadOptions = make([]DownloadOption, 0)
 
 	defaultConfig.ConfigVersion = 3
 
@@ -96,6 +104,7 @@ func (cs *ConfigService) LoadDefaultConfig() {
 
 }
 
+// ProfileCalled returns the corresponding DownloadProfile, or nil if it does not exist
 func (c *Config) ProfileCalled(name string) *DownloadProfile {
 	for _, p := range c.DownloadProfiles {
 		if p.Name == name {
@@ -105,10 +114,11 @@ func (c *Config) ProfileCalled(name string) *DownloadProfile {
 	return nil
 }
 
-func (c *Config) DestinationCalled(name string) *Destination {
+// DownloadOptionCalled returns the corresponding DownloadOption, or nil if it does not exist
-	for _, p := range c.Destinations {
+func (c *Config) DownloadOptionCalled(name string) *DownloadOption {
-		if p.Name == name {
+	for _, o := range c.DownloadOptions {
-			return &p
+		if o.Name == name {
+			return &o
 		}
 	}
 	return nil
@@ -185,17 +195,6 @@ func (c *Config) UpdateFromJSON(j []byte) error {
 		}
 	}
 
-	// check destinations
-	for _, dest := range newConfig.Destinations {
-		s, err := os.Stat(dest.Path)
-		if err != nil {
-			return fmt.Errorf("destination '%s' (%s) is bad: %s", dest.Name, dest.Path, err)
-		}
-		if !s.IsDir() {
-			return fmt.Errorf("destination '%s' (%s) is not a directory", dest.Name, dest.Path)
-		}
-	}
-
 	*c = newConfig
 	return nil
 }
@@ -293,6 +292,20 @@ func (cs *ConfigService) LoadConfig() error {
 		log.Print("migrated config from version 2 => 3")
 	}
 
+	if c.ConfigVersion == 3 {
+		c.ConfigVersion = 4
+		for i := range c.Destinations {
+			newDownloadOption := DownloadOption{
+				Name: c.Destinations[i].Name,
+				Args: []string{"-o", fmt.Sprintf("%s/%%(title)s [%%(id)s].%%(ext)s", c.Destinations[i].Path)},
+			}
+			c.DownloadOptions = append(c.DownloadOptions, newDownloadOption)
+			c.Destinations = nil
+		}
+		configMigrated = true
+		log.Print("migrated config from version 3 => 4")
+	}
+
 	if configMigrated {
 		log.Print("Writing new config after version migration")
 		cs.WriteConfig()

config/config_test.go

@@ -3,10 +3,12 @@ package config
 import (
 	"os"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
-func TestMigrationV1toV3(t *testing.T) {
+func TestMigrationV1toV4(t *testing.T) {
-	v2Config := `config_version: 1
+	v1Config := `config_version: 1
 server:
   port: 6123
   address: http://localhost:6123
@@ -31,12 +33,12 @@ profiles:
   - --audio-format
   - mp3
 `
-	cs := configServiceFromString(v2Config)
+	cs := configServiceFromString(v1Config)
 	err := cs.LoadConfig()
 	if err != nil {
 		t.Errorf("got error when loading config: %s", err)
 	}
-	if cs.Config.ConfigVersion != 3 {
+	if cs.Config.ConfigVersion != 4 {
 		t.Errorf("did not migrate version (it is '%d')", cs.Config.ConfigVersion)
 	}
 	if cs.Config.Server.MaximumActiveDownloads != 2 {
@@ -48,6 +50,58 @@ profiles:
 	os.Remove(cs.ConfigPath)
 }
 
+func TestMigrateV3toV4(t *testing.T) {
+	v3Config := `config_version: 3
+server:
+  port: 6123
+  address: http://localhost:6123
+  download_path: /tmp/Downloads
+  maximum_active_downloads_per_domain: 2
+ui:
+  popup_width: 900
+  popup_height: 900
+destinations:
+  - name: cool destination
+    path: /tmp/coolness
+profiles:
+  - name: standard video
+    command: yt-dlp
+    args:
+      - --newline
+      - --write-info-json
+      - -f
+      - bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best
+  - name: standard mp3
+    command: yt-dlp
+    args:
+      - --newline
+      - --write-info-json
+      - --extract-audio
+      - --audio-format
+      - mp3`
+	cs := configServiceFromString(v3Config)
+	err := cs.LoadConfig()
+	if err != nil {
+		t.Errorf("got error when loading config: %s", err)
+	}
+	if cs.Config.ConfigVersion != 4 {
+		t.Errorf("did not migrate version (it is '%d')", cs.Config.ConfigVersion)
+	}
+	if cs.Config.Server.MaximumActiveDownloads != 2 {
+		t.Error("did not add MaximumActiveDownloads")
+	}
+	if len(cs.Config.Destinations) != 0 {
+		t.Error("incorrect number of destinations from migrated file")
+	}
+	if assert.Len(t, cs.Config.DownloadOptions, 1) {
+		if assert.Len(t, cs.Config.DownloadOptions[0].Args, 2) {
+			assert.Equal(t, "-o", cs.Config.DownloadOptions[0].Args[0])
+			assert.Equal(t, "/tmp/coolness/%(title)s [%(id)s].%(ext)s", cs.Config.DownloadOptions[0].Args[1])
+		}
+	}
+	os.Remove(cs.ConfigPath)
+}
+
 func configServiceFromString(configString string) *ConfigService {
 	tmpFile, _ := os.CreateTemp("", "gropple_test_*.yml")
 	tmpFile.Write([]byte(configString))

docker-compose.build.yml

@@ -1,6 +0,0 @@
-version: "3.9"
-    
-services:
-  gropple:
-    build: .
-    image: tardisx/gropple:$VERSION

docker-compose.yml

@@ -2,11 +2,10 @@ version: "3.9"
 
 services:
   gropple:
-    build: .
     image: tardisx/gropple:latest
     volumes:
-      - ./gropple-config-dir:/config
+      - /tmp/gropple-config-dir/:/config
-      - ./downloads:/downloads/
+      - /tmp/downloads/:/downloads/
     restart: always
     ports:
       - "6123:6123"

download/download.go

@@ -27,7 +27,7 @@ type Download struct {
 	ExitCode        int                    `json:"exit_code"`
 	State           State                  `json:"state"`
 	DownloadProfile config.DownloadProfile `json:"download_profile"`
-	Destination     *config.Destination    `json:"destination"`
+	DownloadOption  *config.DownloadOption `json:"download_option"`
 	Finished        bool                   `json:"finished"`
 	FinishedTS      time.Time              `json:"finished_ts"`
 	Files           []string               `json:"files"`
@@ -82,7 +82,6 @@ func (m *Manager) ManageQueue() {
 		m.Lock.Lock()
 
 		m.startQueued(m.MaxPerDomain)
-		m.moveToDest()
 		m.cleanup()
 		m.Lock.Unlock()
 
@@ -102,32 +101,6 @@ func (m *Manager) DownloadsAsJSON() ([]byte, error) {
 	return b, err
 }
 
-func (m *Manager) moveToDest() {
-
-	// move any downloads that are complete and have a dest
-	for _, dl := range m.Downloads {
-
-		dl.Lock.Lock()
-		if dl.Destination != nil && dl.State == STATE_COMPLETE {
-			dl.State = STATE_MOVED
-			for _, fn := range dl.Files {
-				src := filepath.Join(dl.Config.Server.DownloadPath, fn)
-				dst := filepath.Join(dl.Destination.Path, fn)
-				err := os.Rename(src, dst)
-				if err != nil {
-					log.Printf("%s", err)
-					dl.Log = append(dl.Log, fmt.Sprintf("Could not move %s to %s - %s", fn, dl.Destination.Path, err))
-					break
-				} else {
-					dl.Log = append(dl.Log, fmt.Sprintf("Moved %s to %s", fn, dl.Destination.Path))
-
-				}
-			}
-		}
-		dl.Lock.Unlock()
-	}
-}
-
 // startQueued starts any downloads that have been queued, we would not exceed
 // maxRunning. If maxRunning is 0, there is no limit.
 func (m *Manager) startQueued(maxRunning int) {
@@ -202,17 +175,6 @@ func (m *Manager) Queue(dl *Download) {
 	dl.State = STATE_QUEUED
 }
 
-func (m *Manager) ChangeDestination(dl *Download, dest *config.Destination) {
-	dl.Lock.Lock()
-	// we can only change destination is certain cases...
-	if dl.State != STATE_FAILED && dl.State != STATE_MOVED {
-		dl.Destination = dest
-	}
-
-	dl.Lock.Unlock()
-
-}
-
 func NewDownload(url string, conf *config.Config) *Download {
 	atomic.AddInt32(&downloadId, 1)
 	dl := Download{
@@ -274,16 +236,52 @@ func (dl *Download) domain() string {
 // It blocks until the download is complete.
 func (dl *Download) Begin() {
 	dl.Lock.Lock()
+	u, err := url.Parse(dl.Url)
+	if err != nil {
+		log.Printf("Bad url '%s': %s", dl.Url, err.Error())
+	}
+
+	// grab the host and path for substitutions
+	host := u.Host
+	path := u.Path
+
+	// strip the leading /
+	if strings.Index(path, "/") == 0 {
+		path = path[1:]
+	}
+
+	// escape them in a way that should mean we can use them as a filepath
+	host = strings.ReplaceAll(host, string(filepath.Separator), "_")
+	host = strings.ReplaceAll(host, string(filepath.ListSeparator), "_")
+	path = strings.ReplaceAll(path, string(filepath.Separator), "_")
+	path = strings.ReplaceAll(path, string(filepath.ListSeparator), "_")
 
 	dl.State = STATE_DOWNLOADING
 	cmdSlice := []string{}
-	cmdSlice = append(cmdSlice, dl.DownloadProfile.Args...)
+
+	for i := range dl.DownloadProfile.Args {
+		arg := dl.DownloadProfile.Args[i]
+		arg = strings.ReplaceAll(arg, "%GROPPLE_HOST%", host)
+		arg = strings.ReplaceAll(arg, "%GROPPLE_PATH%", path)
+		cmdSlice = append(cmdSlice, arg)
+	}
+
+	// add the option, if any
+	if dl.DownloadOption != nil {
+		for i := range dl.DownloadOption.Args {
+			arg := dl.DownloadOption.Args[i]
+			arg = strings.ReplaceAll(arg, "%GROPPLE_HOST%", host)
+			arg = strings.ReplaceAll(arg, "%GROPPLE_PATH%", path)
+			cmdSlice = append(cmdSlice, arg)
+		}
+	}
 
 	// only add the url if it's not empty or an example URL. This helps us with testing
 	if !(dl.Url == "" || strings.Contains(dl.domain(), "example.org")) {
 		cmdSlice = append(cmdSlice, dl.Url)
 	}
 
+	dl.Log = append(dl.Log, fmt.Sprintf("executing: %s with args: %s", dl.DownloadProfile.Command, strings.Join(cmdSlice, " ")))
 	cmd := exec.Command(dl.DownloadProfile.Command, cmdSlice...)
 	cmd.Dir = dl.Config.Server.DownloadPath
 

download/download_test.go

@@ -21,11 +21,11 @@ func TestUpdateMetadata(t *testing.T) {
 	// eta's might be xx:xx:xx or xx:xx
 	newD.updateMetadata("[download]   0.0% of 504.09MiB at 135.71KiB/s ETA 01:03:36")
 	if newD.Eta != "01:03:36" {
-		t.Fatalf("bad long eta in dl\n%v", newD)
+		t.Fatalf("bad long eta in dl\n%#v", newD)
 	}
 	newD.updateMetadata("[download]   0.0% of 504.09MiB at 397.98KiB/s ETA 21:38")
 	if newD.Eta != "21:38" {
-		t.Fatalf("bad short eta in dl\n%v", newD)
+		t.Fatalf("bad short eta in dl\n%#v", newD)
 	}
 
 	// added a new file, now we are tracking two
@@ -237,15 +237,15 @@ func TestUpdateMetadataPlaylist(t *testing.T) {
 
 	output := `
 start of log...
-[download] Downloading playlist: niceuser
+[download] Downloading playlist: nice_user
-[RedGifsUser] niceuser: Downloading JSON metadata page 1
+[RedGifsUser] nice_user: Downloading JSON metadata page 1
-[RedGifsUser] niceuser: Downloading JSON metadata page 2
+[RedGifsUser] nice_user: Downloading JSON metadata page 2
-[RedGifsUser] niceuser: Downloading JSON metadata page 3
+[RedGifsUser] nice_user: Downloading JSON metadata page 3
-[RedGifsUser] niceuser: Downloading JSON metadata page 4
+[RedGifsUser] nice_user: Downloading JSON metadata page 4
-[RedGifsUser] niceuser: Downloading JSON metadata page 5
+[RedGifsUser] nice_user: Downloading JSON metadata page 5
-[RedGifsUser] niceuser: Downloading JSON metadata page 6
+[RedGifsUser] nice_user: Downloading JSON metadata page 6
-[info] Writing playlist metadata as JSON to: niceuser [niceuser].info.json
+[info] Writing playlist metadata as JSON to: nice_user [nice_user].info.json
-[RedGifsUser] playlist niceuser: Downloading 3 videos
+[RedGifsUser] playlist nice_user: Downloading 3 videos
 [download] Downloading video 1 of 3
 [info] wrongpreciouschrysomelid: Downloading 1 format(s): hd
 [info] Writing video metadata as JSON to: Splendid Wonderful Speaker Power Chocolate Drop [wrongpreciouschrysomelid].info.json
@@ -279,8 +279,8 @@ start of log...
 [download]  69.1% of 2.89MiB at 11.63MiB/s ETA 00:00
 [download] 100% of 2.89MiB at 14.25MiB/s ETA 00:00
 [download] 100% of 2.89MiB in 00:00
-[info] Writing updated playlist metadata as JSON to: niceuser [niceuser].info.json
+[info] Writing updated playlist metadata as JSON to: nice_user [nice_user].info.json
-[download] Finished downloading playlist: niceuser
+[download] Finished downloading playlist: nice_user
 `
 	newD := Download{}
 

download/download_testdata.go

@@ -11,13 +11,26 @@ func (m *Manager) AddStressTestData(c *config.ConfigService) {
 		"https://www.youtube.com/watch?v=ZUzhZpQAU40",
 		"https://www.youtube.com/watch?v=kVxM3eRWGak",
 		"https://www.youtube.com/watch?v=pl-y9869y0w",
-		"https://vimeo.com/783453809",
 		"https://www.youtube.com/watch?v=Uw4NEPE4l3A",
 		"https://www.youtube.com/watch?v=2RF0lcTuuYE",
 		"https://www.youtube.com/watch?v=lymwNQY0dus",
 		"https://www.youtube.com/watch?v=NTc-I4Z_duc",
 		"https://www.youtube.com/watch?v=wNSm1TJ84Ac",
+		"https://www.youtube.com/watch?v=tyixMpuGEL8",
+		"https://www.youtube.com/watch?v=VnxbkH_3E_4",
+		"https://www.youtube.com/watch?v=VStscvYLYLs",
+		"https://www.youtube.com/watch?v=vYMiSz-WlEY",
+
 		"https://vimeo.com/786570322",
+		"https://vimeo.com/783453809",
+
+		"https://www.gamespot.com/videos/survival-fps-how-metro-2033-solidified-a-subgenre/2300-6408243/",
+		"https://www.gamespot.com/videos/dirt-3-right-back-where-you-started-gameplay-movie/2300-6314712/",
+		"https://www.gamespot.com/videos/the-b-list-driver-san-francisco/2300-6405593/",
+
+		"https://www.imdb.com/video/vi1914750745/?listId=ls053181649&ref_=hm_hp_i_hero-video-1_1",
+		"https://www.imdb.com/video/vi3879585561/?listId=ls053181649&ref_=vp_pl_ap_6",
+		"https://www.imdb.com/video/vi54445849/?listId=ls053181649&ref_=vp_nxt_btn",
 	}
 	for _, u := range urls {
 		d := NewDownload(u, c.Config)

go.mod

@@ -3,7 +3,14 @@ module github.com/tardisx/gropple
 go 1.20
 
 require (
-	github.com/gorilla/mux v1.8.0
+	github.com/gorilla/mux v1.8.1
-	golang.org/x/mod v0.9.0
+	github.com/stretchr/testify v1.8.4
+	golang.org/x/mod v0.14.0
 	gopkg.in/yaml.v2 v2.4.0
 )
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

go.sum

@@ -1,8 +1,16 @@
-github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs=
+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
-golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
+golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -1,53 +1,30 @@
 package main
 
 import (
-	"embed"
-	"encoding/json"
 	"flag"
 	"fmt"
-	"html/template"
-	"io"
 	"log"
 	"net/http"
-	"strings"
 	"time"
 
-	"strconv"
-
-	"github.com/gorilla/mux"
 	"github.com/tardisx/gropple/config"
 	"github.com/tardisx/gropple/download"
 	"github.com/tardisx/gropple/version"
+	"github.com/tardisx/gropple/web"
 )
 
-var dm *download.Manager
-var configService *config.ConfigService
-
-var versionInfo = version.Manager{
-	VersionInfo: version.Info{CurrentVersion: "v0.6.0-alpha.4"},
-}
-
-//go:embed web
-var webFS embed.FS
-
-type successResponse struct {
-	Success bool   `json:"success"`
-	Message string `json:"message"`
-}
-
-type errorResponse struct {
-	Success bool   `json:"success"`
-	Error   string `json:"error"`
-}
-
 func main() {
+	versionInfo := &version.Manager{
+		VersionInfo: version.Info{CurrentVersion: "v1.0.0-alpha.2"},
+	}
 	log.Printf("Starting gropple %s - https://github.com/tardisx/gropple", versionInfo.GetInfo().CurrentVersion)
 
 	var configPath string
 	flag.StringVar(&configPath, "config-path", "", "path to config file")
+
 	flag.Parse()
 
-	configService = &config.ConfigService{}
+	configService := &config.ConfigService{}
 	if configPath != "" {
 		configService.ConfigPath = configPath
 	} else {
@@ -72,23 +49,10 @@ func main() {
 	}
 
 	// create the download manager
-	dm = &download.Manager{MaxPerDomain: configService.Config.Server.MaximumActiveDownloads}
+	downloadManager := &download.Manager{MaxPerDomain: configService.Config.Server.MaximumActiveDownloads}
 
-	r := mux.NewRouter()
+	// create the web handlers
-	r.HandleFunc("/", homeHandler)
+	r := web.CreateRoutes(configService, downloadManager, versionInfo)
-	r.HandleFunc("/static/{filename}", staticHandler)
-	r.HandleFunc("/config", configHandler)
-	r.HandleFunc("/fetch", fetchHandler)
-	r.HandleFunc("/fetch/{id}", fetchHandler)
-
-	// info for the list
-	r.HandleFunc("/rest/fetch", fetchInfoRESTHandler)
-	// info for one, including update
-	r.HandleFunc("/rest/fetch/{id}", fetchInfoOneRESTHandler)
-	r.HandleFunc("/rest/version", versionRESTHandler)
-	r.HandleFunc("/rest/config", configRESTHandler)
-
-	http.Handle("/", r)
 
 	srv := &http.Server{
 		Handler: r,
@@ -111,321 +75,12 @@ func main() {
 
 	// start downloading queued downloads when slots available, and clean up
 	// old entries
-	go dm.ManageQueue()
+	go downloadManager.ManageQueue()
-	dm.AddStressTestData(configService)
+
+	// add testdata if compiled with the '-tags testdata' flag
+	downloadManager.AddStressTestData(configService)
 
 	log.Printf("Visit %s for details on installing the bookmarklet and to check status", configService.Config.Server.Address)
 	log.Fatal(srv.ListenAndServe())
 
 }
-
-// versionRESTHandler returns the version information, if we have up-to-date info from github
-func versionRESTHandler(w http.ResponseWriter, r *http.Request) {
-	if versionInfo.GetInfo().GithubVersionFetched {
-		b, _ := json.Marshal(versionInfo.GetInfo())
-		_, err := w.Write(b)
-		if err != nil {
-			log.Printf("could not write to client: %s", err)
-		}
-	} else {
-		w.WriteHeader(400)
-	}
-}
-
-// homeHandler returns the main index page
-func homeHandler(w http.ResponseWriter, r *http.Request) {
-	w.WriteHeader(http.StatusOK)
-
-	bookmarkletURL := fmt.Sprintf("javascript:(function(f,s,n,o){window.open(f+encodeURIComponent(s),n,o)}('%s/fetch?url=',window.location,'yourform','width=%d,height=%d'));", configService.Config.Server.Address, configService.Config.UI.PopupWidth, configService.Config.UI.PopupHeight)
-
-	t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/menu.tmpl", "web/index.html")
-	if err != nil {
-		panic(err)
-	}
-
-	type Info struct {
-		Manager        *download.Manager
-		BookmarkletURL template.URL
-		Config         *config.Config
-		Version        version.Info
-	}
-
-	info := Info{
-		Manager:        dm,
-		BookmarkletURL: template.URL(bookmarkletURL),
-		Config:         configService.Config,
-		Version:        versionInfo.GetInfo(),
-	}
-
-	dm.Lock.Lock()
-	defer dm.Lock.Unlock()
-	err = t.ExecuteTemplate(w, "layout", info)
-	if err != nil {
-		panic(err)
-	}
-}
-
-// staticHandler handles requests for static files
-func staticHandler(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-
-	filename := vars["filename"]
-	if strings.Index(filename, ".js") == len(filename)-3 {
-		f, err := webFS.Open("web/" + filename)
-		if err != nil {
-			log.Printf("error accessing %s - %v", filename, err)
-			w.WriteHeader(http.StatusNotFound)
-			return
-		}
-		w.WriteHeader(http.StatusOK)
-		_, err = io.Copy(w, f)
-		if err != nil {
-			log.Printf("could not write to client: %s", err)
-		}
-		return
-	}
-	w.WriteHeader(http.StatusNotFound)
-}
-
-// configHandler returns the configuration page
-func configHandler(w http.ResponseWriter, r *http.Request) {
-	w.WriteHeader(http.StatusOK)
-
-	t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/menu.tmpl", "web/config.html")
-	if err != nil {
-		panic(err)
-	}
-
-	err = t.ExecuteTemplate(w, "layout", nil)
-	if err != nil {
-		panic(err)
-	}
-}
-
-// configRESTHandler handles both reading and writing of the configuration
-func configRESTHandler(w http.ResponseWriter, r *http.Request) {
-
-	if r.Method == "POST" {
-		log.Printf("Updating config")
-		b, err := io.ReadAll(r.Body)
-		if err != nil {
-			panic(err)
-		}
-		err = configService.Config.UpdateFromJSON(b)
-
-		if err != nil {
-			errorRes := errorResponse{Success: false, Error: err.Error()}
-			errorResB, _ := json.Marshal(errorRes)
-			w.WriteHeader(400)
-			_, err = w.Write(errorResB)
-			if err != nil {
-				log.Printf("could not write to client: %s", err)
-			}
-			return
-		}
-		configService.WriteConfig()
-	}
-	b, _ := json.Marshal(configService.Config)
-	_, err := w.Write(b)
-	if err != nil {
-		log.Printf("could not write config to client: %s", err)
-	}
-}
-
-func fetchInfoOneRESTHandler(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	idString := vars["id"]
-	if idString != "" {
-		id, err := strconv.Atoi(idString)
-		if err != nil {
-			http.NotFound(w, r)
-			return
-		}
-
-		thisDownload, err := dm.GetDlById(id)
-		if err != nil {
-			http.NotFound(w, r)
-			return
-		}
-		if thisDownload == nil {
-			panic("should not happen")
-		}
-
-		if r.Method == "POST" {
-
-			type updateRequest struct {
-				Action      string `json:"action"`
-				Profile     string `json:"profile"`
-				Destination string `json:"destination"`
-			}
-
-			thisReq := updateRequest{}
-
-			b, err := io.ReadAll(r.Body)
-			if err != nil {
-				panic(err)
-			}
-
-			err = json.Unmarshal(b, &thisReq)
-			if err != nil {
-				errorRes := errorResponse{Success: false, Error: err.Error()}
-				errorResB, _ := json.Marshal(errorRes)
-				w.WriteHeader(400)
-				_, err = w.Write(errorResB)
-				if err != nil {
-					log.Printf("could not write to client: %s", err)
-				}
-				return
-			}
-
-			if thisReq.Action == "start" {
-				// find the profile they asked for
-				profile := configService.Config.ProfileCalled(thisReq.Profile)
-				if profile == nil {
-					panic("bad profile name?")
-				}
-				// set the profile
-				thisDownload.Lock.Lock()
-				thisDownload.DownloadProfile = *profile
-				thisDownload.Lock.Unlock()
-
-				dm.Queue(thisDownload)
-
-				succRes := successResponse{Success: true, Message: "download started"}
-				succResB, _ := json.Marshal(succRes)
-				_, err = w.Write(succResB)
-				if err != nil {
-					log.Printf("could not write to client: %s", err)
-				}
-				return
-			}
-
-			if thisReq.Action == "change_destination" {
-
-				// nil means (probably) that they chose "don't move" - which is fine,
-				// and maps to nil on the Download (the default state).
-				destination := configService.Config.DestinationCalled(thisReq.Destination)
-				dm.ChangeDestination(thisDownload, destination)
-
-				//				log.Printf("%#v", thisDownload)
-
-				succRes := successResponse{Success: true, Message: "destination changed"}
-				succResB, _ := json.Marshal(succRes)
-				_, err = w.Write(succResB)
-				if err != nil {
-					log.Printf("could not write to client: %s", err)
-				}
-				return
-			}
-
-			if thisReq.Action == "stop" {
-
-				thisDownload.Stop()
-				succRes := successResponse{Success: true, Message: "download stopped"}
-				succResB, _ := json.Marshal(succRes)
-				_, err = w.Write(succResB)
-				if err != nil {
-					log.Printf("could not write to client: %s", err)
-				}
-				return
-			}
-		}
-
-		// just a get, return the object
-		thisDownload.Lock.Lock()
-		defer thisDownload.Lock.Unlock()
-
-		b, _ := json.Marshal(thisDownload)
-
-		_, err = w.Write(b)
-		if err != nil {
-			log.Printf("could not write to client: %s", err)
-		}
-		return
-	} else {
-		http.NotFound(w, r)
-	}
-}
-
-func fetchInfoRESTHandler(w http.ResponseWriter, r *http.Request) {
-
-	b, err := dm.DownloadsAsJSON()
-	if err != nil {
-		panic(err)
-	}
-	_, err = w.Write(b)
-	if err != nil {
-		log.Printf("could not write to client: %s", err)
-	}
-}
-
-func fetchHandler(w http.ResponseWriter, r *http.Request) {
-
-	// if they refreshed the popup, just load the existing object, don't
-	// create a new one
-	vars := mux.Vars(r)
-	idString := vars["id"]
-
-	idInt, err := strconv.ParseInt(idString, 10, 32)
-
-	// existing, load it up
-	if err == nil && idInt > 0 {
-
-		dl, err := dm.GetDlById(int(idInt))
-		if err != nil {
-			log.Printf("not found")
-			w.WriteHeader(404)
-			return
-		}
-
-		t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/popup.html")
-		if err != nil {
-			panic(err)
-		}
-
-		templateData := map[string]interface{}{"dl": dl, "config": configService.Config, "canStop": download.CanStopDownload}
-
-		err = t.ExecuteTemplate(w, "layout", templateData)
-		if err != nil {
-			panic(err)
-		}
-		return
-	}
-
-	query := r.URL.Query()
-	url, present := query["url"]
-
-	if !present {
-		w.WriteHeader(400)
-		fmt.Fprint(w, "No url supplied")
-		return
-	} else {
-
-		log.Printf("popup for %s", url)
-		// check the URL for a sudden but inevitable betrayal
-		if strings.Contains(url[0], configService.Config.Server.Address) {
-			w.WriteHeader(400)
-			fmt.Fprint(w, "you mustn't gropple your gropple :-)")
-			return
-		}
-
-		// create the new download
-		newDL := download.NewDownload(url[0], configService.Config)
-		dm.AddDownload(newDL)
-
-		t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/popup.html")
-		if err != nil {
-			panic(err)
-		}
-
-		newDL.Lock.Lock()
-		defer newDL.Lock.Unlock()
-
-		templateData := map[string]interface{}{"Version": versionInfo.GetInfo(), "dl": newDL, "config": configService.Config, "canStop": download.CanStopDownload}
-
-		err = t.ExecuteTemplate(w, "layout", templateData)
-		if err != nil {
-			panic(err)
-		}
-	}
-}

web/data/templates/config.tmpl

@@ -11,7 +11,7 @@
 
     <div class="pure-g">
         <div class="pure-u-1">
-            <button class="pure-button pure-button-primary" @click="save_config();" href="#">Save Config</button>
+            <button class="button-small pure-button button-small pure-button-primary" @click="save_config();" href="#">Save Config</button>
         </div>
     </div>
 
@@ -78,7 +78,7 @@
                             </label>
 
                             <input type="text" x-bind:id="'config-profiles-'+i+'-name'" class="input-long" placeholder="name" x-model="profile.name" />
-                            <button class="pure-button button-del" href="#" @click.prevent="config.profiles.splice(i, 1);;">delete profile</button>
+                            <button class="button-small pure-button button-del" href="#" @click.prevent="config.profiles.splice(i, 1);;">delete profile</button>
 
                             <span class="pure-form-message">The name of this profile. For your information only.</span>
 
@@ -92,11 +92,11 @@
                             <template x-for="(arg, j) in profile.args">
                                 <div>
                                     <input type="text" x-bind:id="'config-profiles-'+i+'-arg-'+j" placeholder="arg" x-model="profile.args[j]" />
-                                    <button class="pure-button button-del" href="#" @click.prevent="profile.args.splice(j, 1);;">delete arg</button>
+                                    <button class="button-small pure-button button-del" href="#" @click.prevent="profile.args.splice(j, 1);;">delete arg</button>
                                 </div>
                             </template>
 
-                            <button class="pure-button button-add" href="#" @click.prevent="profile.args.push('');">add arg</button>
+                            <button class="button-small pure-button button-add" href="#" @click.prevent="profile.args.push('');">add arg</button>
                             <span class="pure-form-message">Arguments for the command. Note that the shell is not used, so there is no need to quote or escape arguments, including those with spaces.</span>
 
                             <hr>
@@ -104,7 +104,7 @@
                         </div>
                     </template>
 
-                    <button class="pure-button button-add" href="#" @click.prevent="config.profiles.push({name: 'new profile', command: 'youtube-dl', args: []});">add profile</button>
+                    <button class="button-small pure-button button-add" href="#" @click.prevent="config.profiles.push({name: 'new profile', command: 'youtube-dl', args: []});">add profile</button>
 
                 </fieldset>
             </form>
@@ -113,39 +113,44 @@
         <div class="pure-u-lg-1-3 pure-u-1 l-box">
             <form class="pure-form gropple-config">
                 <fieldset>
-                    <legend>Destinations</legend>
+                    <legend>Download Options</legend>
-                    <p>You can specify custom destinations (directories) here. Downloads can be
+                    <p>You can specify custom download options here. These are (optionally) selectable in addition
-                        moved to one of these directories after completion from the index page, 
+                    to the profile when starting a download. They append extra arguments to the downloader command.
-                        if you do not want them to be left in the download path above.</p>
+                    The most common use is to specify a particular <tt>-o</tt> argument to <tt>yt-dlp</tt> to allow files to be downloaded
+                    to a custom path.</p>
                     </p>
-                    <template x-for="(dest, i) in config.destinations">
+                    <template x-for="(download_option, i) in config.download_options">
                     <div>
-                        <label x-bind:for="'config-destinations-'+i+'-name'">Name of destination <span x-text="i+1"></span>
+                        <label x-bind:for="'config-download-option-'+i+'-name'">Name of option <span x-text="i+1"></span>
                         </label>
 
-                        <input type="text" x-bind:id="'config-destinations-'+i+'-name'" class="input-long" placeholder="name" x-model="dest.name" />
+                        <input type="text" x-bind:id="'config-download-option-'+i+'-name'" class="input-long" placeholder="name" x-model="download_option.name" />
 
-                        <span class="pure-form-message">The name of this destination. For your information only.</span>
+                        <span class="pure-form-message">The name of this option. For your information only.</span>
 
-                        <label x-bind:for="'config-destinations-'+i+'-command'">Path</label>
+                            <label>Arguments</label>
-                        <input type="text" x-bind:id="'config-destinations-'+i+'-command'" class="input-long" placeholder="name" x-model="dest.path" />
-                        <span class="pure-form-message">Path to move completed downloads to.</span>
 
-                        <button class="pure-button button-del" href="#" @click.prevent="config.destinations.splice(i, 1);">delete destination</button>
+                            <template x-for="(arg, j) in download_option.args">
+                                <div>
+                                    <input type="text" x-bind:id="'config-download-option-'+i+'-arg-'+j" placeholder="arg" x-model="download_option.args[j]" />
+                                    <button class="button-small pure-button button-del" href="#" @click.prevent="download_option.args.splice(j, 1);;">delete arg</button>
+                                </div>
+                            </template>
+                        <button class="button-small pure-button button-del" href="#" @click.prevent="config.download_options.splice(i, 1);">delete option</button>
 
                         <hr>
 
                     </div>
                 </template>
 
-                <button class="pure-button button-add" href="#" @click.prevent="config.destinations.push({name: 'new destination', path: '/tmp'});">add destination</button>
+                <button class="button-small pure-button button-add" href="#" @click.prevent="config.download_options.push({name: 'new option', args: ['-o', 'someting']});">add option</button>
 
             </fieldset>
         </form>
     </div>
     <div class="pure-g">
         <div class="pure-u-1">
-            <button class="pure-button pure-button-primary" @click="save_config();" href="#">Save Config</button>
+            <button class="button-small pure-button button-small pure-button-primary" @click="save_config();" href="#">Save Config</button>
         </div>
     </div>
 
@@ -158,7 +163,7 @@
 <script>
     function config() {
         return {
-            config: { server : {}, ui : {}, profiles: [], destinations: []}, 
+            config: { server : {}, ui : {}, profiles: [], download_options: []},
             error_message: '',
             success_message: '',
 

web/data/templates/index.tmpl

@@ -24,27 +24,43 @@
     <table class="pure-table">
         <thead>
             <tr>
-		    <th>id</th><th>filename</th><th>url</th><th>show</th><th>state</th><th>percent</th><th>eta</th><th>finished</th>
+                <th>id</th>
+                <th>filename</th>
+                <th>url</th>
+                <th>state</th>
+                <th>percent</th>
+                <th>eta</th>
+                <th>finished</th>
             </tr>
         </thead>
         <tbody>
             <template x-for="item in items">
                 <tr>
-                    <td x-text="item.id"></td>
                     <td>
-                        <span x-show="item.files && item.files.length > 0">
+                        <a class="int-link" @click="show_popup(item)" href="#">
-                            <ul>
+                          <span x-text="item.id">
-                                <template x-for="file in item.files">
+                        </a>
-                                    <li x-text="file"></li>
+                    </td>
-                                </template>
+                    <td>
-                            </ul>
+                        <span x-show="item.files && item.files.length == 1">
+                            <span class="filelist" x-text="item.files[0]"></span>
                         </span>
-                        <span x-show="! item.files || item.files.length == 0"
+                        <span x-data="{open: false}" x-show="item.files && item.files.length > 1">
-                              x-text="item.url">
+                            <span class="filelist" x-text="item.files.length + ' files...'"></span>
+                            <button class="pure-button button-small" @click="open = ! open" x-text="open ? 'hide' : 'show'"></button>
+                            <div x-show="open" x-transition>
+                                <ul class="filelist">
+                                    <template x-for="file in item.files">
+                                        <li x-text="file"></li>
+                                    </template>
+                                </ul>
+                            </div>
+                        </span>
+                        <span class="filelist" x-show="! item.files || item.files.length == 0"
+                              x-text="'fetching ' + item.url + '...'">
                         </span>
                     </td>
-                    <td><a class="int-link" x-bind:href="item.url">&#x2197;</a></td>
+                    <td><a class="int-link" x-bind:href="item.url">&#x1F517;</a></td>
-                    <td><a class="int-link" @click="show_popup(item)" href="#">&#x1F4C4;</a></td>
                     <td :class="'state-'+item.state" x-text="item.state"></td>
                     <td x-text="item.percent"></td>
                     <td x-text="item.eta"></td>

web/data/templates/layout.tmpl

@@ -5,9 +5,24 @@
     <title>gropple</title>
     <script src="/static/alpine.min.js" defer></script>
     <meta name="viewport" content="width=device-width, initial-scale=1">
-    <link rel="stylesheet" href="https://unpkg.com/purecss@2.0.6/build/pure-min.css" integrity="sha384-Uu6IeWbM+gzNVXJcM9XV3SohHtmWE+3VGi496jvgX1jyvDTXfdK+rfZc8C1Aehk5" crossorigin="anonymous">
+    <link rel="preconnect" href="https://rsms.me/">
-    <link rel="stylesheet" href="https://unpkg.com/purecss@2.0.6/build/grids-responsive-min.css">
+    <link rel="stylesheet" href="https://rsms.me/inter/inter.css">
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/purecss@3.0.0/build/pure-min.css" integrity="sha384-X38yfunGUhNzHpBaEBsWLO+A0HDYOQi8ufWDkZ0k9e0eXz/tH3II7uKZ9msv++Ls" crossorigin="anonymous">
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/purecss@3.0.0/build/grids-responsive-min.css">
     <style>
+
+:root {
+  font-family: Inter, sans-serif;
+  font-feature-settings: 'liga' 1, 'calt' 1; /* fix for Chrome */
+}
+@supports (font-variation-settings: normal) {
+  :root { font-family: InterVariable, sans-serif; }
+}
+
+        .button-small {
+            font-size: 85%;
+        }
+
         .pure-g > div {
           box-sizing: border-box;
         }
@@ -19,6 +34,9 @@
           height: 100px;
           overflow:auto;
         }
+        .filelist {
+          font-size: 60%;
+        }
         footer {
           padding-top: 50px;
           font-size: 30%;
@@ -45,9 +63,6 @@
         .gropple-config input.input-long {
           width: 27em;
         }
-        .gropple-config button {
-          border-radius: 12px;
-        }
         .gropple-config button.button-del {
           background: rgb(202, 60, 60);
         }
@@ -60,7 +75,7 @@
         }
         .error {
           color: red;
-          font-size: 150%;
+          font-size: 120%;
         }
         .success {
           color: green;

web/data/templates/popup.tmpl

@@ -2,28 +2,17 @@
     <div id="layout" class="pure-g pure-u-1" x-data="popup()" x-init="fetch_data()">
         <h2>Download started</h2>
         <p>Fetching <tt>{{ .dl.Url }}</tt></p>
+        <form class="pure-form">
         <table class="pure-table" >
             <tr>
                 <th>profile</th>
-                <td>
+                <td>{{ .dl.DownloadProfile.Name }}</td>
-                    <select x-bind:disabled="profile_chosen" x-on:change="update_profile()" class="pure-input-1-2" x-model="profile_chosen">
-                        <option value="">choose a profile to start</option>
-                    {{ range $i := .config.DownloadProfiles }}
-                        <option>{{ $i.Name }}</option>
-                    {{ end }}
-                    </select>
-                </td>
             </tr>
             <tr><th>current filename</th><td x-text="filename"></td></tr>
             <tr>
-                <th>destination</th>
+                <th>option</th>
                 <td>
-                    <select  x-on:change="update_destination()" class="pure-input-1-2" x-model="destination_chosen">
+                  {{ if .dl.DownloadOption }} {{ .dl.DownloadOption.Name }} {{ else }} n/a {{ end }}
-                        <option value="-">leave in {{ .config.Server.DownloadPath }}</option>
-                    {{ range $i := .config.Destinations }}
-                        <option>{{ $i.Name }}</option>
-                    {{ end }}
-                    </select>
                 </td>
             </tr>
             <tr><th>state</th><td x-text="state"></td></tr>
@@ -34,8 +23,9 @@
         </table>
         <p>You can close this window and your download will continue. Check the <a href="/" target="_gropple_status">Status page</a> to see all downloads in progress.</p>
         {{ if .canStop }}
-        <button x-show="state=='Downloading'" class="pure-button" @click="stop()">stop</button>
+        <button x-show="state=='Downloading'" class="button-small pure-button" @click="stop()">stop</button>
         {{ end }}
+        </form>
         <div>
             <h4>Logs</h4>
             <pre x-text="log" style="height: auto;">
@@ -50,36 +40,6 @@
         return {
             eta: '', percent: 0.0, state: '??', filename: '', finished: false, log :'',
             playlist_current: 0, playlist_total: 0,
-            profile_chosen: null,
-            destination_chosen: null,
-            watch_profile() {
-                this.$watch('profile_chosen', value => this.profile_chosen(value))
-            },
-            update_profile(name) {
-                console.log('you chose name', this.profile_chosen);
-                let op = {
-                   method: 'POST',
-                   body: JSON.stringify({action: 'start', profile: this.profile_chosen}),
-                   headers: { 'Content-Type': 'application/json' }
-                };
-                fetch('/rest/fetch/{{ .dl.Id }}', op)
-                .then(response => response.json())
-                .then(info => {
-                    console.log(info)
-                })
-            },
-            update_destination(name) {
-                let op = {
-                   method: 'POST',
-                   body: JSON.stringify({action: 'change_destination', destination: this.destination_chosen}),
-                   headers: { 'Content-Type': 'application/json' }
-                };
-                fetch('/rest/fetch/{{ .dl.Id }}', op)
-                .then(response => response.json())
-                .then(info => {
-                    console.log(info)
-                })
-            },
             stop() {
                 let op = {
                    method: 'POST',
@@ -101,13 +61,6 @@
                     this.state = info.state;
                     this.playlist_current = info.playlist_current;
                     this.playlist_total = info.playlist_total;
-                    this.destination_chosen = null;
-                    if (info.destination) {
-                        this.destination_chosen = info.destination.name;
-                    }
-                    if (this.state != 'Choose Profile') {
-                        this.profile_chosen = true;
-                    }
                     this.finished = info.finished;
                     if (info.files && info.files.length > 0) {
                         this.filename = info.files[info.files.length - 1];

web/data/templates/popup_create.tmpl

@@ -0,0 +1,74 @@
+{{ define "content" }}
+    <div id="layout" class="pure-g pure-u-1" x-data="popup_create()" >
+
+        <h2>Download create</h2>
+        <p>URL: <tt>{{ .url }}</tt></p>
+
+        <p class="error"  x-show="error_message"  x-transition.duration.500ms x-text="error_message"></p>
+
+        <table class="pure-table" >
+            <tr>
+                <th>profile</th>
+                <td>
+                    <select class="pure-input-1-2" x-model="profile_chosen">
+                        <option value="">choose a profile</option>
+                    {{ range $i := .config.DownloadProfiles }}
+                        <option>{{ $i.Name }}</option>
+                    {{ end }}
+                    </select>
+                </td>
+            </tr>
+            <tr>
+                <th>download option</th>
+                <td>
+                    <select  class="pure-input-1-2" x-model="download_option_chosen">
+                        <option value="">no option</option>
+                    {{ range $i := .config.DownloadOptions }}
+                        <option>{{ $i.Name }}</option>
+                    {{ end }}
+                    </select>
+                </td>
+            </tr>
+            <tr>
+                <th>&nbsp;</th>
+                <td>
+                    <button class="button-small pure-button" @click="start()">start download</button>
+                </td>
+            </tr>
+
+        </table>
+    </div>
+{{ end }}
+{{ define "js" }}
+<script>
+    function popup_create() {
+        return {
+            profile_chosen: "",
+            download_option_chosen: "",
+            error_message: "",
+            start() {
+                let op = {
+                   method: 'POST',
+                   body: JSON.stringify({action: 'start', url: '{{ .url }}', profile: this.profile_chosen, download_option: this.download_option_chosen}),
+                   headers: { 'Content-Type': 'application/json' }
+                };
+                fetch('/fetch', op)
+                .then(response => response.json())
+                .then(response => {
+                    console.log(response)
+                     if (response.error) {
+                        this.error_message = response.error;
+                        this.success_message = '';
+                        document.body.scrollTop = document.documentElement.scrollTop = 0;
+                    } else {
+                        this.error_message = '';
+                        console.log(response.location)
+                        window.location = response.location
+                    }
+                })
+            }
+        }
+    }
+</script>
+{{ end }}
+

web/web.go

@@ -0,0 +1,401 @@
+package web
+
+import (
+	"embed"
+	"encoding/json"
+	"fmt"
+	"html/template"
+	"io"
+	"log"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/gorilla/mux"
+	"github.com/tardisx/gropple/config"
+	"github.com/tardisx/gropple/download"
+	"github.com/tardisx/gropple/version"
+)
+
+type successResponse struct {
+	Success bool   `json:"success"`
+	Message string `json:"message"`
+}
+
+type queuedResponse struct {
+	Success  bool   `json:"success"`
+	Location string `json:"location"`
+}
+
+type errorResponse struct {
+	Success bool   `json:"success"`
+	Error   string `json:"error"`
+}
+
+//go:embed data/**
+var webFS embed.FS
+
+func CreateRoutes(cs *config.ConfigService, dm *download.Manager, vm *version.Manager) *mux.Router {
+	r := mux.NewRouter()
+
+	// main index page
+	r.HandleFunc("/", homeHandler(cs, vm, dm))
+	// update info on the status page
+	r.HandleFunc("/rest/fetch", fetchInfoRESTHandler(dm))
+
+	// return static files
+	r.HandleFunc("/static/{filename}", staticHandler())
+
+	// return the config page
+	r.HandleFunc("/config", configHandler())
+	// handle config fetches/updates
+	r.HandleFunc("/rest/config", configRESTHandler(cs))
+
+	// create or present a download in the popup
+	r.HandleFunc("/fetch", fetchHandler(cs, vm, dm))
+	r.HandleFunc("/fetch/{id}", fetchHandler(cs, vm, dm))
+
+	// get/update info on a download
+	r.HandleFunc("/rest/fetch/{id}", fetchInfoOneRESTHandler(cs, dm))
+
+	// version information
+	r.HandleFunc("/rest/version", versionRESTHandler(vm))
+
+	http.Handle("/", r)
+	return r
+}
+
+// versionRESTHandler returns the version information, if we have up-to-date info from github
+func versionRESTHandler(vm *version.Manager) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if vm.GetInfo().GithubVersionFetched {
+			b, _ := json.Marshal(vm.GetInfo())
+			_, err := w.Write(b)
+			if err != nil {
+				log.Printf("could not write to client: %s", err)
+			}
+		} else {
+			w.WriteHeader(400)
+		}
+	}
+}
+
+// homeHandler returns the main index page
+func homeHandler(cs *config.ConfigService, vm *version.Manager, dm *download.Manager) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+
+		bookmarkletURL := fmt.Sprintf("javascript:(function(f,s,n,o){window.open(f+encodeURIComponent(s),n,o)}('%s/fetch?url=',window.location,'yourform','width=%d,height=%d'));", cs.Config.Server.Address, cs.Config.UI.PopupWidth, cs.Config.UI.PopupHeight)
+
+		t, err := template.ParseFS(webFS, "data/templates/layout.tmpl", "data/templates/menu.tmpl", "data/templates/index.tmpl")
+		if err != nil {
+			panic(err)
+		}
+
+		type Info struct {
+			Manager        *download.Manager
+			BookmarkletURL template.URL
+			Config         *config.Config
+			Version        version.Info
+		}
+
+		info := Info{
+			Manager:        dm,
+			BookmarkletURL: template.URL(bookmarkletURL),
+			Config:         cs.Config,
+			Version:        vm.GetInfo(),
+		}
+
+		dm.Lock.Lock()
+		defer dm.Lock.Unlock()
+		err = t.ExecuteTemplate(w, "layout", info)
+		if err != nil {
+			panic(err)
+		}
+	}
+}
+
+// staticHandler handles requests for static files
+func staticHandler() func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		vars := mux.Vars(r)
+
+		filename := vars["filename"]
+		if strings.Index(filename, ".js") == len(filename)-3 {
+			f, err := webFS.Open("data/js/" + filename)
+			if err != nil {
+				log.Printf("error accessing %s - %v", filename, err)
+				w.WriteHeader(http.StatusNotFound)
+				return
+			}
+			w.WriteHeader(http.StatusOK)
+			_, err = io.Copy(w, f)
+			if err != nil {
+				log.Printf("could not write to client: %s", err)
+			}
+			return
+		}
+		w.WriteHeader(http.StatusNotFound)
+	}
+}
+
+// configHandler returns the configuration page
+func configHandler() func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+
+		t, err := template.ParseFS(webFS, "data/templates/layout.tmpl", "data/templates/menu.tmpl", "data/templates/config.tmpl")
+		if err != nil {
+			panic(err)
+		}
+
+		err = t.ExecuteTemplate(w, "layout", nil)
+		if err != nil {
+			panic(err)
+		}
+	}
+}
+
+// configRESTHandler handles both reading and writing of the configuration
+func configRESTHandler(cs *config.ConfigService) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+
+		if r.Method == "POST" {
+			log.Printf("Updating config")
+			b, err := io.ReadAll(r.Body)
+			if err != nil {
+				panic(err)
+			}
+			err = cs.Config.UpdateFromJSON(b)
+
+			if err != nil {
+				errorRes := errorResponse{Success: false, Error: err.Error()}
+				errorResB, _ := json.Marshal(errorRes)
+				w.WriteHeader(400)
+				_, err = w.Write(errorResB)
+				if err != nil {
+					log.Printf("could not write to client: %s", err)
+				}
+				return
+			}
+			cs.WriteConfig()
+		}
+		b, _ := json.Marshal(cs.Config)
+		_, err := w.Write(b)
+		if err != nil {
+			log.Printf("could not write config to client: %s", err)
+		}
+	}
+}
+
+func fetchInfoOneRESTHandler(cs *config.ConfigService, dm *download.Manager) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		vars := mux.Vars(r)
+		idString := vars["id"]
+		if idString != "" {
+			id, err := strconv.Atoi(idString)
+			if err != nil {
+				http.NotFound(w, r)
+				return
+			}
+
+			thisDownload, err := dm.GetDlById(id)
+			if err != nil {
+				http.NotFound(w, r)
+				return
+			}
+			if thisDownload == nil {
+				panic("should not happen")
+			}
+
+			if r.Method == "POST" {
+
+				type updateRequest struct {
+					Action string `json:"action"`
+				}
+
+				thisReq := updateRequest{}
+
+				b, err := io.ReadAll(r.Body)
+				if err != nil {
+					panic(err)
+				}
+
+				err = json.Unmarshal(b, &thisReq)
+				if err != nil {
+					errorRes := errorResponse{Success: false, Error: err.Error()}
+					errorResB, _ := json.Marshal(errorRes)
+					w.WriteHeader(400)
+					_, err = w.Write(errorResB)
+					if err != nil {
+						log.Printf("could not write to client: %s", err)
+					}
+					return
+				}
+
+				if thisReq.Action == "stop" {
+
+					thisDownload.Stop()
+					succRes := successResponse{Success: true, Message: "download stopped"}
+					succResB, _ := json.Marshal(succRes)
+					_, err = w.Write(succResB)
+					if err != nil {
+						log.Printf("could not write to client: %s", err)
+					}
+					return
+				}
+			}
+
+			// just a get, return the object
+			thisDownload.Lock.Lock()
+			defer thisDownload.Lock.Unlock()
+
+			b, _ := json.Marshal(thisDownload)
+
+			_, err = w.Write(b)
+			if err != nil {
+				log.Printf("could not write to client: %s", err)
+			}
+			return
+		} else {
+			http.NotFound(w, r)
+		}
+	}
+}
+
+func fetchInfoRESTHandler(dm *download.Manager) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+
+		b, err := dm.DownloadsAsJSON()
+		if err != nil {
+			panic(err)
+		}
+		_, err = w.Write(b)
+		if err != nil {
+			log.Printf("could not write to client: %s", err)
+		}
+	}
+}
+
+// fetchHandler shows the popup, either the initial form (for create) or the form when in
+// progress (to be updated by REST). It also handles the form POST for creating a new download.
+func fetchHandler(cs *config.ConfigService, vm *version.Manager, dm *download.Manager) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		log.Printf("fetchHandler")
+
+		method := r.Method
+
+		// if they refreshed the popup, just load the existing object, don't
+		// create a new one
+		vars := mux.Vars(r)
+		idString := vars["id"]
+
+		idInt, idOK := strconv.ParseInt(idString, 10, 32)
+
+		if method == "GET" && idOK == nil && idInt > 0 {
+			// existing, load it up
+			log.Printf("loading popup for id %d", idInt)
+			dl, err := dm.GetDlById(int(idInt))
+			if err != nil {
+				log.Printf("not found")
+				w.WriteHeader(404)
+				return
+			}
+
+			t, err := template.ParseFS(webFS, "data/templates/layout.tmpl", "data/templates/popup.tmpl")
+			if err != nil {
+				panic(err)
+			}
+
+			templateData := map[string]interface{}{"dl": dl, "config": cs.Config, "canStop": download.CanStopDownload, "Version": vm.GetInfo()}
+
+			err = t.ExecuteTemplate(w, "layout", templateData)
+			if err != nil {
+				panic(err)
+			}
+			return
+		} else if method == "POST" {
+			// creating a new one
+			type reqType struct {
+				URL                  string `json:"url"`
+				ProfileChosen        string `json:"profile"`
+				DownloadOptionChosen string `json:"download_option"`
+			}
+
+			req := reqType{}
+			json.NewDecoder(r.Body).Decode(&req)
+
+			log.Printf("popup POST request: %#v", req)
+
+			if req.URL == "" {
+				w.WriteHeader(400)
+				json.NewEncoder(w).Encode(errorResponse{
+					Success: false,
+					Error:   "No URL supplied",
+				})
+				return
+			} else {
+
+				if req.ProfileChosen == "" {
+
+					w.WriteHeader(400)
+					json.NewEncoder(w).Encode(errorResponse{
+						Success: false,
+						Error:   "you must choose a profile",
+					})
+					return
+				}
+
+				profile := cs.Config.ProfileCalled(req.ProfileChosen)
+				if profile == nil {
+					w.WriteHeader(400)
+					json.NewEncoder(w).Encode(errorResponse{
+						Success: false,
+						Error:   fmt.Sprintf("no such profile: '%s'", req.ProfileChosen),
+					})
+					return
+				}
+
+				option := cs.Config.DownloadOptionCalled(req.DownloadOptionChosen)
+
+				// create the new download
+				newDL := download.NewDownload(req.URL, cs.Config)
+				id := newDL.Id
+				newDL.DownloadOption = option
+				newDL.DownloadProfile = *profile
+				dm.AddDownload(newDL)
+				dm.Queue(newDL)
+
+				w.WriteHeader(200)
+				json.NewEncoder(w).Encode(queuedResponse{
+					Success:  true,
+					Location: fmt.Sprintf("/fetch/%d", id),
+				})
+			}
+		} else {
+			// a GET, show the popup so they can start the download
+			log.Print("loading popup for a new download")
+			query := r.URL.Query()
+			url, present := query["url"]
+
+			if !present {
+				w.WriteHeader(400)
+				fmt.Fprint(w, "No url supplied")
+				return
+			}
+
+			t, err := template.ParseFS(webFS, "data/templates/layout.tmpl", "data/templates/popup_create.tmpl")
+			if err != nil {
+				panic(err)
+			}
+			templateData := map[string]interface{}{"config": cs.Config, "url": url[0], "Version": vm.GetInfo()}
+
+			err = t.ExecuteTemplate(w, "layout", templateData)
+			if err != nil {
+				panic(err)
+			}
+
+		}
+
+	}
+}