Version bump

.goreleaser.yaml

@@ -12,6 +12,7 @@ before:
   hooks:
     # You may remove this if you don't use go modules.
     - go mod tidy
+    - go test ./...
 
 builds:
   - env:
@@ -36,12 +37,8 @@ archives:
       - goos: windows
         format: zip
 
-# changelog:
-#   sort: asc
-#   filters:
-#     exclude:
-#       - "^docs:"
-#       - "^test:"
+changelog:
+  skip: true
 
 dockers:
   - image_templates:

CHANGELOG.md

@@ -2,7 +2,27 @@
 
 All notable changes to this project will be documented in this file.
 
-## [Unreleased]
+## [v1.1.1] - 2023-12-08
+
+- Fix bug where a brand-new config was created with an out-of-date version
+- Fix for portable mode and using executable in the current working directory
+
+## [v1.1.0] - 2023-11-25
+
+- Add feature to bulk add URL's for downloading
+
+## [v1.0.1] - 2023-11-24
+
+- Fix crash on migrating a config that had > 1 destinations
+
+## [v1.0.0] - 2023-11-23
+
+- Don't start downloads until "start download" is pressed
+- Add "download option" for more per-download customisability, especially for destinations
+- Removed "destinations" as that is now possible more flexibly with download options.
+  - Existing configurations using destinations are automatically migrated to an appropriate `yt-dlp -o ...` download options
+- Gropple now available via docker
+- Clean up web interface display on index page, especially when a playlist with many files is downloading
 
 ## [v0.6.0] - 2023-03-15
 

README.md

@@ -1,41 +1,56 @@
 # gropple
 
-A frontend to youtube-dl (or compatible forks, like yt-dlp) to download videos with a single click, straight from your web browser.
+A frontend to youtube-dl (or compatible forks, like yt-dlp) to download videos
+with a single click, straight from your web browser.
 
 ![Screencast](/screencast.gif)
 
-## Pre-requisites
+## Installing
 
-* some familiarity with the command line
-* youtube-dl (plus any of its required dependencies, like ffmpeg)
-* golang compiler (only if you'd like to build from source)
-
-## Build
+### From Source
 
     go build
 
-## Binaries
+### Standalone Binaries
 
-Binaries are available at <https://github.com/tardisx/gropple/releases>
-
-Gropple will automatically check for available updates and prompt you to
-upgrade.
+Binaries are available at <https://github.com/tardisx/gropple/releases> for most
+platforms.
 
 ## Running
 
-    ./gropple
+### From Binaries
+
+     ./gropple
 
 There are no command line arguments. All configuration is done via the web
 interface. The address will be printed after startup:
 
-    2021/09/30 23:53:00 starting gropple v0.5.0 - https://github.com/tardisx/gropple
-    2021/09/30 23:53:00 go to http://localhost:6123 for details on installing the bookmarklet and to check status
+    2023/11/22 22:42:06 Starting gropple v1.0.0 - https://github.com/tardisx/gropple
+    2023/11/22 22:42:07 Configuration loaded from /Users/username/path/config.yml
+    2023/11/22 22:42:07 Visit http://localhost:6123 for details on installing the bookmarklet and to check status
+
+### Docker
+
+Copy the `docker-compose.yml` to a directory somewhere.
+
+Edit the two `volume` entries to point to local paths where you would like to
+store the config file, and the downloads (the path on the left hand side of the
+colon).
+
+Run `docker-compose up -d` to start the program.
+
+Note that the docker images include `yt-dlp` and `ffmpeg` and are thus
+completely self-contained.
+
+Run `docker-compose logs` to see the output of the program, if you are having
+problems.
 
 ## Using
 
-Bring up `http://localhost:6283` (or your configured address) in your browser.
-You should see a link to the bookmarklet at the top of the screen, and the list
-of downloads (currently empty).
+Bring up `http://localhost:6283` (or the appropriate host if you are running it
+on a different machine) in your browser. You should see a link to the
+bookmarklet at the top of the screen, and the list of downloads (currently
+empty).
 
 Drag the bookmarklet to your favourites bar, or otherwise bookmark it as you see
 fit. Any kind of browser bookmark should work. The bookmarklet contains embedded
@@ -48,44 +63,114 @@ the bookmarklet.
 A popup window will appear. Choose a download profile and the download will
 start. The status will be shown in the window, updating in real time.
 
+There is also an optional "download option" you can choose. These are discussed
+below.
+
 You may close this window at any time without stopping the download, the status
-of all downloads is available on the index page.
+of all downloads is available on the index page. Clicking on the id number will
+show the popup again.
 
 ## Configuration
 
-Click the "config" link on the index page to configure gropple. The default
-options are fine if you are running on your local machine. If you are running it
-remotely you will need to set the "server address" to ensure the bookmarklet has
-the correct URL in it.
+Click the "config" link on the index page to configure gropple.
 
-### Configuring Downloaders
+The options in each part are dicussed below.
+
+### Server
+
+#### Port and Server Address
+
+You can configure the port number here if you do not want the default of `6123`.
+
+If you are running it on a machine other than `localhost` you will need to set
+the "server address" to ensure the bookmarklet has the correct URL in it.
+Similarly, if you are running it behind a reverse proxy, the address here must
+match what you would type in the browser so that the bookmarklet will work
+correctly.
+
+#### Download path
+
+The download path specifies where downloads will end up, *if* no specific `-o`
+options are passed to `yt-dlp`.
+
+#### Maximum active downloads per domain
+
+Gropple will limit the number of downloads per domain to this number. Increasing
+this will likely result in failed downloads when server rate limiters notice
+you.
+
+#### UI popup size
+
+Changes the size of the popup window.
+
+### Download Profiles
 
 Gropple's default configuration uses `yt-dlp` and has two profiles set up, one
 for downloading video, the other for downloading audio (mp3).
 
+Each download profile consists of a name (for your reference), a command to run,
+and a number of arguments.
+
 Note that gropple does not include any downloaders, you have to install them
-separately.
+separately (unless using the docker image).
 
 If you would like to use a youtube-dl compatible fork or change the options you
-can do so on the right hand side. Create as many profiles as you wish, whenever
-you start a download you can choose the appropriate profile.
+can do so here. Create as many profiles as you wish, whenever you start a
+download you can choose the appropriate profile.
 
 Note that the command arguments must each be specified separately - see the
-default configuration for an example.
+default configuration. For example, if you have a single argument like
+`--audio-format mp3`, it will be parsed by the `yt-dlp` as a single, long
+unknown argument, and will fail. This needs to be configured as two arguments,
+`--audio-format` and `mp3`.
 
 While gropple will use your `PATH` to find the executable, you can also specify
 a full path instead. Note that any tools that the downloader calls itself (for
 instance, `ffmpeg`) will need to be available on your path.
 
-### Alternate destinations
+### Download Options
 
-Gropple supports adding additional optional destinations. By default, all
-downloads will be stored in the main download path specified in the config. You
-can also add one or more destinations, and you can choose one of these
-destinations when queueing a new download, or while it is still downloading from
-the popup.
+There are also an arbitrary amount of Download Options you can configure. Each
+one specifies one or more extra arguments to add to the downloader command line.
+The most common use for this is to have customised download paths. For instance,
+sometimes you might want to bundle all files into a single directory, other
+times you might want to separate files by download playlist URL or similar.
 
-The file will be moved after downloading is complete.
+Most of this is done directly through appropriate options for `yt-dlp`, see the
+[output template
+documentation](https://github.com/yt-dlp/yt-dlp#output-template).
+
+However, gropple offers two extra substitutions:
+
+  * `%GROPPLE_HOST%`
+  * `%GROPPLE_PATH%`
+
+These will be replaced with the hostname, and path of the download,
+respectively.
+
+So, a playlist URL `https://www.youtube.com/@UsernameHere`
+
+With a download option setup like this:
+
+    * Name of Option: Split by Host and Path
+    * Arguments:
+      * -o
+      * /Downloads/%GROPPLE_HOST%/%GROPPLE_PATH%/%(title)s [%(id)s].%(ext)s
+
+Will result in downloads going into the path
+`/Downloads/www.youtube.com/@UsernameHere/...`.
+
+Note that this also means that `yt-dlp` can resume partially downloaded files, and
+also automatically 'backfill', downloading only files that have not been
+downloaded yet from that playlist.
+
+## Downloading a list of URL's in bulk
+
+From main index page you can click the "Bulk" link in the menu to bring up the
+bulk queue page.
+
+In all respects this acts the same as the usual bookmarklet, but it has a
+textbox for pasting many URLs at once. All downloads will be queued immediately.
 
 ## Portable mode
 

build_release.pl

@@ -1,49 +0,0 @@
-#!/usr/bin/env perl
-
-use strict;
-use warnings;
-
-open my $fh, "<", "main.go" || die $!;
-
-my $version;
-while (<$fh>) {
-  # CurrentVersion: "v0.04"
-  $version = $1 if /CurrentVersion:\s*"(v.*?)"/;
-}
-close $fh;
-
-die "no version?" unless defined $version;
-
-# quit if tests fail
-system("go test ./...") && die "not building release with failing tests";
-
-# so lazy
-system "rm", "-rf", "release", "dist";
-system "mkdir", "release";
-system "mkdir", "dist";
-
-my %build = (
-  win   => { env => { GOOS => 'windows', GOARCH => 'amd64' }, filename => 'gropple.exe' },
-  linux => { env => { GOOS => 'linux',   GOARCH => 'amd64' }, filename => 'gropple' },
-  mac   => { env => { GOOS => 'darwin',  GOARCH => 'amd64' }, filename => 'gropple' },
-); 
-
-foreach my $type (keys %build) {
-  mkdir "release/$type";
-}
-
-foreach my $type (keys %build) {
-  local $ENV{GOOS}   = $build{$type}->{env}->{GOOS};
-  local $ENV{GOARCH} = $build{$type}->{env}->{GOARCH};
-  system "go", "build", "-o", "release/$type/" . $build{$type}->{filename};
-  system "zip", "-j", "dist/gropple-$type-$version.zip", ( glob "release/$type/*" );
-}
-
-# now docker
-exit 0;
-$ENV{VERSION}="$version";
-system "docker-compose", "-f", "docker-compose.build.yml", "build";
-system "docker", "tag", "tardisx/gropple:$version", "tardisx/gropple:latest";
-system "docker", "push", "tardisx/gropple:$version";
-system "docker", "push", "tardisx/gropple:latest";
-

config/config.go

@@ -65,7 +65,7 @@ type ConfigService struct {
 func (cs *ConfigService) LoadTestConfig() {
 	cs.LoadDefaultConfig()
 	cs.Config.Server.DownloadPath = "/tmp"
-	cs.Config.DownloadProfiles = []DownloadProfile{{Name: "test profile", Command: "sleep", Args: []string{"5"}}}
+	cs.Config.DownloadProfiles = []DownloadProfile{{Name: "test profile", Command: "/bin/sleep", Args: []string{"5"}}}
 }
 
 func (cs *ConfigService) LoadDefaultConfig() {
@@ -98,7 +98,7 @@ func (cs *ConfigService) LoadDefaultConfig() {
 	defaultConfig.Destinations = nil
 	defaultConfig.DownloadOptions = make([]DownloadOption, 0)
 
-	defaultConfig.ConfigVersion = 3
+	defaultConfig.ConfigVersion = 4
 
 	cs.Config = &defaultConfig
 
@@ -189,9 +189,10 @@ func (c *Config) UpdateFromJSON(j []byte) error {
 		}
 
 		// check the command exists
-		_, err := exec.LookPath(newConfig.DownloadProfiles[i].Command)
+
+		_, err := AbsPathToExecutable(newConfig.DownloadProfiles[i].Command)
 		if err != nil {
-			return fmt.Errorf("Could not find %s on the path", newConfig.DownloadProfiles[i].Command)
+			return fmt.Errorf("problem with command '%s': %s", newConfig.DownloadProfiles[i].Command, err)
 		}
 	}
 
@@ -300,8 +301,8 @@ func (cs *ConfigService) LoadConfig() error {
 				Args: []string{"-o", fmt.Sprintf("%s/%%(title)s [%%(id)s].%%(ext)s", c.Destinations[i].Path)},
 			}
 			c.DownloadOptions = append(c.DownloadOptions, newDownloadOption)
-			c.Destinations = nil
 		}
+		c.Destinations = nil
 		configMigrated = true
 		log.Print("migrated config from version 3 => 4")
 	}
@@ -337,3 +338,28 @@ func (cs *ConfigService) WriteConfig() {
 	}
 	file.Close()
 }
+
+// AbsPathToExecutable takes a command name, which may or may not be path-qualified,
+// and returns the fully qualified path to it, or an error if could not be found, or
+// if it does not appear to be a file.
+func AbsPathToExecutable(cmd string) (string, error) {
+
+	pathCmd, err := exec.LookPath(cmd)
+	if err != nil {
+		return "", fmt.Errorf("could not LookPath '%s': %w", cmd, err)
+	}
+
+	execAbsolutePath, err := filepath.Abs(pathCmd)
+	if err != nil {
+		return "", fmt.Errorf("could not get absolute path to '%s': %w", cmd, err)
+	}
+	fi, err := os.Stat(execAbsolutePath)
+	if err != nil {
+		return "", fmt.Errorf("could not get stat '%s': %w", cmd, err)
+	}
+	if !fi.Mode().IsRegular() {
+		return "", fmt.Errorf("'%s' is not a regular file: %w", cmd, err)
+	}
+
+	return execAbsolutePath, nil
+}

config/config_test.go

@@ -2,6 +2,8 @@ package config
 
 import (
 	"os"
+	"os/exec"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -102,6 +104,66 @@ profiles:
 	os.Remove(cs.ConfigPath)
 }
 
+func TestMigrateV3toV4CrashBug(t *testing.T) {
+	v3Config := `config_version: 3
+server:
+  port: 6123
+  address: https://superaddress.here.com
+  download_path: /home/path/gropple
+  maximum_active_downloads_per_domain: 2
+ui:
+  popup_width: 500
+  popup_height: 500
+destinations:
+- name: somegifs
+  path: /home/path/somegifs
+- name: otherstuff
+  path: /home/path/otherstuff
+profiles:
+- name: standard video
+  command: yt-dlp
+  args:
+  - --newline
+  - --write-info-json
+  - -f
+  - bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best
+  - --verbose
+  - --embed-metadata
+  - --embed-subs
+  - --embed-thumbnail
+- name: standard mp3
+  command: yt-dlp
+  args:
+  - --extract-audio
+  - --audio-format
+  - mp3
+  - --prefer-ffmpeg
+`
+	cs := configServiceFromString(v3Config)
+	err := cs.LoadConfig()
+	if err != nil {
+		t.Errorf("got error when loading config: %s", err)
+	}
+	if cs.Config.ConfigVersion != 4 {
+		t.Errorf("did not migrate version (it is '%d')", cs.Config.ConfigVersion)
+	}
+	if cs.Config.Server.MaximumActiveDownloads != 2 {
+		t.Error("did not add MaximumActiveDownloads")
+	}
+	if len(cs.Config.Destinations) != 0 {
+		t.Error("incorrect number of destinations from migrated file")
+	}
+	if assert.Len(t, cs.Config.DownloadOptions, 2) {
+		if assert.Len(t, cs.Config.DownloadOptions[0].Args, 2) {
+			assert.Equal(t, "-o", cs.Config.DownloadOptions[0].Args[0])
+			assert.Equal(t, "/home/path/somegifs/%(title)s [%(id)s].%(ext)s", cs.Config.DownloadOptions[0].Args[1])
+			assert.Equal(t, "-o", cs.Config.DownloadOptions[1].Args[0])
+			assert.Equal(t, "/home/path/otherstuff/%(title)s [%(id)s].%(ext)s", cs.Config.DownloadOptions[1].Args[1])
+		}
+	}
+	os.Remove(cs.ConfigPath)
+}
+
 func configServiceFromString(configString string) *ConfigService {
 	tmpFile, _ := os.CreateTemp("", "gropple_test_*.yml")
 	tmpFile.Write([]byte(configString))
@@ -112,3 +174,41 @@ func configServiceFromString(configString string) *ConfigService {
 	}
 	return &cs
 }
+
+func TestLookForExecutable(t *testing.T) {
+	cmdPath, err := exec.LookPath("sleep")
+	if err != nil {
+		t.Errorf("cannot run this test without knowing about sleep: %s", err)
+		t.FailNow()
+	}
+	cmdDir := filepath.Dir(cmdPath)
+
+	cmd := "sleep"
+	path, err := AbsPathToExecutable(cmd)
+	if assert.NoError(t, err) {
+		assert.Equal(t, cmdPath, path)
+	}
+
+	cmd = cmdPath
+	path, err = AbsPathToExecutable(cmd)
+	if assert.NoError(t, err) {
+		assert.Equal(t, cmdPath, path)
+	}
+
+	cmd = "../../../../../../../../.." + cmdPath
+	path, err = AbsPathToExecutable(cmd)
+	if assert.NoError(t, err) {
+		assert.Equal(t, cmdPath, path)
+	}
+	cmd = "./sleep"
+	_, err = AbsPathToExecutable(cmd)
+	assert.Error(t, err)
+
+	os.Chdir(cmdDir)
+	cmd = "./sleep"
+	path, err = AbsPathToExecutable(cmd)
+	if assert.NoError(t, err) {
+		assert.Equal(t, cmdPath, path)
+	}
+
+}

docker-compose.build.yml

@@ -1,6 +0,0 @@
-version: "3.9"
-    
-services:
-  gropple:
-    build: .
-    image: tardisx/gropple:$VERSION

docker-compose.yml

@@ -1,12 +1,11 @@
 version: "3.9"
-    
+
 services:
   gropple:
-    build: .
     image: tardisx/gropple:latest
     volumes:
-      - ./gropple-config-dir:/config
-      - ./downloads:/downloads/
+      - /tmp/gropple-config-dir/:/config
+      - /tmp/downloads/:/downloads/
     restart: always
     ports:
       - "6123:6123"

download/download.go

@@ -281,9 +281,21 @@ func (dl *Download) Begin() {
 		cmdSlice = append(cmdSlice, dl.Url)
 	}
 
-	dl.Log = append(dl.Log, fmt.Sprintf("executing: %s with args: %s", dl.DownloadProfile.Command, strings.Join(cmdSlice, " ")))
-	cmd := exec.Command(dl.DownloadProfile.Command, cmdSlice...)
+	cmdPath, err := config.AbsPathToExecutable(dl.DownloadProfile.Command)
+	if err != nil {
+		dl.State = STATE_FAILED
+		dl.Finished = true
+		dl.FinishedTS = time.Now()
+		dl.Log = append(dl.Log, fmt.Sprintf("error finding executable for downloader: %s", err.Error()))
+		dl.Lock.Unlock()
+		return
+	}
+
+	dl.Log = append(dl.Log, fmt.Sprintf("executing: %s (%s) with args: %s", dl.DownloadProfile.Command, cmdPath, strings.Join(cmdSlice, " ")))
+
+	cmd := exec.Command(cmdPath, cmdSlice...)
 	cmd.Dir = dl.Config.Server.DownloadPath
+	log.Printf("Executing command executable: %s) in %s", cmdPath, dl.Config.Server.DownloadPath)
 
 	stdout, err := cmd.StdoutPipe()
 	if err != nil {
@@ -292,7 +304,6 @@ func (dl *Download) Begin() {
 		dl.FinishedTS = time.Now()
 		dl.Log = append(dl.Log, fmt.Sprintf("error setting up stdout pipe: %v", err))
 		dl.Lock.Unlock()
-
 		return
 	}
 
@@ -307,9 +318,10 @@ func (dl *Download) Begin() {
 		return
 	}
 
-	log.Printf("Executing command: %v", cmd)
 	err = cmd.Start()
 	if err != nil {
+		log.Printf("Executing command failed: %s", err.Error())
+
 		dl.State = STATE_FAILED
 		dl.Finished = true
 		dl.FinishedTS = time.Now()
@@ -363,7 +375,6 @@ func (dl *Download) Begin() {
 		}
 	}
 	dl.Lock.Unlock()
-
 }
 
 // updateDownload updates the download based on data from the reader. Expects the

main.go

@@ -15,7 +15,7 @@ import (
 
 func main() {
 	versionInfo := &version.Manager{
-		VersionInfo: version.Info{CurrentVersion: "v0.7.0"},
+		VersionInfo: version.Info{CurrentVersion: "v1.1.1"},
 	}
 	log.Printf("Starting gropple %s - https://github.com/tardisx/gropple", versionInfo.GetInfo().CurrentVersion)
 

web/data/templates/bulk.tmpl

@@ -0,0 +1,91 @@
+{{ define "content" }}
+
+{{ template "menu.tmpl" . }}
+
+<div id="layout" class="pure-g pure-u-1" x-data="bulk_create()" >
+
+<h1>Bulk upload</h1>
+
+    <p class="error"  x-show="error_message"  x-transition.duration.500ms x-text="error_message"></p>
+    <p class="success" x-show="success_message" x-transition.duration.500ms x-text="success_message"></p>
+
+    <p>Paste URLs here, one per line:</p>
+
+    <textarea x-model="urls" rows="20" cols="80">
+    </textarea>
+
+    <br><br>
+
+    <table class="pure-table" >
+        <tr>
+            <th>profile</th>
+            <td>
+                <select class="pure-input-1-2" x-model="profile_chosen">
+                    <option value="">choose a profile</option>
+                {{ range $i := .config.DownloadProfiles }}
+                    <option name="{{$i.Name}}">{{ $i.Name }}</option>
+                {{ end }}
+                </select>
+            </td>
+        </tr>
+        <tr>
+            <th>download option</th>
+            <td>
+                <select  class="pure-input-1-2" x-model="download_option_chosen">
+                    <option value="">no option</option>
+                {{ range $i := .config.DownloadOptions }}
+                    <option name="{{$i.Name}}">{{ $i.Name }}</option>
+                {{ end }}
+                </select>
+            </td>
+        </tr>
+        <tr>
+            <th>&nbsp;</th>
+            <td>
+                <button class="button-small pure-button" @click="start()">add to queue</button>
+            </td>
+        </tr>
+
+    </table>
+
+
+
+
+</div>
+
+{{ end }}
+
+{{ define "js" }}
+<script>
+    function bulk_create() {
+        return {
+            profile_chosen: "",
+            download_option_chosen: "",
+            urls: "",
+            error_message: "",
+            success_message: "",
+            start() {
+                let op = {
+                   method: 'POST',
+                   body: JSON.stringify({action: 'start', urls: this.urls, profile: this.profile_chosen, download_option: this.download_option_chosen}),
+                   headers: { 'Content-Type': 'application/json' }
+                };
+                fetch('/bulk', op)
+                .then(response => response.json())
+                .then(response => {
+                    console.log(response)
+                     if (response.error) {
+                        this.error_message = response.error;
+                        this.success_message = '';
+                        document.body.scrollTop = document.documentElement.scrollTop = 0;
+                    } else {
+                        this.error_message = '';
+                        this.success_message = response.message;
+                        this.urls = '';
+                    }
+                })
+            }
+        }
+    }
+</script>
+{{ end }}

web/data/templates/config.tmpl

@@ -84,7 +84,10 @@
 
                             <label x-bind:for="'config-profiles-'+i+'-command'">Command to run</label>
                             <input type="text" x-bind:id="'config-profiles-'+i+'-command'" class="input-long" placeholder="name" x-model="profile.command" />
-                            <span class="pure-form-message">Which command to run. Your path will be searched, or you can specify the full path here.</span>
+                            <span class="pure-form-message">Which command to run. Your path will be searched, or you can specify the full path here.
+                            If you are using gropple in portable mode and store the executables with the gropple executable, use a prefix of
+                            <tt>./</tt>, for instance <tt>yt-dlp.exe</tt>.
+                            </span>
 
 
                             <label>Arguments</label>

web/data/templates/menu.tmpl

@@ -7,9 +7,11 @@
             <li class="pure-menu-item">
                 <a href="/config" class="pure-menu-link">Config</a>
             </li>
+            <li class="pure-menu-item">
+                <a href="/bulk" class="pure-menu-link">Bulk</a>
+            </li>
             <li class="pure-menu-item">
                 <a href="https://github.com/tardisx/gropple" class="pure-menu-link">Github</a>
             </li>
         </ul>
     </div>
-    

web/data/templates/popup_create.tmpl

@@ -13,7 +13,7 @@
                     <select class="pure-input-1-2" x-model="profile_chosen">
                         <option value="">choose a profile</option>
                     {{ range $i := .config.DownloadProfiles }}
-                        <option>{{ $i.Name }}</option>
+                        <option name="{{$i.Name}}">{{ $i.Name }}</option>
                     {{ end }}
                     </select>
                 </td>
@@ -24,7 +24,7 @@
                     <select  class="pure-input-1-2" x-model="download_option_chosen">
                         <option value="">no option</option>
                     {{ range $i := .config.DownloadOptions }}
-                        <option>{{ $i.Name }}</option>
+                        <option name="{{$i.Name}}">{{ $i.Name }}</option>
                     {{ end }}
                     </select>
                 </td>

web/web.go

@@ -55,6 +55,9 @@ func CreateRoutes(cs *config.ConfigService, dm *download.Manager, vm *version.Ma
 	r.HandleFunc("/fetch", fetchHandler(cs, vm, dm))
 	r.HandleFunc("/fetch/{id}", fetchHandler(cs, vm, dm))
 
+	// handle the bulk uploader
+	r.HandleFunc("/bulk", bulkHandler(cs, vm, dm))
+
 	// get/update info on a download
 	r.HandleFunc("/rest/fetch/{id}", fetchInfoOneRESTHandler(cs, dm))
 
@@ -399,3 +402,92 @@ func fetchHandler(cs *config.ConfigService, vm *version.Manager, dm *download.Ma
 
 	}
 }
+
+func bulkHandler(cs *config.ConfigService, vm *version.Manager, dm *download.Manager) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		log.Printf("bulkHandler")
+
+		method := r.Method
+		if method == "GET" {
+
+			t, err := template.ParseFS(webFS, "data/templates/layout.tmpl", "data/templates/menu.tmpl", "data/templates/bulk.tmpl")
+			if err != nil {
+				panic(err)
+			}
+			templateData := map[string]interface{}{"config": cs.Config, "Version": vm.GetInfo()}
+
+			err = t.ExecuteTemplate(w, "layout", templateData)
+			if err != nil {
+				panic(err)
+			}
+
+			return
+
+		} else if method == "POST" {
+			type reqBulkType struct {
+				URLs                 string `json:"urls"`
+				ProfileChosen        string `json:"profile"`
+				DownloadOptionChosen string `json:"download_option"`
+			}
+
+			req := reqBulkType{}
+			json.NewDecoder(r.Body).Decode(&req)
+
+			log.Printf("bulk POST request: %#v", req)
+
+			if req.URLs == "" {
+				w.WriteHeader(400)
+				json.NewEncoder(w).Encode(errorResponse{
+					Success: false,
+					Error:   "No URLs supplied",
+				})
+				return
+			}
+
+			if req.ProfileChosen == "" {
+
+				w.WriteHeader(400)
+				json.NewEncoder(w).Encode(errorResponse{
+					Success: false,
+					Error:   "you must choose a profile",
+				})
+				return
+			}
+
+			profile := cs.Config.ProfileCalled(req.ProfileChosen)
+			if profile == nil {
+				w.WriteHeader(400)
+				json.NewEncoder(w).Encode(errorResponse{
+					Success: false,
+					Error:   fmt.Sprintf("no such profile: '%s'", req.ProfileChosen),
+				})
+				return
+			}
+
+			option := cs.Config.DownloadOptionCalled(req.DownloadOptionChosen)
+
+			// create the new downloads
+			urls := strings.Split(req.URLs, "\n")
+			count := 0
+			for _, thisURL := range urls {
+				thisURL = strings.TrimSpace(thisURL)
+				if thisURL != "" {
+					newDL := download.NewDownload(thisURL, cs.Config)
+					newDL.DownloadOption = option
+					newDL.DownloadProfile = *profile
+					dm.AddDownload(newDL)
+					dm.Queue(newDL)
+					log.Printf("queued %s", thisURL)
+					count++
+				}
+			}
+
+			w.WriteHeader(200)
+			json.NewEncoder(w).Encode(successResponse{
+				Success: true,
+				Message: fmt.Sprintf("queued %d downloads", count),
+			})
+			return
+		}
+	}
+}