diff --git a/config/config.go b/config/config.go index e1d0640..0fbbfc9 100644 --- a/config/config.go +++ b/config/config.go @@ -27,6 +27,12 @@ type DownloadProfile struct { Args []string `yaml:"args" json:"args"` } +// DownloadOption contains configuration for extra arguments to pass to the download command +type DownloadOption struct { + Name string `yaml:"name" json:"name"` + Args []string `yaml:"args" json:"args"` +} + // UI holds the configuration for the user interface type UI struct { PopupWidth int `yaml:"popup_width" json:"popup_width"` @@ -44,8 +50,9 @@ type Config struct { ConfigVersion int `yaml:"config_version" json:"config_version"` Server Server `yaml:"server" json:"server"` UI UI `yaml:"ui" json:"ui"` - Destinations []Destination `yaml:"destinations" json:"destinations"` + Destinations []Destination `yaml:"destinations" json:"destinations"` // no longer in use, see DownloadOptions DownloadProfiles []DownloadProfile `yaml:"profiles" json:"profiles"` + DownloadOptions []DownloadOption `yaml:"download_options" json:"download_options"` } // ConfigService is a struct to handle configuration requests, allowing for the @@ -88,7 +95,8 @@ func (cs *ConfigService) LoadDefaultConfig() { defaultConfig.Server.MaximumActiveDownloads = 2 - defaultConfig.Destinations = make([]Destination, 0) + defaultConfig.Destinations = nil + defaultConfig.DownloadOptions = make([]DownloadOption, 0) defaultConfig.ConfigVersion = 3 @@ -96,7 +104,7 @@ func (cs *ConfigService) LoadDefaultConfig() { } -// ProfileCalled returns the corresponding profile, or nil if it does not exist +// ProfileCalled returns the corresponding DownloadProfile, or nil if it does not exist func (c *Config) ProfileCalled(name string) *DownloadProfile { for _, p := range c.DownloadProfiles { if p.Name == name { @@ -106,11 +114,11 @@ func (c *Config) ProfileCalled(name string) *DownloadProfile { return nil } -// DestinationCalled returns the corresponding destination, or nil if it does not exist -func (c *Config) DestinationCalled(name string) *Destination { - for _, p := range c.Destinations { - if p.Name == name { - return &p +// DownloadOptionCalled returns the corresponding DownloadOption, or nil if it does not exist +func (c *Config) DownloadOptionCalled(name string) *DownloadOption { + for _, o := range c.DownloadOptions { + if o.Name == name { + return &o } } return nil @@ -187,17 +195,6 @@ func (c *Config) UpdateFromJSON(j []byte) error { } } - // check destinations - for _, dest := range newConfig.Destinations { - s, err := os.Stat(dest.Path) - if err != nil { - return fmt.Errorf("destination '%s' (%s) is bad: %s", dest.Name, dest.Path, err) - } - if !s.IsDir() { - return fmt.Errorf("destination '%s' (%s) is not a directory", dest.Name, dest.Path) - } - } - *c = newConfig return nil } @@ -295,6 +292,20 @@ func (cs *ConfigService) LoadConfig() error { log.Print("migrated config from version 2 => 3") } + if c.ConfigVersion == 3 { + c.ConfigVersion = 4 + for i := range c.Destinations { + newDownloadOption := DownloadOption{ + Name: c.Destinations[i].Name, + Args: []string{"-o", fmt.Sprintf("%s/%%(title)s [%%(id)s].%%(ext)s", c.Destinations[i].Path)}, + } + c.DownloadOptions = append(c.DownloadOptions, newDownloadOption) + c.Destinations = nil + } + configMigrated = true + log.Print("migrated config from version 3 => 4") + } + if configMigrated { log.Print("Writing new config after version migration") cs.WriteConfig() diff --git a/config/config_test.go b/config/config_test.go index fc41f5e..39b5743 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -3,10 +3,12 @@ package config import ( "os" "testing" + + "github.com/stretchr/testify/assert" ) -func TestMigrationV1toV3(t *testing.T) { - v2Config := `config_version: 1 +func TestMigrationV1toV4(t *testing.T) { + v1Config := `config_version: 1 server: port: 6123 address: http://localhost:6123 @@ -31,12 +33,12 @@ profiles: - --audio-format - mp3 ` - cs := configServiceFromString(v2Config) + cs := configServiceFromString(v1Config) err := cs.LoadConfig() if err != nil { t.Errorf("got error when loading config: %s", err) } - if cs.Config.ConfigVersion != 3 { + if cs.Config.ConfigVersion != 4 { t.Errorf("did not migrate version (it is '%d')", cs.Config.ConfigVersion) } if cs.Config.Server.MaximumActiveDownloads != 2 { @@ -48,6 +50,58 @@ profiles: os.Remove(cs.ConfigPath) } +func TestMigrateV3toV4(t *testing.T) { + v3Config := `config_version: 3 +server: + port: 6123 + address: http://localhost:6123 + download_path: /tmp/Downloads + maximum_active_downloads_per_domain: 2 +ui: + popup_width: 900 + popup_height: 900 +destinations: + - name: cool destination + path: /tmp/coolness +profiles: + - name: standard video + command: yt-dlp + args: + - --newline + - --write-info-json + - -f + - bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best + - name: standard mp3 + command: yt-dlp + args: + - --newline + - --write-info-json + - --extract-audio + - --audio-format + - mp3` + cs := configServiceFromString(v3Config) + err := cs.LoadConfig() + if err != nil { + t.Errorf("got error when loading config: %s", err) + } + if cs.Config.ConfigVersion != 4 { + t.Errorf("did not migrate version (it is '%d')", cs.Config.ConfigVersion) + } + if cs.Config.Server.MaximumActiveDownloads != 2 { + t.Error("did not add MaximumActiveDownloads") + } + if len(cs.Config.Destinations) != 0 { + t.Error("incorrect number of destinations from migrated file") + } + if assert.Len(t, cs.Config.DownloadOptions, 1) { + if assert.Len(t, cs.Config.DownloadOptions[0].Args, 2) { + assert.Equal(t, "-o", cs.Config.DownloadOptions[0].Args[0]) + assert.Equal(t, "/tmp/coolness/%(title)s [%(id)s].%(ext)s", cs.Config.DownloadOptions[0].Args[1]) + } + } + os.Remove(cs.ConfigPath) +} + func configServiceFromString(configString string) *ConfigService { tmpFile, _ := os.CreateTemp("", "gropple_test_*.yml") tmpFile.Write([]byte(configString)) diff --git a/download/download.go b/download/download.go index 68007fa..bd891eb 100644 --- a/download/download.go +++ b/download/download.go @@ -8,7 +8,6 @@ import ( "net/url" "os" "os/exec" - "path/filepath" "regexp" "strconv" "strings" @@ -27,7 +26,7 @@ type Download struct { ExitCode int `json:"exit_code"` State State `json:"state"` DownloadProfile config.DownloadProfile `json:"download_profile"` - Destination *config.Destination `json:"destination"` + DownloadOption *config.DownloadOption `json:"download_option"` Finished bool `json:"finished"` FinishedTS time.Time `json:"finished_ts"` Files []string `json:"files"` @@ -82,7 +81,6 @@ func (m *Manager) ManageQueue() { m.Lock.Lock() m.startQueued(m.MaxPerDomain) - m.moveToDest() m.cleanup() m.Lock.Unlock() @@ -102,32 +100,6 @@ func (m *Manager) DownloadsAsJSON() ([]byte, error) { return b, err } -func (m *Manager) moveToDest() { - - // move any downloads that are complete and have a dest - for _, dl := range m.Downloads { - - dl.Lock.Lock() - if dl.Destination != nil && dl.State == STATE_COMPLETE { - dl.State = STATE_MOVED - for _, fn := range dl.Files { - src := filepath.Join(dl.Config.Server.DownloadPath, fn) - dst := filepath.Join(dl.Destination.Path, fn) - err := os.Rename(src, dst) - if err != nil { - log.Printf("%s", err) - dl.Log = append(dl.Log, fmt.Sprintf("Could not move %s to %s - %s", fn, dl.Destination.Path, err)) - break - } else { - dl.Log = append(dl.Log, fmt.Sprintf("Moved %s to %s", fn, dl.Destination.Path)) - - } - } - } - dl.Lock.Unlock() - } -} - // startQueued starts any downloads that have been queued, we would not exceed // maxRunning. If maxRunning is 0, there is no limit. func (m *Manager) startQueued(maxRunning int) { @@ -202,17 +174,6 @@ func (m *Manager) Queue(dl *Download) { dl.State = STATE_QUEUED } -func (m *Manager) ChangeDestination(dl *Download, dest *config.Destination) { - dl.Lock.Lock() - // we can only change destination is certain cases... - if dl.State != STATE_FAILED && dl.State != STATE_MOVED { - dl.Destination = dest - } - - dl.Lock.Unlock() - -} - func NewDownload(url string, conf *config.Config) *Download { atomic.AddInt32(&downloadId, 1) dl := Download{ diff --git a/download/download_test.go b/download/download_test.go index 5e6ead1..d398e6a 100644 --- a/download/download_test.go +++ b/download/download_test.go @@ -237,15 +237,15 @@ func TestUpdateMetadataPlaylist(t *testing.T) { output := ` start of log... -[download] Downloading playlist: niceuser -[RedGifsUser] niceuser: Downloading JSON metadata page 1 -[RedGifsUser] niceuser: Downloading JSON metadata page 2 -[RedGifsUser] niceuser: Downloading JSON metadata page 3 -[RedGifsUser] niceuser: Downloading JSON metadata page 4 -[RedGifsUser] niceuser: Downloading JSON metadata page 5 -[RedGifsUser] niceuser: Downloading JSON metadata page 6 -[info] Writing playlist metadata as JSON to: niceuser [niceuser].info.json -[RedGifsUser] playlist niceuser: Downloading 3 videos +[download] Downloading playlist: nice_user +[RedGifsUser] nice_user: Downloading JSON metadata page 1 +[RedGifsUser] nice_user: Downloading JSON metadata page 2 +[RedGifsUser] nice_user: Downloading JSON metadata page 3 +[RedGifsUser] nice_user: Downloading JSON metadata page 4 +[RedGifsUser] nice_user: Downloading JSON metadata page 5 +[RedGifsUser] nice_user: Downloading JSON metadata page 6 +[info] Writing playlist metadata as JSON to: nice_user [nice_user].info.json +[RedGifsUser] playlist nice_user: Downloading 3 videos [download] Downloading video 1 of 3 [info] wrongpreciouschrysomelid: Downloading 1 format(s): hd [info] Writing video metadata as JSON to: Splendid Wonderful Speaker Power Chocolate Drop [wrongpreciouschrysomelid].info.json @@ -279,8 +279,8 @@ start of log... [download] 69.1% of 2.89MiB at 11.63MiB/s ETA 00:00 [download] 100% of 2.89MiB at 14.25MiB/s ETA 00:00 [download] 100% of 2.89MiB in 00:00 -[info] Writing updated playlist metadata as JSON to: niceuser [niceuser].info.json -[download] Finished downloading playlist: niceuser +[info] Writing updated playlist metadata as JSON to: nice_user [nice_user].info.json +[download] Finished downloading playlist: nice_user ` newD := Download{} diff --git a/go.mod b/go.mod index a570861..f49754f 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,13 @@ go 1.20 require ( github.com/gorilla/mux v1.8.1 + github.com/stretchr/testify v1.8.4 golang.org/x/mod v0.14.0 gopkg.in/yaml.v2 v2.4.0 ) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum index 14ccd3d..5a945ce 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,16 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/web/data/templates/config.tmpl b/web/data/templates/config.tmpl index 1547e1e..70e13c7 100644 --- a/web/data/templates/config.tmpl +++ b/web/data/templates/config.tmpl @@ -11,14 +11,14 @@
- +
- +
@@ -65,8 +65,8 @@ Download Profiles -

Gropple supports multiple download profiles. Each profile specifies a different youtube-dl - compatible command, and arguments. When starting a download, you may choose which profile +

Gropple supports multiple download profiles. Each profile specifies a different youtube-dl + compatible command, and arguments. When starting a download, you may choose which profile to use. The URL will be appended to the argument list at the end.

@@ -75,10 +75,10 @@ - +
@@ -113,39 +113,44 @@
- Destinations -

You can specify custom destinations (directories) here. Downloads can be - moved to one of these directories after completion from the index page, - if you do not want them to be left in the download path above.

+ Download Options +

You can specify custom download options here. These are (optionally) selectable in addition + to the profile when starting a download. They append extra arguments to the downloader command. + The most common use is to specify a particular -o argument to yt-dlp to allow files to be downloaded + to a custom path.

-