8 Commits

Author SHA1 Message Date
ea70f47f76 Update Changelog 2023-03-13 10:54:47 +10:30
2e3156ef65 New alpha release 2023-03-13 10:50:25 +10:30
08e2c1c377 Use constants and constructors in the test 2023-03-13 10:48:38 +10:30
b40dd218f1 Add move to destination functionality 2023-03-13 10:32:20 +10:30
ba87b943ea Add some stress-test data 2023-03-13 10:25:59 +10:30
3e7a3a2f3b Fix some more races 2023-03-10 00:07:29 +10:30
f2c05d0144 Restore test 2023-03-09 23:24:03 +10:30
3d72b8b16a Remove debugging test URLs 2023-03-09 21:46:28 +10:30
9 changed files with 295 additions and 105 deletions

View File

@@ -7,6 +7,8 @@ All notable changes to this project will be documented in this file.
## [v0.6.0] - 2023-03-09
- Configurable destinations for downloads
- Multiple destination directories can be configured
- When queueing a download, an alternate destination can be selected
- When downloading from a playlist, show the total number of videos and how many have been downloaded
- Show version in web UI
- Improve index page (show URL of queued downloads instead of nothing)

View File

@@ -57,6 +57,7 @@ type ConfigService struct {
func (cs *ConfigService) LoadTestConfig() {
cs.LoadDefaultConfig()
cs.Config.Server.DownloadPath = "/tmp"
cs.Config.DownloadProfiles = []DownloadProfile{{Name: "test profile", Command: "sleep", Args: []string{"5"}}}
}

View File

@@ -1,12 +1,14 @@
package download
import (
"encoding/json"
"fmt"
"io"
"log"
"net/url"
"os"
"os/exec"
"path/filepath"
"regexp"
"strconv"
"strings"
@@ -23,7 +25,7 @@ type Download struct {
PopupUrl string `json:"popup_url"`
Process *os.Process `json:"-"`
ExitCode int `json:"exit_code"`
State string `json:"state"`
State State `json:"state"`
DownloadProfile config.DownloadProfile `json:"download_profile"`
Destination *config.Destination `json:"destination"`
Finished bool `json:"finished"`
@@ -45,6 +47,32 @@ type Manager struct {
Lock sync.Mutex
}
func (m *Manager) String() string {
m.Lock.Lock()
defer m.Lock.Unlock()
out := fmt.Sprintf("Max per domain: %d, downloads: %d\n", m.MaxPerDomain, len(m.Downloads))
for _, dl := range m.Downloads {
out = out + fmt.Sprintf("%3d: (%10s) %30s\n", dl.Id, dl.State, dl.Url)
}
return out
}
type State string
const (
STATE_PREPARING State = "Preparing to start"
STATE_CHOOSE_PROFILE State = "Choose Profile"
STATE_QUEUED State = "Queued"
STATE_DOWNLOADING State = "Downloading"
STATE_DOWNLOADING_METADATA State = "Downloading metadata"
STATE_FAILED State = "Failed"
STATE_COMPLETE State = "Complete"
STATE_MOVED State = "Moved"
)
var CanStopDownload = false
var downloadId int32 = 0
@@ -54,6 +82,7 @@ func (m *Manager) ManageQueue() {
m.Lock.Lock()
m.startQueued(m.MaxPerDomain)
m.moveToDest()
// m.cleanup()
m.Lock.Unlock()
@@ -61,6 +90,44 @@ func (m *Manager) ManageQueue() {
}
}
func (m *Manager) DownloadsAsJSON() ([]byte, error) {
m.Lock.Lock()
defer m.Lock.Unlock()
for _, dl := range m.Downloads {
dl.Lock.Lock()
defer dl.Lock.Unlock()
}
b, err := json.Marshal(m.Downloads)
return b, err
}
func (m *Manager) moveToDest() {
// move any downloads that are complete and have a dest
for _, dl := range m.Downloads {
dl.Lock.Lock()
if dl.Destination != nil && dl.State == STATE_COMPLETE {
dl.State = STATE_MOVED
for _, fn := range dl.Files {
src := filepath.Join(dl.Config.Server.DownloadPath, fn)
dst := filepath.Join(dl.Destination.Path, fn)
err := os.Rename(src, dst)
if err != nil {
log.Printf("%s", err)
dl.Log = append(dl.Log, fmt.Sprintf("Could not move %s to %s - %s", fn, dl.Destination.Path, err))
break
} else {
dl.Log = append(dl.Log, fmt.Sprintf("Moved %s to %s", fn, dl.Destination.Path))
}
}
}
dl.Lock.Unlock()
}
}
// startQueued starts any downloads that have been queued, we would not exceed
// maxRunning. If maxRunning is 0, there is no limit.
func (m *Manager) startQueued(maxRunning int) {
@@ -70,7 +137,7 @@ func (m *Manager) startQueued(maxRunning int) {
for _, dl := range m.Downloads {
dl.Lock.Lock()
if dl.State == "downloading" || dl.State == "preparing to start" {
if dl.State == STATE_DOWNLOADING || dl.State == STATE_PREPARING {
active[dl.domain()]++
}
dl.Lock.Unlock()
@@ -81,8 +148,8 @@ func (m *Manager) startQueued(maxRunning int) {
dl.Lock.Lock()
if dl.State == "queued" && (maxRunning == 0 || active[dl.domain()] < maxRunning) {
dl.State = "preparing to start"
if dl.State == STATE_QUEUED && (maxRunning == 0 || active[dl.domain()] < maxRunning) {
dl.State = STATE_PREPARING
active[dl.domain()]++
log.Printf("Starting download for id:%d (%s)", dl.Id, dl.Url)
@@ -131,7 +198,18 @@ func (m *Manager) GetDlById(id int) (*Download, error) {
func (m *Manager) Queue(dl *Download) {
dl.Lock.Lock()
defer dl.Lock.Unlock()
dl.State = "queued"
dl.State = STATE_QUEUED
}
func (m *Manager) ChangeDestination(dl *Download, dest *config.Destination) {
dl.Lock.Lock()
// we can only change destination is certain cases...
if dl.State != STATE_FAILED && dl.State != STATE_MOVED {
dl.Destination = dest
}
dl.Lock.Unlock()
}
func NewDownload(url string, conf *config.Config) *Download {
@@ -140,7 +218,7 @@ func NewDownload(url string, conf *config.Config) *Download {
Id: int(downloadId),
Url: url,
PopupUrl: fmt.Sprintf("/fetch/%d", int(downloadId)),
State: "choose profile",
State: STATE_CHOOSE_PROFILE,
Files: []string{},
Log: []string{},
Config: conf,
@@ -193,7 +271,7 @@ func (dl *Download) domain() string {
func (dl *Download) Begin() {
dl.Lock.Lock()
dl.State = "downloading"
dl.State = STATE_DOWNLOADING
cmdSlice := []string{}
cmdSlice = append(cmdSlice, dl.DownloadProfile.Args...)
@@ -207,7 +285,7 @@ func (dl *Download) Begin() {
stdout, err := cmd.StdoutPipe()
if err != nil {
dl.State = "failed"
dl.State = STATE_FAILED
dl.Finished = true
dl.FinishedTS = time.Now()
dl.Log = append(dl.Log, fmt.Sprintf("error setting up stdout pipe: %v", err))
@@ -218,7 +296,7 @@ func (dl *Download) Begin() {
stderr, err := cmd.StderrPipe()
if err != nil {
dl.State = "failed"
dl.State = STATE_FAILED
dl.Finished = true
dl.FinishedTS = time.Now()
dl.Log = append(dl.Log, fmt.Sprintf("error setting up stderr pipe: %v", err))
@@ -230,7 +308,7 @@ func (dl *Download) Begin() {
log.Printf("Executing command: %v", cmd)
err = cmd.Start()
if err != nil {
dl.State = "failed"
dl.State = STATE_FAILED
dl.Finished = true
dl.FinishedTS = time.Now()
dl.Log = append(dl.Log, fmt.Sprintf("error starting command '%s': %v", dl.DownloadProfile.Command, err))
@@ -263,13 +341,13 @@ func (dl *Download) Begin() {
log.Printf("Process finished for id: %d (%v)", dl.Id, cmd)
dl.State = "complete"
dl.State = STATE_COMPLETE
dl.Finished = true
dl.FinishedTS = time.Now()
dl.ExitCode = cmd.ProcessState.ExitCode()
if dl.ExitCode != 0 {
dl.State = "failed"
dl.State = STATE_FAILED
}
dl.Lock.Unlock()
@@ -317,7 +395,7 @@ func (dl *Download) updateMetadata(s string) {
matches := etaRE.FindStringSubmatch(s)
if len(matches) == 2 {
dl.Eta = matches[1]
dl.State = "downloading"
dl.State = STATE_DOWNLOADING
}
@@ -378,7 +456,7 @@ func (dl *Download) updateMetadata(s string) {
metadataDL := regexp.MustCompile(`Downloading JSON metadata page (\d+)`)
matches = metadataDL.FindStringSubmatch(s)
if len(matches) == 2 {
dl.State = "Downloading metadata, page " + matches[1]
dl.State = STATE_DOWNLOADING_METADATA
}
// [FixupM3u8] Fixing MPEG-TS in MP4 container of "file [-168849776_456239489].mp4"

View File

@@ -0,0 +1,8 @@
//go:build !testdata
package download
import "github.com/tardisx/gropple/config"
func (m *Manager) AddStressTestData(c *config.ConfigService) {
}

View File

@@ -2,7 +2,11 @@ package download
import (
"strings"
"sync"
"testing"
"time"
"github.com/tardisx/gropple/config"
)
func TestUpdateMetadata(t *testing.T) {
@@ -76,79 +80,158 @@ func TestUpdateMetadata(t *testing.T) {
// [download] 100% of 4.64MiB in 00:00
// [ffmpeg] Merging formats into "Halo Infinite Flight 4K Gameplay-wi7Agv1M6PY.mp4"
// func TestQueue(t *testing.T) {
// cs := config.ConfigService{}
// cs.LoadTestConfig()
// conf := cs.Config
func TestQueue(t *testing.T) {
cs := config.ConfigService{}
cs.LoadTestConfig()
conf := cs.Config
// new1 := Download{Id: 1, Url: "http://sub.example.org/foo1", State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
// new2 := Download{Id: 2, Url: "http://sub.example.org/foo2", State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
// new3 := Download{Id: 3, Url: "http://sub.example.org/foo3", State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
// new4 := Download{Id: 4, Url: "http://example.org/", State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf}
new1 := NewDownload("http://sub.example.org/foo1", conf)
new2 := NewDownload("http://sub.example.org/foo2", conf)
new3 := NewDownload("http://sub.example.org/foo3", conf)
new4 := NewDownload("http://example.org/", conf)
// dls := Downloads{&new1, &new2, &new3, &new4}
// dls.StartQueued(1)
// time.Sleep(time.Millisecond * 100)
// if dls[0].State == "queued" {
// t.Error("#1 was not started")
// }
// if dls[1].State != "queued" {
// t.Error("#2 is not queued")
// }
// if dls[3].State == "queued" {
// t.Error("#4 is not started")
// }
// pretend the user chose a profile for each
new1.DownloadProfile = *conf.ProfileCalled("test profile")
new2.DownloadProfile = *conf.ProfileCalled("test profile")
new3.DownloadProfile = *conf.ProfileCalled("test profile")
new4.DownloadProfile = *conf.ProfileCalled("test profile")
new1.State = STATE_QUEUED
new2.State = STATE_QUEUED
new3.State = STATE_QUEUED
new4.State = STATE_QUEUED
// // this should start no more, as one is still going
// dls.StartQueued(1)
// time.Sleep(time.Millisecond * 100)
// if dls[1].State != "queued" {
// t.Error("#2 was started when it should not be")
// }
q := Manager{
Downloads: []*Download{},
MaxPerDomain: 2,
Lock: sync.Mutex{},
}
// dls.StartQueued(2)
// time.Sleep(time.Millisecond * 100)
// if dls[1].State == "queued" {
// t.Error("#2 was not started but it should be")
q.AddDownload(new1)
q.AddDownload(new2)
q.AddDownload(new3)
q.AddDownload(new4)
// }
q.startQueued(1)
// dls.StartQueued(2)
// time.Sleep(time.Millisecond * 100)
// if dls[3].State == "queued" {
// t.Error("#4 was not started but it should be")
// }
// two should start, one from each of the two domains
time.Sleep(time.Millisecond * 100)
if q.Downloads[0].State != STATE_DOWNLOADING {
t.Errorf("#1 was not downloading - %s instead ", q.Downloads[0].State)
t.Log(q.String())
}
if q.Downloads[1].State != STATE_QUEUED {
t.Errorf("#2 is not queued - %s instead", q.Downloads[1].State)
t.Log(q.String())
}
if q.Downloads[2].State != STATE_QUEUED {
t.Errorf("#3 is not queued - %s instead", q.Downloads[2].State)
t.Log(q.String())
}
if q.Downloads[3].State != STATE_DOWNLOADING {
t.Errorf("#4 is not downloading - %s instead", q.Downloads[3].State)
t.Log(q.String())
}
// // reset them all
// dls[0].State = "queued"
// dls[1].State = "queued"
// dls[2].State = "queued"
// dls[3].State = "queued"
// this should start no more, as one is still going
q.startQueued(1)
time.Sleep(time.Millisecond * 100)
if q.Downloads[0].State != STATE_DOWNLOADING {
t.Errorf("#1 was not downloading - %s instead ", q.Downloads[0].State)
t.Log(q.String())
}
if q.Downloads[1].State != STATE_QUEUED {
t.Errorf("#2 is not queued - %s instead", q.Downloads[1].State)
t.Log(q.String())
}
if q.Downloads[2].State != STATE_QUEUED {
t.Errorf("#3 is not queued - %s instead", q.Downloads[2].State)
t.Log(q.String())
}
if q.Downloads[3].State != STATE_DOWNLOADING {
t.Errorf("#4 is not downloading - %s instead", q.Downloads[3].State)
t.Log(q.String())
}
// dls.StartQueued(0)
// time.Sleep(time.Millisecond * 100)
// wait until the two finish, check
time.Sleep(time.Second * 5.0)
if q.Downloads[0].State != STATE_COMPLETE {
t.Errorf("#1 was not complete - %s instead ", q.Downloads[0].State)
t.Log(q.String())
}
if q.Downloads[1].State != STATE_QUEUED {
t.Errorf("#2 is not queued - %s instead", q.Downloads[1].State)
t.Log(q.String())
}
if q.Downloads[2].State != STATE_QUEUED {
t.Errorf("#3 is not queued - %s instead", q.Downloads[2].State)
t.Log(q.String())
}
if q.Downloads[3].State != STATE_COMPLETE {
t.Errorf("#4 is not complete - %s instead", q.Downloads[3].State)
t.Log(q.String())
}
// // they should all be going
// if dls[0].State == "queued" || dls[1].State == "queued" || dls[2].State == "queued" || dls[3].State == "queued" {
// t.Error("none should be queued")
// }
// this should start one more, as one is still going
q.startQueued(1)
time.Sleep(time.Millisecond * 100)
if q.Downloads[0].State != STATE_COMPLETE {
t.Errorf("#1 was not complete - %s instead ", q.Downloads[0].State)
t.Log(q.String())
}
if q.Downloads[1].State != STATE_DOWNLOADING {
t.Errorf("#2 is not downloading - %s instead", q.Downloads[1].State)
t.Log(q.String())
}
if q.Downloads[2].State != STATE_QUEUED {
t.Errorf("#3 is not queued - %s instead", q.Downloads[2].State)
t.Log(q.String())
}
if q.Downloads[3].State != STATE_COMPLETE {
t.Errorf("#4 is not complete - %s instead", q.Downloads[3].State)
t.Log(q.String())
}
// // reset them all
// dls[0].State = "queued"
// dls[1].State = "queued"
// dls[2].State = "queued"
// dls[3].State = "queued"
// this should start no more, as one is still going
q.startQueued(1)
time.Sleep(time.Millisecond * 100)
if q.Downloads[0].State != STATE_COMPLETE {
t.Errorf("#1 was not complete - %s instead ", q.Downloads[0].State)
t.Log(q.String())
}
if q.Downloads[1].State != STATE_DOWNLOADING {
t.Errorf("#2 is not downloading - %s instead", q.Downloads[1].State)
t.Log(q.String())
}
if q.Downloads[2].State != STATE_QUEUED {
t.Errorf("#3 is not queued - %s instead", q.Downloads[2].State)
t.Log(q.String())
}
if q.Downloads[3].State != STATE_COMPLETE {
t.Errorf("#4 is not complete - %s instead", q.Downloads[3].State)
t.Log(q.String())
}
// dls.StartQueued(2)
// time.Sleep(time.Millisecond * 100)
// but if we allow two per domain, the other queued one will start
q.startQueued(2)
time.Sleep(time.Millisecond * 100)
if q.Downloads[0].State != STATE_COMPLETE {
t.Errorf("#1 was not complete - %s instead ", q.Downloads[0].State)
t.Log(q.String())
}
if q.Downloads[1].State != STATE_DOWNLOADING {
t.Errorf("#2 is not downloading - %s instead", q.Downloads[1].State)
t.Log(q.String())
}
if q.Downloads[2].State != STATE_DOWNLOADING {
t.Errorf("#3 is not downloading - %s instead", q.Downloads[2].State)
t.Log(q.String())
}
if q.Downloads[3].State != STATE_COMPLETE {
t.Errorf("#4 is not complete - %s instead", q.Downloads[3].State)
t.Log(q.String())
}
// // first two should be running, third not (same domain) and 4th running (different domain)
// if dls[0].State == "queued" || dls[1].State == "queued" || dls[2].State != "queued" || dls[3].State == "queued" {
// t.Error("incorrect queued")
// }
// }
}
func TestUpdateMetadataPlaylist(t *testing.T) {

View File

@@ -0,0 +1,28 @@
//go:build testdata
package download
import "github.com/tardisx/gropple/config"
func (m *Manager) AddStressTestData(c *config.ConfigService) {
urls := []string{
"https://www.youtube.com/watch?v=qG_rRkuGBW8",
"https://www.youtube.com/watch?v=ZUzhZpQAU40",
"https://www.youtube.com/watch?v=kVxM3eRWGak",
"https://www.youtube.com/watch?v=pl-y9869y0w",
"https://vimeo.com/783453809",
"https://www.youtube.com/watch?v=Uw4NEPE4l3A",
"https://www.youtube.com/watch?v=2RF0lcTuuYE",
"https://www.youtube.com/watch?v=lymwNQY0dus",
"https://www.youtube.com/watch?v=NTc-I4Z_duc",
"https://www.youtube.com/watch?v=wNSm1TJ84Ac",
"https://vimeo.com/786570322",
}
for _, u := range urls {
d := NewDownload(u, c.Config)
d.DownloadProfile = *c.Config.ProfileCalled("standard video")
m.AddDownload(d)
m.Queue(d)
}
}

41
main.go
View File

@@ -21,11 +21,10 @@ import (
)
var dm *download.Manager
var downloadId = 0
var configService *config.ConfigService
var versionInfo = version.Manager{
VersionInfo: version.Info{CurrentVersion: "v0.6.0-alpha.1"},
VersionInfo: version.Info{CurrentVersion: "v0.6.0-alpha.3"},
}
//go:embed web
@@ -110,25 +109,7 @@ func main() {
// start downloading queued downloads when slots available, and clean up
// old entries
go dm.ManageQueue()
urls := []string{
"https://www.youtube.com/watch?v=qG_rRkuGBW8",
"https://www.youtube.com/watch?v=ZUzhZpQAU40",
// "https://www.youtube.com/watch?v=kVxM3eRWGak",
// "https://www.youtube.com/watch?v=pl-y9869y0w",
// "https://www.youtube.com/watch?v=Uw4NEPE4l3A",
// "https://www.youtube.com/watch?v=6tIsT57_nS0",
// "https://www.youtube.com/watch?v=2RF0lcTuuYE",
// "https://www.youtube.com/watch?v=lymwNQY0dus",
// "https://www.youtube.com/watch?v=NTc-I4Z_duc",
// "https://www.youtube.com/watch?v=wNSm1TJ84Ac",
}
for _, u := range urls {
d := download.NewDownload(u, configService.Config)
d.DownloadProfile = *configService.Config.ProfileCalled("standard video")
dm.AddDownload(d)
dm.Queue(d)
}
dm.AddStressTestData(configService)
log.Printf("Visit %s for details on installing the bookmarklet and to check status", configService.Config.Server.Address)
log.Fatal(srv.ListenAndServe())
@@ -303,12 +284,9 @@ func fetchInfoOneRESTHandler(w http.ResponseWriter, r *http.Request) {
// nil means (probably) that they chose "don't move" - which is fine,
// and maps to nil on the Download (the default state).
destination := configService.Config.DestinationCalled(thisReq.Destination)
dm.ChangeDestination(thisDownload, destination)
thisDownload.Lock.Lock()
thisDownload.Destination = destination
thisDownload.Lock.Unlock()
log.Printf("%#v", thisDownload)
// log.Printf("%#v", thisDownload)
succRes := successResponse{Success: true, Message: "destination changed"}
succResB, _ := json.Marshal(succRes)
@@ -327,7 +305,11 @@ func fetchInfoOneRESTHandler(w http.ResponseWriter, r *http.Request) {
}
// just a get, return the object
thisDownload.Lock.Lock()
defer thisDownload.Lock.Unlock()
b, _ := json.Marshal(thisDownload)
w.Write(b)
return
} else {
@@ -337,9 +319,10 @@ func fetchInfoOneRESTHandler(w http.ResponseWriter, r *http.Request) {
func fetchInfoRESTHandler(w http.ResponseWriter, r *http.Request) {
dm.Lock.Lock()
defer dm.Lock.Unlock()
b, _ := json.Marshal(dm.Downloads)
b, err := dm.DownloadsAsJSON()
if err != nil {
panic(err)
}
w.Write(b)
}

View File

@@ -33,6 +33,9 @@
.state-downloading {
color: blue;
}
.state-moved {
color: green;
}
.state-complete {
color: green;
}

View File

@@ -101,7 +101,11 @@
this.state = info.state;
this.playlist_current = info.playlist_current;
this.playlist_total = info.playlist_total;
if (this.state != 'choose profile') {
this.destination_chosen = null;
if (info.destination) {
this.destination_chosen = info.destination.name;
}
if (this.state != 'Choose Profile') {
this.profile_chosen = true;
}
this.finished = info.finished;