diff --git a/config/config.go b/config/config.go index c00604c..a327b13 100644 --- a/config/config.go +++ b/config/config.go @@ -65,6 +65,15 @@ func DefaultConfig() *Config { return &defaultConfig } +func (c *Config) ProfileCalled(name string) *DownloadProfile { + for _, p := range c.DownloadProfiles { + if p.Name == name { + return &p + } + } + return nil +} + func (c *Config) UpdateFromJSON(j []byte) error { newConfig := Config{} err := json.Unmarshal(j, &newConfig) diff --git a/download/download.go b/download/download.go new file mode 100644 index 0000000..fbf75b6 --- /dev/null +++ b/download/download.go @@ -0,0 +1,172 @@ +package download + +import ( + "fmt" + "io" + "os/exec" + "regexp" + "strconv" + "strings" + "sync" + + "github.com/tardisx/gropple/config" +) + +type Download struct { + Id int `json:"id"` + Url string `json:"url"` + Pid int `json:"pid"` + ExitCode int `json:"exit_code"` + State string `json:"state"` + DownloadProfile config.DownloadProfile `json:"download_profile"` + Finished bool `json:"finished"` + Files []string `json:"files"` + Eta string `json:"eta"` + Percent float32 `json:"percent"` + Log []string `json:"log"` + Config *config.Config +} + +// Begin starts a download, by starting the command specified in the DownloadProfile. +// It blocks until the download is complete. +func (dl *Download) Begin() { + cmdSlice := []string{} + cmdSlice = append(cmdSlice, dl.DownloadProfile.Args...) + cmdSlice = append(cmdSlice, dl.Url) + + cmd := exec.Command(dl.DownloadProfile.Command, cmdSlice...) + cmd.Dir = dl.Config.Server.DownloadPath + + stdout, err := cmd.StdoutPipe() + if err != nil { + dl.State = "failed" + dl.Finished = true + dl.Log = append(dl.Log, fmt.Sprintf("error setting up stdout pipe: %v", err)) + return + } + + stderr, err := cmd.StderrPipe() + if err != nil { + dl.State = "failed" + dl.Finished = true + dl.Log = append(dl.Log, fmt.Sprintf("error setting up stderr pipe: %v", err)) + return + } + + err = cmd.Start() + if err != nil { + dl.State = "failed" + dl.Finished = true + dl.Log = append(dl.Log, fmt.Sprintf("error starting youtube-dl: %v", err)) + return + } + dl.Pid = cmd.Process.Pid + + var wg sync.WaitGroup + + wg.Add(2) + go func() { + defer wg.Done() + dl.updateDownload(stdout) + }() + + go func() { + defer wg.Done() + dl.updateDownload(stderr) + }() + + wg.Wait() + cmd.Wait() + + dl.State = "complete" + dl.Finished = true + dl.ExitCode = cmd.ProcessState.ExitCode() + + if dl.ExitCode != 0 { + dl.State = "failed" + } + +} + +func (dl *Download) updateDownload(r io.Reader) { + // XXX not sure if we might get a partial line? + buf := make([]byte, 1024) + for { + n, err := r.Read(buf) + if n > 0 { + s := string(buf[:n]) + lines := strings.Split(s, "\n") + + for _, l := range lines { + + if l == "" { + continue + } + + // append the raw log + dl.Log = append(dl.Log, l) + + // look for the percent and eta and other metadata + dl.updateMetadata(l) + } + } + if err != nil { + break + } + } +} + +func (dl *Download) updateMetadata(s string) { + + // [download] 49.7% of ~15.72MiB at 5.83MiB/s ETA 00:07 + etaRE := regexp.MustCompile(`download.+ETA +(\d\d:\d\d)`) + matches := etaRE.FindStringSubmatch(s) + if len(matches) == 2 { + dl.Eta = matches[1] + dl.State = "downloading" + + } + + percentRE := regexp.MustCompile(`download.+?([\d\.]+)%`) + matches = percentRE.FindStringSubmatch(s) + if len(matches) == 2 { + p, err := strconv.ParseFloat(matches[1], 32) + if err == nil { + dl.Percent = float32(p) + } else { + panic(err) + } + } + + // This appears once per destination file + // [download] Destination: Filename with spaces and other punctuation here be careful!.mp4 + filename := regexp.MustCompile(`download.+?Destination: (.+)$`) + matches = filename.FindStringSubmatch(s) + if len(matches) == 2 { + dl.Files = append(dl.Files, matches[1]) + } + + // This means a file has been "created" by merging others + // [ffmpeg] Merging formats into "Toto - Africa (Official HD Video)-FTQbiNvZqaY.mp4" + mergedFilename := regexp.MustCompile(`Merging formats into "(.+)"$`) + matches = mergedFilename.FindStringSubmatch(s) + if len(matches) == 2 { + dl.Files = append(dl.Files, matches[1]) + } + + // This means a file has been deleted + // Gross - this time it's unquoted and has trailing guff + // Deleting original file Toto - Africa (Official HD Video)-FTQbiNvZqaY.f137.mp4 (pass -k to keep) + // This is very fragile + deletedFile := regexp.MustCompile(`Deleting original file (.+) \(pass -k to keep\)$`) + matches = deletedFile.FindStringSubmatch(s) + if len(matches) == 2 { + // find the index + for i, f := range dl.Files { + if f == matches[1] { + dl.Files = append(dl.Files[:i], dl.Files[i+1:]...) + break + } + } + } +} diff --git a/main.go b/main.go index a0812fa..7c0f2f7 100644 --- a/main.go +++ b/main.go @@ -8,41 +8,35 @@ import ( "io" "log" "net/http" - "os/exec" - "regexp" "strings" - "sync" "time" "strconv" "github.com/gorilla/mux" "github.com/tardisx/gropple/config" + "github.com/tardisx/gropple/download" "github.com/tardisx/gropple/version" ) -type download struct { - Id int `json:"id"` - Url string `json:"url"` - Pid int `json:"pid"` - ExitCode int `json:"exit_code"` - State string `json:"state"` - Finished bool `json:"finished"` - Files []string `json:"files"` - Eta string `json:"eta"` - Percent float32 `json:"percent"` - Log []string `json:"log"` -} - -var downloads []*download +var downloads []*download.Download var downloadId = 0 +var conf *config.Config var versionInfo = version.Info{CurrentVersion: "v0.5.0"} //go:embed web var webFS embed.FS -var conf *config.Config +type successResponse struct { + Success bool `json:"success"` + Message string `json:"message"` +} + +type errorResponse struct { + Success bool `json:"success"` + Error string `json:"error"` +} func main() { if !config.ConfigFileExists() { @@ -58,14 +52,16 @@ func main() { } r := mux.NewRouter() - r.HandleFunc("/", HomeHandler) - r.HandleFunc("/config", ConfigHandler) - r.HandleFunc("/fetch", FetchHandler) + r.HandleFunc("/", homeHandler) + r.HandleFunc("/config", configHandler) + r.HandleFunc("/fetch", fetchHandler) - r.HandleFunc("/rest/fetch/info", FetchInfoHandler) - r.HandleFunc("/rest/fetch/info/{id}", FetchInfoOneHandler) - r.HandleFunc("/rest/version", VersionHandler) - r.HandleFunc("/rest/config", ConfigRESTHandler) + // info for the list + r.HandleFunc("/rest/fetch", fetchInfoRESTHandler) + // info for one, including update + r.HandleFunc("/rest/fetch/{id}", fetchInfoOneRESTHandler) + r.HandleFunc("/rest/version", versionRESTHandler) + r.HandleFunc("/rest/config", configRESTHandler) http.Handle("/", r) @@ -90,7 +86,8 @@ func main() { log.Fatal(srv.ListenAndServe()) } -func VersionHandler(w http.ResponseWriter, r *http.Request) { +// versionRESTHandler returns the version information, if we have up-to-date info from github +func versionRESTHandler(w http.ResponseWriter, r *http.Request) { if versionInfo.GithubVersionFetched { b, _ := json.Marshal(versionInfo) w.Write(b) @@ -99,7 +96,8 @@ func VersionHandler(w http.ResponseWriter, r *http.Request) { } } -func HomeHandler(w http.ResponseWriter, r *http.Request) { +// homeHandler returns the main index page +func homeHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) bookmarkletURL := fmt.Sprintf("javascript:(function(f,s,n,o){window.open(f+encodeURIComponent(s),n,o)}('%s/fetch?url=',window.location,'yourform','width=%d,height=%d'));", conf.Server.Address, conf.UI.PopupWidth, conf.UI.PopupHeight) @@ -110,7 +108,7 @@ func HomeHandler(w http.ResponseWriter, r *http.Request) { } type Info struct { - Downloads []*download + Downloads []*download.Download BookmarkletURL template.URL } @@ -123,10 +121,10 @@ func HomeHandler(w http.ResponseWriter, r *http.Request) { if err != nil { panic(err) } - } -func ConfigHandler(w http.ResponseWriter, r *http.Request) { +// configHandler returns the configuration page +func configHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/menu.tmpl", "web/config.html") @@ -140,11 +138,8 @@ func ConfigHandler(w http.ResponseWriter, r *http.Request) { } } -func ConfigRESTHandler(w http.ResponseWriter, r *http.Request) { - - type errorResponse struct { - Error string `json:"error"` - } +// configRESTHandler handles both reading and writing of the configuration +func configRESTHandler(w http.ResponseWriter, r *http.Request) { if r.Method == "POST" { log.Printf("Updating config") @@ -155,7 +150,7 @@ func ConfigRESTHandler(w http.ResponseWriter, r *http.Request) { err = conf.UpdateFromJSON(b) if err != nil { - errorRes := errorResponse{Error: err.Error()} + errorRes := errorResponse{Success: false, Error: err.Error()} errorResB, _ := json.Marshal(errorRes) w.WriteHeader(400) w.Write(errorResB) @@ -167,7 +162,8 @@ func ConfigRESTHandler(w http.ResponseWriter, r *http.Request) { w.Write(b) } -func FetchInfoOneHandler(w http.ResponseWriter, r *http.Request) { +// +func fetchInfoOneRESTHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) idString := vars["id"] if idString != "" { @@ -177,24 +173,74 @@ func FetchInfoOneHandler(w http.ResponseWriter, r *http.Request) { return } + // find the download + var thisDownload *download.Download for _, dl := range downloads { if dl.Id == id { - b, _ := json.Marshal(dl) - w.Write(b) + thisDownload = dl + } + } + if thisDownload == nil { + http.NotFound(w, r) + return + } + + if r.Method == "POST" { + log.Printf("Updating download") + + type updateRequest struct { + Action string `json:"action"` + Profile string `json:"profile"` + } + + thisReq := updateRequest{} + + b, err := io.ReadAll(r.Body) + if err != nil { + panic(err) + } + + err = json.Unmarshal(b, &thisReq) + if err != nil { + errorRes := errorResponse{Success: false, Error: err.Error()} + errorResB, _ := json.Marshal(errorRes) + w.WriteHeader(400) + w.Write(errorResB) + return + } + + if thisReq.Action == "start" { + // find the profile they asked for + profile := conf.ProfileCalled(thisReq.Profile) + if profile == nil { + panic("bad profile name?") + } + // set the profile + thisDownload.DownloadProfile = *profile + + go func() { thisDownload.Begin() }() + succRes := successResponse{Success: true, Message: "download started"} + succResB, _ := json.Marshal(succRes) + w.Write(succResB) return } } + + // just a get, return the object + b, _ := json.Marshal(thisDownload) + w.Write(b) + return } else { http.NotFound(w, r) } } -func FetchInfoHandler(w http.ResponseWriter, r *http.Request) { +func fetchInfoRESTHandler(w http.ResponseWriter, r *http.Request) { b, _ := json.Marshal(downloads) w.Write(b) } -func FetchHandler(w http.ResponseWriter, r *http.Request) { +func fetchHandler(w http.ResponseWriter, r *http.Request) { query := r.URL.Query() url, present := query["url"] @@ -215,10 +261,12 @@ func FetchHandler(w http.ResponseWriter, r *http.Request) { // create the record // XXX should be atomic! downloadId++ - newDownload := download{ + newDownload := download.Download{ + Config: conf, + Id: downloadId, Url: url[0], - State: "starting", + State: "choose profile", Finished: false, Eta: "?", Percent: 0.0, @@ -229,160 +277,20 @@ func FetchHandler(w http.ResponseWriter, r *http.Request) { newDownload.Log = append(newDownload.Log, "start of log...") - go func() { - queue(&newDownload) - }() + // go func() { + // newDownload.Begin() + // }() t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/popup.html") if err != nil { panic(err) } - err = t.ExecuteTemplate(w, "layout", newDownload) + + templateData := map[string]interface{}{"dl": newDownload, "config": conf} + + err = t.ExecuteTemplate(w, "layout", templateData) if err != nil { panic(err) } } } - -func queue(dl *download) { - cmdSlice := []string{} - cmdSlice = append(cmdSlice, conf.DownloadProfiles[0].Args...) - cmdSlice = append(cmdSlice, dl.Url) - - cmd := exec.Command(conf.DownloadProfiles[0].Command, cmdSlice...) - cmd.Dir = conf.Server.DownloadPath - - stdout, err := cmd.StdoutPipe() - if err != nil { - dl.State = "failed" - dl.Finished = true - dl.Log = append(dl.Log, fmt.Sprintf("error setting up stdout pipe: %v", err)) - return - } - - stderr, err := cmd.StderrPipe() - if err != nil { - dl.State = "failed" - dl.Finished = true - dl.Log = append(dl.Log, fmt.Sprintf("error setting up stderr pipe: %v", err)) - return - } - - err = cmd.Start() - if err != nil { - dl.State = "failed" - dl.Finished = true - dl.Log = append(dl.Log, fmt.Sprintf("error starting youtube-dl: %v", err)) - return - } - dl.Pid = cmd.Process.Pid - - var wg sync.WaitGroup - - wg.Add(2) - go func() { - defer wg.Done() - updateDownload(stdout, dl) - }() - - go func() { - defer wg.Done() - updateDownload(stderr, dl) - }() - - wg.Wait() - cmd.Wait() - - dl.State = "complete" - dl.Finished = true - dl.ExitCode = cmd.ProcessState.ExitCode() - - if dl.ExitCode != 0 { - dl.State = "failed" - } - -} - -func updateDownload(r io.Reader, dl *download) { - // XXX not sure if we might get a partial line? - buf := make([]byte, 1024) - for { - n, err := r.Read(buf) - if n > 0 { - s := string(buf[:n]) - lines := strings.Split(s, "\n") - - for _, l := range lines { - - if l == "" { - continue - } - - // append the raw log - dl.Log = append(dl.Log, l) - - // look for the percent and eta and other metadata - updateMetadata(dl, l) - } - } - if err != nil { - break - } - } -} - -func updateMetadata(dl *download, s string) { - - // [download] 49.7% of ~15.72MiB at 5.83MiB/s ETA 00:07 - etaRE := regexp.MustCompile(`download.+ETA +(\d\d:\d\d)`) - matches := etaRE.FindStringSubmatch(s) - if len(matches) == 2 { - dl.Eta = matches[1] - dl.State = "downloading" - - } - - percentRE := regexp.MustCompile(`download.+?([\d\.]+)%`) - matches = percentRE.FindStringSubmatch(s) - if len(matches) == 2 { - p, err := strconv.ParseFloat(matches[1], 32) - if err == nil { - dl.Percent = float32(p) - } else { - panic(err) - } - } - - // This appears once per destination file - // [download] Destination: Filename with spaces and other punctuation here be careful!.mp4 - filename := regexp.MustCompile(`download.+?Destination: (.+)$`) - matches = filename.FindStringSubmatch(s) - if len(matches) == 2 { - dl.Files = append(dl.Files, matches[1]) - } - - // This means a file has been "created" by merging others - // [ffmpeg] Merging formats into "Toto - Africa (Official HD Video)-FTQbiNvZqaY.mp4" - mergedFilename := regexp.MustCompile(`Merging formats into "(.+)"$`) - matches = mergedFilename.FindStringSubmatch(s) - if len(matches) == 2 { - dl.Files = append(dl.Files, matches[1]) - } - - // This means a file has been deleted - // Gross - this time it's unquoted and has trailing guff - // Deleting original file Toto - Africa (Official HD Video)-FTQbiNvZqaY.f137.mp4 (pass -k to keep) - // This is very fragile - deletedFile := regexp.MustCompile(`Deleting original file (.+) \(pass -k to keep\)$`) - matches = deletedFile.FindStringSubmatch(s) - if len(matches) == 2 { - // find the index - for i, f := range dl.Files { - if f == matches[1] { - dl.Files = append(dl.Files[:i], dl.Files[i+1:]...) - break - } - } - } - -} diff --git a/web/config.html b/web/config.html index ad08c05..2fb3558 100644 --- a/web/config.html +++ b/web/config.html @@ -131,9 +131,9 @@ save_config() { let op = { method: 'POST', - body: JSON.stringify(this.config), - headers: { 'Content-Type': 'application/json' } - } + body: JSON.stringify(this.config), + headers: { 'Content-Type': 'application/json' } + }; fetch('/rest/config', op) .then(response => { return response.json(); diff --git a/web/popup.html b/web/popup.html index b100f07..99801b2 100644 --- a/web/popup.html +++ b/web/popup.html @@ -1,8 +1,19 @@ {{ define "content" }} -
+

Download started

-

Fetching {{ .Url }}

+

Fetching {{ .dl.Url }}

+ + + + @@ -21,8 +32,26 @@ function popup() { return { eta: '', percent: 0.0, state: '??', filename: '', finished: false, log :'', + profile_chosen: null, + watch_profile() { + console.log('will wtch profile'); + this.$watch('profile_chosen', value => this.profile_chosen(value)) + }, + update_profile(name) { + console.log('you chose name', this.profile_chosen); + let op = { + method: 'POST', + body: JSON.stringify({action: 'start', profile: this.profile_chosen}), + headers: { 'Content-Type': 'application/json' } + }; + fetch('/rest/fetch/{{ .dl.Id }}', op) + .then(response => response.json()) + .then(info => { + console.log(info) + }) + }, fetch_data() { - fetch('/rest/fetch/info/{{ .Id }}') + fetch('/rest/fetch/{{ .dl.Id }}') .then(response => response.json()) .then(info => { this.eta = info.eta;
profile + +
current filename
state
progress