justin/gropple
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases 24 Wiki Activity

Allow downloads to be created, prompt the user for a profile.

Browse Source
This commit is contained in:
Justin Hawkins 2021-09-30 23:48:56 +09:30
parent d47e2af2a4
commit fc0d6a32c3
5 changed files with 314 additions and 196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
config/config.go
View File

@ -65,6 +65,15 @@ func DefaultConfig() *Config {
return &defaultConfig
}
func (c *Config) ProfileCalled(name string) *DownloadProfile {
for _, p := range c.DownloadProfiles {
if p.Name == name {
return &p
}
}
return nil
}
func (c *Config) UpdateFromJSON(j []byte) error {
newConfig := Config{}
err := json.Unmarshal(j, &newConfig)

172
download/download.go Normal file
View File

@ -0,0 +1,172 @@
package download
import (
"fmt"
"io"
"os/exec"
"regexp"
"strconv"
"strings"
"sync"
"github.com/tardisx/gropple/config"
)
type Download struct {
Id int `json:"id"`
Url string `json:"url"`
Pid int `json:"pid"`
ExitCode int `json:"exit_code"`
State string `json:"state"`
DownloadProfile config.DownloadProfile `json:"download_profile"`
Finished bool `json:"finished"`
Files []string `json:"files"`
Eta string `json:"eta"`
Percent float32 `json:"percent"`
Log []string `json:"log"`
Config *config.Config
}
// Begin starts a download, by starting the command specified in the DownloadProfile.
// It blocks until the download is complete.
func (dl *Download) Begin() {
cmdSlice := []string{}
cmdSlice = append(cmdSlice, dl.DownloadProfile.Args...)
cmdSlice = append(cmdSlice, dl.Url)
cmd := exec.Command(dl.DownloadProfile.Command, cmdSlice...)
cmd.Dir = dl.Config.Server.DownloadPath
stdout, err := cmd.StdoutPipe()
if err != nil {
dl.State = "failed"
dl.Finished = true
dl.Log = append(dl.Log, fmt.Sprintf("error setting up stdout pipe: %v", err))
return
}
stderr, err := cmd.StderrPipe()
if err != nil {
dl.State = "failed"
dl.Finished = true
dl.Log = append(dl.Log, fmt.Sprintf("error setting up stderr pipe: %v", err))
return
}
err = cmd.Start()
if err != nil {
dl.State = "failed"
dl.Finished = true
dl.Log = append(dl.Log, fmt.Sprintf("error starting youtube-dl: %v", err))
return
}
dl.Pid = cmd.Process.Pid
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
dl.updateDownload(stdout)
}()
go func() {
defer wg.Done()
dl.updateDownload(stderr)
}()
wg.Wait()
cmd.Wait()
dl.State = "complete"
dl.Finished = true
dl.ExitCode = cmd.ProcessState.ExitCode()
if dl.ExitCode != 0 {
dl.State = "failed"
}
}
func (dl *Download) updateDownload(r io.Reader) {
// XXX not sure if we might get a partial line?
buf := make([]byte, 1024)
for {
n, err := r.Read(buf)
if n > 0 {
s := string(buf[:n])
lines := strings.Split(s, "\n")
for _, l := range lines {
if l == "" {
continue
}
// append the raw log
dl.Log = append(dl.Log, l)
// look for the percent and eta and other metadata
dl.updateMetadata(l)
}
}
if err != nil {
break
}
}
}
func (dl *Download) updateMetadata(s string) {
// [download] 49.7% of ~15.72MiB at 5.83MiB/s ETA 00:07
etaRE := regexp.MustCompile(`download.+ETA +(\d\d:\d\d)`)
matches := etaRE.FindStringSubmatch(s)
if len(matches) == 2 {
dl.Eta = matches[1]
dl.State = "downloading"
}
percentRE := regexp.MustCompile(`download.+?([\d\.]+)%`)
matches = percentRE.FindStringSubmatch(s)
if len(matches) == 2 {
p, err := strconv.ParseFloat(matches[1], 32)
if err == nil {
dl.Percent = float32(p)
} else {
panic(err)
}
}
// This appears once per destination file
// [download] Destination: Filename with spaces and other punctuation here be careful!.mp4
filename := regexp.MustCompile(`download.+?Destination: (.+)$`)
matches = filename.FindStringSubmatch(s)
if len(matches) == 2 {
dl.Files = append(dl.Files, matches[1])
}
// This means a file has been "created" by merging others
// [ffmpeg] Merging formats into "Toto - Africa (Official HD Video)-FTQbiNvZqaY.mp4"
mergedFilename := regexp.MustCompile(`Merging formats into "(.+)"$`)
matches = mergedFilename.FindStringSubmatch(s)
if len(matches) == 2 {
dl.Files = append(dl.Files, matches[1])
}
// This means a file has been deleted
// Gross - this time it's unquoted and has trailing guff
// Deleting original file Toto - Africa (Official HD Video)-FTQbiNvZqaY.f137.mp4 (pass -k to keep)
// This is very fragile
deletedFile := regexp.MustCompile(`Deleting original file (.+) \(pass -k to keep\)$`)
matches = deletedFile.FindStringSubmatch(s)
if len(matches) == 2 {
// find the index
for i, f := range dl.Files {
if f == matches[1] {
dl.Files = append(dl.Files[:i], dl.Files[i+1:]...)
break
}
}
}
}

288
main.go
View File

@ -8,41 +8,35 @@ import (
"io"
"log"
"net/http"
"os/exec"
"regexp"
"strings"
"sync"
"time"
"strconv"
"github.com/gorilla/mux"
"github.com/tardisx/gropple/config"
"github.com/tardisx/gropple/download"
"github.com/tardisx/gropple/version"
)
type download struct {
Id int `json:"id"`
Url string `json:"url"`
Pid int `json:"pid"`
ExitCode int `json:"exit_code"`
State string `json:"state"`
Finished bool `json:"finished"`
Files []string `json:"files"`
Eta string `json:"eta"`
Percent float32 `json:"percent"`
Log []string `json:"log"`
}
var downloads []*download
var downloads []*download.Download
var downloadId = 0
var conf *config.Config
var versionInfo = version.Info{CurrentVersion: "v0.5.0"}
//go:embed web
var webFS embed.FS
var conf *config.Config
type successResponse struct {
Success bool `json:"success"`
Message string `json:"message"`
}
type errorResponse struct {
Success bool `json:"success"`
Error string `json:"error"`
}
func main() {
if !config.ConfigFileExists() {
@ -58,14 +52,16 @@ func main() {
}
r := mux.NewRouter()
r.HandleFunc("/", HomeHandler)
r.HandleFunc("/config", ConfigHandler)
r.HandleFunc("/fetch", FetchHandler)
r.HandleFunc("/", homeHandler)
r.HandleFunc("/config", configHandler)
r.HandleFunc("/fetch", fetchHandler)
r.HandleFunc("/rest/fetch/info", FetchInfoHandler)
r.HandleFunc("/rest/fetch/info/{id}", FetchInfoOneHandler)
r.HandleFunc("/rest/version", VersionHandler)
r.HandleFunc("/rest/config", ConfigRESTHandler)
// info for the list
r.HandleFunc("/rest/fetch", fetchInfoRESTHandler)
// info for one, including update
r.HandleFunc("/rest/fetch/{id}", fetchInfoOneRESTHandler)
r.HandleFunc("/rest/version", versionRESTHandler)
r.HandleFunc("/rest/config", configRESTHandler)
http.Handle("/", r)
@ -90,7 +86,8 @@ func main() {
log.Fatal(srv.ListenAndServe())
}
func VersionHandler(w http.ResponseWriter, r *http.Request) {
// versionRESTHandler returns the version information, if we have up-to-date info from github
func versionRESTHandler(w http.ResponseWriter, r *http.Request) {
if versionInfo.GithubVersionFetched {
b, _ := json.Marshal(versionInfo)
w.Write(b)
@ -99,7 +96,8 @@ func VersionHandler(w http.ResponseWriter, r *http.Request) {
}
}
func HomeHandler(w http.ResponseWriter, r *http.Request) {
// homeHandler returns the main index page
func homeHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
bookmarkletURL := fmt.Sprintf("javascript:(function(f,s,n,o){window.open(f+encodeURIComponent(s),n,o)}('%s/fetch?url=',window.location,'yourform','width=%d,height=%d'));", conf.Server.Address, conf.UI.PopupWidth, conf.UI.PopupHeight)
@ -110,7 +108,7 @@ func HomeHandler(w http.ResponseWriter, r *http.Request) {
}
type Info struct {
Downloads []*download
Downloads []*download.Download
BookmarkletURL template.URL
}
@ -123,10 +121,10 @@ func HomeHandler(w http.ResponseWriter, r *http.Request) {
if err != nil {
panic(err)
}
}
func ConfigHandler(w http.ResponseWriter, r *http.Request) {
// configHandler returns the configuration page
func configHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/menu.tmpl", "web/config.html")
@ -140,11 +138,8 @@ func ConfigHandler(w http.ResponseWriter, r *http.Request) {
}
}
func ConfigRESTHandler(w http.ResponseWriter, r *http.Request) {
type errorResponse struct {
Error string `json:"error"`
}
// configRESTHandler handles both reading and writing of the configuration
func configRESTHandler(w http.ResponseWriter, r *http.Request) {
if r.Method == "POST" {
log.Printf("Updating config")
@ -155,7 +150,7 @@ func ConfigRESTHandler(w http.ResponseWriter, r *http.Request) {
err = conf.UpdateFromJSON(b)
if err != nil {
errorRes := errorResponse{Error: err.Error()}
errorRes := errorResponse{Success: false, Error: err.Error()}
errorResB, _ := json.Marshal(errorRes)
w.WriteHeader(400)
w.Write(errorResB)
@ -167,7 +162,8 @@ func ConfigRESTHandler(w http.ResponseWriter, r *http.Request) {
w.Write(b)
}
func FetchInfoOneHandler(w http.ResponseWriter, r *http.Request) {
//
func fetchInfoOneRESTHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
idString := vars["id"]
if idString != "" {
@ -177,24 +173,74 @@ func FetchInfoOneHandler(w http.ResponseWriter, r *http.Request) {
return
}
// find the download
var thisDownload *download.Download
for _, dl := range downloads {
if dl.Id == id {
b, _ := json.Marshal(dl)
w.Write(b)
thisDownload = dl
}
}
if thisDownload == nil {
http.NotFound(w, r)
return
}
if r.Method == "POST" {
log.Printf("Updating download")
type updateRequest struct {
Action string `json:"action"`
Profile string `json:"profile"`
}
thisReq := updateRequest{}
b, err := io.ReadAll(r.Body)
if err != nil {
panic(err)
}
err = json.Unmarshal(b, &thisReq)
if err != nil {
errorRes := errorResponse{Success: false, Error: err.Error()}
errorResB, _ := json.Marshal(errorRes)
w.WriteHeader(400)
w.Write(errorResB)
return
}
if thisReq.Action == "start" {
// find the profile they asked for
profile := conf.ProfileCalled(thisReq.Profile)
if profile == nil {
panic("bad profile name?")
}
// set the profile
thisDownload.DownloadProfile = *profile
go func() { thisDownload.Begin() }()
succRes := successResponse{Success: true, Message: "download started"}
succResB, _ := json.Marshal(succRes)
w.Write(succResB)
return
}
}
// just a get, return the object
b, _ := json.Marshal(thisDownload)
w.Write(b)
return
} else {
http.NotFound(w, r)
}
}
func FetchInfoHandler(w http.ResponseWriter, r *http.Request) {
func fetchInfoRESTHandler(w http.ResponseWriter, r *http.Request) {
b, _ := json.Marshal(downloads)
w.Write(b)
}
func FetchHandler(w http.ResponseWriter, r *http.Request) {
func fetchHandler(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query()
url, present := query["url"]
@ -215,10 +261,12 @@ func FetchHandler(w http.ResponseWriter, r *http.Request) {
// create the record
// XXX should be atomic!
downloadId++
newDownload := download{
newDownload := download.Download{
Config: conf,
Id: downloadId,
Url: url[0],
State: "starting",
State: "choose profile",
Finished: false,
Eta: "?",
Percent: 0.0,
@ -229,160 +277,20 @@ func FetchHandler(w http.ResponseWriter, r *http.Request) {
newDownload.Log = append(newDownload.Log, "start of log...")
go func() {
queue(&newDownload)
}()
// go func() {
// newDownload.Begin()
// }()
t, err := template.ParseFS(webFS, "web/layout.tmpl", "web/popup.html")
if err != nil {
panic(err)
}
err = t.ExecuteTemplate(w, "layout", newDownload)
templateData := map[string]interface{}{"dl": newDownload, "config": conf}
err = t.ExecuteTemplate(w, "layout", templateData)
if err != nil {
panic(err)
}
}
}
func queue(dl *download) {
cmdSlice := []string{}
cmdSlice = append(cmdSlice, conf.DownloadProfiles[0].Args...)
cmdSlice = append(cmdSlice, dl.Url)
cmd := exec.Command(conf.DownloadProfiles[0].Command, cmdSlice...)
cmd.Dir = conf.Server.DownloadPath
stdout, err := cmd.StdoutPipe()
if err != nil {
dl.State = "failed"
dl.Finished = true
dl.Log = append(dl.Log, fmt.Sprintf("error setting up stdout pipe: %v", err))
return
}
stderr, err := cmd.StderrPipe()
if err != nil {
dl.State = "failed"
dl.Finished = true
dl.Log = append(dl.Log, fmt.Sprintf("error setting up stderr pipe: %v", err))
return
}
err = cmd.Start()
if err != nil {
dl.State = "failed"
dl.Finished = true
dl.Log = append(dl.Log, fmt.Sprintf("error starting youtube-dl: %v", err))
return
}
dl.Pid = cmd.Process.Pid
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
updateDownload(stdout, dl)
}()
go func() {
defer wg.Done()
updateDownload(stderr, dl)
}()
wg.Wait()
cmd.Wait()
dl.State = "complete"
dl.Finished = true
dl.ExitCode = cmd.ProcessState.ExitCode()
if dl.ExitCode != 0 {
dl.State = "failed"
}
}
func updateDownload(r io.Reader, dl *download) {
// XXX not sure if we might get a partial line?
buf := make([]byte, 1024)
for {
n, err := r.Read(buf)
if n > 0 {
s := string(buf[:n])
lines := strings.Split(s, "\n")
for _, l := range lines {
if l == "" {
continue
}
// append the raw log
dl.Log = append(dl.Log, l)
// look for the percent and eta and other metadata
updateMetadata(dl, l)
}
}
if err != nil {
break
}
}
}
func updateMetadata(dl *download, s string) {
// [download] 49.7% of ~15.72MiB at 5.83MiB/s ETA 00:07
etaRE := regexp.MustCompile(`download.+ETA +(\d\d:\d\d)`)
matches := etaRE.FindStringSubmatch(s)
if len(matches) == 2 {
dl.Eta = matches[1]
dl.State = "downloading"
}
percentRE := regexp.MustCompile(`download.+?([\d\.]+)%`)
matches = percentRE.FindStringSubmatch(s)
if len(matches) == 2 {
p, err := strconv.ParseFloat(matches[1], 32)
if err == nil {
dl.Percent = float32(p)
} else {
panic(err)
}
}
// This appears once per destination file
// [download] Destination: Filename with spaces and other punctuation here be careful!.mp4
filename := regexp.MustCompile(`download.+?Destination: (.+)$`)
matches = filename.FindStringSubmatch(s)
if len(matches) == 2 {
dl.Files = append(dl.Files, matches[1])
}
// This means a file has been "created" by merging others
// [ffmpeg] Merging formats into "Toto - Africa (Official HD Video)-FTQbiNvZqaY.mp4"
mergedFilename := regexp.MustCompile(`Merging formats into "(.+)"$`)
matches = mergedFilename.FindStringSubmatch(s)
if len(matches) == 2 {
dl.Files = append(dl.Files, matches[1])
}
// This means a file has been deleted
// Gross - this time it's unquoted and has trailing guff
// Deleting original file Toto - Africa (Official HD Video)-FTQbiNvZqaY.f137.mp4 (pass -k to keep)
// This is very fragile
deletedFile := regexp.MustCompile(`Deleting original file (.+) \(pass -k to keep\)$`)
matches = deletedFile.FindStringSubmatch(s)
if len(matches) == 2 {
// find the index
for i, f := range dl.Files {
if f == matches[1] {
dl.Files = append(dl.Files[:i], dl.Files[i+1:]...)
break
}
}
}
}

6
web/config.html
View File

@ -131,9 +131,9 @@
save_config() {
let op = {
method: 'POST',
body: JSON.stringify(this.config),
headers: { 'Content-Type': 'application/json' }
}
body: JSON.stringify(this.config),
headers: { 'Content-Type': 'application/json' }
};
fetch('/rest/config', op)
.then(response => {
return response.json();

35
web/popup.html
View File

@ -1,8 +1,19 @@
{{ define "content" }}
<div id="layout" class="pure-g pure-u-1" x-data="popup()" x-init="fetch_data()">
<div id="layout" class="pure-g pure-u-1" x-data="popup()" x-init="fetch_data()">
<h2>Download started</h2>
<p>Fetching <tt>{{ .Url }}</tt></p>
<p>Fetching <tt>{{ .dl.Url }}</tt></p>
<table class="pure-table" >
<tr>
<th>profile</th>
<td>
<select x-bind:disabled="profile_chosen" x-on:change="update_profile()" class="pure-input-1-2" x-model="profile_chosen">
<option value="">choose a profile to start</option>
{{ range $i := .config.DownloadProfiles }}
<option>{{ $i.Name }}</option>
{{ end }}
</select>
</td>
</tr>
<tr><th>current filename</th><td x-text="filename"></td></tr>
<tr><th>state</th><td x-text="state"></td></tr>
<tr><th>progress</th><td x-text="percent"></td></tr>
@ -21,8 +32,26 @@
function popup() {
return {
eta: '', percent: 0.0, state: '??', filename: '', finished: false, log :'',
profile_chosen: null,
watch_profile() {
console.log('will wtch profile');
this.$watch('profile_chosen', value => this.profile_chosen(value))
},
update_profile(name) {
console.log('you chose name', this.profile_chosen);
let op = {
method: 'POST',
body: JSON.stringify({action: 'start', profile: this.profile_chosen}),
headers: { 'Content-Type': 'application/json' }
};
fetch('/rest/fetch/{{ .dl.Id }}', op)
.then(response => response.json())
.then(info => {
console.log(info)
})
},
fetch_data() {
fetch('/rest/fetch/info/{{ .Id }}')
fetch('/rest/fetch/{{ .dl.Id }}')
.then(response => response.json())
.then(info => {
this.eta = info.eta;