Add files.

go.mod

@@ -0,0 +1,5 @@
+module github.com/tardisx/gropple
+
+go 1.16
+
+require github.com/gorilla/mux v1.8.0

go.sum

@@ -0,0 +1,2 @@
+github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=

main.go

@@ -0,0 +1,273 @@
+package main
+
+import (
+	"embed"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"html/template"
+	"io"
+	"log"
+	"net/http"
+	"os/exec"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"strconv"
+
+	"github.com/gorilla/mux"
+)
+
+type download struct {
+	Id       int      `json:"id"`
+	Url      string   `json:"url"`
+	Pid      int      `json:"pid"`
+	ExitCode int      `json:"exit_code"`
+	State    string   `json:"state"`
+	Files    []string `json:"files"`
+	Eta      string   `json:"eta"`
+	Percent  float32  `json:"percent"`
+	Log      []string `json:"log"`
+}
+
+var downloads map[int]*download
+var downloadId = 0
+var downloadPath = "./"
+
+//go:embed web
+var webFS embed.FS
+
+func main() {
+	var address string
+	flag.StringVar(&address, "address", "", "address for the service")
+	flag.StringVar(&downloadPath, "path", "", "path for downloaded files - defaults to current directory")
+	flag.Parse()
+
+	r := mux.NewRouter()
+	r.HandleFunc("/", HomeHandler)
+	r.HandleFunc("/fetch", FetchHandler)
+	r.HandleFunc("/fetch/info/{id}", FetchInfoHandler)
+
+	http.Handle("/", r)
+
+	downloads = make(map[int]*download)
+
+	srv := &http.Server{
+		Handler: r,
+		Addr:    "127.0.0.1:8000",
+		// Good practice: enforce timeouts for servers you create!
+		WriteTimeout: 0 * time.Second,
+		ReadTimeout:  0 * time.Second,
+	}
+
+	go func() {
+		for {
+			//fmt.Printf("\n\n%#V\n\n", downloads)
+			time.Sleep(time.Second)
+		}
+	}()
+
+	log.Fatal(srv.ListenAndServe())
+
+}
+
+func HomeHandler(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+
+	bookmarkletURL := "javascript:(function(f,s,n,o){window.open(f+encodeURIComponent(s),n,o)}('http://localhost:8000/fetch?url=',window.location,'yourform','width=500,height=500'));"
+
+	t, err := template.ParseFS(webFS, "web/index.html")
+	if err != nil {
+		panic(err)
+	}
+
+	type Info struct {
+		Downloads      map[int]*download
+		BookmarkletURL template.URL
+	}
+
+	info := Info{
+		Downloads:      downloads,
+		BookmarkletURL: template.URL(bookmarkletURL),
+	}
+
+	log.Printf("%s", info.BookmarkletURL)
+	err = t.Execute(w, info)
+	if err != nil {
+		panic(err)
+	}
+
+}
+
+func FetchInfoHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	idString := vars["id"]
+	if idString != "" {
+		id, err := strconv.Atoi(idString)
+		if err != nil {
+			http.NotFound(w, r)
+			return
+		}
+
+		b, _ := json.Marshal(downloads[id])
+		w.Write(b)
+	} else {
+		http.NotFound(w, r)
+	}
+}
+
+func FetchHandler(w http.ResponseWriter, r *http.Request) {
+
+	query := r.URL.Query()
+	url, present := query["url"] //filters=["color", "price", "brand"]
+
+	if !present {
+		fmt.Fprint(w, "something")
+	} else {
+
+		// create the record
+		// XXX should be atomic!
+		downloadId++
+		newDownload := download{
+			Id:      downloadId,
+			Url:     url[0],
+			State:   "starting",
+			Eta:     "?",
+			Percent: 0.0,
+			Log:     make([]string, 0, 1000),
+		}
+		downloads[downloadId] = &newDownload
+		// XXX atomic ^^
+		newDownload.Log = append(newDownload.Log, "start of log...")
+
+		go func() {
+			queue(&newDownload)
+		}()
+		t, err := template.ParseFS(webFS, "web/popup.html")
+		if err != nil {
+			panic(err)
+		}
+		err = t.Execute(w, newDownload)
+		if err != nil {
+			panic(err)
+		}
+
+		//		fmt.Fprintf(w, "Started DL %d!", downloadId)
+	}
+}
+
+func queue(dl *download) {
+
+	cmd := exec.Command(
+		"youtube-dl",
+		"--write-info-json",
+		"-f", "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best",
+		"--newline", dl.Url,
+	)
+	cmd.Dir = downloadPath
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		dl.State = "ended"
+		dl.Log = append(dl.Log, fmt.Sprintf("error setting up stdout pipe: %v", err))
+		return
+	}
+
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		dl.State = "ended"
+		dl.Log = append(dl.Log, fmt.Sprintf("error setting up stderr pipe: %v", err))
+		return
+	}
+
+	err = cmd.Start()
+	if err != nil {
+		dl.State = "ended"
+		dl.Log = append(dl.Log, fmt.Sprintf("error starting youtube-dl: %v", err))
+		return
+	}
+	dl.Pid = cmd.Process.Pid
+
+	var wg sync.WaitGroup
+
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		updateDownload(stdout, dl)
+	}()
+
+	go func() {
+		defer wg.Done()
+		updateDownload(stderr, dl)
+	}()
+
+	wg.Wait()
+
+	dl.State = "ended"
+	dl.ExitCode = cmd.ProcessState.ExitCode()
+
+	fmt.Printf("OBJ %#v\n", dl)
+
+	return
+}
+
+func updateDownload(r io.Reader, dl *download) {
+	// XXX not sure if we might get a partial line?
+	buf := make([]byte, 1024)
+	for {
+		n, err := r.Read(buf)
+		if n > 0 {
+			s := string(buf[:n])
+			lines := strings.Split(s, "\n")
+
+			for _, l := range lines {
+
+				if l == "" {
+					continue
+				}
+
+				// append the raw log
+				dl.Log = append(dl.Log, l)
+
+				// look for the percent and eta and other metadata
+				updateMetadata(dl, l)
+			}
+		}
+		if err != nil {
+			break
+		}
+	}
+}
+
+func updateMetadata(dl *download, s string) {
+
+	// [download]  49.7% of ~15.72MiB at  5.83MiB/s ETA 00:07
+	etaRE := regexp.MustCompile(`download.+ETA +(\d\d:\d\d)`)
+	matches := etaRE.FindStringSubmatch(s)
+	if len(matches) == 2 {
+		dl.Eta = matches[1]
+		dl.State = "downloading"
+
+	}
+
+	percentRE := regexp.MustCompile(`download.+?([\d\.]+)%`)
+	matches = percentRE.FindStringSubmatch(s)
+	if len(matches) == 2 {
+		p, err := strconv.ParseFloat(matches[1], 32)
+		if err == nil {
+			dl.Percent = float32(p)
+		} else {
+			panic(err)
+		}
+	}
+
+	// This appears once per destination file
+	// [download] Destination: Filename with spaces and other punctuation here be careful!.mp4
+	filename := regexp.MustCompile(`download.+?Destination: (.+)$`)
+	matches = filename.FindStringSubmatch(s)
+	if len(matches) == 2 {
+		dl.Files = append(dl.Files, matches[1])
+	}
+}

web/index.html

@@ -0,0 +1,29 @@
+<html>
+    <head>
+        <title>index</title>
+        <link rel="stylesheet" href="https://unpkg.com/purecss@2.0.6/build/pure-min.css" integrity="sha384-Uu6IeWbM+gzNVXJcM9XV3SohHtmWE+3VGi496jvgX1jyvDTXfdK+rfZc8C1Aehk5" crossorigin="anonymous">
+    </head>
+    <body>
+        <p>
+            Drag this bookmarklet: <a href="{{ .BookmarkletURL }}">Gropple</a> to your bookmark bar, and click it
+            on any page you want to grab the video from.
+        </p>
+        {{ range $k, $v := .Downloads }}
+        <div>
+            <h4>{{ $v.Url }}</h4>
+            <table>
+                <tr><th>state</th><td>{{ $v.State }}</td></tr>
+                <tr><th>percent</th><td>{{ $v.Percent }}</td></tr>
+                <tr><th>files</th><td>{{ range $i, $f := $v.Files }}{{ $f }}<br>{{ end }}</td></tr>
+                <tr><th>exit code</th><td>{{ $v.ExitCode }}</td></tr>
+
+            </table>
+            <pre>
+                {{ range $i, $l := $v.Log }}
+                line: {{ $l }}
+                {{- end -}}
+            </pre>
+        </div>
+        {{ end }}
+               
+    </body>

web/popup.html

@@ -0,0 +1,46 @@
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <title>fetching {{ .Url }}</title>
+    <script src="//unpkg.com/alpinejs" defer></script>
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link rel="stylesheet" href="https://unpkg.com/purecss@2.0.6/build/pure-min.css" integrity="sha384-Uu6IeWbM+gzNVXJcM9XV3SohHtmWE+3VGi496jvgX1jyvDTXfdK+rfZc8C1Aehk5" crossorigin="anonymous">
+    <link rel="stylesheet" href="https://unpkg.com/purecss@2.0.6/build/grids-responsive-min.css">
+</head>
+<body>
+    <div id="layout" class="pure-g pure-u-1">
+        <p>Fetching <tt>{{ .Url }}</tt></p>
+        <table class="pure-table" x-data="popup()"  x-init="fetch_data()">
+            <tr><th>current filename</th><td x-text="filename"></td></tr>
+            <tr><th>state</th><td x-text="state"></td></tr>
+            <tr><th>progress</th><td x-text="percent"></td></tr>
+            <tr><th>ETA</th><td x-text="eta"></td></tr>
+        </table>
+        <p><a href="/" target="_grobbler">Status page</a> </p>
+    </div>
+</body>
+<script>
+    function popup() {
+        return { 
+            eta: '', percent: 0.0, state: '??', filename: '',
+            fetch_data() {
+                fetch('/fetch/info/{{ .Id }}')
+                .then(response => response.json())
+                .then(info => {
+                    this.eta = info.eta;
+                    this.percent = info.percent + "%";
+                    this.state = info.state;
+                    if (info.files && info.files.length > 0) {
+                        this.filename = info.files[info.files.length - 1];
+                    }
+                    if (this.state != "ended") {
+                        setTimeout(() => { this.fetch_data() }, 100);
+                    }
+                });
+            },
+        }
+    }
+    
+</script>
+</html>
+