diff --git a/CHANGELOG.md b/CHANGELOG.md index 94141ee..e42f8ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ All notable changes to this project will be documented in this file. ## [Unreleased] - Check the chosen command exists when configuring a profile +- Add a stop button in the popup to abort a download +- Move included JS to local app instead of accessing from a CDN +- Make the simultaneous download limit apply to each unique domain ## [v0.5.3] - 2021-11-21 diff --git a/download/download.go b/download/download.go index 06db447..4f3eba7 100644 --- a/download/download.go +++ b/download/download.go @@ -3,11 +3,14 @@ package download import ( "fmt" "io" + "log" + "net/url" "os/exec" "regexp" "strconv" "strings" "sync" + "syscall" "time" "github.com/tardisx/gropple/config" @@ -28,6 +31,7 @@ type Download struct { Percent float32 `json:"percent"` Log []string `json:"log"` Config *config.Config + mutex sync.Mutex } type Downloads []*Download @@ -35,26 +39,31 @@ type Downloads []*Download // StartQueued starts any downloads that have been queued, we would not exceed // maxRunning. If maxRunning is 0, there is no limit. func (dls Downloads) StartQueued(maxRunning int) { - active := 0 - queued := 0 + active := make(map[string]int) for _, dl := range dls { + + dl.mutex.Lock() + if dl.State == "downloading" { - active++ - } - if dl.State == "queued" { - queued++ + active[dl.domain()]++ } + dl.mutex.Unlock() + } - // there is room, so start one - if queued > 0 && (active < maxRunning || maxRunning == 0) { - for _, dl := range dls { - if dl.State == "queued" { - dl.State = "downloading" - go func() { dl.Begin() }() - return - } + for _, dl := range dls { + + dl.mutex.Lock() + + if dl.State == "queued" && (maxRunning == 0 || active[dl.domain()] < maxRunning) { + dl.State = "downloading" + active[dl.domain()]++ + log.Printf("Starting download for %#v", dl) + dl.mutex.Unlock() + go func() { dl.Begin() }() + } else { + dl.mutex.Unlock() } } @@ -65,23 +74,57 @@ func (dls Downloads) StartQueued(maxRunning int) { func (dls Downloads) Cleanup() Downloads { newDLs := Downloads{} for _, dl := range dls { + + dl.mutex.Lock() + if dl.Finished && time.Since(dl.FinishedTS) > time.Duration(time.Hour) { // do nothing } else { newDLs = append(newDLs, dl) } + dl.mutex.Unlock() + } return newDLs } // Queue queues a download func (dl *Download) Queue() { + + dl.mutex.Lock() + defer dl.mutex.Unlock() + dl.State = "queued" + +} + +func (dl *Download) Stop() { + log.Printf("stopping the download") + dl.mutex.Lock() + defer dl.mutex.Unlock() + + syscall.Kill(dl.Pid, syscall.SIGTERM) +} + +func (dl *Download) domain() string { + + // note that we expect to already have the mutex locked by the caller + url, err := url.Parse(dl.Url) + if err != nil { + log.Printf("Unknown domain for url: %s", dl.Url) + return "unknown" + } + + return url.Hostname() + } // Begin starts a download, by starting the command specified in the DownloadProfile. // It blocks until the download is complete. func (dl *Download) Begin() { + + dl.mutex.Lock() + dl.State = "downloading" cmdSlice := []string{} cmdSlice = append(cmdSlice, dl.DownloadProfile.Args...) @@ -112,6 +155,7 @@ func (dl *Download) Begin() { return } + log.Printf("Starting %v", cmd) err = cmd.Start() if err != nil { dl.State = "failed" @@ -124,6 +168,8 @@ func (dl *Download) Begin() { var wg sync.WaitGroup + dl.mutex.Unlock() + wg.Add(2) go func() { defer wg.Done() @@ -138,6 +184,8 @@ func (dl *Download) Begin() { wg.Wait() cmd.Wait() + dl.mutex.Lock() + dl.State = "complete" dl.Finished = true dl.FinishedTS = time.Now() @@ -146,6 +194,7 @@ func (dl *Download) Begin() { if dl.ExitCode != 0 { dl.State = "failed" } + dl.mutex.Unlock() } @@ -164,9 +213,13 @@ func (dl *Download) updateDownload(r io.Reader) { continue } + dl.mutex.Lock() + // append the raw log dl.Log = append(dl.Log, l) + dl.mutex.Unlock() + // look for the percent and eta and other metadata dl.updateMetadata(l) } @@ -179,6 +232,10 @@ func (dl *Download) updateDownload(r io.Reader) { func (dl *Download) updateMetadata(s string) { + dl.mutex.Lock() + + defer dl.mutex.Unlock() + // [download] 49.7% of ~15.72MiB at 5.83MiB/s ETA 00:07 etaRE := regexp.MustCompile(`download.+ETA +(\d\d:\d\d(?::\d\d)?)$`) matches := etaRE.FindStringSubmatch(s) diff --git a/download/download_test.go b/download/download_test.go index 12eed73..2594f53 100644 --- a/download/download_test.go +++ b/download/download_test.go @@ -71,7 +71,7 @@ func TestQueue(t *testing.T) { new1 := Download{Id: 1, State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf} new2 := Download{Id: 2, State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf} new3 := Download{Id: 3, State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf} - new4 := Download{Id: 4, State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf} + new4 := Download{Id: 4, Url: "http://company.org/", State: "queued", DownloadProfile: conf.DownloadProfiles[0], Config: conf} dls := Downloads{&new1, &new2, &new3, &new4} dls.StartQueued(1) @@ -81,6 +81,9 @@ func TestQueue(t *testing.T) { if dls[1].State != "queued" { t.Error("#2 is not queued") } + if dls[3].State == "queued" { + t.Error("#4 is not started") + } // this should start no more, as one is still going dls.StartQueued(1) @@ -93,4 +96,9 @@ func TestQueue(t *testing.T) { t.Error("#2 was not started but it should be") } + dls.StartQueued(2) + if dls[3].State == "queued" { + t.Error("#4 was not started but it should be") + } + } diff --git a/main.go b/main.go index fd7d47b..2f6d828 100644 --- a/main.go +++ b/main.go @@ -53,6 +53,7 @@ func main() { r := mux.NewRouter() r.HandleFunc("/", homeHandler) + r.HandleFunc("/static/{filename}", staticHandler) r.HandleFunc("/config", configHandler) r.HandleFunc("/fetch", fetchHandler) r.HandleFunc("/fetch/{id}", fetchHandler) @@ -136,6 +137,26 @@ func homeHandler(w http.ResponseWriter, r *http.Request) { } } +// staticHandler handles requests for static files +func staticHandler(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + + filename := vars["filename"] + log.Printf("WOw :%s", filename) + if strings.Index(filename, ".js") == len(filename)-3 { + f, err := webFS.Open("web/" + filename) + if err != nil { + log.Printf("error accessing %s - %v", filename, err) + w.WriteHeader(http.StatusNotFound) + return + } + w.WriteHeader(http.StatusOK) + io.Copy(w, f) + return + } + w.WriteHeader(http.StatusNotFound) +} + // configHandler returns the configuration page func configHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) @@ -236,6 +257,15 @@ func fetchInfoOneRESTHandler(w http.ResponseWriter, r *http.Request) { w.Write(succResB) return } + + if thisReq.Action == "stop" { + + thisDownload.Stop() + succRes := successResponse{Success: true, Message: "download stopped"} + succResB, _ := json.Marshal(succRes) + w.Write(succResB) + return + } } // just a get, return the object diff --git a/web/alpine.min.js b/web/alpine.min.js new file mode 100644 index 0000000..d07cf4a --- /dev/null +++ b/web/alpine.min.js @@ -0,0 +1,5 @@ +(()=>{var Be=!1,Ve=!1,X=[];function Rt(e){Gr(e)}function Gr(e){X.includes(e)||X.push(e),Yr()}function Yr(){!Ve&&!Be&&(Be=!0,queueMicrotask(Jr))}function Jr(){Be=!1,Ve=!0;for(let e=0;ee.effect(t,{scheduler:r=>{qe?Rt(r):r()}}),He=e.raw}function Ue(e){k=e}function kt(e){let t=()=>{};return[n=>{let i=k(n);e._x_effects||(e._x_effects=new Set,e._x_runEffects=()=>{e._x_effects.forEach(o=>o())}),e._x_effects.add(i),t=()=>{i!==void 0&&(e._x_effects.delete(i),G(i))}},()=>{t()}]}var It=[],Dt=[],Pt=[];function $t(e){Pt.push(e)}function Lt(e){Dt.push(e)}function Ft(e){It.push(e)}function jt(e,t,r){e._x_attributeCleanups||(e._x_attributeCleanups={}),e._x_attributeCleanups[t]||(e._x_attributeCleanups[t]=[]),e._x_attributeCleanups[t].push(r)}function We(e,t){!e._x_attributeCleanups||Object.entries(e._x_attributeCleanups).forEach(([r,n])=>{(t===void 0||t.includes(r))&&(n.forEach(i=>i()),delete e._x_attributeCleanups[r])})}var Ye=new MutationObserver(Ge),Je=!1;function Ze(){Ye.observe(document,{subtree:!0,childList:!0,attributes:!0,attributeOldValue:!0}),Je=!0}function Qr(){Zr(),Ye.disconnect(),Je=!1}var ee=[],Qe=!1;function Zr(){ee=ee.concat(Ye.takeRecords()),ee.length&&!Qe&&(Qe=!0,queueMicrotask(()=>{Xr(),Qe=!1}))}function Xr(){Ge(ee),ee.length=0}function m(e){if(!Je)return e();Qr();let t=e();return Ze(),t}var Xe=!1,me=[];function Kt(){Xe=!0}function zt(){Xe=!1,Ge(me),me=[]}function Ge(e){if(Xe){me=me.concat(e);return}let t=[],r=[],n=new Map,i=new Map;for(let o=0;os.nodeType===1&&t.push(s)),e[o].removedNodes.forEach(s=>s.nodeType===1&&r.push(s))),e[o].type==="attributes")){let s=e[o].target,a=e[o].attributeName,c=e[o].oldValue,l=()=>{n.has(s)||n.set(s,[]),n.get(s).push({name:a,value:s.getAttribute(a)})},u=()=>{i.has(s)||i.set(s,[]),i.get(s).push(a)};s.hasAttribute(a)&&c===null?l():s.hasAttribute(a)?(u(),l()):u()}i.forEach((o,s)=>{We(s,o)}),n.forEach((o,s)=>{It.forEach(a=>a(s,o))});for(let o of r)t.includes(o)||Dt.forEach(s=>s(o));t.forEach(o=>{o._x_ignoreSelf=!0,o._x_ignore=!0});for(let o of t)r.includes(o)||!o.isConnected||(delete o._x_ignoreSelf,delete o._x_ignore,Pt.forEach(s=>s(o)),o._x_ignore=!0,o._x_ignoreSelf=!0);t.forEach(o=>{delete o._x_ignoreSelf,delete o._x_ignore}),t=null,r=null,n=null,i=null}function C(e,t,r){return e._x_dataStack=[t,...I(r||e)],()=>{e._x_dataStack=e._x_dataStack.filter(n=>n!==t)}}function et(e,t){let r=e._x_dataStack[0];Object.entries(t).forEach(([n,i])=>{r[n]=i})}function I(e){return e._x_dataStack?e._x_dataStack:typeof ShadowRoot=="function"&&e instanceof ShadowRoot?I(e.host):e.parentNode?I(e.parentNode):[]}function D(e){let t=new Proxy({},{ownKeys:()=>Array.from(new Set(e.flatMap(r=>Object.keys(r)))),has:(r,n)=>e.some(i=>i.hasOwnProperty(n)),get:(r,n)=>(e.find(i=>{if(i.hasOwnProperty(n)){let o=Object.getOwnPropertyDescriptor(i,n);if(o.get&&o.get._x_alreadyBound||o.set&&o.set._x_alreadyBound)return!0;if((o.get||o.set)&&o.enumerable){let s=o.get,a=o.set,c=o;s=s&&s.bind(t),a=a&&a.bind(t),s&&(s._x_alreadyBound=!0),a&&(a._x_alreadyBound=!0),Object.defineProperty(i,n,{...c,get:s,set:a})}return!0}return!1})||{})[n],set:(r,n,i)=>{let o=e.find(s=>s.hasOwnProperty(n));return o?o[n]=i:e[e.length-1][n]=i,!0}});return t}function he(e){let t=n=>typeof n=="object"&&!Array.isArray(n)&&n!==null,r=(n,i="")=>{Object.entries(Object.getOwnPropertyDescriptors(n)).forEach(([o,{value:s,enumerable:a}])=>{if(a===!1||s===void 0)return;let c=i===""?o:`${i}.${o}`;typeof s=="object"&&s!==null&&s._x_interceptor?n[o]=s.initialize(e,c,o):t(s)&&s!==n&&!(s instanceof Element)&&r(s,c)})};return r(e)}function _e(e,t=()=>{}){let r={initialValue:void 0,_x_interceptor:!0,initialize(n,i,o){return e(this.initialValue,()=>en(n,i),s=>tt(n,i,s),i,o)}};return t(r),n=>{if(typeof n=="object"&&n!==null&&n._x_interceptor){let i=r.initialize.bind(r);r.initialize=(o,s,a)=>{let c=n.initialize(o,s,a);return r.initialValue=c,i(o,s,a)}}else r.initialValue=n;return r}}function en(e,t){return t.split(".").reduce((r,n)=>r[n],e)}function tt(e,t,r){if(typeof t=="string"&&(t=t.split(".")),t.length===1)e[t[0]]=r;else{if(t.length===0)throw error;return e[t[0]]||(e[t[0]]={}),tt(e[t[0]],t.slice(1),r)}}var Bt={};function y(e,t){Bt[e]=t}function te(e,t){return Object.entries(Bt).forEach(([r,n])=>{Object.defineProperty(e,`$${r}`,{get(){return n(t,{Alpine:R,interceptor:_e})},enumerable:!1})}),e}function Vt(e,t,r,...n){try{return r(...n)}catch(i){Y(i,e,t)}}function Y(e,t,r=void 0){Object.assign(e,{el:t,expression:r}),console.warn(`Alpine Expression Error: ${e.message} + +${r?'Expression: "'+r+`" + +`:""}`,t),setTimeout(()=>{throw e},0)}function w(e,t,r={}){let n;return h(e,t)(i=>n=i,r),n}function h(...e){return Ht(...e)}var Ht=rt;function qt(e){Ht=e}function rt(e,t){let r={};te(r,e);let n=[r,...I(e)];if(typeof t=="function")return tn(n,t);let i=rn(n,t,e);return Vt.bind(null,e,t,i)}function tn(e,t){return(r=()=>{},{scope:n={},params:i=[]}={})=>{let o=t.apply(D([n,...e]),i);ge(r,o)}}var nt={};function nn(e,t){if(nt[e])return nt[e];let r=Object.getPrototypeOf(async function(){}).constructor,n=/^[\n\s]*if.*\(.*\)/.test(e)||/^(let|const)\s/.test(e)?`(() => { ${e} })()`:e,o=(()=>{try{return new r(["__self","scope"],`with (scope) { __self.result = ${n} }; __self.finished = true; return __self.result;`)}catch(s){return Y(s,t,e),Promise.resolve()}})();return nt[e]=o,o}function rn(e,t,r){let n=nn(t,r);return(i=()=>{},{scope:o={},params:s=[]}={})=>{n.result=void 0,n.finished=!1;let a=D([o,...e]);if(typeof n=="function"){let c=n(n,a).catch(l=>Y(l,r,t));n.finished?(ge(i,n.result,a,s,r),n.result=void 0):c.then(l=>{ge(i,l,a,s,r)}).catch(l=>Y(l,r,t)).finally(()=>n.result=void 0)}}}function ge(e,t,r,n,i){if(typeof t=="function"){let o=t.apply(r,n);o instanceof Promise?o.then(s=>ge(e,s,r,n)).catch(s=>Y(s,i,t)):e(o)}else e(t)}var it="x-";function E(e=""){return it+e}function Ut(e){it=e}var Wt={};function d(e,t){Wt[e]=t}function re(e,t,r){let n={};return Array.from(t).map(Gt((o,s)=>n[o]=s)).filter(Yt).map(sn(n,r)).sort(an).map(o=>on(e,o))}function Jt(e){return Array.from(e).map(Gt()).filter(t=>!Yt(t))}var ot=!1,ne=new Map,Zt=Symbol();function Qt(e){ot=!0;let t=Symbol();Zt=t,ne.set(t,[]);let r=()=>{for(;ne.get(t).length;)ne.get(t).shift()();ne.delete(t)},n=()=>{ot=!1,r()};e(r),n()}function on(e,t){let r=()=>{},n=Wt[t.type]||r,i=[],o=p=>i.push(p),[s,a]=kt(e);i.push(a);let c={Alpine:R,effect:s,cleanup:o,evaluateLater:h.bind(h,e),evaluate:w.bind(w,e)},l=()=>i.forEach(p=>p());jt(e,t.original,l);let u=()=>{e._x_ignore||e._x_ignoreSelf||(n.inline&&n.inline(e,t,c),n=n.bind(n,e,t,c),ot?ne.get(Zt).push(n):n())};return u.runCleanups=l,u}var xe=(e,t)=>({name:r,value:n})=>(r.startsWith(e)&&(r=r.replace(e,t)),{name:r,value:n}),ye=e=>e;function Gt(e=()=>{}){return({name:t,value:r})=>{let{name:n,value:i}=Xt.reduce((o,s)=>s(o),{name:t,value:r});return n!==t&&e(n,t),{name:n,value:i}}}var Xt=[];function J(e){Xt.push(e)}function Yt({name:e}){return er().test(e)}var er=()=>new RegExp(`^${it}([^:^.]+)\\b`);function sn(e,t){return({name:r,value:n})=>{let i=r.match(er()),o=r.match(/:([a-zA-Z0-9\-:]+)/),s=r.match(/\.[^.\]]+(?=[^\]]*$)/g)||[],a=t||e[r]||r;return{type:i?i[1]:null,value:o?o[1]:null,modifiers:s.map(c=>c.replace(".","")),expression:n,original:a}}}var st="DEFAULT",be=["ignore","ref","data","id","bind","init","for","model","transition","show","if",st,"teleport","element"];function an(e,t){let r=be.indexOf(e.type)===-1?st:e.type,n=be.indexOf(t.type)===-1?st:t.type;return be.indexOf(r)-be.indexOf(n)}function K(e,t,r={}){e.dispatchEvent(new CustomEvent(t,{detail:r,bubbles:!0,composed:!0,cancelable:!0}))}var at=[],ct=!1;function we(e){at.push(e),queueMicrotask(()=>{ct||setTimeout(()=>{ve()})})}function ve(){for(ct=!1;at.length;)at.shift()()}function tr(){ct=!0}function P(e,t){if(typeof ShadowRoot=="function"&&e instanceof ShadowRoot){Array.from(e.children).forEach(i=>P(i,t));return}let r=!1;if(t(e,()=>r=!0),r)return;let n=e.firstElementChild;for(;n;)P(n,t,!1),n=n.nextElementSibling}function z(e,...t){console.warn(`Alpine Warning: ${e}`,...t)}function nr(){document.body||z("Unable to initialize. Trying to load Alpine before `` is available. Did you forget to add `defer` in Alpine's ` + diff --git a/web/popup.html b/web/popup.html index c103c61..81fbbf9 100644 --- a/web/popup.html +++ b/web/popup.html @@ -18,8 +18,10 @@ state progress ETA +

You can close this window and your download will continue. Check the Status page to see all downloads in progress.

+

Logs

@@ -50,6 +52,18 @@
                     console.log(info)
                 })
             },
+            stop() {
+                let op = {
+                   method: 'POST',
+                   body: JSON.stringify({action: 'stop'}),
+                   headers: { 'Content-Type': 'application/json' }
+                };
+                fetch('/rest/fetch/{{ .dl.Id }}', op)
+                .then(response => response.json())
+                .then(info => {
+                    console.log(info)
+                })
+            },
             fetch_data() {
                 fetch('/rest/fetch/{{ .dl.Id }}')
                 .then(response => response.json())