From 9ef6ab71c7b24152a55c465eb7b387e86ac76bc8 Mon Sep 17 00:00:00 2001 From: Justin Hawkins Date: Mon, 7 Jun 2021 21:13:57 +0930 Subject: [PATCH] Add upload package and update dependencies --- dau.go | 196 +---------------------------------- go.mod | 3 +- go.sum | 6 +- uploads/uploads.go | 249 +++++++++++++++++++++++++++++++++++++++++++++ web/server.go | 9 ++ 5 files changed, 264 insertions(+), 199 deletions(-) create mode 100644 uploads/uploads.go diff --git a/dau.go b/dau.go index 9b52d4c..6c38922 100644 --- a/dau.go +++ b/dau.go @@ -3,33 +3,27 @@ package main //go:generate go-bindata -pkg assets -o assets/static.go -prefix data/ data import ( - "bytes" "encoding/json" "fmt" - "io" "io/ioutil" "log" - "mime/multipart" "net/http" "os" "path/filepath" "strings" "time" - "image" - _ "image/gif" _ "image/jpeg" _ "image/png" - "github.com/fogleman/gg" "github.com/pborman/getopt" // "github.com/skratchdot/open-golang/open" - "golang.org/x/image/font/inconsolata" "github.com/tardisx/discord-auto-upload/config" daulog "github.com/tardisx/discord-auto-upload/log" + "github.com/tardisx/discord-auto-upload/uploads" "github.com/tardisx/discord-auto-upload/web" ) @@ -172,190 +166,6 @@ func fileEligible(file string) bool { func processFile(file string) { - if !config.Config.NoWatermark { - daulog.SendLog("Copying to temp location and watermarking ", daulog.LogTypeInfo) - file = mungeFile(file) - } - - if config.Config.WebHookURL == "" { - daulog.SendLog("WebHookURL is not configured - cannot upload!", daulog.LogTypeError) - return - } - - daulog.SendLog("Uploading", daulog.LogTypeInfo) - - extraParams := map[string]string{} - - if config.Config.Username != "" { - log.Print("Overriding username with " + config.Config.Username) - extraParams["username"] = config.Config.Username - } - - type DiscordAPIResponseAttachment struct { - URL string - ProxyURL string - Size int - Width int - Height int - Filename string - } - - type DiscordAPIResponse struct { - Attachments []DiscordAPIResponseAttachment - ID int64 `json:",string"` - } - - var retriesRemaining = 5 - for retriesRemaining > 0 { - - request, err := newfileUploadRequest(config.Config.WebHookURL, extraParams, "file", file) - if err != nil { - log.Fatal(err) - } - start := time.Now() - client := &http.Client{Timeout: time.Second * 30} - resp, err := client.Do(request) - if err != nil { - log.Print("Error performing request:", err) - retriesRemaining-- - sleepForRetries(retriesRemaining) - continue - } else { - - if resp.StatusCode != 200 { - log.Print("Bad response from server:", resp.StatusCode) - if b, err := ioutil.ReadAll(resp.Body); err == nil { - log.Print("Body:", string(b)) - } - retriesRemaining-- - sleepForRetries(retriesRemaining) - continue - } - - resBody, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Print("could not deal with body: ", err) - retriesRemaining-- - sleepForRetries(retriesRemaining) - continue - } - resp.Body.Close() - - var res DiscordAPIResponse - err = json.Unmarshal(resBody, &res) - - if err != nil { - log.Print("could not parse JSON: ", err) - fmt.Println("Response was:", string(resBody[:])) - retriesRemaining-- - sleepForRetries(retriesRemaining) - continue - } - if len(res.Attachments) < 1 { - log.Print("bad response - no attachments?") - retriesRemaining-- - sleepForRetries(retriesRemaining) - continue - } - var a = res.Attachments[0] - elapsed := time.Since(start) - rate := float64(a.Size) / elapsed.Seconds() / 1024.0 - - log.Printf("Uploaded to %s %dx%d", a.URL, a.Width, a.Height) - log.Printf("id: %d, %d bytes transferred in %.2f seconds (%.2f KiB/s)", res.ID, a.Size, elapsed.Seconds(), rate) - break - } - } - - if !config.Config.NoWatermark { - daulog.SendLog(fmt.Sprintf("Removing temporary file: %s", file), daulog.LogTypeDebug) - os.Remove(file) - } - - if retriesRemaining == 0 { - log.Fatal("Failed to upload, even after retries") - } -} - -func sleepForRetries(retry int) { - if retry == 0 { - return - } - retryTime := (6-retry)*(6-retry) + 6 - daulog.SendLog(fmt.Sprintf("Will retry in %d seconds (%d remaining attempts)", retryTime, retry), daulog.LogTypeError) - time.Sleep(time.Duration(retryTime) * time.Second) -} - -func newfileUploadRequest(uri string, params map[string]string, paramName, path string) (*http.Request, error) { - file, err := os.Open(path) - if err != nil { - return nil, err - } - defer file.Close() - - body := &bytes.Buffer{} - writer := multipart.NewWriter(body) - part, err := writer.CreateFormFile(paramName, filepath.Base(path)) - if err != nil { - return nil, err - } - _, err = io.Copy(part, file) - if err != nil { - log.Fatal("Could not copy: ", err) - } - - for key, val := range params { - _ = writer.WriteField(key, val) - } - err = writer.Close() - if err != nil { - return nil, err - } - - req, err := http.NewRequest("POST", uri, body) - req.Header.Set("Content-Type", writer.FormDataContentType()) - return req, err -} - -func mungeFile(path string) string { - - reader, err := os.Open(path) - if err != nil { - log.Fatal(err) - } - defer reader.Close() - - im, _, err := image.Decode(reader) - if err != nil { - log.Fatal(err) - } - bounds := im.Bounds() - // var S float64 = float64(bounds.Max.X) - - dc := gg.NewContext(bounds.Max.X, bounds.Max.Y) - dc.Clear() - dc.SetRGB(0, 0, 0) - - dc.SetFontFace(inconsolata.Regular8x16) - - dc.DrawImage(im, 0, 0) - - dc.DrawRoundedRectangle(0, float64(bounds.Max.Y-18.0), 320, float64(bounds.Max.Y), 0) - dc.SetRGB(0, 0, 0) - dc.Fill() - - dc.SetRGB(1, 1, 1) - - dc.DrawString("github.com/tardisx/discord-auto-upload", 5.0, float64(bounds.Max.Y)-5.0) - - tempfile, err := ioutil.TempFile("", "dau") - if err != nil { - log.Fatal(err) - } - tempfile.Close() - os.Remove(tempfile.Name()) - actualName := tempfile.Name() + ".png" - - dc.SavePNG(actualName) - return actualName + daulog.SendLog("Sending to uploader", daulog.LogTypeInfo) + uploads.AddFile(file) } diff --git a/go.mod b/go.mod index a1e753f..01e1df5 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,5 @@ require ( github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/mitchellh/go-homedir v1.1.0 github.com/pborman/getopt v1.1.0 - github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 - golang.org/x/image v0.0.0-20201208152932-35266b937fa6 + golang.org/x/image v0.0.0-20210504121937-7319ad40d33e ) diff --git a/go.sum b/go.sum index e25f30d..5898072 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,6 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/pborman/getopt v1.1.0 h1:eJ3aFZroQqq0bWmraivjQNt6Dmm5M0h2JcDW38/Azb0= github.com/pborman/getopt v1.1.0/go.mod h1:FxXoW1Re00sQG/+KIkuSqRL/LwQgSkv7uyac+STFsbk= -github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= -github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= -golang.org/x/image v0.0.0-20201208152932-35266b937fa6 h1:nfeHNc1nAqecKCy2FCy4HY+soOOe5sDLJ/gZLbx6GYI= -golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210504121937-7319ad40d33e h1:PzJMNfFQx+QO9hrC1GwZ4BoPGeNGhfeQEgcQFArEjPk= +golang.org/x/image v0.0.0-20210504121937-7319ad40d33e/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/uploads/uploads.go b/uploads/uploads.go new file mode 100644 index 0000000..6be229d --- /dev/null +++ b/uploads/uploads.go @@ -0,0 +1,249 @@ +// The uploads pacakge encapsulates dealing with file uploads to +// discord +package uploads + +import ( + "bytes" + "encoding/json" + "fmt" + "image" + "io" + "io/ioutil" + "log" + "mime/multipart" + "net/http" + "os" + "path/filepath" + "time" + + "github.com/fogleman/gg" + "github.com/tardisx/discord-auto-upload/config" + daulog "github.com/tardisx/discord-auto-upload/log" + "golang.org/x/image/font/inconsolata" +) + +type Upload struct { + Uploaded bool `json:"uploaded"` // has this file been uploaded to discord + UploadedAt time.Time `json:"uploaded_at"` + originalFilename string // path on the local disk + mungedFilename string // post-watermark + Url string `json:"url"` // url on the discord CDN + Width int `json:"width"` + Height int `json:"height"` +} + +var Uploads []*Upload + +func AddFile(file string) { + thisUpload := Upload{ + Uploaded: false, + originalFilename: file, + } + Uploads = append(Uploads, &thisUpload) + + ProcessUpload(&thisUpload) +} + +func ProcessUpload(up *Upload) { + + file := up.originalFilename + + if !config.Config.NoWatermark { + daulog.SendLog("Copying to temp location and watermarking ", daulog.LogTypeInfo) + file = mungeFile(file) + up.mungedFilename = file + } + + if config.Config.WebHookURL == "" { + daulog.SendLog("WebHookURL is not configured - cannot upload!", daulog.LogTypeError) + return + } + + extraParams := map[string]string{} + + if config.Config.Username != "" { + daulog.SendLog("Overriding username with "+config.Config.Username, daulog.LogTypeInfo) + extraParams["username"] = config.Config.Username + } + + type DiscordAPIResponseAttachment struct { + URL string + ProxyURL string + Size int + Width int + Height int + Filename string + } + + type DiscordAPIResponse struct { + Attachments []DiscordAPIResponseAttachment + ID int64 `json:",string"` + } + + var retriesRemaining = 5 + for retriesRemaining > 0 { + + request, err := newfileUploadRequest(config.Config.WebHookURL, extraParams, "file", file) + if err != nil { + log.Fatal(err) + } + start := time.Now() + client := &http.Client{Timeout: time.Second * 30} + resp, err := client.Do(request) + if err != nil { + log.Print("Error performing request:", err) + retriesRemaining-- + sleepForRetries(retriesRemaining) + continue + } else { + + if resp.StatusCode != 200 { + // {"message": "Request entity too large", "code": 40005} + log.Print("Bad response from server:", resp.StatusCode) + if b, err := ioutil.ReadAll(resp.Body); err == nil { + log.Print("Body:", string(b)) + daulog.SendLog(fmt.Sprintf("Bad response: %s", string(b)), daulog.LogTypeError) + } + retriesRemaining-- + sleepForRetries(retriesRemaining) + continue + } + + resBody, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Print("could not deal with body: ", err) + retriesRemaining-- + sleepForRetries(retriesRemaining) + continue + } + resp.Body.Close() + + var res DiscordAPIResponse + err = json.Unmarshal(resBody, &res) + + // {"id": "851092588608880670", "type": 0, "content": "", "channel_id": "849615269706203171", "author": {"bot": true, "id": "849615314274484224", "username": "abcdedf", "avatar": null, "discriminator": "0000"}, "attachments": [{"id": "851092588332449812", "filename": "dau480457962.png", "size": 859505, "url": "https://cdn.discordapp.com/attachments/849615269706203171/851092588332449812/dau480457962.png", "proxy_url": "https://media.discordapp.net/attachments/849615269706203171/851092588332449812/dau480457962.png", "width": 640, "height": 640, "content_type": "image/png"}], "embeds": [], "mentions": [], "mention_roles": [], "pinned": false, "mention_everyone": false, "tts": false, "timestamp": "2021-06-06T13:38:05.660000+00:00", "edited_timestamp": null, "flags": 0, "components": [], "webhook_id": "849615314274484224"} + + daulog.SendLog(fmt.Sprintf("Response: %s", string(resBody[:])), daulog.LogTypeDebug) + + if err != nil { + log.Print("could not parse JSON: ", err) + fmt.Println("Response was:", string(resBody[:])) + retriesRemaining-- + sleepForRetries(retriesRemaining) + continue + } + if len(res.Attachments) < 1 { + log.Print("bad response - no attachments?") + retriesRemaining-- + sleepForRetries(retriesRemaining) + continue + } + var a = res.Attachments[0] + elapsed := time.Since(start) + rate := float64(a.Size) / elapsed.Seconds() / 1024.0 + + daulog.SendLog(fmt.Sprintf("Uploaded to %s %dx%d", a.URL, a.Width, a.Height), daulog.LogTypeInfo) + daulog.SendLog(fmt.Sprintf("id: %d, %d bytes transferred in %.2f seconds (%.2f KiB/s)", res.ID, a.Size, elapsed.Seconds(), rate), daulog.LogTypeInfo) + + up.Url = a.URL + up.Uploaded = true + up.Width = a.Width + up.Height = a.Height + up.UploadedAt = time.Now() + + break + } + } + + if !config.Config.NoWatermark { + daulog.SendLog(fmt.Sprintf("Removing temporary file: %s", file), daulog.LogTypeDebug) + os.Remove(file) + } + + if retriesRemaining == 0 { + log.Fatal("Failed to upload, even after retries") + } +} + +func newfileUploadRequest(uri string, params map[string]string, paramName, path string) (*http.Request, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + part, err := writer.CreateFormFile(paramName, filepath.Base(path)) + if err != nil { + return nil, err + } + _, err = io.Copy(part, file) + if err != nil { + log.Fatal("Could not copy: ", err) + } + + for key, val := range params { + _ = writer.WriteField(key, val) + } + err = writer.Close() + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", uri, body) + req.Header.Set("Content-Type", writer.FormDataContentType()) + return req, err +} + +func mungeFile(path string) string { + + reader, err := os.Open(path) + if err != nil { + log.Fatal(err) + } + defer reader.Close() + + im, _, err := image.Decode(reader) + if err != nil { + log.Fatal(err) + } + bounds := im.Bounds() + // var S float64 = float64(bounds.Max.X) + + dc := gg.NewContext(bounds.Max.X, bounds.Max.Y) + dc.Clear() + dc.SetRGB(0, 0, 0) + + dc.SetFontFace(inconsolata.Regular8x16) + + dc.DrawImage(im, 0, 0) + + dc.DrawRoundedRectangle(0, float64(bounds.Max.Y-18.0), 320, float64(bounds.Max.Y), 0) + dc.SetRGB(0, 0, 0) + dc.Fill() + + dc.SetRGB(1, 1, 1) + + dc.DrawString("github.com/tardisx/discord-auto-upload", 5.0, float64(bounds.Max.Y)-5.0) + + tempfile, err := ioutil.TempFile("", "dau") + if err != nil { + log.Fatal(err) + } + tempfile.Close() + os.Remove(tempfile.Name()) + actualName := tempfile.Name() + ".png" + + dc.SavePNG(actualName) + return actualName +} + +func sleepForRetries(retry int) { + if retry == 0 { + return + } + retryTime := (6-retry)*(6-retry) + 6 + daulog.SendLog(fmt.Sprintf("Will retry in %d seconds (%d remaining attempts)", retryTime, retry), daulog.LogTypeError) + time.Sleep(time.Duration(retryTime) * time.Second) +} diff --git a/web/server.go b/web/server.go index d7c37a6..0ddc158 100644 --- a/web/server.go +++ b/web/server.go @@ -15,6 +15,7 @@ import ( "github.com/tardisx/discord-auto-upload/assets" "github.com/tardisx/discord-auto-upload/config" daulog "github.com/tardisx/discord-auto-upload/log" + "github.com/tardisx/discord-auto-upload/uploads" ) // DAUWebServer - stuff for the web server @@ -292,7 +293,13 @@ func getLogs(w http.ResponseWriter, r *http.Request) { // js, _ := json.Marshal(daulog.LogEntries) w.Write([]byte(text)) +} +func getUploads(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + ups := uploads.Uploads + text, _ := json.Marshal(ups) + w.Write([]byte(text)) } func StartWebServer() { @@ -306,6 +313,8 @@ func StartWebServer() { http.HandleFunc("/rest/config/exclude", getSetExclude) http.HandleFunc("/rest/logs", getLogs) + http.HandleFunc("/rest/uploads", getUploads) + go func() { log.Print("Starting web server on http://localhost:9090") err := http.ListenAndServe(":9090", nil) // set listen port