justin/discord-auto-upload
1
0
Fork 0
Code Issues 15 Pull Requests 1 Actions Packages Projects Releases 20 Wiki Activity

Refactor how logs are handled.

Browse Source
This commit is contained in:
Justin Hawkins 2022-04-04 19:10:07 +09:30
parent fbd267e687
commit ba7ae21248
9 changed files with 171 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
config/config.go
View File

@ -54,11 +54,11 @@ func DefaultConfigService() *ConfigService {
// LoadOrInit loads the current configuration from the config file, or creates
// a new config file if none exists.
func (c *ConfigService) LoadOrInit() error {
daulog.SendLog(fmt.Sprintf("Trying to load config from %s\n", c.ConfigFilename), daulog.LogTypeDebug)
daulog.Debugf("Trying to load config from %s\n", c.ConfigFilename)
_, err := os.Stat(c.ConfigFilename)
if os.IsNotExist(err) {
daulog.SendLog("NOTE: No config file, writing out sample configuration", daulog.LogTypeInfo)
daulog.SendLog("You need to set the configuration via the web interface", daulog.LogTypeInfo)
daulog.Info("NOTE: No config file, writing out sample configuration")
daulog.Info("You need to set the configuration via the web interface")
c.Config = DefaultConfig()
return c.Save()
} else {
@ -84,7 +84,7 @@ func DefaultConfig() *ConfigV2 {
// Load will load the configuration from a known-to-exist config file.
func (c *ConfigService) Load() error {
fmt.Printf("Loading from %s\n\n", c.ConfigFilename)
daulog.Debugf("Loading from %s", c.ConfigFilename)
data, err := ioutil.ReadFile(c.ConfigFilename)
if err != nil {
@ -98,7 +98,7 @@ func (c *ConfigService) Load() error {
// Version 0 predates config migrations
if c.Config.Version == 0 {
// need to migrate this
daulog.SendLog("Migrating config to V2", daulog.LogTypeInfo)
daulog.Info("Migrating config to V2")
configV1 := ConfigV1{}
err = json.Unmarshal([]byte(data), &configV1)
@ -126,7 +126,7 @@ func (c *ConfigService) Load() error {
}
func (c *ConfigService) Save() error {
daulog.SendLog("saving configuration", daulog.LogTypeInfo)
daulog.Info("saving configuration")
// sanity checks
for _, watcher := range c.Config.Watchers {

26
dau.go
View File

@ -54,12 +54,12 @@ func main() {
go func() {
version.GetOnlineVersion()
if version.UpdateAvailable() {
fmt.Printf("You are currently on version %s, but version %s is available\n", version.CurrentVersion, version.LatestVersionInfo.TagName)
fmt.Println("----------- Release Info -----------")
fmt.Println(version.LatestVersionInfo.Body)
fmt.Println("------------------------------------")
fmt.Println("Upgrade at https://github.com/tardisx/discord-auto-upload/releases/latest")
daulog.SendLog(fmt.Sprintf("New version available: %s - download at https://github.com/tardisx/discord-auto-upload/releases/latest", version.LatestVersionInfo.TagName), daulog.LogTypeInfo)
daulog.Info("*** NEW VERSION AVAILABLE ***")
daulog.Infof("You are currently on version %s, but version %s is available\n", version.CurrentVersion, version.LatestVersionInfo.TagName)
daulog.Info("----------- Release Info -----------")
daulog.Info(version.LatestVersionInfo.Body)
daulog.Info("------------------------------------")
daulog.Info("Upgrade at https://github.com/tardisx/discord-auto-upload/releases/latest")
}
}()
@ -71,17 +71,17 @@ func main() {
func startWatchers(config *config.ConfigService, up *upload.Uploader, configChange chan bool) {
for {
daulog.SendLog("Creating watchers", daulog.LogTypeInfo)
daulog.Debug("Creating watchers")
ctx, cancel := context.WithCancel(context.Background())
for _, c := range config.Config.Watchers {
log.Printf("Creating watcher for %s interval %d", c.Path, config.Config.WatchInterval)
daulog.Infof("Creating watcher for %s with interval %d", c.Path, config.Config.WatchInterval)
watcher := watch{uploader: up, lastCheck: time.Now(), newLastCheck: time.Now(), config: c}
go watcher.Watch(config.Config.WatchInterval, ctx)
}
// wait for single that the config changed
<-configChange
cancel()
daulog.SendLog("starting new watchers due to config change", daulog.LogTypeInfo)
daulog.Info("starting new watchers due to config change")
}
}
@ -90,7 +90,7 @@ func (w *watch) Watch(interval int, ctx context.Context) {
for {
select {
case <-ctx.Done():
daulog.SendLog("Killing old watcher", daulog.LogTypeInfo)
daulog.Info("Killing old watcher")
return
default:
newFiles := w.ProcessNewFiles()
@ -99,7 +99,7 @@ func (w *watch) Watch(interval int, ctx context.Context) {
}
// upload them
w.uploader.Upload()
daulog.SendLog(fmt.Sprintf("sleeping for %ds before next check of %s", interval, w.config.Path), daulog.LogTypeDebug)
daulog.Debugf("sleeping for %ds before next check of %s", interval, w.config.Path)
time.Sleep(time.Duration(interval) * time.Second)
}
}
@ -130,11 +130,11 @@ func (w *watch) ProcessNewFiles() []string {
func (w *watch) checkPath() bool {
src, err := os.Stat(w.config.Path)
if err != nil {
daulog.SendLog(fmt.Sprintf("Problem with path '%s': %s", w.config.Path, err), daulog.LogTypeError)
daulog.Errorf("Problem with path '%s': %s", w.config.Path, err)
return false
}
if !src.IsDir() {
daulog.SendLog(fmt.Sprintf("Problem with path '%s': is not a directory", w.config.Path), daulog.LogTypeError)
daulog.Errorf("Problem with path '%s': is not a directory", w.config.Path)
return false
}
return true

65
log/log.go
View File

@ -1,10 +1,14 @@
package log
import (
"log"
"fmt"
"time"
)
type Logger interface {
WriteEntry(l LogEntry)
}
type LogEntryType string
type LogEntry struct {
@ -19,28 +23,73 @@ const (
LogTypeDebug = "debug"
)
var LogEntries []LogEntry
var loggers []Logger
var logInput chan LogEntry
var Memory *MemoryLogger
func init() {
// create some loggers
Memory = &MemoryLogger{maxsize: 100}
stdout := &StdoutLogger{}
loggers = []Logger{Memory, stdout}
// wait for log entries
logInput = make(chan LogEntry)
go func() {
for {
aLog := <-logInput
LogEntries = append(LogEntries, aLog)
for len(LogEntries) > 100 {
LogEntries = LogEntries[1:]
for _, l := range loggers {
l.WriteEntry(aLog)
}
}
}()
}
func SendLog(entry string, entryType LogEntryType) {
func Debug(entry string) {
logInput <- LogEntry{
Timestamp: time.Now(),
Entry: entry,
Type: entryType,
Type: LogTypeDebug,
}
}
func Debugf(entry string, args ...interface{}) {
logInput <- LogEntry{
Timestamp: time.Now(),
Entry: fmt.Sprintf(entry, args...),
Type: LogTypeDebug,
}
}
func Info(entry string) {
logInput <- LogEntry{
Timestamp: time.Now(),
Entry: entry,
Type: LogTypeInfo,
}
}
func Infof(entry string, args ...interface{}) {
logInput <- LogEntry{
Timestamp: time.Now(),
Entry: fmt.Sprintf(entry, args...),
Type: LogTypeInfo,
}
}
func Error(entry string) {
logInput <- LogEntry{
Timestamp: time.Now(),
Entry: entry,
Type: LogTypeError,
}
}
func Errorf(entry string, args ...interface{}) {
logInput <- LogEntry{
Timestamp: time.Now(),
Entry: fmt.Sprintf(entry, args...),
Type: LogTypeError,
}
log.Printf("%6s: %s", entryType, entry)
}

29
log/memory.go Normal file
View File

@ -0,0 +1,29 @@
package log
import (
"sync"
)
type MemoryLogger struct {
size int
entries []LogEntry
maxsize int
lock sync.Mutex
}
func (m *MemoryLogger) WriteEntry(l LogEntry) {
// xxx needs mutex
// if m.entries == nil {
// m.entries = make([]LogEntry, 0)
// }
m.lock.Lock()
m.entries = append(m.entries, l)
if len(m.entries) > m.maxsize {
m.entries = m.entries[1:]
}
m.lock.Unlock()
}
func (m *MemoryLogger) Entries() []LogEntry {
return m.entries
}

12
log/stdout.go Normal file
View File

@ -0,0 +1,12 @@
package log
import (
"log"
)
type StdoutLogger struct {
}
func (m StdoutLogger) WriteEntry(l LogEntry) {
log.Printf("%-6s %s", l.Type, l.Entry)
}

45
upload/upload.go
View File

@ -119,19 +119,19 @@ func (u *Upload) RemoveMarkupTempFile() {
}
func (u *Upload) processUpload() error {
daulog.SendLog(fmt.Sprintf("Uploading: %s", u.OriginalFilename), daulog.LogTypeInfo)
daulog.Infof("Uploading: %s", u.OriginalFilename)
baseFilename := filepath.Base(u.OriginalFilename)
if u.webhookURL == "" {
daulog.SendLog("WebHookURL is not configured - cannot upload!", daulog.LogTypeError)
daulog.Error("WebHookURL is not configured - cannot upload!")
return errors.New("webhook url not configured")
}
extraParams := map[string]string{}
if u.usernameOverride != "" {
daulog.SendLog("Overriding username with "+u.usernameOverride, daulog.LogTypeInfo)
daulog.Infof("Overriding username with '%s'", u.usernameOverride)
extraParams["username"] = u.usernameOverride
}
@ -158,7 +158,7 @@ func (u *Upload) processUpload() error {
if len(u.MarkedUpFilename) > 0 {
filedata, err = os.Open(u.MarkedUpFilename)
if err != nil {
log.Print("Error opening marked up file:", err)
daulog.Errorf("Error opening marked up file: %s", err)
retriesRemaining--
sleepForRetries(retriesRemaining)
continue
@ -166,7 +166,7 @@ func (u *Upload) processUpload() error {
} else {
filedata, err = os.Open(u.OriginalFilename)
if err != nil {
log.Print("Error opening original file:", err)
daulog.Errorf("Error opening original file: %s", err)
retriesRemaining--
sleepForRetries(retriesRemaining)
continue
@ -175,10 +175,10 @@ func (u *Upload) processUpload() error {
var imageData io.Reader
if u.watermark {
daulog.SendLog("Watermarking image", daulog.LogTypeInfo)
daulog.Info("Watermarking image")
imageData, err = u.applyWatermark(filedata)
if err != nil {
log.Print("Error watermarking:", err)
daulog.Errorf("Error watermarking: %s", err)
retriesRemaining--
sleepForRetries(retriesRemaining)
continue
@ -189,7 +189,7 @@ func (u *Upload) processUpload() error {
request, err := newfileUploadRequest(u.webhookURL, extraParams, "file", baseFilename, imageData)
if err != nil {
log.Printf("error creating upload request: %s", err)
daulog.Errorf("error creating upload request: %s", err)
return fmt.Errorf("could not create upload request: %s", err)
}
start := time.Now()
@ -202,24 +202,23 @@ func (u *Upload) processUpload() error {
resp, err := u.Client.Do(request)
if err != nil {
log.Print("Error performing request:", err)
daulog.Errorf("Error performing request: %s", err)
retriesRemaining--
sleepForRetries(retriesRemaining)
continue
} else {
if resp.StatusCode == 413 {
// just fail immediately, we know this means the file was too big
daulog.SendLog("413 received - file too large", daulog.LogTypeError)
daulog.Error("413 received - file too large")
u.State = StateFailed
return errors.New("received 413 - file too large")
}
if resp.StatusCode != 200 {
// {"message": "Request entity too large", "code": 40005}
log.Print("Bad response from server:", resp.StatusCode)
daulog.Errorf("Bad response code from server: %d", resp.StatusCode)
if b, err := ioutil.ReadAll(resp.Body); err == nil {
log.Print("Body:", string(b))
daulog.SendLog(fmt.Sprintf("Bad response: %s", string(b)), daulog.LogTypeError)
daulog.Errorf("Body:\n%s", string(b))
}
retriesRemaining--
sleepForRetries(retriesRemaining)
@ -228,7 +227,7 @@ func (u *Upload) processUpload() error {
resBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Print("could not deal with body: ", err)
daulog.Errorf("could not deal with body: %s", err)
retriesRemaining--
sleepForRetries(retriesRemaining)
continue
@ -240,17 +239,17 @@ func (u *Upload) processUpload() error {
// {"id": "851092588608880670", "type": 0, "content": "", "channel_id": "849615269706203171", "author": {"bot": true, "id": "849615314274484224", "username": "abcdedf", "avatar": null, "discriminator": "0000"}, "attachments": [{"id": "851092588332449812", "filename": "dau480457962.png", "size": 859505, "url": "https://cdn.discordapp.com/attachments/849615269706203171/851092588332449812/dau480457962.png", "proxy_url": "https://media.discordapp.net/attachments/849615269706203171/851092588332449812/dau480457962.png", "width": 640, "height": 640, "content_type": "image/png"}], "embeds": [], "mentions": [], "mention_roles": [], "pinned": false, "mention_everyone": false, "tts": false, "timestamp": "2021-06-06T13:38:05.660000+00:00", "edited_timestamp": null, "flags": 0, "components": [], "webhook_id": "849615314274484224"}
daulog.SendLog(fmt.Sprintf("Response: %s", string(resBody[:])), daulog.LogTypeDebug)
daulog.Debugf("Response: %s", string(resBody[:]))
if err != nil {
log.Print("could not parse JSON: ", err)
fmt.Println("Response was:", string(resBody[:]))
daulog.Errorf("could not parse JSON: %s", err)
daulog.Errorf("Response was: %s", string(resBody[:]))
retriesRemaining--
sleepForRetries(retriesRemaining)
continue
}
if len(res.Attachments) < 1 {
log.Print("bad response - no attachments?")
daulog.Error("bad response - no attachments?")
retriesRemaining--
sleepForRetries(retriesRemaining)
continue
@ -259,8 +258,8 @@ func (u *Upload) processUpload() error {
elapsed := time.Since(start)
rate := float64(a.Size) / elapsed.Seconds() / 1024.0
daulog.SendLog(fmt.Sprintf("Uploaded to %s %dx%d", a.URL, a.Width, a.Height), daulog.LogTypeInfo)
daulog.SendLog(fmt.Sprintf("id: %d, %d bytes transferred in %.2f seconds (%.2f KiB/s)", res.ID, a.Size, elapsed.Seconds(), rate), daulog.LogTypeInfo)
daulog.Infof("Uploaded to %s %dx%d", a.URL, a.Width, a.Height)
daulog.Infof("id: %d, %d bytes transferred in %.2f seconds (%.2f KiB/s)", res.ID, a.Size, elapsed.Seconds(), rate)
u.Url = a.URL
u.State = StateComplete
@ -276,7 +275,7 @@ func (u *Upload) processUpload() error {
u.RemoveMarkupTempFile()
if retriesRemaining == 0 {
daulog.SendLog("Failed to upload, even after all retries", daulog.LogTypeError)
daulog.Error("Failed to upload, even after all retries")
u.State = StateFailed
return errors.New("could not upload after all retries")
}
@ -317,7 +316,7 @@ func (u *Upload) applyWatermark(in *os.File) (io.Reader, error) {
im, _, err := image.Decode(in)
if err != nil {
daulog.SendLog(fmt.Sprintf("Cannot decode image: %v - skipping watermarking", err), daulog.LogTypeError)
daulog.Errorf("Cannot decode image: %v - skipping watermarking", err)
return nil, errors.New("cannot decode image")
}
bounds := im.Bounds()
@ -349,6 +348,6 @@ func sleepForRetries(retry int) {
return
}
retryTime := (6-retry)*(6-retry) + 6
daulog.SendLog(fmt.Sprintf("Will retry in %d seconds (%d remaining attempts)", retryTime, retry), daulog.LogTypeError)
daulog.Errorf("Will retry in %d seconds (%d remaining attempts)", retryTime, retry)
time.Sleep(time.Duration(retryTime) * time.Second)
}

16
version/version.go
View File

@ -38,7 +38,7 @@ func UpdateAvailable() bool {
if !semver.IsValid(LatestVersion) {
// maybe this should just be a warning
log.Printf("online version '%s' is not valid - assuming no new version", LatestVersion)
daulog.Errorf("online version '%s' is not valid - assuming no new version", LatestVersion)
return false
}
comp := semver.Compare(LatestVersion, CurrentVersion)
@ -53,20 +53,12 @@ func UpdateAvailable() bool {
func GetOnlineVersion() {
daulog.SendLog("checking for new version", daulog.LogTypeInfo)
LatestVersion = "v0.12.0"
LatestVersionInfo = GithubRelease{
HTMLURL: "https://github.com/tardisx/discord-auto-upload/releases/tag/v0.13.0",
TagName: "v0.13.0",
Name: "v0.13.0",
Body: "- cool things\n-wow things\n",
}
return
daulog.Info("checking for new version")
client := &http.Client{Timeout: time.Second * 5}
resp, err := client.Get("https://api.github.com/repos/tardisx/discord-auto-upload/releases/latest")
if err != nil {
daulog.SendLog(fmt.Sprintf("WARNING: Update check failed: %v", err), daulog.LogTypeError)
daulog.Errorf("WARNING: Update check failed: %s", err)
return
}
defer resp.Body.Close()
@ -84,4 +76,6 @@ func GetOnlineVersion() {
LatestVersion = latest.TagName
LatestVersionInfo = latest
daulog.Debugf("Latest version: %s", LatestVersion)
}

56
web/data/logs.html
View File

@ -1,24 +1,21 @@
{{ define "content" }}
<main role="main" class="inner DAU">
<main role="main" class="inner DAU" x-data="logs()" x-init="get_logs()">
<h1 class="DAU-heading">Logs</h1>
<p class="lead">Discord-auto-upload logs</p>
<div class="container">
<div class="row">
<div class="col-sm">
<button type="button" onClick="debug=1; get_logs();" class="btn btn-primary">all logs</button>
<button type="button" @click="debug = !debug" class="btn btn-primary" x-text="debug ? 'debug' : 'no debug'"></button>
</div>
<div class="col-sm">
<button type="button" onClick="debug=0; get_logs();" class="btn btn-primary">no debug</button>
</div>
<div class="col-sm">
<button type="button" id="scroll-button" onClick="toggle_scroll();" class="btn btn-primary">disable auto-scroll</button>
<button type="button" @click="scroll = !scroll" class="btn btn-primary" x-text="scroll ? 'auto-scroll' : 'no scroll'"></button>
</div>
</div>
</div>
<pre id="logs" class="text-left pre-scrollable">
<pre id="logs" x-text="text" class="text-left pre-scrollable">
</pre>
</main>
@ -27,35 +24,24 @@
{{ define "js" }}
<script>
var debug = 0;
var scrl = true;
$(document).ready(function() {
get_logs();
setInterval(function() { get_logs(); }, 1000);
});
function toggle_scroll() {
scrl = !scrl;
if (scrl) {
$('#scroll-button').text('disable auto-scroll');
function logs() {
return {
text: '', scroll: true, debug: false,
get_logs() {
fetch('/rest/logs?' + new URLSearchParams({ debug: this.debug ? "1" : "0" }))
.then(response => response.text())
.then(text => {
console.log(text);
this.text = text;
if (this.scroll) {
document.getElementById('logs').scrollTop =10000;
}
let self = this;
setTimeout(function() { self.get_logs(); }, 1000)
})
},
}
}
else {
$('#scroll-button').text('auto-scroll');
}
}
function get_logs() {
$.ajax({ method: 'get', url: '/rest/logs', data: { debug : debug }})
.done(function(data) {
$('#logs').text(data);
console.log('scrl is ', scrl);
if (scrl) {
$('#logs').scrollTop(10000);
}
});
}
</script>
{{ end }}

14
web/server.go
View File

@ -66,7 +66,7 @@ func (ws *WebService) getStatic(w http.ResponseWriter, r *http.Request) {
t, err := template.ParseFS(webFS, "data/wrapper.tmpl", "data/"+path)
if err != nil {
daulog.SendLog(fmt.Sprintf("when fetching: %s got: %s", path, err), daulog.LogTypeError)
daulog.Errorf("when fetching: %s got: %s", path, err)
w.Header().Add("Content-Type", "text/plain")
w.WriteHeader(http.StatusNotFound)
w.Write([]byte("not found"))
@ -94,7 +94,7 @@ func (ws *WebService) getStatic(w http.ResponseWriter, r *http.Request) {
otherStatic, err := webFS.ReadFile("data/" + path)
if err != nil {
daulog.SendLog(fmt.Sprintf("when fetching: %s got: %s", path, err), daulog.LogTypeError)
daulog.Errorf("when fetching: %s got: %s", path, err)
w.Header().Add("Content-Type", "text/plain")
w.WriteHeader(http.StatusNotFound)
w.Write([]byte("not found"))
@ -118,7 +118,7 @@ func (ws *WebService) getLogs(w http.ResponseWriter, r *http.Request) {
}
text := ""
for _, log := range daulog.LogEntries {
for _, log := range daulog.Memory.Entries() {
if !showDebug && log.Type == daulog.LogTypeDebug {
continue
}
@ -127,7 +127,6 @@ func (ws *WebService) getLogs(w http.ResponseWriter, r *http.Request) {
)
}
// js, _ := json.Marshal(daulog.LogEntries)
w.Write([]byte(text))
}
@ -347,12 +346,11 @@ func (ws *WebService) StartWebServer() {
go func() {
listen := fmt.Sprintf(":%d", ws.Config.Config.Port)
log.Printf("Starting web server on http://localhost%s", listen)
daulog.Infof("Starting web server on http://localhost%s", listen)
srv := &http.Server{
Handler: r,
Addr: listen,
// Good practice: enforce timeouts for servers you create!
Handler: r,
Addr: listen,
WriteTimeout: 15 * time.Second,
ReadTimeout: 15 * time.Second,
}