Provide indicator when editing bookmarks

.goreleaser.yaml

@@ -90,3 +90,4 @@ changelog:
       - '^docs:'
       - '^test:'
       - '^[Bb]ump'
+      - '^[Cc]lean'

README.md

@@ -32,6 +32,11 @@ A self-hosted bookmark database with full-text page content search.
     mountpoint.
 * Run `docker-compose up -d`
 
+To upgrade:
+
+* `docker-compose pull`
+* `docker-compose up -d`
+
 ## Packages (deb/rpm)
 
 * Download the .deb or .rpm from the releases

cmd/linkwallet/linkwallet.go

@@ -25,17 +25,29 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
+
 	bmm := db.NewBookmarkManager(&dbh)
 	cmm := db.NewConfigManager(&dbh)
 
 	go func() {
 		for {
-			version.UpdateVersionInfo()
+			version.VersionInfo.UpdateVersionInfo()
 			time.Sleep(time.Hour * 6)
 		}
 	}()
 
-	log.Printf("linkwallet version %s starting", version.Is())
+	// update stats every 5 minutes
+	go func() {
+		for {
+			err := dbh.UpdateBookmarkStats()
+			if err != nil {
+				panic(err)
+			}
+			time.Sleep(time.Minute * 5)
+		}
+	}()
+
+	log.Printf("linkwallet version %s starting", version.VersionInfo.Local.Tag)
 
 	server := web.Create(bmm, cmm)
 	go bmm.RunQueue()

db/bookmarks.go

@@ -100,7 +100,6 @@ func (m *BookmarkManager) SaveBookmark(bm *entity.Bookmark) error {
 func (m *BookmarkManager) LoadBookmarkByID(id uint64) entity.Bookmark {
 	// log.Printf("loading %v", ids)
 	ret := entity.Bookmark{}
-	log.Printf("loading id %d", id)
 	err := m.db.store.Get(id, &ret)
 	if err != nil {
 		panic(err)
@@ -179,6 +178,8 @@ func (m *BookmarkManager) Search(opts SearchOptions) ([]entity.Bookmark, error)
 		panic(err)
 	}
 
+	m.db.IncrementSearches()
+
 	return out, nil
 }
 
@@ -205,7 +206,6 @@ func (m *BookmarkManager) ScrapeAndIndex(bm *entity.Bookmark) error {
 func (m *BookmarkManager) UpdateIndexForBookmark(bm *entity.Bookmark) {
 	words := content.Words(bm)
 	words = append(words, bm.Tags...)
-	log.Printf("index for %d %s (%d words)", bm.ID, bm.URL, len(words))
 	m.db.UpdateIndexForWordsByID(words, bm.ID)
 }
 
@@ -279,3 +279,12 @@ func (m *BookmarkManager) UpdateContent() {
 		time.Sleep(time.Second * 5)
 	}
 }
+
+func (m *BookmarkManager) Stats() (entity.DBStats, error) {
+	stats := entity.DBStats{}
+	err := m.db.store.Get("stats", &stats)
+	if err != nil && err != bolthold.ErrNotFound {
+		return stats, fmt.Errorf("could not load stats: %s", err)
+	}
+	return stats, nil
+}

db/db.go

@@ -2,7 +2,7 @@ package db
 
 import (
 	"fmt"
-	"log"
+	"time"
 
 	"github.com/tardisx/linkwallet/entity"
 	bolthold "github.com/timshannon/bolthold"
@@ -28,8 +28,84 @@ func (db *DB) Close() {
 	db.store.Close()
 }
 
-func (db *DB) Dumpy() {
+// func (db *DB) Dumpy() {
-	res := make([]entity.Bookmark, 0, 0)
+// 	res := make([]entity.Bookmark, 0, 0)
-	db.store.Find(&res, &bolthold.Query{})
+// 	db.store.Find(&res, &bolthold.Query{})
-	log.Printf("%v", res)
+// 	log.Printf("%v", res)
+// }
+
+// IncrementSearches increments the number of searches we have ever performed by one.
+func (db *DB) IncrementSearches() error {
+	txn, err := db.store.Bolt().Begin(true)
+	if err != nil {
+		return fmt.Errorf("could not start transaction for increment searches: %s", err)
+	}
+
+	stats := entity.DBStats{}
+	err = db.store.TxGet(txn, "stats", &stats)
+	if err != nil && err != bolthold.ErrNotFound {
+		txn.Rollback()
+		return fmt.Errorf("could not get stats for incrementing searches: %s", err)
+	}
+
+	stats.Searches += 1
+	err = db.store.TxUpsert(txn, "stats", &stats)
+	if err != nil {
+		txn.Rollback()
+		return fmt.Errorf("could not upsert stats for incrementing searches: %s", err)
+	}
+	err = txn.Commit()
+	if err != nil {
+		return fmt.Errorf("could not commit increment searches transaction: %s", err)
+	}
+
+	return nil
+}
+
+// UpdateBookmarkStats updates the history on the number of bookmarks and words indexed.
+func (db *DB) UpdateBookmarkStats() error {
+
+	txn, err := db.store.Bolt().Begin(true)
+	if err != nil {
+		return fmt.Errorf("could not start transaction for update stats: %s", err)
+	}
+	// count bookmarks and words indexed
+	bmI := entity.Bookmark{}
+	wiI := entity.WordIndex{}
+	bookmarkCount, err := db.store.TxCount(txn, &bmI, &bolthold.Query{})
+	if err != nil {
+		txn.Rollback()
+		return fmt.Errorf("could not get bookmark count: %s", err)
+	}
+	indexWordCount, err := db.store.TxCount(txn, &wiI, &bolthold.Query{})
+	if err != nil {
+		txn.Rollback()
+		return fmt.Errorf("could not get index word count: %s", err)
+	}
+
+	// bucket these stats by day
+	now := time.Now().Truncate(time.Hour * 24)
+
+	stats := entity.DBStats{}
+	err = db.store.TxGet(txn, "stats", &stats)
+	if err != nil && err != bolthold.ErrNotFound {
+		txn.Rollback()
+		return fmt.Errorf("could not get stats: %s", err)
+	}
+	if stats.History == nil {
+		stats.History = make(map[time.Time]entity.BookmarkInfo)
+	}
+	stats.History[now] = entity.BookmarkInfo{Bookmarks: bookmarkCount, IndexedWords: indexWordCount}
+	err = db.store.TxUpsert(txn, "stats", &stats)
+	if err != nil {
+		txn.Rollback()
+		return fmt.Errorf("could not upsert stats: %s", err)
+	}
+
+	err = txn.Commit()
+	if err != nil {
+		return fmt.Errorf("could not commit stats transaction: %s", err)
+	}
+
+	return nil
 }

db/index.go

@@ -20,7 +20,20 @@ func (db *DB) UpdateIndexForWordsByID(words []string, id uint64) {
 	}
 	db.store.TxForEach(txn, &bolthold.Query{}, func(wi *entity.WordIndex) error {
 		delete(wi.Bitmap, id)
-		db.store.TxUpdate(txn, "word_index_"+wi.Word, wi)
+		// if the index is now completely empty, nuke it entirely
+		empty := true
+		for _, v := range wi.Bitmap {
+			if v {
+				empty = false
+				break
+			}
+		}
+
+		if empty {
+			db.store.TxDelete(txn, "word_index_"+wi.Word, wi)
+		} else {
+			db.store.TxUpdate(txn, "word_index_"+wi.Word, wi)
+		}
 		return nil
 	})
 

entity/meta.go

@@ -0,0 +1,47 @@
+package entity
+
+import (
+	"fmt"
+	"sort"
+	"time"
+)
+
+type DBStats struct {
+	History  map[time.Time]BookmarkInfo
+	Searches int
+}
+
+type BookmarkInfo struct {
+	Bookmarks    int
+	IndexedWords int
+}
+
+func (stats DBStats) String() string {
+	out := fmt.Sprintf("searches: %d\n", stats.Searches)
+
+	dates := []time.Time{}
+
+	for k := range stats.History {
+		dates = append(dates, k)
+	}
+
+	sort.Slice(dates, func(i, j int) bool { return dates[i].Before(dates[j]) })
+
+	for _, k := range dates {
+		out += fmt.Sprintf("%s - %d bookmarks, %d words indexed\n", k, stats.History[k].Bookmarks, stats.History[k].IndexedWords)
+	}
+	return out
+}
+
+func (stats DBStats) MostRecentBookmarkInfo() BookmarkInfo {
+	mostRecent := time.Time{}
+	for k := range stats.History {
+		if k.After(mostRecent) {
+			mostRecent = k
+		}
+	}
+	if !mostRecent.IsZero() {
+		return stats.History[mostRecent]
+	}
+	return BookmarkInfo{}
+}

go.mod

@@ -37,6 +37,7 @@ require (
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/gomarkdown/markdown v0.0.0-20220627144906-e9a81102ebeb
 	github.com/google/go-github/v44 v44.1.0
 	github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b
 	github.com/kennygrant/sanitize v1.2.4 // indirect

go.sum

@@ -38,6 +38,8 @@ github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaW
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/gomarkdown/markdown v0.0.0-20220627144906-e9a81102ebeb h1:5b/eFaSaKPFG9ygDBaPKkydKU5nFJYk08g9jPIVogMg=
+github.com/gomarkdown/markdown v0.0.0-20220627144906-e9a81102ebeb/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=

meta/meta.go

@@ -0,0 +1,14 @@
+package meta
+
+import (
+	"fmt"
+	"runtime"
+)
+
+func MemInfo() string {
+	stats := runtime.MemStats{}
+	runtime.ReadMemStats(&stats)
+
+	return fmt.Sprintf("%.3fMb", float64(stats.Alloc)/1024.0/1024.0)
+
+}

version/version.go

@@ -2,15 +2,17 @@ package version
 
 import (
 	"context"
+	"fmt"
+	"strings"
 	"sync"
 
 	"github.com/google/go-github/v44/github"
 	"golang.org/x/mod/semver"
 )
 
-const Tag = "v0.0.27"
+const Tag = "v0.0.31"
 
-var versionInfo struct {
+type Info struct {
 	Local struct {
 		Tag string
 	}
@@ -18,39 +20,30 @@ var versionInfo struct {
 		Valid bool
 		Tag   string
 	}
-	m sync.Mutex
+	UpgradeReleaseNotes string
+	m                   sync.Mutex
 }
 
+var VersionInfo Info
+
 func init() {
-	versionInfo.Remote.Valid = false
+	VersionInfo.Remote.Valid = false
-	versionInfo.Local.Tag = Tag
+	VersionInfo.Local.Tag = Tag
 }
 
-func Is() string {
+func (vi *Info) UpgradeAvailable() bool {
-	return versionInfo.Local.Tag
+	vi.m.Lock()
-}
+	defer vi.m.Unlock()
-
+	if !vi.Remote.Valid {
-func UpgradeAvailable() (bool, string) {
+		return false
-	versionInfo.m.Lock()
-	defer versionInfo.m.Unlock()
-	if !versionInfo.Remote.Valid {
-		return false, ""
 	}
-	if semver.Compare(versionInfo.Local.Tag, versionInfo.Remote.Tag) < 0 {
+	if semver.Compare(vi.Local.Tag, vi.Remote.Tag) < 0 {
-		return true, versionInfo.Remote.Tag
+		return true
 	}
-	return false, ""
+	return false
 }
 
-func UpgradeAvailableString() string {
+func (vi *Info) UpdateVersionInfo() {
-	upgrade, ver := UpgradeAvailable()
-	if upgrade {
-		return ver
-	}
-	return ""
-}
-
-func UpdateVersionInfo() {
 	client := github.NewClient(nil)
 
 	rels, _, err := client.Repositories.ListReleases(context.Background(), "tardisx", "linkwallet", nil)
@@ -60,9 +53,25 @@ func UpdateVersionInfo() {
 	if len(rels) == 0 {
 		return
 	}
-	versionInfo.m.Lock()
+
-	versionInfo.Remote.Tag = *rels[0].TagName
+	vi.m.Lock()
-	versionInfo.Remote.Valid = true
+	vi.Remote.Tag = *rels[0].TagName
-	versionInfo.m.Unlock()
+	vi.Remote.Valid = true
+	vi.UpgradeReleaseNotes = ""
+	for _, r := range rels {
+		if semver.Compare(VersionInfo.Local.Tag, *r.TagName) < 0 {
+			vi.UpgradeReleaseNotes += fmt.Sprintf("*Version %s*\n\n", *r.TagName)
+			bodyLines := strings.Split(*r.Body, "\n")
+			for _, l := range bodyLines {
+				if strings.Index(l, "#") == 0 && strings.Contains(l, "Changelog") {
+					// do nothing, ignore the changelog heading
+				} else {
+					vi.UpgradeReleaseNotes += l + "\n"
+				}
+			}
+		}
+	}
+
+	vi.m.Unlock()
 
 }

web/templates/_layout.html

@@ -34,13 +34,12 @@
     </div>
     <div class="top-bar-right">
       <ul class="menu">
-        {{ if newVersion }}
         <li>
-          <div><a href="https://github.com/tardisx/linkwallet/releases/tag/{{ newVersion }}">{{ newVersion }} available</a></div>
+          <a href="/info">{{ version.Local.Tag }}
-        </li>
+            {{ if version.UpgradeAvailable }}
-        {{ end }}
+            ❗
-        <li class="menu-text">
+            {{ end }}
-          {{ version }}
+          </a>
         </li>
         <li>
           <a href="https://github.com/tardisx/linkwallet">
@@ -64,6 +63,8 @@
       {{ template "config.html" . }}
       {{ else if eq .page "edit" }}
       {{ template "edit.html" . }}
+      {{ else if eq .page "info" }}
+      {{ template "info.html" . }}
       {{ end }}
       {{/* template "foundation_sample.html" . */}}
     </div>

web/templates/edit_form.html

@@ -30,7 +30,10 @@
         </tr>
     </table>
     <p>
-        <button type="button" hx-confirm="Delete this bookmark permanently?" hx-delete="/edit/{{.bookmark.ID}}" class="alert button">delete</button>
+        <button type="button" hx-confirm="Delete this bookmark permanently?" hx-delete="/edit/{{.bookmark.ID}}" class="alert button">Delete</button>
-        <button type="button" class="button" hx-post="/edit/{{.bookmark.ID}}">save</button>
+        <button type="button" class="button" hx-indicator="#saving" hx-post="/edit/{{.bookmark.ID}}"> {{ if .saved }} Saved {{ else }} Save {{ end }}</button>
+        <span id="saving" class="htmx-indicator">
+            <img style="height:1em;" src="/assets/image/beating.gif" /> Saving...
+        </span>
     </p>
 </form>

web/templates/info.html

@@ -0,0 +1,30 @@
+<div class="grid-x grid-padding-x">
+    <div class="large-12 cell">
+
+        <h5>System information</h5>
+        <table>
+            <tr><th>Memory in use</th><td>{{ meminfo }}</td></tr>
+            <tr><th>Bookmarks</th><td>{{ .stats.MostRecentBookmarkInfo.Bookmarks }}</td></tr>
+            <tr><th>Words Indexed</th><td>{{ .stats.MostRecentBookmarkInfo.IndexedWords }}</td></tr>
+        </table>
+
+        <h5>Release info</h5>
+        {{ if not version.Remote.Valid }}
+        <p>GitHub version information not yet fetched.</p>
+        {{ else }}
+        {{ if version.UpgradeAvailable }}
+        <p>
+            A new version is available: 
+            <a href="https://github.com/tardisx/linkwallet/releases/tag/{{ version.Remote.Tag }}">
+                {{ version.Remote.Tag }}
+            </a>
+            (you have {{ version.Local.Tag }}).
+        </p>
+
+        {{ markdown version.UpgradeReleaseNotes }}
+
+        {{ else }}
+        <p>You are currently running the most recent version.</p>
+        {{ end }}
+        {{ end }}
+</div>

web/web.go

@@ -14,8 +14,10 @@ import (
 
 	"github.com/tardisx/linkwallet/db"
 	"github.com/tardisx/linkwallet/entity"
+	"github.com/tardisx/linkwallet/meta"
 	"github.com/tardisx/linkwallet/version"
 
+	"github.com/gomarkdown/markdown"
 	"github.com/hako/durafmt"
 
 	"github.com/gin-contrib/gzip"
@@ -75,7 +77,15 @@ func Create(bmm *db.BookmarkManager, cmm *db.ConfigManager) *Server {
 	}
 
 	// templ := template.Must(template.New("").Funcs(template.FuncMap{"dict": dictHelper}).ParseFS(templateFiles, "templates/*.html"))
-	templ := template.Must(template.New("").Funcs(template.FuncMap{"nicetime": niceTime, "niceURL": niceURL, "join": strings.Join, "version": version.Is, "newVersion": version.UpgradeAvailableString}).ParseFS(templateFiles, "templates/*.html"))
+	templ := template.Must(template.New("").Funcs(
+		template.FuncMap{
+			"nicetime": niceTime,
+			"niceURL":  niceURL,
+			"join":     strings.Join,
+			"version":  func() *version.Info { return &version.VersionInfo },
+			"meminfo":  meta.MemInfo,
+			"markdown": func(s string) template.HTML { return template.HTML(string(markdown.ToHTML([]byte(s), nil, nil))) },
+		}).ParseFS(templateFiles, "templates/*.html"))
 
 	config, err := cmm.LoadConfig()
 	if err != nil {
@@ -361,7 +371,7 @@ func Create(bmm *db.BookmarkManager, cmm *db.ConfigManager) *Server {
 		bmm.SaveBookmark(&bookmark)
 		bmm.UpdateIndexForBookmark(&bookmark) // because title may have changed
 
-		meta := gin.H{"page": "edit", "bookmark": bookmark, "tw": gin.H{"tags": bookmark.Tags, "tags_hidden": strings.Join(bookmark.Tags, "|")}}
+		meta := gin.H{"page": "edit", "bookmark": bookmark, "saved": true, "tw": gin.H{"tags": bookmark.Tags, "tags_hidden": strings.Join(bookmark.Tags, "|")}}
 
 		c.HTML(http.StatusOK,
 			"edit_form.html", meta,
@@ -386,6 +396,17 @@ func Create(bmm *db.BookmarkManager, cmm *db.ConfigManager) *Server {
 		)
 	})
 
+	r.GET("/info", func(c *gin.Context) {
+		dbStats, err := bmm.Stats()
+		if err != nil {
+			panic("could not load stats for info page")
+		}
+		meta := gin.H{"page": "info", "stats": dbStats, "config": config}
+		c.HTML(http.StatusOK,
+			"_layout.html", meta,
+		)
+	})
+
 	return server
 }