Record statistics on how many bookmarks and words are indexed, and the number of searches performed.

cmd/linkwallet/linkwallet.go

@@ -25,6 +25,7 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
+
 	bmm := db.NewBookmarkManager(&dbh)
 	cmm := db.NewConfigManager(&dbh)
 
@@ -35,6 +36,17 @@ func main() {
 		}
 	}()
 
+	// update stats every 5 minutes
+	go func() {
+		for {
+			err := dbh.UpdateBookmarkStats()
+			if err != nil {
+				panic(err)
+			}
+			time.Sleep(time.Minute * 5)
+		}
+	}()
+
 	log.Printf("linkwallet version %s starting", version.VersionInfo.Local.Tag)
 
 	server := web.Create(bmm, cmm)

db/bookmarks.go

@@ -179,6 +179,8 @@ func (m *BookmarkManager) Search(opts SearchOptions) ([]entity.Bookmark, error)
 		panic(err)
 	}
 
+	m.db.IncrementSearches()
+
 	return out, nil
 }
 

db/db.go

@@ -2,7 +2,7 @@ package db
 
 import (
 	"fmt"
-	"log"
+	"time"
 
 	"github.com/tardisx/linkwallet/entity"
 	bolthold "github.com/timshannon/bolthold"
@@ -28,8 +28,84 @@ func (db *DB) Close() {
 	db.store.Close()
 }
 
-func (db *DB) Dumpy() {
+// func (db *DB) Dumpy() {
-	res := make([]entity.Bookmark, 0, 0)
+// 	res := make([]entity.Bookmark, 0, 0)
-	db.store.Find(&res, &bolthold.Query{})
+// 	db.store.Find(&res, &bolthold.Query{})
-	log.Printf("%v", res)
+// 	log.Printf("%v", res)
+// }
+
+// IncrementSearches increments the number of searches we have ever performed by one.
+func (db *DB) IncrementSearches() error {
+	txn, err := db.store.Bolt().Begin(true)
+	if err != nil {
+		return fmt.Errorf("could not start transaction for increment searches: %s", err)
+	}
+
+	stats := entity.DBStats{}
+	err = db.store.TxGet(txn, "stats", &stats)
+	if err != nil && err != bolthold.ErrNotFound {
+		txn.Rollback()
+		return fmt.Errorf("could not get stats for incrementing searches: %s", err)
+	}
+
+	stats.Searches += 1
+	err = db.store.TxUpsert(txn, "stats", &stats)
+	if err != nil {
+		txn.Rollback()
+		return fmt.Errorf("could not upsert stats for incrementing searches: %s", err)
+	}
+	err = txn.Commit()
+	if err != nil {
+		return fmt.Errorf("could not commit increment searches transaction: %s", err)
+	}
+
+	return nil
+}
+
+// UpdateBookmarkStats updates the history on the number of bookmarks and words indexed.
+func (db *DB) UpdateBookmarkStats() error {
+
+	txn, err := db.store.Bolt().Begin(true)
+	if err != nil {
+		return fmt.Errorf("could not start transaction for update stats: %s", err)
+	}
+	// count bookmarks and words indexed
+	bmI := entity.Bookmark{}
+	wiI := entity.WordIndex{}
+	bookmarkCount, err := db.store.TxCount(txn, &bmI, &bolthold.Query{})
+	if err != nil {
+		txn.Rollback()
+		return fmt.Errorf("could not get bookmark count: %s", err)
+	}
+	indexWordCount, err := db.store.TxCount(txn, &wiI, &bolthold.Query{})
+	if err != nil {
+		txn.Rollback()
+		return fmt.Errorf("could not get index word count: %s", err)
+	}
+
+	// bucket these stats by day
+	now := time.Now().Truncate(time.Hour * 24)
+
+	stats := entity.DBStats{}
+	err = db.store.TxGet(txn, "stats", &stats)
+	if err != nil && err != bolthold.ErrNotFound {
+		txn.Rollback()
+		return fmt.Errorf("could not get stats: %s", err)
+	}
+	if stats.History == nil {
+		stats.History = make(map[time.Time]entity.BookmarkInfo)
+	}
+	stats.History[now] = entity.BookmarkInfo{Bookmarks: bookmarkCount, IndexedWords: indexWordCount}
+	err = db.store.TxUpsert(txn, "stats", &stats)
+	if err != nil {
+		txn.Rollback()
+		return fmt.Errorf("could not upsert stats: %s", err)
+	}
+
+	err = txn.Commit()
+	if err != nil {
+		return fmt.Errorf("could not commit stats transaction: %s", err)
+	}
+
+	return nil
 }

db/index.go

@@ -20,7 +20,20 @@ func (db *DB) UpdateIndexForWordsByID(words []string, id uint64) {
 	}
 	db.store.TxForEach(txn, &bolthold.Query{}, func(wi *entity.WordIndex) error {
 		delete(wi.Bitmap, id)
-		db.store.TxUpdate(txn, "word_index_"+wi.Word, wi)
+		// if the index is now completely empty, nuke it entirely
+		empty := true
+		for _, v := range wi.Bitmap {
+			if v {
+				empty = false
+				break
+			}
+		}
+
+		if empty {
+			db.store.TxDelete(txn, "word_index_"+wi.Word, wi)
+		} else {
+			db.store.TxUpdate(txn, "word_index_"+wi.Word, wi)
+		}
 		return nil
 	})
 

entity/meta.go

@@ -0,0 +1,34 @@
+package entity
+
+import (
+	"fmt"
+	"sort"
+	"time"
+)
+
+type DBStats struct {
+	History  map[time.Time]BookmarkInfo
+	Searches int
+}
+
+type BookmarkInfo struct {
+	Bookmarks    int
+	IndexedWords int
+}
+
+func (stats DBStats) String() string {
+	out := fmt.Sprintf("searches: %d\n", stats.Searches)
+
+	dates := []time.Time{}
+
+	for k := range stats.History {
+		dates = append(dates, k)
+	}
+
+	sort.Slice(dates, func(i, j int) bool { return dates[i].Before(dates[j]) })
+
+	for _, k := range dates {
+		out += fmt.Sprintf("%s - %d bookmarks, %d words indexed\n", k, stats.History[k].Bookmarks, stats.History[k].IndexedWords)
+	}
+	return out
+}

version/version.go

@@ -10,7 +10,7 @@ import (
 	"golang.org/x/mod/semver"
 )
 
-const Tag = "v0.0.29"
+const Tag = "v0.0.30"
 
 type Info struct {
 	Local struct {