From ccf373b863dffae62ea4c0171675cd825734513e Mon Sep 17 00:00:00 2001 From: Justin Hawkins Date: Sat, 20 Aug 2022 23:44:47 +0930 Subject: [PATCH] Record statistics on how many bookmarks and words are indexed, and the number of searches performed. --- cmd/linkwallet/linkwallet.go | 12 +++++ db/bookmarks.go | 2 + db/db.go | 86 +++++++++++++++++++++++++++++++++--- db/index.go | 15 ++++++- entity/meta.go | 34 ++++++++++++++ version/version.go | 2 +- 6 files changed, 144 insertions(+), 7 deletions(-) create mode 100644 entity/meta.go diff --git a/cmd/linkwallet/linkwallet.go b/cmd/linkwallet/linkwallet.go index c603de7..d2028b0 100644 --- a/cmd/linkwallet/linkwallet.go +++ b/cmd/linkwallet/linkwallet.go @@ -25,6 +25,7 @@ func main() { if err != nil { log.Fatal(err) } + bmm := db.NewBookmarkManager(&dbh) cmm := db.NewConfigManager(&dbh) @@ -35,6 +36,17 @@ func main() { } }() + // update stats every 5 minutes + go func() { + for { + err := dbh.UpdateBookmarkStats() + if err != nil { + panic(err) + } + time.Sleep(time.Minute * 5) + } + }() + log.Printf("linkwallet version %s starting", version.VersionInfo.Local.Tag) server := web.Create(bmm, cmm) diff --git a/db/bookmarks.go b/db/bookmarks.go index 74456df..e1fe2fc 100644 --- a/db/bookmarks.go +++ b/db/bookmarks.go @@ -179,6 +179,8 @@ func (m *BookmarkManager) Search(opts SearchOptions) ([]entity.Bookmark, error) panic(err) } + m.db.IncrementSearches() + return out, nil } diff --git a/db/db.go b/db/db.go index 819559b..5182cda 100644 --- a/db/db.go +++ b/db/db.go @@ -2,7 +2,7 @@ package db import ( "fmt" - "log" + "time" "github.com/tardisx/linkwallet/entity" bolthold "github.com/timshannon/bolthold" @@ -28,8 +28,84 @@ func (db *DB) Close() { db.store.Close() } -func (db *DB) Dumpy() { - res := make([]entity.Bookmark, 0, 0) - db.store.Find(&res, &bolthold.Query{}) - log.Printf("%v", res) +// func (db *DB) Dumpy() { +// res := make([]entity.Bookmark, 0, 0) +// db.store.Find(&res, &bolthold.Query{}) +// log.Printf("%v", res) +// } + +// IncrementSearches increments the number of searches we have ever performed by one. +func (db *DB) IncrementSearches() error { + txn, err := db.store.Bolt().Begin(true) + if err != nil { + return fmt.Errorf("could not start transaction for increment searches: %s", err) + } + + stats := entity.DBStats{} + err = db.store.TxGet(txn, "stats", &stats) + if err != nil && err != bolthold.ErrNotFound { + txn.Rollback() + return fmt.Errorf("could not get stats for incrementing searches: %s", err) + } + + stats.Searches += 1 + err = db.store.TxUpsert(txn, "stats", &stats) + if err != nil { + txn.Rollback() + return fmt.Errorf("could not upsert stats for incrementing searches: %s", err) + } + err = txn.Commit() + if err != nil { + return fmt.Errorf("could not commit increment searches transaction: %s", err) + } + + return nil +} + +// UpdateBookmarkStats updates the history on the number of bookmarks and words indexed. +func (db *DB) UpdateBookmarkStats() error { + + txn, err := db.store.Bolt().Begin(true) + if err != nil { + return fmt.Errorf("could not start transaction for update stats: %s", err) + } + // count bookmarks and words indexed + bmI := entity.Bookmark{} + wiI := entity.WordIndex{} + bookmarkCount, err := db.store.TxCount(txn, &bmI, &bolthold.Query{}) + if err != nil { + txn.Rollback() + return fmt.Errorf("could not get bookmark count: %s", err) + } + indexWordCount, err := db.store.TxCount(txn, &wiI, &bolthold.Query{}) + if err != nil { + txn.Rollback() + return fmt.Errorf("could not get index word count: %s", err) + } + + // bucket these stats by day + now := time.Now().Truncate(time.Hour * 24) + + stats := entity.DBStats{} + err = db.store.TxGet(txn, "stats", &stats) + if err != nil && err != bolthold.ErrNotFound { + txn.Rollback() + return fmt.Errorf("could not get stats: %s", err) + } + if stats.History == nil { + stats.History = make(map[time.Time]entity.BookmarkInfo) + } + stats.History[now] = entity.BookmarkInfo{Bookmarks: bookmarkCount, IndexedWords: indexWordCount} + err = db.store.TxUpsert(txn, "stats", &stats) + if err != nil { + txn.Rollback() + return fmt.Errorf("could not upsert stats: %s", err) + } + + err = txn.Commit() + if err != nil { + return fmt.Errorf("could not commit stats transaction: %s", err) + } + + return nil } diff --git a/db/index.go b/db/index.go index a2832ae..540d605 100644 --- a/db/index.go +++ b/db/index.go @@ -20,7 +20,20 @@ func (db *DB) UpdateIndexForWordsByID(words []string, id uint64) { } db.store.TxForEach(txn, &bolthold.Query{}, func(wi *entity.WordIndex) error { delete(wi.Bitmap, id) - db.store.TxUpdate(txn, "word_index_"+wi.Word, wi) + // if the index is now completely empty, nuke it entirely + empty := true + for _, v := range wi.Bitmap { + if v { + empty = false + break + } + } + + if empty { + db.store.TxDelete(txn, "word_index_"+wi.Word, wi) + } else { + db.store.TxUpdate(txn, "word_index_"+wi.Word, wi) + } return nil }) diff --git a/entity/meta.go b/entity/meta.go new file mode 100644 index 0000000..ce4bd7c --- /dev/null +++ b/entity/meta.go @@ -0,0 +1,34 @@ +package entity + +import ( + "fmt" + "sort" + "time" +) + +type DBStats struct { + History map[time.Time]BookmarkInfo + Searches int +} + +type BookmarkInfo struct { + Bookmarks int + IndexedWords int +} + +func (stats DBStats) String() string { + out := fmt.Sprintf("searches: %d\n", stats.Searches) + + dates := []time.Time{} + + for k := range stats.History { + dates = append(dates, k) + } + + sort.Slice(dates, func(i, j int) bool { return dates[i].Before(dates[j]) }) + + for _, k := range dates { + out += fmt.Sprintf("%s - %d bookmarks, %d words indexed\n", k, stats.History[k].Bookmarks, stats.History[k].IndexedWords) + } + return out +} diff --git a/version/version.go b/version/version.go index ef693cd..917f5ba 100644 --- a/version/version.go +++ b/version/version.go @@ -10,7 +10,7 @@ import ( "golang.org/x/mod/semver" ) -const Tag = "v0.0.29" +const Tag = "v0.0.30" type Info struct { Local struct {