justin/linkwallet
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases 36 Wiki Activity

Rework to use bolthold/boltdb instead of badgerhold/badgerdb, much more memory efficient

Browse Source
This commit is contained in:
Justin Hawkins
2022-05-28 16:16:08 +09:30
parent 10e205dd8e
commit 8e08329711
10 changed files with 47 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
db/bookmarks.go
View File

@@ -10,7 +10,7 @@ import (
"github.com/tardisx/linkwallet/content"
"github.com/tardisx/linkwallet/entity"
"github.com/timshannon/badgerhold/v4"
bolthold "github.com/timshannon/bolthold"
)
type BookmarkManager struct {
@@ -27,12 +27,12 @@ func NewBookmarkManager(db *DB) *BookmarkManager {
// The entity.Bookmark ID field will be updated.
func (m *BookmarkManager) AddBookmark(bm *entity.Bookmark) error {
existing := entity.Bookmark{}
err := m.db.store.FindOne(&existing, badgerhold.Where("URL").Eq(bm.URL))
if err != badgerhold.ErrNotFound {
err := m.db.store.FindOne(&existing, bolthold.Where("URL").Eq(bm.URL))
if err != bolthold.ErrNotFound {
return fmt.Errorf("bookmark already exists")
}
bm.TimestampCreated = time.Now()
err = m.db.store.Insert(badgerhold.NextSequence(), bm)
err = m.db.store.Insert(bolthold.NextSequence(), bm)
if err != nil {
return fmt.Errorf("addBookmark returned: %w", err)
}
@@ -42,7 +42,7 @@ func (m *BookmarkManager) AddBookmark(bm *entity.Bookmark) error {
// ListBookmarks returns all bookmarks.
func (m *BookmarkManager) ListBookmarks() ([]entity.Bookmark, error) {
bookmarks := make([]entity.Bookmark, 0, 0)
err := m.db.store.Find(&bookmarks, &badgerhold.Query{})
err := m.db.store.Find(&bookmarks, &bolthold.Query{})
if err != nil {
panic(err)
}
@@ -52,7 +52,7 @@ func (m *BookmarkManager) ListBookmarks() ([]entity.Bookmark, error) {
// ExportBookmarks exports all bookmarks to an io.Writer
func (m *BookmarkManager) ExportBookmarks(w io.Writer) error {
bms := []entity.Bookmark{}
err := m.db.store.Find(&bms, &badgerhold.Query{})
err := m.db.store.Find(&bms, &bolthold.Query{})
if err != nil {
return fmt.Errorf("could not export bookmarks: %w", err)
}
@@ -90,7 +90,7 @@ func (m *BookmarkManager) LoadBookmarksByIDs(ids []uint64) []entity.Bookmark {
s[i] = v
}
err := m.db.store.Find(&ret, badgerhold.Where("ID").In(s...))
err := m.db.store.Find(&ret, bolthold.Where("ID").In(s...))
if err != nil {
panic(err)
}
@@ -107,7 +107,7 @@ func (m *BookmarkManager) Search(query string) ([]entity.Bookmark, error) {
for _, word := range words {
var wi *entity.WordIndex
err := m.db.store.Get("word_index_"+word, &wi)
if err == badgerhold.ErrNotFound {
if err == bolthold.ErrNotFound {
continue
}
if err != nil {
@@ -202,8 +202,8 @@ func (m *BookmarkManager) UpdateContent() {
for {
ret = []entity.Bookmark{}
deadline := time.Now().Add(time.Hour * -24 * 7)
err := m.db.store.Find(&ret, badgerhold.Where("TimestampLastScraped").Lt(deadline))
if err == badgerhold.ErrNotFound {
err := m.db.store.Find(&ret, bolthold.Where("TimestampLastScraped").Lt(deadline))
if err == bolthold.ErrNotFound {
log.Printf("none qualify")
time.Sleep(time.Second)
continue

6
db/config.go
View File

@@ -5,7 +5,7 @@ import (
"log"
"github.com/tardisx/linkwallet/entity"
"github.com/timshannon/badgerhold/v4"
bolthold "github.com/timshannon/bolthold"
)
type ConfigManager struct {
@@ -18,14 +18,14 @@ func NewConfigManager(db *DB) *ConfigManager {
func (cmm *ConfigManager) LoadConfig() (entity.Config, error) {
config := entity.Config{}
err := cmm.db.store.FindOne(&config, &badgerhold.Query{})
err := cmm.db.store.FindOne(&config, &bolthold.Query{})
if err == nil {
if config.Version == 1 {
return config, nil
} else {
return entity.Config{}, fmt.Errorf("failed to load config - wrong version %d", config.Version)
}
} else if err == badgerhold.ErrNotFound {
} else if err == bolthold.ErrNotFound {
log.Printf("using default config")
return cmm.DefaultConfig(), nil
} else {

15
db/db.go
View File

@@ -4,19 +4,18 @@ import (
"log"
"github.com/tardisx/linkwallet/entity"
badgerhold "github.com/timshannon/badgerhold/v4"
bolthold "github.com/timshannon/bolthold"
)
type DB struct {
store *badgerhold.Store
store *bolthold.Store
}
func (db *DB) Open(dir string) {
options := badgerhold.DefaultOptions
options.Dir = dir
options.ValueDir = dir
store, err := badgerhold.Open(options)
// options := bolthold.DefaultOptions
// options.Dir = dir
// options.ValueDir = dir
store, err := bolthold.Open("bolt.db", 0666, nil)
if err != nil {
panic(err)
@@ -30,6 +29,6 @@ func (db *DB) Close() {
func (db *DB) Dumpy() {
res := make([]entity.Bookmark, 0, 0)
db.store.Find(&res, &badgerhold.Query{})
db.store.Find(&res, &bolthold.Query{})
log.Printf("%v", res)
}

22
db/index.go
View File

@@ -5,20 +5,22 @@ import (
"time"
"github.com/tardisx/linkwallet/entity"
badgerhold "github.com/timshannon/badgerhold/v4"
bolthold "github.com/timshannon/bolthold"
)
func (db *DB) InitIndices() {
wi := entity.WordIndex{}
db.store.DeleteMatching(wi, &badgerhold.Query{})
db.store.DeleteMatching(wi, &bolthold.Query{})
}
func (db *DB) UpdateIndexForWordsByID(words []string, id uint64) {
// delete this id from all indices
txn := db.store.Badger().NewTransaction(true)
txn, err := db.store.Bolt().Begin(true)
if err != nil {
panic(err)
}
db.store.TxForEach(txn, &badgerhold.Query{}, func(wi *entity.WordIndex) {
db.store.TxForEach(txn, &bolthold.Query{}, func(wi *entity.WordIndex) {
// log.Printf("considering this one: %s", wi.Word)
delete(wi.Bitmap, id)
})
@@ -30,8 +32,7 @@ func (db *DB) UpdateIndexForWordsByID(words []string, id uint64) {
tF := time.Now()
thisWI := entity.WordIndex{Word: word}
err := db.store.TxGet(txn, "word_index_"+word, &thisWI)
// err := db.store.TxFindOne(txn, &thisWI, badgerhold.Where("Word").Eq(word).Index("Word"))
if err == badgerhold.ErrNotFound {
if err == bolthold.ErrNotFound {
// create it
thisWI.Bitmap = map[uint64]bool{}
} else if err != nil {
@@ -52,7 +53,10 @@ func (db *DB) UpdateIndexForWordsByID(words []string, id uint64) {
if i > 0 && i%100 == 0 {
txn.Commit()
txn = db.store.Badger().NewTransaction(true)
txn, err = db.store.Bolt().Begin(true)
if err != nil {
panic(err)
}
}
}
@@ -64,7 +68,7 @@ func (db *DB) UpdateIndexForWordsByID(words []string, id uint64) {
func (db *DB) DumpIndex() {
// delete this id from all indices
err := db.store.ForEach(&badgerhold.Query{}, func(wi *entity.WordIndex) error {
err := db.store.ForEach(&bolthold.Query{}, func(wi *entity.WordIndex) error {
log.Printf("%10s: %v", wi.Word, wi.Bitmap)
return nil
})