diff --git a/cmd/linkwallet/linkwallet.go b/cmd/linkwallet/linkwallet.go index 7384a89..39ec42c 100644 --- a/cmd/linkwallet/linkwallet.go +++ b/cmd/linkwallet/linkwallet.go @@ -21,7 +21,7 @@ func main() { } dbh := db.DB{} - err := dbh.Open(dbPath) + rescrape, err := dbh.Open(dbPath) if err != nil { log.Fatal(err) } @@ -52,5 +52,19 @@ func main() { server := web.Create(bmm, cmm) go bmm.RunQueue() go bmm.UpdateContent() + + if rescrape { + log.Printf("queueing all bookmarks for rescraping, as index was just created") + bookmarks, err := bmm.AllBookmarks() + if err != nil { + log.Printf("could not load all bookmarks: %s", err.Error()) + } else { + for _, bm := range bookmarks { + bmm.QueueScrape(&bm) + } + } + log.Printf("queued %d bookmarks for scraping", len(bookmarks)) + } + server.Start() } diff --git a/db/bookmarks.go b/db/bookmarks.go index 5adc509..7e82e7f 100644 --- a/db/bookmarks.go +++ b/db/bookmarks.go @@ -262,6 +262,17 @@ func (m *BookmarkManager) UpdateContent() { } } +// AllBookmarks returns all bookmarks. It does not use the index for this +// operation. +func (m *BookmarkManager) AllBookmarks() ([]entity.Bookmark, error) { + bookmarks := make([]entity.Bookmark, 0) + err := m.db.store.Find(&bookmarks, &bolthold.Query{}) + if err != nil { + panic(err) + } + return bookmarks, nil +} + func (m *BookmarkManager) Stats() (entity.DBStats, error) { stats := entity.DBStats{} err := m.db.store.Get("stats", &stats) diff --git a/db/db.go b/db/db.go index c5d0a7d..8cb0ec8 100644 --- a/db/db.go +++ b/db/db.go @@ -19,14 +19,17 @@ type DB struct { bleve bleve.Index } -// Open opens the bookmark boltdb, and the bleve index. -func (db *DB) Open(path string) error { +// Open opens the bookmark boltdb, and the bleve index. It returns +// true if the index was newly created, so the caller knows all bookmarks +// need to be re-scraped +func (db *DB) Open(path string) (bool, error) { // options := bolthold.DefaultOptions // options.Dir = dir // options.ValueDir = dir + rescrapeNeeded := false store, err := bolthold.Open(path, 0666, nil) if err != nil { - return fmt.Errorf("cannot open '%s' - %s", path, err) + return false, fmt.Errorf("cannot open '%s' - %s", path, err) } blevePath := path + ".bleve" @@ -37,17 +40,21 @@ func (db *DB) Open(path string) error { if err == bleve.ErrorIndexPathExists { index, err = bleve.Open(blevePath) if err != nil { - return fmt.Errorf("cannot open bleve '%s' - %s", path, err) + return false, fmt.Errorf("cannot open bleve '%s' - %s", path, err) } } else { - return fmt.Errorf("cannot open bleve '%s' - %s", path, err) + return false, fmt.Errorf("cannot open bleve '%s' - %s", path, err) } + } else { + // we just created an index, one didn't exist, so we need to queue + // all bookmarks to be scraped + rescrapeNeeded = true } db.store = store db.file = path db.bleve = index - return nil + return rescrapeNeeded, nil } func createIndexMapping() mapping.IndexMapping {