Rescrape all links if needed on startup

cmd/linkwallet/linkwallet.go

@@ -21,7 +21,7 @@ func main() {
 	}
 
 	dbh := db.DB{}
-	err := dbh.Open(dbPath)
+	rescrape, err := dbh.Open(dbPath)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -52,5 +52,19 @@ func main() {
 	server := web.Create(bmm, cmm)
 	go bmm.RunQueue()
 	go bmm.UpdateContent()
+
+	if rescrape {
+		log.Printf("queueing all bookmarks for rescraping, as index was just created")
+		bookmarks, err := bmm.AllBookmarks()
+		if err != nil {
+			log.Printf("could not load all bookmarks: %s", err.Error())
+		} else {
+			for _, bm := range bookmarks {
+				bmm.QueueScrape(&bm)
+			}
+		}
+		log.Printf("queued %d bookmarks for scraping", len(bookmarks))
+	}
+
 	server.Start()
 }

db/bookmarks.go

@@ -262,6 +262,17 @@ func (m *BookmarkManager) UpdateContent() {
 	}
 }
 
+// AllBookmarks returns all bookmarks. It does not use the index for this
+// operation.
+func (m *BookmarkManager) AllBookmarks() ([]entity.Bookmark, error) {
+	bookmarks := make([]entity.Bookmark, 0)
+	err := m.db.store.Find(&bookmarks, &bolthold.Query{})
+	if err != nil {
+		panic(err)
+	}
+	return bookmarks, nil
+}
+
 func (m *BookmarkManager) Stats() (entity.DBStats, error) {
 	stats := entity.DBStats{}
 	err := m.db.store.Get("stats", &stats)

db/db.go

@@ -19,14 +19,17 @@ type DB struct {
 	bleve bleve.Index
 }
 
-// Open opens the bookmark boltdb, and the bleve index.
-func (db *DB) Open(path string) error {
+// Open opens the bookmark boltdb, and the bleve index. It returns
+// true if the index was newly created, so the caller knows all bookmarks
+// need to be re-scraped
+func (db *DB) Open(path string) (bool, error) {
 	// options := bolthold.DefaultOptions
 	// options.Dir = dir
 	// options.ValueDir = dir
+	rescrapeNeeded := false
 	store, err := bolthold.Open(path, 0666, nil)
 	if err != nil {
-		return fmt.Errorf("cannot open '%s' - %s", path, err)
+		return false, fmt.Errorf("cannot open '%s' - %s", path, err)
 	}
 
 	blevePath := path + ".bleve"
@@ -37,17 +40,21 @@ func (db *DB) Open(path string) error {
 		if err == bleve.ErrorIndexPathExists {
 			index, err = bleve.Open(blevePath)
 			if err != nil {
-				return fmt.Errorf("cannot open bleve '%s' - %s", path, err)
+				return false, fmt.Errorf("cannot open bleve '%s' - %s", path, err)
 			}
 		} else {
-			return fmt.Errorf("cannot open bleve '%s' - %s", path, err)
+			return false, fmt.Errorf("cannot open bleve '%s' - %s", path, err)
 		}
+	} else {
+		// we just created an index, one didn't exist, so we need to queue
+		// all bookmarks to be scraped
+		rescrapeNeeded = true
 	}
 
 	db.store = store
 	db.file = path
 	db.bleve = index
-	return nil
+	return rescrapeNeeded, nil
 }
 
 func createIndexMapping() mapping.IndexMapping {