Merge the feature branch to introduce the bleve indexing.

commit e8c2bc7e4a
Author: Justin Hawkins <justin@hawkins.id.au>
Date:   Sat May 3 12:36:18 2025 +0930

    Clean up menu/version

commit 1993533a46
Author: Justin Hawkins <justin@hawkins.id.au>
Date:   Sat May 3 12:31:50 2025 +0930

    Update README

commit 044cc830dc
Author: Justin Hawkins <justin@hawkins.id.au>
Date:   Sat May 3 12:22:33 2025 +0930

    No longer needed

commit a7c37ad7c5
Author: Justin Hawkins <justin@hawkins.id.au>
Date:   Sat May 3 12:14:59 2025 +0930

    Fixup version handling

commit ade0b748e9
Author: Justin Hawkins <justin@hawkins.id.au>
Date:   Sat May 3 11:58:15 2025 +0930

    Use the correct analyser for searches

commit e5a65cf5cf
Author: Justin Hawkins <justin@hawkins.id.au>
Date:   Fri May 2 19:51:54 2025 +0930

    Fix version in template

commit 0171be0ee4
Author: Justin Hawkins <justin@hawkins.id.au>
Date:   Fri May 2 19:51:41 2025 +0930

    Rescrape all links if needed on startup

commit ae654998f7
Author: Justin Hawkins <justin@hawkins.id.au>
Date:   Fri May 2 19:51:25 2025 +0930

    Spelling

commit bfe9bbee02
Author: Justin Hawkins <justin@hawkins.id.au>
Date:   Fri May 2 19:34:06 2025 +0930

    Make goreleaser set the version

commit 4436313413
Author: Justin Hawkins <justin@hawkins.id.au>
Date:   Fri May 2 11:43:58 2025 +0930

    Make release matrix sane

commit 7b467ecee7
Author: Justin Hawkins <justin@hawkins.id.au>
Date:   Fri May 2 11:40:07 2025 +0930

    I hate YAML, so much.

commit b578e0f044
Author: Justin Hawkins <justin@hawkins.id.au>
Date:   Thu May 1 23:47:07 2025 +0930

    Update goreleaser

commit fba84f0827
Author: Justin Hawkins <justin@hawkins.id.au>
Date:   Thu May 1 23:45:46 2025 +0930

    Update version

commit e4edb08bd1
Author: Justin Hawkins <justin@hawkins.id.au>
Date:   Thu May 1 23:42:59 2025 +0930

    Deps

commit 58b6692d1b
Author: Justin Hawkins <justin@hawkins.id.au>
Date:   Thu May 1 23:39:51 2025 +0930

    Mostly done, first cut

commit badbe5e92f
Author: Justin Hawkins <justin@hawkins.id.au>
Date:   Sun Apr 27 20:28:37 2025 +0930

    Remove unused code

commit 903240dd18
Author: Justin Hawkins <justin@hawkins.id.au>
Date:   Sun Apr 27 20:26:19 2025 +0930

    Update deps

commit de90b9951a
Author: Justin Hawkins <justin@hawkins.id.au>
Date:   Sun Apr 27 20:21:33 2025 +0930

    Keep on bleving

commit 9b15528510
Author: Justin Hawkins <justin@hawkins.id.au>
Date:   Fri Apr 25 23:57:04 2025 +0930

    Start of blevification

.goreleaser.yaml

@@ -6,12 +6,32 @@ before:
     - go test ./...
 
 builds:
-  - env:
+  - main: ./cmd/linkwallet/
+    ldflags:
+      - -s -w -X github.com/tardisx/linkwallet/version.version={{.Version}} -X github.com/tardisx/linkwallet/version.commit={{.Commit}} -X github.com/tardisx/linkwallet/version.date={{.Date}}
+    env:
       - CGO_ENABLED=0
     goos:
       - linux
       - windows
       - darwin
+      - freebsd
+    goarch:
+      - arm
+      - arm64
+      - amd64
+    goarm:
+      - 6
+      - 7
+    ignore:
+      - goos: darwin
+        goarch: arm
+      - goos: windows
+        goarch: arm
+      - goos: windows
+        goarch: arm64
+      - goos: freebsd
+        goarch: arm
 
 archives:
   - formats: [tar.gz]

.vscode/settings.json

@@ -10,5 +10,8 @@
         "serialised",
         "stopword",
         "Upsert"
+    ],
+    "cSpell.ignoreWords": [
+        "rescrape"
     ]
 }

README.md

@@ -5,14 +5,16 @@
 
 A self-hosted bookmark database with full-text page content search.
 
-Searching uses English stemming, providing matches against similar words, in both page
+linkwallet uses the [Bleve](https://blevesearch.com) indexing library, providing
-titles and page content. Searches are lightning fast.
+excellent support for free-text queries over the content of all your bookmarked
+pages.
 
 ![Search][screenshot_search]
 
-Bookmark content is automatically re-scraped periodically. Tags can be applied (though with 
+linkwallet indexes the page content, and automatically re-scrapes the pages
-the full-text search they are often not needed). Bookmarks can be easily managed, and can be
+periodically. Tags can be applied (though with the full-text search they are
-imported or exported in bulk.
+often not needed). Bookmarks can be easily managed, and can be imported or
+exported in bulk.
 
 ![Admin][screenshot_admin]
 
@@ -30,14 +32,11 @@ Bookmarks can be added with two clicks via the bookmarklet.
   * Page content periodically refreshed automatically
   * Interactively search across titles and content
   * Rippingly fast results, as you type
-    * full text search ~60ms (over full text content of 600 bookmarks)
+    * full text search ~30ms (over full text content of 600 bookmarks)
   * No need to remember how you filed something, you just need a keyword
     or two to discover it again
 * Embedded database, no separate database required
-* Light on resources
+* Extremely light on resources
-  * ~21Mb binary
-  * ~40Mb memory
-  * ~24Mb database (600 bookmarks, full text content indexed)
 * Easily export your bookmarks to a plain text file - your data is yours
 
 # Installation
@@ -57,11 +56,7 @@ To upgrade:
 
 ## Packages (deb/rpm)
 
-* Download the .deb or .rpm from the releases
+[not yet migrated to new goreleaser - please message me if you need packages]
-* Install using apt/dpkg/rpm
-  * Automatically creates a systemd service, enabled and started
-  * Runs as user `linkwallet`
-  * Database stored in `/var/lib/linkwallet`
 
 ## Binary
 

cmd/linkwallet/linkwallet.go

@@ -6,7 +6,7 @@ import (
 	"time"
 
 	"github.com/tardisx/linkwallet/db"
-	"github.com/tardisx/linkwallet/version"
+	v "github.com/tardisx/linkwallet/version"
 	"github.com/tardisx/linkwallet/web"
 )
 
@@ -21,7 +21,7 @@ func main() {
 	}
 
 	dbh := db.DB{}
-	err := dbh.Open(dbPath)
+	rescrape, err := dbh.Open(dbPath)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -31,7 +31,7 @@ func main() {
 
 	go func() {
 		for {
-			version.VersionInfo.UpdateVersionInfo()
+			v.VersionInfo.UpdateVersionInfo()
 			time.Sleep(time.Hour * 6)
 		}
 	}()
@@ -47,10 +47,24 @@ func main() {
 		}
 	}()
 
-	log.Printf("linkwallet version %s starting", version.VersionInfo.Local.Tag)
+	log.Printf("linkwallet version %s starting", v.VersionInfo.Local.Version)
 
 	server := web.Create(bmm, cmm)
 	go bmm.RunQueue()
 	go bmm.UpdateContent()
+
+	if rescrape {
+		log.Printf("queueing all bookmarks for rescraping, as index was just created")
+		bookmarks, err := bmm.AllBookmarks()
+		if err != nil {
+			log.Printf("could not load all bookmarks: %s", err.Error())
+		} else {
+			for _, bm := range bookmarks {
+				bmm.QueueScrape(&bm)
+			}
+		}
+		log.Printf("queued %d bookmarks for scraping", len(bookmarks))
+	}
+
 	server.Start()
 }

db/bookmarks.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/blevesearch/bleve/v2"
+	"github.com/blevesearch/bleve/v2/analysis/lang/en"
 	"github.com/blevesearch/bleve/v2/search/query"
 	"github.com/tardisx/linkwallet/content"
 	"github.com/tardisx/linkwallet/entity"
@@ -124,11 +125,11 @@ func (m *BookmarkManager) Search(opts SearchOptions) ([]entity.BookmarkSearchRes
 	if opts.All {
 		q = bleve.NewMatchAllQuery()
 	} else {
+		mq := bleve.NewMatchQuery(opts.Query)
+		mq.Analyzer = en.AnalyzerName
+		tq := bleve.NewTermQuery(opts.Query)
 
-		q = bleve.NewDisjunctionQuery(
+		q = bleve.NewDisjunctionQuery(mq, tq)
-			bleve.NewMatchQuery(opts.Query),
-			bleve.NewTermQuery(opts.Query),
-		)
 	}
 
 	req := bleve.NewSearchRequest(q)
@@ -188,7 +189,6 @@ func (m *BookmarkManager) UpdateIndexForBookmark(bm *entity.Bookmark) {
 	if err != nil {
 		panic(err)
 	}
-	log.Printf("done bleving")
 }
 
 func (m *BookmarkManager) QueueScrape(bm *entity.Bookmark) {
@@ -262,6 +262,17 @@ func (m *BookmarkManager) UpdateContent() {
 	}
 }
 
+// AllBookmarks returns all bookmarks. It does not use the index for this
+// operation.
+func (m *BookmarkManager) AllBookmarks() ([]entity.Bookmark, error) {
+	bookmarks := make([]entity.Bookmark, 0)
+	err := m.db.store.Find(&bookmarks, &bolthold.Query{})
+	if err != nil {
+		panic(err)
+	}
+	return bookmarks, nil
+}
+
 func (m *BookmarkManager) Stats() (entity.DBStats, error) {
 	stats := entity.DBStats{}
 	err := m.db.store.Get("stats", &stats)

db/db.go

@@ -19,14 +19,17 @@ type DB struct {
 	bleve bleve.Index
 }
 
-// Open opens the bookmark boltdb, and the bleve index.
+// Open opens the bookmark boltdb, and the bleve index. It returns
-func (db *DB) Open(path string) error {
+// true if the index was newly created, so the caller knows all bookmarks
+// need to be re-scraped
+func (db *DB) Open(path string) (bool, error) {
 	// options := bolthold.DefaultOptions
 	// options.Dir = dir
 	// options.ValueDir = dir
+	rescrapeNeeded := false
 	store, err := bolthold.Open(path, 0666, nil)
 	if err != nil {
-		return fmt.Errorf("cannot open '%s' - %s", path, err)
+		return false, fmt.Errorf("cannot open '%s' - %s", path, err)
 	}
 
 	blevePath := path + ".bleve"
@@ -37,17 +40,21 @@ func (db *DB) Open(path string) error {
 		if err == bleve.ErrorIndexPathExists {
 			index, err = bleve.Open(blevePath)
 			if err != nil {
-				return fmt.Errorf("cannot open bleve '%s' - %s", path, err)
+				return false, fmt.Errorf("cannot open bleve '%s' - %s", path, err)
 			}
 		} else {
-			return fmt.Errorf("cannot open bleve '%s' - %s", path, err)
+			return false, fmt.Errorf("cannot open bleve '%s' - %s", path, err)
 		}
+	} else {
+		// we just created an index, one didn't exist, so we need to queue
+		// all bookmarks to be scraped
+		rescrapeNeeded = true
 	}
 
 	db.store = store
 	db.file = path
 	db.bleve = index
-	return nil
+	return rescrapeNeeded, nil
 }
 
 func createIndexMapping() mapping.IndexMapping {

release_tag.pl

@@ -1,15 +0,0 @@
-#!/usr/bin/perl
-
-open my $fh, "<", "version/version.go" || die "oops";
-while (my $l =  <$fh>) {
-    if ($l =~ m/const Tag = "(.+)"/) {
-        $tag = $1;
-        system ('git', 'tag', '-a', $tag, '-m', "version $tag for release") ;
-        die "could not tag?\n" if $? != 0;
-        system ('git', 'push', 'origin', $tag);
-        die "could not push tag?\n" if $? != 0;
-        exit 0;
-    }
-}
-
-die "no version in version/version.go?\n";

version/version.go

@@ -3,6 +3,7 @@ package version
 import (
 	"context"
 	"fmt"
+	"log"
 	"strings"
 	"sync"
 
@@ -10,11 +11,20 @@ import (
 	"golang.org/x/mod/semver"
 )
 
-const Tag = "v0.0.36"
+var version string // populated by goreleaser, without leading 'v'
+var commit string
+var date string
+
+var VersionInfo Info
+
+func init() {
+	VersionInfo.Remote.Valid = false
+	VersionInfo.Local.Version = "v" + version
+}
 
 type Info struct {
 	Local struct {
-		Tag string
+		Version string
 	}
 	Remote struct {
 		Valid bool
@@ -24,25 +34,31 @@ type Info struct {
 	m                   sync.Mutex
 }
 
-var VersionInfo Info
-
-func init() {
-	VersionInfo.Remote.Valid = false
-	VersionInfo.Local.Tag = Tag
-}
-
 func (vi *Info) UpgradeAvailable() bool {
 	vi.m.Lock()
 	defer vi.m.Unlock()
 	if !vi.Remote.Valid {
 		return false
 	}
-	if semver.Compare(vi.Local.Tag, vi.Remote.Tag) < 0 {
+
-		return true
+	log.Printf("checking if upgrade available - local %s remote %s", vi.Local.Version, vi.Remote.Tag)
+	localValid := semver.IsValid(vi.Local.Version)
+	remoteValid := semver.IsValid(vi.Remote.Tag)
+
+	if !localValid {
+		log.Printf("version %s invalid", vi.Local.Version)
 	}
+	if !remoteValid {
+		log.Printf("version %s invalid", vi.Remote.Tag)
+	}
+
+	if !localValid || !remoteValid {
 		return false
 	}
 
+	return semver.Compare(vi.Local.Version, vi.Remote.Tag) < 0
+}
+
 func (vi *Info) UpdateVersionInfo() {
 	client := github.NewClient(nil)
 
@@ -59,7 +75,7 @@ func (vi *Info) UpdateVersionInfo() {
 	vi.Remote.Valid = true
 	vi.UpgradeReleaseNotes = ""
 	for _, r := range rels {
-		if semver.Compare(VersionInfo.Local.Tag, *r.TagName) < 0 {
+		if semver.Compare(VersionInfo.Local.Version, *r.TagName) < 0 {
 			vi.UpgradeReleaseNotes += fmt.Sprintf("*Version %s*\n\n", *r.TagName)
 			bodyLines := strings.Split(*r.Body, "\n")
 			for _, l := range bodyLines {

web/templates/_layout.html

@@ -23,6 +23,7 @@
         <li>
           <a href="#">Admin</a>
           <ul class="menu vertical">
+            <li><a href="/info">System Info</a></li>
             <li><a href="/config">Configuration</a></li>
             <li><a href="/manage">Manage links</a></li>
             <li><a href="/export">Export all URLs</a></li>
@@ -34,12 +35,11 @@
     </div>
     <div class="top-bar-right">
       <ul class="menu">
-        <li>
+        <li class="menu-text">
-          <a href="/info">{{ version.Local.Tag }}
+          {{ version.Local.Version }}
             {{ if version.UpgradeAvailable }}
-            ❗
+            <a href="/info">❗</a>
             {{ end }}
-          </a>
         </li>
         <li>
           <a href="https://github.com/tardisx/linkwallet">

web/templates/info.html

@@ -26,7 +26,7 @@
             <a href="https://github.com/tardisx/linkwallet/releases/tag/{{ version.Remote.Tag }}">
                 {{ version.Remote.Tag }}
             </a>
-            (you have {{ version.Local.Tag }}).
+            (you have {{ version.Local.Version }}).
         </p>
 
         {{ markdown version.UpgradeReleaseNotes }}