Deps

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,72 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+  schedule:
+    - cron: '18 22 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go', 'javascript' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        
+        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+        
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #   echo "Run, Build Application using script"
+    #   ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2

.github/workflows/go.yml

@@ -0,0 +1,25 @@
+name: Go
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.18
+
+    - name: Build
+      run: go build -v ./...
+
+    - name: Test
+      run: go test -v ./...

.goreleaser.yaml

@@ -6,32 +6,12 @@ before:
     - go test ./...
 
 builds:
-  - main: ./cmd/linkwallet/
-    ldflags:
-      - -s -w -X github.com/tardisx/linkwallet/version.version={{.Version}} -X github.com/tardisx/linkwallet/version.commit={{.Commit}} -X github.com/tardisx/linkwallet/version.date={{.Date}}
-    env:
+  - env:
       - CGO_ENABLED=0
     goos:
       - linux
       - windows
       - darwin
-      - freebsd
-    goarch:
-      - arm
-      - arm64
-      - amd64
-    goarm:
-      - 6
-      - 7
-    ignore:
-      - goos: darwin
-        goarch: arm
-      - goos: windows
-        goarch: arm
-      - goos: windows
-        goarch: arm64
-      - goos: freebsd
-        goarch: arm
 
 archives:
   - formats: [tar.gz]

.vscode/settings.json

@@ -10,8 +10,5 @@
         "serialised",
         "stopword",
         "Upsert"
-    ],
-    "cSpell.ignoreWords": [
-        "rescrape"
     ]
 }

README.md

@@ -5,16 +5,14 @@
 
 A self-hosted bookmark database with full-text page content search.
 
-linkwallet uses the [Bleve](https://blevesearch.com) indexing library, providing
-excellent support for free-text queries over the content of all your bookmarked
-pages.
+Searching uses English stemming, providing matches against similar words, in both page
+titles and page content. Searches are lightning fast.
 
 ![Search][screenshot_search]
 
-linkwallet indexes the page content, and automatically re-scrapes the pages
-periodically. Tags can be applied (though with the full-text search they are
-often not needed). Bookmarks can be easily managed, and can be imported or
-exported in bulk.
+Bookmark content is automatically re-scraped periodically. Tags can be applied (though with 
+the full-text search they are often not needed). Bookmarks can be easily managed, and can be
+imported or exported in bulk.
 
 ![Admin][screenshot_admin]
 
@@ -32,11 +30,14 @@ Bookmarks can be added with two clicks via the bookmarklet.
   * Page content periodically refreshed automatically
   * Interactively search across titles and content
   * Rippingly fast results, as you type
-    * full text search ~30ms (over full text content of 600 bookmarks)
+    * full text search ~60ms (over full text content of 600 bookmarks)
   * No need to remember how you filed something, you just need a keyword
     or two to discover it again
 * Embedded database, no separate database required
-* Extremely light on resources
+* Light on resources
+  * ~21Mb binary
+  * ~40Mb memory
+  * ~24Mb database (600 bookmarks, full text content indexed)
 * Easily export your bookmarks to a plain text file - your data is yours
 
 # Installation
@@ -56,7 +57,11 @@ To upgrade:
 
 ## Packages (deb/rpm)
 
-[not yet migrated to new goreleaser - please message me if you need packages]
+* Download the .deb or .rpm from the releases
+* Install using apt/dpkg/rpm
+  * Automatically creates a systemd service, enabled and started
+  * Runs as user `linkwallet`
+  * Database stored in `/var/lib/linkwallet`
 
 ## Binary
 

cmd/linkwallet/linkwallet.go

@@ -6,7 +6,7 @@ import (
 	"time"
 
 	"github.com/tardisx/linkwallet/db"
-	v "github.com/tardisx/linkwallet/version"
+	"github.com/tardisx/linkwallet/version"
 	"github.com/tardisx/linkwallet/web"
 )
 
@@ -21,7 +21,7 @@ func main() {
 	}
 
 	dbh := db.DB{}
-	rescrape, err := dbh.Open(dbPath)
+	err := dbh.Open(dbPath)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -31,7 +31,7 @@ func main() {
 
 	go func() {
 		for {
-			v.VersionInfo.UpdateVersionInfo()
+			version.VersionInfo.UpdateVersionInfo()
 			time.Sleep(time.Hour * 6)
 		}
 	}()
@@ -47,24 +47,10 @@ func main() {
 		}
 	}()
 
-	log.Printf("linkwallet version %s starting", v.VersionInfo.Local.Version)
+	log.Printf("linkwallet version %s starting", version.VersionInfo.Local.Tag)
 
 	server := web.Create(bmm, cmm)
 	go bmm.RunQueue()
 	go bmm.UpdateContent()
-
-	if rescrape {
-		log.Printf("queueing all bookmarks for rescraping, as index was just created")
-		bookmarks, err := bmm.AllBookmarks()
-		if err != nil {
-			log.Printf("could not load all bookmarks: %s", err.Error())
-		} else {
-			for _, bm := range bookmarks {
-				bmm.QueueScrape(&bm)
-			}
-		}
-		log.Printf("queued %d bookmarks for scraping", len(bookmarks))
-	}
-
 	server.Start()
 }

db/bookmarks.go

@@ -14,7 +14,6 @@ import (
 	"time"
 
 	"github.com/blevesearch/bleve/v2"
-	"github.com/blevesearch/bleve/v2/analysis/lang/en"
 	"github.com/blevesearch/bleve/v2/search/query"
 	"github.com/tardisx/linkwallet/content"
 	"github.com/tardisx/linkwallet/entity"
@@ -125,11 +124,11 @@ func (m *BookmarkManager) Search(opts SearchOptions) ([]entity.BookmarkSearchRes
 	if opts.All {
 		q = bleve.NewMatchAllQuery()
 	} else {
-		mq := bleve.NewMatchQuery(opts.Query)
-		mq.Analyzer = en.AnalyzerName
-		tq := bleve.NewTermQuery(opts.Query)
 
-		q = bleve.NewDisjunctionQuery(mq, tq)
+		q = bleve.NewDisjunctionQuery(
+			bleve.NewMatchQuery(opts.Query),
+			bleve.NewTermQuery(opts.Query),
+		)
 	}
 
 	req := bleve.NewSearchRequest(q)
@@ -189,6 +188,7 @@ func (m *BookmarkManager) UpdateIndexForBookmark(bm *entity.Bookmark) {
 	if err != nil {
 		panic(err)
 	}
+	log.Printf("done bleving")
 }
 
 func (m *BookmarkManager) QueueScrape(bm *entity.Bookmark) {
@@ -262,17 +262,6 @@ func (m *BookmarkManager) UpdateContent() {
 	}
 }
 
-// AllBookmarks returns all bookmarks. It does not use the index for this
-// operation.
-func (m *BookmarkManager) AllBookmarks() ([]entity.Bookmark, error) {
-	bookmarks := make([]entity.Bookmark, 0)
-	err := m.db.store.Find(&bookmarks, &bolthold.Query{})
-	if err != nil {
-		panic(err)
-	}
-	return bookmarks, nil
-}
-
 func (m *BookmarkManager) Stats() (entity.DBStats, error) {
 	stats := entity.DBStats{}
 	err := m.db.store.Get("stats", &stats)

db/db.go

@@ -19,17 +19,14 @@ type DB struct {
 	bleve bleve.Index
 }
 
-// Open opens the bookmark boltdb, and the bleve index. It returns
-// true if the index was newly created, so the caller knows all bookmarks
-// need to be re-scraped
-func (db *DB) Open(path string) (bool, error) {
+// Open opens the bookmark boltdb, and the bleve index.
+func (db *DB) Open(path string) error {
 	// options := bolthold.DefaultOptions
 	// options.Dir = dir
 	// options.ValueDir = dir
-	rescrapeNeeded := false
 	store, err := bolthold.Open(path, 0666, nil)
 	if err != nil {
-		return false, fmt.Errorf("cannot open '%s' - %s", path, err)
+		return fmt.Errorf("cannot open '%s' - %s", path, err)
 	}
 
 	blevePath := path + ".bleve"
@@ -40,21 +37,17 @@ func (db *DB) Open(path string) (bool, error) {
 		if err == bleve.ErrorIndexPathExists {
 			index, err = bleve.Open(blevePath)
 			if err != nil {
-				return false, fmt.Errorf("cannot open bleve '%s' - %s", path, err)
+				return fmt.Errorf("cannot open bleve '%s' - %s", path, err)
 			}
 		} else {
-			return false, fmt.Errorf("cannot open bleve '%s' - %s", path, err)
+			return fmt.Errorf("cannot open bleve '%s' - %s", path, err)
 		}
-	} else {
-		// we just created an index, one didn't exist, so we need to queue
-		// all bookmarks to be scraped
-		rescrapeNeeded = true
 	}
 
 	db.store = store
 	db.file = path
 	db.bleve = index
-	return rescrapeNeeded, nil
+	return nil
 }
 
 func createIndexMapping() mapping.IndexMapping {

entity/bookmark.go

@@ -2,7 +2,6 @@ package entity
 
 import (
 	"html/template"
-	"strings"
 	"time"
 )
 
@@ -20,13 +19,6 @@ func (bm Bookmark) Type() string {
 	return "bookmark"
 }
 
-func (bm Bookmark) DisplayTitle() string {
-	if strings.TrimSpace(bm.Info.Title) == "" {
-		return bm.URL
-	}
-	return bm.Info.Title
-}
-
 type PageInfo struct {
 	Fetched    time.Time
 	Title      string

entity/bookmark_test.go

@@ -1,22 +0,0 @@
-package entity
-
-import (
-	"testing"
-)
-
-func TestTitle(t *testing.T) {
-	bm := Bookmark{
-		URL: "http://example.org",
-		Info: PageInfo{
-			Title: "",
-		},
-	}
-	if bm.DisplayTitle() != "http://example.org" {
-		t.Errorf("title incorrect - got %s", bm.DisplayTitle())
-	}
-	bm.Info.Title = "Example Site"
-	if bm.DisplayTitle() != "Example Site" {
-		t.Errorf("title incorrect - got %s", bm.DisplayTitle())
-	}
-
-}

release_tag.pl

@@ -0,0 +1,15 @@
+#!/usr/bin/perl
+
+open my $fh, "<", "version/version.go" || die "oops";
+while (my $l =  <$fh>) {
+    if ($l =~ m/const Tag = "(.+)"/) {
+        $tag = $1;
+        system ('git', 'tag', '-a', $tag, '-m', "version $tag for release") ;
+        die "could not tag?\n" if $? != 0;
+        system ('git', 'push', 'origin', $tag);
+        die "could not push tag?\n" if $? != 0;
+        exit 0;
+    }
+}
+
+die "no version in version/version.go?\n";

version/version.go

@@ -3,7 +3,6 @@ package version
 import (
 	"context"
 	"fmt"
-	"log"
 	"strings"
 	"sync"
 
@@ -11,20 +10,11 @@ import (
 	"golang.org/x/mod/semver"
 )
 
-var version string // populated by goreleaser, without leading 'v'
-var commit string
-var date string
-
-var VersionInfo Info
-
-func init() {
-	VersionInfo.Remote.Valid = false
-	VersionInfo.Local.Version = "v" + version
-}
+const Tag = "v0.0.36"
 
 type Info struct {
 	Local struct {
-		Version string
+		Tag string
 	}
 	Remote struct {
 		Valid bool
@@ -34,29 +24,23 @@ type Info struct {
 	m                   sync.Mutex
 }
 
+var VersionInfo Info
+
+func init() {
+	VersionInfo.Remote.Valid = false
+	VersionInfo.Local.Tag = Tag
+}
+
 func (vi *Info) UpgradeAvailable() bool {
 	vi.m.Lock()
 	defer vi.m.Unlock()
 	if !vi.Remote.Valid {
 		return false
 	}
-
-	log.Printf("checking if upgrade available - local %s remote %s", vi.Local.Version, vi.Remote.Tag)
-	localValid := semver.IsValid(vi.Local.Version)
-	remoteValid := semver.IsValid(vi.Remote.Tag)
-
-	if !localValid {
-		log.Printf("version %s invalid", vi.Local.Version)
+	if semver.Compare(vi.Local.Tag, vi.Remote.Tag) < 0 {
+		return true
 	}
-	if !remoteValid {
-		log.Printf("version %s invalid", vi.Remote.Tag)
-	}
-
-	if !localValid || !remoteValid {
-		return false
-	}
-
-	return semver.Compare(vi.Local.Version, vi.Remote.Tag) < 0
+	return false
 }
 
 func (vi *Info) UpdateVersionInfo() {
@@ -75,7 +59,7 @@ func (vi *Info) UpdateVersionInfo() {
 	vi.Remote.Valid = true
 	vi.UpgradeReleaseNotes = ""
 	for _, r := range rels {
-		if semver.Compare(VersionInfo.Local.Version, *r.TagName) < 0 {
+		if semver.Compare(VersionInfo.Local.Tag, *r.TagName) < 0 {
 			vi.UpgradeReleaseNotes += fmt.Sprintf("*Version %s*\n\n", *r.TagName)
 			bodyLines := strings.Split(*r.Body, "\n")
 			for _, l := range bodyLines {

web/templates/_layout.html

@@ -23,7 +23,6 @@
         <li>
           <a href="#">Admin</a>
           <ul class="menu vertical">
-            <li><a href="/info">System Info</a></li>
             <li><a href="/config">Configuration</a></li>
             <li><a href="/manage">Manage links</a></li>
             <li><a href="/export">Export all URLs</a></li>
@@ -35,11 +34,12 @@
     </div>
     <div class="top-bar-right">
       <ul class="menu">
-        <li class="menu-text">
-          {{ version.Local.Version }}
+        <li>
+          <a href="/info">{{ version.Local.Tag }}
             {{ if version.UpgradeAvailable }}
-            <a href="/info">❗</a>
+            ❗
             {{ end }}
+          </a>
         </li>
         <li>
           <a href="https://github.com/tardisx/linkwallet">

web/templates/info.html

@@ -26,7 +26,7 @@
             <a href="https://github.com/tardisx/linkwallet/releases/tag/{{ version.Remote.Tag }}">
                 {{ version.Remote.Tag }}
             </a>
-            (you have {{ version.Local.Version }}).
+            (you have {{ version.Local.Tag }}).
         </p>
 
         {{ markdown version.UpgradeReleaseNotes }}

web/templates/search_results.html

@@ -1,7 +1,7 @@
 <ul>
     {{ range .results }}
     <li>
-         <a href="{{ .Bookmark.URL }}">{{ .Bookmark.DisplayTitle }}</a><br>
+         <a href="{{ .Bookmark.URL }}">{{ .Bookmark.Info.Title }}</a><br>
          {{ .Highlight }}
         </li>
     {{ end }}