Update roadmap

.gitattributes

@@ -0,0 +1 @@
+content/corpora/* linguist-vendored

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,72 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+  schedule:
+    - cron: '18 22 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go', 'javascript' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        
+        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+        
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #   echo "Run, Build Application using script"
+    #   ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2

.github/workflows/go.yml

@@ -0,0 +1,25 @@
+name: Go
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.18
+
+    - name: Build
+      run: go build -v ./...
+
+    - name: Test
+      run: go test -v ./...

.goreleaser.yaml

@@ -21,7 +21,7 @@ dockers:
     build_flag_templates:
     - "--platform=linux/amd64"
 nfpms:
-  - 
+  -
     id: default
     package_name: linkwallet
     file_name_template: "{{ .ConventionalFileName }}"
@@ -86,6 +86,7 @@ changelog:
   sort: asc
   filters:
     exclude:
+      - '^Merge:'
       - '^docs:'
       - '^test:'
       - '^[Bb]ump'

README.md

@@ -32,6 +32,14 @@ A self-hosted bookmark database with full-text page content search.
     mountpoint.
 * Run `docker-compose up -d`
 
+## Packages (deb/rpm)
+
+* Download the .deb or .rpm from the releases
+* Install using apt/dpkg/rpm
+  * Automatically creates a systemd service, enabled and started
+  * Runs as user `linkwallet`
+  * Database stored in `/var/lib/linkwallet`
+
 ## Binary
 
 * Download the appropriate binary from the releases page
@@ -44,10 +52,6 @@ A self-hosted bookmark database with full-text page content search.
 * Checkout the code
 * `go build cmd/linkwallet/linkwallet.go`
 
-## deb/rpm packages
-
-Coming soon.
-
 # Using
 
 linkwallet is a 100% web-driven app. After running, hit the web interface
@@ -63,8 +67,7 @@ will not work.
 # Roadmap
 
 * More options when managing links
-  * delete
   * sorting
 * More tag options
-  * search for tags
-  * bookmarklet with pre-filled tags
+  * bookmarklet with pre-filled tags
+  * search/filter on tags

cmd/linkwallet/linkwallet.go

@@ -3,6 +3,7 @@ package main
 import (
 	"flag"
 	"log"
+	"time"
 
 	"github.com/tardisx/linkwallet/db"
 	"github.com/tardisx/linkwallet/version"
@@ -27,7 +28,12 @@ func main() {
 	bmm := db.NewBookmarkManager(&dbh)
 	cmm := db.NewConfigManager(&dbh)
 
-	go func() { version.UpdateVersionInfo() }()
+	go func() {
+		for {
+			version.UpdateVersionInfo()
+			time.Sleep(time.Hour * 6)
+		}
+	}()
 
 	log.Printf("linkwallet version %s starting", version.Is())
 

db/bookmarks.go

@@ -131,8 +131,6 @@ func (m *BookmarkManager) Search(query string) ([]entity.Bookmark, error) {
 		}
 	}
 
-	// log.Printf("counts: %#v", counts)
-
 	for k, v := range counts {
 		if v == uint8(len(words)) {
 			rets = append(rets, k)
@@ -149,6 +147,10 @@ func (m *BookmarkManager) ScrapeAndIndex(bm *entity.Bookmark) error {
 
 	log.Printf("Start scrape for %s", bm.URL)
 	info := content.FetchPageInfo(*bm)
+	// keep the existing title if necessary
+	if bm.PreserveTitle {
+		info.Title = bm.Info.Title
+	}
 	bm.Info = info
 	bm.TimestampLastScraped = time.Now()
 	err := m.SaveBookmark(bm)
@@ -156,11 +158,16 @@ func (m *BookmarkManager) ScrapeAndIndex(bm *entity.Bookmark) error {
 		panic(err)
 	}
 
+	m.UpdateIndexForBookmark(bm)
+	return nil
+
+}
+
+func (m *BookmarkManager) UpdateIndexForBookmark(bm *entity.Bookmark) {
 	words := content.Words(bm)
+	words = append(words, bm.Tags...)
 	log.Printf("index for %d %s (%d words)", bm.ID, bm.URL, len(words))
 	m.db.UpdateIndexForWordsByID(words, bm.ID)
-
-	return nil
 }
 
 func (m *BookmarkManager) QueueScrape(bm *entity.Bookmark) {

db/index_test.go

@@ -92,3 +92,52 @@ func TestAddRemove(t *testing.T) {
 	}
 
 }
+
+func TestTagIndexing(t *testing.T) {
+	ts := newTestServer()
+	defer ts.Close()
+	serverResponse = "<p>the quick brown fox</p>"
+
+	db := DB{}
+	f, _ := os.CreateTemp("", "test_boltdb_*")
+	f.Close()
+	defer os.Remove(f.Name())
+	db.Open(f.Name())
+
+	bmm := NewBookmarkManager(&db)
+	bm := entity.Bookmark{URL: ts.URL}
+
+	err := bmm.AddBookmark(&bm)
+	if err != nil {
+		t.Fatalf("error adding: %s", err)
+	}
+	if bm.ID == 0 {
+		t.Error("bookmark did not get an id")
+	}
+	err = bmm.ScrapeAndIndex(&bm)
+	if err != nil {
+		t.Errorf("scrape index returned %s", err)
+	}
+
+	searchRes, err := bmm.Search("fox")
+	if err != nil {
+		t.Errorf("search returned %s", err)
+	}
+	if len(searchRes) != 1 {
+		t.Error("did not get one id")
+	}
+
+	// add a tag
+	bm.Tags = []string{"sloth"}
+	err = bmm.ScrapeAndIndex(&bm)
+	if err != nil {
+		t.Errorf("scrape index returned %s", err)
+	}
+	searchRes, err = bmm.Search("sloth")
+	if err != nil {
+		t.Errorf("search returned %s", err)
+	}
+	if len(searchRes) != 1 {
+		t.Error("did not get one id for sloth")
+	}
+}

entity/bookmark.go

@@ -7,6 +7,7 @@ type Bookmark struct {
 	URL                  string
 	Info                 PageInfo
 	Tags                 []string
+	PreserveTitle        bool
 	TimestampCreated     time.Time
 	TimestampLastScraped time.Time
 }

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/mattn/go-isatty v0.0.12 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
 	github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect
-	github.com/stretchr/testify v1.6.1 // indirect
+	github.com/stretchr/testify v1.7.0 // indirect
 	github.com/ugorji/go/codec v1.1.7 // indirect
 	go.etcd.io/bbolt v1.3.6 // indirect
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
@@ -34,6 +34,7 @@ require (
 	github.com/antchfx/htmlquery v1.2.4 // indirect
 	github.com/antchfx/xmlquery v1.3.10 // indirect
 	github.com/antchfx/xpath v1.2.0 // indirect
+	github.com/gin-contrib/gzip v0.0.5
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect

go.sum

@@ -11,8 +11,11 @@ github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwq
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gin-contrib/gzip v0.0.5 h1:mhnVU32YnnBh2LPH2iqRqsA/eR7SAqRaD388jL2s/j0=
+github.com/gin-contrib/gzip v0.0.5/go.mod h1:OPIK6HR0Um2vNmBUTlayD7qle4yVVRZT0PyhdUigrKk=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
+github.com/gin-gonic/gin v1.7.4/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY=
 github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs=
 github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U=
 github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
@@ -73,6 +76,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
 github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
 github.com/timshannon/bolthold v0.0.0-20210913165410-232392fc8a6a h1:oIi7H/bwFUYKYhzKbHc+3MvHRWqhQwXVB4LweLMiVy0=

version/version.go

@@ -8,7 +8,7 @@ import (
 	"golang.org/x/mod/semver"
 )
 
-const Tag = "v0.0.15"
+const Tag = "v0.0.19"
 
 var versionInfo struct {
 	Local struct {
@@ -40,7 +40,14 @@ func UpgradeAvailable() (bool, string) {
 		return true, versionInfo.Remote.Tag
 	}
 	return false, ""
+}
 
+func UpgradeAvailableString() string {
+	upgrade, ver := UpgradeAvailable()
+	if upgrade {
+		return ver
+	}
+	return ""
 }
 
 func UpdateVersionInfo() {

web/templates/_layout.html

@@ -6,9 +6,11 @@
     <meta http-equiv="x-ua-compatible" content="ie=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>linkwallet</title>
-    <link rel="stylesheet" href="/assets/css/foundation.css">
+    <link rel="stylesheet" href="/assets/css/foundation.min.css">
     <link rel="stylesheet" href="/assets/css/app.css">
-    <script src="/assets/js/htmx.min.js" defer></script>
+    <script src="/assets/js/vendor/htmx.min.js" defer></script>
+    <script src="/assets/js/vendor/hyperscript_web.min.js"></script>
+
 
   </head>
 <body>
@@ -32,10 +34,17 @@
     </div>
     <div class="top-bar-right">
       <ul class="menu">
+        {{ if newVersion }}
+        <li>
+          <div><a href="https://github.com/tardisx/linkwallet/releases/tag/{{ newVersion }}">{{ newVersion }} available</a></div>
+        </li>
+        {{ end }}
+        <li class="menu-text">
+          {{ version }}
+        </li>
         <li>
           <a href="https://github.com/tardisx/linkwallet">
-            {{ version }}
-            <img src="/assets/image/GitHub-Mark-32px.png" />
+            <img src="/assets/image/GitHub-Mark-32px.png" width="16px" height="16px"/>
           </a>
         </li>
         <!-- <li><input type="search" placeholder="Search"></li>
@@ -60,9 +69,9 @@
     </div>
 
 
-    <script src="/assets/js/vendor/jquery.js"></script>
+    <!-- <script src="/assets/js/vendor/jquery.js"></script>
     <script src="/assets/js/vendor/what-input.js"></script>
     <script src="/assets/js/vendor/foundation.js"></script>
-    <script src="/assets/js/app.js"></script>
+    <script src="/assets/js/app.js"></script> -->
   </body>
 </html>

web/templates/add_url_form.html

@@ -7,7 +7,7 @@
     <form onsubmit="return false">
         <div class="grid-x grid-padding-x">
             <div class="large-6 cell">
-                <label>Paste a URL</label>
+                <label>URL</label>
                 <input type="text" name="url" value="{{ .url }}"
                       hx-trigger=""
                 />

web/templates/bookmarklet.html

@@ -9,7 +9,7 @@
         <p>Then whenever you are on a webpage you would like to bookmark, just
             click the bookmarklet.</p>
 
-        <a class="button" href="javascript:void(window.open('{{ .config.BaseURL }}/bookmarklet?url=' +encodeURIComponent(window.location), 'windowName', 'width=640,height=480'))">Bookmarklet</a>
+        <a class="button" href="javascript:void(window.open('{{ .config.BaseURL }}/bookmarklet?url=' +encodeURIComponent(window.location)+'&title='+encodeURIComponent(document.title), 'windowName', 'width=640,height=480'))">Bookmarklet</a>
 
 
         {{ else }}

web/templates/edit_form.html

@@ -3,7 +3,11 @@
     <table>
         <tr>
             <th>Title</th>
-            <td>{{ .bookmark.Info.Title }}</td>
+            <td>
+                <input type="text" name="title" {{ if not .bookmark.PreserveTitle }}disabled{{ end }} id="title" value="{{ .bookmark.Info.Title }}">
+                <input id="checkbox" name="override_title" value="on" {{ if  .bookmark.PreserveTitle }}checked{{end}} type="checkbox" _="on click toggle @disabled on #title">
+                <label for="checkbox">override scraped title (ignore page title when scraping)</label>
+            </td>
         </tr>
         <tr>
             <th>URL</th>

web/web.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/hako/durafmt"
 
+	"github.com/gin-contrib/gzip"
 	"github.com/gin-gonic/gin"
 )
 
@@ -51,7 +52,7 @@ func Create(bmm *db.BookmarkManager, cmm *db.ConfigManager) *Server {
 	}
 
 	// templ := template.Must(template.New("").Funcs(template.FuncMap{"dict": dictHelper}).ParseFS(templateFiles, "templates/*.html"))
-	templ := template.Must(template.New("").Funcs(template.FuncMap{"nicetime": niceTime, "niceURL": niceURL, "join": strings.Join, "version": version.Is}).ParseFS(templateFiles, "templates/*.html"))
+	templ := template.Must(template.New("").Funcs(template.FuncMap{"nicetime": niceTime, "niceURL": niceURL, "join": strings.Join, "version": version.Is, "newVersion": version.UpgradeAvailableString}).ParseFS(templateFiles, "templates/*.html"))
 
 	config, err := cmm.LoadConfig()
 	if err != nil {
@@ -67,6 +68,7 @@ func Create(bmm *db.BookmarkManager, cmm *db.ConfigManager) *Server {
 	}
 
 	r.Use(headersByURI())
+	r.Use(gzip.Gzip(gzip.DefaultCompression, gzip.WithExcludedExtensions([]string{".pdf", ".mp4"})))
 
 	r.SetHTMLTemplate(templ)
 	r.StaticFS("/assets", http.FS(staticFS))
@@ -95,6 +97,7 @@ func Create(bmm *db.BookmarkManager, cmm *db.ConfigManager) *Server {
 
 	r.POST("/config", func(c *gin.Context) {
 		config.BaseURL = c.PostForm("baseurl")
+		config.BaseURL = strings.TrimRight(config.BaseURL, "/")
 		cmm.SaveConfig(&config)
 		meta := gin.H{"config": config}
 
@@ -160,7 +163,6 @@ func Create(bmm *db.BookmarkManager, cmm *db.ConfigManager) *Server {
 			}
 		}
 
-		log.Printf("well done %v, %d", totalErrors, added)
 		data := gin.H{
 			"added":  added,
 			"errors": totalErrors,
@@ -261,6 +263,16 @@ func Create(bmm *db.BookmarkManager, cmm *db.ConfigManager) *Server {
 
 		bookmark := bmm.LoadBookmarkByID(bookmarkID)
 
+		// update title and override title
+		overrideTitle := c.PostForm("override_title")
+		if overrideTitle != "" {
+			title := c.PostForm("title")
+			bookmark.Info.Title = title
+			bookmark.PreserveTitle = true
+		} else {
+			bookmark.PreserveTitle = false
+		}
+
 		// freshen tags
 		if c.PostForm("tags_hidden") == "" {
 			// empty
@@ -268,7 +280,9 @@ func Create(bmm *db.BookmarkManager, cmm *db.ConfigManager) *Server {
 		} else {
 			bookmark.Tags = strings.Split(c.PostForm("tags_hidden"), "|")
 		}
+
 		bmm.SaveBookmark(&bookmark)
+		bmm.UpdateIndexForBookmark(&bookmark) // because title may have changed
 
 		meta := gin.H{"page": "edit", "bookmark": bookmark, "tw": gin.H{"tags": bookmark.Tags, "tags_hidden": strings.Join(bookmark.Tags, "|")}}