Updated dependencies.

.github/workflows/go.yml

@@ -34,9 +34,8 @@ jobs:
     - name: Download
       run: go mod download
 
-    # Fixed in go 1.21: https://go.dev/issue/54372
-    # - name: Verify
-    #   run: go mod verify
+    - name: Verify
+      run: go mod verify
 
     - name: Vet
       run: go vet ./...
@@ -48,8 +47,12 @@ jobs:
     - name: Test
       run: go test -v ./...
 
+    - name: Test no locks
+      run: go test -v -tags sqlite3_nolock .
+      if: matrix.os == 'ubuntu-latest'
+
     - name: Test BSD locks
-      run: go test -v -tags sqlite3_bsd ./...
+      run: go test -v -tags sqlite3_flock ./...
       if: matrix.os == 'macos-latest'
 
     - name: Coverage report
@@ -59,6 +62,6 @@ jobs:
         amend: 'true'
         reuse-go: 'true'
       if: |
-        matrix.os == 'ubuntu-latest' &&
-        github.event_name == 'push'
+        github.event_name == 'push' &&
+        matrix.os == 'ubuntu-latest'
       continue-on-error: true

README.md

@@ -7,18 +7,26 @@
 Go module `github.com/ncruces/go-sqlite3` wraps a [WASM](https://webassembly.org/) build of [SQLite](https://sqlite.org/),
 and uses [wazero](https://wazero.io/) to provide `cgo`-free SQLite bindings.
 
-- Package [`github.com/ncruces/go-sqlite3`](https://pkg.go.dev/github.com/ncruces/go-sqlite3)
-wraps the [C SQLite API](https://www.sqlite.org/cintro.html)
-([example usage](https://pkg.go.dev/github.com/ncruces/go-sqlite3#example-package)).
-- Package [`github.com/ncruces/go-sqlite3/driver`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/driver)
-provides a [`database/sql`](https://pkg.go.dev/database/sql) driver
-([example usage](https://pkg.go.dev/github.com/ncruces/go-sqlite3/driver#example-package)).
-- Package [`github.com/ncruces/go-sqlite3/embed`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/embed)
-embeds a build of SQLite into your application.
-- Package [`github.com/ncruces/go-sqlite3/vfs`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs)
-wraps the [C SQLite VFS API](https://www.sqlite.org/vfs.html) and provides a pure Go implementation.
-- Package [`github.com/ncruces/go-sqlite3/gormlite`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/gormlite)
-provides a [GORM](https://gorm.io) driver.
+- [`github.com/ncruces/go-sqlite3`](https://pkg.go.dev/github.com/ncruces/go-sqlite3)
+  wraps the [C SQLite API](https://www.sqlite.org/cintro.html)
+  ([example usage](https://pkg.go.dev/github.com/ncruces/go-sqlite3#example-package)).
+- [`github.com/ncruces/go-sqlite3/driver`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/driver)
+  provides a [`database/sql`](https://pkg.go.dev/database/sql) driver
+  ([example usage](https://pkg.go.dev/github.com/ncruces/go-sqlite3/driver#example-package)).
+- [`github.com/ncruces/go-sqlite3/embed`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/embed)
+  embeds a build of SQLite into your application.
+- [`github.com/ncruces/go-sqlite3/vfs`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs)
+  wraps the [C SQLite VFS API](https://www.sqlite.org/vfs.html) and provides a pure Go implementation.
+- [`github.com/ncruces/go-sqlite3/vfs/memdb`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/memdb)
+  implements an in-memory VFS.
+- [`github.com/ncruces/go-sqlite3/vfs/readervfs`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/readervfs)
+  implements a VFS for immutable databases.
+- [`github.com/ncruces/go-sqlite3/ext/unicode`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/unicode)
+  registers Unicode aware functions.
+- [`github.com/ncruces/go-sqlite3/ext/stats`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/stats)
+  registers [statistics functions](https://www.oreilly.com/library/view/sql-in-a/9780596155322/ch04s02.html).
+- [`github.com/ncruces/go-sqlite3/gormlite`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/gormlite)
+  provides a [GORM](https://gorm.io) driver.
 
 ### Caveats
 
@@ -35,30 +43,39 @@ To work around this limitation, SQLite is [patched](sqlite3/locking_mode.patch)
 to always use `EXCLUSIVE` locking mode for WAL databases.
 
 Because connection pooling is incompatible with `EXCLUSIVE` locking mode,
-to open WAL databases you should disable connection pooling by calling
+to use the [`database/sql`](https://pkg.go.dev/database/sql)
+driver with WAL mode databases you should disable connection pooling by calling
 [`db.SetMaxOpenConns(1)`](https://pkg.go.dev/database/sql#DB.SetMaxOpenConns).
 
 #### POSIX Advisory Locks
 
 POSIX advisory locks, which SQLite uses, are
-[broken by design](https://www.sqlite.org/src/artifact/90c4fa?ln=1073-1161).
+[broken by design](https://www.sqlite.org/src/artifact/2e8b12?ln=1073-1161).
 
 On Linux, macOS and illumos, this module uses
 [OFD locks](https://www.gnu.org/software/libc/manual/html_node/Open-File-Description-Locks.html)
 to synchronize access to database files.
 OFD locks are fully compatible with process-associated POSIX advisory locks.
 
-On BSD Unixes, this module uses
+On BSD Unixes, this module may use
 [BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2).
-BSD locks may _not_ be compatible with process-associated POSIX advisory locks
-(they are on FreeBSD).
+BSD locks may _not_ be compatible with process-associated POSIX advisory locks.
+
+##### TL;DR
+
+In all platforms for which this package builds,
+it should be safe to use it to access databases concurrently,
+from multiple goroutines, processes, and
+with _other_ implementations of SQLite.
+
+If the package does not build for your platform,
+see [this](vfs/README.md#portability).
 
 #### Testing
 
 The pure Go VFS is tested by running SQLite's
 [mptest](https://github.com/sqlite/sqlite/blob/master/mptest/mptest.c)
-on Linux, macOS and Windows;
-BSD code paths are tested on macOS using the `sqlite3_bsd` build tag.
+on Linux, macOS and Windows.
 Performance is tested by running
 [speedtest1](https://github.com/sqlite/sqlite/blob/master/test/speedtest1.c).
 
@@ -75,11 +92,10 @@ Performance is tested by running
   - [x] in-memory VFS
   - [x] read-only VFS, wrapping an [`io.ReaderAt`](https://pkg.go.dev/io#ReaderAt)
   - [ ] cloud-based VFS, based on [Cloud Backed SQLite](https://sqlite.org/cloudsqlite/doc/trunk/www/index.wiki)
-  - [ ] [MVCC](https://en.wikipedia.org/wiki/Multiversion_concurrency_control) VFS, using [BadgerDB](https://github.com/dgraph-io/badger)
 
 ### Alternatives
 
 - [`modernc.org/sqlite`](https://pkg.go.dev/modernc.org/sqlite)
 - [`crawshaw.io/sqlite`](https://pkg.go.dev/crawshaw.io/sqlite)
 - [`github.com/mattn/go-sqlite3`](https://pkg.go.dev/github.com/mattn/go-sqlite3)
-- [`github.com/zombiezen/go-sqlite`](https://pkg.go.dev/github.com/zombiezen/go-sqlite)
+- [`github.com/zombiezen/go-sqlite`](https://pkg.go.dev/github.com/zombiezen/go-sqlite)

embed/build.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 cd -P -- "$(dirname -- "$0")"
 
 ROOT=../
-BINARYEN="$ROOT/tools/binaryen-version_114/bin"
+BINARYEN="$ROOT/tools/binaryen-version_116/bin"
 WASI_SDK="$ROOT/tools/wasi-sdk-20.0/bin"
 
 "$WASI_SDK/clang" --target=wasm32-wasi -flto -g0 -O2 \

ext/stats/stats.go

@@ -0,0 +1,109 @@
+// Package stats provides aggregate functions for statistics.
+//
+// Functions:
+//   - stddev_pop: population standard deviation
+//   - stddev_samp: sample standard deviation
+//   - var_pop: population variance
+//   - var_samp: sample variance
+//   - covar_pop: population covariance
+//   - covar_samp: sample covariance
+//   - corr: correlation coefficient
+//
+// See: [ANSI SQL Aggregate Functions]
+//
+// [ANSI SQL Aggregate Functions]: https://www.oreilly.com/library/view/sql-in-a/9780596155322/ch04s02.html
+package stats
+
+import "github.com/ncruces/go-sqlite3"
+
+// Register registers statistics functions.
+func Register(db *sqlite3.Conn) {
+	flags := sqlite3.DETERMINISTIC | sqlite3.INNOCUOUS
+	db.CreateWindowFunction("var_pop", 1, flags, newVariance(var_pop))
+	db.CreateWindowFunction("var_samp", 1, flags, newVariance(var_samp))
+	db.CreateWindowFunction("stddev_pop", 1, flags, newVariance(stddev_pop))
+	db.CreateWindowFunction("stddev_samp", 1, flags, newVariance(stddev_samp))
+	db.CreateWindowFunction("covar_pop", 2, flags, newCovariance(var_pop))
+	db.CreateWindowFunction("covar_samp", 2, flags, newCovariance(var_samp))
+	db.CreateWindowFunction("corr", 2, flags, newCovariance(corr))
+}
+
+const (
+	var_pop = iota
+	var_samp
+	stddev_pop
+	stddev_samp
+	corr
+)
+
+func newVariance(kind int) func() sqlite3.AggregateFunction {
+	return func() sqlite3.AggregateFunction { return &variance{kind: kind} }
+}
+
+type variance struct {
+	kind int
+	welford
+}
+
+func (fn *variance) Value(ctx sqlite3.Context) {
+	var r float64
+	switch fn.kind {
+	case var_pop:
+		r = fn.var_pop()
+	case var_samp:
+		r = fn.var_samp()
+	case stddev_pop:
+		r = fn.stddev_pop()
+	case stddev_samp:
+		r = fn.stddev_samp()
+	}
+	ctx.ResultFloat(r)
+}
+
+func (fn *variance) Step(ctx sqlite3.Context, arg ...sqlite3.Value) {
+	if a := arg[0]; a.Type() != sqlite3.NULL {
+		fn.enqueue(a.Float())
+	}
+}
+
+func (fn *variance) Inverse(ctx sqlite3.Context, arg ...sqlite3.Value) {
+	if a := arg[0]; a.Type() != sqlite3.NULL {
+		fn.dequeue(a.Float())
+	}
+}
+
+func newCovariance(kind int) func() sqlite3.AggregateFunction {
+	return func() sqlite3.AggregateFunction { return &covariance{kind: kind} }
+}
+
+type covariance struct {
+	kind int
+	welford2
+}
+
+func (fn *covariance) Value(ctx sqlite3.Context) {
+	var r float64
+	switch fn.kind {
+	case var_pop:
+		r = fn.covar_pop()
+	case var_samp:
+		r = fn.covar_samp()
+	case corr:
+		r = fn.correlation()
+	}
+	ctx.ResultFloat(r)
+}
+
+func (fn *covariance) Step(ctx sqlite3.Context, arg ...sqlite3.Value) {
+	a, b := arg[0], arg[1]
+	if a.Type() != sqlite3.NULL && b.Type() != sqlite3.NULL {
+		fn.enqueue(a.Float(), b.Float())
+	}
+}
+
+func (fn *covariance) Inverse(ctx sqlite3.Context, arg ...sqlite3.Value) {
+	a, b := arg[0], arg[1]
+	if a.Type() != sqlite3.NULL && b.Type() != sqlite3.NULL {
+		fn.dequeue(a.Float(), b.Float())
+	}
+}

ext/stats/stats_test.go

@@ -0,0 +1,140 @@
+package stats
+
+import (
+	"math"
+	"testing"
+
+	"github.com/ncruces/go-sqlite3"
+	_ "github.com/ncruces/go-sqlite3/embed"
+)
+
+func TestRegister_variance(t *testing.T) {
+	t.Parallel()
+
+	db, err := sqlite3.Open(":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	Register(db)
+
+	err = db.Exec(`CREATE TABLE IF NOT EXISTS data (x)`)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = db.Exec(`INSERT INTO data (x) VALUES (4), (7.0), ('13'), (NULL), (16)`)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	stmt, _, err := db.Prepare(`
+		SELECT
+			sum(x), avg(x),
+			var_samp(x), var_pop(x),
+			stddev_samp(x), stddev_pop(x)
+		FROM data`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer stmt.Close()
+
+	if stmt.Step() {
+		if got := stmt.ColumnFloat(0); got != 40 {
+			t.Errorf("got %v, want 40", got)
+		}
+		if got := stmt.ColumnFloat(1); got != 10 {
+			t.Errorf("got %v, want 10", got)
+		}
+		if got := stmt.ColumnFloat(2); got != 30 {
+			t.Errorf("got %v, want 30", got)
+		}
+		if got := stmt.ColumnFloat(3); got != 22.5 {
+			t.Errorf("got %v, want 22.5", got)
+		}
+		if got := stmt.ColumnFloat(4); got != math.Sqrt(30) {
+			t.Errorf("got %v, want √30", got)
+		}
+		if got := stmt.ColumnFloat(5); got != math.Sqrt(22.5) {
+			t.Errorf("got %v, want √22.5", got)
+		}
+	}
+
+	{
+		stmt, _, err := db.Prepare(`SELECT var_samp(x) OVER (ROWS 1 PRECEDING) FROM data`)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer stmt.Close()
+
+		want := [...]float64{0, 4.5, 18, 0, 0}
+		for i := 0; stmt.Step(); i++ {
+			if got := stmt.ColumnFloat(0); got != want[i] {
+				t.Errorf("got %v, want %v", got, want[i])
+			}
+			if got := stmt.ColumnType(0); (got == sqlite3.FLOAT) != (want[i] != 0) {
+				t.Errorf("got %v, want %v", got, want[i])
+			}
+		}
+	}
+}
+
+func TestRegister_covariance(t *testing.T) {
+	t.Parallel()
+
+	db, err := sqlite3.Open(":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	Register(db)
+
+	err = db.Exec(`CREATE TABLE IF NOT EXISTS data (x, y)`)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = db.Exec(`INSERT INTO data (x, y) VALUES (3, 70), (5, 80), (2, 60), (7, 90), (4, 75)`)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	stmt, _, err := db.Prepare(`SELECT
+		corr(x, y), covar_samp(x, y), covar_pop(x, y) FROM data`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer stmt.Close()
+
+	if stmt.Step() {
+		if got := stmt.ColumnFloat(0); got != 0.9881049293224639 {
+			t.Errorf("got %v, want 0.9881049293224639", got)
+		}
+		if got := stmt.ColumnFloat(1); got != 21.25 {
+			t.Errorf("got %v, want 21.25", got)
+		}
+		if got := stmt.ColumnFloat(2); got != 17 {
+			t.Errorf("got %v, want 17", got)
+		}
+	}
+
+	{
+		stmt, _, err := db.Prepare(`SELECT covar_samp(x, y) OVER (ROWS 1 PRECEDING) FROM data`)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer stmt.Close()
+
+		want := [...]float64{0, 10, 30, 75, 22.5}
+		for i := 0; stmt.Step(); i++ {
+			if got := stmt.ColumnFloat(0); got != want[i] {
+				t.Errorf("got %v, want %v", got, want[i])
+			}
+			if got := stmt.ColumnType(0); (got == sqlite3.FLOAT) != (want[i] != 0) {
+				t.Errorf("got %v, want %v", got, want[i])
+			}
+		}
+	}
+}

ext/stats/welford.go

@@ -0,0 +1,109 @@
+package stats
+
+import "math"
+
+// Welford's algorithm with Kahan summation:
+// https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm
+// https://en.wikipedia.org/wiki/Kahan_summation_algorithm
+
+type welford struct {
+	m1, m2 kahan
+	n      uint64
+}
+
+func (w welford) average() float64 {
+	return w.m1.hi
+}
+
+func (w welford) var_pop() float64 {
+	return w.m2.hi / float64(w.n)
+}
+
+func (w welford) var_samp() float64 {
+	return w.m2.hi / float64(w.n-1) // Bessel's correction
+}
+
+func (w welford) stddev_pop() float64 {
+	return math.Sqrt(w.var_pop())
+}
+
+func (w welford) stddev_samp() float64 {
+	return math.Sqrt(w.var_samp())
+}
+
+func (w *welford) enqueue(x float64) {
+	w.n++
+	d1 := x - w.m1.hi - w.m1.lo
+	w.m1.add(d1 / float64(w.n))
+	d2 := x - w.m1.hi - w.m1.lo
+	w.m2.add(d1 * d2)
+}
+
+func (w *welford) dequeue(x float64) {
+	w.n--
+	d1 := x - w.m1.hi - w.m1.lo
+	w.m1.sub(d1 / float64(w.n))
+	d2 := x - w.m1.hi - w.m1.lo
+	w.m2.sub(d1 * d2)
+}
+
+type welford2 struct {
+	m1x, m2x kahan
+	m1y, m2y kahan
+	cov      kahan
+	n        uint64
+}
+
+func (w welford2) covar_pop() float64 {
+	return w.cov.hi / float64(w.n)
+}
+
+func (w welford2) covar_samp() float64 {
+	return w.cov.hi / float64(w.n-1) // Bessel's correction
+}
+
+func (w welford2) correlation() float64 {
+	return w.cov.hi / math.Sqrt(w.m2x.hi*w.m2y.hi)
+}
+
+func (w *welford2) enqueue(x, y float64) {
+	w.n++
+	d1x := x - w.m1x.hi - w.m1x.lo
+	d1y := y - w.m1y.hi - w.m1y.lo
+	w.m1x.add(d1x / float64(w.n))
+	w.m1y.add(d1y / float64(w.n))
+	d2x := x - w.m1x.hi - w.m1x.lo
+	d2y := y - w.m1y.hi - w.m1y.lo
+	w.m2x.add(d1x * d2x)
+	w.m2y.add(d1y * d2y)
+	w.cov.add(d1x * d2y)
+}
+
+func (w *welford2) dequeue(x, y float64) {
+	w.n--
+	d1x := x - w.m1x.hi - w.m1x.lo
+	d1y := y - w.m1y.hi - w.m1y.lo
+	w.m1x.sub(d1x / float64(w.n))
+	w.m1y.sub(d1y / float64(w.n))
+	d2x := x - w.m1x.hi - w.m1x.lo
+	d2y := y - w.m1y.hi - w.m1y.lo
+	w.m2x.sub(d1x * d2x)
+	w.m2y.sub(d1y * d2y)
+	w.cov.sub(d1x * d2y)
+}
+
+type kahan struct{ hi, lo float64 }
+
+func (k *kahan) add(x float64) {
+	y := k.lo + x
+	t := k.hi + y
+	k.lo = y - (t - k.hi)
+	k.hi = t
+}
+
+func (k *kahan) sub(x float64) {
+	y := k.lo - x
+	t := k.hi + y
+	k.lo = y - (t - k.hi)
+	k.hi = t
+}

ext/stats/welford_test.go

@@ -0,0 +1,75 @@
+package stats
+
+import (
+	"math"
+	"testing"
+)
+
+func Test_welford(t *testing.T) {
+	var s1, s2 welford
+
+	s1.enqueue(4)
+	s1.enqueue(7)
+	s1.enqueue(13)
+	s1.enqueue(16)
+	if got := s1.average(); got != 10 {
+		t.Errorf("got %v, want 10", got)
+	}
+	if got := s1.var_samp(); got != 30 {
+		t.Errorf("got %v, want 30", got)
+	}
+	if got := s1.var_pop(); got != 22.5 {
+		t.Errorf("got %v, want 22.5", got)
+	}
+	if got := s1.stddev_samp(); got != math.Sqrt(30) {
+		t.Errorf("got %v, want √30", got)
+	}
+	if got := s1.stddev_pop(); got != math.Sqrt(22.5) {
+		t.Errorf("got %v, want √22.5", got)
+	}
+
+	s1.dequeue(4)
+	s2.enqueue(7)
+	s2.enqueue(13)
+	s2.enqueue(16)
+	if s1.var_pop() != s2.var_pop() {
+		t.Errorf("got %v, want %v", s1, s2)
+	}
+}
+
+func Test_covar(t *testing.T) {
+	var c1, c2 welford2
+
+	c1.enqueue(3, 70)
+	c1.enqueue(5, 80)
+	c1.enqueue(2, 60)
+	c1.enqueue(7, 90)
+	c1.enqueue(4, 75)
+
+	if got := c1.covar_samp(); got != 21.25 {
+		t.Errorf("got %v, want 21.25", got)
+	}
+	if got := c1.covar_pop(); got != 17 {
+		t.Errorf("got %v, want 17", got)
+	}
+
+	c1.dequeue(3, 70)
+	c2.enqueue(5, 80)
+	c2.enqueue(2, 60)
+	c2.enqueue(7, 90)
+	c2.enqueue(4, 75)
+	if c1.covar_pop() != c2.covar_pop() {
+		t.Errorf("got %v, want %v", c1.covar_pop(), c2.covar_pop())
+	}
+}
+
+func Test_correlation(t *testing.T) {
+	var c welford2
+	c.enqueue(1, 3)
+	c.enqueue(2, 2)
+	c.enqueue(3, 1)
+
+	if got := c.correlation(); got != -1 {
+		t.Errorf("got %v, want -1", got)
+	}
+}

ext/unicode/unicode.go

@@ -1,17 +1,19 @@
 // Package unicode provides an alternative to the SQLite ICU extension.
 //
-// Provides Unicode aware:
-//   - upper and lower functions,
+// Like the [ICU extension], it provides Unicode aware:
+//   - upper() and lower() functions,
 //   - LIKE and REGEXP operators,
 //   - collation sequences.
 //
-// This package is not 100% compatible with the ICU extension:
-//   - upper and lower use [strings.ToUpper], [strings.ToLower] and [cases];
+// The implementation is not 100% compatible with the [ICU extension]:
+//   - upper() and lower() use [strings.ToUpper], [strings.ToLower] and [cases];
 //   - the LIKE operator follows [strings.EqualFold] rules;
 //   - the REGEXP operator uses Go [regex/syntax];
 //   - collation sequences use [collate].
 //
 // Expect subtle differences (e.g.) in the handling of Turkish case folding.
+//
+// [ICU extension]: https://sqlite.org/src/dir/ext/icu
 package unicode
 
 import (
@@ -45,7 +47,7 @@ func Register(db *sqlite3.Conn) {
 				return
 			}
 
-			err := RegisterCollation(db, name, arg[0].Text())
+			err := RegisterCollation(db, arg[0].Text(), name)
 			if err != nil {
 				ctx.ResultError(err)
 				return
@@ -53,8 +55,9 @@ func Register(db *sqlite3.Conn) {
 		})
 }
 
-func RegisterCollation(db *sqlite3.Conn, name, lang string) error {
-	tag, err := language.Parse(lang)
+// RegisterCollation registers a Unicode collation sequence for a database connection.
+func RegisterCollation(db *sqlite3.Conn, locale, name string) error {
+	tag, err := language.Parse(locale)
 	if err != nil {
 		return err
 	}

func.go

@@ -12,7 +12,7 @@ import (
 // for any unknown collating sequence.
 // The fake collating function works like BINARY.
 //
-// This extension can be used to load schemas that contain
+// This can be used to load schemas that contain
 // one or more unknown collating sequences.
 func (c *Conn) AnyCollationNeeded() {
 	c.call(c.api.anyCollation, uint64(c.handle), 0, 0)

func_win_test.go

@@ -26,7 +26,7 @@ func ExampleConn_CreateWindowFunction() {
 		log.Fatal(err)
 	}
 
-	err = db.CreateWindowFunction("count_ascii", 1, sqlite3.INNOCUOUS, newASCIICounter)
+	err = db.CreateWindowFunction("count_ascii", 1, sqlite3.DETERMINISTIC|sqlite3.INNOCUOUS, newASCIICounter)
 	if err != nil {
 		log.Fatal(err)
 	}

go.mod

@@ -7,8 +7,8 @@ require (
 	github.com/psanford/httpreadat v0.1.0
 	github.com/tetratelabs/wazero v1.5.0
 	golang.org/x/sync v0.3.0
-	golang.org/x/sys v0.11.1-0.20230807062127-60ecf133770b
-	golang.org/x/text v0.12.0
+	golang.org/x/sys v0.12.0
+	golang.org/x/text v0.13.0
 )
 
 retract v0.4.0 // tagged from the wrong branch

go.sum

@@ -6,7 +6,7 @@ github.com/tetratelabs/wazero v1.5.0 h1:Yz3fZHivfDiZFUXnWMPUoiW7s8tC1sjdBtlJn08q
 github.com/tetratelabs/wazero v1.5.0/go.mod h1:0U0G41+ochRKoPKCJlh0jMg1CHkyfK8kDqiirMmKY8A=
 golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
 golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
-golang.org/x/sys v0.11.1-0.20230807062127-60ecf133770b h1:SVkT2ohjl5jbz5YNEn1KslYMggyeWG0FbUBDnQOZ3s8=
-golang.org/x/sys v0.11.1-0.20230807062127-60ecf133770b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
-golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=

go.work.sum

@@ -1,4 +1,4 @@
-github.com/ncruces/go-sqlite3 v0.8.6/go.mod h1:LAWp8Mj+m2ki1HeRJo7flS3luCUqG+GQKjK4BKLc++0=
+github.com/ncruces/go-sqlite3 v0.8.8/go.mod h1:IyRoNwT0Z+mNRXIVeP2DgWPNl78Kmc/B+pO9i6GNgRg=
 golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=

gormlite/go.mod

@@ -1,9 +1,9 @@
 module github.com/ncruces/go-sqlite3/gormlite
 
-go 1.20
+go 1.21
 
 require (
-	github.com/ncruces/go-sqlite3 v0.8.6
+	github.com/ncruces/go-sqlite3 v0.8.8
 	gorm.io/gorm v1.25.4
 )
 
@@ -12,5 +12,5 @@ require (
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/ncruces/julianday v0.1.5 // indirect
 	github.com/tetratelabs/wazero v1.5.0 // indirect
-	golang.org/x/sys v0.11.1-0.20230807062127-60ecf133770b // indirect
+	golang.org/x/sys v0.12.0 // indirect
 )

gormlite/go.sum

@@ -2,14 +2,15 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
 github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
-github.com/ncruces/go-sqlite3 v0.8.6 h1:y0b+rwqkrvV1lscEVk93SeH2trnlKMD5XKtIqVqm9QE=
-github.com/ncruces/go-sqlite3 v0.8.6/go.mod h1:LAWp8Mj+m2ki1HeRJo7flS3luCUqG+GQKjK4BKLc++0=
+github.com/ncruces/go-sqlite3 v0.8.8 h1:Gb6tkeGOSN/m6RRf7gt3mLM4DMwO/InAXh2J/1jt5iM=
+github.com/ncruces/go-sqlite3 v0.8.8/go.mod h1:IyRoNwT0Z+mNRXIVeP2DgWPNl78Kmc/B+pO9i6GNgRg=
 github.com/ncruces/julianday v0.1.5 h1:hDJ9ejiMp3DHsoZ5KW4c1lwfMjbARS7u/gbYcd0FBZk=
 github.com/ncruces/julianday v0.1.5/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
 github.com/tetratelabs/wazero v1.5.0 h1:Yz3fZHivfDiZFUXnWMPUoiW7s8tC1sjdBtlJn08qYa0=
 github.com/tetratelabs/wazero v1.5.0/go.mod h1:0U0G41+ochRKoPKCJlh0jMg1CHkyfK8kDqiirMmKY8A=
-golang.org/x/sys v0.11.1-0.20230807062127-60ecf133770b h1:SVkT2ohjl5jbz5YNEn1KslYMggyeWG0FbUBDnQOZ3s8=
-golang.org/x/sys v0.11.1-0.20230807062127-60ecf133770b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
+golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 gorm.io/gorm v1.25.4 h1:iyNd8fNAe8W9dvtlgeRI5zSVZPsq3OpcTu37cYcpCmw=
 gorm.io/gorm v1.25.4/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=

sqlite3/download.sh

@@ -3,7 +3,7 @@ set -euo pipefail
 
 cd -P -- "$(dirname -- "$0")"
 
-curl -#OL "https://sqlite.org/2023/sqlite-amalgamation-3430000.zip"
+curl -#OL "https://sqlite.org/2023/sqlite-amalgamation-3430100.zip"
 unzip -d . sqlite-amalgamation-*.zip
 mv sqlite-amalgamation-*/sqlite3* .
 rm -rf sqlite-amalgamation-*
@@ -12,24 +12,24 @@ cat *.patch | patch --posix
 
 mkdir -p ext/
 cd ext/
-curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.0/ext/misc/decimal.c"
-curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.0/ext/misc/uint.c"
-curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.0/ext/misc/uuid.c"
-curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.0/ext/misc/base64.c"
-curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.0/ext/misc/regexp.c"
-curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.0/ext/misc/series.c"
-curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.0/ext/misc/anycollseq.c"
+curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.1/ext/misc/decimal.c"
+curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.1/ext/misc/uint.c"
+curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.1/ext/misc/uuid.c"
+curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.1/ext/misc/base64.c"
+curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.1/ext/misc/regexp.c"
+curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.1/ext/misc/series.c"
+curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.1/ext/misc/anycollseq.c"
 cd ~-
 
 cd ../vfs/tests/mptest/testdata/
-curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.0/mptest/mptest.c"
-curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.0/mptest/config01.test"
-curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.0/mptest/config02.test"
-curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.0/mptest/crash01.test"
-curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.0/mptest/crash02.subtest"
-curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.0/mptest/multiwrite01.test"
+curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.1/mptest/mptest.c"
+curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.1/mptest/config01.test"
+curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.1/mptest/config02.test"
+curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.1/mptest/crash01.test"
+curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.1/mptest/crash02.subtest"
+curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.1/mptest/multiwrite01.test"
 cd ~-
 
 cd ../vfs/tests/speedtest1/testdata/
-curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.0/test/speedtest1.c"
+curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.43.1/test/speedtest1.c"
 cd ~-

vfs/README.md

@@ -2,8 +2,30 @@
 
 This package implements the SQLite [OS Interface](https://www.sqlite.org/vfs.html) (aka VFS).
 
-It replaces the default VFS with a pure Go implementation,
-that is tested on Linux, macOS and Windows,
-but which should also work on illumos and the various BSDs.
+It replaces the default SQLite VFS with a pure Go implementation.
 
-It also exposes interfaces that should allow you to implement your own custom VFSes.
+It also exposes interfaces that should allow you to implement your own custom VFSes.
+
+## Portability
+
+This package is tested on Linux, macOS and Windows,
+but it should also work on FreeBSD and illumos
+(code paths for those plaforms are tested on macOS and Linux, respectively).
+
+In all platforms for which this package builds,
+it should be safe to use it to access databases concurrently,
+from multiple goroutines, processes, and
+with _other_ implementations of SQLite.
+
+If the package does not build for your platform,
+you may try to use the `sqlite3_flock` and `sqlite3_nolock` build tags.
+These are only minimally tested and concurrency test failures should be expected.
+
+The `sqlite3_flock` tag uses
+[BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2).
+It should be safe to access databases concurrently from multiple goroutines and processes,
+but **not** with _other_ implementations of SQLite
+(_unless_ these are _also_ configured to use `flock`).
+
+The `sqlite3_nolock` tag uses no locking at all.
+Database corruption is the likely result from concurrent write access.

vfs/lock.go

@@ -1,3 +1,5 @@
+//go:build !sqlite3_nolock
+
 package vfs
 
 import (

vfs/lock_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"os"
 	"path/filepath"
-	"runtime"
 	"testing"
 
 	"github.com/ncruces/go-sqlite3/internal/util"
@@ -12,13 +11,6 @@ import (
 )
 
 func Test_vfsLock(t *testing.T) {
-	switch runtime.GOOS {
-	case "linux", "darwin", "windows":
-		break
-	default:
-		t.Skip("OS lacks OFD locks")
-	}
-
 	name := filepath.Join(t.TempDir(), "test.db")
 
 	// Create a temporary file.

vfs/nolock.go

@@ -0,0 +1,22 @@
+//go:build sqlite3_nolock
+
+package vfs
+
+const (
+	_PENDING_BYTE  = 0x40000000
+	_RESERVED_BYTE = (_PENDING_BYTE + 1)
+	_SHARED_FIRST  = (_PENDING_BYTE + 2)
+	_SHARED_SIZE   = 510
+)
+
+func (f *vfsFile) Lock(lock LockLevel) error {
+	return nil
+}
+
+func (f *vfsFile) Unlock(lock LockLevel) error {
+	return nil
+}
+
+func (f *vfsFile) CheckReservedLock() (bool, error) {
+	return false, nil
+}

vfs/os_bsd.go

@@ -1,4 +1,4 @@
-//go:build freebsd || openbsd || netbsd || dragonfly || (darwin && sqlite3_bsd)
+//go:build sqlite3_flock || freebsd
 
 package vfs
 

vfs/os_darwin.go

@@ -1,4 +1,4 @@
-//go:build !sqlite3_bsd
+//go:build !sqlite3_flock
 
 package vfs
 

vfs/os_nolock.go

@@ -0,0 +1,28 @@
+//go:build sqlite3_nolock && unix && !(linux || darwin || freebsd || illumos)
+
+package vfs
+
+import (
+	"os"
+	"time"
+)
+
+func osUnlock(file *os.File, start, len int64) _ErrorCode {
+	return _OK
+}
+
+func osLock(file *os.File, typ int16, start, len int64, timeout time.Duration, def _ErrorCode) _ErrorCode {
+	return _OK
+}
+
+func osReadLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode {
+	return _OK
+}
+
+func osWriteLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode {
+	return _OK
+}
+
+func osCheckLock(file *os.File, start, len int64) (bool, _ErrorCode) {
+	return false, _OK
+}

vfs/os_ofd.go

@@ -1,4 +1,4 @@
-//go:build linux || illumos
+//go:build (linux || illumos) && !sqlite3_flock
 
 package vfs
 

vfs/os_std_access.go

@@ -0,0 +1,36 @@
+//go:build !unix
+
+package vfs
+
+import (
+	"io/fs"
+	"os"
+)
+
+const (
+	_S_IREAD  = 0400
+	_S_IWRITE = 0200
+	_S_IEXEC  = 0100
+)
+
+func osAccess(path string, flags AccessFlag) error {
+	fi, err := os.Stat(path)
+	if err != nil {
+		return err
+	}
+	if flags == ACCESS_EXISTS {
+		return nil
+	}
+
+	var want fs.FileMode = _S_IREAD
+	if flags == ACCESS_READWRITE {
+		want |= _S_IWRITE
+	}
+	if fi.IsDir() {
+		want |= _S_IEXEC
+	}
+	if fi.Mode()&want != want {
+		return fs.ErrPermission
+	}
+	return nil
+}

vfs/os_std_alloc.go

@@ -1,4 +1,4 @@
-//go:build !linux && (!darwin || sqlite3_bsd)
+//go:build !linux && (!darwin || sqlite3_flock)
 
 package vfs
 
@@ -7,10 +7,6 @@ import (
 	"os"
 )
 
-func osSync(file *os.File, fullsync, dataonly bool) error {
-	return file.Sync()
-}
-
 func osAllocate(file *os.File, size int64) error {
 	off, err := file.Seek(0, io.SeekEnd)
 	if err != nil {

vfs/os_std_mode.go

@@ -0,0 +1,14 @@
+//go:build !unix
+
+package vfs
+
+import "os"
+
+func osSetMode(file *os.File, modeof string) error {
+	fi, err := os.Stat(modeof)
+	if err != nil {
+		return err
+	}
+	file.Chmod(fi.Mode())
+	return nil
+}

vfs/os_std_open.go

@@ -0,0 +1,12 @@
+//go:build !windows
+
+package vfs
+
+import (
+	"io/fs"
+	"os"
+)
+
+func osOpenFile(name string, flag int, perm fs.FileMode) (*os.File, error) {
+	return os.OpenFile(name, flag, perm)
+}

vfs/os_std_sync.go

@@ -0,0 +1,9 @@
+//go:build !linux && (!darwin || sqlite3_flock)
+
+package vfs
+
+import "os"
+
+func osSync(file *os.File, fullsync, dataonly bool) error {
+	return file.Sync()
+}

vfs/os_unix.go

@@ -3,7 +3,6 @@
 package vfs
 
 import (
-	"io/fs"
 	"os"
 	"syscall"
 	"time"
@@ -11,10 +10,6 @@ import (
 	"golang.org/x/sys/unix"
 )
 
-func osOpenFile(name string, flag int, perm fs.FileMode) (*os.File, error) {
-	return os.OpenFile(name, flag, perm)
-}
-
 func osAccess(path string, flags AccessFlag) error {
 	var access uint32 // unix.F_OK
 	switch flags {

vfs/os_windows.go

@@ -25,37 +25,6 @@ func osOpenFile(name string, flag int, perm fs.FileMode) (*os.File, error) {
 	return os.NewFile(uintptr(r), name), nil
 }
 
-func osAccess(path string, flags AccessFlag) error {
-	fi, err := os.Stat(path)
-	if err != nil {
-		return err
-	}
-	if flags == ACCESS_EXISTS {
-		return nil
-	}
-
-	var want fs.FileMode = windows.S_IRUSR
-	if flags == ACCESS_READWRITE {
-		want |= windows.S_IWUSR
-	}
-	if fi.IsDir() {
-		want |= windows.S_IXUSR
-	}
-	if fi.Mode()&want != want {
-		return fs.ErrPermission
-	}
-	return nil
-}
-
-func osSetMode(file *os.File, modeof string) error {
-	fi, err := os.Stat(modeof)
-	if err != nil {
-		return err
-	}
-	file.Chmod(fi.Mode())
-	return nil
-}
-
 func osGetSharedLock(file *os.File, timeout time.Duration) _ErrorCode {
 	// Acquire the PENDING lock temporarily before acquiring a new SHARED lock.
 	rc := osReadLock(file, _PENDING_BYTE, 1, timeout)

vfs/tests/mptest/testdata/build.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 cd -P -- "$(dirname -- "$0")"
 
 ROOT=../../../../
-BINARYEN="$ROOT/tools/binaryen-version_114/bin"
+BINARYEN="$ROOT/tools/binaryen-version_116/bin"
 WASI_SDK="$ROOT/tools/wasi-sdk-20.0/bin"
 
 "$WASI_SDK/clang" --target=wasm32-wasi -flto -g0 -O2 \

vfs/tests/mptest/testdata/mptest.wasm

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d23b37d507077cdcbb616852185370a227278b599187dc134200ed274a7a3a02
-size 1441194
+oid sha256:5b77e9e13a487e976a6e71bc698542098433d1cc586ad8f24784f1f325ffb8dd
+size 1459145

vfs/tests/speedtest1/testdata/build.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 cd -P -- "$(dirname -- "$0")"
 
 ROOT=../../../../
-BINARYEN="$ROOT/tools/binaryen-version_114/bin"
+BINARYEN="$ROOT/tools/binaryen-version_116/bin"
 WASI_SDK="$ROOT/tools/wasi-sdk-20.0/bin"
 
 "$WASI_SDK/clang" --target=wasm32-wasi -flto -g0 -O2 \

vfs/tests/speedtest1/testdata/speedtest1.wasm

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:83d67feda51cc974634e245ac2b072f9587c607c7ad97321f2de9dde2188e63a
-size 1481348
+oid sha256:3b52de3306965ac3f812592be29697d75232802a13bb16a34344f8d81dbf0637
+size 1499410