BSD WAL support. (#90)

Uses in-memory locks.
Also supports illumos.

.github/workflows/test.yml

@@ -131,7 +131,7 @@ jobs:
     - name: Test riscv64 (interpreter)
       run: GOARCH=riscv64 go test -v -short ./...
 
-    - name: Test s390x (big-endian, z/OS)
+    - name: Test s390x (big-endian, z/OS demo)
       run: GOARCH=s390x go test -v -short -tags sqlite3_flock ./...
 
   test-vm:

driver/driver_test.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/ncruces/go-sqlite3"
 	_ "github.com/ncruces/go-sqlite3/embed"
+	_ "github.com/ncruces/go-sqlite3/internal/testcfg"
 	"github.com/ncruces/go-sqlite3/internal/util"
 	"github.com/ncruces/go-sqlite3/vfs"
 )

func_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/ncruces/go-sqlite3"
 	_ "github.com/ncruces/go-sqlite3/embed"
 	"github.com/ncruces/go-sqlite3/ext/unicode"
+	_ "github.com/ncruces/go-sqlite3/internal/testcfg"
 )
 
 func ExampleConn_CreateCollation() {

go.mod

@@ -6,10 +6,10 @@ require (
 	github.com/ncruces/julianday v1.0.0
 	github.com/psanford/httpreadat v0.1.0
 	github.com/tetratelabs/wazero v1.7.2
-	golang.org/x/crypto v0.23.0
+	golang.org/x/crypto v0.24.0
 	golang.org/x/sync v0.7.0
-	golang.org/x/sys v0.20.0
+	golang.org/x/sys v0.21.0
-	golang.org/x/text v0.15.0
+	golang.org/x/text v0.16.0
 	lukechampine.com/adiantum v1.1.1
 )
 

go.sum

@@ -6,13 +6,13 @@ github.com/psanford/httpreadat v0.1.0 h1:VleW1HS2zO7/4c7c7zNl33fO6oYACSagjJIyMIw
 github.com/psanford/httpreadat v0.1.0/go.mod h1:Zg7P+TlBm3bYbyHTKv/EdtSJZn3qwbPwpfZ/I9GKCRE=
 github.com/tetratelabs/wazero v1.7.2 h1:1+z5nXJNwMLPAWaTePFi49SSTL0IMx/i3Fg8Yc25GDc=
 github.com/tetratelabs/wazero v1.7.2/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
-golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
+golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
-golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
 golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
 golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
-golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 lukechampine.com/adiantum v1.1.1 h1:4fp6gTxWCqpEbLy40ExiYDDED3oUNWx5cTqBCtPdZqA=
 lukechampine.com/adiantum v1.1.1/go.mod h1:LrAYVnTYLnUtE/yMp5bQr0HstAf060YUF8nM0B6+rUw=

go.work.sum

@@ -1,4 +1,7 @@
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=

gormlite/go.mod

@@ -12,5 +12,5 @@ require (
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/ncruces/julianday v1.0.0 // indirect
 	github.com/tetratelabs/wazero v1.7.2 // indirect
-	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
 )

gormlite/go.sum

@@ -8,8 +8,8 @@ github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt
 github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
 github.com/tetratelabs/wazero v1.7.2 h1:1+z5nXJNwMLPAWaTePFi49SSTL0IMx/i3Fg8Yc25GDc=
 github.com/tetratelabs/wazero v1.7.2/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
-golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 gorm.io/gorm v1.25.10 h1:dQpO+33KalOA+aFYGlK+EfxcI5MbO7EP2yYygwh9h+s=

internal/testcfg/testcfg.go

@@ -1,6 +1,7 @@
 package testcfg
 
 import (
+	"math/bits"
 	"os"
 	"path/filepath"
 
@@ -9,6 +10,10 @@ import (
 )
 
 func init() {
+	if bits.UintSize < 64 {
+		return
+	}
+
 	sqlite3.RuntimeConfig = wazero.NewRuntimeConfig().
 		WithMemoryCapacityFromMax(true).
 		WithMemoryLimitPages(1024)

internal/util/mmap.go

@@ -1,4 +1,4 @@
-//go:build (darwin || linux) && (amd64 || arm64 || riscv64) && !(sqlite3_noshm || sqlite3_nosys)
+//go:build unix && (amd64 || arm64 || riscv64) && !(sqlite3_noshm || sqlite3_nosys)
 
 package util
 

internal/util/mmap_other.go

@@ -1,4 +1,4 @@
-//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64) || sqlite3_noshm || sqlite3_nosys
+//go:build !unix || !(amd64 || arm64 || riscv64) || sqlite3_noshm || sqlite3_nosys
 
 package util
 

tests/stmt_test.go

@@ -3,6 +3,7 @@ package tests
 import (
 	"encoding/json"
 	"math"
+	"math/bits"
 	"testing"
 	"time"
 
@@ -617,6 +618,9 @@ func TestStmt_ColumnTime(t *testing.T) {
 }
 
 func TestStmt_Error(t *testing.T) {
+	if bits.UintSize < 64 {
+		t.Skip("skipping on 32-bit")
+	}
 	t.Parallel()
 
 	db, err := sqlite3.Open(":memory:")

vfs/README.md

@@ -46,7 +46,7 @@ to check if your build supports file locking.
 
 ### Write-Ahead Logging
 
-On 64-bit Linux and macOS, this module uses `mmap` to implement
+On 64-bit Unix, this module uses `mmap` to implement
 [shared-memory for the WAL-index](https://sqlite.org/wal.html#implementation_of_shared_memory_for_the_wal_index),
 like SQLite.
 

vfs/shm.go

@@ -51,12 +51,7 @@ type vfsShm struct {
 	readOnly bool
 }
 
-func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, error) {
+func (s *vfsShm) shmOpen() error {
-	// Ensure size is a multiple of the OS page size.
-	if int(size)&(unix.Getpagesize()-1) != 0 {
-		return 0, _IOERR_SHMMAP
-	}
-
 	if s.File == nil {
 		var flag int
 		if s.readOnly {
@@ -67,29 +62,41 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext
 		f, err := os.OpenFile(s.path,
 			flag|unix.O_CREAT|unix.O_NOFOLLOW, 0666)
 		if err != nil {
-			return 0, _CANTOPEN
+			return _CANTOPEN
 		}
 		s.File = f
 	}
 
 	// Dead man's switch.
 	if lock, rc := osGetLock(s.File, _SHM_DMS, 1); rc != _OK {
-		return 0, _IOERR_LOCK
+		return _IOERR_LOCK
 	} else if lock == unix.F_WRLCK {
-		return 0, _BUSY
+		return _BUSY
 	} else if lock == unix.F_UNLCK {
 		if s.readOnly {
-			return 0, _READONLY_CANTINIT
+			return _READONLY_CANTINIT
 		}
 		if rc := osWriteLock(s.File, _SHM_DMS, 1, 0); rc != _OK {
-			return 0, rc
+			return rc
 		}
 		if err := s.Truncate(0); err != nil {
-			return 0, _IOERR_SHMOPEN
+			return _IOERR_SHMOPEN
 		}
 	}
 	if rc := osReadLock(s.File, _SHM_DMS, 1, 0); rc != _OK {
-		return 0, rc
+		return rc
+	}
+	return nil
+}
+
+func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, error) {
+	// Ensure size is a multiple of the OS page size.
+	if int(size)&(unix.Getpagesize()-1) != 0 {
+		return 0, _IOERR_SHMMAP
+	}
+
+	if err := s.shmOpen(); err != nil {
+		return 0, err
 	}
 
 	// Check if file is big enough.
@@ -165,9 +172,9 @@ func (s *vfsShm) shmUnmap(delete bool) {
 	s.regions = s.regions[:0]
 
 	// Close the file.
-	defer s.Close()
 	if delete {
-		os.Remove(s.Name())
+		os.Remove(s.path)
 	}
+	s.Close()
 	s.File = nil
 }

vfs/shm_bsd.go

@@ -0,0 +1,261 @@
+//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (amd64 || arm64 || riscv64) && !(sqlite3_noshm || sqlite3_nosys)
+
+package vfs
+
+import (
+	"context"
+	"io"
+	"os"
+	"sync"
+
+	"github.com/ncruces/go-sqlite3/internal/util"
+	"github.com/tetratelabs/wazero/api"
+	"golang.org/x/sys/unix"
+)
+
+// SupportsSharedMemory is false on platforms that do not support shared memory.
+// To use [WAL without shared-memory], you need to set [EXCLUSIVE locking mode].
+//
+// [WAL without shared-memory]: https://sqlite.org/wal.html#noshm
+// [EXCLUSIVE locking mode]: https://sqlite.org/pragma.html#pragma_locking_mode
+const SupportsSharedMemory = true
+
+const (
+	_SHM_NLOCK = 8
+	_SHM_BASE  = 120
+	_SHM_DMS   = _SHM_BASE + _SHM_NLOCK
+)
+
+func (f *vfsFile) SharedMemory() SharedMemory { return f.shm }
+
+// NewSharedMemory returns a shared-memory WAL-index
+// backed by a file with the given path.
+// It will return nil if shared-memory is not supported,
+// or not appropriate for the given flags.
+// Only [OPEN_MAIN_DB] databases may need a WAL-index.
+// You must ensure all concurrent accesses to a database
+// use shared-memory instances created with the same path.
+func NewSharedMemory(path string, flags OpenFlag) SharedMemory {
+	if flags&OPEN_MAIN_DB == 0 || flags&(OPEN_DELETEONCLOSE|OPEN_MEMORY) != 0 {
+		return nil
+	}
+	return &vfsShm{
+		path:     path,
+		readOnly: flags&OPEN_READONLY != 0,
+	}
+}
+
+type vfsShmFile struct {
+	*os.File
+	info os.FileInfo
+
+	// +checklocks:vfsShmFilesMtx
+	refs int
+
+	// +checklocks:lockMtx
+	lock    [_SHM_NLOCK]int16
+	lockMtx sync.Mutex
+}
+
+var (
+	// +checklocks:vfsShmFilesMtx
+	vfsShmFiles    []*vfsShmFile
+	vfsShmFilesMtx sync.Mutex
+)
+
+type vfsShm struct {
+	*vfsShmFile
+	path     string
+	lock     [_SHM_NLOCK]bool
+	regions  []*util.MappedRegion
+	readOnly bool
+}
+
+func (s *vfsShm) Close() error {
+	if s.vfsShmFile == nil {
+		return nil
+	}
+
+	// Unlock everything.
+	s.shmLock(0, _SHM_NLOCK, _SHM_UNLOCK)
+
+	vfsShmFilesMtx.Lock()
+	defer vfsShmFilesMtx.Unlock()
+
+	// Decrease reference count.
+	if s.vfsShmFile.refs > 1 {
+		s.vfsShmFile.refs--
+		s.vfsShmFile = nil
+		return nil
+	}
+	for i, g := range vfsShmFiles {
+		if g == s.vfsShmFile {
+			vfsShmFiles[i] = nil
+			break
+		}
+	}
+
+	err := s.File.Close()
+	s.vfsShmFile = nil
+	return err
+}
+
+func (s *vfsShm) shmOpen() error {
+	if s.vfsShmFile != nil {
+		return nil
+	}
+
+	var flag int
+	if s.readOnly {
+		flag = unix.O_RDONLY
+	} else {
+		flag = unix.O_RDWR
+	}
+	f, err := os.OpenFile(s.path,
+		flag|unix.O_CREAT|unix.O_NOFOLLOW, 0666)
+	if err != nil {
+		return _CANTOPEN
+	}
+
+	fi, err := f.Stat()
+	if err != nil {
+		return _IOERR_FSTAT
+	}
+
+	vfsShmFilesMtx.Lock()
+	defer vfsShmFilesMtx.Unlock()
+
+	for _, g := range vfsShmFiles {
+		if g != nil && os.SameFile(fi, g.info) {
+			f.Close()
+			g.refs++
+			s.vfsShmFile = g
+			return nil
+		}
+	}
+	s.vfsShmFile = &vfsShmFile{
+		File: f,
+		info: fi,
+		refs: 1,
+	}
+	add := true
+	for i, g := range vfsShmFiles {
+		if g == nil {
+			vfsShmFiles[i] = s.vfsShmFile
+			add = false
+		}
+	}
+	if add {
+		vfsShmFiles = append(vfsShmFiles, s.vfsShmFile)
+	}
+
+	if s.readOnly {
+		return _READONLY_CANTINIT
+	}
+	if rc := osWriteLock(f, _SHM_DMS, 1, 0); rc != _OK {
+		return rc
+	}
+	if err := f.Truncate(0); err != nil {
+		return _IOERR_SHMOPEN
+	}
+	return nil
+}
+
+func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, error) {
+	// Ensure size is a multiple of the OS page size.
+	if int(size)&(unix.Getpagesize()-1) != 0 {
+		return 0, _IOERR_SHMMAP
+	}
+
+	if err := s.shmOpen(); err != nil {
+		return 0, err
+	}
+
+	// Check if file is big enough.
+	o, err := s.Seek(0, io.SeekEnd)
+	if err != nil {
+		return 0, _IOERR_SHMSIZE
+	}
+	if n := (int64(id) + 1) * int64(size); n > o {
+		if !extend {
+			return 0, nil
+		}
+		err := osAllocate(s.File, n)
+		if err != nil {
+			return 0, _IOERR_SHMSIZE
+		}
+	}
+
+	var prot int
+	if s.readOnly {
+		prot = unix.PROT_READ
+	} else {
+		prot = unix.PROT_READ | unix.PROT_WRITE
+	}
+	r, err := util.MapRegion(ctx, mod, s.File, int64(id)*int64(size), size, prot)
+	if err != nil {
+		return 0, err
+	}
+	s.regions = append(s.regions, r)
+	return r.Ptr, nil
+}
+
+func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) error {
+	s.lockMtx.Lock()
+	defer s.lockMtx.Unlock()
+
+	switch {
+	case flags&_SHM_UNLOCK != 0:
+		for i := offset; i < offset+n; i++ {
+			if s.lock[i] {
+				if s.vfsShmFile.lock[i] <= 0 {
+					s.vfsShmFile.lock[i] = 0
+				} else {
+					s.vfsShmFile.lock[i]--
+				}
+			}
+		}
+	case flags&_SHM_SHARED != 0:
+		for i := offset; i < offset+n; i++ {
+			if s.vfsShmFile.lock[i] < 0 {
+				return _BUSY
+			}
+		}
+		for i := offset; i < offset+n; i++ {
+			s.vfsShmFile.lock[i]++
+			s.lock[i] = true
+		}
+	case flags&_SHM_EXCLUSIVE != 0:
+		for i := offset; i < offset+n; i++ {
+			if s.vfsShmFile.lock[i] != 0 {
+				return _BUSY
+			}
+		}
+		for i := offset; i < offset+n; i++ {
+			s.vfsShmFile.lock[i] = -1
+			s.lock[i] = true
+		}
+	}
+
+	return nil
+}
+
+func (s *vfsShm) shmUnmap(delete bool) {
+	if s.vfsShmFile == nil {
+		return
+	}
+
+	// Unmap regions.
+	for _, r := range s.regions {
+		r.Unmap()
+	}
+	clear(s.regions)
+	s.regions = s.regions[:0]
+
+	// Close the file.
+	if delete {
+		os.Remove(s.path)
+	}
+	s.Close()
+	s.vfsShmFile = nil
+}

vfs/shm_other.go

@@ -1,4 +1,4 @@
-//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64) || sqlite3_flock || sqlite3_noshm || sqlite3_nosys
+//go:build !(darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || !(amd64 || arm64 || riscv64) || sqlite3_noshm || sqlite3_nosys
 
 package vfs