BSD WAL support. (#90)

Uses in-memory locks.
Also supports illumos.

vfs/README.md

@@ -46,7 +46,7 @@ to check if your build supports file locking.
 
 ### Write-Ahead Logging
 
-On 64-bit Linux and macOS, this module uses `mmap` to implement
+On 64-bit Unix, this module uses `mmap` to implement
 [shared-memory for the WAL-index](https://sqlite.org/wal.html#implementation_of_shared_memory_for_the_wal_index),
 like SQLite.
 

vfs/shm.go

@@ -51,12 +51,7 @@ type vfsShm struct {
 	readOnly bool
 }
 
-func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, error) {
-	// Ensure size is a multiple of the OS page size.
-	if int(size)&(unix.Getpagesize()-1) != 0 {
-		return 0, _IOERR_SHMMAP
-	}
-
+func (s *vfsShm) shmOpen() error {
 	if s.File == nil {
 		var flag int
 		if s.readOnly {
@@ -67,29 +62,41 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext
 		f, err := os.OpenFile(s.path,
 			flag|unix.O_CREAT|unix.O_NOFOLLOW, 0666)
 		if err != nil {
-			return 0, _CANTOPEN
+			return _CANTOPEN
 		}
 		s.File = f
 	}
 
 	// Dead man's switch.
 	if lock, rc := osGetLock(s.File, _SHM_DMS, 1); rc != _OK {
-		return 0, _IOERR_LOCK
+		return _IOERR_LOCK
 	} else if lock == unix.F_WRLCK {
-		return 0, _BUSY
+		return _BUSY
 	} else if lock == unix.F_UNLCK {
 		if s.readOnly {
-			return 0, _READONLY_CANTINIT
+			return _READONLY_CANTINIT
 		}
 		if rc := osWriteLock(s.File, _SHM_DMS, 1, 0); rc != _OK {
-			return 0, rc
+			return rc
 		}
 		if err := s.Truncate(0); err != nil {
-			return 0, _IOERR_SHMOPEN
+			return _IOERR_SHMOPEN
 		}
 	}
 	if rc := osReadLock(s.File, _SHM_DMS, 1, 0); rc != _OK {
-		return 0, rc
+		return rc
+	}
+	return nil
+}
+
+func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, error) {
+	// Ensure size is a multiple of the OS page size.
+	if int(size)&(unix.Getpagesize()-1) != 0 {
+		return 0, _IOERR_SHMMAP
+	}
+
+	if err := s.shmOpen(); err != nil {
+		return 0, err
 	}
 
 	// Check if file is big enough.
@@ -165,9 +172,9 @@ func (s *vfsShm) shmUnmap(delete bool) {
 	s.regions = s.regions[:0]
 
 	// Close the file.
-	defer s.Close()
 	if delete {
-		os.Remove(s.Name())
+		os.Remove(s.path)
 	}
+	s.Close()
 	s.File = nil
 }

vfs/shm_bsd.go

@@ -0,0 +1,261 @@
+//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (amd64 || arm64 || riscv64) && !(sqlite3_noshm || sqlite3_nosys)
+
+package vfs
+
+import (
+	"context"
+	"io"
+	"os"
+	"sync"
+
+	"github.com/ncruces/go-sqlite3/internal/util"
+	"github.com/tetratelabs/wazero/api"
+	"golang.org/x/sys/unix"
+)
+
+// SupportsSharedMemory is false on platforms that do not support shared memory.
+// To use [WAL without shared-memory], you need to set [EXCLUSIVE locking mode].
+//
+// [WAL without shared-memory]: https://sqlite.org/wal.html#noshm
+// [EXCLUSIVE locking mode]: https://sqlite.org/pragma.html#pragma_locking_mode
+const SupportsSharedMemory = true
+
+const (
+	_SHM_NLOCK = 8
+	_SHM_BASE  = 120
+	_SHM_DMS   = _SHM_BASE + _SHM_NLOCK
+)
+
+func (f *vfsFile) SharedMemory() SharedMemory { return f.shm }
+
+// NewSharedMemory returns a shared-memory WAL-index
+// backed by a file with the given path.
+// It will return nil if shared-memory is not supported,
+// or not appropriate for the given flags.
+// Only [OPEN_MAIN_DB] databases may need a WAL-index.
+// You must ensure all concurrent accesses to a database
+// use shared-memory instances created with the same path.
+func NewSharedMemory(path string, flags OpenFlag) SharedMemory {
+	if flags&OPEN_MAIN_DB == 0 || flags&(OPEN_DELETEONCLOSE|OPEN_MEMORY) != 0 {
+		return nil
+	}
+	return &vfsShm{
+		path:     path,
+		readOnly: flags&OPEN_READONLY != 0,
+	}
+}
+
+type vfsShmFile struct {
+	*os.File
+	info os.FileInfo
+
+	// +checklocks:vfsShmFilesMtx
+	refs int
+
+	// +checklocks:lockMtx
+	lock    [_SHM_NLOCK]int16
+	lockMtx sync.Mutex
+}
+
+var (
+	// +checklocks:vfsShmFilesMtx
+	vfsShmFiles    []*vfsShmFile
+	vfsShmFilesMtx sync.Mutex
+)
+
+type vfsShm struct {
+	*vfsShmFile
+	path     string
+	lock     [_SHM_NLOCK]bool
+	regions  []*util.MappedRegion
+	readOnly bool
+}
+
+func (s *vfsShm) Close() error {
+	if s.vfsShmFile == nil {
+		return nil
+	}
+
+	// Unlock everything.
+	s.shmLock(0, _SHM_NLOCK, _SHM_UNLOCK)
+
+	vfsShmFilesMtx.Lock()
+	defer vfsShmFilesMtx.Unlock()
+
+	// Decrease reference count.
+	if s.vfsShmFile.refs > 1 {
+		s.vfsShmFile.refs--
+		s.vfsShmFile = nil
+		return nil
+	}
+	for i, g := range vfsShmFiles {
+		if g == s.vfsShmFile {
+			vfsShmFiles[i] = nil
+			break
+		}
+	}
+
+	err := s.File.Close()
+	s.vfsShmFile = nil
+	return err
+}
+
+func (s *vfsShm) shmOpen() error {
+	if s.vfsShmFile != nil {
+		return nil
+	}
+
+	var flag int
+	if s.readOnly {
+		flag = unix.O_RDONLY
+	} else {
+		flag = unix.O_RDWR
+	}
+	f, err := os.OpenFile(s.path,
+		flag|unix.O_CREAT|unix.O_NOFOLLOW, 0666)
+	if err != nil {
+		return _CANTOPEN
+	}
+
+	fi, err := f.Stat()
+	if err != nil {
+		return _IOERR_FSTAT
+	}
+
+	vfsShmFilesMtx.Lock()
+	defer vfsShmFilesMtx.Unlock()
+
+	for _, g := range vfsShmFiles {
+		if g != nil && os.SameFile(fi, g.info) {
+			f.Close()
+			g.refs++
+			s.vfsShmFile = g
+			return nil
+		}
+	}
+	s.vfsShmFile = &vfsShmFile{
+		File: f,
+		info: fi,
+		refs: 1,
+	}
+	add := true
+	for i, g := range vfsShmFiles {
+		if g == nil {
+			vfsShmFiles[i] = s.vfsShmFile
+			add = false
+		}
+	}
+	if add {
+		vfsShmFiles = append(vfsShmFiles, s.vfsShmFile)
+	}
+
+	if s.readOnly {
+		return _READONLY_CANTINIT
+	}
+	if rc := osWriteLock(f, _SHM_DMS, 1, 0); rc != _OK {
+		return rc
+	}
+	if err := f.Truncate(0); err != nil {
+		return _IOERR_SHMOPEN
+	}
+	return nil
+}
+
+func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, error) {
+	// Ensure size is a multiple of the OS page size.
+	if int(size)&(unix.Getpagesize()-1) != 0 {
+		return 0, _IOERR_SHMMAP
+	}
+
+	if err := s.shmOpen(); err != nil {
+		return 0, err
+	}
+
+	// Check if file is big enough.
+	o, err := s.Seek(0, io.SeekEnd)
+	if err != nil {
+		return 0, _IOERR_SHMSIZE
+	}
+	if n := (int64(id) + 1) * int64(size); n > o {
+		if !extend {
+			return 0, nil
+		}
+		err := osAllocate(s.File, n)
+		if err != nil {
+			return 0, _IOERR_SHMSIZE
+		}
+	}
+
+	var prot int
+	if s.readOnly {
+		prot = unix.PROT_READ
+	} else {
+		prot = unix.PROT_READ | unix.PROT_WRITE
+	}
+	r, err := util.MapRegion(ctx, mod, s.File, int64(id)*int64(size), size, prot)
+	if err != nil {
+		return 0, err
+	}
+	s.regions = append(s.regions, r)
+	return r.Ptr, nil
+}
+
+func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) error {
+	s.lockMtx.Lock()
+	defer s.lockMtx.Unlock()
+
+	switch {
+	case flags&_SHM_UNLOCK != 0:
+		for i := offset; i < offset+n; i++ {
+			if s.lock[i] {
+				if s.vfsShmFile.lock[i] <= 0 {
+					s.vfsShmFile.lock[i] = 0
+				} else {
+					s.vfsShmFile.lock[i]--
+				}
+			}
+		}
+	case flags&_SHM_SHARED != 0:
+		for i := offset; i < offset+n; i++ {
+			if s.vfsShmFile.lock[i] < 0 {
+				return _BUSY
+			}
+		}
+		for i := offset; i < offset+n; i++ {
+			s.vfsShmFile.lock[i]++
+			s.lock[i] = true
+		}
+	case flags&_SHM_EXCLUSIVE != 0:
+		for i := offset; i < offset+n; i++ {
+			if s.vfsShmFile.lock[i] != 0 {
+				return _BUSY
+			}
+		}
+		for i := offset; i < offset+n; i++ {
+			s.vfsShmFile.lock[i] = -1
+			s.lock[i] = true
+		}
+	}
+
+	return nil
+}
+
+func (s *vfsShm) shmUnmap(delete bool) {
+	if s.vfsShmFile == nil {
+		return
+	}
+
+	// Unmap regions.
+	for _, r := range s.regions {
+		r.Unmap()
+	}
+	clear(s.regions)
+	s.regions = s.regions[:0]
+
+	// Close the file.
+	if delete {
+		os.Remove(s.path)
+	}
+	s.Close()
+	s.vfsShmFile = nil
+}

vfs/shm_other.go

@@ -1,4 +1,4 @@
-//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64) || sqlite3_flock || sqlite3_noshm || sqlite3_nosys
+//go:build !(darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || !(amd64 || arm64 || riscv64) || sqlite3_noshm || sqlite3_nosys
 
 package vfs