diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 98470e1..c0585ab 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -131,7 +131,7 @@ jobs: - name: Test riscv64 (interpreter) run: GOARCH=riscv64 go test -v -short ./... - - name: Test s390x (big-endian, z/OS) + - name: Test s390x (big-endian, z/OS demo) run: GOARCH=s390x go test -v -short -tags sqlite3_flock ./... test-vm: diff --git a/driver/driver_test.go b/driver/driver_test.go index 266e840..c7ea1c7 100644 --- a/driver/driver_test.go +++ b/driver/driver_test.go @@ -13,6 +13,7 @@ import ( "github.com/ncruces/go-sqlite3" _ "github.com/ncruces/go-sqlite3/embed" + _ "github.com/ncruces/go-sqlite3/internal/testcfg" "github.com/ncruces/go-sqlite3/internal/util" "github.com/ncruces/go-sqlite3/vfs" ) diff --git a/func_test.go b/func_test.go index 03b0fdd..357be87 100644 --- a/func_test.go +++ b/func_test.go @@ -9,6 +9,7 @@ import ( "github.com/ncruces/go-sqlite3" _ "github.com/ncruces/go-sqlite3/embed" "github.com/ncruces/go-sqlite3/ext/unicode" + _ "github.com/ncruces/go-sqlite3/internal/testcfg" ) func ExampleConn_CreateCollation() { diff --git a/go.mod b/go.mod index 622aa4d..f2c52e5 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,10 @@ require ( github.com/ncruces/julianday v1.0.0 github.com/psanford/httpreadat v0.1.0 github.com/tetratelabs/wazero v1.7.2 - golang.org/x/crypto v0.23.0 + golang.org/x/crypto v0.24.0 golang.org/x/sync v0.7.0 - golang.org/x/sys v0.20.0 - golang.org/x/text v0.15.0 + golang.org/x/sys v0.21.0 + golang.org/x/text v0.16.0 lukechampine.com/adiantum v1.1.1 ) diff --git a/go.sum b/go.sum index e4df547..f40e765 100644 --- a/go.sum +++ b/go.sum @@ -6,13 +6,13 @@ github.com/psanford/httpreadat v0.1.0 h1:VleW1HS2zO7/4c7c7zNl33fO6oYACSagjJIyMIw github.com/psanford/httpreadat v0.1.0/go.mod h1:Zg7P+TlBm3bYbyHTKv/EdtSJZn3qwbPwpfZ/I9GKCRE= github.com/tetratelabs/wazero v1.7.2 h1:1+z5nXJNwMLPAWaTePFi49SSTL0IMx/i3Fg8Yc25GDc= github.com/tetratelabs/wazero v1.7.2/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= lukechampine.com/adiantum v1.1.1 h1:4fp6gTxWCqpEbLy40ExiYDDED3oUNWx5cTqBCtPdZqA= lukechampine.com/adiantum v1.1.1/go.mod h1:LrAYVnTYLnUtE/yMp5bQr0HstAf060YUF8nM0B6+rUw= diff --git a/go.work.sum b/go.work.sum index 0349376..27b395c 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,4 +1,7 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= diff --git a/gormlite/go.mod b/gormlite/go.mod index ba905fe..b244e0b 100644 --- a/gormlite/go.mod +++ b/gormlite/go.mod @@ -12,5 +12,5 @@ require ( github.com/jinzhu/now v1.1.5 // indirect github.com/ncruces/julianday v1.0.0 // indirect github.com/tetratelabs/wazero v1.7.2 // indirect - golang.org/x/sys v0.20.0 // indirect + golang.org/x/sys v0.21.0 // indirect ) diff --git a/gormlite/go.sum b/gormlite/go.sum index 9f68f28..4c2cf25 100644 --- a/gormlite/go.sum +++ b/gormlite/go.sum @@ -8,8 +8,8 @@ github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g= github.com/tetratelabs/wazero v1.7.2 h1:1+z5nXJNwMLPAWaTePFi49SSTL0IMx/i3Fg8Yc25GDc= github.com/tetratelabs/wazero v1.7.2/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= gorm.io/gorm v1.25.10 h1:dQpO+33KalOA+aFYGlK+EfxcI5MbO7EP2yYygwh9h+s= diff --git a/internal/testcfg/testcfg.go b/internal/testcfg/testcfg.go index c7169ba..23e0d3b 100644 --- a/internal/testcfg/testcfg.go +++ b/internal/testcfg/testcfg.go @@ -1,6 +1,7 @@ package testcfg import ( + "math/bits" "os" "path/filepath" @@ -9,6 +10,10 @@ import ( ) func init() { + if bits.UintSize < 64 { + return + } + sqlite3.RuntimeConfig = wazero.NewRuntimeConfig(). WithMemoryCapacityFromMax(true). WithMemoryLimitPages(1024) diff --git a/internal/util/mmap.go b/internal/util/mmap.go index ada07e1..a4fa2a2 100644 --- a/internal/util/mmap.go +++ b/internal/util/mmap.go @@ -1,4 +1,4 @@ -//go:build (darwin || linux) && (amd64 || arm64 || riscv64) && !(sqlite3_noshm || sqlite3_nosys) +//go:build unix && (amd64 || arm64 || riscv64) && !(sqlite3_noshm || sqlite3_nosys) package util diff --git a/internal/util/mmap_other.go b/internal/util/mmap_other.go index 15e0d3e..a0a3ba6 100644 --- a/internal/util/mmap_other.go +++ b/internal/util/mmap_other.go @@ -1,4 +1,4 @@ -//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64) || sqlite3_noshm || sqlite3_nosys +//go:build !unix || !(amd64 || arm64 || riscv64) || sqlite3_noshm || sqlite3_nosys package util diff --git a/tests/stmt_test.go b/tests/stmt_test.go index 3fcdcf1..a314314 100644 --- a/tests/stmt_test.go +++ b/tests/stmt_test.go @@ -3,6 +3,7 @@ package tests import ( "encoding/json" "math" + "math/bits" "testing" "time" @@ -617,6 +618,9 @@ func TestStmt_ColumnTime(t *testing.T) { } func TestStmt_Error(t *testing.T) { + if bits.UintSize < 64 { + t.Skip("skipping on 32-bit") + } t.Parallel() db, err := sqlite3.Open(":memory:") diff --git a/vfs/README.md b/vfs/README.md index 88059a4..4d3ae8c 100644 --- a/vfs/README.md +++ b/vfs/README.md @@ -46,7 +46,7 @@ to check if your build supports file locking. ### Write-Ahead Logging -On 64-bit Linux and macOS, this module uses `mmap` to implement +On 64-bit Unix, this module uses `mmap` to implement [shared-memory for the WAL-index](https://sqlite.org/wal.html#implementation_of_shared_memory_for_the_wal_index), like SQLite. diff --git a/vfs/shm.go b/vfs/shm.go index 2b76dd5..850c132 100644 --- a/vfs/shm.go +++ b/vfs/shm.go @@ -51,12 +51,7 @@ type vfsShm struct { readOnly bool } -func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, error) { - // Ensure size is a multiple of the OS page size. - if int(size)&(unix.Getpagesize()-1) != 0 { - return 0, _IOERR_SHMMAP - } - +func (s *vfsShm) shmOpen() error { if s.File == nil { var flag int if s.readOnly { @@ -67,29 +62,41 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext f, err := os.OpenFile(s.path, flag|unix.O_CREAT|unix.O_NOFOLLOW, 0666) if err != nil { - return 0, _CANTOPEN + return _CANTOPEN } s.File = f } // Dead man's switch. if lock, rc := osGetLock(s.File, _SHM_DMS, 1); rc != _OK { - return 0, _IOERR_LOCK + return _IOERR_LOCK } else if lock == unix.F_WRLCK { - return 0, _BUSY + return _BUSY } else if lock == unix.F_UNLCK { if s.readOnly { - return 0, _READONLY_CANTINIT + return _READONLY_CANTINIT } if rc := osWriteLock(s.File, _SHM_DMS, 1, 0); rc != _OK { - return 0, rc + return rc } if err := s.Truncate(0); err != nil { - return 0, _IOERR_SHMOPEN + return _IOERR_SHMOPEN } } if rc := osReadLock(s.File, _SHM_DMS, 1, 0); rc != _OK { - return 0, rc + return rc + } + return nil +} + +func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, error) { + // Ensure size is a multiple of the OS page size. + if int(size)&(unix.Getpagesize()-1) != 0 { + return 0, _IOERR_SHMMAP + } + + if err := s.shmOpen(); err != nil { + return 0, err } // Check if file is big enough. @@ -165,9 +172,9 @@ func (s *vfsShm) shmUnmap(delete bool) { s.regions = s.regions[:0] // Close the file. - defer s.Close() if delete { - os.Remove(s.Name()) + os.Remove(s.path) } + s.Close() s.File = nil } diff --git a/vfs/shm_bsd.go b/vfs/shm_bsd.go new file mode 100644 index 0000000..94e5274 --- /dev/null +++ b/vfs/shm_bsd.go @@ -0,0 +1,261 @@ +//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (amd64 || arm64 || riscv64) && !(sqlite3_noshm || sqlite3_nosys) + +package vfs + +import ( + "context" + "io" + "os" + "sync" + + "github.com/ncruces/go-sqlite3/internal/util" + "github.com/tetratelabs/wazero/api" + "golang.org/x/sys/unix" +) + +// SupportsSharedMemory is false on platforms that do not support shared memory. +// To use [WAL without shared-memory], you need to set [EXCLUSIVE locking mode]. +// +// [WAL without shared-memory]: https://sqlite.org/wal.html#noshm +// [EXCLUSIVE locking mode]: https://sqlite.org/pragma.html#pragma_locking_mode +const SupportsSharedMemory = true + +const ( + _SHM_NLOCK = 8 + _SHM_BASE = 120 + _SHM_DMS = _SHM_BASE + _SHM_NLOCK +) + +func (f *vfsFile) SharedMemory() SharedMemory { return f.shm } + +// NewSharedMemory returns a shared-memory WAL-index +// backed by a file with the given path. +// It will return nil if shared-memory is not supported, +// or not appropriate for the given flags. +// Only [OPEN_MAIN_DB] databases may need a WAL-index. +// You must ensure all concurrent accesses to a database +// use shared-memory instances created with the same path. +func NewSharedMemory(path string, flags OpenFlag) SharedMemory { + if flags&OPEN_MAIN_DB == 0 || flags&(OPEN_DELETEONCLOSE|OPEN_MEMORY) != 0 { + return nil + } + return &vfsShm{ + path: path, + readOnly: flags&OPEN_READONLY != 0, + } +} + +type vfsShmFile struct { + *os.File + info os.FileInfo + + // +checklocks:vfsShmFilesMtx + refs int + + // +checklocks:lockMtx + lock [_SHM_NLOCK]int16 + lockMtx sync.Mutex +} + +var ( + // +checklocks:vfsShmFilesMtx + vfsShmFiles []*vfsShmFile + vfsShmFilesMtx sync.Mutex +) + +type vfsShm struct { + *vfsShmFile + path string + lock [_SHM_NLOCK]bool + regions []*util.MappedRegion + readOnly bool +} + +func (s *vfsShm) Close() error { + if s.vfsShmFile == nil { + return nil + } + + // Unlock everything. + s.shmLock(0, _SHM_NLOCK, _SHM_UNLOCK) + + vfsShmFilesMtx.Lock() + defer vfsShmFilesMtx.Unlock() + + // Decrease reference count. + if s.vfsShmFile.refs > 1 { + s.vfsShmFile.refs-- + s.vfsShmFile = nil + return nil + } + for i, g := range vfsShmFiles { + if g == s.vfsShmFile { + vfsShmFiles[i] = nil + break + } + } + + err := s.File.Close() + s.vfsShmFile = nil + return err +} + +func (s *vfsShm) shmOpen() error { + if s.vfsShmFile != nil { + return nil + } + + var flag int + if s.readOnly { + flag = unix.O_RDONLY + } else { + flag = unix.O_RDWR + } + f, err := os.OpenFile(s.path, + flag|unix.O_CREAT|unix.O_NOFOLLOW, 0666) + if err != nil { + return _CANTOPEN + } + + fi, err := f.Stat() + if err != nil { + return _IOERR_FSTAT + } + + vfsShmFilesMtx.Lock() + defer vfsShmFilesMtx.Unlock() + + for _, g := range vfsShmFiles { + if g != nil && os.SameFile(fi, g.info) { + f.Close() + g.refs++ + s.vfsShmFile = g + return nil + } + } + s.vfsShmFile = &vfsShmFile{ + File: f, + info: fi, + refs: 1, + } + add := true + for i, g := range vfsShmFiles { + if g == nil { + vfsShmFiles[i] = s.vfsShmFile + add = false + } + } + if add { + vfsShmFiles = append(vfsShmFiles, s.vfsShmFile) + } + + if s.readOnly { + return _READONLY_CANTINIT + } + if rc := osWriteLock(f, _SHM_DMS, 1, 0); rc != _OK { + return rc + } + if err := f.Truncate(0); err != nil { + return _IOERR_SHMOPEN + } + return nil +} + +func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, error) { + // Ensure size is a multiple of the OS page size. + if int(size)&(unix.Getpagesize()-1) != 0 { + return 0, _IOERR_SHMMAP + } + + if err := s.shmOpen(); err != nil { + return 0, err + } + + // Check if file is big enough. + o, err := s.Seek(0, io.SeekEnd) + if err != nil { + return 0, _IOERR_SHMSIZE + } + if n := (int64(id) + 1) * int64(size); n > o { + if !extend { + return 0, nil + } + err := osAllocate(s.File, n) + if err != nil { + return 0, _IOERR_SHMSIZE + } + } + + var prot int + if s.readOnly { + prot = unix.PROT_READ + } else { + prot = unix.PROT_READ | unix.PROT_WRITE + } + r, err := util.MapRegion(ctx, mod, s.File, int64(id)*int64(size), size, prot) + if err != nil { + return 0, err + } + s.regions = append(s.regions, r) + return r.Ptr, nil +} + +func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) error { + s.lockMtx.Lock() + defer s.lockMtx.Unlock() + + switch { + case flags&_SHM_UNLOCK != 0: + for i := offset; i < offset+n; i++ { + if s.lock[i] { + if s.vfsShmFile.lock[i] <= 0 { + s.vfsShmFile.lock[i] = 0 + } else { + s.vfsShmFile.lock[i]-- + } + } + } + case flags&_SHM_SHARED != 0: + for i := offset; i < offset+n; i++ { + if s.vfsShmFile.lock[i] < 0 { + return _BUSY + } + } + for i := offset; i < offset+n; i++ { + s.vfsShmFile.lock[i]++ + s.lock[i] = true + } + case flags&_SHM_EXCLUSIVE != 0: + for i := offset; i < offset+n; i++ { + if s.vfsShmFile.lock[i] != 0 { + return _BUSY + } + } + for i := offset; i < offset+n; i++ { + s.vfsShmFile.lock[i] = -1 + s.lock[i] = true + } + } + + return nil +} + +func (s *vfsShm) shmUnmap(delete bool) { + if s.vfsShmFile == nil { + return + } + + // Unmap regions. + for _, r := range s.regions { + r.Unmap() + } + clear(s.regions) + s.regions = s.regions[:0] + + // Close the file. + if delete { + os.Remove(s.path) + } + s.Close() + s.vfsShmFile = nil +} diff --git a/vfs/shm_other.go b/vfs/shm_other.go index 2119197..7c89975 100644 --- a/vfs/shm_other.go +++ b/vfs/shm_other.go @@ -1,4 +1,4 @@ -//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64) || sqlite3_flock || sqlite3_noshm || sqlite3_nosys +//go:build !(darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || !(amd64 || arm64 || riscv64) || sqlite3_noshm || sqlite3_nosys package vfs