From cc0b011e8dddd0723e5d4114126560ead4b914d7 Mon Sep 17 00:00:00 2001 From: Nuno Cruces Date: Sat, 13 Apr 2024 13:43:24 +0100 Subject: [PATCH] Readonly WAL. --- internal/util/mmap.go | 10 +++++----- internal/util/mmap_other.go | 2 +- tests/wal_test.go | 32 ++++++++++++++++++++++++++++++++ vfs/README.md | 9 ++++----- vfs/shm.go | 17 ++++++++++++----- vfs/shm_other.go | 2 +- 6 files changed, 55 insertions(+), 17 deletions(-) diff --git a/internal/util/mmap.go b/internal/util/mmap.go index c3daaab..eda2790 100644 --- a/internal/util/mmap.go +++ b/internal/util/mmap.go @@ -1,4 +1,4 @@ -//go:build (darwin || linux || illumos) && (amd64 || arm64) && !sqlite3_flock && !sqlite3_noshm && !sqlite3_nosys +//go:build (darwin || linux || illumos) && (amd64 || arm64 || riscv64) && !sqlite3_flock && !sqlite3_noshm && !sqlite3_nosys package util @@ -69,10 +69,10 @@ type MappedRegion struct { used bool } -func MapRegion(ctx context.Context, mod api.Module, f *os.File, offset int64, size int32) (*MappedRegion, error) { +func MapRegion(ctx context.Context, mod api.Module, f *os.File, offset int64, size int32, prot int) (*MappedRegion, error) { s := ctx.Value(moduleKey{}).(*moduleState) r := s.new(ctx, mod, size) - err := r.mmap(f, offset) + err := r.mmap(f, offset, prot) if err != nil { return nil, err } @@ -90,9 +90,9 @@ func (r *MappedRegion) Unmap() error { return err } -func (r *MappedRegion) mmap(f *os.File, offset int64) error { +func (r *MappedRegion) mmap(f *os.File, offset int64, prot int) error { _, err := mmap(r.addr, uintptr(r.size), - unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED|unix.MAP_FIXED, + prot, unix.MAP_SHARED|unix.MAP_FIXED, int(f.Fd()), offset) r.used = err == nil return err diff --git a/internal/util/mmap_other.go b/internal/util/mmap_other.go index 8e46e83..41a510f 100644 --- a/internal/util/mmap_other.go +++ b/internal/util/mmap_other.go @@ -1,4 +1,4 @@ -//go:build !(darwin || linux || illumos) || !(amd64 || arm64) || sqlite3_flock || sqlite3_noshm || sqlite3_nosys +//go:build !(darwin || linux || illumos) || !(amd64 || arm64 || riscv64) || sqlite3_flock || sqlite3_noshm || sqlite3_nosys package util diff --git a/tests/wal_test.go b/tests/wal_test.go index 4ed6d17..2f525b2 100644 --- a/tests/wal_test.go +++ b/tests/wal_test.go @@ -1,10 +1,12 @@ package tests import ( + "os" "path/filepath" "testing" "github.com/ncruces/go-sqlite3" + "github.com/ncruces/go-sqlite3/vfs" ) func TestWAL_enter_exit(t *testing.T) { @@ -32,6 +34,36 @@ func TestWAL_enter_exit(t *testing.T) { } } +func TestWAL_readonly(t *testing.T) { + if !vfs.SupportsSharedMemory { + t.Skip("skipping without shared memory") + } + + t.Parallel() + + tmp := filepath.Join(t.TempDir(), "test.db") + err := os.WriteFile(tmp, waldb, 0666) + if err != nil { + t.Fatal(err) + } + + db, err := sqlite3.OpenFlags(tmp, sqlite3.OPEN_READONLY) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + stmt, _, err := db.Prepare(`SELECT * FROM sqlite_master`) + if err != nil { + t.Fatal(err) + } + defer stmt.Close() + + if stmt.Step() { + t.Error("want no rows") + } +} + func TestConn_WalCheckpoint(t *testing.T) { t.Parallel() diff --git a/vfs/README.md b/vfs/README.md index 5d33e59..40bfbe5 100644 --- a/vfs/README.md +++ b/vfs/README.md @@ -49,19 +49,18 @@ On 64-bit Linux, macOS and illumos, this module uses `mmap` to implement [shared-memory for the WAL-index](https://sqlite.org/wal.html#implementation_of_shared_memory_for_the_wal_index), like SQLite. -To allow `mmap` to work, each connection needs to reserve a lot of address space.\ +To allow `mmap` to work, each connection needs to reserve up to 4GB of address space.\ To limit the amount of address space each connection needs, use [`WithMemoryLimitPages`](../tests/parallel/parallel_test.go#L21). On all other platforms, [WAL](https://sqlite.org/wal.html) support is [limited](https://sqlite.org/wal.html#noshm). -To work around that limitation, SQLite is [patched](sqlite3/locking_mode.patch) +To work around this limitation, SQLite is [patched](sqlite3/locking_mode.patch) to automatically use `EXCLUSIVE` locking mode for WAL databases on such platforms. -Because connection pooling is incompatible with `EXCLUSIVE` locking mode, -to use the [`database/sql`](https://pkg.go.dev/database/sql) driver -with WAL mode databases you should disable connection pooling by calling +To use the [`database/sql`](https://pkg.go.dev/database/sql) driver +with `EXCLUSIVE` locking mode you should disable connection pooling by calling [`db.SetMaxOpenConns(1)`](https://pkg.go.dev/database/sql#DB.SetMaxOpenConns). You can use [`vfs.SupportsSharedMemory`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs#SupportsSharedMemory) diff --git a/vfs/shm.go b/vfs/shm.go index 42da5b9..381498d 100644 --- a/vfs/shm.go +++ b/vfs/shm.go @@ -1,4 +1,4 @@ -//go:build (linux || darwin) && (amd64 || arm64) && !sqlite3_flock && !sqlite3_noshm && !sqlite3_nosys +//go:build (darwin || linux || illumos) && (amd64 || arm64 || riscv64) && !sqlite3_flock && !sqlite3_noshm && !sqlite3_nosys package vfs @@ -42,11 +42,12 @@ func (f *vfsFile) shmMap(ctx context.Context, mod api.Module, id, size int32, ex if f.shm.File == nil { var flag int if f.readOnly { - flag = unix.O_RDONLY | unix.O_NOFOLLOW + flag = unix.O_RDONLY } else { - flag = unix.O_RDWR | unix.O_CREAT | unix.O_NOFOLLOW + flag = unix.O_RDWR } - s, err := os.OpenFile(f.Name()+"-shm", flag, 0666) + s, err := os.OpenFile(f.Name()+"-shm", + flag|unix.O_CREAT|unix.O_NOFOLLOW, 0666) if err != nil { return 0, _CANTOPEN } @@ -88,7 +89,13 @@ func (f *vfsFile) shmMap(ctx context.Context, mod api.Module, id, size int32, ex } } - r, err := util.MapRegion(ctx, mod, f.shm.File, int64(id)*int64(size), size) + var prot int + if f.readOnly { + prot = unix.PROT_READ + } else { + prot = unix.PROT_READ | unix.PROT_WRITE + } + r, err := util.MapRegion(ctx, mod, f.shm.File, int64(id)*int64(size), size, prot) if err != nil { return 0, err } diff --git a/vfs/shm_other.go b/vfs/shm_other.go index 922e147..de11ce4 100644 --- a/vfs/shm_other.go +++ b/vfs/shm_other.go @@ -1,4 +1,4 @@ -//go:build !(linux || darwin) || !(amd64 || arm64) || sqlite3_flock || sqlite3_noshm || sqlite3_nosys +//go:build !(darwin || linux || illumos) || !(amd64 || arm64 || riscv64) || sqlite3_flock || sqlite3_noshm || sqlite3_nosys package vfs