go/sqlite3
1
0
Fork 0
mirror of https://github.com/ncruces/go-sqlite3.git synced 2026-01-11 21:49:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Configure memory, 32-bit WAL. (#170)

Browse Source
This commit is contained in:
Nuno Cruces
2024-10-17 13:04:23 +01:00
committed by GitHub
parent 50c8517603
commit c900889848
21 changed files with 47 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/test.yml vendored
View File

@@ -160,7 +160,7 @@ jobs:
- name: Test ppc64le (interpreter)
run: GOARCH=ppc64le go test -v -short ./...
- name: Test s390x (big-endian, z/OS demo)
- name: Test s390x (big-endian, z/OS like)
run: GOARCH=s390x go test -v -short -tags sqlite3_flock ./...
test-vm:

19
internal/alloc/alloc_other.go
View File

@@ -4,6 +4,21 @@ package alloc
import "github.com/tetratelabs/wazero/experimental"
func Virtual(cap, max uint64) experimental.LinearMemory {
return Slice(cap, max)
func NewMemory(cap, max uint64) experimental.LinearMemory {
return &sliceMemory{make([]byte, 0, cap)}
}
type sliceMemory struct {
buf []byte
}
func (b *sliceMemory) Free() {}
func (b *sliceMemory) Reallocate(size uint64) []byte {
if cap := uint64(cap(b.buf)); size > cap {
b.buf = append(b.buf[:cap], make([]byte, size-cap)...)
} else {
b.buf = b.buf[:size]
}
return b.buf
}

24
internal/alloc/alloc_slice.go
View File

@@ -1,24 +0,0 @@
//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys
package alloc
import "github.com/tetratelabs/wazero/experimental"
func Slice(cap, _ uint64) experimental.LinearMemory {
return &sliceMemory{make([]byte, 0, cap)}
}
type sliceMemory struct {
buf []byte
}
func (b *sliceMemory) Free() {}
func (b *sliceMemory) Reallocate(size uint64) []byte {
if cap := uint64(cap(b.buf)); size > cap {
b.buf = append(b.buf[:cap], make([]byte, size-cap)...)
} else {
b.buf = b.buf[:size]
}
return b.buf
}

2
internal/alloc/alloc_test.go
View File

@@ -9,6 +9,6 @@ import (
func TestVirtual(t *testing.T) {
defer func() { _ = recover() }()
alloc.Virtual(math.MaxInt+2, math.MaxInt+2)
alloc.NewMemory(math.MaxInt+2, math.MaxInt+2)
t.Error("want panic")
}

2
internal/alloc/alloc_unix.go
View File

@@ -9,7 +9,7 @@ import (
"golang.org/x/sys/unix"
)
func Virtual(_, max uint64) experimental.LinearMemory {
func NewMemory(_, max uint64) experimental.LinearMemory {
// Round up to the page size.
rnd := uint64(unix.Getpagesize() - 1)
max = (max + rnd) &^ rnd

2
internal/alloc/alloc_windows.go
View File

@@ -11,7 +11,7 @@ import (
"golang.org/x/sys/windows"
)
func Virtual(_, max uint64) experimental.LinearMemory {
func NewMemory(_, max uint64) experimental.LinearMemory {
// Round up to the page size.
rnd := uint64(windows.Getpagesize() - 1)
max = (max + rnd) &^ rnd

8
internal/testcfg/testcfg.go
View File

@@ -1,7 +1,6 @@
package testcfg
import (
"math/bits"
"os"
"path/filepath"
@@ -12,13 +11,8 @@ import (
// notest
func init() {
if bits.UintSize < 64 {
return
}
sqlite3.RuntimeConfig = wazero.NewRuntimeConfig().
WithMemoryCapacityFromMax(true).
WithMemoryLimitPages(1024)
WithMemoryLimitPages(512)
if os.Getenv("CI") != "" {
path := filepath.Join(os.TempDir(), "wazero")

9
internal/util/mmap.go
View File

@@ -1,4 +1,4 @@
//go:build unix && (amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys)
//go:build unix && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys)
package util
@@ -7,17 +7,10 @@ import (
"os"
"unsafe"
"github.com/ncruces/go-sqlite3/internal/alloc"
"github.com/tetratelabs/wazero/api"
"github.com/tetratelabs/wazero/experimental"
"golang.org/x/sys/unix"
)
func withAllocator(ctx context.Context) context.Context {
return experimental.WithMemoryAllocator(ctx,
experimental.MemoryAllocatorFunc(alloc.Virtual))
}
type mmapState struct {
regions []*MappedRegion
}

19
internal/util/mmap_other.go
View File

@@ -1,22 +1,5 @@
//go:build !unix || !(amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys
//go:build !unix || !(386 || arm || amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys
package util
import (
"context"
"github.com/ncruces/go-sqlite3/internal/alloc"
"github.com/tetratelabs/wazero/experimental"
)
type mmapState struct{}
func withAllocator(ctx context.Context) context.Context {
return experimental.WithMemoryAllocator(ctx,
experimental.MemoryAllocatorFunc(func(cap, max uint64) experimental.LinearMemory {
if cap == max {
return alloc.Virtual(cap, max)
}
return alloc.Slice(cap, max)
}))
}

3
internal/util/module.go
View File

@@ -3,6 +3,7 @@ package util
import (
"context"
"github.com/ncruces/go-sqlite3/internal/alloc"
"github.com/tetratelabs/wazero/experimental"
)
@@ -14,7 +15,7 @@ type moduleState struct {
func NewContext(ctx context.Context) context.Context {
state := new(moduleState)
ctx = withAllocator(ctx)
ctx = experimental.WithMemoryAllocator(ctx, experimental.MemoryAllocatorFunc(alloc.NewMemory))
ctx = experimental.WithCloseNotifier(ctx, state)
ctx = context.WithValue(ctx, moduleKey{}, state)
return ctx

9
sqlite.go
View File

@@ -49,10 +49,15 @@ func compileSQLite() {
cfg := RuntimeConfig
if cfg == nil {
cfg = wazero.NewRuntimeConfig()
if bits.UintSize >= 64 {
cfg = cfg.WithMemoryLimitPages(4096) // 256MB
} else {
cfg = cfg.WithMemoryLimitPages(512) // 32MB
}
}
cfg = cfg.WithCoreFeatures(api.CoreFeaturesV2 | experimental.CoreFeaturesThreads)
instance.runtime = wazero.NewRuntimeWithConfig(ctx,
cfg.WithCoreFeatures(api.CoreFeaturesV2|experimental.CoreFeaturesThreads))
instance.runtime = wazero.NewRuntimeWithConfig(ctx, cfg)
env := instance.runtime.NewHostModuleBuilder("env")
env = vfs.ExportHostFunctions(env)

2
sqlite_test.go
View File

@@ -6,12 +6,10 @@ import (
"testing"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/tetratelabs/wazero"
)
func init() {
Path = "./embed/sqlite3.wasm"
RuntimeConfig = wazero.NewRuntimeConfig().WithMemoryLimitPages(1024)
}
func Test_sqlite_error_OOM(t *testing.T) {

3
tests/time_test.go
View File

@@ -223,6 +223,9 @@ func TestDB_timeCollation(t *testing.T) {
}
func TestDB_isoWeek(t *testing.T) {
if testing.Short() {
t.Skip("skipping in short mode")
}
t.Parallel()
db, err := sqlite3.Open(":memory:")

8
vfs/README.md
View File

@@ -46,18 +46,14 @@ to check if your build supports file locking.
### Write-Ahead Logging
On 64-bit little-endian Unix, this module uses `mmap` to implement
On little-endian Unix, this module uses `mmap` to implement
[shared-memory for the WAL-index](https://sqlite.org/wal.html#implementation_of_shared_memory_for_the_wal_index),
like SQLite.
To allow `mmap` to work, each connection needs to reserve up to 4GB of address space.
To limit the address space each connection reserves,
use [`WithMemoryLimitPages`](../tests/testcfg/testcfg.go).
With [BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2)
a WAL database can only be accessed by a single proccess.
Other processes that attempt to access a database locked with BSD locks,
will fail with the `SQLITE_PROTOCOL` error code.
will fail with the [`SQLITE_PROTOCOL`](https://sqlite.org/rescode.html#protocol) error code.
Otherwise, [WAL support is limited](https://sqlite.org/wal.html#noshm),
and `EXCLUSIVE` locking mode must be set to create, read, and write WAL databases.

4
vfs/os_f2fs_linux.go
View File

@@ -1,4 +1,4 @@
//go:build (amd64 || arm64 || riscv64 || ppc64le) && !sqlite3_nosys
//go:build (amd64 || arm64 || riscv64) && !sqlite3_nosys
package vfs
@@ -13,7 +13,7 @@ const (
_F2FS_IOC_START_ATOMIC_WRITE = 62721
_F2FS_IOC_COMMIT_ATOMIC_WRITE = 62722
_F2FS_IOC_ABORT_ATOMIC_WRITE = 62725
_F2FS_IOC_GET_FEATURES = 2147808524
_F2FS_IOC_GET_FEATURES = 2147808524 // -2147158772
_F2FS_FEATURE_ATOMIC_WRITE = 4
)

2
vfs/os_std_atomic.go
View File

@@ -1,4 +1,4 @@
//go:build !linux || !(amd64 || arm64 || riscv64 || ppc64le) || sqlite3_nosys
//go:build !linux || !(amd64 || arm64 || riscv64) || sqlite3_nosys
package vfs

2
vfs/shm.go
View File

@@ -1,4 +1,4 @@
//go:build (darwin || linux) && (amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_flock || sqlite3_noshm || sqlite3_nosys)
//go:build (darwin || linux) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_flock || sqlite3_noshm || sqlite3_nosys)
package vfs

2
vfs/shm_bsd.go
View File

@@ -1,4 +1,4 @@
//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys)
//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys)
package vfs

2
vfs/shm_other.go
View File

@@ -1,4 +1,4 @@
//go:build !(darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || !(amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys
//go:build !(darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || !(386 || arm || amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys
package vfs

3
vfs/tests/mptest/mptest_test.go
View File

@@ -43,10 +43,9 @@ func TestMain(m *testing.M) {
ctx := context.Background()
cfg := wazero.NewRuntimeConfig().
WithCoreFeatures(api.CoreFeaturesV2 | experimental.CoreFeaturesThreads).
WithMemoryLimitPages(1024)
WithMemoryLimitPages(512)
rt = wazero.NewRuntimeWithConfig(ctx, cfg)
wasi_snapshot_preview1.MustInstantiate(ctx, rt)
env := vfs.ExportHostFunctions(rt.NewHostModuleBuilder("env"))
env.NewFunctionBuilder().WithFunc(system).Export("system")
_, err := env.Instantiate(ctx)

3
vfs/tests/speedtest1/speedtest1_test.go
View File

@@ -42,7 +42,8 @@ func TestMain(m *testing.M) {
ctx := context.Background()
cfg := wazero.NewRuntimeConfig().
WithCoreFeatures(api.CoreFeaturesV2 | experimental.CoreFeaturesThreads)
WithCoreFeatures(api.CoreFeaturesV2 | experimental.CoreFeaturesThreads).
WithMemoryLimitPages(512)
rt = wazero.NewRuntimeWithConfig(ctx, cfg)
wasi_snapshot_preview1.MustInstantiate(ctx, rt)
env := vfs.ExportHostFunctions(rt.NewHostModuleBuilder("env"))