diff --git a/README.md b/README.md index 5568a29..5b55923 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,9 @@ Go module `github.com/ncruces/go-sqlite3` is a `cgo`-free [SQLite](https://sqlit It provides a [`database/sql`](https://pkg.go.dev/database/sql) compatible driver, as well as direct access to most of the [C SQLite API](https://sqlite.org/cintro.html). -It wraps a [Wasm](https://webassembly.org/) build of SQLite, and uses [wazero](https://wazero.io/) as the runtime.\ -Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ runtime dependencies. +It wraps a [Wasm](https://webassembly.org/) [build](embed/) of SQLite, +and uses [wazero](https://wazero.io/) as the runtime.\ +Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ runtime dependencies [^1]. ### Packages @@ -54,6 +55,8 @@ Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ run implements an in-memory VFS. - [`github.com/ncruces/go-sqlite3/vfs/readervfs`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/readervfs) implements a VFS for immutable databases. +- [`github.com/ncruces/go-sqlite3/vfs/adiantum`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/adiantum) + wraps a VFS to offer encryption at rest. ### Advanced features @@ -101,3 +104,6 @@ The Wasm and VFS layers are also tested by running SQLite's - [`crawshaw.io/sqlite`](https://pkg.go.dev/crawshaw.io/sqlite) - [`github.com/mattn/go-sqlite3`](https://pkg.go.dev/github.com/mattn/go-sqlite3) - [`github.com/zombiezen/go-sqlite`](https://pkg.go.dev/github.com/zombiezen/go-sqlite) + +[^1]: anything else you find in [`go.mod`](./go.mod) is either a test dependency, + or needed by one of the extensions. diff --git a/embed/exports.txt b/embed/exports.txt index 0f9fc46..d01ee7e 100644 --- a/embed/exports.txt +++ b/embed/exports.txt @@ -61,6 +61,9 @@ sqlite3_errmsg sqlite3_error_offset sqlite3_errstr sqlite3_exec +sqlite3_filename_database +sqlite3_filename_journal +sqlite3_filename_wal sqlite3_finalize sqlite3_get_autocommit sqlite3_get_auxdata diff --git a/embed/sqlite3.wasm b/embed/sqlite3.wasm index 3e99106..52b1995 100755 Binary files a/embed/sqlite3.wasm and b/embed/sqlite3.wasm differ diff --git a/go.mod b/go.mod index 2ac4362..18dd674 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,9 @@ require ( golang.org/x/sync v0.7.0 golang.org/x/sys v0.19.0 golang.org/x/text v0.14.0 + lukechampine.com/adiantum v1.0.0 ) +require github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect + retract v0.4.0 // tagged from the wrong branch diff --git a/go.sum b/go.sum index 942146f..85aa985 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,25 @@ +github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY= +github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA= github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M= github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g= github.com/psanford/httpreadat v0.1.0 h1:VleW1HS2zO7/4c7c7zNl33fO6oYACSagjJIyMIwZLUE= github.com/psanford/httpreadat v0.1.0/go.mod h1:Zg7P+TlBm3bYbyHTKv/EdtSJZn3qwbPwpfZ/I9GKCRE= github.com/tetratelabs/wazero v1.7.1 h1:QtSfd6KLc41DIMpDYlJdoMc6k7QTN246DM2+n2Y/Dx8= github.com/tetratelabs/wazero v1.7.1/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +lukechampine.com/adiantum v1.0.0 h1:xxLFgKHyno8ES1XiKLbQfU9DGiMaM2xsIJI2czgT7es= +lukechampine.com/adiantum v1.0.0/go.mod h1:kjMpBiZFjVX/FeEYcN81jyt3//7J3XjJgH9OkAXV4n0= diff --git a/internal/util/mmap.go b/internal/util/mmap.go index 94476a1..8265800 100644 --- a/internal/util/mmap.go +++ b/internal/util/mmap.go @@ -25,7 +25,7 @@ func (s *mmapState) init(ctx context.Context, enabled bool) context.Context { return ctx } -func CanMap(ctx context.Context) bool { +func CanMapFiles(ctx context.Context) bool { s := ctx.Value(moduleKey{}).(*moduleState) return s.mmapState.enabled } diff --git a/internal/util/mmap_other.go b/internal/util/mmap_other.go index 41a510f..2e89f80 100644 --- a/internal/util/mmap_other.go +++ b/internal/util/mmap_other.go @@ -10,6 +10,6 @@ func (s *mmapState) init(ctx context.Context, _ bool) context.Context { return ctx } -func CanMap(ctx context.Context) bool { +func CanMapFiles(ctx context.Context) bool { return false } diff --git a/internal/util/unwrap.go b/internal/util/unwrap.go new file mode 100644 index 0000000..1cb09a2 --- /dev/null +++ b/internal/util/unwrap.go @@ -0,0 +1,8 @@ +package util + +func Unwrap[T any](v T) T { + if u, ok := any(v).(interface{ Unwrap() T }); ok { + return u.Unwrap() + } + return v +} diff --git a/tests/db_test.go b/tests/db_test.go index 2322e7b..6f32fc2 100644 --- a/tests/db_test.go +++ b/tests/db_test.go @@ -10,6 +10,7 @@ import ( "github.com/ncruces/go-sqlite3" _ "github.com/ncruces/go-sqlite3/embed" _ "github.com/ncruces/go-sqlite3/tests/testcfg" + _ "github.com/ncruces/go-sqlite3/vfs/adiantum" _ "github.com/ncruces/go-sqlite3/vfs/memdb" ) @@ -56,10 +57,18 @@ func TestDB_utf16(t *testing.T) { testDB(t, tmp) } -func TestDB_vfs(t *testing.T) { +func TestDB_memdb(t *testing.T) { + t.Parallel() testDB(t, "file:test.db?vfs=memdb") } +func TestDB_adiantum(t *testing.T) { + t.Parallel() + testDB(t, "file:"+ + filepath.ToSlash(filepath.Join(t.TempDir(), "test.db"))+ + "?vfs=adiantum&textkey=correct+horse+battery+staple") +} + func testDB(t testing.TB, name string) { db, err := sqlite3.Open(name) if err != nil { diff --git a/tests/parallel/parallel_test.go b/tests/parallel/parallel_test.go index 97806a1..c9db08a 100644 --- a/tests/parallel/parallel_test.go +++ b/tests/parallel/parallel_test.go @@ -14,10 +14,11 @@ import ( _ "github.com/ncruces/go-sqlite3/embed" _ "github.com/ncruces/go-sqlite3/tests/testcfg" "github.com/ncruces/go-sqlite3/vfs" + _ "github.com/ncruces/go-sqlite3/vfs/adiantum" "github.com/ncruces/go-sqlite3/vfs/memdb" ) -func TestParallel(t *testing.T) { +func Test_parallel(t *testing.T) { var iter int if testing.Short() { iter = 1000 @@ -34,7 +35,7 @@ func TestParallel(t *testing.T) { testIntegrity(t, name) } -func TestWAL(t *testing.T) { +func Test_wal(t *testing.T) { if !vfs.SupportsSharedMemory { t.Skip("skipping without shared memory") } @@ -48,7 +49,7 @@ func TestWAL(t *testing.T) { testIntegrity(t, name) } -func TestMemory(t *testing.T) { +func Test_memdb(t *testing.T) { var iter int if testing.Short() { iter = 1000 @@ -61,6 +62,21 @@ func TestMemory(t *testing.T) { testIntegrity(t, name) } +func Test_adiantum(t *testing.T) { + var iter int + if testing.Short() { + iter = 1000 + } else { + iter = 5000 + } + + name := "file:" + + filepath.ToSlash(filepath.Join(t.TempDir(), "test.db")) + + "?vfs=adiantum&textkey=correct+horse+battery+staple" + testParallel(t, name, iter) + testIntegrity(t, name) +} + func TestMultiProcess(t *testing.T) { if testing.Short() { t.Skip("skipping in short mode") @@ -112,7 +128,7 @@ func TestChildProcess(t *testing.T) { testParallel(t, name, 1000) } -func BenchmarkMemory(b *testing.B) { +func Benchmark_memdb(b *testing.B) { memdb.Delete("test.db") name := "file:/test.db?vfs=memdb" testParallel(b, name, b.N) diff --git a/util/ioutil/seek.go b/util/ioutil/seek.go index 6f033e6..0e0e124 100644 --- a/util/ioutil/seek.go +++ b/util/ioutil/seek.go @@ -8,6 +8,7 @@ import ( // SeekingReaderAt implements [io.ReaderAt] // through an underlying [io.ReadSeeker]. type SeekingReaderAt struct { + // +checklocks:l r io.ReadSeeker l sync.Mutex } diff --git a/vfs/adiantum/README.md b/vfs/adiantum/README.md new file mode 100644 index 0000000..b037954 --- /dev/null +++ b/vfs/adiantum/README.md @@ -0,0 +1,29 @@ +# Go `"adiantum"` SQLite VFS + +This package wraps an SQLite VFS to offer encryption at rest. + +> [!WARNING] +> This work was not certified by a cryptographer. +> If you need vetted encryption, you should purchase the +> [SQLite Encryption Extension](https://sqlite.org/see), +> and either wrap it, or seek assistance wrapping it. + +The `"adiantum"` VFS wraps the default SQLite VFS using the +[Adiantum](https://github.com/lukechampine/adiantum) +tweakable and length-preserving encryption. + +In general, any HBSH construction can be used to wrap any VFS. + +The default Adiantum construction uses XChaCha12 for its stream cipher, +AES for its block cipher, and NH and Poly1305 for hashing. +It uses Argon2id to derive keys from plain text. + +> [!IMPORTANT] +> Adiantum is typically used for disk encryption. +> The standard threat model for disk encryption considers an adversary +> that can read multiple snapshots of a disk. +> The security property that disk encryption provides is that +> the only information such an adversary can determine is +> whether the data in a sector has or has not changed over time. + +The VFS encrypts database files, rollback and statement journals, and WAL files. \ No newline at end of file diff --git a/vfs/adiantum/adiantum.go b/vfs/adiantum/adiantum.go new file mode 100644 index 0000000..13d1bdd --- /dev/null +++ b/vfs/adiantum/adiantum.go @@ -0,0 +1,52 @@ +package adiantum + +import ( + "sync" + + "golang.org/x/crypto/argon2" + "lukechampine.com/adiantum" + "lukechampine.com/adiantum/hbsh" +) + +const pepper = "github.com/ncruces/go-sqlite3/vfs/adiantum" + +type adiantumCreator struct{} + +func (adiantumCreator) HBSH(key []byte) *hbsh.HBSH { + return adiantum.New(key) +} + +func (adiantumCreator) KDF(text string) []byte { + if key := keyCacheGet(text); key != nil { + return key[:] + } + + key := argon2.IDKey([]byte(text), []byte(pepper), 1, 64*1024, 4, 32) + keyCachePut(text, (*[32]byte)(key)) + return key +} + +const keyCacheMaxEntries = 100 + +var ( + // +checklocks:keyCacheMtx + keyCache = map[string]*[32]byte{} + keyCacheMtx sync.RWMutex +) + +func keyCacheGet(text string) *[32]byte { + keyCacheMtx.RLock() + defer keyCacheMtx.RUnlock() + return keyCache[text] +} + +func keyCachePut(text string, key *[32]byte) { + keyCacheMtx.Lock() + defer keyCacheMtx.Unlock() + if len(keyCache) >= keyCacheMaxEntries { + for k := range keyCache { + delete(keyCache, k) + } + } + keyCache[text] = key +} diff --git a/vfs/adiantum/api.go b/vfs/adiantum/api.go new file mode 100644 index 0000000..c5c7db9 --- /dev/null +++ b/vfs/adiantum/api.go @@ -0,0 +1,50 @@ +// Package adiantum wraps an SQLite VFS to offer encryption at rest. +// +// The "adiantum" [vfs.VFS] wraps the default VFS using the +// Adiantum tweakable length-preserving encryption. +// +// Importing package adiantum registers that VFS. +// +// import _ "github.com/ncruces/go-sqlite3/vfs/adiantum" +// +// To open an encrypted database you need to provide key material. +// This is done through [URI] parameters: +// +// - key: key material in binary (32 bytes) +// - hexkey: key material in hex (64 hex digits) +// - textkey: key material in text (any length) +// +// [URI]: https://sqlite.org/uri.html +package adiantum + +import ( + "github.com/ncruces/go-sqlite3/vfs" + "lukechampine.com/adiantum/hbsh" +) + +func init() { + Register("adiantum", vfs.Find(""), nil) +} + +// Register registers an encrypting VFS, wrapping a base VFS, +// and possibly using a custom HBSH cipher construction. +// To use the default Adiantum construction, set cipher to nil. +func Register(name string, base vfs.VFS, cipher HBSHCreator) { + if cipher == nil { + cipher = adiantumCreator{} + } + vfs.Register("adiantum", &hbshVFS{ + VFS: base, + hbsh: cipher, + }) +} + +// HBSHCreator creates an [hbsh.HBSH] cipher, +// given key material. +type HBSHCreator interface { + // KDF maps a secret (text) to a key of the appropriate size. + KDF(text string) (key []byte) + + // HBSH creates an HBSH cipher given an appropriate key. + HBSH(key []byte) *hbsh.HBSH +} diff --git a/vfs/adiantum/hbsh.go b/vfs/adiantum/hbsh.go new file mode 100644 index 0000000..8d40d33 --- /dev/null +++ b/vfs/adiantum/hbsh.go @@ -0,0 +1,202 @@ +package adiantum + +import ( + "encoding/binary" + "encoding/hex" + "io" + "net/url" + + "github.com/ncruces/go-sqlite3" + "github.com/ncruces/go-sqlite3/internal/util" + "github.com/ncruces/go-sqlite3/vfs" + "lukechampine.com/adiantum/hbsh" +) + +type hbshVFS struct { + vfs.VFS + hbsh HBSHCreator +} + +func (h *hbshVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) { + return h.OpenParams(name, flags, url.Values{}) +} + +func (h *hbshVFS) OpenParams(name string, flags vfs.OpenFlag, params url.Values) (file vfs.File, _ vfs.OpenFlag, err error) { + if h, ok := h.VFS.(vfs.VFSParams); ok { + file, flags, err = h.OpenParams(name, flags, params) + } else { + file, flags, err = h.Open(name, flags) + } + if err != nil || flags&(0| + vfs.OPEN_MAIN_DB| + vfs.OPEN_MAIN_JOURNAL| + vfs.OPEN_SUBJOURNAL| + vfs.OPEN_WAL) == 0 { + return file, flags, err + } + + var key []byte + if t, ok := params["key"]; ok { + key = []byte(t[0]) + } else if t, ok := params["hexkey"]; ok { + key, err = hex.DecodeString(t[0]) + } else if t, ok := params["textkey"]; ok { + key = h.hbsh.KDF(t[0]) + } + + return &hbshFile{File: file, hbsh: h.hbsh.HBSH(key)}, flags, err +} + +const ( + blockSize = 4096 + tweakSize = 8 +) + +type hbshFile struct { + vfs.File + hbsh *hbsh.HBSH + block [blockSize]byte + tweak [tweakSize]byte +} + +func (h *hbshFile) ReadAt(p []byte, off int64) (n int, err error) { + min := (off) &^ (blockSize - 1) // round down + max := (off + int64(len(p)) + blockSize - 1) &^ (blockSize - 1) // round up + + for ; min < max; min += blockSize { + // Read full block. + m, err := h.File.ReadAt(h.block[:], min) + if m != blockSize { + return n, err + } + + binary.LittleEndian.PutUint64(h.tweak[:], uint64(min)) + data := h.hbsh.Decrypt(h.block[:], h.tweak[:]) + + if off > min { + data = data[off-min:] + } + n += copy(p[n:], data) + } + + if n != len(p) { + panic(util.AssertErr()) + } + return n, nil +} + +func (h *hbshFile) WriteAt(p []byte, off int64) (n int, err error) { + min := (off) &^ (blockSize - 1) // round down + max := (off + int64(len(p)) + blockSize - 1) &^ (blockSize - 1) // round up + + for ; min < max; min += blockSize { + binary.LittleEndian.PutUint64(h.tweak[:], uint64(min)) + data := h.block[:] + + if min < off || len(p[n:]) < blockSize { + // Read full block. + m, err := h.File.ReadAt(h.block[:], min) + switch { + case m == 0 && err == io.EOF: + clear(data) + case m != blockSize: + return n, err + default: + // Partial update. + data = h.hbsh.Decrypt(h.block[:], h.tweak[:]) + if off > min { + data = data[off-min:] + } + } + } + + t := copy(data, p[n:]) + h.hbsh.Encrypt(h.block[:], h.tweak[:]) + + // Write full block. + m, err := h.File.WriteAt(h.block[:], min) + if m != blockSize { + return n, err + } + n += t + } + + if n != len(p) { + panic(util.AssertErr()) + } + return n, nil +} + +func (h *hbshFile) Truncate(size int64) error { + size = (size + blockSize - 1) &^ (blockSize - 1) // round up + return h.File.Truncate(size) +} + +func (h *hbshFile) SectorSize() int { + return max(h.File.SectorSize(), blockSize) +} + +func (h *hbshFile) DeviceCharacteristics() vfs.DeviceCharacteristic { + return h.File.DeviceCharacteristics() & (0 | + // The only safe flags are these: + vfs.IOCAP_UNDELETABLE_WHEN_OPEN | + vfs.IOCAP_IMMUTABLE | + vfs.IOCAP_BATCH_ATOMIC) +} + +// This is needed for shared memory. +func (h *hbshFile) Unwrap() vfs.File { + return h.File +} + +// Wrap optional methods. + +func (h *hbshFile) SizeHint(size int64) error { + if f, ok := h.File.(vfs.FileSizeHint); ok { + size = (size + blockSize - 1) &^ (blockSize - 1) // round up + return f.SizeHint(size) + } + return sqlite3.NOTFOUND +} + +func (h *hbshFile) HasMoved() (bool, error) { + if f, ok := h.File.(vfs.FileHasMoved); ok { + return f.HasMoved() + } + return false, sqlite3.NOTFOUND +} + +func (h *hbshFile) Overwrite() error { + if f, ok := h.File.(vfs.FileOverwrite); ok { + return f.Overwrite() + } + return sqlite3.NOTFOUND +} + +func (h *hbshFile) CommitPhaseTwo() error { + if f, ok := h.File.(vfs.FileCommitPhaseTwo); ok { + return f.CommitPhaseTwo() + } + return sqlite3.NOTFOUND +} + +func (h *hbshFile) BeginAtomicWrite() error { + if f, ok := h.File.(vfs.FileBatchAtomicWrite); ok { + return f.BeginAtomicWrite() + } + return sqlite3.NOTFOUND +} + +func (h *hbshFile) CommitAtomicWrite() error { + if f, ok := h.File.(vfs.FileBatchAtomicWrite); ok { + return f.CommitAtomicWrite() + } + return sqlite3.NOTFOUND +} + +func (h *hbshFile) RollbackAtomicWrite() error { + if f, ok := h.File.(vfs.FileBatchAtomicWrite); ok { + return f.RollbackAtomicWrite() + } + return sqlite3.NOTFOUND +} diff --git a/vfs/tests/mptest/mptest_test.go b/vfs/tests/mptest/mptest_test.go index 8f78b45..313bc9b 100644 --- a/vfs/tests/mptest/mptest_test.go +++ b/vfs/tests/mptest/mptest_test.go @@ -19,6 +19,7 @@ import ( "github.com/ncruces/go-sqlite3/internal/util" "github.com/ncruces/go-sqlite3/vfs" + _ "github.com/ncruces/go-sqlite3/vfs/adiantum" "github.com/ncruces/go-sqlite3/vfs/memdb" "github.com/tetratelabs/wazero" "github.com/tetratelabs/wazero/api" @@ -231,6 +232,49 @@ func Test_multiwrite01_wal(t *testing.T) { mod.Close(ctx) } +func Test_crash01_adiantum(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + if os.Getenv("CI") != "" { + t.Skip("skipping in CI") + } + + ctx := util.NewContext(newContext(t), true) + name := "file:" + filepath.Join(t.TempDir(), "test.db") + + "?textkey=correct+horse+battery+staple" + cfg := config(ctx).WithArgs("mptest", name, "crash01.test", + "--vfs", "adiantum") + mod, err := rt.InstantiateModule(ctx, module, cfg) + if err != nil { + t.Fatal(err) + } + mod.Close(ctx) +} + +func Test_crash01_adiantum_wal(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + if os.Getenv("CI") != "" { + t.Skip("skipping in CI") + } + if !vfs.SupportsSharedMemory { + t.Skip("skipping without shared memory") + } + + ctx := util.NewContext(newContext(t), true) + name := "file:" + filepath.Join(t.TempDir(), "test.db") + + "?textkey=correct+horse+battery+staple" + cfg := config(ctx).WithArgs("mptest", name, "crash01.test", + "--vfs", "adiantum", "--journalmode", "wal") + mod, err := rt.InstantiateModule(ctx, module, cfg) + if err != nil { + t.Fatal(err) + } + mod.Close(ctx) +} + func newContext(t *testing.T) context.Context { return context.WithValue(context.Background(), logger{}, &testWriter{T: t}) } diff --git a/vfs/tests/mptest/testdata/build.sh b/vfs/tests/mptest/testdata/build.sh index c55c6d2..1b5645f 100755 --- a/vfs/tests/mptest/testdata/build.sh +++ b/vfs/tests/mptest/testdata/build.sh @@ -20,9 +20,9 @@ WASI_SDK="$ROOT/tools/wasi-sdk-22.0/bin" -DSQLITE_DEFAULT_SYNCHRONOUS=0 \ -DSQLITE_DEFAULT_LOCKING_MODE=0 \ -DSQLITE_NO_SYNC -DSQLITE_THREADSAFE=0 \ - -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_USE_URI \ -D_WASI_EMULATED_GETPID -lwasi-emulated-getpid \ - -Wl,--export=aligned_alloc + $(awk '{print "-Wl,--export="$0}' exports.txt) "$BINARYEN/wasm-opt" -g --strip --strip-producers -c -O3 \ mptest.wasm -o mptest.tmp \ diff --git a/vfs/tests/mptest/testdata/exports.txt b/vfs/tests/mptest/testdata/exports.txt new file mode 100644 index 0000000..28b4473 --- /dev/null +++ b/vfs/tests/mptest/testdata/exports.txt @@ -0,0 +1,8 @@ +aligned_alloc +free +malloc +sqlite3_filename_database +sqlite3_filename_journal +sqlite3_filename_wal +sqlite3_uri_key +sqlite3_uri_parameter \ No newline at end of file diff --git a/vfs/tests/mptest/testdata/mptest.wasm.bz2 b/vfs/tests/mptest/testdata/mptest.wasm.bz2 index 3cac291..507e831 100644 --- a/vfs/tests/mptest/testdata/mptest.wasm.bz2 +++ b/vfs/tests/mptest/testdata/mptest.wasm.bz2 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7bf8ef8a52def8dec9b4af62b21cbea22a6cd2a702770aa9c75f445691f1abd5 -size 469700 +oid sha256:35d27c1ec2c14d82ee6757d768435dde8dd0324ed6963fb00889a44364e19071 +size 470365 diff --git a/vfs/vfs.go b/vfs/vfs.go index a63ed21..f7b175d 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -167,8 +167,10 @@ func vfsOpen(ctx context.Context, mod api.Module, pVfs, zPath, pFile uint32, fla if pOutFlags != 0 { util.WriteUint32(mod, pOutFlags, uint32(flags)) } - if pOutVFS != 0 && util.CanMap(ctx) { - util.WriteUint32(mod, pOutVFS, 1) + if pOutVFS != 0 && util.CanMapFiles(ctx) { + if _, ok := util.Unwrap(file).(fileShm); ok { + util.WriteUint32(mod, pOutVFS, 1) + } } vfsFileRegister(ctx, mod, pFile, file) return _OK @@ -361,7 +363,7 @@ func vfsShmBarrier(ctx context.Context, mod api.Module, pFile uint32) { } func vfsShmMap(ctx context.Context, mod api.Module, pFile uint32, iRegion, szRegion int32, bExtend, pp uint32) _ErrorCode { - file := vfsFileGet(ctx, mod, pFile).(fileShm) + file := util.Unwrap(vfsFileGet(ctx, mod, pFile)).(fileShm) p, err := file.shmMap(ctx, mod, iRegion, szRegion, bExtend != 0) if err != nil { return vfsErrorCode(err, _IOERR_SHMMAP) @@ -371,31 +373,45 @@ func vfsShmMap(ctx context.Context, mod api.Module, pFile uint32, iRegion, szReg } func vfsShmLock(ctx context.Context, mod api.Module, pFile uint32, offset, n int32, flags _ShmFlag) _ErrorCode { - file := vfsFileGet(ctx, mod, pFile).(fileShm) + file := util.Unwrap(vfsFileGet(ctx, mod, pFile)).(fileShm) err := file.shmLock(offset, n, flags) return vfsErrorCode(err, _IOERR_SHMLOCK) } func vfsShmUnmap(ctx context.Context, mod api.Module, pFile, bDelete uint32) _ErrorCode { - file := vfsFileGet(ctx, mod, pFile).(fileShm) + file := util.Unwrap(vfsFileGet(ctx, mod, pFile)).(fileShm) file.shmUnmap(bDelete != 0) return _OK } func vfsURIParameters(ctx context.Context, mod api.Module, zPath uint32, flags OpenFlag) url.Values { - if flags&OPEN_URI == 0 { + switch { + case flags&(OPEN_URI|OPEN_MAIN_DB) == OPEN_URI|OPEN_MAIN_DB: + // database file + case flags&(OPEN_MAIN_JOURNAL|OPEN_SUBJOURNAL|OPEN_SUPER_JOURNAL|OPEN_WAL) != 0: + // journal or WAL file + default: return nil } - uriParam := mod.ExportedFunction("sqlite3_uri_parameter") + nameDB := mod.ExportedFunction("sqlite3_filename_database") uriKey := mod.ExportedFunction("sqlite3_uri_key") - if uriParam == nil || uriKey == nil { + uriParam := mod.ExportedFunction("sqlite3_uri_parameter") + if nameDB == nil || uriKey == nil || uriParam == nil { return nil } var stack [2]uint64 var params url.Values + if flags&OPEN_MAIN_DB == 0 { + stack[0] = uint64(zPath) + if err := nameDB.CallWithStack(ctx, stack[:]); err != nil { + panic(err) + } + zPath = uint32(stack[0]) + } + for i := 0; ; i++ { stack[1] = uint64(i) stack[0] = uint64(zPath)