diff --git a/embed/exports.txt b/embed/exports.txt index d01ee7e..3b73ddb 100644 --- a/embed/exports.txt +++ b/embed/exports.txt @@ -51,6 +51,7 @@ sqlite3_create_collation_go sqlite3_create_function_go sqlite3_create_module_go sqlite3_create_window_function_go +sqlite3_database_file_object sqlite3_db_config sqlite3_db_name sqlite3_db_readonly @@ -61,9 +62,6 @@ sqlite3_errmsg sqlite3_error_offset sqlite3_errstr sqlite3_exec -sqlite3_filename_database -sqlite3_filename_journal -sqlite3_filename_wal sqlite3_finalize sqlite3_get_autocommit sqlite3_get_auxdata diff --git a/embed/sqlite3.wasm b/embed/sqlite3.wasm index 52b1995..bb3a888 100755 Binary files a/embed/sqlite3.wasm and b/embed/sqlite3.wasm differ diff --git a/vfs/adiantum/README.md b/vfs/adiantum/README.md index db26510..9b90c30 100644 --- a/vfs/adiantum/README.md +++ b/vfs/adiantum/README.md @@ -16,14 +16,24 @@ In general, any HBSH construction can be used to wrap any VFS. The default Adiantum construction uses XChaCha12 for its stream cipher, AES for its block cipher, and NH and Poly1305 for hashing. -It uses Argon2id to derive 256-bit keys from plain text. +Additionally, we use Argon2id to derive 256-bit keys from plain text. -The VFS encrypts database files, rollback and statement journals, and WAL files. +The VFS encrypts all files _except_ +[super journals](https://sqlite.org/tempfiles.html#super_journal_files): +these _never_ contain database data, only filenames, +and padding them to the block size is problematic. + +Temporary files _are_ encrypted with **random** keys, +as they _may_ contain database data. +To avoid the overhead of encrypting temporary files, +keep them in memory: + + PRAGMA temp_store = memory; > [!IMPORTANT] > Adiantum is typically used for disk encryption. > The standard threat model for disk encryption considers an adversary > that can read multiple snapshots of a disk. -> The security property that disk encryption provides is that -> the only information such an adversary can determine is -> whether the data in a sector has or has not changed over time. +> The only security property that disk encryption (and this package) +> provides is that the only information such an adversary can determine +> is whether the data in a sector has or has not changed over time. diff --git a/vfs/adiantum/adiantum.go b/vfs/adiantum/adiantum.go index 39e45bb..acc6503 100644 --- a/vfs/adiantum/adiantum.go +++ b/vfs/adiantum/adiantum.go @@ -1,6 +1,7 @@ package adiantum import ( + "crypto/rand" "sync" "golang.org/x/crypto/argon2" @@ -20,6 +21,12 @@ func (adiantumCreator) HBSH(key []byte) *hbsh.HBSH { } func (adiantumCreator) KDF(text string) []byte { + if text == "" { + key := make([]byte, 32) + n, _ := rand.Read(key) + return key[:n] + } + if key := keyCacheGet(text); key != nil { return key[:] } diff --git a/vfs/adiantum/api.go b/vfs/adiantum/api.go index c5c7db9..ecc95b7 100644 --- a/vfs/adiantum/api.go +++ b/vfs/adiantum/api.go @@ -33,18 +33,20 @@ func Register(name string, base vfs.VFS, cipher HBSHCreator) { if cipher == nil { cipher = adiantumCreator{} } - vfs.Register("adiantum", &hbshVFS{ + vfs.Register(name, &hbshVFS{ VFS: base, hbsh: cipher, }) } -// HBSHCreator creates an [hbsh.HBSH] cipher, +// HBSHCreator creates an [hbsh.HBSH] cipher // given key material. type HBSHCreator interface { - // KDF maps a secret (text) to a key of the appropriate size. - KDF(text string) (key []byte) + // KDF derives an HBSH key from a secret. + // If no secret is given, a random key is generated. + KDF(secret string) (key []byte) - // HBSH creates an HBSH cipher given an appropriate key. + // HBSH creates an HBSH cipher given a key. + // If key is not appropriate, nil is returned. HBSH(key []byte) *hbsh.HBSH } diff --git a/vfs/adiantum/hbsh.go b/vfs/adiantum/hbsh.go index 6b40e41..ec8605b 100644 --- a/vfs/adiantum/hbsh.go +++ b/vfs/adiantum/hbsh.go @@ -18,18 +18,14 @@ type hbshVFS struct { } func (h *hbshVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) { - return h.OpenParams(name, flags, url.Values{}) + return h.OpenParams(name, flags, nil) } func (h *hbshVFS) OpenParams(name string, flags vfs.OpenFlag, params url.Values) (file vfs.File, _ vfs.OpenFlag, err error) { - encrypt := flags&(0| - vfs.OPEN_MAIN_DB| - vfs.OPEN_MAIN_JOURNAL| - vfs.OPEN_SUBJOURNAL| - vfs.OPEN_WAL) != 0 - var hbsh *hbsh.HBSH - if encrypt { + + // Encrypt everything except super journals. + if flags&vfs.OPEN_SUPER_JOURNAL == 0 { var key []byte if t, ok := params["key"]; ok { key = []byte(t[0]) @@ -37,6 +33,8 @@ func (h *hbshVFS) OpenParams(name string, flags vfs.OpenFlag, params url.Values) key, _ = hex.DecodeString(t[0]) } else if t, ok := params["textkey"]; ok { key = h.hbsh.KDF(t[0]) + } else if name == "" { + key = h.hbsh.KDF("") } if hbsh = h.hbsh.HBSH(key); hbsh == nil { @@ -45,7 +43,6 @@ func (h *hbshVFS) OpenParams(name string, flags vfs.OpenFlag, params url.Values) } if h, ok := h.VFS.(vfs.VFSParams); ok { - delete(params, "vfs") delete(params, "key") delete(params, "hexkey") delete(params, "textkey") @@ -53,7 +50,7 @@ func (h *hbshVFS) OpenParams(name string, flags vfs.OpenFlag, params url.Values) } else { file, flags, err = h.Open(name, flags) } - if err != nil || hbsh == nil { + if err != nil || hbsh == nil || flags&vfs.OPEN_MEMORY != 0 { return file, flags, err } return &hbshFile{File: file, hbsh: hbsh}, flags, err diff --git a/vfs/api.go b/vfs/api.go index f191495..e0484e7 100644 --- a/vfs/api.go +++ b/vfs/api.go @@ -24,6 +24,15 @@ type VFSParams interface { OpenParams(name string, flags OpenFlag, params url.Values) (File, OpenFlag, error) } +// VFSJournal extends VFS with the ability to open journals +// that need a reference to their corresponding database files. +// +// https://sqlite.org/c3ref/database_file_object.html +type VFSJournal interface { + VFS + OpenJournal(name string, flags OpenFlag, db File) (File, OpenFlag, error) +} + // A File represents an open file in the OS interface layer. // // Use sqlite3.ErrorCode or sqlite3.ExtendedErrorCode to return specific error codes to SQLite. diff --git a/vfs/memdb/memdb.go b/vfs/memdb/memdb.go index 4d94083..57d887b 100644 --- a/vfs/memdb/memdb.go +++ b/vfs/memdb/memdb.go @@ -11,12 +11,22 @@ import ( "github.com/ncruces/go-sqlite3/vfs" ) +// Must be a multiple of 64K (the largest page size). +const sectorSize = 65536 + type memVFS struct{} func (memVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) { - // Allowed file types: + // For simplicity, we do not support reading or writing data + // across "sector" boundaries. + // + // This is not a problem for most SQLite file types: // - databases, which only do page aligned reads/writes; - // - temp journals, used by the sorter, which does the same. + // - temp journals, as used by the sorter, which does the same: + // https://sqlite.org/src/artifact/237840?ln=409-412 + // + // We refuse to open all other file types, + // but returning OPEN_MEMORY means SQLite won't ask us to. const types = vfs.OPEN_MAIN_DB | vfs.OPEN_TRANSIENT_DB | vfs.OPEN_TEMP_DB | @@ -61,9 +71,6 @@ func (memVFS) FullPathname(name string) (string, error) { return name, nil } -// Must be a multiple of 64K (the largest page size). -const sectorSize = 65536 - type memDB struct { // +checklocks:lockMtx pending *memFile diff --git a/vfs/registry.go b/vfs/registry.go index 7009444..42a2106 100644 --- a/vfs/registry.go +++ b/vfs/registry.go @@ -23,6 +23,7 @@ func Find(name string) VFS { } // Register registers a VFS. +// Empty and "os" are reserved names. // // https://sqlite.org/c3ref/vfs_find.html func Register(name string, vfs VFS) { diff --git a/vfs/tests/mptest/testdata/build.sh b/vfs/tests/mptest/testdata/build.sh index 1b5645f..c6fb46f 100755 --- a/vfs/tests/mptest/testdata/build.sh +++ b/vfs/tests/mptest/testdata/build.sh @@ -16,11 +16,11 @@ WASI_SDK="$ROOT/tools/wasi-sdk-22.0/bin" -fno-stack-protector -fno-stack-clash-protection \ -Wl,--stack-first \ -Wl,--import-undefined \ - -D_HAVE_SQLITE_CONFIG_H -DHAVE_USLEEP \ + -D_HAVE_SQLITE_CONFIG_H -DSQLITE_USE_URI \ -DSQLITE_DEFAULT_SYNCHRONOUS=0 \ -DSQLITE_DEFAULT_LOCKING_MODE=0 \ -DSQLITE_NO_SYNC -DSQLITE_THREADSAFE=0 \ - -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_USE_URI \ + -DSQLITE_OMIT_LOAD_EXTENSION -DHAVE_USLEEP \ -D_WASI_EMULATED_GETPID -lwasi-emulated-getpid \ $(awk '{print "-Wl,--export="$0}' exports.txt) diff --git a/vfs/tests/mptest/testdata/exports.txt b/vfs/tests/mptest/testdata/exports.txt index 28b4473..8dc0c4c 100644 --- a/vfs/tests/mptest/testdata/exports.txt +++ b/vfs/tests/mptest/testdata/exports.txt @@ -1,8 +1,6 @@ aligned_alloc free malloc -sqlite3_filename_database -sqlite3_filename_journal -sqlite3_filename_wal +sqlite3_database_file_object sqlite3_uri_key sqlite3_uri_parameter \ No newline at end of file diff --git a/vfs/tests/mptest/testdata/mptest.wasm.bz2 b/vfs/tests/mptest/testdata/mptest.wasm.bz2 index 507e831..85a069f 100644 --- a/vfs/tests/mptest/testdata/mptest.wasm.bz2 +++ b/vfs/tests/mptest/testdata/mptest.wasm.bz2 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:35d27c1ec2c14d82ee6757d768435dde8dd0324ed6963fb00889a44364e19071 -size 470365 +oid sha256:afe1db5aea2a3ab996370fa85052cafaae10ab4b2f8154885da2c1d2a8503840 +size 470240 diff --git a/vfs/tests/speedtest1/speedtest1_test.go b/vfs/tests/speedtest1/speedtest1_test.go index 30fca3b..5e4ab4e 100644 --- a/vfs/tests/speedtest1/speedtest1_test.go +++ b/vfs/tests/speedtest1/speedtest1_test.go @@ -21,6 +21,7 @@ import ( "github.com/ncruces/go-sqlite3/internal/util" "github.com/ncruces/go-sqlite3/vfs" + _ "github.com/ncruces/go-sqlite3/vfs/adiantum" _ "github.com/ncruces/go-sqlite3/vfs/memdb" ) @@ -38,8 +39,7 @@ func TestMain(m *testing.M) { initFlags() ctx := context.Background() - cfg := wazero.NewRuntimeConfig().WithMemoryLimitPages(1024) - rt = wazero.NewRuntimeWithConfig(ctx, cfg) + rt = wazero.NewRuntime(ctx) wasi_snapshot_preview1.MustInstantiate(ctx, rt) env := vfs.ExportHostFunctions(rt.NewHostModuleBuilder("env")) _, err := env.Instantiate(ctx) @@ -100,3 +100,22 @@ func Benchmark_speedtest1(b *testing.B) { } mod.Close(ctx) } + +func Benchmark_adiantum(b *testing.B) { + output.Reset() + ctx := util.NewContext(context.Background(), true) + name := "file:" + filepath.Join(b.TempDir(), "test.db") + + "?textkey=correct+horse+battery+staple" + args := append(options, "--vfs", "adiantum", "--size", strconv.Itoa(b.N), name) + cfg := wazero.NewModuleConfig(). + WithArgs(args...).WithName("speedtest1"). + WithStdout(&output).WithStderr(&output). + WithSysWalltime().WithSysNanotime().WithSysNanosleep(). + WithOsyield(runtime.Gosched). + WithRandSource(rand.Reader) + mod, err := rt.InstantiateModule(ctx, module, cfg) + if err != nil { + b.Fatal(err) + } + mod.Close(ctx) +} diff --git a/vfs/tests/speedtest1/testdata/build.sh b/vfs/tests/speedtest1/testdata/build.sh index 9a5c421..875daf6 100755 --- a/vfs/tests/speedtest1/testdata/build.sh +++ b/vfs/tests/speedtest1/testdata/build.sh @@ -16,8 +16,8 @@ WASI_SDK="$ROOT/tools/wasi-sdk-22.0/bin" -fno-stack-protector -fno-stack-clash-protection \ -Wl,--stack-first \ -Wl,--import-undefined \ - -D_HAVE_SQLITE_CONFIG_H \ - -Wl,--export=aligned_alloc + -D_HAVE_SQLITE_CONFIG_H -DSQLITE_USE_URI \ + $(awk '{print "-Wl,--export="$0}' exports.txt) "$BINARYEN/wasm-opt" -g --strip --strip-producers -c -O3 \ speedtest1.wasm -o speedtest1.tmp \ diff --git a/vfs/tests/speedtest1/testdata/exports.txt b/vfs/tests/speedtest1/testdata/exports.txt new file mode 100644 index 0000000..8dc0c4c --- /dev/null +++ b/vfs/tests/speedtest1/testdata/exports.txt @@ -0,0 +1,6 @@ +aligned_alloc +free +malloc +sqlite3_database_file_object +sqlite3_uri_key +sqlite3_uri_parameter \ No newline at end of file diff --git a/vfs/tests/speedtest1/testdata/speedtest1.wasm.bz2 b/vfs/tests/speedtest1/testdata/speedtest1.wasm.bz2 index 2886a97..d953c0c 100644 --- a/vfs/tests/speedtest1/testdata/speedtest1.wasm.bz2 +++ b/vfs/tests/speedtest1/testdata/speedtest1.wasm.bz2 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7f669eede3ba9c9a104d37c3a90ecec26086a87aea54c1f650df202923d75cb9 -size 483426 +oid sha256:e4f9ed81c9497a9d8b91517416d122ae04c2c517d3e0612d869e128dcee3fa81 +size 483519 diff --git a/vfs/vfs.go b/vfs/vfs.go index f7b175d..9f5e7b8 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -143,7 +143,10 @@ func vfsOpen(ctx context.Context, mod api.Module, pVfs, zPath, pFile uint32, fla var err error var parsed bool var params url.Values - if pfs, ok := vfs.(VFSParams); ok { + if jfs, ok := vfs.(VFSJournal); ok && flags&(OPEN_WAL|OPEN_MAIN_JOURNAL) != 0 { + db := vfsDatabaseFileObject(ctx, mod, zPath) + file, flags, err = jfs.OpenJournal(path, flags, db) + } else if pfs, ok := vfs.(VFSParams); ok { parsed = true params = vfsURIParameters(ctx, mod, zPath, flags) file, flags, err = pfs.OpenParams(path, flags, params) @@ -387,30 +390,17 @@ func vfsShmUnmap(ctx context.Context, mod api.Module, pFile, bDelete uint32) _Er func vfsURIParameters(ctx context.Context, mod api.Module, zPath uint32, flags OpenFlag) url.Values { switch { case flags&(OPEN_URI|OPEN_MAIN_DB) == OPEN_URI|OPEN_MAIN_DB: - // database file - case flags&(OPEN_MAIN_JOURNAL|OPEN_SUBJOURNAL|OPEN_SUPER_JOURNAL|OPEN_WAL) != 0: + // database file with URI + case flags&(OPEN_WAL|OPEN_MAIN_JOURNAL) != 0: // journal or WAL file default: return nil } - nameDB := mod.ExportedFunction("sqlite3_filename_database") - uriKey := mod.ExportedFunction("sqlite3_uri_key") - uriParam := mod.ExportedFunction("sqlite3_uri_parameter") - if nameDB == nil || uriKey == nil || uriParam == nil { - return nil - } - var stack [2]uint64 var params url.Values - - if flags&OPEN_MAIN_DB == 0 { - stack[0] = uint64(zPath) - if err := nameDB.CallWithStack(ctx, stack[:]); err != nil { - panic(err) - } - zPath = uint32(stack[0]) - } + uriKey := mod.ExportedFunction("sqlite3_uri_key") + uriParam := mod.ExportedFunction("sqlite3_uri_parameter") for i := 0; ; i++ { stack[1] = uint64(i) @@ -438,6 +428,15 @@ func vfsURIParameters(ctx context.Context, mod api.Module, zPath uint32, flags O } } +func vfsDatabaseFileObject(ctx context.Context, mod api.Module, zPath uint32) File { + stack := [...]uint64{uint64(zPath)} + fn := mod.ExportedFunction("sqlite3_database_file_object") + if err := fn.CallWithStack(ctx, stack[:]); err != nil { + panic(err) + } + return vfsFileGet(ctx, mod, uint32(stack[0])) +} + func vfsGet(mod api.Module, pVfs uint32) VFS { var name string if pVfs != 0 {