Adiantum encrypting VFS improvements. (#80)

Encrypt temporary files.

embed/exports.txt

@@ -51,6 +51,7 @@ sqlite3_create_collation_go
 sqlite3_create_function_go
 sqlite3_create_module_go
 sqlite3_create_window_function_go
+sqlite3_database_file_object
 sqlite3_db_config
 sqlite3_db_name
 sqlite3_db_readonly
@@ -61,9 +62,6 @@ sqlite3_errmsg
 sqlite3_error_offset
 sqlite3_errstr
 sqlite3_exec
-sqlite3_filename_database
-sqlite3_filename_journal
-sqlite3_filename_wal
 sqlite3_finalize
 sqlite3_get_autocommit
 sqlite3_get_auxdata

vfs/adiantum/README.md

@@ -16,14 +16,24 @@ In general, any HBSH construction can be used to wrap any VFS.
 
 The default Adiantum construction uses XChaCha12 for its stream cipher,
 AES for its block cipher, and NH and Poly1305 for hashing.
-It uses Argon2id to derive 256-bit keys from plain text.
+Additionally, we use Argon2id to derive 256-bit keys from plain text.
 
-The VFS encrypts database files, rollback and statement journals, and WAL files.
+The VFS encrypts all files _except_
+[super journals](https://sqlite.org/tempfiles.html#super_journal_files):
+these _never_ contain database data, only filenames,
+and padding them to the block size is problematic.
+
+Temporary files _are_ encrypted with **random** keys,
+as they _may_ contain database data.
+To avoid the overhead of encrypting temporary files,
+keep them in memory:
+
+    PRAGMA temp_store = memory;
 
 > [!IMPORTANT]
 > Adiantum is typically used for disk encryption.
 > The standard threat model for disk encryption considers an adversary
 > that can read multiple snapshots of a disk.
-> The security property that disk encryption provides is that
-> the only information such an adversary can determine is
-> whether the data in a sector has or has not changed over time.
+> The only security property that disk encryption (and this package)
+> provides is that the only information such an adversary can determine
+> is whether the data in a sector has or has not changed over time.

vfs/adiantum/adiantum.go

@@ -1,6 +1,7 @@
 package adiantum
 
 import (
+	"crypto/rand"
 	"sync"
 
 	"golang.org/x/crypto/argon2"
@@ -20,6 +21,12 @@ func (adiantumCreator) HBSH(key []byte) *hbsh.HBSH {
 }
 
 func (adiantumCreator) KDF(text string) []byte {
+	if text == "" {
+		key := make([]byte, 32)
+		n, _ := rand.Read(key)
+		return key[:n]
+	}
+
 	if key := keyCacheGet(text); key != nil {
 		return key[:]
 	}

vfs/adiantum/api.go

@@ -33,18 +33,20 @@ func Register(name string, base vfs.VFS, cipher HBSHCreator) {
 	if cipher == nil {
 		cipher = adiantumCreator{}
 	}
-	vfs.Register("adiantum", &hbshVFS{
+	vfs.Register(name, &hbshVFS{
 		VFS:  base,
 		hbsh: cipher,
 	})
 }
 
-// HBSHCreator creates an [hbsh.HBSH] cipher,
+// HBSHCreator creates an [hbsh.HBSH] cipher
 // given key material.
 type HBSHCreator interface {
-	// KDF maps a secret (text) to a key of the appropriate size.
-	KDF(text string) (key []byte)
+	// KDF derives an HBSH key from a secret.
+	// If no secret is given, a random key is generated.
+	KDF(secret string) (key []byte)
 
-	// HBSH creates an HBSH cipher given an appropriate key.
+	// HBSH creates an HBSH cipher given a key.
+	// If key is not appropriate, nil is returned.
 	HBSH(key []byte) *hbsh.HBSH
 }

vfs/adiantum/hbsh.go

@@ -18,18 +18,14 @@ type hbshVFS struct {
 }
 
 func (h *hbshVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
-	return h.OpenParams(name, flags, url.Values{})
+	return h.OpenParams(name, flags, nil)
 }
 
 func (h *hbshVFS) OpenParams(name string, flags vfs.OpenFlag, params url.Values) (file vfs.File, _ vfs.OpenFlag, err error) {
-	encrypt := flags&(0|
-		vfs.OPEN_MAIN_DB|
-		vfs.OPEN_MAIN_JOURNAL|
-		vfs.OPEN_SUBJOURNAL|
-		vfs.OPEN_WAL) != 0
-
 	var hbsh *hbsh.HBSH
-	if encrypt {
+
+	// Encrypt everything except super journals.
+	if flags&vfs.OPEN_SUPER_JOURNAL == 0 {
 		var key []byte
 		if t, ok := params["key"]; ok {
 			key = []byte(t[0])
@@ -37,6 +33,8 @@ func (h *hbshVFS) OpenParams(name string, flags vfs.OpenFlag, params url.Values)
 			key, _ = hex.DecodeString(t[0])
 		} else if t, ok := params["textkey"]; ok {
 			key = h.hbsh.KDF(t[0])
+		} else if name == "" {
+			key = h.hbsh.KDF("")
 		}
 
 		if hbsh = h.hbsh.HBSH(key); hbsh == nil {
@@ -45,7 +43,6 @@ func (h *hbshVFS) OpenParams(name string, flags vfs.OpenFlag, params url.Values)
 	}
 
 	if h, ok := h.VFS.(vfs.VFSParams); ok {
-		delete(params, "vfs")
 		delete(params, "key")
 		delete(params, "hexkey")
 		delete(params, "textkey")
@@ -53,7 +50,7 @@ func (h *hbshVFS) OpenParams(name string, flags vfs.OpenFlag, params url.Values)
 	} else {
 		file, flags, err = h.Open(name, flags)
 	}
-	if err != nil || hbsh == nil {
+	if err != nil || hbsh == nil || flags&vfs.OPEN_MEMORY != 0 {
 		return file, flags, err
 	}
 	return &hbshFile{File: file, hbsh: hbsh}, flags, err

vfs/api.go

@@ -24,6 +24,15 @@ type VFSParams interface {
 	OpenParams(name string, flags OpenFlag, params url.Values) (File, OpenFlag, error)
 }
 
+// VFSJournal extends VFS with the ability to open journals
+// that need a reference to their corresponding database files.
+//
+// https://sqlite.org/c3ref/database_file_object.html
+type VFSJournal interface {
+	VFS
+	OpenJournal(name string, flags OpenFlag, db File) (File, OpenFlag, error)
+}
+
 // A File represents an open file in the OS interface layer.
 //
 // Use sqlite3.ErrorCode or sqlite3.ExtendedErrorCode to return specific error codes to SQLite.

vfs/memdb/memdb.go

@@ -11,12 +11,22 @@ import (
 	"github.com/ncruces/go-sqlite3/vfs"
 )
 
+// Must be a multiple of 64K (the largest page size).
+const sectorSize = 65536
+
 type memVFS struct{}
 
 func (memVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
-	// Allowed file types:
+	// For simplicity, we do not support reading or writing data
+	// across "sector" boundaries.
+	//
+	// This is not a problem for most SQLite file types:
 	// - databases, which only do page aligned reads/writes;
-	// - temp journals, used by the sorter, which does the same.
+	// - temp journals, as used by the sorter, which does the same:
+	//   https://sqlite.org/src/artifact/237840?ln=409-412
+	//
+	// We refuse to open all other file types,
+	// but returning OPEN_MEMORY means SQLite won't ask us to.
 	const types = vfs.OPEN_MAIN_DB |
 		vfs.OPEN_TRANSIENT_DB |
 		vfs.OPEN_TEMP_DB |
@@ -61,9 +71,6 @@ func (memVFS) FullPathname(name string) (string, error) {
 	return name, nil
 }
 
-// Must be a multiple of 64K (the largest page size).
-const sectorSize = 65536
-
 type memDB struct {
 	// +checklocks:lockMtx
 	pending *memFile

vfs/registry.go

@@ -23,6 +23,7 @@ func Find(name string) VFS {
 }
 
 // Register registers a VFS.
+// Empty and "os" are reserved names.
 //
 // https://sqlite.org/c3ref/vfs_find.html
 func Register(name string, vfs VFS) {

vfs/tests/mptest/testdata/build.sh

@@ -16,11 +16,11 @@ WASI_SDK="$ROOT/tools/wasi-sdk-22.0/bin"
 	-fno-stack-protector -fno-stack-clash-protection \
 	-Wl,--stack-first \
 	-Wl,--import-undefined \
-	-D_HAVE_SQLITE_CONFIG_H -DHAVE_USLEEP \
+	-D_HAVE_SQLITE_CONFIG_H -DSQLITE_USE_URI \
 	-DSQLITE_DEFAULT_SYNCHRONOUS=0 \
 	-DSQLITE_DEFAULT_LOCKING_MODE=0 \
 	-DSQLITE_NO_SYNC -DSQLITE_THREADSAFE=0 \
-	-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_USE_URI \
+	-DSQLITE_OMIT_LOAD_EXTENSION -DHAVE_USLEEP \
 	-D_WASI_EMULATED_GETPID -lwasi-emulated-getpid \
 	$(awk '{print "-Wl,--export="$0}' exports.txt)
 

vfs/tests/mptest/testdata/exports.txt

@@ -1,8 +1,6 @@
 aligned_alloc
 free
 malloc
-sqlite3_filename_database
-sqlite3_filename_journal
-sqlite3_filename_wal
+sqlite3_database_file_object
 sqlite3_uri_key
 sqlite3_uri_parameter

vfs/tests/mptest/testdata/mptest.wasm.bz2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:35d27c1ec2c14d82ee6757d768435dde8dd0324ed6963fb00889a44364e19071
-size 470365
+oid sha256:afe1db5aea2a3ab996370fa85052cafaae10ab4b2f8154885da2c1d2a8503840
+size 470240

vfs/tests/speedtest1/speedtest1_test.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/ncruces/go-sqlite3/internal/util"
 	"github.com/ncruces/go-sqlite3/vfs"
+	_ "github.com/ncruces/go-sqlite3/vfs/adiantum"
 	_ "github.com/ncruces/go-sqlite3/vfs/memdb"
 )
 
@@ -38,8 +39,7 @@ func TestMain(m *testing.M) {
 	initFlags()
 
 	ctx := context.Background()
-	cfg := wazero.NewRuntimeConfig().WithMemoryLimitPages(1024)
-	rt = wazero.NewRuntimeWithConfig(ctx, cfg)
+	rt = wazero.NewRuntime(ctx)
 	wasi_snapshot_preview1.MustInstantiate(ctx, rt)
 	env := vfs.ExportHostFunctions(rt.NewHostModuleBuilder("env"))
 	_, err := env.Instantiate(ctx)
@@ -100,3 +100,22 @@ func Benchmark_speedtest1(b *testing.B) {
 	}
 	mod.Close(ctx)
 }
+
+func Benchmark_adiantum(b *testing.B) {
+	output.Reset()
+	ctx := util.NewContext(context.Background(), true)
+	name := "file:" + filepath.Join(b.TempDir(), "test.db") +
+		"?textkey=correct+horse+battery+staple"
+	args := append(options, "--vfs", "adiantum", "--size", strconv.Itoa(b.N), name)
+	cfg := wazero.NewModuleConfig().
+		WithArgs(args...).WithName("speedtest1").
+		WithStdout(&output).WithStderr(&output).
+		WithSysWalltime().WithSysNanotime().WithSysNanosleep().
+		WithOsyield(runtime.Gosched).
+		WithRandSource(rand.Reader)
+	mod, err := rt.InstantiateModule(ctx, module, cfg)
+	if err != nil {
+		b.Fatal(err)
+	}
+	mod.Close(ctx)
+}

vfs/tests/speedtest1/testdata/build.sh

@@ -16,8 +16,8 @@ WASI_SDK="$ROOT/tools/wasi-sdk-22.0/bin"
 	-fno-stack-protector -fno-stack-clash-protection \
 	-Wl,--stack-first \
 	-Wl,--import-undefined \
-	-D_HAVE_SQLITE_CONFIG_H \
-	-Wl,--export=aligned_alloc
+	-D_HAVE_SQLITE_CONFIG_H -DSQLITE_USE_URI \
+	$(awk '{print "-Wl,--export="$0}' exports.txt)
 
 "$BINARYEN/wasm-opt" -g --strip --strip-producers -c -O3 \
 	speedtest1.wasm -o speedtest1.tmp \

vfs/tests/speedtest1/testdata/exports.txt

@@ -0,0 +1,6 @@
+aligned_alloc
+free
+malloc
+sqlite3_database_file_object
+sqlite3_uri_key
+sqlite3_uri_parameter

vfs/tests/speedtest1/testdata/speedtest1.wasm.bz2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f669eede3ba9c9a104d37c3a90ecec26086a87aea54c1f650df202923d75cb9
-size 483426
+oid sha256:e4f9ed81c9497a9d8b91517416d122ae04c2c517d3e0612d869e128dcee3fa81
+size 483519

vfs/vfs.go

@@ -143,7 +143,10 @@ func vfsOpen(ctx context.Context, mod api.Module, pVfs, zPath, pFile uint32, fla
 	var err error
 	var parsed bool
 	var params url.Values
-	if pfs, ok := vfs.(VFSParams); ok {
+	if jfs, ok := vfs.(VFSJournal); ok && flags&(OPEN_WAL|OPEN_MAIN_JOURNAL) != 0 {
+		db := vfsDatabaseFileObject(ctx, mod, zPath)
+		file, flags, err = jfs.OpenJournal(path, flags, db)
+	} else if pfs, ok := vfs.(VFSParams); ok {
 		parsed = true
 		params = vfsURIParameters(ctx, mod, zPath, flags)
 		file, flags, err = pfs.OpenParams(path, flags, params)
@@ -387,30 +390,17 @@ func vfsShmUnmap(ctx context.Context, mod api.Module, pFile, bDelete uint32) _Er
 func vfsURIParameters(ctx context.Context, mod api.Module, zPath uint32, flags OpenFlag) url.Values {
 	switch {
 	case flags&(OPEN_URI|OPEN_MAIN_DB) == OPEN_URI|OPEN_MAIN_DB:
-		// database file
-	case flags&(OPEN_MAIN_JOURNAL|OPEN_SUBJOURNAL|OPEN_SUPER_JOURNAL|OPEN_WAL) != 0:
+		// database file with URI
+	case flags&(OPEN_WAL|OPEN_MAIN_JOURNAL) != 0:
 		// journal or WAL file
 	default:
 		return nil
 	}
 
-	nameDB := mod.ExportedFunction("sqlite3_filename_database")
-	uriKey := mod.ExportedFunction("sqlite3_uri_key")
-	uriParam := mod.ExportedFunction("sqlite3_uri_parameter")
-	if nameDB == nil || uriKey == nil || uriParam == nil {
-		return nil
-	}
-
 	var stack [2]uint64
 	var params url.Values
-
-	if flags&OPEN_MAIN_DB == 0 {
-		stack[0] = uint64(zPath)
-		if err := nameDB.CallWithStack(ctx, stack[:]); err != nil {
-			panic(err)
-		}
-		zPath = uint32(stack[0])
-	}
+	uriKey := mod.ExportedFunction("sqlite3_uri_key")
+	uriParam := mod.ExportedFunction("sqlite3_uri_parameter")
 
 	for i := 0; ; i++ {
 		stack[1] = uint64(i)
@@ -438,6 +428,15 @@ func vfsURIParameters(ctx context.Context, mod api.Module, zPath uint32, flags O
 	}
 }
 
+func vfsDatabaseFileObject(ctx context.Context, mod api.Module, zPath uint32) File {
+	stack := [...]uint64{uint64(zPath)}
+	fn := mod.ExportedFunction("sqlite3_database_file_object")
+	if err := fn.CallWithStack(ctx, stack[:]); err != nil {
+		panic(err)
+	}
+	return vfsFileGet(ctx, mod, uint32(stack[0]))
+}
+
 func vfsGet(mod api.Module, pVfs uint32) VFS {
 	var name string
 	if pVfs != 0 {