VFS improvements.

ext/serdes/serdes.go

@@ -2,9 +2,8 @@
 package serdes
 
 import (
-	"io"
-
 	"github.com/ncruces/go-sqlite3"
+	"github.com/ncruces/go-sqlite3/util/vfsutil"
 	"github.com/ncruces/go-sqlite3/vfs"
 )
 
@@ -14,16 +13,16 @@ func init() {
 	vfs.Register(vfsName, sliceVFS{})
 }
 
-var fileToOpen = make(chan *sliceFile, 1)
+var fileToOpen = make(chan *[]byte, 1)
 
 // Serialize backs up a database into a byte slice.
 //
 // https://sqlite.org/c3ref/serialize.html
 func Serialize(db *sqlite3.Conn, schema string) ([]byte, error) {
-	var file sliceFile
+	var file []byte
 	fileToOpen <- &file
 	err := db.Backup(schema, "file:serdes.db?vfs="+vfsName)
-	return file.data, err
+	return file, err
 }
 
 // Deserialize restores a database from a byte slice,
@@ -41,7 +40,7 @@ func Serialize(db *sqlite3.Conn, schema string) ([]byte, error) {
 // ["memdb"]: https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/memdb
 // ["reader"]: https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/readervfs
 func Deserialize(db *sqlite3.Conn, schema string, data []byte) error {
-	fileToOpen <- &sliceFile{data}
+	fileToOpen <- &data
 	return db.Restore(schema, "file:serdes.db?vfs="+vfsName)
 }
 
@@ -53,7 +52,7 @@ func (sliceVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, e
 	}
 	select {
 	case file := <-fileToOpen:
-		return file, flags | vfs.OPEN_MEMORY, nil
+		return (*vfsutil.SliceFile)(file), flags | vfs.OPEN_MEMORY, nil
 	default:
 		return nil, flags, sqlite3.MISUSE
 	}
@@ -71,70 +70,3 @@ func (sliceVFS) Access(name string, flag vfs.AccessFlag) (bool, error) {
 func (sliceVFS) FullPathname(name string) (string, error) {
 	return name, nil
 }
-
-type sliceFile struct{ data []byte }
-
-func (f *sliceFile) ReadAt(b []byte, off int64) (n int, err error) {
-	if d := f.data; off < int64(len(d)) {
-		n = copy(b, d[off:])
-	}
-	if n == 0 {
-		err = io.EOF
-	}
-	return
-}
-
-func (f *sliceFile) WriteAt(b []byte, off int64) (n int, err error) {
-	if d := f.data; off > int64(len(d)) {
-		f.data = append(d, make([]byte, off-int64(len(d)))...)
-	}
-	d := append(f.data[:off], b...)
-	if len(d) > len(f.data) {
-		f.data = d
-	}
-	return len(b), nil
-}
-
-func (f *sliceFile) Size() (int64, error) {
-	return int64(len(f.data)), nil
-}
-
-func (f *sliceFile) Truncate(size int64) error {
-	if d := f.data; size < int64(len(d)) {
-		f.data = d[:size]
-	}
-	return nil
-}
-
-func (f *sliceFile) SizeHint(size int64) error {
-	if d := f.data; size > int64(len(d)) {
-		f.data = append(d, make([]byte, size-int64(len(d)))...)
-	}
-	return nil
-}
-
-func (*sliceFile) Close() error { return nil }
-
-func (*sliceFile) Sync(flag vfs.SyncFlag) error { return nil }
-
-func (*sliceFile) Lock(lock vfs.LockLevel) error { return nil }
-
-func (*sliceFile) Unlock(lock vfs.LockLevel) error { return nil }
-
-func (*sliceFile) CheckReservedLock() (bool, error) {
-	// notest // OPEN_MEMORY
-	return false, nil
-}
-
-func (*sliceFile) SectorSize() int {
-	// notest // IOCAP_POWERSAFE_OVERWRITE
-	return 0
-}
-
-func (*sliceFile) DeviceCharacteristics() vfs.DeviceCharacteristic {
-	return vfs.IOCAP_ATOMIC |
-		vfs.IOCAP_SAFE_APPEND |
-		vfs.IOCAP_SEQUENTIAL |
-		vfs.IOCAP_POWERSAFE_OVERWRITE |
-		vfs.IOCAP_SUBPAGE_READ
-}

util/vfsutil/slice.go

@@ -0,0 +1,96 @@
+package vfsutil
+
+import (
+	"io"
+
+	"github.com/ncruces/go-sqlite3"
+	"github.com/ncruces/go-sqlite3/vfs"
+)
+
+// SliceFile implements [vfs.File] with a byte slice.
+// It is suitable for temporary files (such as [vfs.OPEN_TEMP_JOURNAL]),
+// but not concurrency safe.
+type SliceFile []byte
+
+var (
+	// Ensure these interfaces are implemented:
+	_ vfs.FileSizeHint = &SliceFile{}
+)
+
+// ReadAt implements [io.ReaderAt].
+func (f *SliceFile) ReadAt(b []byte, off int64) (n int, err error) {
+	if d := *f; off < int64(len(d)) {
+		n = copy(b, d[off:])
+	}
+	if n < len(b) {
+		err = io.EOF
+	}
+	return
+}
+
+// WriteAt implements [io.WriterAt].
+func (f *SliceFile) WriteAt(b []byte, off int64) (n int, err error) {
+	d := *f
+	if off > int64(len(d)) {
+		d = append(d, make([]byte, off-int64(len(d)))...)
+	}
+	d = append(d[:off], b...)
+	if len(d) > len(*f) {
+		*f = d
+	}
+	return len(b), nil
+}
+
+// Size implements [vfs.File].
+func (f *SliceFile) Size() (int64, error) {
+	return int64(len(*f)), nil
+}
+
+// Truncate implements [vfs.File].
+func (f *SliceFile) Truncate(size int64) error {
+	if d := *f; size < int64(len(d)) {
+		*f = d[:size]
+	}
+	return nil
+}
+
+// SizeHint implements [vfs.FileSizeHint].
+func (f *SliceFile) SizeHint(size int64) error {
+	if d := *f; size > int64(len(d)) {
+		*f = append(d, make([]byte, size-int64(len(d)))...)
+	}
+	return nil
+}
+
+// Close implements [io.Closer].
+func (*SliceFile) Close() error { return nil }
+
+// Sync implements [vfs.File].
+func (*SliceFile) Sync(flags vfs.SyncFlag) error { return nil }
+
+// Lock implements [vfs.File].
+func (*SliceFile) Lock(lock vfs.LockLevel) error { return nil }
+
+// Unlock implements [vfs.File].
+func (*SliceFile) Unlock(lock vfs.LockLevel) error { return nil }
+
+// CheckReservedLock implements [vfs.File].
+func (*SliceFile) CheckReservedLock() (bool, error) {
+	// notest // OPEN_MEMORY
+	return false, sqlite3.IOERR_CHECKRESERVEDLOCK
+}
+
+// SectorSize implements [vfs.File].
+func (*SliceFile) SectorSize() int {
+	// notest // IOCAP_POWERSAFE_OVERWRITE
+	return 0
+}
+
+// DeviceCharacteristics implements [vfs.File].
+func (*SliceFile) DeviceCharacteristics() vfs.DeviceCharacteristic {
+	return vfs.IOCAP_ATOMIC |
+		vfs.IOCAP_SEQUENTIAL |
+		vfs.IOCAP_SAFE_APPEND |
+		vfs.IOCAP_POWERSAFE_OVERWRITE |
+		vfs.IOCAP_SUBPAGE_READ
+}

vfs/adiantum/hbsh.go

@@ -160,10 +160,11 @@ func (h *hbshFile) WriteAt(p []byte, off int64) (n int, err error) {
 		if off > min || len(p[n:]) < blockSize {
 			// Partial block write: read-update-write.
 			m, err := h.File.ReadAt(h.block[:], min)
-			if m != blockSize {
-				if err != io.EOF {
-					return n, err
-				}
+			if m == blockSize {
+				data = h.hbsh.Decrypt(h.block[:], h.tweak[:])
+			} else if err != io.EOF {
+				return n, err
+			} else {
 				// Writing past the EOF.
 				// We're either appending an entirely new block,
 				// or the final block was only partially written.
@@ -171,8 +172,6 @@ func (h *hbshFile) WriteAt(p []byte, off int64) (n int, err error) {
 				// and is as good as corrupt.
 				// Either way, zero pad the file to the next block size.
 				clear(data)
-			} else {
-				data = h.hbsh.Decrypt(h.block[:], h.tweak[:])
 			}
 			if off > min {
 				data = data[off-min:]
@@ -223,16 +222,16 @@ func (h *hbshFile) SizeHint(size int64) error {
 	return vfsutil.WrapSizeHint(h.File, roundUp(size))
 }
 
+// Wrap optional methods.
+
 func (h *hbshFile) Unwrap() vfs.File {
-	return h.File
+	return h.File // notest
 }
 
 func (h *hbshFile) SharedMemory() vfs.SharedMemory {
-	return vfsutil.WrapSharedMemory(h.File)
+	return vfsutil.WrapSharedMemory(h.File) // notest
 }
 
-// Wrap optional methods.
-
 func (h *hbshFile) LockState() vfs.LockLevel {
 	return vfsutil.WrapLockState(h.File) // notest
 }

vfs/api.go

@@ -36,9 +36,9 @@ type VFSFilename interface {
 //
 // https://sqlite.org/c3ref/io_methods.html
 type File interface {
-	Close() error
-	ReadAt(p []byte, off int64) (n int, err error)
-	WriteAt(p []byte, off int64) (n int, err error)
+	io.Closer
+	io.ReaderAt
+	io.WriterAt
 	Truncate(size int64) error
 	Sync(flags SyncFlag) error
 	Size() (int64, error)

vfs/memdb/memdb.go

@@ -2,40 +2,39 @@ package memdb
 
 import (
 	"io"
+	"strings"
 	"sync"
 	"time"
 
 	"github.com/ncruces/go-sqlite3"
+	"github.com/ncruces/go-sqlite3/util/vfsutil"
 	"github.com/ncruces/go-sqlite3/vfs"
 )
 
 const sectorSize = 65536
 
-// Ensure sectorSize is a multiple of 64K (the largest page size).
-var _ [0]struct{} = [sectorSize & 65535]struct{}{}
-
 type memVFS struct{}
 
 func (memVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
 	// For simplicity, we do not support reading or writing data
 	// across "sector" boundaries.
-	//
-	// This is not a problem for most SQLite file types:
-	// - databases, which only do page aligned reads/writes;
-	// - temp journals, as used by the sorter, which does the same:
-	//   https://github.com/sqlite/sqlite/blob/b74eb0/src/vdbesort.c#L409-L412
-	//
-	// We refuse to open all other file types,
-	// but returning OPEN_MEMORY means SQLite won't ask us to.
-	const types = vfs.OPEN_MAIN_DB | vfs.OPEN_TEMP_DB |
-		vfs.OPEN_TRANSIENT_DB | vfs.OPEN_TEMP_JOURNAL
-	if flags&types == 0 {
+	// This is not a problem for SQLite database files.
+	const databases = vfs.OPEN_MAIN_DB | vfs.OPEN_TEMP_DB | vfs.OPEN_TRANSIENT_DB
+
+	// Temp journals, as used by the sorter, use SliceFile.
+	if flags&vfs.OPEN_TEMP_JOURNAL != 0 {
+		return &vfsutil.SliceFile{}, flags | vfs.OPEN_MEMORY, nil
+	}
+
+	// Refuse to open all other file types.
+	// Returning OPEN_MEMORY means SQLite won't ask us to.
+	if flags&databases == 0 {
 		// notest // OPEN_MEMORY
 		return nil, flags, sqlite3.CANTOPEN
 	}
 
 	// A shared database has a name that begins with "/".
-	shared := len(name) > 1 && name[0] == '/'
+	shared := strings.HasPrefix(name, "/")
 
 	var db *memDB
 	if shared {
@@ -158,12 +157,27 @@ func (m *memFile) WriteAt(b []byte, off int64) (n int, err error) {
 	return n, nil
 }
 
+func (m *memFile) Size() (int64, error) {
+	m.dataMtx.RLock()
+	defer m.dataMtx.RUnlock()
+	return m.size, nil
+}
+
 func (m *memFile) Truncate(size int64) error {
 	m.dataMtx.Lock()
 	defer m.dataMtx.Unlock()
 	return m.truncate(size)
 }
 
+func (m *memFile) SizeHint(size int64) error {
+	m.dataMtx.Lock()
+	defer m.dataMtx.Unlock()
+	if size > m.size {
+		return m.truncate(size)
+	}
+	return nil
+}
+
 // +checklocks:m.dataMtx
 func (m *memFile) truncate(size int64) error {
 	if size < m.size {
@@ -183,16 +197,6 @@ func (m *memFile) truncate(size int64) error {
 	return nil
 }
 
-func (m *memFile) Sync(flag vfs.SyncFlag) error {
-	return nil
-}
-
-func (m *memFile) Size() (int64, error) {
-	m.dataMtx.RLock()
-	defer m.dataMtx.RUnlock()
-	return m.size, nil
-}
-
 func (m *memFile) Lock(lock vfs.LockLevel) error {
 	if m.lock >= lock {
 		return nil
@@ -276,31 +280,24 @@ func (m *memFile) CheckReservedLock() (bool, error) {
 	return m.reserved, nil
 }
 
-func (m *memFile) SectorSize() int {
+func (m *memFile) LockState() vfs.LockLevel {
+	return m.lock
+}
+
+func (*memFile) Sync(flag vfs.SyncFlag) error { return nil }
+
+func (*memFile) SectorSize() int {
 	// notest // IOCAP_POWERSAFE_OVERWRITE
 	return sectorSize
 }
 
-func (m *memFile) DeviceCharacteristics() vfs.DeviceCharacteristic {
+func (*memFile) DeviceCharacteristics() vfs.DeviceCharacteristic {
 	return vfs.IOCAP_ATOMIC |
 		vfs.IOCAP_SEQUENTIAL |
 		vfs.IOCAP_SAFE_APPEND |
 		vfs.IOCAP_POWERSAFE_OVERWRITE
 }
 
-func (m *memFile) SizeHint(size int64) error {
-	m.dataMtx.Lock()
-	defer m.dataMtx.Unlock()
-	if size > m.size {
-		return m.truncate(size)
-	}
-	return nil
-}
-
-func (m *memFile) LockState() vfs.LockLevel {
-	return m.lock
-}
-
 func divRoundUp(a, b int64) int64 {
 	return (a + b - 1) / b
 }

vfs/readervfs/reader.go

@@ -28,7 +28,7 @@ func (readerVFS) Delete(name string, dirSync bool) error {
 
 func (readerVFS) Access(name string, flag vfs.AccessFlag) (bool, error) {
 	// notest
-	return false, nil
+	return false, sqlite3.IOERR_ACCESS
 }
 
 func (readerVFS) FullPathname(name string) (string, error) {
@@ -43,32 +43,32 @@ func (readerFile) Close() error {
 
 func (readerFile) WriteAt(b []byte, off int64) (n int, err error) {
 	// notest
-	return 0, sqlite3.READONLY
+	return 0, sqlite3.IOERR_WRITE
 }
 
 func (readerFile) Truncate(size int64) error {
 	// notest
-	return sqlite3.READONLY
+	return sqlite3.IOERR_TRUNCATE
 }
 
 func (readerFile) Sync(flag vfs.SyncFlag) error {
 	// notest
-	return nil
+	return sqlite3.IOERR_FSYNC
 }
 
 func (readerFile) Lock(lock vfs.LockLevel) error {
 	// notest
-	return nil
+	return sqlite3.IOERR_LOCK
 }
 
 func (readerFile) Unlock(lock vfs.LockLevel) error {
 	// notest
-	return nil
+	return sqlite3.IOERR_UNLOCK
 }
 
 func (readerFile) CheckReservedLock() (bool, error) {
 	// notest
-	return false, nil
+	return false, sqlite3.IOERR_CHECKRESERVEDLOCK
 }
 
 func (readerFile) SectorSize() int {

vfs/tests/speedtest1/speedtest1_test.go

@@ -35,7 +35,7 @@ func TestMain(m *testing.M) {
 	initFlags()
 
 	ctx := context.Background()
-	cfg := wazero.NewRuntimeConfig().WithMemoryLimitPages(512)
+	cfg := wazero.NewRuntimeConfig().WithMemoryLimitPages(2048)
 	rt = wazero.NewRuntimeWithConfig(ctx, cfg)
 	wasi_snapshot_preview1.MustInstantiate(ctx, rt)
 	env := vfs.ExportHostFunctions(rt.NewHostModuleBuilder("env"))

vfs/xts/xts.go

@@ -124,10 +124,10 @@ func (x *xtsFile) ReadAt(p []byte, off int64) (n int, err error) {
 			return n, err
 		}
 
-		sectorNum := uint64(min / sectorSize)
-		x.cipher.Decrypt(x.sector[:], x.sector[:], sectorNum)
-
 		data := x.sector[:]
+		sectorNum := uint64(min / sectorSize)
+		x.cipher.Decrypt(data, x.sector[:], sectorNum)
+
 		if off > min {
 			data = data[off-min:]
 		}
@@ -150,16 +150,17 @@ func (x *xtsFile) WriteAt(p []byte, off int64) (n int, err error) {
 
 	// Write one block at a time.
 	for ; min < max; min += sectorSize {
-		sectorNum := uint64(min / sectorSize)
 		data := x.sector[:]
+		sectorNum := uint64(min / sectorSize)
 
 		if off > min || len(p[n:]) < sectorSize {
 			// Partial block write: read-update-write.
 			m, err := x.File.ReadAt(x.sector[:], min)
-			if m != sectorSize {
-				if err != io.EOF {
-					return n, err
-				}
+			if m == sectorSize {
+				x.cipher.Decrypt(data, x.sector[:], sectorNum)
+			} else if err != io.EOF {
+				return n, err
+			} else {
 				// Writing past the EOF.
 				// We're either appending an entirely new block,
 				// or the final block was only partially written.
@@ -167,8 +168,6 @@ func (x *xtsFile) WriteAt(p []byte, off int64) (n int, err error) {
 				// and is as good as corrupt.
 				// Either way, zero pad the file to the next block size.
 				clear(data)
-			} else {
-				x.cipher.Decrypt(data, data, sectorNum)
 			}
 			if off > min {
 				data = data[off-min:]
@@ -219,16 +218,16 @@ func (x *xtsFile) SizeHint(size int64) error {
 	return vfsutil.WrapSizeHint(x.File, roundUp(size))
 }
 
+// Wrap optional methods.
+
 func (x *xtsFile) Unwrap() vfs.File {
-	return x.File
+	return x.File // notest
 }
 
 func (x *xtsFile) SharedMemory() vfs.SharedMemory {
-	return vfsutil.WrapSharedMemory(x.File)
+	return vfsutil.WrapSharedMemory(x.File) // notest
 }
 
-// Wrap optional methods.
-
 func (x *xtsFile) LockState() vfs.LockLevel {
 	return vfsutil.WrapLockState(x.File) // notest
 }