From 0399f10c0640cfb7a335762dbe961c30ba269130 Mon Sep 17 00:00:00 2001 From: Nuno Cruces Date: Tue, 22 Jul 2025 22:41:29 +0100 Subject: [PATCH] VFS improvements. --- ext/serdes/serdes.go | 80 ++------------------- util/vfsutil/slice.go | 96 +++++++++++++++++++++++++ vfs/adiantum/hbsh.go | 19 +++-- vfs/api.go | 6 +- vfs/memdb/memdb.go | 77 ++++++++++---------- vfs/readervfs/reader.go | 14 ++-- vfs/tests/speedtest1/speedtest1_test.go | 2 +- vfs/xts/xts.go | 27 ++++--- 8 files changed, 172 insertions(+), 149 deletions(-) create mode 100644 util/vfsutil/slice.go diff --git a/ext/serdes/serdes.go b/ext/serdes/serdes.go index a1bcd87..378a1e2 100644 --- a/ext/serdes/serdes.go +++ b/ext/serdes/serdes.go @@ -2,9 +2,8 @@ package serdes import ( - "io" - "github.com/ncruces/go-sqlite3" + "github.com/ncruces/go-sqlite3/util/vfsutil" "github.com/ncruces/go-sqlite3/vfs" ) @@ -14,16 +13,16 @@ func init() { vfs.Register(vfsName, sliceVFS{}) } -var fileToOpen = make(chan *sliceFile, 1) +var fileToOpen = make(chan *[]byte, 1) // Serialize backs up a database into a byte slice. // // https://sqlite.org/c3ref/serialize.html func Serialize(db *sqlite3.Conn, schema string) ([]byte, error) { - var file sliceFile + var file []byte fileToOpen <- &file err := db.Backup(schema, "file:serdes.db?vfs="+vfsName) - return file.data, err + return file, err } // Deserialize restores a database from a byte slice, @@ -41,7 +40,7 @@ func Serialize(db *sqlite3.Conn, schema string) ([]byte, error) { // ["memdb"]: https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/memdb // ["reader"]: https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/readervfs func Deserialize(db *sqlite3.Conn, schema string, data []byte) error { - fileToOpen <- &sliceFile{data} + fileToOpen <- &data return db.Restore(schema, "file:serdes.db?vfs="+vfsName) } @@ -53,7 +52,7 @@ func (sliceVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, e } select { case file := <-fileToOpen: - return file, flags | vfs.OPEN_MEMORY, nil + return (*vfsutil.SliceFile)(file), flags | vfs.OPEN_MEMORY, nil default: return nil, flags, sqlite3.MISUSE } @@ -71,70 +70,3 @@ func (sliceVFS) Access(name string, flag vfs.AccessFlag) (bool, error) { func (sliceVFS) FullPathname(name string) (string, error) { return name, nil } - -type sliceFile struct{ data []byte } - -func (f *sliceFile) ReadAt(b []byte, off int64) (n int, err error) { - if d := f.data; off < int64(len(d)) { - n = copy(b, d[off:]) - } - if n == 0 { - err = io.EOF - } - return -} - -func (f *sliceFile) WriteAt(b []byte, off int64) (n int, err error) { - if d := f.data; off > int64(len(d)) { - f.data = append(d, make([]byte, off-int64(len(d)))...) - } - d := append(f.data[:off], b...) - if len(d) > len(f.data) { - f.data = d - } - return len(b), nil -} - -func (f *sliceFile) Size() (int64, error) { - return int64(len(f.data)), nil -} - -func (f *sliceFile) Truncate(size int64) error { - if d := f.data; size < int64(len(d)) { - f.data = d[:size] - } - return nil -} - -func (f *sliceFile) SizeHint(size int64) error { - if d := f.data; size > int64(len(d)) { - f.data = append(d, make([]byte, size-int64(len(d)))...) - } - return nil -} - -func (*sliceFile) Close() error { return nil } - -func (*sliceFile) Sync(flag vfs.SyncFlag) error { return nil } - -func (*sliceFile) Lock(lock vfs.LockLevel) error { return nil } - -func (*sliceFile) Unlock(lock vfs.LockLevel) error { return nil } - -func (*sliceFile) CheckReservedLock() (bool, error) { - // notest // OPEN_MEMORY - return false, nil -} - -func (*sliceFile) SectorSize() int { - // notest // IOCAP_POWERSAFE_OVERWRITE - return 0 -} - -func (*sliceFile) DeviceCharacteristics() vfs.DeviceCharacteristic { - return vfs.IOCAP_ATOMIC | - vfs.IOCAP_SAFE_APPEND | - vfs.IOCAP_SEQUENTIAL | - vfs.IOCAP_POWERSAFE_OVERWRITE | - vfs.IOCAP_SUBPAGE_READ -} diff --git a/util/vfsutil/slice.go b/util/vfsutil/slice.go new file mode 100644 index 0000000..4a1f547 --- /dev/null +++ b/util/vfsutil/slice.go @@ -0,0 +1,96 @@ +package vfsutil + +import ( + "io" + + "github.com/ncruces/go-sqlite3" + "github.com/ncruces/go-sqlite3/vfs" +) + +// SliceFile implements [vfs.File] with a byte slice. +// It is suitable for temporary files (such as [vfs.OPEN_TEMP_JOURNAL]), +// but not concurrency safe. +type SliceFile []byte + +var ( + // Ensure these interfaces are implemented: + _ vfs.FileSizeHint = &SliceFile{} +) + +// ReadAt implements [io.ReaderAt]. +func (f *SliceFile) ReadAt(b []byte, off int64) (n int, err error) { + if d := *f; off < int64(len(d)) { + n = copy(b, d[off:]) + } + if n < len(b) { + err = io.EOF + } + return +} + +// WriteAt implements [io.WriterAt]. +func (f *SliceFile) WriteAt(b []byte, off int64) (n int, err error) { + d := *f + if off > int64(len(d)) { + d = append(d, make([]byte, off-int64(len(d)))...) + } + d = append(d[:off], b...) + if len(d) > len(*f) { + *f = d + } + return len(b), nil +} + +// Size implements [vfs.File]. +func (f *SliceFile) Size() (int64, error) { + return int64(len(*f)), nil +} + +// Truncate implements [vfs.File]. +func (f *SliceFile) Truncate(size int64) error { + if d := *f; size < int64(len(d)) { + *f = d[:size] + } + return nil +} + +// SizeHint implements [vfs.FileSizeHint]. +func (f *SliceFile) SizeHint(size int64) error { + if d := *f; size > int64(len(d)) { + *f = append(d, make([]byte, size-int64(len(d)))...) + } + return nil +} + +// Close implements [io.Closer]. +func (*SliceFile) Close() error { return nil } + +// Sync implements [vfs.File]. +func (*SliceFile) Sync(flags vfs.SyncFlag) error { return nil } + +// Lock implements [vfs.File]. +func (*SliceFile) Lock(lock vfs.LockLevel) error { return nil } + +// Unlock implements [vfs.File]. +func (*SliceFile) Unlock(lock vfs.LockLevel) error { return nil } + +// CheckReservedLock implements [vfs.File]. +func (*SliceFile) CheckReservedLock() (bool, error) { + // notest // OPEN_MEMORY + return false, sqlite3.IOERR_CHECKRESERVEDLOCK +} + +// SectorSize implements [vfs.File]. +func (*SliceFile) SectorSize() int { + // notest // IOCAP_POWERSAFE_OVERWRITE + return 0 +} + +// DeviceCharacteristics implements [vfs.File]. +func (*SliceFile) DeviceCharacteristics() vfs.DeviceCharacteristic { + return vfs.IOCAP_ATOMIC | + vfs.IOCAP_SEQUENTIAL | + vfs.IOCAP_SAFE_APPEND | + vfs.IOCAP_POWERSAFE_OVERWRITE | + vfs.IOCAP_SUBPAGE_READ +} diff --git a/vfs/adiantum/hbsh.go b/vfs/adiantum/hbsh.go index b5e96be..d8c7801 100644 --- a/vfs/adiantum/hbsh.go +++ b/vfs/adiantum/hbsh.go @@ -160,10 +160,11 @@ func (h *hbshFile) WriteAt(p []byte, off int64) (n int, err error) { if off > min || len(p[n:]) < blockSize { // Partial block write: read-update-write. m, err := h.File.ReadAt(h.block[:], min) - if m != blockSize { - if err != io.EOF { - return n, err - } + if m == blockSize { + data = h.hbsh.Decrypt(h.block[:], h.tweak[:]) + } else if err != io.EOF { + return n, err + } else { // Writing past the EOF. // We're either appending an entirely new block, // or the final block was only partially written. @@ -171,8 +172,6 @@ func (h *hbshFile) WriteAt(p []byte, off int64) (n int, err error) { // and is as good as corrupt. // Either way, zero pad the file to the next block size. clear(data) - } else { - data = h.hbsh.Decrypt(h.block[:], h.tweak[:]) } if off > min { data = data[off-min:] @@ -223,16 +222,16 @@ func (h *hbshFile) SizeHint(size int64) error { return vfsutil.WrapSizeHint(h.File, roundUp(size)) } +// Wrap optional methods. + func (h *hbshFile) Unwrap() vfs.File { - return h.File + return h.File // notest } func (h *hbshFile) SharedMemory() vfs.SharedMemory { - return vfsutil.WrapSharedMemory(h.File) + return vfsutil.WrapSharedMemory(h.File) // notest } -// Wrap optional methods. - func (h *hbshFile) LockState() vfs.LockLevel { return vfsutil.WrapLockState(h.File) // notest } diff --git a/vfs/api.go b/vfs/api.go index d5bb3a7..a0d36b2 100644 --- a/vfs/api.go +++ b/vfs/api.go @@ -36,9 +36,9 @@ type VFSFilename interface { // // https://sqlite.org/c3ref/io_methods.html type File interface { - Close() error - ReadAt(p []byte, off int64) (n int, err error) - WriteAt(p []byte, off int64) (n int, err error) + io.Closer + io.ReaderAt + io.WriterAt Truncate(size int64) error Sync(flags SyncFlag) error Size() (int64, error) diff --git a/vfs/memdb/memdb.go b/vfs/memdb/memdb.go index 5eec5a5..e304f6d 100644 --- a/vfs/memdb/memdb.go +++ b/vfs/memdb/memdb.go @@ -2,40 +2,39 @@ package memdb import ( "io" + "strings" "sync" "time" "github.com/ncruces/go-sqlite3" + "github.com/ncruces/go-sqlite3/util/vfsutil" "github.com/ncruces/go-sqlite3/vfs" ) const sectorSize = 65536 -// Ensure sectorSize is a multiple of 64K (the largest page size). -var _ [0]struct{} = [sectorSize & 65535]struct{}{} - type memVFS struct{} func (memVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) { // For simplicity, we do not support reading or writing data // across "sector" boundaries. - // - // This is not a problem for most SQLite file types: - // - databases, which only do page aligned reads/writes; - // - temp journals, as used by the sorter, which does the same: - // https://github.com/sqlite/sqlite/blob/b74eb0/src/vdbesort.c#L409-L412 - // - // We refuse to open all other file types, - // but returning OPEN_MEMORY means SQLite won't ask us to. - const types = vfs.OPEN_MAIN_DB | vfs.OPEN_TEMP_DB | - vfs.OPEN_TRANSIENT_DB | vfs.OPEN_TEMP_JOURNAL - if flags&types == 0 { + // This is not a problem for SQLite database files. + const databases = vfs.OPEN_MAIN_DB | vfs.OPEN_TEMP_DB | vfs.OPEN_TRANSIENT_DB + + // Temp journals, as used by the sorter, use SliceFile. + if flags&vfs.OPEN_TEMP_JOURNAL != 0 { + return &vfsutil.SliceFile{}, flags | vfs.OPEN_MEMORY, nil + } + + // Refuse to open all other file types. + // Returning OPEN_MEMORY means SQLite won't ask us to. + if flags&databases == 0 { // notest // OPEN_MEMORY return nil, flags, sqlite3.CANTOPEN } // A shared database has a name that begins with "/". - shared := len(name) > 1 && name[0] == '/' + shared := strings.HasPrefix(name, "/") var db *memDB if shared { @@ -158,12 +157,27 @@ func (m *memFile) WriteAt(b []byte, off int64) (n int, err error) { return n, nil } +func (m *memFile) Size() (int64, error) { + m.dataMtx.RLock() + defer m.dataMtx.RUnlock() + return m.size, nil +} + func (m *memFile) Truncate(size int64) error { m.dataMtx.Lock() defer m.dataMtx.Unlock() return m.truncate(size) } +func (m *memFile) SizeHint(size int64) error { + m.dataMtx.Lock() + defer m.dataMtx.Unlock() + if size > m.size { + return m.truncate(size) + } + return nil +} + // +checklocks:m.dataMtx func (m *memFile) truncate(size int64) error { if size < m.size { @@ -183,16 +197,6 @@ func (m *memFile) truncate(size int64) error { return nil } -func (m *memFile) Sync(flag vfs.SyncFlag) error { - return nil -} - -func (m *memFile) Size() (int64, error) { - m.dataMtx.RLock() - defer m.dataMtx.RUnlock() - return m.size, nil -} - func (m *memFile) Lock(lock vfs.LockLevel) error { if m.lock >= lock { return nil @@ -276,31 +280,24 @@ func (m *memFile) CheckReservedLock() (bool, error) { return m.reserved, nil } -func (m *memFile) SectorSize() int { +func (m *memFile) LockState() vfs.LockLevel { + return m.lock +} + +func (*memFile) Sync(flag vfs.SyncFlag) error { return nil } + +func (*memFile) SectorSize() int { // notest // IOCAP_POWERSAFE_OVERWRITE return sectorSize } -func (m *memFile) DeviceCharacteristics() vfs.DeviceCharacteristic { +func (*memFile) DeviceCharacteristics() vfs.DeviceCharacteristic { return vfs.IOCAP_ATOMIC | vfs.IOCAP_SEQUENTIAL | vfs.IOCAP_SAFE_APPEND | vfs.IOCAP_POWERSAFE_OVERWRITE } -func (m *memFile) SizeHint(size int64) error { - m.dataMtx.Lock() - defer m.dataMtx.Unlock() - if size > m.size { - return m.truncate(size) - } - return nil -} - -func (m *memFile) LockState() vfs.LockLevel { - return m.lock -} - func divRoundUp(a, b int64) int64 { return (a + b - 1) / b } diff --git a/vfs/readervfs/reader.go b/vfs/readervfs/reader.go index 34ed5e8..043951d 100644 --- a/vfs/readervfs/reader.go +++ b/vfs/readervfs/reader.go @@ -28,7 +28,7 @@ func (readerVFS) Delete(name string, dirSync bool) error { func (readerVFS) Access(name string, flag vfs.AccessFlag) (bool, error) { // notest - return false, nil + return false, sqlite3.IOERR_ACCESS } func (readerVFS) FullPathname(name string) (string, error) { @@ -43,32 +43,32 @@ func (readerFile) Close() error { func (readerFile) WriteAt(b []byte, off int64) (n int, err error) { // notest - return 0, sqlite3.READONLY + return 0, sqlite3.IOERR_WRITE } func (readerFile) Truncate(size int64) error { // notest - return sqlite3.READONLY + return sqlite3.IOERR_TRUNCATE } func (readerFile) Sync(flag vfs.SyncFlag) error { // notest - return nil + return sqlite3.IOERR_FSYNC } func (readerFile) Lock(lock vfs.LockLevel) error { // notest - return nil + return sqlite3.IOERR_LOCK } func (readerFile) Unlock(lock vfs.LockLevel) error { // notest - return nil + return sqlite3.IOERR_UNLOCK } func (readerFile) CheckReservedLock() (bool, error) { // notest - return false, nil + return false, sqlite3.IOERR_CHECKRESERVEDLOCK } func (readerFile) SectorSize() int { diff --git a/vfs/tests/speedtest1/speedtest1_test.go b/vfs/tests/speedtest1/speedtest1_test.go index f733aeb..9fafb2c 100644 --- a/vfs/tests/speedtest1/speedtest1_test.go +++ b/vfs/tests/speedtest1/speedtest1_test.go @@ -35,7 +35,7 @@ func TestMain(m *testing.M) { initFlags() ctx := context.Background() - cfg := wazero.NewRuntimeConfig().WithMemoryLimitPages(512) + cfg := wazero.NewRuntimeConfig().WithMemoryLimitPages(2048) rt = wazero.NewRuntimeWithConfig(ctx, cfg) wasi_snapshot_preview1.MustInstantiate(ctx, rt) env := vfs.ExportHostFunctions(rt.NewHostModuleBuilder("env")) diff --git a/vfs/xts/xts.go b/vfs/xts/xts.go index 6a59e9f..643df28 100644 --- a/vfs/xts/xts.go +++ b/vfs/xts/xts.go @@ -124,10 +124,10 @@ func (x *xtsFile) ReadAt(p []byte, off int64) (n int, err error) { return n, err } - sectorNum := uint64(min / sectorSize) - x.cipher.Decrypt(x.sector[:], x.sector[:], sectorNum) - data := x.sector[:] + sectorNum := uint64(min / sectorSize) + x.cipher.Decrypt(data, x.sector[:], sectorNum) + if off > min { data = data[off-min:] } @@ -150,16 +150,17 @@ func (x *xtsFile) WriteAt(p []byte, off int64) (n int, err error) { // Write one block at a time. for ; min < max; min += sectorSize { - sectorNum := uint64(min / sectorSize) data := x.sector[:] + sectorNum := uint64(min / sectorSize) if off > min || len(p[n:]) < sectorSize { // Partial block write: read-update-write. m, err := x.File.ReadAt(x.sector[:], min) - if m != sectorSize { - if err != io.EOF { - return n, err - } + if m == sectorSize { + x.cipher.Decrypt(data, x.sector[:], sectorNum) + } else if err != io.EOF { + return n, err + } else { // Writing past the EOF. // We're either appending an entirely new block, // or the final block was only partially written. @@ -167,8 +168,6 @@ func (x *xtsFile) WriteAt(p []byte, off int64) (n int, err error) { // and is as good as corrupt. // Either way, zero pad the file to the next block size. clear(data) - } else { - x.cipher.Decrypt(data, data, sectorNum) } if off > min { data = data[off-min:] @@ -219,16 +218,16 @@ func (x *xtsFile) SizeHint(size int64) error { return vfsutil.WrapSizeHint(x.File, roundUp(size)) } +// Wrap optional methods. + func (x *xtsFile) Unwrap() vfs.File { - return x.File + return x.File // notest } func (x *xtsFile) SharedMemory() vfs.SharedMemory { - return vfsutil.WrapSharedMemory(x.File) + return vfsutil.WrapSharedMemory(x.File) // notest } -// Wrap optional methods. - func (x *xtsFile) LockState() vfs.LockLevel { return vfsutil.WrapLockState(x.File) // notest }