From beba988824afe60cb418676a1d9220f482010306 Mon Sep 17 00:00:00 2001 From: Nuno Cruces Date: Sun, 28 Apr 2024 10:33:39 +0100 Subject: [PATCH] Multiple fixes. --- driver/driver.go | 2 +- internal/util/module.go | 2 +- tests/conn_test.go | 2 +- vfs/adiantum/hbsh.go | 32 ++++++++++++++++---------------- vfs/api.go | 4 +++- vfs/file.go | 2 +- vfs/memdb/memdb.go | 7 +++---- vfs/shm.go | 27 +++++++++++++++------------ vfs/shm_other.go | 15 +++++++-------- vfs/vfs.go | 22 ++++++++++------------ 10 files changed, 58 insertions(+), 57 deletions(-) diff --git a/driver/driver.go b/driver/driver.go index 377936c..b496f76 100644 --- a/driver/driver.go +++ b/driver/driver.go @@ -327,7 +327,7 @@ func (c *conn) ExecContext(ctx context.Context, query string, args []driver.Name return newResult(c.Conn), nil } -func (*conn) CheckNamedValue(arg *driver.NamedValue) error { +func (c *conn) CheckNamedValue(arg *driver.NamedValue) error { return nil } diff --git a/internal/util/module.go b/internal/util/module.go index f5ee4bd..20b17b2 100644 --- a/internal/util/module.go +++ b/internal/util/module.go @@ -8,8 +8,8 @@ import ( type moduleKey struct{} type moduleState struct { - handleState mmapState + handleState } func NewContext(ctx context.Context) context.Context { diff --git a/tests/conn_test.go b/tests/conn_test.go index 61d9361..819a2fb 100644 --- a/tests/conn_test.go +++ b/tests/conn_test.go @@ -139,7 +139,7 @@ func TestConn_SetInterrupt(t *testing.T) { SELECT 0, 1 UNION ALL SELECT next, curr + next FROM fibonacci - LIMIT 1e6 + LIMIT 1e7 ) SELECT min(curr) FROM fibonacci `) diff --git a/vfs/adiantum/hbsh.go b/vfs/adiantum/hbsh.go index 7dcdef0..41a1fac 100644 --- a/vfs/adiantum/hbsh.go +++ b/vfs/adiantum/hbsh.go @@ -51,16 +51,16 @@ func (h *hbshVFS) OpenFilename(name *vfs.Filename, flags vfs.OpenFlag) (file vfs } const ( - blockSize = 4096 tweakSize = 8 + blockSize = 4096 ) type hbshFile struct { vfs.File hbsh *hbsh.HBSH reset HBSHCreator - block [blockSize]byte tweak [tweakSize]byte + block [blockSize]byte } func (h *hbshFile) Pragma(name string, value string) (string, error) { @@ -95,8 +95,8 @@ func (h *hbshFile) ReadAt(p []byte, off int64) (n int, err error) { return 0, sqlite3.CANTOPEN } - min := (off) &^ (blockSize - 1) // round down - max := (off + int64(len(p)) + blockSize - 1) &^ (blockSize - 1) // round up + min := (off) &^ (blockSize - 1) // round down + max := (off + int64(len(p)) + (blockSize - 1)) &^ (blockSize - 1) // round up // Read one block at a time. for ; min < max; min += blockSize { @@ -125,8 +125,8 @@ func (h *hbshFile) WriteAt(p []byte, off int64) (n int, err error) { return 0, sqlite3.READONLY } - min := (off) &^ (blockSize - 1) // round down - max := (off + int64(len(p)) + blockSize - 1) &^ (blockSize - 1) // round up + min := (off) &^ (blockSize - 1) // round down + max := (off + int64(len(p)) + (blockSize - 1)) &^ (blockSize - 1) // round up // Write one block at a time. for ; min < max; min += blockSize { @@ -143,13 +143,13 @@ func (h *hbshFile) WriteAt(p []byte, off int64) (n int, err error) { // Writing past the EOF: // We're either appending an entirely new block, // or the final block was only partially written. - // A partially written block can't be decripted, + // A partially written block can't be decrypted, // and is as good as corrupt. // Either way, zero pad the file to the next block size. clear(data) + } else { + data = h.hbsh.Decrypt(h.block[:], h.tweak[:]) } - - data = h.hbsh.Decrypt(h.block[:], h.tweak[:]) if off > min { data = data[off-min:] } @@ -172,7 +172,7 @@ func (h *hbshFile) WriteAt(p []byte, off int64) (n int, err error) { } func (h *hbshFile) Truncate(size int64) error { - size = (size + blockSize - 1) &^ (blockSize - 1) // round up + size = (size + (blockSize - 1)) &^ (blockSize - 1) // round up return h.File.Truncate(size) } @@ -188,25 +188,25 @@ func (h *hbshFile) DeviceCharacteristics() vfs.DeviceCharacteristic { vfs.IOCAP_BATCH_ATOMIC) } +// Wrap optional methods. + func (h *hbshFile) SharedMemory() vfs.SharedMemory { - if shm, ok := h.File.(vfs.FileSharedMemory); ok { - return shm.SharedMemory() + if f, ok := h.File.(vfs.FileSharedMemory); ok { + return f.SharedMemory() } return nil } -// Wrap optional methods. - func (h *hbshFile) ChunkSize(size int) { if f, ok := h.File.(vfs.FileChunkSize); ok { - size = (size + blockSize - 1) &^ (blockSize - 1) // round up + size = (size + (blockSize - 1)) &^ (blockSize - 1) // round up f.ChunkSize(size) } } func (h *hbshFile) SizeHint(size int64) error { if f, ok := h.File.(vfs.FileSizeHint); ok { - size = (size + blockSize - 1) &^ (blockSize - 1) // round up + size = (size + (blockSize - 1)) &^ (blockSize - 1) // round up return f.SizeHint(size) } return sqlite3.NOTFOUND diff --git a/vfs/api.go b/vfs/api.go index 87c9df8..19c22ae 100644 --- a/vfs/api.go +++ b/vfs/api.go @@ -157,6 +157,8 @@ type FileCheckpoint interface { // FileSharedMemory extends File to possibly implement // shared-memory for the WAL-index. +// The same shared-memory instance must be returned +// for the entire life of the file. // It's OK for SharedMemory to return nil. type FileSharedMemory interface { File @@ -164,7 +166,7 @@ type FileSharedMemory interface { } // SharedMemory is a shared-memory WAL-index implementation. -// This cannot be externally implemented. +// Use [NewSharedMemory] to create a shared-memory. type SharedMemory interface { shmMap(context.Context, api.Module, int32, int32, bool) (uint32, error) shmLock(int32, int32, _ShmFlag) error diff --git a/vfs/file.go b/vfs/file.go index 92170ca..ca8cf84 100644 --- a/vfs/file.go +++ b/vfs/file.go @@ -176,7 +176,7 @@ func (f *vfsFile) Size() (int64, error) { return f.Seek(0, io.SeekEnd) } -func (*vfsFile) SectorSize() int { +func (f *vfsFile) SectorSize() int { return _DEFAULT_SECTOR_SIZE } diff --git a/vfs/memdb/memdb.go b/vfs/memdb/memdb.go index f20b792..09ffa4e 100644 --- a/vfs/memdb/memdb.go +++ b/vfs/memdb/memdb.go @@ -28,7 +28,6 @@ func (memVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, err // We refuse to open all other file types, // but returning OPEN_MEMORY means SQLite won't ask us to. const types = vfs.OPEN_MAIN_DB | - vfs.OPEN_TRANSIENT_DB | vfs.OPEN_TEMP_DB | vfs.OPEN_TEMP_JOURNAL if flags&types == 0 { @@ -173,7 +172,7 @@ func (m *memFile) truncate(size int64) error { return nil } -func (*memFile) Sync(flag vfs.SyncFlag) error { +func (m *memFile) Sync(flag vfs.SyncFlag) error { return nil } @@ -263,11 +262,11 @@ func (m *memFile) CheckReservedLock() (bool, error) { return m.reserved != nil, nil } -func (*memFile) SectorSize() int { +func (m *memFile) SectorSize() int { return sectorSize } -func (*memFile) DeviceCharacteristics() vfs.DeviceCharacteristic { +func (m *memFile) DeviceCharacteristics() vfs.DeviceCharacteristic { return vfs.IOCAP_ATOMIC | vfs.IOCAP_SEQUENTIAL | vfs.IOCAP_SAFE_APPEND | diff --git a/vfs/shm.go b/vfs/shm.go index bfc1805..2b76dd5 100644 --- a/vfs/shm.go +++ b/vfs/shm.go @@ -12,11 +12,8 @@ import ( "golang.org/x/sys/unix" ) -// SupportsSharedMemory is true on platforms that support shared memory. -// To enable shared memory support on those platforms, -// you need to set the appropriate [wazero.RuntimeConfig]; -// otherwise, [EXCLUSIVE locking mode] is activated automatically -// to use [WAL without shared-memory]. +// SupportsSharedMemory is false on platforms that do not support shared memory. +// To use [WAL without shared-memory], you need to set [EXCLUSIVE locking mode]. // // [WAL without shared-memory]: https://sqlite.org/wal.html#noshm // [EXCLUSIVE locking mode]: https://sqlite.org/pragma.html#pragma_locking_mode @@ -31,12 +28,14 @@ const ( func (f *vfsFile) SharedMemory() SharedMemory { return f.shm } // NewSharedMemory returns a shared-memory WAL-index -// backed a file with the given path. -// It may return nil if shared-memory is not supported, +// backed by a file with the given path. +// It will return nil if shared-memory is not supported, // or not appropriate for the given flags. -// Only [OPEN_MAIN_DB] databases support WAL mode. +// Only [OPEN_MAIN_DB] databases may need a WAL-index. +// You must ensure all concurrent accesses to a database +// use shared-memory instances created with the same path. func NewSharedMemory(path string, flags OpenFlag) SharedMemory { - if flags&OPEN_MAIN_DB == 0 { + if flags&OPEN_MAIN_DB == 0 || flags&(OPEN_DELETEONCLOSE|OPEN_MEMORY) != 0 { return nil } return &vfsShm{ @@ -47,8 +46,8 @@ func NewSharedMemory(path string, flags OpenFlag) SharedMemory { type vfsShm struct { *os.File - regions []*util.MappedRegion path string + regions []*util.MappedRegion readOnly bool } @@ -154,6 +153,10 @@ func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) error { } func (s *vfsShm) shmUnmap(delete bool) { + if s.File == nil { + return + } + // Unmap regions. for _, r := range s.regions { r.Unmap() @@ -162,9 +165,9 @@ func (s *vfsShm) shmUnmap(delete bool) { s.regions = s.regions[:0] // Close the file. - if delete && s.File != nil { + defer s.Close() + if delete { os.Remove(s.Name()) } - s.Close() s.File = nil } diff --git a/vfs/shm_other.go b/vfs/shm_other.go index b5c2eed..2119197 100644 --- a/vfs/shm_other.go +++ b/vfs/shm_other.go @@ -2,21 +2,20 @@ package vfs -// SupportsSharedMemory is true on platforms that support shared memory. -// To enable shared memory support on those platforms, -// you need to set the appropriate [wazero.RuntimeConfig]; -// otherwise, [EXCLUSIVE locking mode] is activated automatically -// to use [WAL without shared-memory]. +// SupportsSharedMemory is false on platforms that do not support shared memory. +// To use [WAL without shared-memory], you need to set [EXCLUSIVE locking mode]. // // [WAL without shared-memory]: https://sqlite.org/wal.html#noshm // [EXCLUSIVE locking mode]: https://sqlite.org/pragma.html#pragma_locking_mode const SupportsSharedMemory = false // NewSharedMemory returns a shared-memory WAL-index -// backed a file with the given path. -// It may return nil if shared-memory is not supported, +// backed by a file with the given path. +// It will return nil if shared-memory is not supported, // or not appropriate for the given flags. -// Only [OPEN_MAIN_DB] databases support WAL mode. +// Only [OPEN_MAIN_DB] databases may need a WAL-index. +// You must ensure all concurrent accesses to a database +// use shared-memory instances created with the same path. func NewSharedMemory(path string, flags OpenFlag) SharedMemory { return nil } diff --git a/vfs/vfs.go b/vfs/vfs.go index 805716e..1887e9f 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -146,7 +146,6 @@ func vfsOpen(ctx context.Context, mod api.Module, pVfs, zPath, pFile uint32, fla } else { file, flags, err = vfs.Open(path, flags) } - if err != nil { return vfsErrorCode(err, _CANTOPEN) } @@ -157,14 +156,13 @@ func vfsOpen(ctx context.Context, mod api.Module, pVfs, zPath, pFile uint32, fla file.SetPowersafeOverwrite(b) } } - + if file, ok := file.(FileSharedMemory); ok && + pOutVFS != 0 && file.SharedMemory() != nil { + util.WriteUint32(mod, pOutVFS, 1) + } if pOutFlags != 0 { util.WriteUint32(mod, pOutFlags, uint32(flags)) } - if f, ok := file.(FileSharedMemory); ok && flags&OPEN_MAIN_DB != 0 && - pOutVFS != 0 && f.SharedMemory() != nil { - util.WriteUint32(mod, pOutVFS, 1) - } vfsFileRegister(ctx, mod, pFile, file) return _OK } @@ -398,8 +396,8 @@ func vfsShmBarrier(ctx context.Context, mod api.Module, pFile uint32) { } func vfsShmMap(ctx context.Context, mod api.Module, pFile uint32, iRegion, szRegion int32, bExtend, pp uint32) _ErrorCode { - file := vfsFileGet(ctx, mod, pFile).(FileSharedMemory).SharedMemory() - p, err := file.shmMap(ctx, mod, iRegion, szRegion, bExtend != 0) + shm := vfsFileGet(ctx, mod, pFile).(FileSharedMemory).SharedMemory() + p, err := shm.shmMap(ctx, mod, iRegion, szRegion, bExtend != 0) if err != nil { return vfsErrorCode(err, _IOERR_SHMMAP) } @@ -408,14 +406,14 @@ func vfsShmMap(ctx context.Context, mod api.Module, pFile uint32, iRegion, szReg } func vfsShmLock(ctx context.Context, mod api.Module, pFile uint32, offset, n int32, flags _ShmFlag) _ErrorCode { - file := vfsFileGet(ctx, mod, pFile).(FileSharedMemory).SharedMemory() - err := file.shmLock(offset, n, flags) + shm := vfsFileGet(ctx, mod, pFile).(FileSharedMemory).SharedMemory() + err := shm.shmLock(offset, n, flags) return vfsErrorCode(err, _IOERR_SHMLOCK) } func vfsShmUnmap(ctx context.Context, mod api.Module, pFile, bDelete uint32) _ErrorCode { - file := vfsFileGet(ctx, mod, pFile).(FileSharedMemory).SharedMemory() - file.shmUnmap(bDelete != 0) + shm := vfsFileGet(ctx, mod, pFile).(FileSharedMemory).SharedMemory() + shm.shmUnmap(bDelete != 0) return _OK }