From 15dec13f156393607aef7a65badfdbbcf39fba2b Mon Sep 17 00:00:00 2001 From: Nuno Cruces Date: Fri, 17 Mar 2023 17:13:03 +0000 Subject: [PATCH] FCNTL_SIZE_HINT, refactor. --- README.md | 2 +- vfs.go | 40 ++++++++++++++-- vfs_lock_test.go | 2 +- vfs_os_darwin.go | 60 ++++++++++++++++++++++++ vfs_os_linux.go | 39 +++++++++++++++ vfs_os_posix.go | 30 ++++++++++++ vfs_unix.go => vfs_os_unix.go | 73 +---------------------------- vfs_windows.go => vfs_os_windows.go | 67 +++++++++++++------------- 8 files changed, 201 insertions(+), 112 deletions(-) create mode 100644 vfs_os_darwin.go create mode 100644 vfs_os_linux.go create mode 100644 vfs_os_posix.go rename vfs_unix.go => vfs_os_unix.go (55%) rename vfs_windows.go => vfs_os_windows.go (88%) diff --git a/README.md b/README.md index e9f11ac..0a087ec 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ to synchronize access to database files. POSIX advisory locks, which SQLite uses, are [broken by design](https://www.sqlite.org/src/artifact/90c4fa?ln=1073-1161). OFD locks are fully compatible with process-associated POSIX advisory locks, -and are supported on Linux, macOS and illumos. +and are supported on Linux and macOS. As a work around for other Unixes, you can use [`nolock=1`](https://www.sqlite.org/uri.html). #### Testing diff --git a/vfs.go b/vfs.go index 35931f4..ecef74c 100644 --- a/vfs.go +++ b/vfs.go @@ -194,11 +194,30 @@ func vfsDelete(ctx context.Context, mod api.Module, pVfs, zPath, syncDir uint32) func vfsAccess(ctx context.Context, mod api.Module, pVfs, zPath uint32, flags _AccessFlag, pResOut uint32) uint32 { path := memory{mod}.readString(zPath, _MAX_PATHNAME) - ok, rc := vfsOS.Access(path, flags) + err := vfsOS.Access(path, flags) + var res uint32 - if ok { - res = 1 + var rc xErrorCode + if flags == _ACCESS_EXISTS { + switch { + case err == nil: + res = 1 + case errors.Is(err, fs.ErrNotExist): + res = 0 + default: + rc = IOERR_ACCESS + } + } else { + switch { + case err == nil: + res = 1 + case errors.Is(err, fs.ErrPermission): + res = 0 + default: + rc = IOERR_ACCESS + } } + memory{mod}.writeUint32(pResOut, res) return uint32(rc) } @@ -312,13 +331,26 @@ func vfsFileSize(ctx context.Context, mod api.Module, pFile, pSize uint32) uint3 func vfsFileControl(ctx context.Context, mod api.Module, pFile uint32, op _FcntlOpcode, pArg uint32) uint32 { switch op { case _FCNTL_SIZE_HINT: - // + return vfsSizeHint(ctx, mod, pFile, pArg) case _FCNTL_HAS_MOVED: return vfsFileMoved(ctx, mod, pFile, pArg) } return uint32(NOTFOUND) } +func vfsSizeHint(ctx context.Context, mod api.Module, pFile, pArg uint32) uint32 { + file := vfsFile.GetOS(ctx, mod, pFile) + size := memory{mod}.readUint64(pArg) + err := vfsOS.Allocate(file, int64(size)) + if err == notImplErr { + return uint32(NOTFOUND) + } + if err != nil { + return uint32(IOERR_TRUNCATE) + } + return _OK +} + func vfsFileMoved(ctx context.Context, mod api.Module, pFile, pResOut uint32) uint32 { file := vfsFile.GetOS(ctx, mod, pFile) fi, err := file.Stat() diff --git a/vfs_lock_test.go b/vfs_lock_test.go index 88d463f..0b11512 100644 --- a/vfs_lock_test.go +++ b/vfs_lock_test.go @@ -10,7 +10,7 @@ import ( func Test_vfsLock(t *testing.T) { switch runtime.GOOS { - case "linux", "darwin", "illumos", "windows": + case "linux", "darwin", "windows": break default: t.Skip("OS lacks OFD locks") diff --git a/vfs_os_darwin.go b/vfs_os_darwin.go new file mode 100644 index 0000000..eb7410f --- /dev/null +++ b/vfs_os_darwin.go @@ -0,0 +1,60 @@ +package sqlite3 + +import ( + "os" + "time" + + "golang.org/x/sys/unix" +) + +func (vfsOSMethods) Sync(file *os.File, fullsync, dataonly bool) error { + if !fullsync { + return unix.Fsync(int(file.Fd())) + } + return file.Sync() +} + +func (vfsOSMethods) Allocate(file *os.File, size int64) error { + // https://stackoverflow.com/a/11497568/867786 + store := unix.Fstore_t{ + Flags: unix.F_ALLOCATECONTIG, + Posmode: unix.F_PEOFPOSMODE, + Offset: 0, + Length: size, + } + + // Try to get a continous chunk of disk space. + err := unix.FcntlFstore(file.Fd(), unix.F_PREALLOCATE, &store) + if err != nil { + // OK, perhaps we are too fragmented, allocate non-continuous. + store.Flags = unix.F_ALLOCATEALL + return unix.FcntlFstore(file.Fd(), unix.F_PREALLOCATE, &store) + } + return nil +} + +func (vfsOSMethods) fcntlGetLock(file *os.File, lock *unix.Flock_t) error { + const F_OFD_GETLK = 92 // https://github.com/apple/darwin-xnu/blob/main/bsd/sys/fcntl.h + return unix.FcntlFlock(file.Fd(), F_OFD_GETLK, lock) +} + +func (vfsOSMethods) fcntlSetLock(file *os.File, lock unix.Flock_t) error { + const F_OFD_SETLK = 90 // https://github.com/apple/darwin-xnu/blob/main/bsd/sys/fcntl.h + return unix.FcntlFlock(file.Fd(), F_OFD_SETLK, &lock) +} + +func (vfsOSMethods) fcntlSetLockTimeout(timeout time.Duration, file *os.File, lock unix.Flock_t) error { + if timeout == 0 { + return vfsOS.fcntlSetLock(file, lock) + } + + const F_OFD_SETLKWTIMEOUT = 93 // https://github.com/apple/darwin-xnu/blob/main/bsd/sys/fcntl.h + flocktimeout := &struct { + unix.Flock_t + unix.Timespec + }{ + Flock_t: lock, + Timespec: unix.NsecToTimespec(int64(timeout / time.Nanosecond)), + } + return unix.FcntlFlock(file.Fd(), F_OFD_SETLKWTIMEOUT, &flocktimeout.Flock_t) +} diff --git a/vfs_os_linux.go b/vfs_os_linux.go new file mode 100644 index 0000000..a82ddc8 --- /dev/null +++ b/vfs_os_linux.go @@ -0,0 +1,39 @@ +package sqlite3 + +import ( + "os" + "time" + + "golang.org/x/sys/unix" +) + +func (vfsOSMethods) Sync(file *os.File, fullsync, dataonly bool) error { + if dataonly { + //lint:ignore SA1019 OK on linux + _, _, err := unix.Syscall(unix.SYS_FDATASYNC, file.Fd(), 0, 0) + if err != 0 { + return err + } + return nil + } + return file.Sync() +} + +func (vfsOSMethods) Allocate(file *os.File, size int64) error { + if size == 0 { + return nil + } + return unix.Fallocate(int(file.Fd()), 0, 0, size) +} + +func (vfsOSMethods) fcntlGetLock(file *os.File, lock *unix.Flock_t) error { + return unix.FcntlFlock(file.Fd(), unix.F_OFD_GETLK, lock) +} + +func (vfsOSMethods) fcntlSetLock(file *os.File, lock unix.Flock_t) error { + return unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &lock) +} + +func (vfsOSMethods) fcntlSetLockTimeout(timeout time.Duration, file *os.File, lock unix.Flock_t) error { + return vfsOS.fcntlSetLock(file, lock) +} diff --git a/vfs_os_posix.go b/vfs_os_posix.go new file mode 100644 index 0000000..e9abe40 --- /dev/null +++ b/vfs_os_posix.go @@ -0,0 +1,30 @@ +//go:build !windows && !linux && !darwin + +package sqlite3 + +import ( + "os" + "time" + + "golang.org/x/sys/unix" +) + +func (vfsOSMethods) Sync(file *os.File, fullsync, dataonly bool) error { + return file.Sync() +} + +func (vfsOSMethods) Allocate(file *os.File, size int64) error { + return notImplErr +} + +func (vfsOSMethods) fcntlGetLock(file *os.File, lock *unix.Flock_t) error { + return notImplErr +} + +func (vfsOSMethods) fcntlSetLock(file *os.File, lock unix.Flock_t) error { + return notImplErr +} + +func (vfsOSMethods) fcntlSetLockTimeout(timeout time.Duration, file *os.File, lock unix.Flock_t) error { + return notImplErr +} diff --git a/vfs_unix.go b/vfs_os_unix.go similarity index 55% rename from vfs_unix.go rename to vfs_os_unix.go index 8a8f688..7049d73 100644 --- a/vfs_unix.go +++ b/vfs_os_unix.go @@ -5,8 +5,6 @@ package sqlite3 import ( "io/fs" "os" - "runtime" - "time" "golang.org/x/sys/unix" ) @@ -15,7 +13,7 @@ func (vfsOSMethods) OpenFile(name string, flag int, perm fs.FileMode) (*os.File, return os.OpenFile(name, flag, perm) } -func (vfsOSMethods) Access(path string, flags _AccessFlag) (bool, xErrorCode) { +func (vfsOSMethods) Access(path string, flags _AccessFlag) error { var access uint32 = unix.F_OK switch flags { case _ACCESS_READWRITE: @@ -23,27 +21,7 @@ func (vfsOSMethods) Access(path string, flags _AccessFlag) (bool, xErrorCode) { case _ACCESS_READ: access = unix.R_OK } - - err := unix.Access(path, access) - if err == nil { - return true, _OK - } - return false, _OK -} - -func (vfsOSMethods) Sync(file *os.File, fullsync, dataonly bool) error { - if runtime.GOOS == "darwin" && !fullsync { - return unix.Fsync(int(file.Fd())) - } - if runtime.GOOS == "linux" && dataonly { - //lint:ignore SA1019 OK on linux - _, _, err := unix.Syscall(unix.SYS_FDATASYNC, file.Fd(), 0, 0) - if err != 0 { - return err - } - return nil - } - return file.Sync() + return unix.Access(path, access) } func (vfsOSMethods) GetExclusiveLock(file *os.File) xErrorCode { @@ -113,53 +91,6 @@ func (vfsOSMethods) checkLock(file *os.File, start, len int64) (bool, xErrorCode return lock.Type != unix.F_UNLCK, _OK } -func (vfsOSMethods) fcntlGetLock(file *os.File, lock *unix.Flock_t) error { - var F_OFD_GETLK int - switch runtime.GOOS { - case "linux": - F_OFD_GETLK = 36 // https://github.com/torvalds/linux/blob/master/include/uapi/asm-generic/fcntl.h - case "darwin": - F_OFD_GETLK = 92 // https://github.com/apple/darwin-xnu/blob/main/bsd/sys/fcntl.h - case "illumos": - F_OFD_GETLK = 47 // https://github.com/illumos/illumos-gate/blob/master/usr/src/uts/common/sys/fcntl.h - default: - return notImplErr - } - return unix.FcntlFlock(file.Fd(), F_OFD_GETLK, lock) -} - -func (vfsOSMethods) fcntlSetLock(file *os.File, lock unix.Flock_t) error { - var F_OFD_SETLK int - switch runtime.GOOS { - case "linux": - F_OFD_SETLK = 37 // https://github.com/torvalds/linux/blob/master/include/uapi/asm-generic/fcntl.h - case "darwin": - F_OFD_SETLK = 90 // https://github.com/apple/darwin-xnu/blob/main/bsd/sys/fcntl.h - case "illumos": - F_OFD_SETLK = 48 // https://github.com/illumos/illumos-gate/blob/master/usr/src/uts/common/sys/fcntl.h - default: - return notImplErr - } - return unix.FcntlFlock(file.Fd(), F_OFD_SETLK, &lock) -} - -func (vfsOSMethods) fcntlSetLockTimeout(timeout time.Duration, file *os.File, lock unix.Flock_t) error { - if runtime.GOOS != "darwin" || timeout == 0 { - return vfsOS.fcntlSetLock(file, lock) - } - - const F_OFD_SETLKWTIMEOUT = 93 // https://github.com/apple/darwin-xnu/blob/main/bsd/sys/fcntl.h - flocktimeout := &struct { - unix.Flock_t - unix.Timespec - }{ - Flock_t: lock, - Timespec: unix.NsecToTimespec(int64(timeout / time.Nanosecond)), - } - - return unix.FcntlFlock(file.Fd(), F_OFD_SETLKWTIMEOUT, &flocktimeout.Flock_t) -} - func (vfsOSMethods) lockErrorCode(err error, def xErrorCode) xErrorCode { if err == nil { return _OK diff --git a/vfs_windows.go b/vfs_os_windows.go similarity index 88% rename from vfs_windows.go rename to vfs_os_windows.go index 912e2f7..5deb437 100644 --- a/vfs_windows.go +++ b/vfs_os_windows.go @@ -1,7 +1,7 @@ package sqlite3 import ( - "errors" + "io" "io/fs" "os" "syscall" @@ -25,44 +25,41 @@ func (vfsOSMethods) OpenFile(name string, flag int, perm fs.FileMode) (*os.File, return os.NewFile(uintptr(r), name), nil } +func (vfsOSMethods) Access(path string, flags _AccessFlag) error { + fi, err := os.Stat(path) + if err != nil { + return err + } + if flags == _ACCESS_EXISTS { + return nil + } + + var want fs.FileMode = windows.S_IRUSR + if flags == _ACCESS_READWRITE { + want |= windows.S_IWUSR + } + if fi.IsDir() { + want |= windows.S_IXUSR + } + if fi.Mode()&want != want { + return fs.ErrPermission + } + return nil +} + func (vfsOSMethods) Sync(file *os.File, fullsync, dataonly bool) error { return file.Sync() } -func (vfsOSMethods) Access(path string, flags _AccessFlag) (bool, xErrorCode) { - fi, err := os.Stat(path) - - switch { - case flags == _ACCESS_EXISTS: - switch { - case err == nil: - return true, _OK - case errors.Is(err, fs.ErrNotExist): - return false, _OK - default: - return false, IOERR_ACCESS - } - - case err == nil: - var want fs.FileMode = syscall.S_IRUSR - if flags == _ACCESS_READWRITE { - want |= syscall.S_IWUSR - } - if fi.IsDir() { - want |= syscall.S_IXUSR - } - if fi.Mode()&want == want { - return true, _OK - } else { - return false, _OK - } - - case errors.Is(err, fs.ErrPermission): - return false, _OK - - default: - return false, IOERR_ACCESS +func (vfsOSMethods) Allocate(file *os.File, size int64) error { + off, err := file.Seek(0, io.SeekEnd) + if err != nil { + return err } + if size > off { + return file.Truncate(size) + } + return nil } func (vfsOSMethods) GetExclusiveLock(file *os.File) xErrorCode { @@ -160,7 +157,7 @@ func (vfsOSMethods) lockErrorCode(err error, def xErrorCode) xErrorCode { if err == nil { return _OK } - if errno, ok := err.(syscall.Errno); ok { + if errno, ok := err.(windows.Errno); ok { // https://devblogs.microsoft.com/oldnewthing/20140905-00/?p=63 switch errno { case