From d880d6842cebc63cab7767dd67716879dae4c538 Mon Sep 17 00:00:00 2001 From: Nuno Cruces Date: Thu, 23 Mar 2023 13:28:25 +0000 Subject: [PATCH] Refactor VFS. --- README.md | 12 +++++---- vfs_os_bsd.go | 56 +++++++++++++++++++++++++++++++++++++++++ vfs_os_linux.go | 54 ---------------------------------------- vfs_os_ofd.go | 63 +++++++++++++++++++++++++++++++++++++++++++++++ vfs_os_other.go | 51 +------------------------------------- vfs_os_unix.go | 2 +- vfs_os_windows.go | 16 ------------ 7 files changed, 128 insertions(+), 126 deletions(-) create mode 100644 vfs_os_bsd.go create mode 100644 vfs_os_ofd.go diff --git a/README.md b/README.md index f113fc4..031a5e8 100644 --- a/README.md +++ b/README.md @@ -39,15 +39,18 @@ To open WAL databases, or use `EXCLUSIVE` locking mode, disable connection pooling by calling [`db.SetMaxOpenConns(1)`](https://pkg.go.dev/database/sql#DB.SetMaxOpenConns). -#### Open File Description Locks +#### POSIX Advisory Locks -POSIX advisory locks, which SQLite uses, are [broken by design](https://www.sqlite.org/src/artifact/90c4fa?ln=1073-1161). +POSIX advisory locks, which SQLite uses, are +[broken by design](https://www.sqlite.org/src/artifact/90c4fa?ln=1073-1161). -On Linux and macOS, this module uses [OFD locks](https://www.gnu.org/software/libc/manual/html_node/Open-File-Description-Locks.html) +On Linux, macOS and illumos, this module uses +[OFD locks](https://www.gnu.org/software/libc/manual/html_node/Open-File-Description-Locks.html) to synchronize access to database files. OFD locks are fully compatible with process-associated POSIX advisory locks. -On other Unixes, this module uses [BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2). +On BSD Unixes, this module uses +[BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2). BSD locks may _not_ be compatible with process-associated POSIX advisory locks. #### Testing @@ -55,7 +58,6 @@ BSD locks may _not_ be compatible with process-associated POSIX advisory locks. The pure Go VFS is stress tested by running an unmodified build of SQLite's [mptest](https://github.com/sqlite/sqlite/blob/master/mptest/mptest.c) on Linux, macOS and Windows. - Performance is tested by running [speedtest1](https://github.com/sqlite/sqlite/blob/master/test/speedtest1.c). diff --git a/vfs_os_bsd.go b/vfs_os_bsd.go new file mode 100644 index 0000000..5a7f96e --- /dev/null +++ b/vfs_os_bsd.go @@ -0,0 +1,56 @@ +//go:build freebsd || openbsd || netbsd || dragonfly || (darwin && sqlite3_bsd) + +package sqlite3 + +import ( + "os" + "time" + + "golang.org/x/sys/unix" +) + +func (vfsOSMethods) unlock(file *os.File, start, len int64) xErrorCode { + if start == 0 && len == 0 { + err := unix.Flock(int(file.Fd()), unix.LOCK_UN) + if err != nil { + return IOERR_UNLOCK + } + } + return _OK +} + +func (vfsOSMethods) lock(file *os.File, how int, timeout time.Duration, def xErrorCode) xErrorCode { + var err error + for { + err = unix.Flock(int(file.Fd()), how) + if errno, _ := err.(unix.Errno); errno != unix.EAGAIN { + break + } + if timeout < time.Millisecond { + break + } + timeout -= time.Millisecond + time.Sleep(time.Millisecond) + } + return vfsOS.lockErrorCode(err, def) +} + +func (vfsOSMethods) readLock(file *os.File, start, len int64, timeout time.Duration) xErrorCode { + return vfsOS.lock(file, unix.LOCK_SH|unix.LOCK_NB, timeout, IOERR_RDLOCK) +} + +func (vfsOSMethods) writeLock(file *os.File, start, len int64, timeout time.Duration) xErrorCode { + return vfsOS.lock(file, unix.LOCK_EX|unix.LOCK_NB, timeout, IOERR_LOCK) +} + +func (vfsOSMethods) checkLock(file *os.File, start, len int64) (bool, xErrorCode) { + lock := unix.Flock_t{ + Type: unix.F_RDLCK, + Start: start, + Len: len, + } + if unix.FcntlFlock(file.Fd(), unix.F_GETLK, &lock) != nil { + return false, IOERR_CHECKRESERVEDLOCK + } + return lock.Type != unix.F_UNLCK, _OK +} diff --git a/vfs_os_linux.go b/vfs_os_linux.go index 9f06408..7c8f1be 100644 --- a/vfs_os_linux.go +++ b/vfs_os_linux.go @@ -2,7 +2,6 @@ package sqlite3 import ( "os" - "time" "golang.org/x/sys/unix" ) @@ -24,56 +23,3 @@ func (vfsOSMethods) Allocate(file *os.File, size int64) error { } return unix.Fallocate(int(file.Fd()), 0, 0, size) } - -func (vfsOSMethods) unlock(file *os.File, start, len int64) xErrorCode { - err := unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &unix.Flock_t{ - Type: unix.F_UNLCK, - Start: start, - Len: len, - }) - if err != nil { - return IOERR_UNLOCK - } - return _OK -} - -func (vfsOSMethods) lock(file *os.File, typ int16, start, len int64, timeout time.Duration, def xErrorCode) xErrorCode { - lock := unix.Flock_t{ - Type: typ, - Start: start, - Len: len, - } - var err error - for { - err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &lock) - if errno, _ := err.(unix.Errno); errno != unix.EAGAIN { - break - } - if timeout < time.Millisecond { - break - } - timeout -= time.Millisecond - time.Sleep(time.Millisecond) - } - return vfsOS.lockErrorCode(err, def) -} - -func (vfsOSMethods) readLock(file *os.File, start, len int64, timeout time.Duration) xErrorCode { - return vfsOS.lock(file, unix.F_RDLCK, start, len, timeout, IOERR_RDLOCK) -} - -func (vfsOSMethods) writeLock(file *os.File, start, len int64, timeout time.Duration) xErrorCode { - return vfsOS.lock(file, unix.F_WRLCK, start, len, timeout, IOERR_LOCK) -} - -func (vfsOSMethods) checkLock(file *os.File, start, len int64) (bool, xErrorCode) { - lock := unix.Flock_t{ - Type: unix.F_RDLCK, - Start: start, - Len: len, - } - if unix.FcntlFlock(file.Fd(), unix.F_OFD_GETLK, &lock) != nil { - return false, IOERR_CHECKRESERVEDLOCK - } - return lock.Type != unix.F_UNLCK, _OK -} diff --git a/vfs_os_ofd.go b/vfs_os_ofd.go new file mode 100644 index 0000000..9d81d33 --- /dev/null +++ b/vfs_os_ofd.go @@ -0,0 +1,63 @@ +//go:build linux || illumos + +package sqlite3 + +import ( + "os" + "time" + + "golang.org/x/sys/unix" +) + +func (vfsOSMethods) unlock(file *os.File, start, len int64) xErrorCode { + err := unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &unix.Flock_t{ + Type: unix.F_UNLCK, + Start: start, + Len: len, + }) + if err != nil { + return IOERR_UNLOCK + } + return _OK +} + +func (vfsOSMethods) lock(file *os.File, typ int16, start, len int64, timeout time.Duration, def xErrorCode) xErrorCode { + lock := unix.Flock_t{ + Type: typ, + Start: start, + Len: len, + } + var err error + for { + err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &lock) + if errno, _ := err.(unix.Errno); errno != unix.EAGAIN { + break + } + if timeout < time.Millisecond { + break + } + timeout -= time.Millisecond + time.Sleep(time.Millisecond) + } + return vfsOS.lockErrorCode(err, def) +} + +func (vfsOSMethods) readLock(file *os.File, start, len int64, timeout time.Duration) xErrorCode { + return vfsOS.lock(file, unix.F_RDLCK, start, len, timeout, IOERR_RDLOCK) +} + +func (vfsOSMethods) writeLock(file *os.File, start, len int64, timeout time.Duration) xErrorCode { + return vfsOS.lock(file, unix.F_WRLCK, start, len, timeout, IOERR_LOCK) +} + +func (vfsOSMethods) checkLock(file *os.File, start, len int64) (bool, xErrorCode) { + lock := unix.Flock_t{ + Type: unix.F_RDLCK, + Start: start, + Len: len, + } + if unix.FcntlFlock(file.Fd(), unix.F_OFD_GETLK, &lock) != nil { + return false, IOERR_CHECKRESERVEDLOCK + } + return lock.Type != unix.F_UNLCK, _OK +} diff --git a/vfs_os_other.go b/vfs_os_other.go index 23e52ca..3134761 100644 --- a/vfs_os_other.go +++ b/vfs_os_other.go @@ -1,13 +1,10 @@ -//go:build unix && !linux && (!darwin || sqlite3_bsd) +//go:build !linux && (!darwin || sqlite3_bsd) package sqlite3 import ( "io" "os" - "time" - - "golang.org/x/sys/unix" ) func (vfsOSMethods) Sync(file *os.File, fullsync, dataonly bool) error { @@ -24,49 +21,3 @@ func (vfsOSMethods) Allocate(file *os.File, size int64) error { } return file.Truncate(size) } - -func (vfsOSMethods) unlock(file *os.File, start, len int64) xErrorCode { - if start == 0 && len == 0 { - err := unix.Flock(int(file.Fd()), unix.LOCK_UN) - if err != nil { - return IOERR_UNLOCK - } - } - return _OK -} - -func (vfsOSMethods) lock(file *os.File, how int, timeout time.Duration, def xErrorCode) xErrorCode { - var err error - for { - err = unix.Flock(int(file.Fd()), how) - if errno, _ := err.(unix.Errno); errno != unix.EAGAIN { - break - } - if timeout < time.Millisecond { - break - } - timeout -= time.Millisecond - time.Sleep(time.Millisecond) - } - return vfsOS.lockErrorCode(err, def) -} - -func (vfsOSMethods) readLock(file *os.File, start, len int64, timeout time.Duration) xErrorCode { - return vfsOS.lock(file, unix.LOCK_SH|unix.LOCK_NB, timeout, IOERR_RDLOCK) -} - -func (vfsOSMethods) writeLock(file *os.File, start, len int64, timeout time.Duration) xErrorCode { - return vfsOS.lock(file, unix.LOCK_EX|unix.LOCK_NB, timeout, IOERR_LOCK) -} - -func (vfsOSMethods) checkLock(file *os.File, start, len int64) (bool, xErrorCode) { - lock := unix.Flock_t{ - Type: unix.F_RDLCK, - Start: start, - Len: len, - } - if unix.FcntlFlock(file.Fd(), unix.F_GETLK, &lock) != nil { - return false, IOERR_CHECKRESERVEDLOCK - } - return lock.Type != unix.F_UNLCK, _OK -} diff --git a/vfs_os_unix.go b/vfs_os_unix.go index 28860d9..2ad7ed4 100644 --- a/vfs_os_unix.go +++ b/vfs_os_unix.go @@ -15,7 +15,7 @@ func (vfsOSMethods) OpenFile(name string, flag int, perm fs.FileMode) (*os.File, } func (vfsOSMethods) Access(path string, flags _AccessFlag) error { - var access uint32 = unix.F_OK + var access uint32 // unix.F_OK switch flags { case _ACCESS_READWRITE: access = unix.R_OK | unix.W_OK diff --git a/vfs_os_windows.go b/vfs_os_windows.go index 457ee55..2132e18 100644 --- a/vfs_os_windows.go +++ b/vfs_os_windows.go @@ -1,7 +1,6 @@ package sqlite3 import ( - "io" "io/fs" "os" "syscall" @@ -48,21 +47,6 @@ func (vfsOSMethods) Access(path string, flags _AccessFlag) error { return nil } -func (vfsOSMethods) Sync(file *os.File, fullsync, dataonly bool) error { - return file.Sync() -} - -func (vfsOSMethods) Allocate(file *os.File, size int64) error { - off, err := file.Seek(0, io.SeekEnd) - if err != nil { - return err - } - if size <= off { - return nil - } - return file.Truncate(size) -} - func (vfsOSMethods) GetSharedLock(file *os.File, timeout time.Duration) xErrorCode { // Acquire the PENDING lock temporarily before acquiring a new SHARED lock. rc := vfsOS.readLock(file, _PENDING_BYTE, 1, timeout)