From 8d0c654178ce15d868501249fda6065de60afb1b Mon Sep 17 00:00:00 2001 From: Nuno Cruces Date: Tue, 17 Oct 2023 14:04:23 +0100 Subject: [PATCH] Cross compilation. --- .github/workflows/bsd.yml | 28 +++++++++++++ .github/workflows/cross.sh | 22 ++++++++++ .github/workflows/cross.yml | 21 ++++++++++ .github/workflows/go.yml | 21 ++-------- README.md | 22 ++++------ tests/db_test.go | 7 ++++ vfs/README.md | 25 +---------- vfs/lock.go | 23 +---------- vfs/nolock.go | 22 ---------- vfs/os_bsd.go | 2 +- vfs/os_darwin.go | 2 +- vfs/os_linux.go | 2 + vfs/os_nolock.go | 35 +++++++++------- vfs/os_ofd.go | 2 +- vfs/os_std_access.go | 2 +- vfs/os_std_alloc.go | 2 +- vfs/os_std_mode.go | 2 +- vfs/os_std_open.go | 2 +- vfs/os_std_sync.go | 2 +- vfs/os_unix.go | 60 +-------------------------- vfs/os_unix2.go | 82 +++++++++++++++++++++++++++++++++++++ vfs/os_windows.go | 17 ++++++++ 22 files changed, 222 insertions(+), 181 deletions(-) create mode 100644 .github/workflows/bsd.yml create mode 100755 .github/workflows/cross.sh create mode 100644 .github/workflows/cross.yml delete mode 100644 vfs/nolock.go create mode 100644 vfs/os_unix2.go diff --git a/.github/workflows/bsd.yml b/.github/workflows/bsd.yml new file mode 100644 index 0000000..0aefca8 --- /dev/null +++ b/.github/workflows/bsd.yml @@ -0,0 +1,28 @@ +name: BSD + +on: + workflow_dispatch: + +jobs: + test: + runs-on: macos-12 + + steps: + - uses: actions/checkout@v3 + with: + lfs: 'true' + + - name: Set up + uses: actions/setup-go@v4 + with: + go-version: stable + + - name: Build + run: GOOS=freebsd go test -c ./... + + - name: Test + uses: cross-platform-actions/action@v0.19.1 + with: + operating_system: freebsd + version: '13.2' + run: find . -name '*.test' -maxdepth 1 -exec {} -test.v \; diff --git a/.github/workflows/cross.sh b/.github/workflows/cross.sh new file mode 100755 index 0000000..9d9bff7 --- /dev/null +++ b/.github/workflows/cross.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +echo android ; GOOS=android GOARCH=amd64 go build . +echo darwin ; GOOS=darwin GOARCH=amd64 go build . +echo dragonfly ; GOOS=dragonfly GOARCH=amd64 go build . +echo freebsd ; GOOS=freebsd GOARCH=amd64 go build . +echo illumos ; GOOS=illumos GOARCH=amd64 go build . +echo ios ; GOOS=ios GOARCH=amd64 go build . +echo linux ; GOOS=linux GOARCH=amd64 go build . +echo netbsd ; GOOS=netbsd GOARCH=amd64 go build . +echo openbsd ; GOOS=openbsd GOARCH=amd64 go build . +echo plan9 ; GOOS=plan9 GOARCH=amd64 go build . +echo solaris ; GOOS=solaris GOARCH=amd64 go build . +echo windows ; GOOS=windows GOARCH=amd64 go build . +# echo aix ; GOOS=aix GOARCH=ppc64 go build . +echo js ; GOOS=js GOARCH=wasm go build . +echo wasip1 ; GOOS=wasip1 GOARCH=wasm go build . +echo darwin-flock ; GOOS=darwin GOARCH=amd64 go build -tags sqlite3_flock . +echo darwin-nosys ; GOOS=darwin GOARCH=amd64 go build -tags sqlite3_nosys . +echo linux-nosys ; GOOS=linux GOARCH=amd64 go build -tags sqlite3_nosys . +echo windows-nosys ; GOOS=windows GOARCH=amd64 go build -tags sqlite3_nosys . +echo freebsd-nosys ; GOOS=freebsd GOARCH=amd64 go build -tags sqlite3_nosys . diff --git a/.github/workflows/cross.yml b/.github/workflows/cross.yml new file mode 100644 index 0000000..38a7e92 --- /dev/null +++ b/.github/workflows/cross.yml @@ -0,0 +1,21 @@ +name: Cross compile + +on: + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + lfs: 'true' + + - name: Set up + uses: actions/setup-go@v4 + with: + go-version: stable + + - name: Build + run: .github/workflows/cross.sh diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 9cf81ee..67f79de 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -46,27 +46,13 @@ jobs: - name: Test run: go test -v ./... + - name: Test no locks + run: go test -v -tags sqlite3_nosys ./tests -run TestDB_nolock + - name: Test BSD locks run: go test -v -tags sqlite3_flock ./... if: matrix.os == 'macos-latest' - - name: Build FreeBSD tests - run: GOOS=freebsd go test -c ./... - if: matrix.os == 'macos-latest' - - - name: Run FreeBSD tests in a xHyve VM - uses: cross-platform-actions/action@master - with: - operating_system: freebsd - version: '13.2' - run: | - find . -name '*.test' -maxdepth 1 -exec {} -test.v \; - if: matrix.os == 'macos-latest' - - - name: Test no locks - run: go test -v -tags sqlite3_nolock . - if: matrix.os == 'ubuntu-latest' - - name: Coverage report uses: ncruces/go-coverage-report@v0 with: @@ -76,4 +62,3 @@ jobs: if: | github.event_name == 'push' && matrix.os == 'ubuntu-latest' - continue-on-error: true diff --git a/README.md b/README.md index 71d77c4..8e0adfe 100644 --- a/README.md +++ b/README.md @@ -47,35 +47,31 @@ to use the [`database/sql`](https://pkg.go.dev/database/sql) driver with WAL mode databases you should disable connection pooling by calling [`db.SetMaxOpenConns(1)`](https://pkg.go.dev/database/sql#DB.SetMaxOpenConns). -#### POSIX Advisory Locks +#### File Locking -POSIX advisory locks, which SQLite uses, are +POSIX advisory locks, which SQLite uses on Unix, are [broken by design](https://www.sqlite.org/src/artifact/2e8b12?ln=1073-1161). On Linux, macOS and illumos, this module uses [OFD locks](https://www.gnu.org/software/libc/manual/html_node/Open-File-Description-Locks.html) to synchronize access to database files. -OFD locks are fully compatible with process-associated POSIX advisory locks. +OFD locks are fully compatible with POSIX advisory locks. On BSD Unixes, this module uses [BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2). -On BSD Unixes, BSD locks _should_ be compatible with process-associated POSIX advisory locks. +On BSD Unixes, BSD locks are fully compatible with POSIX advisory locks. -##### TL;DR +On Windows, this module uses `LockFile`, `LockFileEx`, and `UnlockFile`, like SQLite. -In all platforms for which this package builds out of the box, -it should be safe to use it to access databases concurrently, -from multiple goroutines, processes, and -with _other_ implementations of SQLite. - -If the package does not build for your platform, -see [this](vfs/README.md#portability). +On all other platforms, file locking is not supported, and you must use +[`nolock=1`](https://www.sqlite.org/uri.html#urinolock) +to open database files. #### Testing The pure Go VFS is tested by running SQLite's [mptest](https://github.com/sqlite/sqlite/blob/master/mptest/mptest.c) -on Linux, macOS and Windows. +on Linux, macOS, Windows and FreeBSD. Performance is tested by running [speedtest1](https://github.com/sqlite/sqlite/blob/master/test/speedtest1.c). diff --git a/tests/db_test.go b/tests/db_test.go index 0fe005f..3ff27f4 100644 --- a/tests/db_test.go +++ b/tests/db_test.go @@ -25,6 +25,13 @@ func TestDB_file(t *testing.T) { testDB(t, filepath.Join(t.TempDir(), "test.db")) } +func TestDB_nolock(t *testing.T) { + t.Parallel() + testDB(t, "file:"+ + filepath.ToSlash(filepath.Join(t.TempDir(), "test.db"))+ + "?nolock=1") +} + func TestDB_wal(t *testing.T) { t.Parallel() wal := filepath.Join(t.TempDir(), "test.db") diff --git a/vfs/README.md b/vfs/README.md index e4cdbd9..ee47315 100644 --- a/vfs/README.md +++ b/vfs/README.md @@ -4,27 +4,4 @@ This package implements the SQLite [OS Interface](https://www.sqlite.org/vfs.htm It replaces the default SQLite VFS with a pure Go implementation. -It also exposes interfaces that should allow you to implement your own custom VFSes. - -## Portability - -This package is continuously tested on Linux, macOS and Windows, -but it should also work on BSD Unixes and illumos -(code paths for those plaforms are tested on macOS and Linux, respectively). - -In all platforms for which this package builds out of the box, -it should be safe to use it to access databases concurrently, -from multiple goroutines, processes, and -with _other_ implementations of SQLite. - -If the package does not build for your platform, -you may try to use the `sqlite3_flock` or `sqlite3_nolock` build tags. - -The `sqlite3_flock` tag uses -[BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2). -It should be safe to access databases concurrently from multiple goroutines and processes, -but **not** with _other_ implementations of SQLite -(_unless_ these are _also_ configured to use `flock`). - -The `sqlite3_nolock` tag uses no locking at all. -Database corruption is the likely result from concurrent write access. \ No newline at end of file +It also exposes interfaces that should allow you to implement your own custom VFSes. \ No newline at end of file diff --git a/vfs/lock.go b/vfs/lock.go index a2b9651..3815fd2 100644 --- a/vfs/lock.go +++ b/vfs/lock.go @@ -1,12 +1,6 @@ -//go:build !sqlite3_nolock - package vfs -import ( - "os" - - "github.com/ncruces/go-sqlite3/internal/util" -) +import "github.com/ncruces/go-sqlite3/internal/util" const ( _PENDING_BYTE = 0x40000000 @@ -134,18 +128,3 @@ func (f *vfsFile) CheckReservedLock() (bool, error) { } return osCheckReservedLock(f.File) } - -func osGetReservedLock(file *os.File) _ErrorCode { - // Acquire the RESERVED lock. - return osWriteLock(file, _RESERVED_BYTE, 1, 0) -} - -func osGetPendingLock(file *os.File) _ErrorCode { - // Acquire the PENDING lock. - return osWriteLock(file, _PENDING_BYTE, 1, 0) -} - -func osCheckReservedLock(file *os.File) (bool, _ErrorCode) { - // Test the RESERVED lock. - return osCheckLock(file, _RESERVED_BYTE, 1) -} diff --git a/vfs/nolock.go b/vfs/nolock.go deleted file mode 100644 index a1f6236..0000000 --- a/vfs/nolock.go +++ /dev/null @@ -1,22 +0,0 @@ -//go:build sqlite3_nolock - -package vfs - -const ( - _PENDING_BYTE = 0x40000000 - _RESERVED_BYTE = (_PENDING_BYTE + 1) - _SHARED_FIRST = (_PENDING_BYTE + 2) - _SHARED_SIZE = 510 -) - -func (f *vfsFile) Lock(lock LockLevel) error { - return nil -} - -func (f *vfsFile) Unlock(lock LockLevel) error { - return nil -} - -func (f *vfsFile) CheckReservedLock() (bool, error) { - return false, nil -} diff --git a/vfs/os_bsd.go b/vfs/os_bsd.go index bcc3f61..0fccfc5 100644 --- a/vfs/os_bsd.go +++ b/vfs/os_bsd.go @@ -1,4 +1,4 @@ -//go:build (freebsd || openbsd || netbsd || dragonfly || sqlite3_flock) && !sqlite3_nolock +//go:build (freebsd || openbsd || netbsd || dragonfly || sqlite3_flock) && !sqlite3_nosys package vfs diff --git a/vfs/os_darwin.go b/vfs/os_darwin.go index c366313..1b032a8 100644 --- a/vfs/os_darwin.go +++ b/vfs/os_darwin.go @@ -1,4 +1,4 @@ -//go:build !sqlite3_flock +//go:build !sqlite3_flock && !sqlite3_nosys package vfs diff --git a/vfs/os_linux.go b/vfs/os_linux.go index 2161380..a19d29d 100644 --- a/vfs/os_linux.go +++ b/vfs/os_linux.go @@ -1,3 +1,5 @@ +//go:build !sqlite3_nosys + package vfs import ( diff --git a/vfs/os_nolock.go b/vfs/os_nolock.go index ec48008..d2beefb 100644 --- a/vfs/os_nolock.go +++ b/vfs/os_nolock.go @@ -1,28 +1,33 @@ -//go:build sqlite3_nolock && unix && !darwin +//go:build !(linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos) || sqlite3_nosys package vfs -import ( - "os" - "time" -) +import "os" -func osUnlock(file *os.File, start, len int64) _ErrorCode { - return _OK +func osGetSharedLock(file *os.File) _ErrorCode { + return _IOERR_RDLOCK } -func osLock(file *os.File, typ int16, start, len int64, timeout time.Duration, def _ErrorCode) _ErrorCode { - return _OK +func osGetReservedLock(file *os.File) _ErrorCode { + return _IOERR_LOCK } -func osReadLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode { - return _OK +func osGetPendingLock(file *os.File) _ErrorCode { + return _IOERR_LOCK } -func osWriteLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode { - return _OK +func osGetExclusiveLock(file *os.File) _ErrorCode { + return _IOERR_LOCK } -func osCheckLock(file *os.File, start, len int64) (bool, _ErrorCode) { - return false, _OK +func osDowngradeLock(file *os.File, state LockLevel) _ErrorCode { + return _IOERR_RDLOCK +} + +func osReleaseLock(file *os.File, _ LockLevel) _ErrorCode { + return _IOERR_UNLOCK +} + +func osCheckReservedLock(file *os.File) (bool, _ErrorCode) { + return false, _IOERR_CHECKRESERVEDLOCK } diff --git a/vfs/os_ofd.go b/vfs/os_ofd.go index 20aeede..d3aa3ce 100644 --- a/vfs/os_ofd.go +++ b/vfs/os_ofd.go @@ -1,4 +1,4 @@ -//go:build (linux || illumos) && !(sqlite3_flock || sqlite3_nolock) +//go:build (linux || illumos) && !sqlite3_nosys package vfs diff --git a/vfs/os_std_access.go b/vfs/os_std_access.go index d6e8fd1..b1ca611 100644 --- a/vfs/os_std_access.go +++ b/vfs/os_std_access.go @@ -1,4 +1,4 @@ -//go:build !unix +//go:build !unix || sqlite3_nosys package vfs diff --git a/vfs/os_std_alloc.go b/vfs/os_std_alloc.go index bb6f8be..60c9218 100644 --- a/vfs/os_std_alloc.go +++ b/vfs/os_std_alloc.go @@ -1,4 +1,4 @@ -//go:build !linux && (!darwin || sqlite3_flock) +//go:build !(linux || darwin) || sqlite3_flock || sqlite3_nosys package vfs diff --git a/vfs/os_std_mode.go b/vfs/os_std_mode.go index dfee7a4..ac49047 100644 --- a/vfs/os_std_mode.go +++ b/vfs/os_std_mode.go @@ -1,4 +1,4 @@ -//go:build !unix +//go:build !unix || sqlite3_nosys package vfs diff --git a/vfs/os_std_open.go b/vfs/os_std_open.go index ab2242e..9a66081 100644 --- a/vfs/os_std_open.go +++ b/vfs/os_std_open.go @@ -1,4 +1,4 @@ -//go:build !windows +//go:build !windows || sqlite3_nosys package vfs diff --git a/vfs/os_std_sync.go b/vfs/os_std_sync.go index 889e95a..6ea6aa1 100644 --- a/vfs/os_std_sync.go +++ b/vfs/os_std_sync.go @@ -1,4 +1,4 @@ -//go:build !linux && (!darwin || sqlite3_flock) +//go:build !(linux || darwin) || sqlite3_flock || sqlite3_nosys package vfs diff --git a/vfs/os_unix.go b/vfs/os_unix.go index 3a0c9c4..bf4b44e 100644 --- a/vfs/os_unix.go +++ b/vfs/os_unix.go @@ -1,11 +1,10 @@ -//go:build unix +//go:build unix && !sqlite3_nosys package vfs import ( "os" "syscall" - "time" "golang.org/x/sys/unix" ) @@ -32,60 +31,3 @@ func osSetMode(file *os.File, modeof string) error { } return nil } - -func osGetSharedLock(file *os.File) _ErrorCode { - // Test the PENDING lock before acquiring a new SHARED lock. - if pending, _ := osCheckLock(file, _PENDING_BYTE, 1); pending { - return _BUSY - } - // Acquire the SHARED lock. - return osReadLock(file, _SHARED_FIRST, _SHARED_SIZE, 0) -} - -func osGetExclusiveLock(file *os.File) _ErrorCode { - // Acquire the EXCLUSIVE lock. - return osWriteLock(file, _SHARED_FIRST, _SHARED_SIZE, time.Millisecond) -} - -func osDowngradeLock(file *os.File, state LockLevel) _ErrorCode { - if state >= LOCK_EXCLUSIVE { - // Downgrade to a SHARED lock. - if rc := osReadLock(file, _SHARED_FIRST, _SHARED_SIZE, 0); rc != _OK { - // In theory, the downgrade to a SHARED cannot fail because another - // process is holding an incompatible lock. If it does, this - // indicates that the other process is not following the locking - // protocol. If this happens, return _IOERR_RDLOCK. Returning - // BUSY would confuse the upper layer. - return _IOERR_RDLOCK - } - } - // Release the PENDING and RESERVED locks. - return osUnlock(file, _PENDING_BYTE, 2) -} - -func osReleaseLock(file *os.File, _ LockLevel) _ErrorCode { - // Release all locks. - return osUnlock(file, 0, 0) -} - -func osLockErrorCode(err error, def _ErrorCode) _ErrorCode { - if err == nil { - return _OK - } - if errno, ok := err.(unix.Errno); ok { - switch errno { - case - unix.EACCES, - unix.EAGAIN, - unix.EBUSY, - unix.EINTR, - unix.ENOLCK, - unix.EDEADLK, - unix.ETIMEDOUT: - return _BUSY - case unix.EPERM: - return _PERM - } - } - return def -} diff --git a/vfs/os_unix2.go b/vfs/os_unix2.go new file mode 100644 index 0000000..b57a032 --- /dev/null +++ b/vfs/os_unix2.go @@ -0,0 +1,82 @@ +//go:build (linux || darwin || freebsd || openbsd || netbsd || dragonfly || illumos) && !sqlite3_nosys + +package vfs + +import ( + "os" + "time" + + "golang.org/x/sys/unix" +) + +func osGetSharedLock(file *os.File) _ErrorCode { + // Test the PENDING lock before acquiring a new SHARED lock. + if pending, _ := osCheckLock(file, _PENDING_BYTE, 1); pending { + return _BUSY + } + // Acquire the SHARED lock. + return osReadLock(file, _SHARED_FIRST, _SHARED_SIZE, 0) +} + +func osGetReservedLock(file *os.File) _ErrorCode { + // Acquire the RESERVED lock. + return osWriteLock(file, _RESERVED_BYTE, 1, 0) +} + +func osGetPendingLock(file *os.File) _ErrorCode { + // Acquire the PENDING lock. + return osWriteLock(file, _PENDING_BYTE, 1, 0) +} + +func osGetExclusiveLock(file *os.File) _ErrorCode { + // Acquire the EXCLUSIVE lock. + return osWriteLock(file, _SHARED_FIRST, _SHARED_SIZE, time.Millisecond) +} + +func osDowngradeLock(file *os.File, state LockLevel) _ErrorCode { + if state >= LOCK_EXCLUSIVE { + // Downgrade to a SHARED lock. + if rc := osReadLock(file, _SHARED_FIRST, _SHARED_SIZE, 0); rc != _OK { + // In theory, the downgrade to a SHARED cannot fail because another + // process is holding an incompatible lock. If it does, this + // indicates that the other process is not following the locking + // protocol. If this happens, return _IOERR_RDLOCK. Returning + // BUSY would confuse the upper layer. + return _IOERR_RDLOCK + } + } + // Release the PENDING and RESERVED locks. + return osUnlock(file, _PENDING_BYTE, 2) +} + +func osReleaseLock(file *os.File, _ LockLevel) _ErrorCode { + // Release all locks. + return osUnlock(file, 0, 0) +} + +func osCheckReservedLock(file *os.File) (bool, _ErrorCode) { + // Test the RESERVED lock. + return osCheckLock(file, _RESERVED_BYTE, 1) +} + +func osLockErrorCode(err error, def _ErrorCode) _ErrorCode { + if err == nil { + return _OK + } + if errno, ok := err.(unix.Errno); ok { + switch errno { + case + unix.EACCES, + unix.EAGAIN, + unix.EBUSY, + unix.EINTR, + unix.ENOLCK, + unix.EDEADLK, + unix.ETIMEDOUT: + return _BUSY + case unix.EPERM: + return _PERM + } + } + return def +} diff --git a/vfs/os_windows.go b/vfs/os_windows.go index 0b2d8f2..e2e2c55 100644 --- a/vfs/os_windows.go +++ b/vfs/os_windows.go @@ -1,3 +1,5 @@ +//go:build !sqlite3_nosys + package vfs import ( @@ -39,6 +41,16 @@ func osGetSharedLock(file *os.File) _ErrorCode { return rc } +func osGetReservedLock(file *os.File) _ErrorCode { + // Acquire the RESERVED lock. + return osWriteLock(file, _RESERVED_BYTE, 1, 0) +} + +func osGetPendingLock(file *os.File) _ErrorCode { + // Acquire the PENDING lock. + return osWriteLock(file, _PENDING_BYTE, 1, 0) +} + func osGetExclusiveLock(file *os.File) _ErrorCode { // Release the SHARED lock. osUnlock(file, _SHARED_FIRST, _SHARED_SIZE) @@ -90,6 +102,11 @@ func osReleaseLock(file *os.File, state LockLevel) _ErrorCode { return _OK } +func osCheckReservedLock(file *os.File) (bool, _ErrorCode) { + // Test the RESERVED lock. + return osCheckLock(file, _RESERVED_BYTE, 1) +} + func osUnlock(file *os.File, start, len uint32) _ErrorCode { err := windows.UnlockFileEx(windows.Handle(file.Fd()), 0, len, 0, &windows.Overlapped{Offset: start})