diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 630bd83..656ea40 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -52,7 +52,8 @@ jobs: if: matrix.os == 'ubuntu-latest' - name: Test no locks - run: go test -v -tags sqlite3_nosys ./tests -run TestDB_nolock + run: go test -v -tags sqlite3_nosys ./... + if: matrix.os == 'ubuntu-latest' - name: Test GORM run: gormlite/test.sh @@ -123,21 +124,7 @@ jobs: - name: Test run: go test -v ./... - test-386: - runs-on: ubuntu-latest - needs: test - - steps: - - uses: actions/checkout@v4 - with: { lfs: 'true' } - - - uses: actions/setup-go@v5 - with: { go-version: stable } - - - name: Test - run: GOARCH=386 go test -v -short ./... - - test-arm: + test-qemu: runs-on: ubuntu-latest needs: test @@ -150,5 +137,11 @@ jobs: - uses: docker/setup-qemu-action@v3 - - name: Test + - name: Test 386 + run: GOARCH=386 go test -v -short ./... + + - name: Test arm64 run: GOARCH=arm64 go test -v -short ./... + + - name: Test riscv64 + run: GOARCH=riscv64 go test -v -short ./... diff --git a/README.md b/README.md index 967ba23..01fbaa9 100644 --- a/README.md +++ b/README.md @@ -87,9 +87,11 @@ This project aims for [high test coverage](https://github.com/ncruces/go-sqlite3 It also benefits greatly from [SQLite's](https://sqlite.org/testing.html) and [wazero's](https://tetrate.io/blog/introducing-wazero-from-tetrate/#:~:text=Rock%2Dsolid%20test%20approach) thorough testing. +Every commit is [tested](.github/workflows/test.yml) on +Linux (amd64/arm64/386/riscv64), macOS (amd64/arm64), Windows, FreeBSD and illumos. + The Go VFS is tested by running SQLite's -[mptest](https://github.com/sqlite/sqlite/blob/master/mptest/mptest.c) -on Linux, macOS, Windows, FreeBSD and illumos. +[mptest](https://github.com/sqlite/sqlite/blob/master/mptest/mptest.c). ### Performance diff --git a/conn.go b/conn.go index d3fa380..fc2d765 100644 --- a/conn.go +++ b/conn.go @@ -348,8 +348,8 @@ func (c *Conn) checkInterrupt() { } func progressCallback(ctx context.Context, mod api.Module, pDB uint32) (interrupt uint32) { - if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.commit != nil { - if c.interrupt != nil && c.interrupt.Err() != nil { + if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.interrupt != nil { + if c.interrupt.Err() != nil { interrupt = 1 } } diff --git a/driver/driver_test.go b/driver/driver_test.go index b839316..1c8c2ed 100644 --- a/driver/driver_test.go +++ b/driver/driver_test.go @@ -1,3 +1,5 @@ +//go:build !sqlite3_nosys + package driver import ( diff --git a/driver/example_test.go b/driver/example_test.go index 85352bc..161c338 100644 --- a/driver/example_test.go +++ b/driver/example_test.go @@ -1,3 +1,5 @@ +//go:build !sqlite3_nosys + package driver_test // Adapted from: https://go.dev/doc/tutorial/database-access diff --git a/embed/sqlite3.wasm b/embed/sqlite3.wasm index 2f36b29..a7057dd 100755 Binary files a/embed/sqlite3.wasm and b/embed/sqlite3.wasm differ diff --git a/ext/lines/lines_test.go b/ext/lines/lines_test.go index 0c6a525..7fe310a 100644 --- a/ext/lines/lines_test.go +++ b/ext/lines/lines_test.go @@ -59,7 +59,7 @@ func Example() { if err := rows.Err(); err != nil { log.Fatal(err) } - // Output: + // Expected output: // US: 141001 // GB: 22560 // CA: 11759 diff --git a/sqlite3/locking_mode.patch b/sqlite3/locking_mode.patch deleted file mode 100644 index ebae0eb..0000000 --- a/sqlite3/locking_mode.patch +++ /dev/null @@ -1,14 +0,0 @@ -# Use exclusive locking mode for WAL databases with v1 VFSes. ---- sqlite3.c.orig -+++ sqlite3.c -@@ -64209,7 +64209,9 @@ - SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager){ - const sqlite3_io_methods *pMethods = pPager->fd->pMethods; - if( pPager->noLock ) return 0; -- return pPager->exclusiveMode || (pMethods->iVersion>=2 && pMethods->xShmMap); -+ if( pMethods->iVersion>=2 && pMethods->xShmMap ) return 1; -+ pPager->exclusiveMode = 1; -+ return 1; - } - - /* diff --git a/tests/backup_test.go b/tests/backup_test.go index cb9ab8b..980ed4c 100644 --- a/tests/backup_test.go +++ b/tests/backup_test.go @@ -1,3 +1,5 @@ +//go:build !sqlite3_nosys + package tests import ( diff --git a/tests/bradfitz/sql_test.go b/tests/bradfitz/sql_test.go index 8c10a8b..2e0e8ae 100644 --- a/tests/bradfitz/sql_test.go +++ b/tests/bradfitz/sql_test.go @@ -1,3 +1,5 @@ +//go:build !sqlite3_nosys + package bradfitz // Adapted from: https://github.com/bradfitz/go-sql-test diff --git a/tests/db_test.go b/tests/db_test.go index 6f32fc2..ed89bf1 100644 --- a/tests/db_test.go +++ b/tests/db_test.go @@ -10,6 +10,7 @@ import ( "github.com/ncruces/go-sqlite3" _ "github.com/ncruces/go-sqlite3/embed" _ "github.com/ncruces/go-sqlite3/tests/testcfg" + "github.com/ncruces/go-sqlite3/vfs" _ "github.com/ncruces/go-sqlite3/vfs/adiantum" _ "github.com/ncruces/go-sqlite3/vfs/memdb" ) @@ -26,18 +27,19 @@ func TestDB_memory(t *testing.T) { } func TestDB_file(t *testing.T) { + if !vfs.SupportsFileLocking { + t.Skip("skipping without locks") + } + t.Parallel() testDB(t, filepath.Join(t.TempDir(), "test.db")) } -func TestDB_nolock(t *testing.T) { - t.Parallel() - testDB(t, "file:"+ - filepath.ToSlash(filepath.Join(t.TempDir(), "test.db"))+ - "?nolock=1") -} - func TestDB_wal(t *testing.T) { + if !vfs.SupportsSharedMemory { + t.Skip("skipping without shared memory") + } + t.Parallel() tmp := filepath.Join(t.TempDir(), "test.db") err := os.WriteFile(tmp, waldb, 0666) @@ -48,6 +50,10 @@ func TestDB_wal(t *testing.T) { } func TestDB_utf16(t *testing.T) { + if !vfs.SupportsFileLocking { + t.Skip("skipping without locks") + } + t.Parallel() tmp := filepath.Join(t.TempDir(), "test.db") err := os.WriteFile(tmp, utf16db, 0666) @@ -64,9 +70,15 @@ func TestDB_memdb(t *testing.T) { func TestDB_adiantum(t *testing.T) { t.Parallel() - testDB(t, "file:"+ - filepath.ToSlash(filepath.Join(t.TempDir(), "test.db"))+ - "?vfs=adiantum&textkey=correct+horse+battery+staple") + tmp := filepath.Join(t.TempDir(), "test.db") + testDB(t, "file:"+filepath.ToSlash(tmp)+"?nolock=1"+ + "&vfs=adiantum&textkey=correct+horse+battery+staple") +} + +func TestDB_nolock(t *testing.T) { + t.Parallel() + tmp := filepath.Join(t.TempDir(), "test.db") + testDB(t, "file:"+filepath.ToSlash(tmp)+"?nolock=1") } func testDB(t testing.TB, name string) { diff --git a/tests/parallel/parallel_test.go b/tests/parallel/parallel_test.go index 6c8064f..e42c13a 100644 --- a/tests/parallel/parallel_test.go +++ b/tests/parallel/parallel_test.go @@ -1,3 +1,5 @@ +//go:build !sqlite3_nosys + package tests import ( diff --git a/tests/testdata/f2fs.sh b/tests/testdata/f2fs.sh index 8ac7bb1..0802eb0 100755 --- a/tests/testdata/f2fs.sh +++ b/tests/testdata/f2fs.sh @@ -14,7 +14,7 @@ sudo mount -nv -o loop f2fs.img f2fs/ mkdir -p f2fs/tmp/ go test -c "$ROOT/tests" -coverpkg github.com/ncruces/go-sqlite3/... -TMPDIR=f2fs/tmp/ ./tests.test -test.v -test.short -test.coverprofile cover.out +TMPDIR=$PWD/f2fs/tmp/ ./tests.test -test.v -test.short -test.coverprofile cover.out go tool cover -html cover.out sudo umount f2fs/ diff --git a/tests/wal_test.go b/tests/wal_test.go index 8f1fcdf..711fc88 100644 --- a/tests/wal_test.go +++ b/tests/wal_test.go @@ -1,3 +1,5 @@ +//go:build !sqlite3_nosys + package tests import ( @@ -22,6 +24,13 @@ func TestWAL_enter_exit(t *testing.T) { } defer db.Close() + if !vfs.SupportsSharedMemory { + err = db.Exec(`PRAGMA locking_mode=EXCLUSIVE`) + if err != nil { + t.Fatal(err) + } + } + err = db.Exec(` CREATE TABLE test (col); PRAGMA journal_mode=WAL; @@ -89,6 +98,7 @@ func TestConn_WalCheckpoint(t *testing.T) { }) err = db.Exec(` + PRAGMA locking_mode=EXCLUSIVE; PRAGMA journal_mode=WAL; CREATE TABLE test (col); `) diff --git a/vfs/README.md b/vfs/README.md index 9730f14..552a93b 100644 --- a/vfs/README.md +++ b/vfs/README.md @@ -24,7 +24,7 @@ OFD locks are fully compatible with POSIX advisory locks. On BSD Unixes, this module uses [BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2). -On BSD Unixes, BSD locks are fully compatible with POSIX advisory locks. +On BSD, these locks are fully compatible with POSIX advisory locks. However, concurrency is reduced with BSD locks (`BEGIN IMMEDIATE` behaves the same as `BEGIN EXCLUSIVE`). @@ -34,8 +34,7 @@ like SQLite. On all other platforms, file locking is not supported, and you must use [`nolock=1`](https://sqlite.org/uri.html#urinolock) (or [`immutable=1`](https://sqlite.org/uri.html#uriimmutable)) -to open database files. - +to open database files.\ To use the [`database/sql`](https://pkg.go.dev/database/sql) driver with `nolock=1` you must disable connection pooling by calling [`db.SetMaxOpenConns(1)`](https://pkg.go.dev/database/sql#DB.SetMaxOpenConns). @@ -53,16 +52,16 @@ To allow `mmap` to work, each connection needs to reserve up to 4GB of address s To limit the amount of address space each connection needs, use [`WithMemoryLimitPages`](../tests/testcfg/testcfg.go). -On all other platforms, [WAL](https://sqlite.org/wal.html) support is +On Windows and BSD, [WAL](https://sqlite.org/wal.html) support is [limited](https://sqlite.org/wal.html#noshm). - -To work around this limitation, SQLite is [patched](sqlite3/locking_mode.patch) -to automatically use `EXCLUSIVE` locking mode for WAL databases on such platforms. - -To use the [`database/sql`](https://pkg.go.dev/database/sql) driver -with `EXCLUSIVE` locking mode you should disable connection pooling by calling +`EXCLUSIVE` locking mode can be set to create, read, and write WAL databases.\ +To use `EXCLUSIVE` locking mode with the +[`database/sql`](https://pkg.go.dev/database/sql) driver +you must disable connection pooling by calling [`db.SetMaxOpenConns(1)`](https://pkg.go.dev/database/sql#DB.SetMaxOpenConns). +On all other platforms, where file locking is not supported, WAL mode does not work. + You can use [`vfs.SupportsSharedMemory`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs#SupportsSharedMemory) to check if your platform supports shared memory. @@ -77,4 +76,4 @@ The VFS can be customized with a few build tags: - `sqlite3_flock` forces the use of BSD locks; it can be used on macOS to test the BSD locking implementation. - `sqlite3_nosys` prevents importing [`x/sys`](https://pkg.go.dev/golang.org/x/sys); disables locking _and_ shared memory on all platforms. -- `sqlite3_noshm` disables shared memory on all platforms. \ No newline at end of file +- `sqlite3_noshm` disables shared memory on all platforms. diff --git a/vfs/lock_test.go b/vfs/lock_test.go index 6cf359c..30abdfe 100644 --- a/vfs/lock_test.go +++ b/vfs/lock_test.go @@ -1,3 +1,5 @@ +//go:build !sqlite3_nosys + package vfs import ( diff --git a/vfs/tests/mptest/mptest_test.go b/vfs/tests/mptest/mptest_test.go index 2fca6b7..33919aa 100644 --- a/vfs/tests/mptest/mptest_test.go +++ b/vfs/tests/mptest/mptest_test.go @@ -1,3 +1,5 @@ +//go:build !sqlite3_nosys + package mptest import ( diff --git a/vfs/tests/mptest/testdata/mptest.wasm.bz2 b/vfs/tests/mptest/testdata/mptest.wasm.bz2 index 85a069f..957e694 100644 --- a/vfs/tests/mptest/testdata/mptest.wasm.bz2 +++ b/vfs/tests/mptest/testdata/mptest.wasm.bz2 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:afe1db5aea2a3ab996370fa85052cafaae10ab4b2f8154885da2c1d2a8503840 -size 470240 +oid sha256:3fc3e666b2bb3e0e27622d43ab35f802111c59155c592a7c822f09d1229e8700 +size 470668 diff --git a/vfs/tests/speedtest1/testdata/speedtest1.wasm.bz2 b/vfs/tests/speedtest1/testdata/speedtest1.wasm.bz2 index d953c0c..75b4de2 100644 --- a/vfs/tests/speedtest1/testdata/speedtest1.wasm.bz2 +++ b/vfs/tests/speedtest1/testdata/speedtest1.wasm.bz2 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e4f9ed81c9497a9d8b91517416d122ae04c2c517d3e0612d869e128dcee3fa81 -size 483519 +oid sha256:4573183e549df979c07040d92c3b2de72b5dd49ad0d8e779de63ed95fba4e898 +size 484084