Compare commits

..

50 Commits

Author SHA1 Message Date
Nuno Cruces
57daee7f59 Update README.md 2024-05-01 12:20:33 +01:00
Nuno Cruces
f976ab0dee Additional check. 2024-04-30 20:56:42 +01:00
Nuno Cruces
beba988824 Multiple fixes. 2024-04-30 01:30:39 +01:00
Nuno Cruces
992676d7ec Improved WAL API. 2024-04-27 20:55:14 +01:00
Nuno Cruces
82d8a2d796 Documentation. 2024-04-27 16:31:32 +01:00
Nuno Cruces
811e6e63be Adiantum pragmas. 2024-04-27 12:19:46 +01:00
Nuno Cruces
3c21784aee Simplify URI parameters. 2024-04-27 10:44:00 +01:00
Nuno Cruces
019246d1be Simplify mmap. 2024-04-26 16:45:32 +01:00
Nuno Cruces
fa259bdc94 Simplify _pragma. 2024-04-26 00:07:04 +01:00
Nuno Cruces
8e327a9783 VFS pragma. 2024-04-25 13:30:47 +01:00
Nuno Cruces
09a0ce04ce Test more. (#84)
Also, fix the progress callback and disable a slow example.
2024-04-24 15:49:45 +01:00
Nuno Cruces
fdb2ed0376 Fix illumos. (#83) 2024-04-24 01:07:17 +01:00
Nuno Cruces
3fb0eeec51 Filename API (#82)
Also remove VFSParams.
2024-04-23 11:43:14 +01:00
Nuno Cruces
7f6446ad31 Remove cache (side-channel for shared keys). 2024-04-23 02:25:26 +01:00
Nuno Cruces
77cdf1841f Documentation nits. 2024-04-22 15:28:19 +01:00
kim
189fbc98ac change driver name to SQLite{}, remove global variable 2024-04-22 14:02:45 +01:00
kim
d4027b0133 export the database/sql driver type and global instance 2024-04-22 14:02:45 +01:00
Nuno Cruces
62b79d2ac3 Shared memory API. 2024-04-21 12:33:38 +01:00
Nuno Cruces
07241d064a Adiantum encrypting VFS improvements. (#80)
Encrypt temporary files.
2024-04-21 01:56:38 +01:00
Nuno Cruces
2c30bc996a Don't panic. 2024-04-18 10:12:17 +01:00
Nuno Cruces
9d2194b4ea Update README.md 2024-04-18 02:13:59 +01:00
Nuno Cruces
b3a1cb3dd6 Update README.md 2024-04-18 01:42:25 +01:00
Nuno Cruces
ec1ed22149 Adiantum encrypting VFS. (#77) 2024-04-18 01:39:47 +01:00
Nuno Cruces
e86789b285 Test config. 2024-04-16 17:35:45 +01:00
Nuno Cruces
a1fcafa780 Formatting. 2024-04-16 14:02:23 +01:00
Nuno Cruces
d3f5745790 Updated dependencies. 2024-04-16 02:54:11 +01:00
Nuno Cruces
ec609ea131 F2FS. 2024-04-16 02:52:37 +01:00
Nuno Cruces
7bab8bb949 wazero 1.7.1. 2024-04-16 01:59:29 +01:00
Nuno Cruces
ce97e820d5 wasi-sdk-22. 2024-04-16 01:58:49 +01:00
Nuno Cruces
7d8249efa5 SQLite 3.45.3. 2024-04-16 01:58:48 +01:00
Nuno Cruces
d2362b0311 Use coroutines. 2024-04-16 01:05:24 +01:00
Nuno Cruces
17f1681477 Remove test. 2024-04-15 13:55:12 +01:00
Nuno Cruces
cc0b011e8d Readonly WAL. 2024-04-13 17:11:13 +01:00
dependabot[bot]
46086916d4 Bump cross-platform-actions/action from 0.23.0 to 0.24.0 (#76)
Bumps [cross-platform-actions/action](https://github.com/cross-platform-actions/action) from 0.23.0 to 0.24.0.
- [Release notes](https://github.com/cross-platform-actions/action/releases)
- [Changelog](https://github.com/cross-platform-actions/action/blob/master/changelog.md)
- [Commits](https://github.com/cross-platform-actions/action/compare/v0.23.0...v0.24.0)

---
updated-dependencies:
- dependency-name: cross-platform-actions/action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2024-04-13 00:45:41 +01:00
Nuno Cruces
4322c71a09 Fix race. 2024-04-12 16:28:53 +01:00
Nuno Cruces
da9077cbea Fix repeat runs. 2024-04-12 15:54:48 +01:00
Nuno Cruces
1c3ad12434 WAL and vacuum hooks. 2024-04-12 15:02:01 +01:00
Nuno Cruces
7260962aba Update README.md 2024-04-11 15:18:49 +01:00
Nuno Cruces
e503be641a Refactors. 2024-04-11 12:00:17 +01:00
Nuno Cruces
11c03a16f9 Implement shared memory WAL. (#71)
- enabled by default on 64-bit macOS and Linux (`amd64`/`arm64`)
- depends on merged but unreleased wazero
- may cause small performance regression
- users may need WithMemoryLimitPages if not enough address space available
- needs docs
2024-04-10 13:15:36 +01:00
dependabot[bot]
f1c376cb49 Bump golang.org/x/sync from 0.6.0 to 0.7.0 (#72)
Bumps [golang.org/x/sync](https://github.com/golang/sync) from 0.6.0 to 0.7.0.
- [Commits](https://github.com/golang/sync/compare/v0.6.0...v0.7.0)

---
updated-dependencies:
- dependency-name: golang.org/x/sync
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2024-04-04 23:52:16 +01:00
dependabot[bot]
91fd1457aa Bump golang.org/x/crypto from 0.21.0 to 0.22.0 (#74)
Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.21.0 to 0.22.0.
- [Commits](https://github.com/golang/crypto/compare/v0.21.0...v0.22.0)

---
updated-dependencies:
- dependency-name: golang.org/x/crypto
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2024-04-04 23:50:02 +01:00
Nuno Cruces
63938d5705 Simplify tests. 2024-04-04 01:25:52 +01:00
Nuno Cruces
10daa594f5 Gorm v1.25.8. 2024-03-28 00:10:07 +00:00
Nuno Cruces
2c2b6835b4 Tweaks, docs. 2024-03-27 07:54:15 +00:00
Nuno Cruces
af7fc3dcb7 Remove deprecations. 2024-03-27 07:54:08 +00:00
Nuno Cruces
0f9ce387b9 Documentation. 2024-03-22 00:21:00 +00:00
Nuno Cruces
b7d22e8fbf Fdatasync. 2024-03-21 15:04:59 +00:00
Nuno Cruces
617982f947 F2FS atomic writes. (#66)
https://sqlite.org/cgi/src/technote/714f6cbbf7
2024-03-21 13:59:47 +00:00
Nuno Cruces
36583542e1 Updated dependencies. 2024-03-17 16:34:02 +00:00
117 changed files with 2483 additions and 727 deletions

View File

@@ -1,6 +1,7 @@
#!/usr/bin/env bash
set -euo pipefail
echo 'set -euo pipefail' > test.sh
echo 'set -eu' > test.sh
for p in $(go list ./...); do
dir=".${p#github.com/ncruces/go-sqlite3}"

View File

@@ -17,7 +17,9 @@ echo aix ; GOOS=aix GOARCH=ppc64 go build .
echo js ; GOOS=js GOARCH=wasm go build .
echo wasip1 ; GOOS=wasip1 GOARCH=wasm go build .
echo darwin-flock ; GOOS=darwin GOARCH=amd64 go build -tags sqlite3_flock .
echo darwin-noshm ; GOOS=darwin GOARCH=amd64 go build -tags sqlite3_noshm .
echo darwin-nosys ; GOOS=darwin GOARCH=amd64 go build -tags sqlite3_nosys .
echo linux-noshm ; GOOS=linux GOARCH=amd64 go build -tags sqlite3_noshm .
echo linux-nosys ; GOOS=linux GOARCH=amd64 go build -tags sqlite3_nosys .
echo windows-nosys ; GOOS=windows GOARCH=amd64 go build -tags sqlite3_nosys .
echo freebsd-nosys ; GOOS=freebsd GOARCH=amd64 go build -tags sqlite3_nosys .

11
.github/workflows/illumos.sh vendored Executable file
View File

@@ -0,0 +1,11 @@
#!/usr/bin/env bash
set -euo pipefail
echo 'set -eu' > test.sh
for p in $(go list ./...); do
dir=".${p#github.com/ncruces/go-sqlite3}"
name="$(basename "$p").test"
(cd ${dir}; GOOS=illumos go test -c)
[ -f "${dir}/${name}" ] && echo "(cd ${dir}; ./${name} -test.v -test.short)" >> test.sh
done

View File

@@ -2,22 +2,22 @@
set -euo pipefail
if [[ "$OSTYPE" == "linux"* ]]; then
WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-21/wasi-sdk-21.0-linux.tar.gz"
WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-22/wasi-sdk-22.0-linux.tar.gz"
BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_117/binaryen-version_117-x86_64-linux.tar.gz"
elif [[ "$OSTYPE" == "darwin"* ]]; then
WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-21/wasi-sdk-21.0-macos.tar.gz"
WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-22/wasi-sdk-22.0-macos.tar.gz"
BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_117/binaryen-version_117-x86_64-macos.tar.gz"
elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then
WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-21/wasi-sdk-21.0.m-mingw.tar.gz"
WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-22/wasi-sdk-22.0.m-mingw.tar.gz"
BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_117/binaryen-version_117-x86_64-windows.tar.gz"
fi
# Download tools
mkdir -p tools
mkdir -p tools/
[ -d "tools/wasi-sdk"* ] || curl -#L "$WASI_SDK" | tar xzC tools &
[ -d "tools/binaryen-version"* ] || curl -#L "$BINARYEN" | tar xzC tools &
wait
sqlite3/download.sh # Download SQLite
embed/build.sh # Build WASM
embed/build.sh # Build Wasm
git diff --exit-code # Check diffs

View File

@@ -47,8 +47,13 @@ jobs:
run: go test -v -tags sqlite3_flock ./...
if: matrix.os == 'macos-latest'
- name: Test no shared memory
run: go test -v -tags sqlite3_noshm ./...
if: matrix.os == 'ubuntu-latest'
- name: Test no locks
run: go test -v -tags sqlite3_nosys ./tests -run TestDB_nolock
run: go test -v -tags sqlite3_nosys ./...
if: matrix.os == 'ubuntu-latest'
- name: Test GORM
run: gormlite/test.sh
@@ -76,14 +81,35 @@ jobs:
run: .github/workflows/bsd.sh
- name: Test
uses: cross-platform-actions/action@v0.23.0
uses: cross-platform-actions/action@v0.24.0
with:
operating_system: freebsd
version: '14.0'
shell: bash
run: source test.sh
run: . ./test.sh
sync_files: runner-to-vm
test-illumos:
runs-on: ubuntu-latest
needs: test
steps:
- uses: actions/checkout@v4
with: { lfs: 'true' }
- uses: actions/setup-go@v5
with: { go-version: stable }
- name: Build
run: .github/workflows/illumos.sh
- name: Test
uses: vmactions/omnios-vm@v1
with:
usesh: true
copyback: false
run: . ./test.sh
test-m1:
runs-on: macos-14
needs: test
@@ -98,21 +124,7 @@ jobs:
- name: Test
run: go test -v ./...
test-386:
runs-on: ubuntu-latest
needs: test
steps:
- uses: actions/checkout@v4
with: { lfs: 'true' }
- uses: actions/setup-go@v5
with: { go-version: stable }
- name: Test
run: GOARCH=386 go test -v -short ./...
test-arm:
test-qemu:
runs-on: ubuntu-latest
needs: test
@@ -125,5 +137,11 @@ jobs:
- uses: docker/setup-qemu-action@v3
- name: Test
- name: Test 386
run: GOARCH=386 go test -v -short ./...
- name: Test arm64
run: GOARCH=arm64 go test -v -short ./...
- name: Test riscv64
run: GOARCH=riscv64 go test -v -short ./...

View File

@@ -4,12 +4,13 @@
[![Go Report](https://goreportcard.com/badge/github.com/ncruces/go-sqlite3)](https://goreportcard.com/report/github.com/ncruces/go-sqlite3)
[![Go Coverage](https://github.com/ncruces/go-sqlite3/wiki/coverage.svg)](https://github.com/ncruces/go-sqlite3/wiki/Test-coverage-report)
Go module `github.com/ncruces/go-sqlite3` is `cgo`-free [SQLite](https://sqlite.org/) wrapper.\
Go module `github.com/ncruces/go-sqlite3` is a `cgo`-free [SQLite](https://sqlite.org/) wrapper.\
It provides a [`database/sql`](https://pkg.go.dev/database/sql) compatible driver,
as well as direct access to most of the [C SQLite API](https://sqlite.org/cintro.html).
It wraps a [WASM](https://webassembly.org/) build of SQLite, and uses [wazero](https://wazero.io/) as the runtime.\
Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ runtime dependencies.
It wraps a [Wasm](https://webassembly.org/) [build](embed/) of SQLite,
and uses [wazero](https://wazero.io/) as the runtime.\
Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ runtime dependencies [^1].
### Packages
@@ -54,6 +55,8 @@ Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ run
implements an in-memory VFS.
- [`github.com/ncruces/go-sqlite3/vfs/readervfs`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/readervfs)
implements a VFS for immutable databases.
- [`github.com/ncruces/go-sqlite3/vfs/adiantum`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/adiantum)
wraps a VFS to offer encryption at rest.
### Advanced features
@@ -67,54 +70,16 @@ Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ run
- [math functions](https://sqlite.org/lang_mathfunc.html)
- [full-text search](https://sqlite.org/fts5.html)
- [geospatial search](https://sqlite.org/geopoly.html)
- [encryption at rest](vfs/adiantum/README.md)
- [and more…](embed/README.md)
### Caveats
This module replaces the SQLite [OS Interface](https://sqlite.org/vfs.html)
(aka VFS) with a [pure Go](vfs/) implementation.
This has benefits, but also comes with some drawbacks.
(aka VFS) with a [pure Go](vfs/) implementation,
which has advantages and disadvantages.
#### Write-Ahead Logging
Because WASM does not support shared memory,
[WAL](https://sqlite.org/wal.html) support is [limited](https://sqlite.org/wal.html#noshm).
To work around this limitation, SQLite is [patched](sqlite3/locking_mode.patch)
to always use `EXCLUSIVE` locking mode for WAL databases.
Because connection pooling is incompatible with `EXCLUSIVE` locking mode,
to use the [`database/sql`](https://pkg.go.dev/database/sql) driver
with WAL mode databases you should disable connection pooling by calling
[`db.SetMaxOpenConns(1)`](https://pkg.go.dev/database/sql#DB.SetMaxOpenConns).
#### File Locking
POSIX advisory locks, which SQLite uses on Unix, are
[broken by design](https://sqlite.org/src/artifact/2e8b12?ln=1073-1161).
On Linux, macOS and illumos, this module uses
[OFD locks](https://www.gnu.org/software/libc/manual/html_node/Open-File-Description-Locks.html)
to synchronize access to database files.
OFD locks are fully compatible with POSIX advisory locks.
On BSD Unixes, this module uses
[BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2).
On BSD Unixes, BSD locks are fully compatible with POSIX advisory locks.
On Windows, this module uses `LockFileEx` and `UnlockFileEx`,
like SQLite.
On all other platforms, file locking is not supported, and you must use
[`nolock=1`](https://sqlite.org/uri.html#urinolock)
(or [`immutable=1`](https://sqlite.org/uri.html#uriimmutable))
to open database files.
You can use [`vfs.SupportsFileLocking`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs#SupportsFileLocking)
to check if your platform supports file locking.
To use the [`database/sql`](https://pkg.go.dev/database/sql) driver
with `nolock=1` you must disable connection pooling by calling
[`db.SetMaxOpenConns(1)`](https://pkg.go.dev/database/sql#DB.SetMaxOpenConns).
Read more about the Go VFS design [here](vfs/README.md).
### Testing
@@ -122,16 +87,17 @@ This project aims for [high test coverage](https://github.com/ncruces/go-sqlite3
It also benefits greatly from [SQLite's](https://sqlite.org/testing.html) and
[wazero's](https://tetrate.io/blog/introducing-wazero-from-tetrate/#:~:text=Rock%2Dsolid%20test%20approach) thorough testing.
The pure Go VFS is tested by running SQLite's
[mptest](https://github.com/sqlite/sqlite/blob/master/mptest/mptest.c)
on Linux, macOS, Windows and FreeBSD.
Every commit is [tested](.github/workflows/test.yml) on
Linux (amd64/arm64/386/riscv64), macOS (amd64/arm64), Windows, FreeBSD and illumos.
The Go VFS is tested by running SQLite's
[mptest](https://github.com/sqlite/sqlite/blob/master/mptest/mptest.c).
### Performance
Perfomance of the [`database/sql`](https://pkg.go.dev/database/sql) driver is
[competitive](https://github.com/cvilsmeier/go-sqlite-bench) with alternatives.
The WASM and VFS layers are also tested by running SQLite's
The Wasm and VFS layers are also tested by running SQLite's
[speedtest1](https://github.com/sqlite/sqlite/blob/master/test/speedtest1.c).
### Alternatives
@@ -140,3 +106,6 @@ The WASM and VFS layers are also tested by running SQLite's
- [`crawshaw.io/sqlite`](https://pkg.go.dev/crawshaw.io/sqlite)
- [`github.com/mattn/go-sqlite3`](https://pkg.go.dev/github.com/mattn/go-sqlite3)
- [`github.com/zombiezen/go-sqlite`](https://pkg.go.dev/github.com/zombiezen/go-sqlite)
[^1]: anything else you find in `go.mod` is either a test dependency,
or needed by one of the extensions.

View File

@@ -82,7 +82,7 @@ func (c *Conn) SetAuthorizer(cb func(action AuthorizerActionCode, name3rd, name4
}
func authorizerCallback(ctx context.Context, mod api.Module, pDB uint32, action AuthorizerActionCode, zName3rd, zName4th, zSchema, zNameInner uint32) AuthorizerReturnCode {
func authorizerCallback(ctx context.Context, mod api.Module, pDB uint32, action AuthorizerActionCode, zName3rd, zName4th, zSchema, zNameInner uint32) (rc AuthorizerReturnCode) {
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.authorizer != nil {
var name3rd, name4th, schema, nameInner string
if zName3rd != 0 {
@@ -97,7 +97,68 @@ func authorizerCallback(ctx context.Context, mod api.Module, pDB uint32, action
if zNameInner != 0 {
nameInner = util.ReadString(mod, zNameInner, _MAX_NAME)
}
return c.authorizer(action, name3rd, name4th, schema, nameInner)
rc = c.authorizer(action, name3rd, name4th, schema, nameInner)
}
return AUTH_OK
return rc
}
// WalCheckpoint checkpoints a WAL database.
//
// https://sqlite.org/c3ref/wal_checkpoint_v2.html
func (c *Conn) WalCheckpoint(schema string, mode CheckpointMode) (nLog, nCkpt int, err error) {
defer c.arena.mark()()
nLogPtr := c.arena.new(ptrlen)
nCkptPtr := c.arena.new(ptrlen)
schemaPtr := c.arena.string(schema)
r := c.call("sqlite3_wal_checkpoint_v2",
uint64(c.handle), uint64(schemaPtr), uint64(mode),
uint64(nLogPtr), uint64(nCkptPtr))
nLog = int(int32(util.ReadUint32(c.mod, nLogPtr)))
nCkpt = int(int32(util.ReadUint32(c.mod, nCkptPtr)))
return nLog, nCkpt, c.error(r)
}
// WalAutoCheckpoint configures WAL auto-checkpoints.
//
// https://sqlite.org/c3ref/wal_autocheckpoint.html
func (c *Conn) WalAutoCheckpoint(pages int) error {
r := c.call("sqlite3_wal_autocheckpoint", uint64(c.handle), uint64(pages))
return c.error(r)
}
// WalHook registers a callback function to be invoked
// each time data is committed to a database in WAL mode.
//
// https://sqlite.org/c3ref/wal_hook.html
func (c *Conn) WalHook(cb func(db *Conn, schema string, pages int) error) {
var enable uint64
if cb != nil {
enable = 1
}
c.call("sqlite3_wal_hook_go", uint64(c.handle), enable)
c.wal = cb
}
func walCallback(ctx context.Context, mod api.Module, _, pDB, zSchema uint32, pages int32) (rc uint32) {
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.wal != nil {
schema := util.ReadString(mod, zSchema, _MAX_NAME)
err := c.wal(c, schema, int(pages))
_, rc = errorCode(err, ERROR)
}
return rc
}
// AutoVacuumPages registers a autovacuum compaction amount callback.
//
// https://sqlite.org/c3ref/autovacuum_pages.html
func (c *Conn) AutoVacuumPages(cb func(schema string, dbPages, freePages, bytesPerPage uint) uint) error {
funcPtr := util.AddHandle(c.ctx, cb)
r := c.call("sqlite3_autovacuum_pages_go", uint64(c.handle), uint64(funcPtr))
return c.error(r)
}
func autoVacuumCallback(ctx context.Context, mod api.Module, pApp, zSchema, nDbPage, nFreePage, nBytePerPage uint32) uint32 {
fn := util.GetHandle(ctx, pApp).(func(schema string, dbPages, freePages, bytesPerPage uint) uint)
schema := util.ReadString(mod, zSchema, _MAX_NAME)
return uint32(fn(schema, uint(nDbPage), uint(nFreePage), uint(nBytePerPage)))
}

71
conn.go
View File

@@ -2,7 +2,6 @@ package sqlite3
import (
"context"
"errors"
"fmt"
"math"
"net/url"
@@ -10,6 +9,7 @@ import (
"time"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/vfs"
"github.com/tetratelabs/wazero/api"
)
@@ -29,6 +29,7 @@ type Conn struct {
update func(AuthorizerActionCode, string, string, int64)
commit func() bool
rollback func()
wal func(*Conn, string, int) error
arena arena
handle uint32
@@ -101,15 +102,14 @@ func (c *Conn) openDB(filename string, flags OpenFlag) (uint32, error) {
pragmas.WriteString(`;`)
}
}
pragmaPtr := c.arena.string(pragmas.String())
r := c.call("sqlite3_exec", uint64(handle), uint64(pragmaPtr), 0, 0, 0)
if err := c.sqlite.error(r, handle, pragmas.String()); err != nil {
if errors.Is(err, ERROR) {
if pragmas.Len() != 0 {
pragmaPtr := c.arena.string(pragmas.String())
r := c.call("sqlite3_exec", uint64(handle), uint64(pragmaPtr), 0, 0, 0)
if err := c.sqlite.error(r, handle, pragmas.String()); err != nil {
err = fmt.Errorf("sqlite3: invalid _pragma: %w", err)
c.closeDB(handle)
return 0, err
}
c.closeDB(handle)
return 0, err
}
}
c.call("sqlite3_progress_handler_go", uint64(handle), 100)
@@ -174,7 +174,7 @@ func (c *Conn) Prepare(sql string) (stmt *Stmt, tail string, err error) {
//
// https://sqlite.org/c3ref/prepare.html
func (c *Conn) PrepareFlags(sql string, flags PrepareFlag) (stmt *Stmt, tail string, err error) {
if len(sql) > _MAX_LENGTH {
if len(sql) > _MAX_SQL_LENGTH {
return nil, "", TOOBIG
}
@@ -215,6 +215,20 @@ func (c *Conn) DBName(n int) string {
return util.ReadString(c.mod, ptr, _MAX_NAME)
}
// Filename returns the filename for a database.
//
// https://sqlite.org/c3ref/db_filename.html
func (c *Conn) Filename(schema string) *vfs.Filename {
var ptr uint32
if schema != "" {
defer c.arena.mark()()
ptr = c.arena.string(schema)
}
r := c.call("sqlite3_db_filename", uint64(c.handle), uint64(ptr))
return vfs.OpenFilename(c.ctx, c.mod, uint32(r), vfs.OPEN_MAIN_DB)
}
// ReadOnly determines if a database is read-only.
//
// https://sqlite.org/c3ref/db_readonly.html
@@ -281,6 +295,12 @@ func (c *Conn) ReleaseMemory() error {
return c.error(r)
}
// GetInterrupt gets the context set with [Conn.SetInterrupt],
// or nil if none was set.
func (c *Conn) GetInterrupt() context.Context {
return c.interrupt
}
// SetInterrupt interrupts a long-running query when a context is done.
//
// Subsequent uses of the connection will return [INTERRUPT]
@@ -325,13 +345,13 @@ func (c *Conn) checkInterrupt() {
}
}
func progressCallback(ctx context.Context, mod api.Module, pDB uint32) uint32 {
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.commit != nil {
if c.interrupt != nil && c.interrupt.Err() != nil {
return 1
func progressCallback(ctx context.Context, mod api.Module, pDB uint32) (interrupt uint32) {
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.interrupt != nil {
if c.interrupt.Err() != nil {
interrupt = 1
}
}
return 0
return interrupt
}
// BusyTimeout sets a busy timeout.
@@ -359,28 +379,13 @@ func (c *Conn) BusyHandler(cb func(count int) (retry bool)) error {
return nil
}
func busyCallback(ctx context.Context, mod api.Module, pDB, count uint32) uint32 {
func busyCallback(ctx context.Context, mod api.Module, pDB uint32, count int32) (retry uint32) {
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.busy != nil {
if retry := c.busy(int(count)); retry {
return 1
if c.busy(int(count)) {
retry = 1
}
}
return 0
}
// Deprecated: executes a PRAGMA statement and returns results.
func (c *Conn) Pragma(str string) ([]string, error) {
stmt, _, err := c.Prepare(`PRAGMA ` + str)
if err != nil {
return nil, err
}
defer stmt.Close()
var pragmas []string
for stmt.Step() {
pragmas = append(pragmas, stmt.ColumnText(0))
}
return pragmas, stmt.Close()
return retry
}
func (c *Conn) error(rc uint64, sql ...string) error {

View File

@@ -256,41 +256,41 @@ const (
type AuthorizerActionCode uint32
const (
/************************************************ 3rd ************ 4th ***********/
CREATE_INDEX AuthorizerActionCode = 1 /* Index Name Table Name */
CREATE_TABLE AuthorizerActionCode = 2 /* Table Name NULL */
CREATE_TEMP_INDEX AuthorizerActionCode = 3 /* Index Name Table Name */
CREATE_TEMP_TABLE AuthorizerActionCode = 4 /* Table Name NULL */
CREATE_TEMP_TRIGGER AuthorizerActionCode = 5 /* Trigger Name Table Name */
CREATE_TEMP_VIEW AuthorizerActionCode = 6 /* View Name NULL */
CREATE_TRIGGER AuthorizerActionCode = 7 /* Trigger Name Table Name */
CREATE_VIEW AuthorizerActionCode = 8 /* View Name NULL */
DELETE AuthorizerActionCode = 9 /* Table Name NULL */
DROP_INDEX AuthorizerActionCode = 10 /* Index Name Table Name */
DROP_TABLE AuthorizerActionCode = 11 /* Table Name NULL */
DROP_TEMP_INDEX AuthorizerActionCode = 12 /* Index Name Table Name */
DROP_TEMP_TABLE AuthorizerActionCode = 13 /* Table Name NULL */
DROP_TEMP_TRIGGER AuthorizerActionCode = 14 /* Trigger Name Table Name */
DROP_TEMP_VIEW AuthorizerActionCode = 15 /* View Name NULL */
DROP_TRIGGER AuthorizerActionCode = 16 /* Trigger Name Table Name */
DROP_VIEW AuthorizerActionCode = 17 /* View Name NULL */
INSERT AuthorizerActionCode = 18 /* Table Name NULL */
PRAGMA AuthorizerActionCode = 19 /* Pragma Name 1st arg or NULL */
READ AuthorizerActionCode = 20 /* Table Name Column Name */
SELECT AuthorizerActionCode = 21 /* NULL NULL */
TRANSACTION AuthorizerActionCode = 22 /* Operation NULL */
UPDATE AuthorizerActionCode = 23 /* Table Name Column Name */
ATTACH AuthorizerActionCode = 24 /* Filename NULL */
DETACH AuthorizerActionCode = 25 /* Database Name NULL */
ALTER_TABLE AuthorizerActionCode = 26 /* Database Name Table Name */
REINDEX AuthorizerActionCode = 27 /* Index Name NULL */
ANALYZE AuthorizerActionCode = 28 /* Table Name NULL */
CREATE_VTABLE AuthorizerActionCode = 29 /* Table Name Module Name */
DROP_VTABLE AuthorizerActionCode = 30 /* Table Name Module Name */
FUNCTION AuthorizerActionCode = 31 /* NULL Function Name */
SAVEPOINT AuthorizerActionCode = 32 /* Operation Savepoint Name */
COPY AuthorizerActionCode = 0 /* No longer used */
RECURSIVE AuthorizerActionCode = 33 /* NULL NULL */
/***************************************************** 3rd ************ 4th ***********/
AUTH_CREATE_INDEX AuthorizerActionCode = 1 /* Index Name Table Name */
AUTH_CREATE_TABLE AuthorizerActionCode = 2 /* Table Name NULL */
AUTH_CREATE_TEMP_INDEX AuthorizerActionCode = 3 /* Index Name Table Name */
AUTH_CREATE_TEMP_TABLE AuthorizerActionCode = 4 /* Table Name NULL */
AUTH_CREATE_TEMP_TRIGGER AuthorizerActionCode = 5 /* Trigger Name Table Name */
AUTH_CREATE_TEMP_VIEW AuthorizerActionCode = 6 /* View Name NULL */
AUTH_CREATE_TRIGGER AuthorizerActionCode = 7 /* Trigger Name Table Name */
AUTH_CREATE_VIEW AuthorizerActionCode = 8 /* View Name NULL */
AUTH_DELETE AuthorizerActionCode = 9 /* Table Name NULL */
AUTH_DROP_INDEX AuthorizerActionCode = 10 /* Index Name Table Name */
AUTH_DROP_TABLE AuthorizerActionCode = 11 /* Table Name NULL */
AUTH_DROP_TEMP_INDEX AuthorizerActionCode = 12 /* Index Name Table Name */
AUTH_DROP_TEMP_TABLE AuthorizerActionCode = 13 /* Table Name NULL */
AUTH_DROP_TEMP_TRIGGER AuthorizerActionCode = 14 /* Trigger Name Table Name */
AUTH_DROP_TEMP_VIEW AuthorizerActionCode = 15 /* View Name NULL */
AUTH_DROP_TRIGGER AuthorizerActionCode = 16 /* Trigger Name Table Name */
AUTH_DROP_VIEW AuthorizerActionCode = 17 /* View Name NULL */
AUTH_INSERT AuthorizerActionCode = 18 /* Table Name NULL */
AUTH_PRAGMA AuthorizerActionCode = 19 /* Pragma Name 1st arg or NULL */
AUTH_READ AuthorizerActionCode = 20 /* Table Name Column Name */
AUTH_SELECT AuthorizerActionCode = 21 /* NULL NULL */
AUTH_TRANSACTION AuthorizerActionCode = 22 /* Operation NULL */
AUTH_UPDATE AuthorizerActionCode = 23 /* Table Name Column Name */
AUTH_ATTACH AuthorizerActionCode = 24 /* Filename NULL */
AUTH_DETACH AuthorizerActionCode = 25 /* Database Name NULL */
AUTH_ALTER_TABLE AuthorizerActionCode = 26 /* Database Name Table Name */
AUTH_REINDEX AuthorizerActionCode = 27 /* Index Name NULL */
AUTH_ANALYZE AuthorizerActionCode = 28 /* Table Name NULL */
AUTH_CREATE_VTABLE AuthorizerActionCode = 29 /* Table Name Module Name */
AUTH_DROP_VTABLE AuthorizerActionCode = 30 /* Table Name Module Name */
AUTH_FUNCTION AuthorizerActionCode = 31 /* NULL Function Name */
AUTH_SAVEPOINT AuthorizerActionCode = 32 /* Operation Savepoint Name */
AUTH_COPY AuthorizerActionCode = 0 /* No longer used */
AUTH_RECURSIVE AuthorizerActionCode = 33 /* NULL NULL */
)
// AuthorizerReturnCode are the integer codes
@@ -305,6 +305,18 @@ const (
AUTH_IGNORE AuthorizerReturnCode = 2 /* Don't allow access, but don't generate an error */
)
// CheckpointMode are all the checkpoint mode values.
//
// https://sqlite.org/c3ref/c_checkpoint_full.html
type CheckpointMode uint32
const (
CHECKPOINT_PASSIVE CheckpointMode = 0 /* Do as much as possible w/o blocking */
CHECKPOINT_FULL CheckpointMode = 1 /* Wait for writers, then checkpoint */
CHECKPOINT_RESTART CheckpointMode = 2 /* Like FULL but wait for readers */
CHECKPOINT_TRUNCATE CheckpointMode = 3 /* Like RESTART but also truncate WAL */
)
// TxnState are the allowed return values from [Conn.TxnState].
//
// https://sqlite.org/c3ref/c_txn_none.html

View File

@@ -63,39 +63,47 @@ var driverName = "sqlite3"
func init() {
if driverName != "" {
sql.Register(driverName, sqlite{})
sql.Register(driverName, &SQLite{})
}
}
// Open opens the SQLite database specified by dataSourceName as a [database/sql.DB].
//
// The init function is called by the driver on new connections.
// The conn can be used to execute queries, register functions, etc.
// Any error return closes the conn and passes the error to [database/sql].
// The [sqlite3.Conn] can be used to execute queries, register functions, etc.
// Any error returned closes the connection and is returned to [database/sql].
func Open(dataSourceName string, init func(*sqlite3.Conn) error) (*sql.DB, error) {
c, err := newConnector(dataSourceName, init)
c, err := (&SQLite{Init: init}).OpenConnector(dataSourceName)
if err != nil {
return nil, err
}
return sql.OpenDB(c), nil
}
type sqlite struct{}
// SQLite implements [database/sql/driver.Driver].
type SQLite struct {
// Init function is called by the driver on new connections.
// The [sqlite3.Conn] can be used to execute queries, register functions, etc.
// Any error returned closes the connection and is returned to [database/sql].
Init func(*sqlite3.Conn) error
}
func (sqlite) Open(name string) (driver.Conn, error) {
c, err := newConnector(name, nil)
// Open implements [database/sql/driver.Driver].
func (d *SQLite) Open(name string) (driver.Conn, error) {
c, err := d.newConnector(name)
if err != nil {
return nil, err
}
return c.Connect(context.Background())
}
func (sqlite) OpenConnector(name string) (driver.Connector, error) {
return newConnector(name, nil)
// OpenConnector implements [database/sql/driver.DriverContext].
func (d *SQLite) OpenConnector(name string) (driver.Connector, error) {
return d.newConnector(name)
}
func newConnector(name string, init func(*sqlite3.Conn) error) (*connector, error) {
c := connector{name: name, init: init}
func (d *SQLite) newConnector(name string) (*connector, error) {
c := connector{driver: d, name: name}
var txlock, timefmt string
if strings.HasPrefix(name, "file:") {
@@ -137,7 +145,7 @@ func newConnector(name string, init func(*sqlite3.Conn) error) (*connector, erro
}
type connector struct {
init func(*sqlite3.Conn) error
driver *SQLite
name string
txBegin string
tmRead sqlite3.TimeFormat
@@ -146,7 +154,7 @@ type connector struct {
}
func (n *connector) Driver() driver.Driver {
return sqlite{}
return n.driver
}
func (n *connector) Connect(ctx context.Context) (_ driver.Conn, err error) {
@@ -175,13 +183,13 @@ func (n *connector) Connect(ctx context.Context) (_ driver.Conn, err error) {
return nil, err
}
}
if n.init != nil {
err = n.init(c.Conn)
if n.driver.Init != nil {
err = n.driver.Init(c.Conn)
if err != nil {
return nil, err
}
}
if n.pragmas || n.init != nil {
if n.pragmas || n.driver.Init != nil {
s, _, err := c.Conn.Prepare(`PRAGMA query_only`)
if err != nil {
return nil, err
@@ -319,7 +327,7 @@ func (c *conn) ExecContext(ctx context.Context, query string, args []driver.Name
return newResult(c.Conn), nil
}
func (*conn) CheckNamedValue(arg *driver.NamedValue) error {
func (c *conn) CheckNamedValue(arg *driver.NamedValue) error {
return nil
}

View File

@@ -1,3 +1,5 @@
//go:build !sqlite3_nosys
package driver
import (
@@ -14,6 +16,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/internal/util"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Test_Open_dir(t *testing.T) {
@@ -153,7 +156,7 @@ func Test_BeginTx(t *testing.T) {
t.Fatal(err)
}
_, err = tx1.Exec(`CREATE TABLE IF NOT EXISTS test (col)`)
_, err = tx1.Exec(`CREATE TABLE test (col)`)
if err == nil {
t.Error("want error")
}
@@ -310,7 +313,7 @@ func Test_time(t *testing.T) {
twosday := time.Date(2022, 2, 22, 22, 22, 22, 0, time.UTC)
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS test (at DATETIME)`)
_, err = db.Exec(`CREATE TABLE test (at DATETIME)`)
if err != nil {
t.Fatal(err)
}

View File

@@ -1,3 +1,5 @@
//go:build !sqlite3_nosys
package driver_test
// Adapted from: https://go.dev/doc/tutorial/database-access

View File

@@ -18,7 +18,7 @@ func Example_json() {
defer db.Close()
_, err = db.Exec(`
CREATE TABLE IF NOT EXISTS orders (
CREATE TABLE orders (
cart_id INTEGER PRIMARY KEY,
user_id INTEGER NOT NULL,
cart TEXT

View File

@@ -6,6 +6,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/vfs/memdb"
)
@@ -16,7 +17,7 @@ func ExampleSavepoint() {
}
defer db.Close()
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS users (id INT, name VARCHAR(10))`)
_, err = db.Exec(`CREATE TABLE users (id INT, name VARCHAR(10))`)
if err != nil {
log.Fatal(err)
}

View File

@@ -1,6 +1,6 @@
# Embeddable WASM build of SQLite
# Embeddable Wasm build of SQLite
This folder includes an embeddable WASM build of SQLite 3.45.1 for use with
This folder includes an embeddable Wasm build of SQLite 3.45.3 for use with
[`github.com/ncruces/go-sqlite3`](https://pkg.go.dev/github.com/ncruces/go-sqlite3).
The following optional features are compiled in:

View File

@@ -5,10 +5,10 @@ cd -P -- "$(dirname -- "$0")"
ROOT=../
BINARYEN="$ROOT/tools/binaryen-version_117/bin"
WASI_SDK="$ROOT/tools/wasi-sdk-21.0/bin"
WASI_SDK="$ROOT/tools/wasi-sdk-22.0/bin"
"$WASI_SDK/clang" --target=wasm32-wasi -std=c17 -flto -g0 -O2 \
-Wall -Wextra -Wno-unused-parameter \
-Wall -Wextra -Wno-unused-parameter \
-o sqlite3.wasm "$ROOT/sqlite3/main.c" \
-I"$ROOT/sqlite3" \
-mexec-model=reactor \

View File

@@ -1,7 +1,9 @@
aligned_alloc
free
malloc
malloc_destructor
sqlite3_anycollseq_init
sqlite3_autovacuum_pages_go
sqlite3_backup_finish
sqlite3_backup_init
sqlite3_backup_pagecount
@@ -49,7 +51,9 @@ sqlite3_create_collation_go
sqlite3_create_function_go
sqlite3_create_module_go
sqlite3_create_window_function_go
sqlite3_database_file_object
sqlite3_db_config
sqlite3_db_filename
sqlite3_db_name
sqlite3_db_readonly
sqlite3_db_release_memory
@@ -59,6 +63,9 @@ sqlite3_errmsg
sqlite3_error_offset
sqlite3_errstr
sqlite3_exec
sqlite3_filename_database
sqlite3_filename_journal
sqlite3_filename_wal
sqlite3_finalize
sqlite3_get_autocommit
sqlite3_get_auxdata
@@ -114,4 +121,7 @@ sqlite3_vtab_in_first
sqlite3_vtab_in_next
sqlite3_vtab_nochange
sqlite3_vtab_on_conflict
sqlite3_vtab_rhs_value
sqlite3_vtab_rhs_value
sqlite3_wal_autocheckpoint
sqlite3_wal_checkpoint_v2
sqlite3_wal_hook_go

Binary file not shown.

View File

@@ -16,7 +16,7 @@ func Example() {
log.Fatal(err)
}
err = db.Exec(`CREATE TABLE IF NOT EXISTS users (id INT, name VARCHAR(10))`)
err = db.Exec(`CREATE TABLE users (id INT, name VARCHAR(10))`)
if err != nil {
log.Fatal(err)
}

View File

@@ -11,6 +11,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/array"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Example_driver() {

View File

@@ -12,6 +12,7 @@ import (
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/array"
"github.com/ncruces/go-sqlite3/ext/blobio"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/vfs/memdb"
)
@@ -27,7 +28,7 @@ func Example() {
}
defer db.Close()
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS test (col)`)
_, err = db.Exec(`CREATE TABLE test (col)`)
if err != nil {
log.Fatal(err)
}
@@ -79,8 +80,8 @@ func Test_readblob(t *testing.T) {
}
err = db.Exec(`
CREATE TABLE IF NOT EXISTS test1 (col);
CREATE TABLE IF NOT EXISTS test2 (col);
CREATE TABLE test1 (col);
CREATE TABLE test2 (col);
INSERT INTO test1 VALUES (x'cafe');
INSERT INTO test2 VALUES (x'babe');
`)
@@ -139,8 +140,8 @@ func Test_openblob(t *testing.T) {
}
err = db.Exec(`
CREATE TABLE IF NOT EXISTS test1 (col);
CREATE TABLE IF NOT EXISTS test2 (col);
CREATE TABLE test1 (col);
CREATE TABLE test2 (col);
INSERT INTO test1 VALUES (x'cafe');
INSERT INTO test2 VALUES (x'babe');
`)

View File

@@ -8,6 +8,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/csv"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Example() {
@@ -20,7 +21,7 @@ func Example() {
csv.Register(db)
err = db.Exec(`
CREATE VIRTUAL TABLE IF NOT EXISTS eurofxref USING csv(
CREATE VIRTUAL TABLE eurofxref USING csv(
filename = 'testdata/eurofxref.csv',
header = YES,
columns = 42,

68
ext/fileio/coro.go Normal file
View File

@@ -0,0 +1,68 @@
package fileio
import (
"fmt"
"github.com/ncruces/go-sqlite3/internal/util"
)
// Adapted from: https://research.swtch.com/coro
const errCoroCanceled = util.ErrorString("coroutine canceled")
func coroNew[In, Out any](f func(In, func(Out) In) Out) (resume func(In) (Out, bool), cancel func()) {
type msg[T any] struct {
panic any
val T
}
cin := make(chan msg[In])
cout := make(chan msg[Out])
running := true
resume = func(in In) (out Out, ok bool) {
if !running {
return
}
cin <- msg[In]{val: in}
m := <-cout
if m.panic != nil {
panic(m.panic)
}
return m.val, running
}
cancel = func() {
if !running {
return
}
e := fmt.Errorf("%w", errCoroCanceled)
cin <- msg[In]{panic: e}
m := <-cout
if m.panic != nil && m.panic != e {
panic(m.panic)
}
}
yield := func(out Out) In {
cout <- msg[Out]{val: out}
m := <-cin
if m.panic != nil {
panic(m.panic)
}
return m.val
}
go func() {
defer func() {
if running {
running = false
cout <- msg[Out]{panic: recover()}
}
}()
var out Out
m := <-cin
if m.panic == nil {
out = f(m.val, yield)
}
running = false
cout <- msg[Out]{val: out}
}()
return resume, cancel
}

View File

@@ -11,6 +11,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/fileio"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Test_lsmode(t *testing.T) {

View File

@@ -53,12 +53,12 @@ func (d fsdir) Open() (sqlite3.VTabCursor, error) {
type cursor struct {
fsdir
curr entry
next chan entry
done chan struct{}
base string
rowID int64
eof bool
base string
resume func(struct{}) (entry, bool)
cancel func()
curr entry
eof bool
rowID int64
}
type entry struct {
@@ -68,12 +68,8 @@ type entry struct {
}
func (c *cursor) Close() error {
if c.done != nil {
close(c.done)
s := <-c.next
c.done = nil
c.next = nil
return s.err
if c.cancel != nil {
c.cancel()
}
return nil
}
@@ -96,16 +92,25 @@ func (c *cursor) Filter(idxNum int, idxStr string, arg ...sqlite3.Value) error {
c.base = base
}
c.rowID = 0
c.resume, c.cancel = coroNew(func(_ struct{}, yield func(entry) struct{}) entry {
walkDir := func(path string, d fs.DirEntry, err error) error {
yield(entry{d, err, path})
return nil
}
if c.fsys != nil {
fs.WalkDir(c.fsys, root, walkDir)
} else {
filepath.WalkDir(root, walkDir)
}
return entry{}
})
c.eof = false
c.next = make(chan entry)
c.done = make(chan struct{})
go c.WalkDir(root)
c.rowID = 0
return c.Next()
}
func (c *cursor) Next() error {
curr, ok := <-c.next
curr, ok := c.resume(struct{}{})
c.curr = curr
c.eof = !ok
c.rowID++
@@ -165,22 +170,3 @@ func (c *cursor) Column(ctx *sqlite3.Context, n int) error {
}
return nil
}
func (c *cursor) WalkDir(path string) {
defer close(c.next)
if c.fsys != nil {
fs.WalkDir(c.fsys, path, c.WalkDirFunc)
} else {
filepath.WalkDir(path, c.WalkDirFunc)
}
}
func (c *cursor) WalkDirFunc(path string, d fs.DirEntry, err error) error {
select {
case <-c.done:
return fs.SkipAll
case c.next <- entry{d, err, path}:
return nil
}
}

View File

@@ -12,6 +12,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/fileio"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Test_fsdir(t *testing.T) {
@@ -28,7 +29,7 @@ func Test_fsdir(t *testing.T) {
}
defer db.Close()
rows, err := db.Query(`SELECT * FROM fsdir('.', '.') LIMIT 4`)
rows, err := db.Query(`SELECT * FROM fsdir('.', '.')`)
if err != nil {
t.Fatal(err)
}

View File

@@ -10,6 +10,7 @@ import (
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Test_writefile(t *testing.T) {

View File

@@ -10,6 +10,7 @@ import (
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "golang.org/x/crypto/blake2b"
_ "golang.org/x/crypto/blake2s"
_ "golang.org/x/crypto/md4"

View File

@@ -14,6 +14,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/lines"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Example() {
@@ -58,7 +59,7 @@ func Example() {
if err := rows.Err(); err != nil {
log.Fatal(err)
}
// Output:
// Expected output:
// US: 141001
// GB: 22560
// CA: 11759

View File

@@ -9,6 +9,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/pivot"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
// https://antonz.org/sqlite-pivot-table/

View File

@@ -8,6 +8,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/statement"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Example() {

View File

@@ -7,6 +7,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/stats"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestRegister_variance(t *testing.T) {
@@ -20,7 +21,7 @@ func TestRegister_variance(t *testing.T) {
stats.Register(db)
err = db.Exec(`CREATE TABLE IF NOT EXISTS data (x)`)
err = db.Exec(`CREATE TABLE data (x)`)
if err != nil {
t.Fatal(err)
}
@@ -92,7 +93,7 @@ func TestRegister_covariance(t *testing.T) {
stats.Register(db)
err = db.Exec(`CREATE TABLE IF NOT EXISTS data (y, x)`)
err = db.Exec(`CREATE TABLE data (y, x)`)
if err != nil {
t.Fatal(err)
}

View File

@@ -7,6 +7,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestRegister(t *testing.T) {
@@ -81,7 +82,7 @@ func TestRegister_collation(t *testing.T) {
Register(db)
err = db.Exec(`CREATE TABLE IF NOT EXISTS words (word VARCHAR(10))`)
err = db.Exec(`CREATE TABLE words (word VARCHAR(10))`)
if err != nil {
t.Fatal(err)
}

View File

@@ -7,6 +7,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/zorder"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestRegister_zorder(t *testing.T) {

View File

@@ -18,7 +18,7 @@ func ExampleConn_CreateCollation() {
}
defer db.Close()
err = db.Exec(`CREATE TABLE IF NOT EXISTS words (word VARCHAR(10))`)
err = db.Exec(`CREATE TABLE words (word VARCHAR(10))`)
if err != nil {
log.Fatal(err)
}
@@ -66,7 +66,7 @@ func ExampleConn_CreateFunction() {
}
defer db.Close()
err = db.Exec(`CREATE TABLE IF NOT EXISTS words (word VARCHAR(10))`)
err = db.Exec(`CREATE TABLE words (word VARCHAR(10))`)
if err != nil {
log.Fatal(err)
}
@@ -111,7 +111,7 @@ func ExampleContext_SetAuxData() {
}
defer db.Close()
err = db.Exec(`CREATE TABLE IF NOT EXISTS words (word VARCHAR(10))`)
err = db.Exec(`CREATE TABLE words (word VARCHAR(10))`)
if err != nil {
log.Fatal(err)
}

View File

@@ -16,7 +16,7 @@ func ExampleConn_CreateWindowFunction() {
}
defer db.Close()
err = db.Exec(`CREATE TABLE IF NOT EXISTS words (word VARCHAR(10))`)
err = db.Exec(`CREATE TABLE words (word VARCHAR(10))`)
if err != nil {
log.Fatal(err)
}

11
go.mod
View File

@@ -5,11 +5,14 @@ go 1.21
require (
github.com/ncruces/julianday v1.0.0
github.com/psanford/httpreadat v0.1.0
github.com/tetratelabs/wazero v1.7.0
golang.org/x/crypto v0.21.0
golang.org/x/sync v0.6.0
golang.org/x/sys v0.18.0
github.com/tetratelabs/wazero v1.7.1
golang.org/x/crypto v0.22.0
golang.org/x/sync v0.7.0
golang.org/x/sys v0.19.0
golang.org/x/text v0.14.0
lukechampine.com/adiantum v1.0.0
)
require github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect
retract v0.4.0 // tagged from the wrong branch

27
go.sum
View File

@@ -1,14 +1,25 @@
github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY=
github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/psanford/httpreadat v0.1.0 h1:VleW1HS2zO7/4c7c7zNl33fO6oYACSagjJIyMIwZLUE=
github.com/psanford/httpreadat v0.1.0/go.mod h1:Zg7P+TlBm3bYbyHTKv/EdtSJZn3qwbPwpfZ/I9GKCRE=
github.com/tetratelabs/wazero v1.7.0 h1:jg5qPydno59wqjpGrHph81lbtHzTrWzwwtD4cD88+hQ=
github.com/tetratelabs/wazero v1.7.0/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
github.com/tetratelabs/wazero v1.7.1 h1:QtSfd6KLc41DIMpDYlJdoMc6k7QTN246DM2+n2Y/Dx8=
github.com/tetratelabs/wazero v1.7.1/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
lukechampine.com/adiantum v1.0.0 h1:xxLFgKHyno8ES1XiKLbQfU9DGiMaM2xsIJI2czgT7es=
lukechampine.com/adiantum v1.0.0/go.mod h1:kjMpBiZFjVX/FeEYcN81jyt3//7J3XjJgH9OkAXV4n0=

View File

@@ -1,4 +1,8 @@
golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=

View File

@@ -3,14 +3,14 @@ module github.com/ncruces/go-sqlite3/gormlite
go 1.21
require (
github.com/ncruces/go-sqlite3 v0.12.2
gorm.io/gorm v1.25.7
github.com/ncruces/go-sqlite3 v0.14.0
gorm.io/gorm v1.25.9
)
require (
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/ncruces/julianday v1.0.0 // indirect
github.com/tetratelabs/wazero v1.7.0-pre.1 // indirect
golang.org/x/sys v0.18.0 // indirect
github.com/tetratelabs/wazero v1.7.1 // indirect
golang.org/x/sys v0.19.0 // indirect
)

View File

@@ -2,15 +2,15 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/ncruces/go-sqlite3 v0.12.2 h1:NO8lFyFTA6aUtDWviQX2Rzqi1RX3X52peWq/MLgV1Gc=
github.com/ncruces/go-sqlite3 v0.12.2/go.mod h1:+8dWcBxb2Yar4EcCwav1a21MpKZbztwOYBLSRYt9bMY=
github.com/ncruces/go-sqlite3 v0.14.0 h1:R1Jmnc7o5ECQeZPNzbLHfE8vz1DLewV+bJypHSad354=
github.com/ncruces/go-sqlite3 v0.14.0/go.mod h1:NRmOFatwnQaZq8niw0f3k/j3a0yUVt250qcVeDuyANY=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/tetratelabs/wazero v1.7.0-pre.1 h1:mOcomS6m5tz4gZgUaocVm0o64uDPPAmErJJmiOVLHvw=
github.com/tetratelabs/wazero v1.7.0-pre.1/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
github.com/tetratelabs/wazero v1.7.1 h1:QtSfd6KLc41DIMpDYlJdoMc6k7QTN246DM2+n2Y/Dx8=
github.com/tetratelabs/wazero v1.7.1/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
gorm.io/gorm v1.25.7 h1:VsD6acwRjz2zFxGO50gPO6AkNs7KKnvfzUjHQhZDz/A=
gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
gorm.io/gorm v1.25.9 h1:wct0gxZIELDk8+ZqF/MVnHLkA1rvYlBWUMv2EdsK1g8=
gorm.io/gorm v1.25.9/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=

View File

@@ -334,7 +334,7 @@ type _Index struct {
// GetIndexes return Indexes []gorm.Index and execErr error,
// See the [doc]
//
// [doc]: https://www.sqlite.org/pragma.html#pragma_index_list
// [doc]: https://sqlite.org/pragma.html#pragma_index_list
func (m _Migrator) GetIndexes(value interface{}) ([]gorm.Index, error) {
indexes := make([]gorm.Index, 0)
err := m.RunWithValue(value, func(stmt *gorm.Statement) error {

View File

@@ -9,6 +9,7 @@ import (
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestDialector(t *testing.T) {

View File

@@ -7,7 +7,7 @@ rm -rf gorm/ tests/
go work use -r .
go test
git clone --branch v1.25.7 --filter=blob:none https://github.com/go-gorm/gorm.git
git clone --branch v1.25.9 --filter=blob:none https://github.com/go-gorm/gorm.git
mv gorm/tests tests
rm -rf gorm/

71
internal/util/alloc.go Normal file
View File

@@ -0,0 +1,71 @@
//go:build unix
package util
import (
"math"
"github.com/tetratelabs/wazero/experimental"
"golang.org/x/sys/unix"
)
func mmappedAllocator(cap, max uint64) experimental.LinearMemory {
// Round up to the page size.
rnd := uint64(unix.Getpagesize() - 1)
max = (max + rnd) &^ rnd
cap = (cap + rnd) &^ rnd
if max > math.MaxInt {
// This ensures int(max) overflows to a negative value,
// and unix.Mmap returns EINVAL.
max = math.MaxUint64
}
// Reserve max bytes of address space, to ensure we won't need to move it.
// A protected, private, anonymous mapping should not commit memory.
b, err := unix.Mmap(-1, 0, int(max), unix.PROT_NONE, unix.MAP_PRIVATE|unix.MAP_ANON)
if err != nil {
panic(err)
}
// Commit the initial cap bytes of memory.
err = unix.Mprotect(b[:cap], unix.PROT_READ|unix.PROT_WRITE)
if err != nil {
unix.Munmap(b)
panic(err)
}
return &mmappedMemory{buf: b[:cap]}
}
// The slice covers the entire mmapped memory:
// - len(buf) is the already committed memory,
// - cap(buf) is the reserved address space.
type mmappedMemory struct {
buf []byte
}
func (m *mmappedMemory) Reallocate(size uint64) []byte {
if com := uint64(len(m.buf)); com < size {
// Round up to the page size.
rnd := uint64(unix.Getpagesize() - 1)
new := (size + rnd) &^ rnd
// Commit additional memory up to new bytes.
err := unix.Mprotect(m.buf[com:new], unix.PROT_READ|unix.PROT_WRITE)
if err != nil {
panic(err)
}
// Update committed memory.
m.buf = m.buf[:new]
}
// Limit returned capacity because bytes beyond
// len(m.buf) have not yet been committed.
return m.buf[:size:len(m.buf)]
}
func (m *mmappedMemory) Free() {
err := unix.Munmap(m.buf[:cap(m.buf)])
if err != nil {
panic(err)
}
m.buf = nil
}

View File

@@ -3,21 +3,11 @@ package util
import (
"context"
"io"
"github.com/tetratelabs/wazero/experimental"
)
type handleKey struct{}
type handleState struct {
handles []any
empty int
}
func NewContext(ctx context.Context) context.Context {
state := new(handleState)
ctx = experimental.WithCloseNotifier(ctx, state)
ctx = context.WithValue(ctx, handleKey{}, state)
return ctx
holes int
}
func (s *handleState) CloseNotify(ctx context.Context, exitCode uint32) {
@@ -27,14 +17,14 @@ func (s *handleState) CloseNotify(ctx context.Context, exitCode uint32) {
}
}
s.handles = nil
s.empty = 0
s.holes = 0
}
func GetHandle(ctx context.Context, id uint32) any {
if id == 0 {
return nil
}
s := ctx.Value(handleKey{}).(*handleState)
s := ctx.Value(moduleKey{}).(*moduleState)
return s.handles[^id]
}
@@ -42,10 +32,10 @@ func DelHandle(ctx context.Context, id uint32) error {
if id == 0 {
return nil
}
s := ctx.Value(handleKey{}).(*handleState)
s := ctx.Value(moduleKey{}).(*moduleState)
a := s.handles[^id]
s.handles[^id] = nil
s.empty++
s.holes++
if c, ok := a.(io.Closer); ok {
return c.Close()
}
@@ -56,13 +46,13 @@ func AddHandle(ctx context.Context, a any) (id uint32) {
if a == nil {
panic(NilErr)
}
s := ctx.Value(handleKey{}).(*handleState)
s := ctx.Value(moduleKey{}).(*moduleState)
// Find an empty slot.
if s.empty > cap(s.handles)-len(s.handles) {
if s.holes > cap(s.handles)-len(s.handles) {
for id, h := range s.handles {
if h == nil {
s.empty--
s.holes--
s.handles[id] = a
return ^uint32(id)
}

97
internal/util/mmap.go Normal file
View File

@@ -0,0 +1,97 @@
//go:build (darwin || linux) && (amd64 || arm64 || riscv64) && !(sqlite3_flock || sqlite3_noshm || sqlite3_nosys)
package util
import (
"context"
"os"
"unsafe"
"github.com/tetratelabs/wazero/api"
"github.com/tetratelabs/wazero/experimental"
"golang.org/x/sys/unix"
)
func withMmappedAllocator(ctx context.Context) context.Context {
return experimental.WithMemoryAllocator(ctx,
experimental.MemoryAllocatorFunc(mmappedAllocator))
}
type mmapState struct {
regions []*MappedRegion
}
func (s *mmapState) new(ctx context.Context, mod api.Module, size int32) *MappedRegion {
// Find unused region.
for _, r := range s.regions {
if !r.used && r.size == size {
return r
}
}
// Allocate page aligned memmory.
alloc := mod.ExportedFunction("aligned_alloc")
stack := [2]uint64{
uint64(unix.Getpagesize()),
uint64(size),
}
if err := alloc.CallWithStack(ctx, stack[:]); err != nil {
panic(err)
}
if stack[0] == 0 {
panic(OOMErr)
}
// Save the newly allocated region.
ptr := uint32(stack[0])
buf := View(mod, ptr, uint64(size))
addr := uintptr(unsafe.Pointer(&buf[0]))
s.regions = append(s.regions, &MappedRegion{
Ptr: ptr,
addr: addr,
size: size,
})
return s.regions[len(s.regions)-1]
}
type MappedRegion struct {
addr uintptr
Ptr uint32
size int32
used bool
}
func MapRegion(ctx context.Context, mod api.Module, f *os.File, offset int64, size int32, prot int) (*MappedRegion, error) {
s := ctx.Value(moduleKey{}).(*moduleState)
r := s.new(ctx, mod, size)
err := r.mmap(f, offset, prot)
if err != nil {
return nil, err
}
return r, nil
}
func (r *MappedRegion) Unmap() error {
// We can't munmap the region, otherwise it could be remaped.
// Instead, convert it to a protected, private, anonymous mapping.
// If successful, it can be reused for a subsequent mmap.
_, err := mmap(r.addr, uintptr(r.size),
unix.PROT_NONE, unix.MAP_PRIVATE|unix.MAP_ANON|unix.MAP_FIXED,
-1, 0)
r.used = err != nil
return err
}
func (r *MappedRegion) mmap(f *os.File, offset int64, prot int) error {
_, err := mmap(r.addr, uintptr(r.size),
prot, unix.MAP_SHARED|unix.MAP_FIXED,
int(f.Fd()), offset)
r.used = err == nil
return err
}
// We need the low level mmap for MAP_FIXED to work.
// Bind the syscall version hoping that it is more stable.
//go:linkname mmap syscall.mmap
func mmap(addr, length uintptr, prot, flag, fd int, pos int64) (*byte, error)

View File

@@ -0,0 +1,11 @@
//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64) || sqlite3_flock || sqlite3_noshm || sqlite3_nosys
package util
import "context"
type mmapState struct{}
func withMmappedAllocator(ctx context.Context) context.Context {
return ctx
}

21
internal/util/module.go Normal file
View File

@@ -0,0 +1,21 @@
package util
import (
"context"
"github.com/tetratelabs/wazero/experimental"
)
type moduleKey struct{}
type moduleState struct {
mmapState
handleState
}
func NewContext(ctx context.Context) context.Context {
state := new(moduleState)
ctx = withMmappedAllocator(ctx)
ctx = experimental.WithCloseNotifier(ctx, state)
ctx = context.WithValue(ctx, moduleKey{}, state)
return ctx
}

View File

@@ -15,14 +15,14 @@ import (
"github.com/tetratelabs/wazero/api"
)
// Configure SQLite WASM.
// Configure SQLite Wasm.
//
// Importing package embed initializes these
// Importing package embed initializes [Binary]
// with an appropriate build of SQLite:
//
// import _ "github.com/ncruces/go-sqlite3/embed"
var (
Binary []byte // WASM binary to load.
Binary []byte // Wasm binary to load.
Path string // Path to load the binary from.
RuntimeConfig wazero.RuntimeConfig
@@ -88,7 +88,7 @@ func instantiateSQLite() (sqlt *sqlite, err error) {
sqlt.ctx = util.NewContext(context.Background())
sqlt.mod, err = instance.runtime.InstantiateModule(sqlt.ctx,
instance.compiled, wazero.NewModuleConfig())
instance.compiled, wazero.NewModuleConfig().WithName(""))
if err != nil {
return nil, err
}
@@ -99,8 +99,8 @@ func instantiateSQLite() (sqlt *sqlite, err error) {
}
sqlt.freer = util.ReadUint32(sqlt.mod, uint32(global.Get()))
if err != nil {
return nil, err
if sqlt.freer == 0 {
return nil, util.BadBinaryErr
}
return sqlt, nil
}
@@ -274,7 +274,7 @@ func (a *arena) new(size uint64) uint32 {
}
func (a *arena) bytes(b []byte) uint32 {
if b == nil {
if (*[0]byte)(b) == nil {
return 0
}
ptr := a.new(uint64(len(b)))
@@ -294,6 +294,8 @@ func exportCallbacks(env wazero.HostModuleBuilder) wazero.HostModuleBuilder {
util.ExportFuncII(env, "go_commit_hook", commitCallback)
util.ExportFuncVI(env, "go_rollback_hook", rollbackCallback)
util.ExportFuncVIIIIJ(env, "go_update_hook", updateCallback)
util.ExportFuncIIIII(env, "go_wal_hook", walCallback)
util.ExportFuncIIIIII(env, "go_autovacuum_pages", autoVacuumCallback)
util.ExportFuncIIIIIII(env, "go_authorizer", authorizerCallback)
util.ExportFuncVIII(env, "go_log", logCallback)
util.ExportFuncVI(env, "go_destroy", destroyCallback)

View File

@@ -12,25 +12,25 @@ cat *.patch | patch --no-backup-if-mismatch
mkdir -p ext/
cd ext/
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.2/ext/misc/anycollseq.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.2/ext/misc/base64.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.2/ext/misc/decimal.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.2/ext/misc/ieee754.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.2/ext/misc/regexp.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.2/ext/misc/series.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.2/ext/misc/uint.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.2/ext/misc/uuid.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/ext/misc/anycollseq.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/ext/misc/base64.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/ext/misc/decimal.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/ext/misc/ieee754.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/ext/misc/regexp.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/ext/misc/series.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/ext/misc/uint.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/ext/misc/uuid.c"
cd ~-
cd ../vfs/tests/mptest/testdata/
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.2/mptest/mptest.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.2/mptest/config01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.2/mptest/config02.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.2/mptest/crash01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.2/mptest/crash02.subtest"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.2/mptest/multiwrite01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/mptest/mptest.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/mptest/config01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/mptest/config02.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/mptest/crash01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/mptest/crash02.subtest"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/mptest/multiwrite01.test"
cd ~-
cd ../vfs/tests/speedtest1/testdata/
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.2/test/speedtest1.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/test/speedtest1.c"
cd ~-

View File

@@ -8,12 +8,16 @@ int go_busy_handler(void *, int);
int go_commit_hook(void *);
void go_rollback_hook(void *);
void go_update_hook(void *, int, char const *, char const *, sqlite3_int64);
int go_wal_hook(void *, sqlite3 *, const char *, int);
int go_authorizer(void *, int, const char *, const char *, const char *,
const char *);
void go_log(void *, int, const char *);
unsigned int go_autovacuum_pages(void *, const char *, unsigned int,
unsigned int, unsigned int);
void sqlite3_progress_handler_go(sqlite3 *db, int n) {
sqlite3_progress_handler(db, n, go_progress_handler, /*arg=*/db);
}
@@ -34,6 +38,10 @@ void sqlite3_update_hook_go(sqlite3 *db, bool enable) {
sqlite3_update_hook(db, enable ? go_update_hook : NULL, /*arg=*/db);
}
void sqlite3_wal_hook_go(sqlite3 *db, bool enable) {
sqlite3_wal_hook(db, enable ? go_wal_hook : NULL, /*arg=*/NULL);
}
int sqlite3_set_authorizer_go(sqlite3 *db, bool enable) {
return sqlite3_set_authorizer(db, enable ? go_authorizer : NULL, /*arg=*/db);
}
@@ -41,4 +49,10 @@ int sqlite3_set_authorizer_go(sqlite3 *db, bool enable) {
int sqlite3_config_log_go(bool enable) {
return sqlite3_config(SQLITE_CONFIG_LOG, enable ? go_log : NULL,
/*arg=*/NULL);
}
int sqlite3_autovacuum_pages_go(sqlite3 *db, go_handle app) {
int rc = sqlite3_autovacuum_pages(db, go_autovacuum_pages, app, go_destroy);
if (rc) go_destroy(app);
return rc;
}

View File

@@ -1,14 +0,0 @@
# Use exclusive locking mode for WAL databases with v1 VFSes.
--- sqlite3.c.orig
+++ sqlite3.c
@@ -64209,7 +64209,9 @@
SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager){
const sqlite3_io_methods *pMethods = pPager->fd->pMethods;
if( pPager->noLock ) return 0;
- return pPager->exclusiveMode || (pMethods->iVersion>=2 && pMethods->xShmMap);
+ if( pMethods->iVersion>=2 && pMethods->xShmMap ) return 1;
+ pPager->exclusiveMode = 1;
+ return 1;
}
/*

View File

@@ -57,8 +57,8 @@
#define SQLITE_ENABLE_ATOMIC_WRITE
#define SQLITE_ENABLE_BATCH_ATOMIC_WRITE
// Because WASM does not support shared memory,
// SQLite disables WAL for WASM builds.
// Because Wasm does not support shared memory,
// SQLite disables WAL for Wasm builds.
// We patch SQLite to use exclusive locking mode instead.
// https://sqlite.org/wal.html#noshm
#undef SQLITE_OMIT_WAL

View File

@@ -5,15 +5,15 @@
#include "include.h"
#include "sqlite3.h"
int go_localtime(struct tm *, sqlite3_int64);
int go_vfs_find(const char *zVfsName);
int go_localtime(struct tm *, sqlite3_int64);
int go_randomness(sqlite3_vfs *, int nByte, char *zOut);
int go_sleep(sqlite3_vfs *, int microseconds);
int go_current_time_64(sqlite3_vfs *, sqlite3_int64 *);
int go_open(sqlite3_vfs *, sqlite3_filename zName, sqlite3_file *, int flags,
int *pOutFlags);
int *pOutFlags, int *pOutVFS);
int go_delete(sqlite3_vfs *, const char *zName, int syncDir);
int go_access(sqlite3_vfs *, const char *zName, int flags, int *pResOut);
int go_full_pathname(sqlite3_vfs *, const char *zName, int nOut, char *zOut);
@@ -32,29 +32,55 @@ int go_lock(sqlite3_file *, int eLock);
int go_unlock(sqlite3_file *, int eLock);
int go_check_reserved_lock(sqlite3_file *, int *pResOut);
int go_shm_map(sqlite3_file *, int iPg, int pgsz, int, void volatile **);
int go_shm_lock(sqlite3_file *, int offset, int n, int flags);
int go_shm_unmap(sqlite3_file *, int deleteFlag);
void go_shm_barrier(sqlite3_file *);
static int go_open_wrapper(sqlite3_vfs *vfs, sqlite3_filename zName,
sqlite3_file *file, int flags, int *pOutFlags) {
static const sqlite3_io_methods os_io = {
.iVersion = 1,
.xClose = go_close,
.xRead = go_read,
.xWrite = go_write,
.xTruncate = go_truncate,
.xSync = go_sync,
.xFileSize = go_file_size,
.xLock = go_lock,
.xUnlock = go_unlock,
.xCheckReservedLock = go_check_reserved_lock,
.xFileControl = go_file_control,
.xSectorSize = go_sector_size,
.xDeviceCharacteristics = go_device_characteristics,
};
static const sqlite3_io_methods go_io[2] = {
{
.iVersion = 1,
.xClose = go_close,
.xRead = go_read,
.xWrite = go_write,
.xTruncate = go_truncate,
.xSync = go_sync,
.xFileSize = go_file_size,
.xLock = go_lock,
.xUnlock = go_unlock,
.xCheckReservedLock = go_check_reserved_lock,
.xFileControl = go_file_control,
.xSectorSize = go_sector_size,
.xDeviceCharacteristics = go_device_characteristics,
},
{
.iVersion = 2,
.xClose = go_close,
.xRead = go_read,
.xWrite = go_write,
.xTruncate = go_truncate,
.xSync = go_sync,
.xFileSize = go_file_size,
.xLock = go_lock,
.xUnlock = go_unlock,
.xCheckReservedLock = go_check_reserved_lock,
.xFileControl = go_file_control,
.xSectorSize = go_sector_size,
.xDeviceCharacteristics = go_device_characteristics,
.xShmMap = go_shm_map,
.xShmLock = go_shm_lock,
.xShmBarrier = go_shm_barrier,
.xShmUnmap = go_shm_unmap,
}};
int vfsID = 0;
memset(file, 0, vfs->szOsFile);
int rc = go_open(vfs, zName, file, flags, pOutFlags);
int rc = go_open(vfs, zName, file, flags, pOutFlags, &vfsID);
if (rc) {
return rc;
}
file->pMethods = &os_io;
file->pMethods = &go_io[vfsID];
return SQLITE_OK;
}

View File

@@ -7,10 +7,12 @@ import (
"testing"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/tetratelabs/wazero"
)
func init() {
Path = "./embed/sqlite3.wasm"
RuntimeConfig = wazero.NewRuntimeConfig().WithMemoryLimitPages(1024)
}
func Test_sqlite_error_OOM(t *testing.T) {

16
stmt.go
View File

@@ -120,7 +120,7 @@ func (s *Stmt) Status(op StmtStatus, reset bool) int {
}
r := s.c.call("sqlite3_stmt_status", uint64(s.handle),
uint64(op), i)
return int(r)
return int(int32(r))
}
// ClearBindings resets all bindings on the prepared statement.
@@ -137,7 +137,7 @@ func (s *Stmt) ClearBindings() error {
func (s *Stmt) BindCount() int {
r := s.c.call("sqlite3_bind_parameter_count",
uint64(s.handle))
return int(r)
return int(int32(r))
}
// BindIndex returns the index of a parameter in the prepared statement
@@ -149,7 +149,7 @@ func (s *Stmt) BindIndex(name string) int {
namePtr := s.c.arena.string(name)
r := s.c.call("sqlite3_bind_parameter_index",
uint64(s.handle), uint64(namePtr))
return int(r)
return int(int32(r))
}
// BindName returns the name of a parameter in the prepared statement.
@@ -357,7 +357,7 @@ func (s *Stmt) BindValue(param int, value Value) error {
func (s *Stmt) ColumnCount() int {
r := s.c.call("sqlite3_column_count",
uint64(s.handle))
return int(r)
return int(int32(r))
}
// ColumnName returns the name of the result column.
@@ -553,6 +553,14 @@ func (s *Stmt) ColumnValue(col int) Value {
}
}
// Columns populates result columns into the provided slice.
// The slice must have [Stmt.ColumnCount] length.
//
// [INTEGER] columns will be retrieved as int64 values,
// [FLOAT] as float64, [NULL] as nil,
// [TEXT] as string, and [BLOB] as []byte.
// Any []byte are owned by SQLite and may be invalidated by
// subsequent calls to [Stmt] methods.
func (s *Stmt) Columns(dest []any) error {
defer s.c.arena.mark()()
count := uint64(len(dest))

View File

@@ -1,3 +1,5 @@
//go:build !sqlite3_nosys
package tests
import (
@@ -6,6 +8,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestBackup(t *testing.T) {
@@ -20,7 +23,7 @@ func TestBackup(t *testing.T) {
}
defer db.Close()
err = db.Exec(`CREATE TABLE IF NOT EXISTS users (id INT, name VARCHAR(10))`)
err = db.Exec(`CREATE TABLE users (id INT, name VARCHAR(10))`)
if err != nil {
t.Fatal(err)
}

View File

@@ -11,6 +11,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestBlob(t *testing.T) {
@@ -22,7 +23,7 @@ func TestBlob(t *testing.T) {
}
defer db.Close()
err = db.Exec(`CREATE TABLE IF NOT EXISTS test (col)`)
err = db.Exec(`CREATE TABLE test (col)`)
if err != nil {
t.Fatal(err)
}
@@ -97,7 +98,7 @@ func TestBlob_large(t *testing.T) {
}
defer db.Close()
err = db.Exec(`CREATE TABLE IF NOT EXISTS test (col)`)
err = db.Exec(`CREATE TABLE test (col)`)
if err != nil {
t.Fatal(err)
}
@@ -158,7 +159,7 @@ func TestBlob_overflow(t *testing.T) {
}
defer db.Close()
err = db.Exec(`CREATE TABLE IF NOT EXISTS test (col)`)
err = db.Exec(`CREATE TABLE test (col)`)
if err != nil {
t.Fatal(err)
}
@@ -217,7 +218,7 @@ func TestBlob_invalid(t *testing.T) {
}
defer db.Close()
err = db.Exec(`CREATE TABLE IF NOT EXISTS test (col)`)
err = db.Exec(`CREATE TABLE test (col)`)
if err != nil {
t.Fatal(err)
}
@@ -242,7 +243,7 @@ func TestBlob_Write_readonly(t *testing.T) {
}
defer db.Close()
err = db.Exec(`CREATE TABLE IF NOT EXISTS test (col)`)
err = db.Exec(`CREATE TABLE test (col)`)
if err != nil {
t.Fatal(err)
}
@@ -273,7 +274,7 @@ func TestBlob_Read_expired(t *testing.T) {
}
defer db.Close()
err = db.Exec(`CREATE TABLE IF NOT EXISTS test (col)`)
err = db.Exec(`CREATE TABLE test (col)`)
if err != nil {
t.Fatal(err)
}
@@ -309,7 +310,7 @@ func TestBlob_Seek(t *testing.T) {
}
defer db.Close()
err = db.Exec(`CREATE TABLE IF NOT EXISTS test (col)`)
err = db.Exec(`CREATE TABLE test (col)`)
if err != nil {
t.Fatal(err)
}
@@ -358,7 +359,7 @@ func TestBlob_Reopen(t *testing.T) {
}
defer db.Close()
err = db.Exec(`CREATE TABLE IF NOT EXISTS test (col)`)
err = db.Exec(`CREATE TABLE test (col)`)
if err != nil {
t.Fatal(err)
}

View File

@@ -1,3 +1,5 @@
//go:build !sqlite3_nosys
package bradfitz
// Adapted from: https://github.com/bradfitz/go-sql-test
@@ -12,6 +14,7 @@ import (
_ "github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
type Tester interface {

View File

@@ -6,12 +6,13 @@ import (
"math"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/vfs/memdb"
)
func TestConn_Open_dir(t *testing.T) {
@@ -111,41 +112,6 @@ func TestConn_Close_BUSY(t *testing.T) {
}
}
func TestConn_Pragma(t *testing.T) {
t.Parallel()
db, err := sqlite3.Open("file::memory:?_pragma=busy_timeout(1000)")
if err != nil {
t.Fatal(err)
}
defer db.Close()
got, err := db.Pragma("busy_timeout")
if err != nil {
t.Fatal(err)
}
want := []string{"1000"}
if !reflect.DeepEqual(got, want) {
t.Errorf("got %v, want %v", got, want)
}
var serr *sqlite3.Error
_, err = db.Pragma("+")
if err == nil {
t.Error("want: error")
}
if !errors.As(err, &serr) {
t.Fatalf("got %T, want sqlite3.Error", err)
}
if rc := serr.Code(); rc != sqlite3.ERROR {
t.Errorf("got %d, want sqlite3.ERROR", rc)
}
if got := err.Error(); got != `sqlite3: SQL logic error: near "+": syntax error` {
t.Error("got message:", got)
}
}
func TestConn_SetInterrupt(t *testing.T) {
t.Parallel()
@@ -173,7 +139,7 @@ func TestConn_SetInterrupt(t *testing.T) {
SELECT 0, 1
UNION ALL
SELECT next, curr + next FROM fibonacci
LIMIT 1e6
LIMIT 1e7
)
SELECT min(curr) FROM fibonacci
`)
@@ -450,6 +416,48 @@ func TestConn_SetLastInsertRowID(t *testing.T) {
}
}
func TestConn_Filename(t *testing.T) {
t.Parallel()
file := filepath.Join(t.TempDir(), "test.db")
db, err := sqlite3.Open(file)
if err != nil {
t.Fatal(err)
}
defer db.Close()
n := db.Filename("")
if n.String() != file {
t.Errorf("got %v", n)
}
if n.Database() != file {
t.Errorf("got %v", n)
}
if n.DatabaseFile() == nil {
t.Errorf("got %v", n)
}
n = db.Filename("xpto")
if n != nil {
t.Errorf("got %v", n)
}
if n.String() != "" {
t.Errorf("got %v", n)
}
if n.Database() != "" {
t.Errorf("got %v", n)
}
if n.Journal() != "" {
t.Errorf("got %v", n)
}
if n.WAL() != "" {
t.Errorf("got %v", n)
}
if n.DatabaseFile() != nil {
t.Errorf("got %v", n)
}
}
func TestConn_ReadOnly(t *testing.T) {
t.Parallel()
@@ -485,3 +493,35 @@ func TestConn_DBName(t *testing.T) {
t.Errorf("got %s", name)
}
}
func TestConn_AutoVacuumPages(t *testing.T) {
t.Parallel()
db, err := sqlite3.Open("file:test.db?vfs=memdb&_pragma=auto_vacuum(FULL)")
if err != nil {
t.Fatal(err)
}
defer db.Close()
err = db.AutoVacuumPages(func(schema string, dbPages, freePages, bytesPerPage uint) uint {
return freePages
})
if err != nil {
t.Fatal(err)
}
err = db.Exec(`CREATE TABLE test (col)`)
if err != nil {
t.Fatal(err)
}
err = db.Exec(`INSERT INTO test VALUES (zeroblob(1024*1024))`)
if err != nil {
t.Fatal(err)
}
err = db.Exec(`DROP TABLE test`)
if err != nil {
t.Fatal(err)
}
}

View File

@@ -9,6 +9,9 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
"github.com/ncruces/go-sqlite3/vfs"
_ "github.com/ncruces/go-sqlite3/vfs/adiantum"
_ "github.com/ncruces/go-sqlite3/vfs/memdb"
)
@@ -24,18 +27,19 @@ func TestDB_memory(t *testing.T) {
}
func TestDB_file(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
t.Parallel()
testDB(t, filepath.Join(t.TempDir(), "test.db"))
}
func TestDB_nolock(t *testing.T) {
t.Parallel()
testDB(t, "file:"+
filepath.ToSlash(filepath.Join(t.TempDir(), "test.db"))+
"?nolock=1")
}
func TestDB_wal(t *testing.T) {
if !vfs.SupportsSharedMemory {
t.Skip("skipping without shared memory")
}
t.Parallel()
tmp := filepath.Join(t.TempDir(), "test.db")
err := os.WriteFile(tmp, waldb, 0666)
@@ -46,6 +50,10 @@ func TestDB_wal(t *testing.T) {
}
func TestDB_utf16(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
t.Parallel()
tmp := filepath.Join(t.TempDir(), "test.db")
err := os.WriteFile(tmp, utf16db, 0666)
@@ -55,10 +63,24 @@ func TestDB_utf16(t *testing.T) {
testDB(t, tmp)
}
func TestDB_vfs(t *testing.T) {
func TestDB_memdb(t *testing.T) {
t.Parallel()
testDB(t, "file:test.db?vfs=memdb")
}
func TestDB_adiantum(t *testing.T) {
t.Parallel()
tmp := filepath.Join(t.TempDir(), "test.db")
testDB(t, "file:"+filepath.ToSlash(tmp)+"?nolock=1"+
"&vfs=adiantum&textkey=correct+horse+battery+staple")
}
func TestDB_nolock(t *testing.T) {
t.Parallel()
tmp := filepath.Join(t.TempDir(), "test.db")
testDB(t, "file:"+filepath.ToSlash(tmp)+"?nolock=1")
}
func testDB(t testing.TB, name string) {
db, err := sqlite3.Open(name)
if err != nil {

View File

@@ -6,6 +6,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestDriver(t *testing.T) {
@@ -27,7 +28,7 @@ func TestDriver(t *testing.T) {
defer conn.Close()
res, err := conn.ExecContext(ctx,
`CREATE TABLE IF NOT EXISTS users (id INT, name VARCHAR(10))`)
`CREATE TABLE users (id INT, name VARCHAR(10))`)
if err != nil {
t.Fatal(err)
}

View File

@@ -5,6 +5,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Test_base64(t *testing.T) {

View File

@@ -6,6 +6,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestCreateFunction(t *testing.T) {
@@ -196,7 +197,7 @@ func TestAnyCollationNeeded(t *testing.T) {
}
defer db.Close()
err = db.Exec(`CREATE TABLE IF NOT EXISTS users (id INT, name VARCHAR(10))`)
err = db.Exec(`CREATE TABLE users (id INT, name VARCHAR(10))`)
if err != nil {
t.Fatal(err)
}

View File

@@ -10,6 +10,7 @@ import (
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
"github.com/ncruces/julianday"
)
@@ -31,7 +32,7 @@ func TestJSON(t *testing.T) {
}
defer conn.Close()
_, err = conn.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS test (col)`)
_, err = conn.ExecContext(ctx, `CREATE TABLE test (col)`)
if err != nil {
t.Fatal(err)
}

View File

@@ -1,3 +1,5 @@
//go:build !sqlite3_nosys
package tests
import (
@@ -12,10 +14,13 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
"github.com/ncruces/go-sqlite3/vfs"
_ "github.com/ncruces/go-sqlite3/vfs/adiantum"
"github.com/ncruces/go-sqlite3/vfs/memdb"
)
func TestParallel(t *testing.T) {
func Test_parallel(t *testing.T) {
var iter int
if testing.Short() {
iter = 1000
@@ -32,7 +37,21 @@ func TestParallel(t *testing.T) {
testIntegrity(t, name)
}
func TestMemory(t *testing.T) {
func Test_wal(t *testing.T) {
if !vfs.SupportsSharedMemory {
t.Skip("skipping without shared memory")
}
name := "file:" +
filepath.Join(t.TempDir(), "test.db") +
"?_pragma=busy_timeout(10000)" +
"&_pragma=journal_mode(wal)" +
"&_pragma=synchronous(off)"
testParallel(t, name, 1000)
testIntegrity(t, name)
}
func Test_memdb(t *testing.T) {
var iter int
if testing.Short() {
iter = 1000
@@ -45,6 +64,22 @@ func TestMemory(t *testing.T) {
testIntegrity(t, name)
}
func Test_adiantum(t *testing.T) {
var iter int
if testing.Short() {
iter = 1000
} else {
iter = 5000
}
name := "file:" +
filepath.ToSlash(filepath.Join(t.TempDir(), "test.db")) +
"?vfs=adiantum" +
"&_pragma=hexkey(e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855)"
testParallel(t, name, iter)
testIntegrity(t, name)
}
func TestMultiProcess(t *testing.T) {
if testing.Short() {
t.Skip("skipping in short mode")
@@ -96,7 +131,7 @@ func TestChildProcess(t *testing.T) {
testParallel(t, name, 1000)
}
func BenchmarkMemory(b *testing.B) {
func Benchmark_memdb(b *testing.B) {
memdb.Delete("test.db")
name := "file:/test.db?vfs=memdb"
testParallel(b, name, b.N)

View File

@@ -8,6 +8,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestStmt(t *testing.T) {
@@ -19,7 +20,7 @@ func TestStmt(t *testing.T) {
}
defer db.Close()
err = db.Exec(`CREATE TABLE IF NOT EXISTS test (col)`)
err = db.Exec(`CREATE TABLE test (col)`)
if err != nil {
t.Fatal(err)
}

10
tests/testcfg/testcfg.go Normal file
View File

@@ -0,0 +1,10 @@
package testcfg
import (
"github.com/ncruces/go-sqlite3"
"github.com/tetratelabs/wazero"
)
func init() {
sqlite3.RuntimeConfig = wazero.NewRuntimeConfig().WithMemoryLimitPages(1024)
}

BIN
tests/testdata/f2fs.img.gz vendored Normal file

Binary file not shown.

21
tests/testdata/f2fs.sh vendored Executable file
View File

@@ -0,0 +1,21 @@
#!/usr/bin/env bash
set -euo pipefail
cd -P -- "$(dirname -- "$0")"
ROOT=../../
if mountpoint -q f2fs/; then
sudo umount f2fs/
fi
mkdir -p f2fs/
gunzip -c f2fs.img.gz > f2fs.img
sudo mount -nv -o loop f2fs.img f2fs/
mkdir -p f2fs/tmp/
go test -c "$ROOT/tests" -coverpkg github.com/ncruces/go-sqlite3/...
TMPDIR=$PWD/f2fs/tmp/ ./tests.test -test.v -test.short -test.coverprofile cover.out
go tool cover -html cover.out
sudo umount f2fs/
rm -r f2fs/ f2fs.img cover.out *.test

View File

@@ -10,6 +10,7 @@ import (
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestTimeFormat_Encode(t *testing.T) {
@@ -146,8 +147,7 @@ func TestTimeFormat_Scanner(t *testing.T) {
}
defer conn.Close()
_, err = conn.ExecContext(ctx,
`CREATE TABLE IF NOT EXISTS test (col)`)
_, err = conn.ExecContext(ctx, `CREATE TABLE test (col)`)
if err != nil {
t.Fatal(err)
}
@@ -178,7 +178,7 @@ func TestDB_timeCollation(t *testing.T) {
}
defer db.Close()
err = db.Exec(`CREATE TABLE IF NOT EXISTS times (tstamp COLLATE TIME)`)
err = db.Exec(`CREATE TABLE times (tstamp COLLATE TIME)`)
if err != nil {
t.Fatal(err)
}

View File

@@ -7,6 +7,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestConn_Transaction_exec(t *testing.T) {
@@ -22,7 +23,7 @@ func TestConn_Transaction_exec(t *testing.T) {
db.CommitHook(func() bool { return true })
db.UpdateHook(func(sqlite3.AuthorizerActionCode, string, string, int64) {})
err = db.Exec(`CREATE TABLE IF NOT EXISTS test (col)`)
err = db.Exec(`CREATE TABLE test (col)`)
if err != nil {
t.Fatal(err)
}
@@ -95,7 +96,7 @@ func TestConn_Transaction_panic(t *testing.T) {
}
defer db.Close()
err = db.Exec(`CREATE TABLE IF NOT EXISTS test (col)`)
err = db.Exec(`CREATE TABLE test (col)`)
if err != nil {
t.Fatal(err)
}
@@ -153,7 +154,7 @@ func TestConn_Transaction_interrupt(t *testing.T) {
}
defer db.Close()
err = db.Exec(`CREATE TABLE IF NOT EXISTS test (col)`)
err = db.Exec(`CREATE TABLE test (col)`)
if err != nil {
t.Fatal(err)
}
@@ -255,7 +256,7 @@ func TestConn_Transaction_rollback(t *testing.T) {
}
defer db.Close()
err = db.Exec(`CREATE TABLE IF NOT EXISTS test (col)`)
err = db.Exec(`CREATE TABLE test (col)`)
if err != nil {
t.Fatal(err)
}
@@ -301,7 +302,7 @@ func TestConn_Savepoint_exec(t *testing.T) {
}
defer db.Close()
err = db.Exec(`CREATE TABLE IF NOT EXISTS test (col)`)
err = db.Exec(`CREATE TABLE test (col)`)
if err != nil {
t.Fatal(err)
}
@@ -369,7 +370,7 @@ func TestConn_Savepoint_panic(t *testing.T) {
}
defer db.Close()
err = db.Exec(`CREATE TABLE IF NOT EXISTS test (col)`)
err = db.Exec(`CREATE TABLE test (col)`)
if err != nil {
t.Fatal(err)
}
@@ -426,7 +427,7 @@ func TestConn_Savepoint_interrupt(t *testing.T) {
}
defer db.Close()
err = db.Exec(`CREATE TABLE IF NOT EXISTS test (col)`)
err = db.Exec(`CREATE TABLE test (col)`)
if err != nil {
t.Fatal(err)
}
@@ -506,7 +507,7 @@ func TestConn_Savepoint_rollback(t *testing.T) {
}
defer db.Close()
err = db.Exec(`CREATE TABLE IF NOT EXISTS test (col)`)
err = db.Exec(`CREATE TABLE test (col)`)
if err != nil {
t.Fatal(err)
}

View File

@@ -6,6 +6,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
"github.com/ncruces/go-sqlite3/vfs/memdb"
"github.com/ncruces/go-sqlite3/vfs/readervfs"
)

108
tests/wal_test.go Normal file
View File

@@ -0,0 +1,108 @@
//go:build !sqlite3_nosys
package tests
import (
"os"
"path/filepath"
"testing"
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
"github.com/ncruces/go-sqlite3/vfs"
)
func TestWAL_enter_exit(t *testing.T) {
t.Parallel()
file := filepath.Join(t.TempDir(), "test.db")
db, err := sqlite3.Open(file)
if err != nil {
t.Fatal(err)
}
defer db.Close()
if !vfs.SupportsSharedMemory {
err = db.Exec(`PRAGMA locking_mode=EXCLUSIVE`)
if err != nil {
t.Fatal(err)
}
}
err = db.Exec(`
CREATE TABLE test (col);
PRAGMA journal_mode=WAL;
SELECT * FROM test;
PRAGMA journal_mode=DELETE;
SELECT * FROM test;
PRAGMA journal_mode=WAL;
SELECT * FROM test;
`)
if err != nil {
t.Fatal(err)
}
}
func TestWAL_readonly(t *testing.T) {
if !vfs.SupportsSharedMemory {
t.Skip("skipping without shared memory")
}
t.Parallel()
tmp := filepath.Join(t.TempDir(), "test.db")
err := os.WriteFile(tmp, waldb, 0666)
if err != nil {
t.Fatal(err)
}
db, err := sqlite3.OpenFlags(tmp, sqlite3.OPEN_READONLY)
if err != nil {
t.Fatal(err)
}
defer db.Close()
stmt, _, err := db.Prepare(`SELECT * FROM sqlite_master`)
if err != nil {
t.Fatal(err)
}
defer stmt.Close()
if stmt.Step() {
t.Error("want no rows")
}
}
func TestConn_WalCheckpoint(t *testing.T) {
t.Parallel()
file := filepath.Join(t.TempDir(), "test.db")
db, err := sqlite3.Open(file)
if err != nil {
t.Fatal(err)
}
defer db.Close()
err = db.WalAutoCheckpoint(1000)
if err != nil {
t.Fatal(err)
}
db.WalHook(func(db *sqlite3.Conn, schema string, pages int) error {
log, ckpt, err := db.WalCheckpoint(schema, sqlite3.CHECKPOINT_FULL)
t.Log(log, ckpt, err)
return err
})
err = db.Exec(`
PRAGMA locking_mode=EXCLUSIVE;
PRAGMA journal_mode=WAL;
CREATE TABLE test (col);
`)
if err != nil {
t.Fatal(err)
}
}

13
txn.go
View File

@@ -230,9 +230,6 @@ func (c *Conn) TxnState(schema string) TxnState {
return TxnState(r)
}
// Deprecated: renamed for consistency with [Conn.TxnState].
type Tx = Txn
// CommitHook registers a callback function to be invoked
// whenever a transaction is committed.
// Return true to allow the commit operation to continue normally.
@@ -260,7 +257,7 @@ func (c *Conn) RollbackHook(cb func()) {
c.rollback = cb
}
// RollbackHook registers a callback function to be invoked
// UpdateHook registers a callback function to be invoked
// whenever a row is updated, inserted or deleted in a rowid table.
//
// https://sqlite.org/c3ref/update_hook.html
@@ -273,13 +270,13 @@ func (c *Conn) UpdateHook(cb func(action AuthorizerActionCode, schema, table str
c.update = cb
}
func commitCallback(ctx context.Context, mod api.Module, pDB uint32) uint32 {
func commitCallback(ctx context.Context, mod api.Module, pDB uint32) (rollback uint32) {
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.commit != nil {
if ok := c.commit(); !ok {
return 1
if !c.commit() {
rollback = 1
}
}
return 0
return rollback
}
func rollbackCallback(ctx context.Context, mod api.Module, pDB uint32) {

View File

@@ -1,3 +1,4 @@
// Package fsutil implements filesystem utility functions.
package fsutil
import (

View File

@@ -8,8 +8,9 @@ import (
// SeekingReaderAt implements [io.ReaderAt]
// through an underlying [io.ReadSeeker].
type SeekingReaderAt struct {
l sync.Mutex
// +checklocks:l
r io.ReadSeeker
l sync.Mutex
}
// NewSeekingReaderAt creates a new SeekingReaderAt.

View File

@@ -2,6 +2,78 @@
This package implements the SQLite [OS Interface](https://sqlite.org/vfs.html) (aka VFS).
It replaces the default SQLite VFS with a pure Go implementation.
It replaces the default SQLite VFS with a **pure Go** implementation.
It also exposes interfaces that should allow you to implement your own custom VFSes.
It also exposes [interfaces](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs#VFS)
that should allow you to implement your own custom VFSes.
Since it is a from scratch reimplementation,
there are naturally some ways it deviates from the original.
The main differences are [file locking](#file-locking) and [WAL mode](write-ahead-logging) support.
### File Locking
POSIX advisory locks, which SQLite uses on Unix, are
[broken by design](https://github.com/sqlite/sqlite/blob/b74eb0/src/os_unix.c#L1073-L1161).
On Linux and macOS, this module uses
[OFD locks](https://www.gnu.org/software/libc/manual/html_node/Open-File-Description-Locks.html)
to synchronize access to database files.
OFD locks are fully compatible with POSIX advisory locks.
On BSD Unixes, this module uses
[BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2).
On BSD, these locks are fully compatible with POSIX advisory locks.
However, concurrency is reduced with BSD locks
(`BEGIN IMMEDIATE` behaves the same as `BEGIN EXCLUSIVE`).
On Windows, this module uses `LockFileEx` and `UnlockFileEx`,
like SQLite.
On all other platforms, file locking is not supported, and you must use
[`nolock=1`](https://sqlite.org/uri.html#urinolock)
(or [`immutable=1`](https://sqlite.org/uri.html#uriimmutable))
to open database files.\
To use the [`database/sql`](https://pkg.go.dev/database/sql) driver
with `nolock=1` you must disable connection pooling by calling
[`db.SetMaxOpenConns(1)`](https://pkg.go.dev/database/sql#DB.SetMaxOpenConns).
You can use [`vfs.SupportsFileLocking`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs#SupportsFileLocking)
to check if your platform supports file locking.
### Write-Ahead Logging
On 64-bit Linux and macOS, this module uses `mmap` to implement
[shared-memory for the WAL-index](https://sqlite.org/wal.html#implementation_of_shared_memory_for_the_wal_index),
like SQLite.
To allow `mmap` to work, each connection needs to reserve up to 4GB of address space.\
To limit the amount of address space each connection needs,
use [`WithMemoryLimitPages`](../tests/testcfg/testcfg.go).
On Windows and BSD, [WAL](https://sqlite.org/wal.html) support is
[limited](https://sqlite.org/wal.html#noshm).
`EXCLUSIVE` locking mode can be set to create, read, and write WAL databases.\
To use `EXCLUSIVE` locking mode with the
[`database/sql`](https://pkg.go.dev/database/sql) driver
you must disable connection pooling by calling
[`db.SetMaxOpenConns(1)`](https://pkg.go.dev/database/sql#DB.SetMaxOpenConns).
On all other platforms, where file locking is not supported, WAL mode does not work.
You can use [`vfs.SupportsSharedMemory`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs#SupportsSharedMemory)
to check if your platform supports shared memory.
### Batch-Atomic Write
On 64-bit Linux, this module supports [batch-atomic writes](https://sqlite.org/cgi/src/technote/714)
with the F2FS filesystem.
### Build tags
The VFS can be customized with a few build tags:
- `sqlite3_flock` forces the use of BSD locks; it can be used on macOS to test the BSD locking implementation.
- `sqlite3_nosys` prevents importing [`x/sys`](https://pkg.go.dev/golang.org/x/sys);
disables locking _and_ shared memory on all platforms.
- `sqlite3_noshm` disables shared memory on all platforms.

64
vfs/adiantum/README.md Normal file
View File

@@ -0,0 +1,64 @@
# Go `"adiantum"` SQLite VFS
This package wraps an SQLite VFS to offer encryption at rest.
> [!WARNING]
> This work was not certified by a cryptographer.
> If you need vetted encryption, you should purchase the
> [SQLite Encryption Extension](https://sqlite.org/see),
> and either wrap it, or seek assistance wrapping it.
The `"adiantum"` VFS wraps the default SQLite VFS using the
[Adiantum](https://github.com/lukechampine/adiantum)
tweakable and length-preserving encryption.\
In general, any HBSH construction can be used to wrap any VFS.
The default Adiantum construction uses XChaCha12 for its stream cipher,
AES for its block cipher, and NH and Poly1305 for hashing.\
Additionally, we use [Argon2id](https://pkg.go.dev/golang.org/x/crypto/argon2#hdr-Argon2id)
to derive 256-bit keys from plain text where needed.
File contents are encrypted in 4K blocks, matching the
[default](https://sqlite.org/pgszchng2016.html) SQLite page size.
The VFS encrypts all files _except_
[super journals](https://sqlite.org/tempfiles.html#super_journal_files):
these _never_ contain database data, only filenames,
and padding them to the block size is problematic.
Temporary files _are_ encrypted with **random** keys,
as they _may_ contain database data.
To avoid the overhead of encrypting temporary files,
keep them in memory:
PRAGMA temp_store = memory;
> [!IMPORTANT]
> Adiantum is a cipher composition for disk encryption.
> The standard threat model for disk encryption considers an adversary
> that can read multiple snapshots of a disk.
> The only security property that disk encryption provides
> is that all information such an adversary can obtain
> is whether the data in a sector has or has not changed over time.
The encryption offered by this package is fully deterministic.
This means that an adversary who can get ahold of multiple snapshots
(e.g. backups) of a database file can learn precisely:
which blocks changed, which ones didn't, which got reverted.
This is slightly weaker than other forms of SQLite encryption
that include *some* nondeterminism; with limited nondeterminism,
an adversary can't distinguish between
blocks that actually changed, and blocks that got reverted.
> [!CAUTION]
> This package does not claim protect databases against tampering or forgery.
The major practical consequence of the above point is that,
if you're keeping `"adiantum"` encrypted backups of your database,
and want to protect against forgery, you should sign your backups,
and verify signatures before restoring them.
This is slightly weaker than other forms of SQLite encryption
that include block-level [MACs](https://en.wikipedia.org/wiki/Message_authentication_code).
Block-level MACs can protect against forging individual blocks,
but can't prevent them from being reverted to former versions of themselves.

29
vfs/adiantum/adiantum.go Normal file
View File

@@ -0,0 +1,29 @@
package adiantum
import (
"crypto/rand"
"golang.org/x/crypto/argon2"
"lukechampine.com/adiantum"
"lukechampine.com/adiantum/hbsh"
)
const pepper = "github.com/ncruces/go-sqlite3/vfs/adiantum"
type adiantumCreator struct{}
func (adiantumCreator) HBSH(key []byte) *hbsh.HBSH {
if len(key) != 32 {
return nil
}
return adiantum.New(key)
}
func (adiantumCreator) KDF(text string) []byte {
if text == "" {
key := make([]byte, 32)
n, _ := rand.Read(key)
return key[:n]
}
return argon2.IDKey([]byte(text), []byte(pepper), 1, 64*1024, 4, 32)
}

62
vfs/adiantum/api.go Normal file
View File

@@ -0,0 +1,62 @@
// Package adiantum wraps an SQLite VFS to offer encryption at rest.
//
// The "adiantum" [vfs.VFS] wraps the default VFS using the
// Adiantum tweakable, length-preserving encryption.
//
// Importing package adiantum registers that VFS:
//
// import _ "github.com/ncruces/go-sqlite3/vfs/adiantum"
//
// To open an encrypted database you need to provide key material.
//
// The simplest way to do that is to specify the key through an [URI] parameter:
//
// - key: key material in binary (32 bytes)
// - hexkey: key material in hex (64 hex digits)
// - textkey: key material in text (any length)
//
// However, this makes your key easily accessible to other parts of
// your application (e.g. through [vfs.Filename.URIParameters]).
//
// To avoid this, use any of the following PRAGMAs:
//
// PRAGMA key='D41d8cD98f00b204e9800998eCf8427e';
// PRAGMA hexkey='e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855';
// PRAGMA textkey='your-secret-key';
//
// [URI]: https://sqlite.org/uri.html
package adiantum
import (
"github.com/ncruces/go-sqlite3/vfs"
"lukechampine.com/adiantum/hbsh"
)
func init() {
Register("adiantum", vfs.Find(""), nil)
}
// Register registers an encrypting VFS, wrapping a base VFS,
// and possibly using a custom HBSH cipher construction.
// To use the default Adiantum construction, set cipher to nil.
func Register(name string, base vfs.VFS, cipher HBSHCreator) {
if cipher == nil {
cipher = adiantumCreator{}
}
vfs.Register(name, &hbshVFS{
VFS: base,
hbsh: cipher,
})
}
// HBSHCreator creates an [hbsh.HBSH] cipher
// given key material.
type HBSHCreator interface {
// KDF derives an HBSH key from a secret.
// If no secret is given, a random key is generated.
KDF(secret string) (key []byte)
// HBSH creates an HBSH cipher given a key.
// If key is not appropriate, nil is returned.
HBSH(key []byte) *hbsh.HBSH
}

277
vfs/adiantum/hbsh.go Normal file
View File

@@ -0,0 +1,277 @@
package adiantum
import (
"encoding/binary"
"encoding/hex"
"io"
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/vfs"
"lukechampine.com/adiantum/hbsh"
)
type hbshVFS struct {
vfs.VFS
hbsh HBSHCreator
}
func (h *hbshVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
return nil, 0, sqlite3.CANTOPEN
}
func (h *hbshVFS) OpenFilename(name *vfs.Filename, flags vfs.OpenFlag) (file vfs.File, _ vfs.OpenFlag, err error) {
if h, ok := h.VFS.(vfs.VFSFilename); ok {
file, flags, err = h.OpenFilename(name, flags)
} else {
file, flags, err = h.Open(name.String(), flags)
}
// Encrypt everything except super journals and memory files.
if err != nil || flags&(vfs.OPEN_SUPER_JOURNAL|vfs.OPEN_MEMORY) != 0 {
return file, flags, err
}
var hbsh *hbsh.HBSH
if f, ok := name.DatabaseFile().(*hbshFile); ok {
hbsh = f.hbsh
} else {
var key []byte
if params := name.URIParameters(); name == nil {
key = h.hbsh.KDF("") // Temporary files get a random key.
} else if t, ok := params["key"]; ok {
key = []byte(t[0])
} else if t, ok := params["hexkey"]; ok {
key, _ = hex.DecodeString(t[0])
} else if t, ok := params["textkey"]; ok {
key = h.hbsh.KDF(t[0])
}
hbsh = h.hbsh.HBSH(key)
}
// Main datatabases may have their key specified later, as a PRAGMA.
if hbsh != nil || flags&vfs.OPEN_MAIN_DB != 0 {
return &hbshFile{File: file, hbsh: hbsh, reset: h.hbsh}, flags, nil
}
return nil, flags, sqlite3.CANTOPEN
}
const (
tweakSize = 8
blockSize = 4096
)
type hbshFile struct {
vfs.File
hbsh *hbsh.HBSH
reset HBSHCreator
tweak [tweakSize]byte
block [blockSize]byte
}
func (h *hbshFile) Pragma(name string, value string) (string, error) {
var key []byte
switch name {
case "key":
key = []byte(value)
case "hexkey":
key, _ = hex.DecodeString(value)
case "textkey":
key = h.reset.KDF(value)
default:
if f, ok := h.File.(vfs.FilePragma); ok {
return f.Pragma(name, value)
}
return "", sqlite3.NOTFOUND
}
if h.hbsh = h.reset.HBSH(key); h.hbsh != nil {
return "ok", nil
}
return "", sqlite3.CANTOPEN
}
func (h *hbshFile) ReadAt(p []byte, off int64) (n int, err error) {
if h.hbsh == nil {
// Only OPEN_MAIN_DB can have a missing key.
if off == 0 && len(p) == 100 {
// SQLite is trying to read the header of a database file.
// Pretend the file is empty so the key may specified later,
// as a PRAGMA.
return 0, io.EOF
}
return 0, sqlite3.CANTOPEN
}
min := (off) &^ (blockSize - 1) // round down
max := (off + int64(len(p)) + (blockSize - 1)) &^ (blockSize - 1) // round up
// Read one block at a time.
for ; min < max; min += blockSize {
m, err := h.File.ReadAt(h.block[:], min)
if m != blockSize {
return n, err
}
binary.LittleEndian.PutUint64(h.tweak[:], uint64(min))
data := h.hbsh.Decrypt(h.block[:], h.tweak[:])
if off > min {
data = data[off-min:]
}
n += copy(p[n:], data)
}
if n != len(p) {
panic(util.AssertErr())
}
return n, nil
}
func (h *hbshFile) WriteAt(p []byte, off int64) (n int, err error) {
if h.hbsh == nil {
return 0, sqlite3.READONLY
}
min := (off) &^ (blockSize - 1) // round down
max := (off + int64(len(p)) + (blockSize - 1)) &^ (blockSize - 1) // round up
// Write one block at a time.
for ; min < max; min += blockSize {
binary.LittleEndian.PutUint64(h.tweak[:], uint64(min))
data := h.block[:]
if off > min || len(p[n:]) < blockSize {
// Partial block write: read-update-write.
m, err := h.File.ReadAt(h.block[:], min)
if m != blockSize {
if err != io.EOF {
return n, err
}
// Writing past the EOF.
// We're either appending an entirely new block,
// or the final block was only partially written.
// A partially written block can't be decrypted,
// and is as good as corrupt.
// Either way, zero pad the file to the next block size.
clear(data)
} else {
data = h.hbsh.Decrypt(h.block[:], h.tweak[:])
}
if off > min {
data = data[off-min:]
}
}
t := copy(data, p[n:])
h.hbsh.Encrypt(h.block[:], h.tweak[:])
m, err := h.File.WriteAt(h.block[:], min)
if m != blockSize {
return n, err
}
n += t
}
if n != len(p) {
panic(util.AssertErr())
}
return n, nil
}
func (h *hbshFile) Truncate(size int64) error {
size = (size + (blockSize - 1)) &^ (blockSize - 1) // round up
return h.File.Truncate(size)
}
func (h *hbshFile) SectorSize() int {
return max(h.File.SectorSize(), blockSize)
}
func (h *hbshFile) DeviceCharacteristics() vfs.DeviceCharacteristic {
return h.File.DeviceCharacteristics() & (0 |
// The only safe flags are these:
vfs.IOCAP_UNDELETABLE_WHEN_OPEN |
vfs.IOCAP_IMMUTABLE |
vfs.IOCAP_BATCH_ATOMIC)
}
// Wrap optional methods.
func (h *hbshFile) SharedMemory() vfs.SharedMemory {
if f, ok := h.File.(vfs.FileSharedMemory); ok {
return f.SharedMemory()
}
return nil
}
func (h *hbshFile) ChunkSize(size int) {
if f, ok := h.File.(vfs.FileChunkSize); ok {
size = (size + (blockSize - 1)) &^ (blockSize - 1) // round up
f.ChunkSize(size)
}
}
func (h *hbshFile) SizeHint(size int64) error {
if f, ok := h.File.(vfs.FileSizeHint); ok {
size = (size + (blockSize - 1)) &^ (blockSize - 1) // round up
return f.SizeHint(size)
}
return sqlite3.NOTFOUND
}
func (h *hbshFile) HasMoved() (bool, error) {
if f, ok := h.File.(vfs.FileHasMoved); ok {
return f.HasMoved()
}
return false, sqlite3.NOTFOUND
}
func (h *hbshFile) Overwrite() error {
if f, ok := h.File.(vfs.FileOverwrite); ok {
return f.Overwrite()
}
return sqlite3.NOTFOUND
}
func (h *hbshFile) CommitPhaseTwo() error {
if f, ok := h.File.(vfs.FileCommitPhaseTwo); ok {
return f.CommitPhaseTwo()
}
return sqlite3.NOTFOUND
}
func (h *hbshFile) BeginAtomicWrite() error {
if f, ok := h.File.(vfs.FileBatchAtomicWrite); ok {
return f.BeginAtomicWrite()
}
return sqlite3.NOTFOUND
}
func (h *hbshFile) CommitAtomicWrite() error {
if f, ok := h.File.(vfs.FileBatchAtomicWrite); ok {
return f.CommitAtomicWrite()
}
return sqlite3.NOTFOUND
}
func (h *hbshFile) RollbackAtomicWrite() error {
if f, ok := h.File.(vfs.FileBatchAtomicWrite); ok {
return f.RollbackAtomicWrite()
}
return sqlite3.NOTFOUND
}
func (h *hbshFile) CheckpointDone() error {
if f, ok := h.File.(vfs.FileCheckpoint); ok {
return f.CheckpointDone()
}
return sqlite3.NOTFOUND
}
func (h *hbshFile) CheckpointStart() error {
if f, ok := h.File.(vfs.FileCheckpoint); ok {
return f.CheckpointStart()
}
return sqlite3.NOTFOUND
}

View File

@@ -1,7 +1,12 @@
// Package vfs wraps the C SQLite VFS API.
package vfs
import "net/url"
import (
"context"
"io"
"github.com/tetratelabs/wazero/api"
)
// A VFS defines the interface between the SQLite core and the underlying operating system.
//
@@ -15,13 +20,13 @@ type VFS interface {
FullPathname(name string) (string, error)
}
// VFSParams extends VFS with the ability to handle URI parameters
// through the OpenParams method.
// VFSFilename extends VFS with the ability to use Filename
// objects for opening files.
//
// https://sqlite.org/c3ref/uri_boolean.html
type VFSParams interface {
// https://sqlite.org/c3ref/filename.html
type VFSFilename interface {
VFS
OpenParams(name string, flags OpenFlag, params url.Values) (File, OpenFlag, error)
OpenFilename(name *Filename, flags OpenFlag) (File, OpenFlag, error)
}
// A File represents an open file in the OS interface layer.
@@ -53,6 +58,15 @@ type FileLockState interface {
LockState() LockLevel
}
// FileChunkSize extends File to implement the
// SQLITE_FCNTL_CHUNK_SIZE file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlchunksize
type FileChunkSize interface {
File
ChunkSize(size int)
}
// FileSizeHint extends File to implement the
// SQLITE_FCNTL_SIZE_HINT file control opcode.
//
@@ -120,3 +134,42 @@ type FileBatchAtomicWrite interface {
CommitAtomicWrite() error
RollbackAtomicWrite() error
}
// FilePragma extends File to implement the
// SQLITE_FCNTL_PRAGMA file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpragma
type FilePragma interface {
File
Pragma(name, value string) (string, error)
}
// FileCheckpoint extends File to implement the
// SQLITE_FCNTL_CKPT_START and SQLITE_FCNTL_CKPT_DONE
// file control opcodes.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlckptstart
type FileCheckpoint interface {
File
CheckpointDone() error
CheckpointStart() error
}
// FileSharedMemory extends File to possibly implement
// shared-memory for the WAL-index.
// The same shared-memory instance must be returned
// for the entire life of the file.
// It's OK for SharedMemory to return nil.
type FileSharedMemory interface {
File
SharedMemory() SharedMemory
}
// SharedMemory is a shared-memory WAL-index implementation.
// Use [NewSharedMemory] to create a shared-memory.
type SharedMemory interface {
shmMap(context.Context, api.Module, int32, int32, bool) (uint32, error)
shmLock(int32, int32, _ShmFlag) error
shmUnmap(bool)
io.Closer
}

View File

@@ -4,8 +4,11 @@ import "github.com/ncruces/go-sqlite3/internal/util"
const (
_MAX_NAME = 1e6 // Self-imposed limit for most NUL terminated strings.
_MAX_SQL_LENGTH = 1e9
_MAX_PATHNAME = 1024
_DEFAULT_SECTOR_SIZE = 4096
ptrlen = 4
)
// https://sqlite.org/rescode.html
@@ -17,6 +20,7 @@ func (e _ErrorCode) Error() string {
const (
_OK _ErrorCode = util.OK
_ERROR _ErrorCode = util.ERROR
_PERM _ErrorCode = util.PERM
_BUSY _ErrorCode = util.BUSY
_READONLY _ErrorCode = util.READONLY
@@ -37,6 +41,10 @@ const (
_IOERR_CHECKRESERVEDLOCK _ErrorCode = util.IOERR_CHECKRESERVEDLOCK
_IOERR_LOCK _ErrorCode = util.IOERR_LOCK
_IOERR_CLOSE _ErrorCode = util.IOERR_CLOSE
_IOERR_SHMOPEN _ErrorCode = util.IOERR_SHMOPEN
_IOERR_SHMSIZE _ErrorCode = util.IOERR_SHMSIZE
_IOERR_SHMLOCK _ErrorCode = util.IOERR_SHMLOCK
_IOERR_SHMMAP _ErrorCode = util.IOERR_SHMMAP
_IOERR_SEEK _ErrorCode = util.IOERR_SEEK
_IOERR_DELETE_NOENT _ErrorCode = util.IOERR_DELETE_NOENT
_IOERR_BEGIN_ATOMIC _ErrorCode = util.IOERR_BEGIN_ATOMIC
@@ -44,6 +52,7 @@ const (
_IOERR_ROLLBACK_ATOMIC _ErrorCode = util.IOERR_ROLLBACK_ATOMIC
_CANTOPEN_FULLPATH _ErrorCode = util.CANTOPEN_FULLPATH
_CANTOPEN_ISDIR _ErrorCode = util.CANTOPEN_ISDIR
_READONLY_CANTINIT _ErrorCode = util.READONLY_CANTINIT
_OK_SYMLINK _ErrorCode = util.OK_SYMLINK
)
@@ -213,3 +222,13 @@ const (
_FCNTL_CKSM_FILE _FcntlOpcode = 41
_FCNTL_RESET_CACHE _FcntlOpcode = 42
)
// https://sqlite.org/c3ref/c_shm_exclusive.html
type _ShmFlag uint32
const (
_SHM_UNLOCK _ShmFlag = 1
_SHM_LOCK _ShmFlag = 2
_SHM_SHARED _ShmFlag = 4
_SHM_EXCLUSIVE _ShmFlag = 8
)

View File

@@ -4,7 +4,6 @@ import (
"errors"
"io"
"io/fs"
"net/url"
"os"
"path/filepath"
"runtime"
@@ -70,10 +69,10 @@ func (vfsOS) Access(name string, flags AccessFlag) (bool, error) {
}
func (vfsOS) Open(name string, flags OpenFlag) (File, OpenFlag, error) {
return vfsOS{}.OpenParams(name, flags, nil)
return nil, 0, _CANTOPEN
}
func (vfsOS) OpenParams(name string, flags OpenFlag, params url.Values) (File, OpenFlag, error) {
func (vfsOS) OpenFilename(name *Filename, flags OpenFlag) (File, OpenFlag, error) {
var oflags int
if flags&OPEN_EXCLUSIVE != 0 {
oflags |= os.O_EXCL
@@ -90,10 +89,10 @@ func (vfsOS) OpenParams(name string, flags OpenFlag, params url.Values) (File, O
var err error
var f *os.File
if name == "" {
if name == nil {
f, err = os.CreateTemp("", "*.db")
} else {
f, err = osutil.OpenFile(name, oflags, 0666)
f, err = osutil.OpenFile(name.String(), oflags, 0666)
}
if err != nil {
if errors.Is(err, syscall.EISDIR) {
@@ -102,7 +101,7 @@ func (vfsOS) OpenParams(name string, flags OpenFlag, params url.Values) (File, O
return nil, flags, err
}
if modeof := params.Get("modeof"); modeof != "" {
if modeof := name.URIParameter("modeof"); modeof != "" {
if err = osSetMode(f, modeof); err != nil {
f.Close()
return nil, flags, _IOERR_FSTAT
@@ -119,12 +118,14 @@ func (vfsOS) OpenParams(name string, flags OpenFlag, params url.Values) (File, O
syncDir: runtime.GOOS != "windows" &&
flags&(OPEN_CREATE) != 0 &&
flags&(OPEN_MAIN_JOURNAL|OPEN_SUPER_JOURNAL|OPEN_WAL) != 0,
shm: NewSharedMemory(name.String()+"-shm", flags),
}
return &file, flags, nil
}
type vfsFile struct {
*os.File
shm SharedMemory
lock LockLevel
readOnly bool
keepWAL bool
@@ -141,6 +142,13 @@ var (
_ FilePowersafeOverwrite = &vfsFile{}
)
func (f *vfsFile) Close() error {
if f.shm != nil {
f.shm.Close()
}
return f.File.Close()
}
func (f *vfsFile) Sync(flags SyncFlag) error {
dataonly := (flags & SYNC_DATAONLY) != 0
fullsync := (flags & 0x0f) == SYNC_FULL
@@ -168,15 +176,19 @@ func (f *vfsFile) Size() (int64, error) {
return f.Seek(0, io.SeekEnd)
}
func (*vfsFile) SectorSize() int {
func (f *vfsFile) SectorSize() int {
return _DEFAULT_SECTOR_SIZE
}
func (f *vfsFile) DeviceCharacteristics() DeviceCharacteristic {
if f.psow {
return IOCAP_POWERSAFE_OVERWRITE
var res DeviceCharacteristic
if osBatchAtomic(f.File) {
res |= IOCAP_BATCH_ATOMIC
}
return 0
if f.psow {
res |= IOCAP_POWERSAFE_OVERWRITE
}
return res
}
func (f *vfsFile) SizeHint(size int64) error {

174
vfs/filename.go Normal file
View File

@@ -0,0 +1,174 @@
package vfs
import (
"context"
"net/url"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/tetratelabs/wazero/api"
)
// Filename is used by SQLite to pass filenames
// to the Open method of a VFS.
//
// https://sqlite.org/c3ref/filename.html
type Filename struct {
ctx context.Context
mod api.Module
zPath uint32
flags OpenFlag
stack [2]uint64
}
// OpenFilename is an internal API users should not call directly.
func OpenFilename(ctx context.Context, mod api.Module, id uint32, flags OpenFlag) *Filename {
if id == 0 {
return nil
}
return &Filename{
ctx: ctx,
mod: mod,
zPath: id,
flags: flags,
}
}
// String returns this filename as a string.
func (n *Filename) String() string {
if n == nil || n.zPath == 0 {
return ""
}
return util.ReadString(n.mod, n.zPath, _MAX_PATHNAME)
}
// Database returns the name of the corresponding database file.
//
// https://sqlite.org/c3ref/filename_database.html
func (n *Filename) Database() string {
return n.path("sqlite3_filename_database")
}
// Journal returns the name of the corresponding rollback journal file.
//
// https://sqlite.org/c3ref/filename_database.html
func (n *Filename) Journal() string {
return n.path("sqlite3_filename_journal")
}
// Journal returns the name of the corresponding WAL file.
//
// https://sqlite.org/c3ref/filename_database.html
func (n *Filename) WAL() string {
return n.path("sqlite3_filename_wal")
}
func (n *Filename) path(method string) string {
if n == nil || n.zPath == 0 {
return ""
}
n.stack[0] = uint64(n.zPath)
fn := n.mod.ExportedFunction(method)
if err := fn.CallWithStack(n.ctx, n.stack[:]); err != nil {
panic(err)
}
return util.ReadString(n.mod, uint32(n.stack[0]), _MAX_PATHNAME)
}
// DatabaseFile returns the main database [File] corresponding to a journal.
//
// https://sqlite.org/c3ref/database_file_object.html
func (n *Filename) DatabaseFile() File {
if n == nil || n.zPath == 0 {
return nil
}
if n.flags&(OPEN_MAIN_DB|OPEN_MAIN_JOURNAL|OPEN_WAL) == 0 {
return nil
}
n.stack[0] = uint64(n.zPath)
fn := n.mod.ExportedFunction("sqlite3_database_file_object")
if err := fn.CallWithStack(n.ctx, n.stack[:]); err != nil {
panic(err)
}
file, _ := vfsFileGet(n.ctx, n.mod, uint32(n.stack[0])).(File)
return file
}
// URIParameter returns the value of a URI parameter.
//
// https://sqlite.org/c3ref/uri_boolean.html
func (n *Filename) URIParameter(key string) string {
if n == nil || n.zPath == 0 {
return ""
}
uriKey := n.mod.ExportedFunction("sqlite3_uri_key")
n.stack[0] = uint64(n.zPath)
n.stack[1] = uint64(0)
if err := uriKey.CallWithStack(n.ctx, n.stack[:]); err != nil {
panic(err)
}
ptr := uint32(n.stack[0])
if ptr == 0 {
return ""
}
// Parse the format from:
// https://github.com/sqlite/sqlite/blob/b74eb0/src/pager.c#L4797-L4840
// This avoids having to alloc/free the key just to find a value.
for {
k := util.ReadString(n.mod, ptr, _MAX_NAME)
if k == "" {
return ""
}
ptr += uint32(len(k)) + 1
v := util.ReadString(n.mod, ptr, _MAX_NAME)
if k == key {
return v
}
ptr += uint32(len(v)) + 1
}
}
// URIParameters obtains values for URI parameters.
//
// https://sqlite.org/c3ref/uri_boolean.html
func (n *Filename) URIParameters() url.Values {
if n == nil || n.zPath == 0 {
return nil
}
uriKey := n.mod.ExportedFunction("sqlite3_uri_key")
n.stack[0] = uint64(n.zPath)
n.stack[1] = uint64(0)
if err := uriKey.CallWithStack(n.ctx, n.stack[:]); err != nil {
panic(err)
}
ptr := uint32(n.stack[0])
if ptr == 0 {
return nil
}
var params url.Values
// Parse the format from:
// https://github.com/sqlite/sqlite/blob/b74eb0/src/pager.c#L4797-L4840
// This is the only way to support multiple valued keys.
for {
k := util.ReadString(n.mod, ptr, _MAX_NAME)
if k == "" {
return params
}
ptr += uint32(len(k)) + 1
v := util.ReadString(n.mod, ptr, _MAX_NAME)
if params == nil {
params = url.Values{}
}
params.Add(k, v)
ptr += uint32(len(v)) + 1
}
}

View File

@@ -1,7 +1,17 @@
//go:build (linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos) && !sqlite3_nosys
package vfs
import "github.com/ncruces/go-sqlite3/internal/util"
// SupportsFileLocking is false on platforms that do not support file locking.
// To open a database file on those platforms,
// you need to use the [nolock] or [immutable] URI parameters.
//
// [nolock]: https://sqlite.org/uri.html#urinolock
// [immutable]: https://sqlite.org/uri.html#uriimmutable
const SupportsFileLocking = true
const (
_PENDING_BYTE = 0x40000000
_RESERVED_BYTE = (_PENDING_BYTE + 1)
@@ -107,11 +117,9 @@ func (f *vfsFile) Unlock(lock LockLevel) error {
switch lock {
case LOCK_SHARED:
if rc := osDowngradeLock(f.File, f.lock); rc != _OK {
return rc
}
rc := osDowngradeLock(f.File, f.lock)
f.lock = LOCK_SHARED
return nil
return rc
case LOCK_NONE:
rc := osReleaseLock(f.File, f.lock)

23
vfs/lock_other.go Normal file
View File

@@ -0,0 +1,23 @@
//go:build !(linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos) || sqlite3_nosys
package vfs
// SupportsFileLocking is false on platforms that do not support file locking.
// To open a database file on those platforms,
// you need to use the [nolock] or [immutable] URI parameters.
//
// [nolock]: https://sqlite.org/uri.html#urinolock
// [immutable]: https://sqlite.org/uri.html#uriimmutable
const SupportsFileLocking = false
func (f *vfsFile) Lock(LockLevel) error {
return _IOERR_LOCK
}
func (f *vfsFile) Unlock(LockLevel) error {
return _IOERR_UNLOCK
}
func (f *vfsFile) CheckReservedLock() (bool, error) {
return false, _IOERR_CHECKRESERVEDLOCK
}

View File

@@ -1,3 +1,5 @@
//go:build !sqlite3_nosys
package vfs
import (

View File

@@ -4,7 +4,7 @@
// among multiple database connections in the same process,
// as long as the database name begins with "/".
//
// Importing package memdb registers the VFS.
// Importing package memdb registers the VFS:
//
// import _ "github.com/ncruces/go-sqlite3/vfs/memdb"
package memdb

View File

@@ -11,14 +11,23 @@ import (
"github.com/ncruces/go-sqlite3/vfs"
)
// Must be a multiple of 64K (the largest page size).
const sectorSize = 65536
type memVFS struct{}
func (memVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
// Allowed file types:
// For simplicity, we do not support reading or writing data
// across "sector" boundaries.
//
// This is not a problem for most SQLite file types:
// - databases, which only do page aligned reads/writes;
// - temp journals, used by the sorter, which does the same.
// - temp journals, as used by the sorter, which does the same:
// https://github.com/sqlite/sqlite/blob/b74eb0/src/vdbesort.c#L409-L412
//
// We refuse to open all other file types,
// but returning OPEN_MEMORY means SQLite won't ask us to.
const types = vfs.OPEN_MAIN_DB |
vfs.OPEN_TRANSIENT_DB |
vfs.OPEN_TEMP_DB |
vfs.OPEN_TEMP_JOURNAL
if flags&types == 0 {
@@ -61,9 +70,6 @@ func (memVFS) FullPathname(name string) (string, error) {
return name, nil
}
// Must be a multiple of 64K (the largest page size).
const sectorSize = 65536
type memDB struct {
// +checklocks:lockMtx
pending *memFile
@@ -166,7 +172,7 @@ func (m *memFile) truncate(size int64) error {
return nil
}
func (*memFile) Sync(flag vfs.SyncFlag) error {
func (m *memFile) Sync(flag vfs.SyncFlag) error {
return nil
}
@@ -256,11 +262,11 @@ func (m *memFile) CheckReservedLock() (bool, error) {
return m.reserved != nil, nil
}
func (*memFile) SectorSize() int {
func (m *memFile) SectorSize() int {
return sectorSize
}
func (*memFile) DeviceCharacteristics() vfs.DeviceCharacteristic {
func (m *memFile) DeviceCharacteristics() vfs.DeviceCharacteristic {
return vfs.IOCAP_ATOMIC |
vfs.IOCAP_SEQUENTIAL |
vfs.IOCAP_SAFE_APPEND |

View File

@@ -7,6 +7,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
//go:embed testdata/wal.db
@@ -21,7 +22,7 @@ func Test_wal(t *testing.T) {
}
defer db.Close()
err = db.Exec(`CREATE TABLE IF NOT EXISTS users (id INT, name VARCHAR(10))`)
err = db.Exec(`CREATE TABLE users (id INT, name VARCHAR(10))`)
if err != nil {
t.Fatal(err)
}

View File

@@ -1,4 +1,4 @@
//go:build (freebsd || openbsd || netbsd || dragonfly || sqlite3_flock) && !sqlite3_nosys
//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && !sqlite3_nosys
package vfs
@@ -31,15 +31,3 @@ func osReadLock(file *os.File, _ /*start*/, _ /*len*/ int64, _ /*timeout*/ time.
func osWriteLock(file *os.File, _ /*start*/, _ /*len*/ int64, _ /*timeout*/ time.Duration) _ErrorCode {
return osLock(file, unix.LOCK_EX|unix.LOCK_NB, _IOERR_LOCK)
}
func osCheckLock(file *os.File, start, len int64) (bool, _ErrorCode) {
lock := unix.Flock_t{
Type: unix.F_RDLCK,
Start: start,
Len: len,
}
if unix.FcntlFlock(file.Fd(), unix.F_GETLK, &lock) != nil {
return false, _IOERR_CHECKRESERVEDLOCK
}
return lock.Type != unix.F_UNLCK, _OK
}

View File

@@ -14,7 +14,6 @@ const (
// https://github.com/apple/darwin-xnu/blob/main/bsd/sys/fcntl.h
_F_OFD_SETLK = 90
_F_OFD_SETLKW = 91
_F_OFD_GETLK = 92
_F_OFD_SETLKWTIMEOUT = 93
)
@@ -94,15 +93,3 @@ func osReadLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCo
func osWriteLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode {
return osLock(file, unix.F_WRLCK, start, len, timeout, _IOERR_LOCK)
}
func osCheckLock(file *os.File, start, len int64) (bool, _ErrorCode) {
lock := unix.Flock_t{
Type: unix.F_RDLCK,
Start: start,
Len: len,
}
if unix.FcntlFlock(file.Fd(), _F_OFD_GETLK, &lock) != nil {
return false, _IOERR_CHECKRESERVEDLOCK
}
return lock.Type != unix.F_UNLCK, _OK
}

34
vfs/os_f2fs_linux.go Normal file
View File

@@ -0,0 +1,34 @@
//go:build (amd64 || arm64 || riscv64) && !sqlite3_nosys
package vfs
import (
"os"
"golang.org/x/sys/unix"
)
const (
_F2FS_IOC_START_ATOMIC_WRITE = 62721
_F2FS_IOC_COMMIT_ATOMIC_WRITE = 62722
_F2FS_IOC_ABORT_ATOMIC_WRITE = 62725
_F2FS_IOC_GET_FEATURES = 2147808524
_F2FS_FEATURE_ATOMIC_WRITE = 4
)
func osBatchAtomic(file *os.File) bool {
flags, err := unix.IoctlGetInt(int(file.Fd()), _F2FS_IOC_GET_FEATURES)
return err == nil && flags&_F2FS_FEATURE_ATOMIC_WRITE != 0
}
func (f *vfsFile) BeginAtomicWrite() error {
return unix.IoctlSetInt(int(f.Fd()), _F2FS_IOC_START_ATOMIC_WRITE, 0)
}
func (f *vfsFile) CommitAtomicWrite() error {
return unix.IoctlSetInt(int(f.Fd()), _F2FS_IOC_COMMIT_ATOMIC_WRITE, 0)
}
func (f *vfsFile) RollbackAtomicWrite() error {
return unix.IoctlSetInt(int(f.Fd()), _F2FS_IOC_ABORT_ATOMIC_WRITE, 0)
}

View File

@@ -3,20 +3,16 @@
package vfs
import (
"math/rand"
"os"
"time"
"golang.org/x/sys/unix"
)
func osSync(file *os.File, _ /*fullsync*/, dataonly bool) error {
if dataonly {
_, _, err := unix.Syscall(unix.SYS_FDATASYNC, file.Fd(), 0, 0)
if err != 0 {
return err
}
return nil
}
return file.Sync()
func osSync(file *os.File, _ /*fullsync*/, _ /*dataonly*/ bool) error {
// SQLite trusts Linux's fdatasync for all fsync's.
return unix.Fdatasync(int(file.Fd()))
}
func osAllocate(file *os.File, size int64) error {
@@ -25,3 +21,51 @@ func osAllocate(file *os.File, size int64) error {
}
return unix.Fallocate(int(file.Fd()), 0, 0, size)
}
func osUnlock(file *os.File, start, len int64) _ErrorCode {
err := unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &unix.Flock_t{
Type: unix.F_UNLCK,
Start: start,
Len: len,
})
if err != nil {
return _IOERR_UNLOCK
}
return _OK
}
func osLock(file *os.File, typ int16, start, len int64, timeout time.Duration, def _ErrorCode) _ErrorCode {
lock := unix.Flock_t{
Type: typ,
Start: start,
Len: len,
}
var err error
switch {
case timeout == 0:
err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &lock)
case timeout < 0:
err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLKW, &lock)
default:
before := time.Now()
for {
err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &lock)
if errno, _ := err.(unix.Errno); errno != unix.EAGAIN {
break
}
if timeout < time.Since(before) {
break
}
osSleep(time.Duration(rand.Int63n(int64(time.Millisecond))))
}
}
return osLockErrorCode(err, def)
}
func osReadLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode {
return osLock(file, unix.F_RDLCK, start, len, timeout, _IOERR_RDLOCK)
}
func osWriteLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode {
return osLock(file, unix.F_WRLCK, start, len, timeout, _IOERR_LOCK)
}

View File

@@ -1,41 +0,0 @@
//go:build !(linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos) || sqlite3_nosys
package vfs
import "os"
// SupportsFileLocking is false on platforms that do not support file locking.
// To open a database file in one such platform,
// you need to use the [nolock] or [immutable] URI parameters.
//
// [nolock]: https://sqlite.org/uri.html#urinolock
// [immutable]: https://sqlite.org/uri.html#uriimmutable
const SupportsFileLocking = false
func osGetSharedLock(_ *os.File) _ErrorCode {
return _IOERR_RDLOCK
}
func osGetReservedLock(_ *os.File) _ErrorCode {
return _IOERR_LOCK
}
func osGetPendingLock(_ *os.File, _ bool) _ErrorCode {
return _IOERR_LOCK
}
func osGetExclusiveLock(_ *os.File, _ bool) _ErrorCode {
return _IOERR_LOCK
}
func osDowngradeLock(_ *os.File, _ LockLevel) _ErrorCode {
return _IOERR_RDLOCK
}
func osReleaseLock(_ *os.File, _ LockLevel) _ErrorCode {
return _IOERR_UNLOCK
}
func osCheckReservedLock(_ *os.File) (bool, _ErrorCode) {
return false, _IOERR_CHECKRESERVEDLOCK
}

View File

@@ -1,70 +0,0 @@
//go:build (linux || illumos) && !sqlite3_nosys
package vfs
import (
"os"
"time"
"golang.org/x/sys/unix"
)
func osUnlock(file *os.File, start, len int64) _ErrorCode {
err := unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &unix.Flock_t{
Type: unix.F_UNLCK,
Start: start,
Len: len,
})
if err != nil {
return _IOERR_UNLOCK
}
return _OK
}
func osLock(file *os.File, typ int16, start, len int64, timeout time.Duration, def _ErrorCode) _ErrorCode {
lock := unix.Flock_t{
Type: typ,
Start: start,
Len: len,
}
var err error
switch {
case timeout == 0:
err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &lock)
case timeout < 0:
err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLKW, &lock)
default:
before := time.Now()
for {
err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &lock)
if errno, _ := err.(unix.Errno); errno != unix.EAGAIN {
break
}
if timeout <= 0 || timeout < time.Since(before) {
break
}
osSleep(time.Millisecond)
}
}
return osLockErrorCode(err, def)
}
func osReadLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode {
return osLock(file, unix.F_RDLCK, start, len, timeout, _IOERR_RDLOCK)
}
func osWriteLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode {
return osLock(file, unix.F_WRLCK, start, len, timeout, _IOERR_LOCK)
}
func osCheckLock(file *os.File, start, len int64) (bool, _ErrorCode) {
lock := unix.Flock_t{
Type: unix.F_RDLCK,
Start: start,
Len: len,
}
if unix.FcntlFlock(file.Fd(), unix.F_OFD_GETLK, &lock) != nil {
return false, _IOERR_CHECKRESERVEDLOCK
}
return lock.Type != unix.F_UNLCK, _OK
}

9
vfs/os_std_atomic.go Normal file
View File

@@ -0,0 +1,9 @@
//go:build !linux || !(amd64 || arm64 || riscv64) || sqlite3_nosys
package vfs
import "os"
func osBatchAtomic(*os.File) bool {
return false
}

Some files were not shown because too many files have changed in this diff Show More