Compare commits

..

26 Commits

Author SHA1 Message Date
Nuno Cruces
57daee7f59 Update README.md 2024-05-01 12:20:33 +01:00
Nuno Cruces
f976ab0dee Additional check. 2024-04-30 20:56:42 +01:00
Nuno Cruces
beba988824 Multiple fixes. 2024-04-30 01:30:39 +01:00
Nuno Cruces
992676d7ec Improved WAL API. 2024-04-27 20:55:14 +01:00
Nuno Cruces
82d8a2d796 Documentation. 2024-04-27 16:31:32 +01:00
Nuno Cruces
811e6e63be Adiantum pragmas. 2024-04-27 12:19:46 +01:00
Nuno Cruces
3c21784aee Simplify URI parameters. 2024-04-27 10:44:00 +01:00
Nuno Cruces
019246d1be Simplify mmap. 2024-04-26 16:45:32 +01:00
Nuno Cruces
fa259bdc94 Simplify _pragma. 2024-04-26 00:07:04 +01:00
Nuno Cruces
8e327a9783 VFS pragma. 2024-04-25 13:30:47 +01:00
Nuno Cruces
09a0ce04ce Test more. (#84)
Also, fix the progress callback and disable a slow example.
2024-04-24 15:49:45 +01:00
Nuno Cruces
fdb2ed0376 Fix illumos. (#83) 2024-04-24 01:07:17 +01:00
Nuno Cruces
3fb0eeec51 Filename API (#82)
Also remove VFSParams.
2024-04-23 11:43:14 +01:00
Nuno Cruces
7f6446ad31 Remove cache (side-channel for shared keys). 2024-04-23 02:25:26 +01:00
Nuno Cruces
77cdf1841f Documentation nits. 2024-04-22 15:28:19 +01:00
kim
189fbc98ac change driver name to SQLite{}, remove global variable 2024-04-22 14:02:45 +01:00
kim
d4027b0133 export the database/sql driver type and global instance 2024-04-22 14:02:45 +01:00
Nuno Cruces
62b79d2ac3 Shared memory API. 2024-04-21 12:33:38 +01:00
Nuno Cruces
07241d064a Adiantum encrypting VFS improvements. (#80)
Encrypt temporary files.
2024-04-21 01:56:38 +01:00
Nuno Cruces
2c30bc996a Don't panic. 2024-04-18 10:12:17 +01:00
Nuno Cruces
9d2194b4ea Update README.md 2024-04-18 02:13:59 +01:00
Nuno Cruces
b3a1cb3dd6 Update README.md 2024-04-18 01:42:25 +01:00
Nuno Cruces
ec1ed22149 Adiantum encrypting VFS. (#77) 2024-04-18 01:39:47 +01:00
Nuno Cruces
e86789b285 Test config. 2024-04-16 17:35:45 +01:00
Nuno Cruces
a1fcafa780 Formatting. 2024-04-16 14:02:23 +01:00
Nuno Cruces
d3f5745790 Updated dependencies. 2024-04-16 02:54:11 +01:00
84 changed files with 1322 additions and 382 deletions

View File

@@ -1,7 +1,7 @@
#!/usr/bin/env bash
set -euo pipefail
echo 'set -euo pipefail' > test.sh
echo 'set -eu' > test.sh
for p in $(go list ./...); do
dir=".${p#github.com/ncruces/go-sqlite3}"

11
.github/workflows/illumos.sh vendored Executable file
View File

@@ -0,0 +1,11 @@
#!/usr/bin/env bash
set -euo pipefail
echo 'set -eu' > test.sh
for p in $(go list ./...); do
dir=".${p#github.com/ncruces/go-sqlite3}"
name="$(basename "$p").test"
(cd ${dir}; GOOS=illumos go test -c)
[ -f "${dir}/${name}" ] && echo "(cd ${dir}; ./${name} -test.v -test.short)" >> test.sh
done

View File

@@ -52,7 +52,8 @@ jobs:
if: matrix.os == 'ubuntu-latest'
- name: Test no locks
run: go test -v -tags sqlite3_nosys ./tests -run TestDB_nolock
run: go test -v -tags sqlite3_nosys ./...
if: matrix.os == 'ubuntu-latest'
- name: Test GORM
run: gormlite/test.sh
@@ -85,9 +86,30 @@ jobs:
operating_system: freebsd
version: '14.0'
shell: bash
run: source test.sh
run: . ./test.sh
sync_files: runner-to-vm
test-illumos:
runs-on: ubuntu-latest
needs: test
steps:
- uses: actions/checkout@v4
with: { lfs: 'true' }
- uses: actions/setup-go@v5
with: { go-version: stable }
- name: Build
run: .github/workflows/illumos.sh
- name: Test
uses: vmactions/omnios-vm@v1
with:
usesh: true
copyback: false
run: . ./test.sh
test-m1:
runs-on: macos-14
needs: test
@@ -102,21 +124,7 @@ jobs:
- name: Test
run: go test -v ./...
test-386:
runs-on: ubuntu-latest
needs: test
steps:
- uses: actions/checkout@v4
with: { lfs: 'true' }
- uses: actions/setup-go@v5
with: { go-version: stable }
- name: Test
run: GOARCH=386 go test -v -short ./...
test-arm:
test-qemu:
runs-on: ubuntu-latest
needs: test
@@ -129,5 +137,11 @@ jobs:
- uses: docker/setup-qemu-action@v3
- name: Test
- name: Test 386
run: GOARCH=386 go test -v -short ./...
- name: Test arm64
run: GOARCH=arm64 go test -v -short ./...
- name: Test riscv64
run: GOARCH=riscv64 go test -v -short ./...

View File

@@ -4,12 +4,13 @@
[![Go Report](https://goreportcard.com/badge/github.com/ncruces/go-sqlite3)](https://goreportcard.com/report/github.com/ncruces/go-sqlite3)
[![Go Coverage](https://github.com/ncruces/go-sqlite3/wiki/coverage.svg)](https://github.com/ncruces/go-sqlite3/wiki/Test-coverage-report)
Go module `github.com/ncruces/go-sqlite3` is `cgo`-free [SQLite](https://sqlite.org/) wrapper.\
Go module `github.com/ncruces/go-sqlite3` is a `cgo`-free [SQLite](https://sqlite.org/) wrapper.\
It provides a [`database/sql`](https://pkg.go.dev/database/sql) compatible driver,
as well as direct access to most of the [C SQLite API](https://sqlite.org/cintro.html).
It wraps a [Wasm](https://webassembly.org/) build of SQLite, and uses [wazero](https://wazero.io/) as the runtime.\
Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ runtime dependencies.
It wraps a [Wasm](https://webassembly.org/) [build](embed/) of SQLite,
and uses [wazero](https://wazero.io/) as the runtime.\
Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ runtime dependencies [^1].
### Packages
@@ -54,6 +55,8 @@ Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ run
implements an in-memory VFS.
- [`github.com/ncruces/go-sqlite3/vfs/readervfs`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/readervfs)
implements a VFS for immutable databases.
- [`github.com/ncruces/go-sqlite3/vfs/adiantum`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/adiantum)
wraps a VFS to offer encryption at rest.
### Advanced features
@@ -67,6 +70,7 @@ Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ run
- [math functions](https://sqlite.org/lang_mathfunc.html)
- [full-text search](https://sqlite.org/fts5.html)
- [geospatial search](https://sqlite.org/geopoly.html)
- [encryption at rest](vfs/adiantum/README.md)
- [and more…](embed/README.md)
### Caveats
@@ -83,9 +87,10 @@ This project aims for [high test coverage](https://github.com/ncruces/go-sqlite3
It also benefits greatly from [SQLite's](https://sqlite.org/testing.html) and
[wazero's](https://tetrate.io/blog/introducing-wazero-from-tetrate/#:~:text=Rock%2Dsolid%20test%20approach) thorough testing.
Every commit is [tested](.github/workflows/test.yml) on
Linux (amd64/arm64/386/riscv64), macOS (amd64/arm64), Windows, FreeBSD and illumos.
The Go VFS is tested by running SQLite's
[mptest](https://github.com/sqlite/sqlite/blob/master/mptest/mptest.c)
on Linux, macOS, Windows and FreeBSD.
[mptest](https://github.com/sqlite/sqlite/blob/master/mptest/mptest.c).
### Performance
@@ -101,3 +106,6 @@ The Wasm and VFS layers are also tested by running SQLite's
- [`crawshaw.io/sqlite`](https://pkg.go.dev/crawshaw.io/sqlite)
- [`github.com/mattn/go-sqlite3`](https://pkg.go.dev/github.com/mattn/go-sqlite3)
- [`github.com/zombiezen/go-sqlite`](https://pkg.go.dev/github.com/zombiezen/go-sqlite)
[^1]: anything else you find in `go.mod` is either a test dependency,
or needed by one of the extensions.

35
conn.go
View File

@@ -2,7 +2,6 @@ package sqlite3
import (
"context"
"errors"
"fmt"
"math"
"net/url"
@@ -10,6 +9,7 @@ import (
"time"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/vfs"
"github.com/tetratelabs/wazero/api"
)
@@ -102,15 +102,14 @@ func (c *Conn) openDB(filename string, flags OpenFlag) (uint32, error) {
pragmas.WriteString(`;`)
}
}
pragmaPtr := c.arena.string(pragmas.String())
r := c.call("sqlite3_exec", uint64(handle), uint64(pragmaPtr), 0, 0, 0)
if err := c.sqlite.error(r, handle, pragmas.String()); err != nil {
if errors.Is(err, ERROR) {
if pragmas.Len() != 0 {
pragmaPtr := c.arena.string(pragmas.String())
r := c.call("sqlite3_exec", uint64(handle), uint64(pragmaPtr), 0, 0, 0)
if err := c.sqlite.error(r, handle, pragmas.String()); err != nil {
err = fmt.Errorf("sqlite3: invalid _pragma: %w", err)
c.closeDB(handle)
return 0, err
}
c.closeDB(handle)
return 0, err
}
}
c.call("sqlite3_progress_handler_go", uint64(handle), 100)
@@ -175,7 +174,7 @@ func (c *Conn) Prepare(sql string) (stmt *Stmt, tail string, err error) {
//
// https://sqlite.org/c3ref/prepare.html
func (c *Conn) PrepareFlags(sql string, flags PrepareFlag) (stmt *Stmt, tail string, err error) {
if len(sql) > _MAX_LENGTH {
if len(sql) > _MAX_SQL_LENGTH {
return nil, "", TOOBIG
}
@@ -216,6 +215,20 @@ func (c *Conn) DBName(n int) string {
return util.ReadString(c.mod, ptr, _MAX_NAME)
}
// Filename returns the filename for a database.
//
// https://sqlite.org/c3ref/db_filename.html
func (c *Conn) Filename(schema string) *vfs.Filename {
var ptr uint32
if schema != "" {
defer c.arena.mark()()
ptr = c.arena.string(schema)
}
r := c.call("sqlite3_db_filename", uint64(c.handle), uint64(ptr))
return vfs.OpenFilename(c.ctx, c.mod, uint32(r), vfs.OPEN_MAIN_DB)
}
// ReadOnly determines if a database is read-only.
//
// https://sqlite.org/c3ref/db_readonly.html
@@ -333,8 +346,8 @@ func (c *Conn) checkInterrupt() {
}
func progressCallback(ctx context.Context, mod api.Module, pDB uint32) (interrupt uint32) {
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.commit != nil {
if c.interrupt != nil && c.interrupt.Err() != nil {
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.interrupt != nil {
if c.interrupt.Err() != nil {
interrupt = 1
}
}

View File

@@ -63,39 +63,47 @@ var driverName = "sqlite3"
func init() {
if driverName != "" {
sql.Register(driverName, sqlite{})
sql.Register(driverName, &SQLite{})
}
}
// Open opens the SQLite database specified by dataSourceName as a [database/sql.DB].
//
// The init function is called by the driver on new connections.
// The conn can be used to execute queries, register functions, etc.
// Any error return closes the conn and passes the error to [database/sql].
// The [sqlite3.Conn] can be used to execute queries, register functions, etc.
// Any error returned closes the connection and is returned to [database/sql].
func Open(dataSourceName string, init func(*sqlite3.Conn) error) (*sql.DB, error) {
c, err := newConnector(dataSourceName, init)
c, err := (&SQLite{Init: init}).OpenConnector(dataSourceName)
if err != nil {
return nil, err
}
return sql.OpenDB(c), nil
}
type sqlite struct{}
// SQLite implements [database/sql/driver.Driver].
type SQLite struct {
// Init function is called by the driver on new connections.
// The [sqlite3.Conn] can be used to execute queries, register functions, etc.
// Any error returned closes the connection and is returned to [database/sql].
Init func(*sqlite3.Conn) error
}
func (sqlite) Open(name string) (driver.Conn, error) {
c, err := newConnector(name, nil)
// Open implements [database/sql/driver.Driver].
func (d *SQLite) Open(name string) (driver.Conn, error) {
c, err := d.newConnector(name)
if err != nil {
return nil, err
}
return c.Connect(context.Background())
}
func (sqlite) OpenConnector(name string) (driver.Connector, error) {
return newConnector(name, nil)
// OpenConnector implements [database/sql/driver.DriverContext].
func (d *SQLite) OpenConnector(name string) (driver.Connector, error) {
return d.newConnector(name)
}
func newConnector(name string, init func(*sqlite3.Conn) error) (*connector, error) {
c := connector{name: name, init: init}
func (d *SQLite) newConnector(name string) (*connector, error) {
c := connector{driver: d, name: name}
var txlock, timefmt string
if strings.HasPrefix(name, "file:") {
@@ -137,7 +145,7 @@ func newConnector(name string, init func(*sqlite3.Conn) error) (*connector, erro
}
type connector struct {
init func(*sqlite3.Conn) error
driver *SQLite
name string
txBegin string
tmRead sqlite3.TimeFormat
@@ -146,7 +154,7 @@ type connector struct {
}
func (n *connector) Driver() driver.Driver {
return sqlite{}
return n.driver
}
func (n *connector) Connect(ctx context.Context) (_ driver.Conn, err error) {
@@ -175,13 +183,13 @@ func (n *connector) Connect(ctx context.Context) (_ driver.Conn, err error) {
return nil, err
}
}
if n.init != nil {
err = n.init(c.Conn)
if n.driver.Init != nil {
err = n.driver.Init(c.Conn)
if err != nil {
return nil, err
}
}
if n.pragmas || n.init != nil {
if n.pragmas || n.driver.Init != nil {
s, _, err := c.Conn.Prepare(`PRAGMA query_only`)
if err != nil {
return nil, err
@@ -319,7 +327,7 @@ func (c *conn) ExecContext(ctx context.Context, query string, args []driver.Name
return newResult(c.Conn), nil
}
func (*conn) CheckNamedValue(arg *driver.NamedValue) error {
func (c *conn) CheckNamedValue(arg *driver.NamedValue) error {
return nil
}

View File

@@ -1,3 +1,5 @@
//go:build !sqlite3_nosys
package driver
import (
@@ -14,6 +16,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/internal/util"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Test_Open_dir(t *testing.T) {

View File

@@ -1,3 +1,5 @@
//go:build !sqlite3_nosys
package driver_test
// Adapted from: https://go.dev/doc/tutorial/database-access

View File

@@ -6,6 +6,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/vfs/memdb"
)

View File

@@ -8,7 +8,7 @@ BINARYEN="$ROOT/tools/binaryen-version_117/bin"
WASI_SDK="$ROOT/tools/wasi-sdk-22.0/bin"
"$WASI_SDK/clang" --target=wasm32-wasi -std=c17 -flto -g0 -O2 \
-Wall -Wextra -Wno-unused-parameter \
-Wall -Wextra -Wno-unused-parameter \
-o sqlite3.wasm "$ROOT/sqlite3/main.c" \
-I"$ROOT/sqlite3" \
-mexec-model=reactor \

View File

@@ -51,7 +51,9 @@ sqlite3_create_collation_go
sqlite3_create_function_go
sqlite3_create_module_go
sqlite3_create_window_function_go
sqlite3_database_file_object
sqlite3_db_config
sqlite3_db_filename
sqlite3_db_name
sqlite3_db_readonly
sqlite3_db_release_memory
@@ -61,6 +63,9 @@ sqlite3_errmsg
sqlite3_error_offset
sqlite3_errstr
sqlite3_exec
sqlite3_filename_database
sqlite3_filename_journal
sqlite3_filename_wal
sqlite3_finalize
sqlite3_get_autocommit
sqlite3_get_auxdata

Binary file not shown.

View File

@@ -11,6 +11,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/array"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Example_driver() {

View File

@@ -12,6 +12,7 @@ import (
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/array"
"github.com/ncruces/go-sqlite3/ext/blobio"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/vfs/memdb"
)

View File

@@ -8,6 +8,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/csv"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Example() {

View File

@@ -11,6 +11,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/fileio"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Test_lsmode(t *testing.T) {

View File

@@ -12,6 +12,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/fileio"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Test_fsdir(t *testing.T) {

View File

@@ -10,6 +10,7 @@ import (
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Test_writefile(t *testing.T) {

View File

@@ -10,6 +10,7 @@ import (
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "golang.org/x/crypto/blake2b"
_ "golang.org/x/crypto/blake2s"
_ "golang.org/x/crypto/md4"

View File

@@ -14,6 +14,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/lines"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Example() {
@@ -58,7 +59,7 @@ func Example() {
if err := rows.Err(); err != nil {
log.Fatal(err)
}
// Output:
// Expected output:
// US: 141001
// GB: 22560
// CA: 11759

View File

@@ -9,6 +9,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/pivot"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
// https://antonz.org/sqlite-pivot-table/

View File

@@ -8,6 +8,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/statement"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Example() {

View File

@@ -7,6 +7,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/stats"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestRegister_variance(t *testing.T) {

View File

@@ -7,6 +7,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestRegister(t *testing.T) {

View File

@@ -7,6 +7,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/zorder"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestRegister_zorder(t *testing.T) {

3
go.mod
View File

@@ -10,6 +10,9 @@ require (
golang.org/x/sync v0.7.0
golang.org/x/sys v0.19.0
golang.org/x/text v0.14.0
lukechampine.com/adiantum v1.0.0
)
require github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect
retract v0.4.0 // tagged from the wrong branch

11
go.sum
View File

@@ -1,14 +1,25 @@
github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY=
github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/psanford/httpreadat v0.1.0 h1:VleW1HS2zO7/4c7c7zNl33fO6oYACSagjJIyMIwZLUE=
github.com/psanford/httpreadat v0.1.0/go.mod h1:Zg7P+TlBm3bYbyHTKv/EdtSJZn3qwbPwpfZ/I9GKCRE=
github.com/tetratelabs/wazero v1.7.1 h1:QtSfd6KLc41DIMpDYlJdoMc6k7QTN246DM2+n2Y/Dx8=
github.com/tetratelabs/wazero v1.7.1/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
lukechampine.com/adiantum v1.0.0 h1:xxLFgKHyno8ES1XiKLbQfU9DGiMaM2xsIJI2czgT7es=
lukechampine.com/adiantum v1.0.0/go.mod h1:kjMpBiZFjVX/FeEYcN81jyt3//7J3XjJgH9OkAXV4n0=

View File

@@ -3,7 +3,7 @@ module github.com/ncruces/go-sqlite3/gormlite
go 1.21
require (
github.com/ncruces/go-sqlite3 v0.13.0
github.com/ncruces/go-sqlite3 v0.14.0
gorm.io/gorm v1.25.9
)
@@ -11,6 +11,6 @@ require (
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/ncruces/julianday v1.0.0 // indirect
github.com/tetratelabs/wazero v1.7.0 // indirect
github.com/tetratelabs/wazero v1.7.1 // indirect
golang.org/x/sys v0.19.0 // indirect
)

View File

@@ -2,12 +2,12 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/ncruces/go-sqlite3 v0.13.0 h1:+N1VHVLnrCJasyXdAKQL9MNTKC3wHZa8pLMUuP8wb3k=
github.com/ncruces/go-sqlite3 v0.13.0/go.mod h1:y9zPUI+C42V9xuuJeN+tGhpOjr4gUHz2Pi2RLFVEdZg=
github.com/ncruces/go-sqlite3 v0.14.0 h1:R1Jmnc7o5ECQeZPNzbLHfE8vz1DLewV+bJypHSad354=
github.com/ncruces/go-sqlite3 v0.14.0/go.mod h1:NRmOFatwnQaZq8niw0f3k/j3a0yUVt250qcVeDuyANY=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/tetratelabs/wazero v1.7.0 h1:jg5qPydno59wqjpGrHph81lbtHzTrWzwwtD4cD88+hQ=
github.com/tetratelabs/wazero v1.7.0/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
github.com/tetratelabs/wazero v1.7.1 h1:QtSfd6KLc41DIMpDYlJdoMc6k7QTN246DM2+n2Y/Dx8=
github.com/tetratelabs/wazero v1.7.1/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=

View File

@@ -9,6 +9,7 @@ import (
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestDialector(t *testing.T) {

View File

@@ -1,4 +1,4 @@
//go:build (darwin || linux || illumos) && (amd64 || arm64 || riscv64) && !sqlite3_flock && !sqlite3_noshm && !sqlite3_nosys
//go:build (darwin || linux) && (amd64 || arm64 || riscv64) && !(sqlite3_flock || sqlite3_noshm || sqlite3_nosys)
package util
@@ -12,22 +12,13 @@ import (
"golang.org/x/sys/unix"
)
func withMmappedAllocator(ctx context.Context) context.Context {
return experimental.WithMemoryAllocator(ctx,
experimental.MemoryAllocatorFunc(mmappedAllocator))
}
type mmapState struct {
regions []*MappedRegion
enabled bool
}
func (s *mmapState) init(ctx context.Context, enabled bool) context.Context {
if s.enabled = enabled; enabled {
return experimental.WithMemoryAllocator(ctx,
experimental.MemoryAllocatorFunc(mmappedAllocator))
}
return ctx
}
func CanMap(ctx context.Context) bool {
s := ctx.Value(moduleKey{}).(*moduleState)
return s.mmapState.enabled
}
func (s *mmapState) new(ctx context.Context, mod api.Module, size int32) *MappedRegion {

View File

@@ -1,4 +1,4 @@
//go:build !(darwin || linux || illumos) || !(amd64 || arm64 || riscv64) || sqlite3_flock || sqlite3_noshm || sqlite3_nosys
//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64) || sqlite3_flock || sqlite3_noshm || sqlite3_nosys
package util
@@ -6,10 +6,6 @@ import "context"
type mmapState struct{}
func (s *mmapState) init(ctx context.Context, _ bool) context.Context {
func withMmappedAllocator(ctx context.Context) context.Context {
return ctx
}
func CanMap(ctx context.Context) bool {
return false
}

View File

@@ -8,14 +8,14 @@ import (
type moduleKey struct{}
type moduleState struct {
handleState
mmapState
handleState
}
func NewContext(ctx context.Context, mappableMemory bool) context.Context {
func NewContext(ctx context.Context) context.Context {
state := new(moduleState)
ctx = context.WithValue(ctx, moduleKey{}, state)
ctx = withMmappedAllocator(ctx)
ctx = experimental.WithCloseNotifier(ctx, state)
ctx = state.mmapState.init(ctx, mappableMemory)
ctx = context.WithValue(ctx, moduleKey{}, state)
return ctx
}

View File

@@ -85,7 +85,7 @@ func instantiateSQLite() (sqlt *sqlite, err error) {
}
sqlt = new(sqlite)
sqlt.ctx = util.NewContext(context.Background(), vfs.SupportsSharedMemory)
sqlt.ctx = util.NewContext(context.Background())
sqlt.mod, err = instance.runtime.InstantiateModule(sqlt.ctx,
instance.compiled, wazero.NewModuleConfig().WithName(""))

View File

@@ -1,14 +0,0 @@
# Use exclusive locking mode for WAL databases with v1 VFSes.
--- sqlite3.c.orig
+++ sqlite3.c
@@ -64209,7 +64209,9 @@
SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager){
const sqlite3_io_methods *pMethods = pPager->fd->pMethods;
if( pPager->noLock ) return 0;
- return pPager->exclusiveMode || (pMethods->iVersion>=2 && pMethods->xShmMap);
+ if( pMethods->iVersion>=2 && pMethods->xShmMap ) return 1;
+ pPager->exclusiveMode = 1;
+ return 1;
}
/*

View File

@@ -7,10 +7,12 @@ import (
"testing"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/tetratelabs/wazero"
)
func init() {
Path = "./embed/sqlite3.wasm"
RuntimeConfig = wazero.NewRuntimeConfig().WithMemoryLimitPages(1024)
}
func Test_sqlite_error_OOM(t *testing.T) {

View File

@@ -1,3 +1,5 @@
//go:build !sqlite3_nosys
package tests
import (
@@ -6,6 +8,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestBackup(t *testing.T) {

View File

@@ -11,6 +11,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestBlob(t *testing.T) {

View File

@@ -1,3 +1,5 @@
//go:build !sqlite3_nosys
package bradfitz
// Adapted from: https://github.com/bradfitz/go-sql-test
@@ -12,6 +14,7 @@ import (
_ "github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
type Tester interface {

View File

@@ -11,6 +11,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/vfs/memdb"
)
@@ -138,7 +139,7 @@ func TestConn_SetInterrupt(t *testing.T) {
SELECT 0, 1
UNION ALL
SELECT next, curr + next FROM fibonacci
LIMIT 1e6
LIMIT 1e7
)
SELECT min(curr) FROM fibonacci
`)
@@ -415,6 +416,48 @@ func TestConn_SetLastInsertRowID(t *testing.T) {
}
}
func TestConn_Filename(t *testing.T) {
t.Parallel()
file := filepath.Join(t.TempDir(), "test.db")
db, err := sqlite3.Open(file)
if err != nil {
t.Fatal(err)
}
defer db.Close()
n := db.Filename("")
if n.String() != file {
t.Errorf("got %v", n)
}
if n.Database() != file {
t.Errorf("got %v", n)
}
if n.DatabaseFile() == nil {
t.Errorf("got %v", n)
}
n = db.Filename("xpto")
if n != nil {
t.Errorf("got %v", n)
}
if n.String() != "" {
t.Errorf("got %v", n)
}
if n.Database() != "" {
t.Errorf("got %v", n)
}
if n.Journal() != "" {
t.Errorf("got %v", n)
}
if n.WAL() != "" {
t.Errorf("got %v", n)
}
if n.DatabaseFile() != nil {
t.Errorf("got %v", n)
}
}
func TestConn_ReadOnly(t *testing.T) {
t.Parallel()

View File

@@ -9,6 +9,9 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
"github.com/ncruces/go-sqlite3/vfs"
_ "github.com/ncruces/go-sqlite3/vfs/adiantum"
_ "github.com/ncruces/go-sqlite3/vfs/memdb"
)
@@ -24,18 +27,19 @@ func TestDB_memory(t *testing.T) {
}
func TestDB_file(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
t.Parallel()
testDB(t, filepath.Join(t.TempDir(), "test.db"))
}
func TestDB_nolock(t *testing.T) {
t.Parallel()
testDB(t, "file:"+
filepath.ToSlash(filepath.Join(t.TempDir(), "test.db"))+
"?nolock=1")
}
func TestDB_wal(t *testing.T) {
if !vfs.SupportsSharedMemory {
t.Skip("skipping without shared memory")
}
t.Parallel()
tmp := filepath.Join(t.TempDir(), "test.db")
err := os.WriteFile(tmp, waldb, 0666)
@@ -46,6 +50,10 @@ func TestDB_wal(t *testing.T) {
}
func TestDB_utf16(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
t.Parallel()
tmp := filepath.Join(t.TempDir(), "test.db")
err := os.WriteFile(tmp, utf16db, 0666)
@@ -55,10 +63,24 @@ func TestDB_utf16(t *testing.T) {
testDB(t, tmp)
}
func TestDB_vfs(t *testing.T) {
func TestDB_memdb(t *testing.T) {
t.Parallel()
testDB(t, "file:test.db?vfs=memdb")
}
func TestDB_adiantum(t *testing.T) {
t.Parallel()
tmp := filepath.Join(t.TempDir(), "test.db")
testDB(t, "file:"+filepath.ToSlash(tmp)+"?nolock=1"+
"&vfs=adiantum&textkey=correct+horse+battery+staple")
}
func TestDB_nolock(t *testing.T) {
t.Parallel()
tmp := filepath.Join(t.TempDir(), "test.db")
testDB(t, "file:"+filepath.ToSlash(tmp)+"?nolock=1")
}
func testDB(t testing.TB, name string) {
db, err := sqlite3.Open(name)
if err != nil {

View File

@@ -6,6 +6,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestDriver(t *testing.T) {

View File

@@ -5,6 +5,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Test_base64(t *testing.T) {

View File

@@ -6,6 +6,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestCreateFunction(t *testing.T) {

View File

@@ -10,6 +10,7 @@ import (
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
"github.com/ncruces/julianday"
)

View File

@@ -1,3 +1,5 @@
//go:build !sqlite3_nosys
package tests
import (
@@ -12,17 +14,13 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
"github.com/ncruces/go-sqlite3/vfs"
_ "github.com/ncruces/go-sqlite3/vfs/adiantum"
"github.com/ncruces/go-sqlite3/vfs/memdb"
"github.com/tetratelabs/wazero"
)
func TestMain(m *testing.M) {
sqlite3.RuntimeConfig = wazero.NewRuntimeConfig().WithMemoryLimitPages(1024)
os.Exit(m.Run())
}
func TestParallel(t *testing.T) {
func Test_parallel(t *testing.T) {
var iter int
if testing.Short() {
iter = 1000
@@ -39,7 +37,7 @@ func TestParallel(t *testing.T) {
testIntegrity(t, name)
}
func TestWAL(t *testing.T) {
func Test_wal(t *testing.T) {
if !vfs.SupportsSharedMemory {
t.Skip("skipping without shared memory")
}
@@ -53,7 +51,7 @@ func TestWAL(t *testing.T) {
testIntegrity(t, name)
}
func TestMemory(t *testing.T) {
func Test_memdb(t *testing.T) {
var iter int
if testing.Short() {
iter = 1000
@@ -66,6 +64,22 @@ func TestMemory(t *testing.T) {
testIntegrity(t, name)
}
func Test_adiantum(t *testing.T) {
var iter int
if testing.Short() {
iter = 1000
} else {
iter = 5000
}
name := "file:" +
filepath.ToSlash(filepath.Join(t.TempDir(), "test.db")) +
"?vfs=adiantum" +
"&_pragma=hexkey(e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855)"
testParallel(t, name, iter)
testIntegrity(t, name)
}
func TestMultiProcess(t *testing.T) {
if testing.Short() {
t.Skip("skipping in short mode")
@@ -117,7 +131,7 @@ func TestChildProcess(t *testing.T) {
testParallel(t, name, 1000)
}
func BenchmarkMemory(b *testing.B) {
func Benchmark_memdb(b *testing.B) {
memdb.Delete("test.db")
name := "file:/test.db?vfs=memdb"
testParallel(b, name, b.N)

View File

@@ -8,6 +8,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestStmt(t *testing.T) {

10
tests/testcfg/testcfg.go Normal file
View File

@@ -0,0 +1,10 @@
package testcfg
import (
"github.com/ncruces/go-sqlite3"
"github.com/tetratelabs/wazero"
)
func init() {
sqlite3.RuntimeConfig = wazero.NewRuntimeConfig().WithMemoryLimitPages(1024)
}

View File

@@ -14,7 +14,7 @@ sudo mount -nv -o loop f2fs.img f2fs/
mkdir -p f2fs/tmp/
go test -c "$ROOT/tests" -coverpkg github.com/ncruces/go-sqlite3/...
TMPDIR=f2fs/tmp/ ./tests.test -test.v -test.short -test.coverprofile cover.out
TMPDIR=$PWD/f2fs/tmp/ ./tests.test -test.v -test.short -test.coverprofile cover.out
go tool cover -html cover.out
sudo umount f2fs/

View File

@@ -10,6 +10,7 @@ import (
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestTimeFormat_Encode(t *testing.T) {

View File

@@ -7,6 +7,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestConn_Transaction_exec(t *testing.T) {

View File

@@ -6,6 +6,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
"github.com/ncruces/go-sqlite3/vfs/memdb"
"github.com/ncruces/go-sqlite3/vfs/readervfs"
)

View File

@@ -1,3 +1,5 @@
//go:build !sqlite3_nosys
package tests
import (
@@ -6,6 +8,8 @@ import (
"testing"
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
"github.com/ncruces/go-sqlite3/vfs"
)
@@ -20,6 +24,13 @@ func TestWAL_enter_exit(t *testing.T) {
}
defer db.Close()
if !vfs.SupportsSharedMemory {
err = db.Exec(`PRAGMA locking_mode=EXCLUSIVE`)
if err != nil {
t.Fatal(err)
}
}
err = db.Exec(`
CREATE TABLE test (col);
PRAGMA journal_mode=WAL;
@@ -87,6 +98,7 @@ func TestConn_WalCheckpoint(t *testing.T) {
})
err = db.Exec(`
PRAGMA locking_mode=EXCLUSIVE;
PRAGMA journal_mode=WAL;
CREATE TABLE test (col);
`)

View File

@@ -8,6 +8,7 @@ import (
// SeekingReaderAt implements [io.ReaderAt]
// through an underlying [io.ReadSeeker].
type SeekingReaderAt struct {
// +checklocks:l
r io.ReadSeeker
l sync.Mutex
}

View File

@@ -15,16 +15,16 @@ The main differences are [file locking](#file-locking) and [WAL mode](write-ahea
### File Locking
POSIX advisory locks, which SQLite uses on Unix, are
[broken by design](https://sqlite.org/src/artifact/2e8b12?ln=1073-1161).
[broken by design](https://github.com/sqlite/sqlite/blob/b74eb0/src/os_unix.c#L1073-L1161).
On Linux, macOS and illumos, this module uses
On Linux and macOS, this module uses
[OFD locks](https://www.gnu.org/software/libc/manual/html_node/Open-File-Description-Locks.html)
to synchronize access to database files.
OFD locks are fully compatible with POSIX advisory locks.
On BSD Unixes, this module uses
[BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2).
On BSD Unixes, BSD locks are fully compatible with POSIX advisory locks.
On BSD, these locks are fully compatible with POSIX advisory locks.
However, concurrency is reduced with BSD locks
(`BEGIN IMMEDIATE` behaves the same as `BEGIN EXCLUSIVE`).
@@ -34,8 +34,7 @@ like SQLite.
On all other platforms, file locking is not supported, and you must use
[`nolock=1`](https://sqlite.org/uri.html#urinolock)
(or [`immutable=1`](https://sqlite.org/uri.html#uriimmutable))
to open database files.
to open database files.\
To use the [`database/sql`](https://pkg.go.dev/database/sql) driver
with `nolock=1` you must disable connection pooling by calling
[`db.SetMaxOpenConns(1)`](https://pkg.go.dev/database/sql#DB.SetMaxOpenConns).
@@ -45,31 +44,31 @@ to check if your platform supports file locking.
### Write-Ahead Logging
On 64-bit Linux, macOS and illumos, this module uses `mmap` to implement
On 64-bit Linux and macOS, this module uses `mmap` to implement
[shared-memory for the WAL-index](https://sqlite.org/wal.html#implementation_of_shared_memory_for_the_wal_index),
like SQLite.
To allow `mmap` to work, each connection needs to reserve up to 4GB of address space.\
To limit the amount of address space each connection needs,
use [`WithMemoryLimitPages`](../tests/parallel/parallel_test.go#L21).
use [`WithMemoryLimitPages`](../tests/testcfg/testcfg.go).
On all other platforms, [WAL](https://sqlite.org/wal.html) support is
On Windows and BSD, [WAL](https://sqlite.org/wal.html) support is
[limited](https://sqlite.org/wal.html#noshm).
To work around this limitation, SQLite is [patched](sqlite3/locking_mode.patch)
to automatically use `EXCLUSIVE` locking mode for WAL databases on such platforms.
To use the [`database/sql`](https://pkg.go.dev/database/sql) driver
with `EXCLUSIVE` locking mode you should disable connection pooling by calling
`EXCLUSIVE` locking mode can be set to create, read, and write WAL databases.\
To use `EXCLUSIVE` locking mode with the
[`database/sql`](https://pkg.go.dev/database/sql) driver
you must disable connection pooling by calling
[`db.SetMaxOpenConns(1)`](https://pkg.go.dev/database/sql#DB.SetMaxOpenConns).
On all other platforms, where file locking is not supported, WAL mode does not work.
You can use [`vfs.SupportsSharedMemory`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs#SupportsSharedMemory)
to check if your platform supports shared memory.
### Batch-Atomic Write
On 64-bit Linux, this module supports [batch-atomic writes](https://sqlite.org/cgi/src/technote/714)
on the F2FS filesystem.
with the F2FS filesystem.
### Build tags
@@ -77,4 +76,4 @@ The VFS can be customized with a few build tags:
- `sqlite3_flock` forces the use of BSD locks; it can be used on macOS to test the BSD locking implementation.
- `sqlite3_nosys` prevents importing [`x/sys`](https://pkg.go.dev/golang.org/x/sys);
disables locking _and_ shared memory on all platforms.
- `sqlite3_noshm` disables shared memory on all platforms.
- `sqlite3_noshm` disables shared memory on all platforms.

64
vfs/adiantum/README.md Normal file
View File

@@ -0,0 +1,64 @@
# Go `"adiantum"` SQLite VFS
This package wraps an SQLite VFS to offer encryption at rest.
> [!WARNING]
> This work was not certified by a cryptographer.
> If you need vetted encryption, you should purchase the
> [SQLite Encryption Extension](https://sqlite.org/see),
> and either wrap it, or seek assistance wrapping it.
The `"adiantum"` VFS wraps the default SQLite VFS using the
[Adiantum](https://github.com/lukechampine/adiantum)
tweakable and length-preserving encryption.\
In general, any HBSH construction can be used to wrap any VFS.
The default Adiantum construction uses XChaCha12 for its stream cipher,
AES for its block cipher, and NH and Poly1305 for hashing.\
Additionally, we use [Argon2id](https://pkg.go.dev/golang.org/x/crypto/argon2#hdr-Argon2id)
to derive 256-bit keys from plain text where needed.
File contents are encrypted in 4K blocks, matching the
[default](https://sqlite.org/pgszchng2016.html) SQLite page size.
The VFS encrypts all files _except_
[super journals](https://sqlite.org/tempfiles.html#super_journal_files):
these _never_ contain database data, only filenames,
and padding them to the block size is problematic.
Temporary files _are_ encrypted with **random** keys,
as they _may_ contain database data.
To avoid the overhead of encrypting temporary files,
keep them in memory:
PRAGMA temp_store = memory;
> [!IMPORTANT]
> Adiantum is a cipher composition for disk encryption.
> The standard threat model for disk encryption considers an adversary
> that can read multiple snapshots of a disk.
> The only security property that disk encryption provides
> is that all information such an adversary can obtain
> is whether the data in a sector has or has not changed over time.
The encryption offered by this package is fully deterministic.
This means that an adversary who can get ahold of multiple snapshots
(e.g. backups) of a database file can learn precisely:
which blocks changed, which ones didn't, which got reverted.
This is slightly weaker than other forms of SQLite encryption
that include *some* nondeterminism; with limited nondeterminism,
an adversary can't distinguish between
blocks that actually changed, and blocks that got reverted.
> [!CAUTION]
> This package does not claim protect databases against tampering or forgery.
The major practical consequence of the above point is that,
if you're keeping `"adiantum"` encrypted backups of your database,
and want to protect against forgery, you should sign your backups,
and verify signatures before restoring them.
This is slightly weaker than other forms of SQLite encryption
that include block-level [MACs](https://en.wikipedia.org/wiki/Message_authentication_code).
Block-level MACs can protect against forging individual blocks,
but can't prevent them from being reverted to former versions of themselves.

29
vfs/adiantum/adiantum.go Normal file
View File

@@ -0,0 +1,29 @@
package adiantum
import (
"crypto/rand"
"golang.org/x/crypto/argon2"
"lukechampine.com/adiantum"
"lukechampine.com/adiantum/hbsh"
)
const pepper = "github.com/ncruces/go-sqlite3/vfs/adiantum"
type adiantumCreator struct{}
func (adiantumCreator) HBSH(key []byte) *hbsh.HBSH {
if len(key) != 32 {
return nil
}
return adiantum.New(key)
}
func (adiantumCreator) KDF(text string) []byte {
if text == "" {
key := make([]byte, 32)
n, _ := rand.Read(key)
return key[:n]
}
return argon2.IDKey([]byte(text), []byte(pepper), 1, 64*1024, 4, 32)
}

62
vfs/adiantum/api.go Normal file
View File

@@ -0,0 +1,62 @@
// Package adiantum wraps an SQLite VFS to offer encryption at rest.
//
// The "adiantum" [vfs.VFS] wraps the default VFS using the
// Adiantum tweakable, length-preserving encryption.
//
// Importing package adiantum registers that VFS:
//
// import _ "github.com/ncruces/go-sqlite3/vfs/adiantum"
//
// To open an encrypted database you need to provide key material.
//
// The simplest way to do that is to specify the key through an [URI] parameter:
//
// - key: key material in binary (32 bytes)
// - hexkey: key material in hex (64 hex digits)
// - textkey: key material in text (any length)
//
// However, this makes your key easily accessible to other parts of
// your application (e.g. through [vfs.Filename.URIParameters]).
//
// To avoid this, use any of the following PRAGMAs:
//
// PRAGMA key='D41d8cD98f00b204e9800998eCf8427e';
// PRAGMA hexkey='e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855';
// PRAGMA textkey='your-secret-key';
//
// [URI]: https://sqlite.org/uri.html
package adiantum
import (
"github.com/ncruces/go-sqlite3/vfs"
"lukechampine.com/adiantum/hbsh"
)
func init() {
Register("adiantum", vfs.Find(""), nil)
}
// Register registers an encrypting VFS, wrapping a base VFS,
// and possibly using a custom HBSH cipher construction.
// To use the default Adiantum construction, set cipher to nil.
func Register(name string, base vfs.VFS, cipher HBSHCreator) {
if cipher == nil {
cipher = adiantumCreator{}
}
vfs.Register(name, &hbshVFS{
VFS: base,
hbsh: cipher,
})
}
// HBSHCreator creates an [hbsh.HBSH] cipher
// given key material.
type HBSHCreator interface {
// KDF derives an HBSH key from a secret.
// If no secret is given, a random key is generated.
KDF(secret string) (key []byte)
// HBSH creates an HBSH cipher given a key.
// If key is not appropriate, nil is returned.
HBSH(key []byte) *hbsh.HBSH
}

277
vfs/adiantum/hbsh.go Normal file
View File

@@ -0,0 +1,277 @@
package adiantum
import (
"encoding/binary"
"encoding/hex"
"io"
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/vfs"
"lukechampine.com/adiantum/hbsh"
)
type hbshVFS struct {
vfs.VFS
hbsh HBSHCreator
}
func (h *hbshVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
return nil, 0, sqlite3.CANTOPEN
}
func (h *hbshVFS) OpenFilename(name *vfs.Filename, flags vfs.OpenFlag) (file vfs.File, _ vfs.OpenFlag, err error) {
if h, ok := h.VFS.(vfs.VFSFilename); ok {
file, flags, err = h.OpenFilename(name, flags)
} else {
file, flags, err = h.Open(name.String(), flags)
}
// Encrypt everything except super journals and memory files.
if err != nil || flags&(vfs.OPEN_SUPER_JOURNAL|vfs.OPEN_MEMORY) != 0 {
return file, flags, err
}
var hbsh *hbsh.HBSH
if f, ok := name.DatabaseFile().(*hbshFile); ok {
hbsh = f.hbsh
} else {
var key []byte
if params := name.URIParameters(); name == nil {
key = h.hbsh.KDF("") // Temporary files get a random key.
} else if t, ok := params["key"]; ok {
key = []byte(t[0])
} else if t, ok := params["hexkey"]; ok {
key, _ = hex.DecodeString(t[0])
} else if t, ok := params["textkey"]; ok {
key = h.hbsh.KDF(t[0])
}
hbsh = h.hbsh.HBSH(key)
}
// Main datatabases may have their key specified later, as a PRAGMA.
if hbsh != nil || flags&vfs.OPEN_MAIN_DB != 0 {
return &hbshFile{File: file, hbsh: hbsh, reset: h.hbsh}, flags, nil
}
return nil, flags, sqlite3.CANTOPEN
}
const (
tweakSize = 8
blockSize = 4096
)
type hbshFile struct {
vfs.File
hbsh *hbsh.HBSH
reset HBSHCreator
tweak [tweakSize]byte
block [blockSize]byte
}
func (h *hbshFile) Pragma(name string, value string) (string, error) {
var key []byte
switch name {
case "key":
key = []byte(value)
case "hexkey":
key, _ = hex.DecodeString(value)
case "textkey":
key = h.reset.KDF(value)
default:
if f, ok := h.File.(vfs.FilePragma); ok {
return f.Pragma(name, value)
}
return "", sqlite3.NOTFOUND
}
if h.hbsh = h.reset.HBSH(key); h.hbsh != nil {
return "ok", nil
}
return "", sqlite3.CANTOPEN
}
func (h *hbshFile) ReadAt(p []byte, off int64) (n int, err error) {
if h.hbsh == nil {
// Only OPEN_MAIN_DB can have a missing key.
if off == 0 && len(p) == 100 {
// SQLite is trying to read the header of a database file.
// Pretend the file is empty so the key may specified later,
// as a PRAGMA.
return 0, io.EOF
}
return 0, sqlite3.CANTOPEN
}
min := (off) &^ (blockSize - 1) // round down
max := (off + int64(len(p)) + (blockSize - 1)) &^ (blockSize - 1) // round up
// Read one block at a time.
for ; min < max; min += blockSize {
m, err := h.File.ReadAt(h.block[:], min)
if m != blockSize {
return n, err
}
binary.LittleEndian.PutUint64(h.tweak[:], uint64(min))
data := h.hbsh.Decrypt(h.block[:], h.tweak[:])
if off > min {
data = data[off-min:]
}
n += copy(p[n:], data)
}
if n != len(p) {
panic(util.AssertErr())
}
return n, nil
}
func (h *hbshFile) WriteAt(p []byte, off int64) (n int, err error) {
if h.hbsh == nil {
return 0, sqlite3.READONLY
}
min := (off) &^ (blockSize - 1) // round down
max := (off + int64(len(p)) + (blockSize - 1)) &^ (blockSize - 1) // round up
// Write one block at a time.
for ; min < max; min += blockSize {
binary.LittleEndian.PutUint64(h.tweak[:], uint64(min))
data := h.block[:]
if off > min || len(p[n:]) < blockSize {
// Partial block write: read-update-write.
m, err := h.File.ReadAt(h.block[:], min)
if m != blockSize {
if err != io.EOF {
return n, err
}
// Writing past the EOF.
// We're either appending an entirely new block,
// or the final block was only partially written.
// A partially written block can't be decrypted,
// and is as good as corrupt.
// Either way, zero pad the file to the next block size.
clear(data)
} else {
data = h.hbsh.Decrypt(h.block[:], h.tweak[:])
}
if off > min {
data = data[off-min:]
}
}
t := copy(data, p[n:])
h.hbsh.Encrypt(h.block[:], h.tweak[:])
m, err := h.File.WriteAt(h.block[:], min)
if m != blockSize {
return n, err
}
n += t
}
if n != len(p) {
panic(util.AssertErr())
}
return n, nil
}
func (h *hbshFile) Truncate(size int64) error {
size = (size + (blockSize - 1)) &^ (blockSize - 1) // round up
return h.File.Truncate(size)
}
func (h *hbshFile) SectorSize() int {
return max(h.File.SectorSize(), blockSize)
}
func (h *hbshFile) DeviceCharacteristics() vfs.DeviceCharacteristic {
return h.File.DeviceCharacteristics() & (0 |
// The only safe flags are these:
vfs.IOCAP_UNDELETABLE_WHEN_OPEN |
vfs.IOCAP_IMMUTABLE |
vfs.IOCAP_BATCH_ATOMIC)
}
// Wrap optional methods.
func (h *hbshFile) SharedMemory() vfs.SharedMemory {
if f, ok := h.File.(vfs.FileSharedMemory); ok {
return f.SharedMemory()
}
return nil
}
func (h *hbshFile) ChunkSize(size int) {
if f, ok := h.File.(vfs.FileChunkSize); ok {
size = (size + (blockSize - 1)) &^ (blockSize - 1) // round up
f.ChunkSize(size)
}
}
func (h *hbshFile) SizeHint(size int64) error {
if f, ok := h.File.(vfs.FileSizeHint); ok {
size = (size + (blockSize - 1)) &^ (blockSize - 1) // round up
return f.SizeHint(size)
}
return sqlite3.NOTFOUND
}
func (h *hbshFile) HasMoved() (bool, error) {
if f, ok := h.File.(vfs.FileHasMoved); ok {
return f.HasMoved()
}
return false, sqlite3.NOTFOUND
}
func (h *hbshFile) Overwrite() error {
if f, ok := h.File.(vfs.FileOverwrite); ok {
return f.Overwrite()
}
return sqlite3.NOTFOUND
}
func (h *hbshFile) CommitPhaseTwo() error {
if f, ok := h.File.(vfs.FileCommitPhaseTwo); ok {
return f.CommitPhaseTwo()
}
return sqlite3.NOTFOUND
}
func (h *hbshFile) BeginAtomicWrite() error {
if f, ok := h.File.(vfs.FileBatchAtomicWrite); ok {
return f.BeginAtomicWrite()
}
return sqlite3.NOTFOUND
}
func (h *hbshFile) CommitAtomicWrite() error {
if f, ok := h.File.(vfs.FileBatchAtomicWrite); ok {
return f.CommitAtomicWrite()
}
return sqlite3.NOTFOUND
}
func (h *hbshFile) RollbackAtomicWrite() error {
if f, ok := h.File.(vfs.FileBatchAtomicWrite); ok {
return f.RollbackAtomicWrite()
}
return sqlite3.NOTFOUND
}
func (h *hbshFile) CheckpointDone() error {
if f, ok := h.File.(vfs.FileCheckpoint); ok {
return f.CheckpointDone()
}
return sqlite3.NOTFOUND
}
func (h *hbshFile) CheckpointStart() error {
if f, ok := h.File.(vfs.FileCheckpoint); ok {
return f.CheckpointStart()
}
return sqlite3.NOTFOUND
}

View File

@@ -1,7 +1,12 @@
// Package vfs wraps the C SQLite VFS API.
package vfs
import "net/url"
import (
"context"
"io"
"github.com/tetratelabs/wazero/api"
)
// A VFS defines the interface between the SQLite core and the underlying operating system.
//
@@ -15,13 +20,13 @@ type VFS interface {
FullPathname(name string) (string, error)
}
// VFSParams extends VFS with the ability to handle URI parameters
// through the OpenParams method.
// VFSFilename extends VFS with the ability to use Filename
// objects for opening files.
//
// https://sqlite.org/c3ref/uri_boolean.html
type VFSParams interface {
// https://sqlite.org/c3ref/filename.html
type VFSFilename interface {
VFS
OpenParams(name string, flags OpenFlag, params url.Values) (File, OpenFlag, error)
OpenFilename(name *Filename, flags OpenFlag) (File, OpenFlag, error)
}
// A File represents an open file in the OS interface layer.
@@ -53,6 +58,15 @@ type FileLockState interface {
LockState() LockLevel
}
// FileChunkSize extends File to implement the
// SQLITE_FCNTL_CHUNK_SIZE file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlchunksize
type FileChunkSize interface {
File
ChunkSize(size int)
}
// FileSizeHint extends File to implement the
// SQLITE_FCNTL_SIZE_HINT file control opcode.
//
@@ -120,3 +134,42 @@ type FileBatchAtomicWrite interface {
CommitAtomicWrite() error
RollbackAtomicWrite() error
}
// FilePragma extends File to implement the
// SQLITE_FCNTL_PRAGMA file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpragma
type FilePragma interface {
File
Pragma(name, value string) (string, error)
}
// FileCheckpoint extends File to implement the
// SQLITE_FCNTL_CKPT_START and SQLITE_FCNTL_CKPT_DONE
// file control opcodes.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlckptstart
type FileCheckpoint interface {
File
CheckpointDone() error
CheckpointStart() error
}
// FileSharedMemory extends File to possibly implement
// shared-memory for the WAL-index.
// The same shared-memory instance must be returned
// for the entire life of the file.
// It's OK for SharedMemory to return nil.
type FileSharedMemory interface {
File
SharedMemory() SharedMemory
}
// SharedMemory is a shared-memory WAL-index implementation.
// Use [NewSharedMemory] to create a shared-memory.
type SharedMemory interface {
shmMap(context.Context, api.Module, int32, int32, bool) (uint32, error)
shmLock(int32, int32, _ShmFlag) error
shmUnmap(bool)
io.Closer
}

View File

@@ -4,8 +4,11 @@ import "github.com/ncruces/go-sqlite3/internal/util"
const (
_MAX_NAME = 1e6 // Self-imposed limit for most NUL terminated strings.
_MAX_SQL_LENGTH = 1e9
_MAX_PATHNAME = 1024
_DEFAULT_SECTOR_SIZE = 4096
ptrlen = 4
)
// https://sqlite.org/rescode.html
@@ -17,6 +20,7 @@ func (e _ErrorCode) Error() string {
const (
_OK _ErrorCode = util.OK
_ERROR _ErrorCode = util.ERROR
_PERM _ErrorCode = util.PERM
_BUSY _ErrorCode = util.BUSY
_READONLY _ErrorCode = util.READONLY

View File

@@ -1,18 +1,15 @@
package vfs
import (
"context"
"errors"
"io"
"io/fs"
"net/url"
"os"
"path/filepath"
"runtime"
"syscall"
"github.com/ncruces/go-sqlite3/util/osutil"
"github.com/tetratelabs/wazero/api"
)
type vfsOS struct{}
@@ -72,10 +69,10 @@ func (vfsOS) Access(name string, flags AccessFlag) (bool, error) {
}
func (vfsOS) Open(name string, flags OpenFlag) (File, OpenFlag, error) {
return vfsOS{}.OpenParams(name, flags, nil)
return nil, 0, _CANTOPEN
}
func (vfsOS) OpenParams(name string, flags OpenFlag, params url.Values) (File, OpenFlag, error) {
func (vfsOS) OpenFilename(name *Filename, flags OpenFlag) (File, OpenFlag, error) {
var oflags int
if flags&OPEN_EXCLUSIVE != 0 {
oflags |= os.O_EXCL
@@ -92,10 +89,10 @@ func (vfsOS) OpenParams(name string, flags OpenFlag, params url.Values) (File, O
var err error
var f *os.File
if name == "" {
if name == nil {
f, err = os.CreateTemp("", "*.db")
} else {
f, err = osutil.OpenFile(name, oflags, 0666)
f, err = osutil.OpenFile(name.String(), oflags, 0666)
}
if err != nil {
if errors.Is(err, syscall.EISDIR) {
@@ -104,7 +101,7 @@ func (vfsOS) OpenParams(name string, flags OpenFlag, params url.Values) (File, O
return nil, flags, err
}
if modeof := params.Get("modeof"); modeof != "" {
if modeof := name.URIParameter("modeof"); modeof != "" {
if err = osSetMode(f, modeof); err != nil {
f.Close()
return nil, flags, _IOERR_FSTAT
@@ -121,13 +118,14 @@ func (vfsOS) OpenParams(name string, flags OpenFlag, params url.Values) (File, O
syncDir: runtime.GOOS != "windows" &&
flags&(OPEN_CREATE) != 0 &&
flags&(OPEN_MAIN_JOURNAL|OPEN_SUPER_JOURNAL|OPEN_WAL) != 0,
shm: NewSharedMemory(name.String()+"-shm", flags),
}
return &file, flags, nil
}
type vfsFile struct {
*os.File
shm vfsShm
shm SharedMemory
lock LockLevel
readOnly bool
keepWAL bool
@@ -145,7 +143,9 @@ var (
)
func (f *vfsFile) Close() error {
f.shm.Close()
if f.shm != nil {
f.shm.Close()
}
return f.File.Close()
}
@@ -176,7 +176,7 @@ func (f *vfsFile) Size() (int64, error) {
return f.Seek(0, io.SeekEnd)
}
func (*vfsFile) SectorSize() int {
func (f *vfsFile) SectorSize() int {
return _DEFAULT_SECTOR_SIZE
}
@@ -215,9 +215,3 @@ func (f *vfsFile) PowersafeOverwrite() bool { return f.psow }
func (f *vfsFile) PersistentWAL() bool { return f.keepWAL }
func (f *vfsFile) SetPowersafeOverwrite(psow bool) { f.psow = psow }
func (f *vfsFile) SetPersistentWAL(keepWAL bool) { f.keepWAL = keepWAL }
type fileShm interface {
shmMap(context.Context, api.Module, int32, int32, bool) (uint32, error)
shmLock(int32, int32, _ShmFlag) error
shmUnmap(bool)
}

174
vfs/filename.go Normal file
View File

@@ -0,0 +1,174 @@
package vfs
import (
"context"
"net/url"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/tetratelabs/wazero/api"
)
// Filename is used by SQLite to pass filenames
// to the Open method of a VFS.
//
// https://sqlite.org/c3ref/filename.html
type Filename struct {
ctx context.Context
mod api.Module
zPath uint32
flags OpenFlag
stack [2]uint64
}
// OpenFilename is an internal API users should not call directly.
func OpenFilename(ctx context.Context, mod api.Module, id uint32, flags OpenFlag) *Filename {
if id == 0 {
return nil
}
return &Filename{
ctx: ctx,
mod: mod,
zPath: id,
flags: flags,
}
}
// String returns this filename as a string.
func (n *Filename) String() string {
if n == nil || n.zPath == 0 {
return ""
}
return util.ReadString(n.mod, n.zPath, _MAX_PATHNAME)
}
// Database returns the name of the corresponding database file.
//
// https://sqlite.org/c3ref/filename_database.html
func (n *Filename) Database() string {
return n.path("sqlite3_filename_database")
}
// Journal returns the name of the corresponding rollback journal file.
//
// https://sqlite.org/c3ref/filename_database.html
func (n *Filename) Journal() string {
return n.path("sqlite3_filename_journal")
}
// Journal returns the name of the corresponding WAL file.
//
// https://sqlite.org/c3ref/filename_database.html
func (n *Filename) WAL() string {
return n.path("sqlite3_filename_wal")
}
func (n *Filename) path(method string) string {
if n == nil || n.zPath == 0 {
return ""
}
n.stack[0] = uint64(n.zPath)
fn := n.mod.ExportedFunction(method)
if err := fn.CallWithStack(n.ctx, n.stack[:]); err != nil {
panic(err)
}
return util.ReadString(n.mod, uint32(n.stack[0]), _MAX_PATHNAME)
}
// DatabaseFile returns the main database [File] corresponding to a journal.
//
// https://sqlite.org/c3ref/database_file_object.html
func (n *Filename) DatabaseFile() File {
if n == nil || n.zPath == 0 {
return nil
}
if n.flags&(OPEN_MAIN_DB|OPEN_MAIN_JOURNAL|OPEN_WAL) == 0 {
return nil
}
n.stack[0] = uint64(n.zPath)
fn := n.mod.ExportedFunction("sqlite3_database_file_object")
if err := fn.CallWithStack(n.ctx, n.stack[:]); err != nil {
panic(err)
}
file, _ := vfsFileGet(n.ctx, n.mod, uint32(n.stack[0])).(File)
return file
}
// URIParameter returns the value of a URI parameter.
//
// https://sqlite.org/c3ref/uri_boolean.html
func (n *Filename) URIParameter(key string) string {
if n == nil || n.zPath == 0 {
return ""
}
uriKey := n.mod.ExportedFunction("sqlite3_uri_key")
n.stack[0] = uint64(n.zPath)
n.stack[1] = uint64(0)
if err := uriKey.CallWithStack(n.ctx, n.stack[:]); err != nil {
panic(err)
}
ptr := uint32(n.stack[0])
if ptr == 0 {
return ""
}
// Parse the format from:
// https://github.com/sqlite/sqlite/blob/b74eb0/src/pager.c#L4797-L4840
// This avoids having to alloc/free the key just to find a value.
for {
k := util.ReadString(n.mod, ptr, _MAX_NAME)
if k == "" {
return ""
}
ptr += uint32(len(k)) + 1
v := util.ReadString(n.mod, ptr, _MAX_NAME)
if k == key {
return v
}
ptr += uint32(len(v)) + 1
}
}
// URIParameters obtains values for URI parameters.
//
// https://sqlite.org/c3ref/uri_boolean.html
func (n *Filename) URIParameters() url.Values {
if n == nil || n.zPath == 0 {
return nil
}
uriKey := n.mod.ExportedFunction("sqlite3_uri_key")
n.stack[0] = uint64(n.zPath)
n.stack[1] = uint64(0)
if err := uriKey.CallWithStack(n.ctx, n.stack[:]); err != nil {
panic(err)
}
ptr := uint32(n.stack[0])
if ptr == 0 {
return nil
}
var params url.Values
// Parse the format from:
// https://github.com/sqlite/sqlite/blob/b74eb0/src/pager.c#L4797-L4840
// This is the only way to support multiple valued keys.
for {
k := util.ReadString(n.mod, ptr, _MAX_NAME)
if k == "" {
return params
}
ptr += uint32(len(k)) + 1
v := util.ReadString(n.mod, ptr, _MAX_NAME)
if params == nil {
params = url.Values{}
}
params.Add(k, v)
ptr += uint32(len(v)) + 1
}
}

View File

@@ -1,3 +1,5 @@
//go:build !sqlite3_nosys
package vfs
import (
@@ -33,7 +35,7 @@ func Test_vfsLock(t *testing.T) {
pOutput = 32
)
mod := wazerotest.NewModule(wazerotest.NewMemory(wazerotest.PageSize))
ctx := util.NewContext(context.TODO(), false)
ctx := util.NewContext(context.TODO())
vfsFileRegister(ctx, mod, pFile1, &vfsFile{File: file1})
vfsFileRegister(ctx, mod, pFile2, &vfsFile{File: file2})

View File

@@ -4,7 +4,7 @@
// among multiple database connections in the same process,
// as long as the database name begins with "/".
//
// Importing package memdb registers the VFS.
// Importing package memdb registers the VFS:
//
// import _ "github.com/ncruces/go-sqlite3/vfs/memdb"
package memdb

View File

@@ -11,14 +11,23 @@ import (
"github.com/ncruces/go-sqlite3/vfs"
)
// Must be a multiple of 64K (the largest page size).
const sectorSize = 65536
type memVFS struct{}
func (memVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
// Allowed file types:
// For simplicity, we do not support reading or writing data
// across "sector" boundaries.
//
// This is not a problem for most SQLite file types:
// - databases, which only do page aligned reads/writes;
// - temp journals, used by the sorter, which does the same.
// - temp journals, as used by the sorter, which does the same:
// https://github.com/sqlite/sqlite/blob/b74eb0/src/vdbesort.c#L409-L412
//
// We refuse to open all other file types,
// but returning OPEN_MEMORY means SQLite won't ask us to.
const types = vfs.OPEN_MAIN_DB |
vfs.OPEN_TRANSIENT_DB |
vfs.OPEN_TEMP_DB |
vfs.OPEN_TEMP_JOURNAL
if flags&types == 0 {
@@ -61,9 +70,6 @@ func (memVFS) FullPathname(name string) (string, error) {
return name, nil
}
// Must be a multiple of 64K (the largest page size).
const sectorSize = 65536
type memDB struct {
// +checklocks:lockMtx
pending *memFile
@@ -166,7 +172,7 @@ func (m *memFile) truncate(size int64) error {
return nil
}
func (*memFile) Sync(flag vfs.SyncFlag) error {
func (m *memFile) Sync(flag vfs.SyncFlag) error {
return nil
}
@@ -256,11 +262,11 @@ func (m *memFile) CheckReservedLock() (bool, error) {
return m.reserved != nil, nil
}
func (*memFile) SectorSize() int {
func (m *memFile) SectorSize() int {
return sectorSize
}
func (*memFile) DeviceCharacteristics() vfs.DeviceCharacteristic {
func (m *memFile) DeviceCharacteristics() vfs.DeviceCharacteristic {
return vfs.IOCAP_ATOMIC |
vfs.IOCAP_SEQUENTIAL |
vfs.IOCAP_SAFE_APPEND |

View File

@@ -7,6 +7,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
//go:embed testdata/wal.db

View File

@@ -1,4 +1,4 @@
//go:build (freebsd || openbsd || netbsd || dragonfly || sqlite3_flock) && !sqlite3_nosys
//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && !sqlite3_nosys
package vfs

View File

@@ -3,7 +3,9 @@
package vfs
import (
"math/rand"
"os"
"time"
"golang.org/x/sys/unix"
)
@@ -19,3 +21,51 @@ func osAllocate(file *os.File, size int64) error {
}
return unix.Fallocate(int(file.Fd()), 0, 0, size)
}
func osUnlock(file *os.File, start, len int64) _ErrorCode {
err := unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &unix.Flock_t{
Type: unix.F_UNLCK,
Start: start,
Len: len,
})
if err != nil {
return _IOERR_UNLOCK
}
return _OK
}
func osLock(file *os.File, typ int16, start, len int64, timeout time.Duration, def _ErrorCode) _ErrorCode {
lock := unix.Flock_t{
Type: typ,
Start: start,
Len: len,
}
var err error
switch {
case timeout == 0:
err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &lock)
case timeout < 0:
err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLKW, &lock)
default:
before := time.Now()
for {
err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &lock)
if errno, _ := err.(unix.Errno); errno != unix.EAGAIN {
break
}
if timeout < time.Since(before) {
break
}
osSleep(time.Duration(rand.Int63n(int64(time.Millisecond))))
}
}
return osLockErrorCode(err, def)
}
func osReadLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode {
return osLock(file, unix.F_RDLCK, start, len, timeout, _IOERR_RDLOCK)
}
func osWriteLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode {
return osLock(file, unix.F_WRLCK, start, len, timeout, _IOERR_LOCK)
}

View File

@@ -1,59 +0,0 @@
//go:build (linux || illumos) && !sqlite3_nosys
package vfs
import (
"math/rand"
"os"
"time"
"golang.org/x/sys/unix"
)
func osUnlock(file *os.File, start, len int64) _ErrorCode {
err := unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &unix.Flock_t{
Type: unix.F_UNLCK,
Start: start,
Len: len,
})
if err != nil {
return _IOERR_UNLOCK
}
return _OK
}
func osLock(file *os.File, typ int16, start, len int64, timeout time.Duration, def _ErrorCode) _ErrorCode {
lock := unix.Flock_t{
Type: typ,
Start: start,
Len: len,
}
var err error
switch {
case timeout == 0:
err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &lock)
case timeout < 0:
err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLKW, &lock)
default:
before := time.Now()
for {
err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &lock)
if errno, _ := err.(unix.Errno); errno != unix.EAGAIN {
break
}
if timeout < time.Since(before) {
break
}
osSleep(time.Duration(rand.Int63n(int64(time.Millisecond))))
}
}
return osLockErrorCode(err, def)
}
func osReadLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode {
return osLock(file, unix.F_RDLCK, start, len, timeout, _IOERR_RDLOCK)
}
func osWriteLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode {
return osLock(file, unix.F_WRLCK, start, len, timeout, _IOERR_LOCK)
}

View File

@@ -3,7 +3,7 @@
// The "reader" [vfs.VFS] permits accessing any [io.ReaderAt]
// as an immutable SQLite database.
//
// Importing package readervfs registers the VFS.
// Importing package readervfs registers the VFS:
//
// import _ "github.com/ncruces/go-sqlite3/vfs/readervfs"
package readervfs

View File

@@ -23,6 +23,7 @@ func Find(name string) VFS {
}
// Register registers a VFS.
// Empty and "os" are reserved names.
//
// https://sqlite.org/c3ref/vfs_find.html
func Register(name string, vfs VFS) {

View File

@@ -1,4 +1,4 @@
//go:build (darwin || linux || illumos) && (amd64 || arm64 || riscv64) && !sqlite3_flock && !sqlite3_noshm && !sqlite3_nosys
//go:build (darwin || linux) && (amd64 || arm64 || riscv64) && !(sqlite3_flock || sqlite3_noshm || sqlite3_nosys)
package vfs
@@ -12,98 +12,116 @@ import (
"golang.org/x/sys/unix"
)
// SupportsSharedMemory is true on platforms that support shared memory.
// To enable shared memory support on those platforms,
// you need to set the appropriate [wazero.RuntimeConfig];
// otherwise, [EXCLUSIVE locking mode] is activated automatically
// to use [WAL without shared-memory].
// SupportsSharedMemory is false on platforms that do not support shared memory.
// To use [WAL without shared-memory], you need to set [EXCLUSIVE locking mode].
//
// [WAL without shared-memory]: https://sqlite.org/wal.html#noshm
// [EXCLUSIVE locking mode]: https://sqlite.org/pragma.html#pragma_locking_mode
const SupportsSharedMemory = true
type vfsShm struct {
*os.File
regions []*util.MappedRegion
}
const (
_SHM_NLOCK = 8
_SHM_BASE = 120
_SHM_DMS = _SHM_BASE + _SHM_NLOCK
)
func (f *vfsFile) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, error) {
func (f *vfsFile) SharedMemory() SharedMemory { return f.shm }
// NewSharedMemory returns a shared-memory WAL-index
// backed by a file with the given path.
// It will return nil if shared-memory is not supported,
// or not appropriate for the given flags.
// Only [OPEN_MAIN_DB] databases may need a WAL-index.
// You must ensure all concurrent accesses to a database
// use shared-memory instances created with the same path.
func NewSharedMemory(path string, flags OpenFlag) SharedMemory {
if flags&OPEN_MAIN_DB == 0 || flags&(OPEN_DELETEONCLOSE|OPEN_MEMORY) != 0 {
return nil
}
return &vfsShm{
path: path,
readOnly: flags&OPEN_READONLY != 0,
}
}
type vfsShm struct {
*os.File
path string
regions []*util.MappedRegion
readOnly bool
}
func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, error) {
// Ensure size is a multiple of the OS page size.
if int(size)&(unix.Getpagesize()-1) != 0 {
return 0, _IOERR_SHMMAP
}
if f.shm.File == nil {
if s.File == nil {
var flag int
if f.readOnly {
if s.readOnly {
flag = unix.O_RDONLY
} else {
flag = unix.O_RDWR
}
s, err := os.OpenFile(f.Name()+"-shm",
f, err := os.OpenFile(s.path,
flag|unix.O_CREAT|unix.O_NOFOLLOW, 0666)
if err != nil {
return 0, _CANTOPEN
}
f.shm.File = s
s.File = f
}
// Dead man's switch.
if lock, rc := osGetLock(f.shm.File, _SHM_DMS, 1); rc != _OK {
if lock, rc := osGetLock(s.File, _SHM_DMS, 1); rc != _OK {
return 0, _IOERR_LOCK
} else if lock == unix.F_WRLCK {
return 0, _BUSY
} else if lock == unix.F_UNLCK {
if f.readOnly {
if s.readOnly {
return 0, _READONLY_CANTINIT
}
if rc := osWriteLock(f.shm.File, _SHM_DMS, 1, 0); rc != _OK {
if rc := osWriteLock(s.File, _SHM_DMS, 1, 0); rc != _OK {
return 0, rc
}
if err := f.shm.Truncate(0); err != nil {
if err := s.Truncate(0); err != nil {
return 0, _IOERR_SHMOPEN
}
}
if rc := osReadLock(f.shm.File, _SHM_DMS, 1, 0); rc != _OK {
if rc := osReadLock(s.File, _SHM_DMS, 1, 0); rc != _OK {
return 0, rc
}
// Check if file is big enough.
s, err := f.shm.Seek(0, io.SeekEnd)
o, err := s.Seek(0, io.SeekEnd)
if err != nil {
return 0, _IOERR_SHMSIZE
}
if n := (int64(id) + 1) * int64(size); n > s {
if n := (int64(id) + 1) * int64(size); n > o {
if !extend {
return 0, nil
}
err := osAllocate(f.shm.File, n)
err := osAllocate(s.File, n)
if err != nil {
return 0, _IOERR_SHMSIZE
}
}
var prot int
if f.readOnly {
if s.readOnly {
prot = unix.PROT_READ
} else {
prot = unix.PROT_READ | unix.PROT_WRITE
}
r, err := util.MapRegion(ctx, mod, f.shm.File, int64(id)*int64(size), size, prot)
r, err := util.MapRegion(ctx, mod, s.File, int64(id)*int64(size), size, prot)
if err != nil {
return 0, err
}
f.shm.regions = append(f.shm.regions, r)
s.regions = append(s.regions, r)
return r.Ptr, nil
}
func (f *vfsFile) shmLock(offset, n int32, flags _ShmFlag) error {
func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) error {
// Argument check.
if n <= 0 || offset < 0 || offset+n > _SHM_NLOCK {
panic(util.AssertErr())
@@ -124,28 +142,32 @@ func (f *vfsFile) shmLock(offset, n int32, flags _ShmFlag) error {
switch {
case flags&_SHM_UNLOCK != 0:
return osUnlock(f.shm.File, _SHM_BASE+int64(offset), int64(n))
return osUnlock(s.File, _SHM_BASE+int64(offset), int64(n))
case flags&_SHM_SHARED != 0:
return osReadLock(f.shm.File, _SHM_BASE+int64(offset), int64(n), 0)
return osReadLock(s.File, _SHM_BASE+int64(offset), int64(n), 0)
case flags&_SHM_EXCLUSIVE != 0:
return osWriteLock(f.shm.File, _SHM_BASE+int64(offset), int64(n), 0)
return osWriteLock(s.File, _SHM_BASE+int64(offset), int64(n), 0)
default:
panic(util.AssertErr())
}
}
func (f *vfsFile) shmUnmap(delete bool) {
func (s *vfsShm) shmUnmap(delete bool) {
if s.File == nil {
return
}
// Unmap regions.
for _, r := range f.shm.regions {
for _, r := range s.regions {
r.Unmap()
}
clear(f.shm.regions)
f.shm.regions = f.shm.regions[:0]
clear(s.regions)
s.regions = s.regions[:0]
// Close the file.
if delete && f.shm.File != nil {
os.Remove(f.shm.Name())
defer s.Close()
if delete {
os.Remove(s.Name())
}
f.shm.Close()
f.shm.File = nil
s.File = nil
}

View File

@@ -1,17 +1,21 @@
//go:build !(darwin || linux || illumos) || !(amd64 || arm64 || riscv64) || sqlite3_flock || sqlite3_noshm || sqlite3_nosys
//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64) || sqlite3_flock || sqlite3_noshm || sqlite3_nosys
package vfs
// SupportsSharedMemory is true on platforms that support shared memory.
// To enable shared memory support on those platforms,
// you need to set the appropriate [wazero.RuntimeConfig];
// otherwise, [EXCLUSIVE locking mode] is activated automatically
// to use [WAL without shared-memory].
// SupportsSharedMemory is false on platforms that do not support shared memory.
// To use [WAL without shared-memory], you need to set [EXCLUSIVE locking mode].
//
// [WAL without shared-memory]: https://sqlite.org/wal.html#noshm
// [EXCLUSIVE locking mode]: https://sqlite.org/pragma.html#pragma_locking_mode
const SupportsSharedMemory = false
type vfsShm struct{}
func (vfsShm) Close() error { return nil }
// NewSharedMemory returns a shared-memory WAL-index
// backed by a file with the given path.
// It will return nil if shared-memory is not supported,
// or not appropriate for the given flags.
// Only [OPEN_MAIN_DB] databases may need a WAL-index.
// You must ensure all concurrent accesses to a database
// use shared-memory instances created with the same path.
func NewSharedMemory(path string, flags OpenFlag) SharedMemory {
return nil
}

View File

@@ -1,3 +1,5 @@
//go:build !sqlite3_nosys
package mptest
import (
@@ -19,6 +21,7 @@ import (
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/vfs"
_ "github.com/ncruces/go-sqlite3/vfs/adiantum"
"github.com/ncruces/go-sqlite3/vfs/memdb"
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/api"
@@ -39,7 +42,8 @@ var (
func TestMain(m *testing.M) {
ctx := context.Background()
rt = wazero.NewRuntime(ctx)
cfg := wazero.NewRuntimeConfig().WithMemoryLimitPages(1024)
rt = wazero.NewRuntimeWithConfig(ctx, cfg)
wasi_snapshot_preview1.MustInstantiate(ctx, rt)
env := vfs.ExportHostFunctions(rt.NewHostModuleBuilder("env"))
@@ -92,7 +96,7 @@ func system(ctx context.Context, mod api.Module, ptr uint32) uint32 {
cfg := config(ctx).WithArgs(args...)
go func() {
ctx := util.NewContext(ctx, true)
ctx := util.NewContext(ctx)
mod, _ := rt.InstantiateModule(ctx, module, cfg)
mod.Close(ctx)
}()
@@ -100,7 +104,7 @@ func system(ctx context.Context, mod api.Module, ptr uint32) uint32 {
}
func Test_config01(t *testing.T) {
ctx := util.NewContext(newContext(t), false)
ctx := util.NewContext(newContext(t))
name := filepath.Join(t.TempDir(), "test.db")
cfg := config(ctx).WithArgs("mptest", name, "config01.test")
mod, err := rt.InstantiateModule(ctx, module, cfg)
@@ -118,7 +122,7 @@ func Test_config02(t *testing.T) {
t.Skip("skipping in CI")
}
ctx := util.NewContext(newContext(t), false)
ctx := util.NewContext(newContext(t))
name := filepath.Join(t.TempDir(), "test.db")
cfg := config(ctx).WithArgs("mptest", name, "config02.test")
mod, err := rt.InstantiateModule(ctx, module, cfg)
@@ -136,7 +140,7 @@ func Test_crash01(t *testing.T) {
t.Skip("skipping in CI")
}
ctx := util.NewContext(newContext(t), false)
ctx := util.NewContext(newContext(t))
name := filepath.Join(t.TempDir(), "test.db")
cfg := config(ctx).WithArgs("mptest", name, "crash01.test")
mod, err := rt.InstantiateModule(ctx, module, cfg)
@@ -151,7 +155,7 @@ func Test_multiwrite01(t *testing.T) {
t.Skip("skipping in short mode")
}
ctx := util.NewContext(newContext(t), false)
ctx := util.NewContext(newContext(t))
name := filepath.Join(t.TempDir(), "test.db")
cfg := config(ctx).WithArgs("mptest", name, "multiwrite01.test")
mod, err := rt.InstantiateModule(ctx, module, cfg)
@@ -163,7 +167,7 @@ func Test_multiwrite01(t *testing.T) {
func Test_config01_memory(t *testing.T) {
memdb.Delete("test.db")
ctx := util.NewContext(newContext(t), false)
ctx := util.NewContext(newContext(t))
cfg := config(ctx).WithArgs("mptest", "/test.db", "config01.test",
"--vfs", "memdb")
mod, err := rt.InstantiateModule(ctx, module, cfg)
@@ -179,7 +183,7 @@ func Test_multiwrite01_memory(t *testing.T) {
}
memdb.Delete("test.db")
ctx := util.NewContext(newContext(t), false)
ctx := util.NewContext(newContext(t))
cfg := config(ctx).WithArgs("mptest", "/test.db", "multiwrite01.test",
"--vfs", "memdb")
mod, err := rt.InstantiateModule(ctx, module, cfg)
@@ -200,7 +204,7 @@ func Test_crash01_wal(t *testing.T) {
t.Skip("skipping without shared memory")
}
ctx := util.NewContext(newContext(t), true)
ctx := util.NewContext(newContext(t))
name := filepath.Join(t.TempDir(), "test.db")
cfg := config(ctx).WithArgs("mptest", name, "crash01.test",
"--journalmode", "wal")
@@ -219,7 +223,7 @@ func Test_multiwrite01_wal(t *testing.T) {
t.Skip("skipping without shared memory")
}
ctx := util.NewContext(newContext(t), true)
ctx := util.NewContext(newContext(t))
name := filepath.Join(t.TempDir(), "test.db")
cfg := config(ctx).WithArgs("mptest", name, "multiwrite01.test",
"--journalmode", "wal")
@@ -230,6 +234,49 @@ func Test_multiwrite01_wal(t *testing.T) {
mod.Close(ctx)
}
func Test_crash01_adiantum(t *testing.T) {
if testing.Short() {
t.Skip("skipping in short mode")
}
if os.Getenv("CI") != "" {
t.Skip("skipping in CI")
}
ctx := util.NewContext(newContext(t))
name := "file:" + filepath.Join(t.TempDir(), "test.db") +
"?hexkey=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
cfg := config(ctx).WithArgs("mptest", name, "crash01.test",
"--vfs", "adiantum")
mod, err := rt.InstantiateModule(ctx, module, cfg)
if err != nil {
t.Fatal(err)
}
mod.Close(ctx)
}
func Test_crash01_adiantum_wal(t *testing.T) {
if testing.Short() {
t.Skip("skipping in short mode")
}
if os.Getenv("CI") != "" {
t.Skip("skipping in CI")
}
if !vfs.SupportsSharedMemory {
t.Skip("skipping without shared memory")
}
ctx := util.NewContext(newContext(t))
name := "file:" + filepath.Join(t.TempDir(), "test.db") +
"?hexkey=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
cfg := config(ctx).WithArgs("mptest", name, "crash01.test",
"--vfs", "adiantum", "--journalmode", "wal")
mod, err := rt.InstantiateModule(ctx, module, cfg)
if err != nil {
t.Fatal(err)
}
mod.Close(ctx)
}
func newContext(t *testing.T) context.Context {
return context.WithValue(context.Background(), logger{}, &testWriter{T: t})
}

View File

@@ -8,7 +8,7 @@ BINARYEN="$ROOT/tools/binaryen-version_117/bin"
WASI_SDK="$ROOT/tools/wasi-sdk-22.0/bin"
"$WASI_SDK/clang" --target=wasm32-wasi -std=c17 -flto -g0 -O2 \
-o mptest.wasm main.c \
-o mptest.wasm main.c \
-I"$ROOT/sqlite3" \
-msimd128 -mmutable-globals \
-mbulk-memory -mreference-types \
@@ -16,13 +16,13 @@ WASI_SDK="$ROOT/tools/wasi-sdk-22.0/bin"
-fno-stack-protector -fno-stack-clash-protection \
-Wl,--stack-first \
-Wl,--import-undefined \
-D_HAVE_SQLITE_CONFIG_H \
-D_HAVE_SQLITE_CONFIG_H -DSQLITE_USE_URI \
-DSQLITE_DEFAULT_SYNCHRONOUS=0 \
-DSQLITE_DEFAULT_LOCKING_MODE=0 \
-DHAVE_USLEEP -DSQLITE_NO_SYNC \
-DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
-DSQLITE_NO_SYNC -DSQLITE_THREADSAFE=0 \
-DSQLITE_OMIT_LOAD_EXTENSION -DHAVE_USLEEP \
-D_WASI_EMULATED_GETPID -lwasi-emulated-getpid \
-Wl,--export=aligned_alloc
$(awk '{print "-Wl,--export="$0}' exports.txt)
"$BINARYEN/wasm-opt" -g --strip --strip-producers -c -O3 \
mptest.wasm -o mptest.tmp \

5
vfs/tests/mptest/testdata/exports.txt vendored Normal file
View File

@@ -0,0 +1,5 @@
aligned_alloc
free
malloc
sqlite3_database_file_object
sqlite3_uri_key

View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:7bf8ef8a52def8dec9b4af62b21cbea22a6cd2a702770aa9c75f445691f1abd5
size 469700
oid sha256:1de925ac1b2a15c0e0ac4231a507d31bdf1ea576f0d768f5f4bf5c3436a1eece
size 470117

View File

@@ -21,6 +21,7 @@ import (
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/vfs"
_ "github.com/ncruces/go-sqlite3/vfs/adiantum"
_ "github.com/ncruces/go-sqlite3/vfs/memdb"
)
@@ -84,7 +85,7 @@ func initFlags() {
func Benchmark_speedtest1(b *testing.B) {
output.Reset()
ctx := util.NewContext(context.Background(), true)
ctx := util.NewContext(context.Background())
name := filepath.Join(b.TempDir(), "test.db")
args := append(options, "--size", strconv.Itoa(b.N), name)
cfg := wazero.NewModuleConfig().
@@ -99,3 +100,22 @@ func Benchmark_speedtest1(b *testing.B) {
}
mod.Close(ctx)
}
func Benchmark_adiantum(b *testing.B) {
output.Reset()
ctx := util.NewContext(context.Background())
name := "file:" + filepath.Join(b.TempDir(), "test.db") +
"?hexkey=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
args := append(options, "--vfs", "adiantum", "--size", strconv.Itoa(b.N), name)
cfg := wazero.NewModuleConfig().
WithArgs(args...).WithName("speedtest1").
WithStdout(&output).WithStderr(&output).
WithSysWalltime().WithSysNanotime().WithSysNanosleep().
WithOsyield(runtime.Gosched).
WithRandSource(rand.Reader)
mod, err := rt.InstantiateModule(ctx, module, cfg)
if err != nil {
b.Fatal(err)
}
mod.Close(ctx)
}

View File

@@ -16,8 +16,8 @@ WASI_SDK="$ROOT/tools/wasi-sdk-22.0/bin"
-fno-stack-protector -fno-stack-clash-protection \
-Wl,--stack-first \
-Wl,--import-undefined \
-D_HAVE_SQLITE_CONFIG_H \
-Wl,--export=aligned_alloc
-D_HAVE_SQLITE_CONFIG_H -DSQLITE_USE_URI \
$(awk '{print "-Wl,--export="$0}' exports.txt)
"$BINARYEN/wasm-opt" -g --strip --strip-producers -c -O3 \
speedtest1.wasm -o speedtest1.tmp \

View File

@@ -0,0 +1,5 @@
aligned_alloc
free
malloc
sqlite3_database_file_object
sqlite3_uri_key

View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:7f669eede3ba9c9a104d37c3a90ecec26086a87aea54c1f650df202923d75cb9
size 483426
oid sha256:262f1702de606413a92fbb9d7d58b812140d072cacabe79556d5fa38e00461d0
size 483792

View File

@@ -4,7 +4,6 @@ import (
"context"
"crypto/rand"
"io"
"net/url"
"reflect"
"sync"
"time"
@@ -141,35 +140,29 @@ func vfsOpen(ctx context.Context, mod api.Module, pVfs, zPath, pFile uint32, fla
var file File
var err error
var parsed bool
var params url.Values
if pfs, ok := vfs.(VFSParams); ok {
parsed = true
params = vfsURIParameters(ctx, mod, zPath, flags)
file, flags, err = pfs.OpenParams(path, flags, params)
if ffs, ok := vfs.(VFSFilename); ok {
name := OpenFilename(ctx, mod, zPath, flags)
file, flags, err = ffs.OpenFilename(name, flags)
} else {
file, flags, err = vfs.Open(path, flags)
}
if err != nil {
return vfsErrorCode(err, _CANTOPEN)
}
if file, ok := file.(FilePowersafeOverwrite); ok {
if !parsed {
params = vfsURIParameters(ctx, mod, zPath, flags)
}
if b, ok := util.ParseBool(params.Get("psow")); ok {
name := OpenFilename(ctx, mod, zPath, flags)
if b, ok := util.ParseBool(name.URIParameter("psow")); ok {
file.SetPowersafeOverwrite(b)
}
}
if file, ok := file.(FileSharedMemory); ok &&
pOutVFS != 0 && file.SharedMemory() != nil {
util.WriteUint32(mod, pOutVFS, 1)
}
if pOutFlags != 0 {
util.WriteUint32(mod, pOutFlags, uint32(flags))
}
if pOutVFS != 0 && util.CanMap(ctx) {
util.WriteUint32(mod, pOutVFS, 1)
}
vfsFileRegister(ctx, mod, pFile, file)
return _OK
}
@@ -183,7 +176,7 @@ func vfsClose(ctx context.Context, mod api.Module, pFile uint32) _ErrorCode {
}
func vfsRead(ctx context.Context, mod api.Module, pFile, zBuf uint32, iAmt int32, iOfst int64) _ErrorCode {
file := vfsFileGet(ctx, mod, pFile)
file := vfsFileGet(ctx, mod, pFile).(File)
buf := util.View(mod, zBuf, uint64(iAmt))
n, err := file.ReadAt(buf, iOfst)
@@ -198,7 +191,7 @@ func vfsRead(ctx context.Context, mod api.Module, pFile, zBuf uint32, iAmt int32
}
func vfsWrite(ctx context.Context, mod api.Module, pFile, zBuf uint32, iAmt int32, iOfst int64) _ErrorCode {
file := vfsFileGet(ctx, mod, pFile)
file := vfsFileGet(ctx, mod, pFile).(File)
buf := util.View(mod, zBuf, uint64(iAmt))
_, err := file.WriteAt(buf, iOfst)
@@ -209,38 +202,38 @@ func vfsWrite(ctx context.Context, mod api.Module, pFile, zBuf uint32, iAmt int3
}
func vfsTruncate(ctx context.Context, mod api.Module, pFile uint32, nByte int64) _ErrorCode {
file := vfsFileGet(ctx, mod, pFile)
file := vfsFileGet(ctx, mod, pFile).(File)
err := file.Truncate(nByte)
return vfsErrorCode(err, _IOERR_TRUNCATE)
}
func vfsSync(ctx context.Context, mod api.Module, pFile uint32, flags SyncFlag) _ErrorCode {
file := vfsFileGet(ctx, mod, pFile)
file := vfsFileGet(ctx, mod, pFile).(File)
err := file.Sync(flags)
return vfsErrorCode(err, _IOERR_FSYNC)
}
func vfsFileSize(ctx context.Context, mod api.Module, pFile, pSize uint32) _ErrorCode {
file := vfsFileGet(ctx, mod, pFile)
file := vfsFileGet(ctx, mod, pFile).(File)
size, err := file.Size()
util.WriteUint64(mod, pSize, uint64(size))
return vfsErrorCode(err, _IOERR_SEEK)
}
func vfsLock(ctx context.Context, mod api.Module, pFile uint32, eLock LockLevel) _ErrorCode {
file := vfsFileGet(ctx, mod, pFile)
file := vfsFileGet(ctx, mod, pFile).(File)
err := file.Lock(eLock)
return vfsErrorCode(err, _IOERR_LOCK)
}
func vfsUnlock(ctx context.Context, mod api.Module, pFile uint32, eLock LockLevel) _ErrorCode {
file := vfsFileGet(ctx, mod, pFile)
file := vfsFileGet(ctx, mod, pFile).(File)
err := file.Unlock(eLock)
return vfsErrorCode(err, _IOERR_UNLOCK)
}
func vfsCheckReservedLock(ctx context.Context, mod api.Module, pFile, pResOut uint32) _ErrorCode {
file := vfsFileGet(ctx, mod, pFile)
file := vfsFileGet(ctx, mod, pFile).(File)
locked, err := file.CheckReservedLock()
var res uint32
@@ -253,7 +246,7 @@ func vfsCheckReservedLock(ctx context.Context, mod api.Module, pFile, pResOut ui
}
func vfsFileControl(ctx context.Context, mod api.Module, pFile uint32, op _FcntlOpcode, pArg uint32) _ErrorCode {
file := vfsFileGet(ctx, mod, pFile)
file := vfsFileGet(ctx, mod, pFile).(File)
switch op {
case _FCNTL_LOCKSTATE:
@@ -286,6 +279,13 @@ func vfsFileControl(ctx context.Context, mod api.Module, pFile uint32, op _Fcntl
return _OK
}
case _FCNTL_CHUNK_SIZE:
if file, ok := file.(FileChunkSize); ok {
size := util.ReadUint32(mod, pArg)
file.ChunkSize(int(size))
return _OK
}
case _FCNTL_SIZE_HINT:
if file, ok := file.(FileSizeHint); ok {
size := util.ReadUint64(mod, pArg)
@@ -331,25 +331,60 @@ func vfsFileControl(ctx context.Context, mod api.Module, pFile uint32, op _Fcntl
err := file.RollbackAtomicWrite()
return vfsErrorCode(err, _IOERR_ROLLBACK_ATOMIC)
}
case _FCNTL_CKPT_DONE:
if file, ok := file.(FileCheckpoint); ok {
err := file.CheckpointDone()
return vfsErrorCode(err, _IOERR)
}
case _FCNTL_CKPT_START:
if file, ok := file.(FileCheckpoint); ok {
err := file.CheckpointStart()
return vfsErrorCode(err, _IOERR)
}
case _FCNTL_PRAGMA:
if file, ok := file.(FilePragma); ok {
ptr := util.ReadUint32(mod, pArg+1*ptrlen)
name := util.ReadString(mod, ptr, _MAX_SQL_LENGTH)
var value string
if ptr := util.ReadUint32(mod, pArg+2*ptrlen); ptr != 0 {
value = util.ReadString(mod, ptr, _MAX_SQL_LENGTH)
}
out, err := file.Pragma(name, value)
ret := vfsErrorCode(err, _ERROR)
if ret == _ERROR {
out = err.Error()
}
if out != "" {
fn := mod.ExportedFunction("malloc")
stack := [...]uint64{uint64(len(out) + 1)}
if err := fn.CallWithStack(ctx, stack[:]); err != nil {
panic(err)
}
util.WriteUint32(mod, pArg, uint32(stack[0]))
util.WriteString(mod, uint32(stack[0]), out)
}
return ret
}
}
// Consider also implementing these opcodes (in use by SQLite):
// _FCNTL_BUSYHANDLER
// _FCNTL_CHUNK_SIZE
// _FCNTL_CKPT_DONE
// _FCNTL_CKPT_START
// _FCNTL_PRAGMA
// _FCNTL_LAST_ERRNO
// _FCNTL_SYNC
return _NOTFOUND
}
func vfsSectorSize(ctx context.Context, mod api.Module, pFile uint32) uint32 {
file := vfsFileGet(ctx, mod, pFile)
file := vfsFileGet(ctx, mod, pFile).(File)
return uint32(file.SectorSize())
}
func vfsDeviceCharacteristics(ctx context.Context, mod api.Module, pFile uint32) DeviceCharacteristic {
file := vfsFileGet(ctx, mod, pFile)
file := vfsFileGet(ctx, mod, pFile).(File)
return file.DeviceCharacteristics()
}
@@ -361,8 +396,8 @@ func vfsShmBarrier(ctx context.Context, mod api.Module, pFile uint32) {
}
func vfsShmMap(ctx context.Context, mod api.Module, pFile uint32, iRegion, szRegion int32, bExtend, pp uint32) _ErrorCode {
file := vfsFileGet(ctx, mod, pFile).(fileShm)
p, err := file.shmMap(ctx, mod, iRegion, szRegion, bExtend != 0)
shm := vfsFileGet(ctx, mod, pFile).(FileSharedMemory).SharedMemory()
p, err := shm.shmMap(ctx, mod, iRegion, szRegion, bExtend != 0)
if err != nil {
return vfsErrorCode(err, _IOERR_SHMMAP)
}
@@ -371,57 +406,17 @@ func vfsShmMap(ctx context.Context, mod api.Module, pFile uint32, iRegion, szReg
}
func vfsShmLock(ctx context.Context, mod api.Module, pFile uint32, offset, n int32, flags _ShmFlag) _ErrorCode {
file := vfsFileGet(ctx, mod, pFile).(fileShm)
err := file.shmLock(offset, n, flags)
shm := vfsFileGet(ctx, mod, pFile).(FileSharedMemory).SharedMemory()
err := shm.shmLock(offset, n, flags)
return vfsErrorCode(err, _IOERR_SHMLOCK)
}
func vfsShmUnmap(ctx context.Context, mod api.Module, pFile, bDelete uint32) _ErrorCode {
file := vfsFileGet(ctx, mod, pFile).(fileShm)
file.shmUnmap(bDelete != 0)
shm := vfsFileGet(ctx, mod, pFile).(FileSharedMemory).SharedMemory()
shm.shmUnmap(bDelete != 0)
return _OK
}
func vfsURIParameters(ctx context.Context, mod api.Module, zPath uint32, flags OpenFlag) url.Values {
if flags&OPEN_URI == 0 {
return nil
}
uriParam := mod.ExportedFunction("sqlite3_uri_parameter")
uriKey := mod.ExportedFunction("sqlite3_uri_key")
if uriParam == nil || uriKey == nil {
return nil
}
var stack [2]uint64
var params url.Values
for i := 0; ; i++ {
stack[1] = uint64(i)
stack[0] = uint64(zPath)
if err := uriKey.CallWithStack(ctx, stack[:]); err != nil {
panic(err)
}
if stack[0] == 0 {
return params
}
key := util.ReadString(mod, uint32(stack[0]), _MAX_NAME)
if params.Has(key) {
continue
}
stack[1] = stack[0]
stack[0] = uint64(zPath)
if err := uriParam.CallWithStack(ctx, stack[:]); err != nil {
panic(err)
}
if params == nil {
params = url.Values{}
}
params.Set(key, util.ReadString(mod, uint32(stack[0]), _MAX_NAME))
}
}
func vfsGet(mod api.Module, pVfs uint32) VFS {
var name string
if pVfs != 0 {
@@ -440,10 +435,10 @@ func vfsFileRegister(ctx context.Context, mod api.Module, pFile uint32, file Fil
util.WriteUint32(mod, pFile+fileHandleOffset, id)
}
func vfsFileGet(ctx context.Context, mod api.Module, pFile uint32) File {
func vfsFileGet(ctx context.Context, mod api.Module, pFile uint32) any {
const fileHandleOffset = 4
id := util.ReadUint32(mod, pFile+fileHandleOffset)
return util.GetHandle(ctx, id).(File)
return util.GetHandle(ctx, id)
}
func vfsFileClose(ctx context.Context, mod api.Module, pFile uint32) error {

View File

@@ -209,7 +209,7 @@ func Test_vfsAccess(t *testing.T) {
func Test_vfsFile(t *testing.T) {
mod := wazerotest.NewModule(wazerotest.NewMemory(wazerotest.PageSize))
ctx := util.NewContext(context.TODO(), false)
ctx := util.NewContext(context.TODO())
// Open a temporary file.
rc := vfsOpen(ctx, mod, 0, 0, 4, OPEN_CREATE|OPEN_EXCLUSIVE|OPEN_READWRITE|OPEN_DELETEONCLOSE, 0, 0)
@@ -281,7 +281,7 @@ func Test_vfsFile(t *testing.T) {
func Test_vfsFile_psow(t *testing.T) {
mod := wazerotest.NewModule(wazerotest.NewMemory(wazerotest.PageSize))
ctx := util.NewContext(context.TODO(), false)
ctx := util.NewContext(context.TODO())
// Open a temporary file.
rc := vfsOpen(ctx, mod, 0, 0, 4, OPEN_CREATE|OPEN_EXCLUSIVE|OPEN_READWRITE|OPEN_DELETEONCLOSE, 0, 0)