Compare commits

..

60 Commits

Author SHA1 Message Date
Nuno Cruces
00d52a873f SQLite 3.46.0. 2024-05-24 11:39:27 +01:00
Nuno Cruces
94fb25e84c Enable sqlite_stat4. 2024-05-24 10:30:14 +01:00
Nuno Cruces
b1f2ff55a0 Remove unnecessary conversions. 2024-05-24 10:30:14 +01:00
Nuno Cruces
53eef1510f Fixed capacity virtual memory. 2024-05-20 14:34:47 +01:00
Nuno Cruces
d23bdcd225 Commiting memory not needed. 2024-05-20 09:41:35 +01:00
Nuno Cruces
321d359663 Ensure benchmarks run. 2024-05-20 01:20:11 +01:00
Nuno Cruces
8f88b687d4 Fix flaky test. 2024-05-20 01:10:13 +01:00
Nuno Cruces
d1075f7dad Fix #87. 2024-05-20 01:04:53 +01:00
Nuno Cruces
ed932ee93b Interrupt busy handlers. 2024-05-19 16:30:09 +01:00
Nuno Cruces
3d30a561f0 Custom allocator. 2024-05-17 17:30:43 +01:00
Nuno Cruces
bdaf77a657 Custom Windows allocator. 2024-05-17 16:57:25 +01:00
Nuno Cruces
323bd6e47e Tweak options early. 2024-05-16 16:24:45 +01:00
Nuno Cruces
5f1c372a65 wazero v1.7.2. 2024-05-13 11:44:34 +01:00
Nuno Cruces
3950be71c1 HPolyC example. 2024-05-12 01:35:40 +01:00
Nuno Cruces
f3dc9bdafc Updated adiantum. 2024-05-10 17:00:35 +01:00
Nuno Cruces
e0720fdb92 Gorm v1.25.10. 2024-05-09 13:24:36 +01:00
Nuno Cruces
5fdcdff7e0 Solaris is flaky. 2024-05-07 17:21:14 +01:00
Nuno Cruces
4d23fc3cee Fix file format. 2024-05-07 16:34:51 +01:00
Nuno Cruces
34882e7c8d Fix z/OS build. 2024-05-07 01:54:13 +01:00
Nuno Cruces
57686a2cf3 Dependencies. 2024-05-06 20:39:37 +01:00
Nuno Cruces
190ca0f0cc NPOT sectors. 2024-05-06 11:57:48 +01:00
Nuno Cruces
1a223fa69f Update README.md 2024-05-06 00:41:56 +01:00
Nuno Cruces
12111a619a Cache LFS. 2024-05-05 23:24:39 +01:00
Nuno Cruces
1c58744f87 Test Solaris. 2024-05-04 15:25:03 +01:00
Nuno Cruces
f0ce3e58eb Docs. 2024-05-04 11:44:54 +01:00
Nuno Cruces
5d5c302ff4 Support for z/OS.
Support is behind sqlite3_flock build tag, and tested through s390x Linux. See #86.
2024-05-04 09:48:50 +01:00
Nuno Cruces
10c494031c EWOULDBLOCK. 2024-05-03 18:09:24 +01:00
Nuno Cruces
d84152dd8d Fix TestTimeFormat. 2024-05-03 18:09:11 +01:00
Nuno Cruces
19209b372c Raise Argon2id iterations. 2024-05-03 14:08:38 +01:00
Nuno Cruces
1e03c6c1fb Add initialize. 2024-05-03 12:39:51 +01:00
Nuno Cruces
bb279cb426 Fixes. 2024-05-02 23:42:31 +01:00
Nuno Cruces
7b646100cb Test endianness. 2024-05-02 23:24:24 +01:00
Nuno Cruces
e0a209908b Enable more tests. 2024-05-02 23:22:59 +01:00
Nuno Cruces
67d859a5b4 Support custom pepper. 2024-05-02 12:09:39 +01:00
Nuno Cruces
57daee7f59 Update README.md 2024-05-01 12:20:33 +01:00
Nuno Cruces
f976ab0dee Additional check. 2024-04-30 20:56:42 +01:00
Nuno Cruces
beba988824 Multiple fixes. 2024-04-30 01:30:39 +01:00
Nuno Cruces
992676d7ec Improved WAL API. 2024-04-27 20:55:14 +01:00
Nuno Cruces
82d8a2d796 Documentation. 2024-04-27 16:31:32 +01:00
Nuno Cruces
811e6e63be Adiantum pragmas. 2024-04-27 12:19:46 +01:00
Nuno Cruces
3c21784aee Simplify URI parameters. 2024-04-27 10:44:00 +01:00
Nuno Cruces
019246d1be Simplify mmap. 2024-04-26 16:45:32 +01:00
Nuno Cruces
fa259bdc94 Simplify _pragma. 2024-04-26 00:07:04 +01:00
Nuno Cruces
8e327a9783 VFS pragma. 2024-04-25 13:30:47 +01:00
Nuno Cruces
09a0ce04ce Test more. (#84)
Also, fix the progress callback and disable a slow example.
2024-04-24 15:49:45 +01:00
Nuno Cruces
fdb2ed0376 Fix illumos. (#83) 2024-04-24 01:07:17 +01:00
Nuno Cruces
3fb0eeec51 Filename API (#82)
Also remove VFSParams.
2024-04-23 11:43:14 +01:00
Nuno Cruces
7f6446ad31 Remove cache (side-channel for shared keys). 2024-04-23 02:25:26 +01:00
Nuno Cruces
77cdf1841f Documentation nits. 2024-04-22 15:28:19 +01:00
kim
189fbc98ac change driver name to SQLite{}, remove global variable 2024-04-22 14:02:45 +01:00
kim
d4027b0133 export the database/sql driver type and global instance 2024-04-22 14:02:45 +01:00
Nuno Cruces
62b79d2ac3 Shared memory API. 2024-04-21 12:33:38 +01:00
Nuno Cruces
07241d064a Adiantum encrypting VFS improvements. (#80)
Encrypt temporary files.
2024-04-21 01:56:38 +01:00
Nuno Cruces
2c30bc996a Don't panic. 2024-04-18 10:12:17 +01:00
Nuno Cruces
9d2194b4ea Update README.md 2024-04-18 02:13:59 +01:00
Nuno Cruces
b3a1cb3dd6 Update README.md 2024-04-18 01:42:25 +01:00
Nuno Cruces
ec1ed22149 Adiantum encrypting VFS. (#77) 2024-04-18 01:39:47 +01:00
Nuno Cruces
e86789b285 Test config. 2024-04-16 17:35:45 +01:00
Nuno Cruces
a1fcafa780 Formatting. 2024-04-16 14:02:23 +01:00
Nuno Cruces
d3f5745790 Updated dependencies. 2024-04-16 02:54:11 +01:00
120 changed files with 2310 additions and 1182 deletions

23
.github/actions/lfs/action.yml vendored Normal file
View File

@@ -0,0 +1,23 @@
name: Git LFS pull
description: Cached Git LFS pull.
runs:
using: "composite"
steps:
- name: Create LFS file list
shell: bash
run: git lfs ls-files --long | cut -d ' ' -f1 | sort > .lfs-assets-id
- name: Restore LFS cache
uses: actions/cache@v4
with:
path: .git/lfs/objects
key: lfs-${{ hashFiles('.lfs-assets-id') }}
restore-keys: lfs-
enableCrossOsArchive: true
- name: Git LFS pull
shell: bash
run: |
git lfs pull
git lfs prune

View File

@@ -1,11 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
echo 'set -euo pipefail' > test.sh
for p in $(go list ./...); do
dir=".${p#github.com/ncruces/go-sqlite3}"
name="$(basename "$p").test"
(cd ${dir}; GOOS=freebsd go test -c)
[ -f "${dir}/${name}" ] && echo "(cd ${dir}; ./${name} -test.v)" >> test.sh
done

11
.github/workflows/build-test.sh vendored Executable file
View File

@@ -0,0 +1,11 @@
#!/usr/bin/env bash
set -euo pipefail
echo 'set -eu' > test.sh
for p in $(go list ./...); do
dir=".${p#github.com/ncruces/go-sqlite3}"
name="$(basename "$p").test"
(cd ${dir}; go test -c)
[ -f "${dir}/${name}" ] && echo "(cd ${dir}; ./${name} ${TESTFLAGS})" >> test.sh
done

View File

@@ -16,10 +16,12 @@ echo windows ; GOOS=windows GOARCH=amd64 go build .
echo aix ; GOOS=aix GOARCH=ppc64 go build .
echo js ; GOOS=js GOARCH=wasm go build .
echo wasip1 ; GOOS=wasip1 GOARCH=wasm go build .
echo linux-flock ; GOOS=linux GOARCH=amd64 go build -tags sqlite3_flock .
echo linux-noshm ; GOOS=linux GOARCH=amd64 go build -tags sqlite3_noshm .
echo linux-nosys ; GOOS=linux GOARCH=amd64 go build -tags sqlite3_nosys .
echo darwin-flock ; GOOS=darwin GOARCH=amd64 go build -tags sqlite3_flock .
echo darwin-noshm ; GOOS=darwin GOARCH=amd64 go build -tags sqlite3_noshm .
echo darwin-nosys ; GOOS=darwin GOARCH=amd64 go build -tags sqlite3_nosys .
echo linux-noshm ; GOOS=linux GOARCH=amd64 go build -tags sqlite3_noshm .
echo linux-nosys ; GOOS=linux GOARCH=amd64 go build -tags sqlite3_nosys .
echo windows-nosys ; GOOS=windows GOARCH=amd64 go build -tags sqlite3_nosys .
echo freebsd-nosys ; GOOS=freebsd GOARCH=amd64 go build -tags sqlite3_nosys .
echo freebsd-nosys ; GOOS=freebsd GOARCH=amd64 go build -tags sqlite3_nosys .
echo solaris-flock ; GOOS=solaris GOARCH=amd64 go build -tags sqlite3_flock .

View File

@@ -9,10 +9,8 @@ jobs:
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: stable
with: { go-version: stable }
- name: Build
run: .github/workflows/cross.sh

View File

@@ -12,12 +12,8 @@ jobs:
steps:
- uses: actions/checkout@v4
with:
lfs: 'true'
- uses: actions/setup-go@v5
with:
go-version: stable
with: { go-version: stable }
- name: Build
run: .github/workflows/repro.sh

View File

@@ -16,11 +16,12 @@ jobs:
steps:
- uses: actions/checkout@v4
with: { lfs: 'true' }
- uses: actions/setup-go@v5
with: { go-version: stable }
- name: Git LFS pull
uses: ./.github/actions/lfs
- name: Format
run: gofmt -s -w . && git diff --exit-code
if: matrix.os != 'windows-latest'
@@ -41,7 +42,7 @@ jobs:
run: go build -v ./...
- name: Test
run: go test -v ./...
run: go test -v ./... -bench . -benchtime=1x
- name: Test BSD locks
run: go test -v -tags sqlite3_flock ./...
@@ -52,7 +53,8 @@ jobs:
if: matrix.os == 'ubuntu-latest'
- name: Test no locks
run: go test -v -tags sqlite3_nosys ./tests -run TestDB_nolock
run: go test -v -tags sqlite3_nosys ./...
if: matrix.os == 'ubuntu-latest'
- name: Test GORM
run: gormlite/test.sh
@@ -65,19 +67,38 @@ jobs:
github.event_name == 'push' &&
matrix.os == 'ubuntu-latest'
test-intel:
runs-on: macos-13
needs: test
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with: { go-version: stable }
- name: Git LFS pull
uses: ./.github/actions/lfs
- name: Test
run: go test -v ./...
test-bsd:
runs-on: ubuntu-latest
needs: test
steps:
- uses: actions/checkout@v4
with: { lfs: 'true' }
- uses: actions/setup-go@v5
with: { go-version: stable }
- name: Git LFS pull
uses: ./.github/actions/lfs
- name: Build
run: .github/workflows/bsd.sh
env:
GOOS: freebsd
TESTFLAGS: '-test.v'
run: .github/workflows/build-test.sh
- name: Test
uses: cross-platform-actions/action@v0.24.0
@@ -85,49 +106,69 @@ jobs:
operating_system: freebsd
version: '14.0'
shell: bash
run: source test.sh
run: . ./test.sh
sync_files: runner-to-vm
test-m1:
runs-on: macos-14
needs: test
steps:
- uses: actions/checkout@v4
with: { lfs: 'true' }
- uses: actions/setup-go@v5
with: { go-version: stable }
- name: Test
run: go test -v ./...
test-386:
test-qemu:
runs-on: ubuntu-latest
needs: test
steps:
- uses: docker/setup-qemu-action@v3
- uses: actions/checkout@v4
with: { lfs: 'true' }
- uses: actions/setup-go@v5
with: { go-version: stable }
- name: Test
- name: Git LFS pull
uses: ./.github/actions/lfs
- name: Test 386 (32-bit)
run: GOARCH=386 go test -v -short ./...
test-arm:
- name: Test arm64 (compiler)
run: GOARCH=arm64 go test -v -short ./...
- name: Test riscv64 (interpreter)
run: GOARCH=riscv64 go test -v -short ./...
- name: Test s390x (big-endian, z/OS)
run: GOARCH=s390x go test -v -short -tags sqlite3_flock ./...
test-vm:
runs-on: ubuntu-latest
needs: test
steps:
- uses: actions/checkout@v4
with: { lfs: 'true' }
- uses: actions/setup-go@v5
with: { go-version: stable }
- uses: docker/setup-qemu-action@v3
- name: Git LFS pull
uses: ./.github/actions/lfs
- name: Test
run: GOARCH=arm64 go test -v -short ./...
- name: Build illumos
env:
GOOS: illumos
TESTFLAGS: '-test.v -test.short'
run: .github/workflows/build-test.sh
- name: Test illumos
uses: vmactions/omnios-vm@v1
with:
usesh: true
copyback: false
run: . ./test.sh
- name: Build Solaris
env:
GOOS: solaris
TESTFLAGS: '-test.v -test.short'
run: .github/workflows/build-test.sh
- name: Test Solaris
uses: vmactions/solaris-vm@v1
with:
usesh: true
copyback: false
run: . ./test.sh
continue-on-error: true

View File

@@ -4,12 +4,13 @@
[![Go Report](https://goreportcard.com/badge/github.com/ncruces/go-sqlite3)](https://goreportcard.com/report/github.com/ncruces/go-sqlite3)
[![Go Coverage](https://github.com/ncruces/go-sqlite3/wiki/coverage.svg)](https://github.com/ncruces/go-sqlite3/wiki/Test-coverage-report)
Go module `github.com/ncruces/go-sqlite3` is `cgo`-free [SQLite](https://sqlite.org/) wrapper.\
Go module `github.com/ncruces/go-sqlite3` is a `cgo`-free [SQLite](https://sqlite.org/) wrapper.\
It provides a [`database/sql`](https://pkg.go.dev/database/sql) compatible driver,
as well as direct access to most of the [C SQLite API](https://sqlite.org/cintro.html).
It wraps a [Wasm](https://webassembly.org/) build of SQLite, and uses [wazero](https://wazero.io/) as the runtime.\
Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ runtime dependencies.
It wraps a [Wasm](https://webassembly.org/) [build](embed/) of SQLite,
and uses [wazero](https://wazero.io/) as the runtime.\
Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ runtime dependencies [^1].
### Packages
@@ -54,6 +55,8 @@ Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ run
implements an in-memory VFS.
- [`github.com/ncruces/go-sqlite3/vfs/readervfs`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/readervfs)
implements a VFS for immutable databases.
- [`github.com/ncruces/go-sqlite3/vfs/adiantum`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/adiantum)
wraps a VFS to offer encryption at rest.
### Advanced features
@@ -67,6 +70,7 @@ Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ run
- [math functions](https://sqlite.org/lang_mathfunc.html)
- [full-text search](https://sqlite.org/fts5.html)
- [geospatial search](https://sqlite.org/geopoly.html)
- [encryption at rest](vfs/adiantum/README.md)
- [and more…](embed/README.md)
### Caveats
@@ -83,9 +87,12 @@ This project aims for [high test coverage](https://github.com/ncruces/go-sqlite3
It also benefits greatly from [SQLite's](https://sqlite.org/testing.html) and
[wazero's](https://tetrate.io/blog/introducing-wazero-from-tetrate/#:~:text=Rock%2Dsolid%20test%20approach) thorough testing.
Every commit is [tested](.github/workflows/test.yml) on
Linux (amd64/arm64/386/riscv64/s390x), macOS (amd64/arm64),
Windows (amd64), FreeBSD (amd64), illumos (amd64), and Solaris (amd64).
The Go VFS is tested by running SQLite's
[mptest](https://github.com/sqlite/sqlite/blob/master/mptest/mptest.c)
on Linux, macOS, Windows and FreeBSD.
[mptest](https://github.com/sqlite/sqlite/blob/master/mptest/mptest.c).
### Performance
@@ -101,3 +108,6 @@ The Wasm and VFS layers are also tested by running SQLite's
- [`crawshaw.io/sqlite`](https://pkg.go.dev/crawshaw.io/sqlite)
- [`github.com/mattn/go-sqlite3`](https://pkg.go.dev/github.com/mattn/go-sqlite3)
- [`github.com/zombiezen/go-sqlite`](https://pkg.go.dev/github.com/zombiezen/go-sqlite)
[^1]: anything else you find in `go.mod` is either a test dependency,
or needed by one of the extensions.

65
conn.go
View File

@@ -2,7 +2,6 @@ package sqlite3
import (
"context"
"errors"
"fmt"
"math"
"net/url"
@@ -10,6 +9,7 @@ import (
"time"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/vfs"
"github.com/tetratelabs/wazero/api"
)
@@ -102,15 +102,14 @@ func (c *Conn) openDB(filename string, flags OpenFlag) (uint32, error) {
pragmas.WriteString(`;`)
}
}
pragmaPtr := c.arena.string(pragmas.String())
r := c.call("sqlite3_exec", uint64(handle), uint64(pragmaPtr), 0, 0, 0)
if err := c.sqlite.error(r, handle, pragmas.String()); err != nil {
if errors.Is(err, ERROR) {
if pragmas.Len() != 0 {
pragmaPtr := c.arena.string(pragmas.String())
r := c.call("sqlite3_exec", uint64(handle), uint64(pragmaPtr), 0, 0, 0)
if err := c.sqlite.error(r, handle, pragmas.String()); err != nil {
err = fmt.Errorf("sqlite3: invalid _pragma: %w", err)
c.closeDB(handle)
return 0, err
}
c.closeDB(handle)
return 0, err
}
}
c.call("sqlite3_progress_handler_go", uint64(handle), 100)
@@ -175,7 +174,7 @@ func (c *Conn) Prepare(sql string) (stmt *Stmt, tail string, err error) {
//
// https://sqlite.org/c3ref/prepare.html
func (c *Conn) PrepareFlags(sql string, flags PrepareFlag) (stmt *Stmt, tail string, err error) {
if len(sql) > _MAX_LENGTH {
if len(sql) > _MAX_SQL_LENGTH {
return nil, "", TOOBIG
}
@@ -216,6 +215,20 @@ func (c *Conn) DBName(n int) string {
return util.ReadString(c.mod, ptr, _MAX_NAME)
}
// Filename returns the filename for a database.
//
// https://sqlite.org/c3ref/db_filename.html
func (c *Conn) Filename(schema string) *vfs.Filename {
var ptr uint32
if schema != "" {
defer c.arena.mark()()
ptr = c.arena.string(schema)
}
r := c.call("sqlite3_db_filename", uint64(c.handle), uint64(ptr))
return vfs.OpenFilename(c.ctx, c.mod, uint32(r), vfs.OPEN_MAIN_DB)
}
// ReadOnly determines if a database is read-only.
//
// https://sqlite.org/c3ref/db_readonly.html
@@ -333,10 +346,9 @@ func (c *Conn) checkInterrupt() {
}
func progressCallback(ctx context.Context, mod api.Module, pDB uint32) (interrupt uint32) {
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.commit != nil {
if c.interrupt != nil && c.interrupt.Err() != nil {
interrupt = 1
}
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB &&
c.interrupt != nil && c.interrupt.Err() != nil {
interrupt = 1
}
return interrupt
}
@@ -350,6 +362,30 @@ func (c *Conn) BusyTimeout(timeout time.Duration) error {
return c.error(r)
}
func timeoutCallback(ctx context.Context, mod api.Module, pDB uint32, count, tmout int32) (retry uint32) {
if c, ok := ctx.Value(connKey{}).(*Conn); ok &&
(c.interrupt == nil || c.interrupt.Err() == nil) {
const delays = "\x01\x02\x05\x0a\x0f\x14\x19\x19\x19\x32\x32\x64"
const totals = "\x00\x01\x03\x08\x12\x21\x35\x4e\x67\x80\xb2\xe4"
const ndelay = int32(len(delays) - 1)
var delay, prior int32
if count <= ndelay {
delay = int32(delays[count])
prior = int32(totals[count])
} else {
delay = int32(delays[ndelay])
prior = int32(totals[ndelay]) + delay*(count-ndelay)
}
if delay = min(delay, tmout-prior); delay > 0 {
time.Sleep(time.Duration(delay) * time.Millisecond)
retry = 1
}
}
return retry
}
// BusyHandler registers a callback to handle [BUSY] errors.
//
// https://sqlite.org/c3ref/busy_handler.html
@@ -367,7 +403,8 @@ func (c *Conn) BusyHandler(cb func(count int) (retry bool)) error {
}
func busyCallback(ctx context.Context, mod api.Module, pDB uint32, count int32) (retry uint32) {
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.busy != nil {
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.busy != nil &&
(c.interrupt == nil || c.interrupt.Err() == nil) {
if c.busy(int(count)) {
retry = 1
}

View File

@@ -63,39 +63,47 @@ var driverName = "sqlite3"
func init() {
if driverName != "" {
sql.Register(driverName, sqlite{})
sql.Register(driverName, &SQLite{})
}
}
// Open opens the SQLite database specified by dataSourceName as a [database/sql.DB].
//
// The init function is called by the driver on new connections.
// The conn can be used to execute queries, register functions, etc.
// Any error return closes the conn and passes the error to [database/sql].
// The [sqlite3.Conn] can be used to execute queries, register functions, etc.
// Any error returned closes the connection and is returned to [database/sql].
func Open(dataSourceName string, init func(*sqlite3.Conn) error) (*sql.DB, error) {
c, err := newConnector(dataSourceName, init)
c, err := (&SQLite{Init: init}).OpenConnector(dataSourceName)
if err != nil {
return nil, err
}
return sql.OpenDB(c), nil
}
type sqlite struct{}
// SQLite implements [database/sql/driver.Driver].
type SQLite struct {
// Init function is called by the driver on new connections.
// The [sqlite3.Conn] can be used to execute queries, register functions, etc.
// Any error returned closes the connection and is returned to [database/sql].
Init func(*sqlite3.Conn) error
}
func (sqlite) Open(name string) (driver.Conn, error) {
c, err := newConnector(name, nil)
// Open implements [database/sql/driver.Driver].
func (d *SQLite) Open(name string) (driver.Conn, error) {
c, err := d.newConnector(name)
if err != nil {
return nil, err
}
return c.Connect(context.Background())
}
func (sqlite) OpenConnector(name string) (driver.Connector, error) {
return newConnector(name, nil)
// OpenConnector implements [database/sql/driver.DriverContext].
func (d *SQLite) OpenConnector(name string) (driver.Connector, error) {
return d.newConnector(name)
}
func newConnector(name string, init func(*sqlite3.Conn) error) (*connector, error) {
c := connector{name: name, init: init}
func (d *SQLite) newConnector(name string) (*connector, error) {
c := connector{driver: d, name: name}
var txlock, timefmt string
if strings.HasPrefix(name, "file:") {
@@ -137,7 +145,7 @@ func newConnector(name string, init func(*sqlite3.Conn) error) (*connector, erro
}
type connector struct {
init func(*sqlite3.Conn) error
driver *SQLite
name string
txBegin string
tmRead sqlite3.TimeFormat
@@ -146,7 +154,7 @@ type connector struct {
}
func (n *connector) Driver() driver.Driver {
return sqlite{}
return n.driver
}
func (n *connector) Connect(ctx context.Context) (_ driver.Conn, err error) {
@@ -175,13 +183,13 @@ func (n *connector) Connect(ctx context.Context) (_ driver.Conn, err error) {
return nil, err
}
}
if n.init != nil {
err = n.init(c.Conn)
if n.driver.Init != nil {
err = n.driver.Init(c.Conn)
if err != nil {
return nil, err
}
}
if n.pragmas || n.init != nil {
if n.pragmas || n.driver.Init != nil {
s, _, err := c.Conn.Prepare(`PRAGMA query_only`)
if err != nil {
return nil, err
@@ -319,7 +327,7 @@ func (c *conn) ExecContext(ctx context.Context, query string, args []driver.Name
return newResult(c.Conn), nil
}
func (*conn) CheckNamedValue(arg *driver.NamedValue) error {
func (c *conn) CheckNamedValue(arg *driver.NamedValue) error {
return nil
}

View File

@@ -14,6 +14,8 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/internal/util"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
"github.com/ncruces/go-sqlite3/vfs"
)
func Test_Open_dir(t *testing.T) {
@@ -79,6 +81,9 @@ func Test_Open_pragma_invalid(t *testing.T) {
}
func Test_Open_txLock(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
t.Parallel()
db, err := sql.Open("sqlite3", "file:"+
@@ -125,6 +130,9 @@ func Test_Open_txLock_invalid(t *testing.T) {
}
func Test_BeginTx(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
t.Parallel()
ctx, cancel := context.WithCancel(context.Background())

View File

@@ -1,3 +1,5 @@
//go:build (linux || darwin || windows || freebsd || illumos) && !sqlite3_nosys
package driver_test
// Adapted from: https://go.dev/doc/tutorial/database-access

View File

@@ -6,6 +6,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/vfs/memdb"
)

View File

@@ -1,6 +1,6 @@
# Embeddable Wasm build of SQLite
This folder includes an embeddable Wasm build of SQLite 3.45.3 for use with
This folder includes an embeddable Wasm build of SQLite 3.46.0 for use with
[`github.com/ncruces/go-sqlite3`](https://pkg.go.dev/github.com/ncruces/go-sqlite3).
The following optional features are compiled in:
@@ -10,6 +10,7 @@ The following optional features are compiled in:
- [R*Tree](https://sqlite.org/rtree.html)
- [GeoPoly](https://sqlite.org/geopoly.html)
- [soundex](https://sqlite.org/lang_corefunc.html#soundex)
- [stat4](https://sqlite.org/compile.html#enable_stat4)
- [base64](https://github.com/sqlite/sqlite/blob/master/ext/misc/base64.c)
- [decimal](https://github.com/sqlite/sqlite/blob/master/ext/misc/decimal.c)
- [ieee754](https://github.com/sqlite/sqlite/blob/master/ext/misc/ieee754.c)

View File

@@ -8,7 +8,7 @@ BINARYEN="$ROOT/tools/binaryen-version_117/bin"
WASI_SDK="$ROOT/tools/wasi-sdk-22.0/bin"
"$WASI_SDK/clang" --target=wasm32-wasi -std=c17 -flto -g0 -O2 \
-Wall -Wextra -Wno-unused-parameter \
-Wall -Wextra -Wno-unused-parameter -Wno-unused-function \
-o sqlite3.wasm "$ROOT/sqlite3/main.c" \
-I"$ROOT/sqlite3" \
-mexec-model=reactor \
@@ -20,6 +20,7 @@ WASI_SDK="$ROOT/tools/wasi-sdk-22.0/bin"
-Wl,--stack-first \
-Wl,--import-undefined \
-D_HAVE_SQLITE_CONFIG_H \
-DSQLITE_CUSTOM_INCLUDE=sqlite_opt.h \
$(awk '{print "-Wl,--export="$0}' exports.txt)
trap 'rm -f sqlite3.tmp' EXIT

View File

@@ -36,10 +36,13 @@ sqlite3_collation_needed_go
sqlite3_column_blob
sqlite3_column_bytes
sqlite3_column_count
sqlite3_column_database_name
sqlite3_column_decltype
sqlite3_column_double
sqlite3_column_int64
sqlite3_column_name
sqlite3_column_origin_name
sqlite3_column_table_name
sqlite3_column_text
sqlite3_column_type
sqlite3_column_value
@@ -51,7 +54,9 @@ sqlite3_create_collation_go
sqlite3_create_function_go
sqlite3_create_module_go
sqlite3_create_window_function_go
sqlite3_database_file_object
sqlite3_db_config
sqlite3_db_filename
sqlite3_db_name
sqlite3_db_readonly
sqlite3_db_release_memory
@@ -61,6 +66,9 @@ sqlite3_errmsg
sqlite3_error_offset
sqlite3_errstr
sqlite3_exec
sqlite3_filename_database
sqlite3_filename_journal
sqlite3_filename_wal
sqlite3_finalize
sqlite3_get_autocommit
sqlite3_get_auxdata

Binary file not shown.

View File

@@ -11,6 +11,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/array"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Example_driver() {

View File

@@ -12,6 +12,7 @@ import (
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/array"
"github.com/ncruces/go-sqlite3/ext/blobio"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/vfs/memdb"
)

View File

@@ -8,6 +8,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/csv"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Example() {

View File

@@ -11,6 +11,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/fileio"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Test_lsmode(t *testing.T) {

View File

@@ -12,6 +12,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/fileio"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Test_fsdir(t *testing.T) {

View File

@@ -10,6 +10,7 @@ import (
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Test_writefile(t *testing.T) {

View File

@@ -10,6 +10,7 @@ import (
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "golang.org/x/crypto/blake2b"
_ "golang.org/x/crypto/blake2s"
_ "golang.org/x/crypto/md4"

View File

@@ -14,6 +14,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/lines"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Example() {
@@ -58,7 +59,7 @@ func Example() {
if err := rows.Err(); err != nil {
log.Fatal(err)
}
// Output:
// Expected output:
// US: 141001
// GB: 22560
// CA: 11759

View File

@@ -9,6 +9,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/pivot"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
// https://antonz.org/sqlite-pivot-table/

View File

@@ -8,6 +8,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/statement"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Example() {

View File

@@ -7,6 +7,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/stats"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestRegister_variance(t *testing.T) {
@@ -179,6 +180,9 @@ func TestRegister_covariance(t *testing.T) {
}
func Benchmark_average(b *testing.B) {
sqlite3.Initialize()
b.ResetTimer()
db, err := sqlite3.Open(":memory:")
if err != nil {
b.Fatal(err)
@@ -210,6 +214,9 @@ func Benchmark_average(b *testing.B) {
}
func Benchmark_variance(b *testing.B) {
sqlite3.Initialize()
b.ResetTimer()
db, err := sqlite3.Open(":memory:")
if err != nil {
b.Fatal(err)

View File

@@ -7,6 +7,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestRegister(t *testing.T) {

View File

@@ -7,6 +7,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/zorder"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestRegister_zorder(t *testing.T) {

9
go.mod
View File

@@ -5,11 +5,12 @@ go 1.21
require (
github.com/ncruces/julianday v1.0.0
github.com/psanford/httpreadat v0.1.0
github.com/tetratelabs/wazero v1.7.1
golang.org/x/crypto v0.22.0
github.com/tetratelabs/wazero v1.7.2
golang.org/x/crypto v0.23.0
golang.org/x/sync v0.7.0
golang.org/x/sys v0.19.0
golang.org/x/text v0.14.0
golang.org/x/sys v0.20.0
golang.org/x/text v0.15.0
lukechampine.com/adiantum v1.1.0
)
retract v0.4.0 // tagged from the wrong branch

18
go.sum
View File

@@ -2,13 +2,15 @@ github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/psanford/httpreadat v0.1.0 h1:VleW1HS2zO7/4c7c7zNl33fO6oYACSagjJIyMIwZLUE=
github.com/psanford/httpreadat v0.1.0/go.mod h1:Zg7P+TlBm3bYbyHTKv/EdtSJZn3qwbPwpfZ/I9GKCRE=
github.com/tetratelabs/wazero v1.7.1 h1:QtSfd6KLc41DIMpDYlJdoMc6k7QTN246DM2+n2Y/Dx8=
github.com/tetratelabs/wazero v1.7.1/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
github.com/tetratelabs/wazero v1.7.2 h1:1+z5nXJNwMLPAWaTePFi49SSTL0IMx/i3Fg8Yc25GDc=
github.com/tetratelabs/wazero v1.7.2/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
lukechampine.com/adiantum v1.1.0 h1:Y56WsdnHGgl62EmxkwJz0qvlnWOUqJmVYljwRPj7ovY=
lukechampine.com/adiantum v1.1.0/go.mod h1:LrAYVnTYLnUtE/yMp5bQr0HstAf060YUF8nM0B6+rUw=

View File

@@ -4,5 +4,6 @@ golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=

View File

@@ -3,14 +3,14 @@ module github.com/ncruces/go-sqlite3/gormlite
go 1.21
require (
github.com/ncruces/go-sqlite3 v0.13.0
gorm.io/gorm v1.25.9
github.com/ncruces/go-sqlite3 v0.15.0
gorm.io/gorm v1.25.10
)
require (
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/ncruces/julianday v1.0.0 // indirect
github.com/tetratelabs/wazero v1.7.0 // indirect
golang.org/x/sys v0.19.0 // indirect
github.com/tetratelabs/wazero v1.7.2 // indirect
golang.org/x/sys v0.20.0 // indirect
)

View File

@@ -2,15 +2,15 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/ncruces/go-sqlite3 v0.13.0 h1:+N1VHVLnrCJasyXdAKQL9MNTKC3wHZa8pLMUuP8wb3k=
github.com/ncruces/go-sqlite3 v0.13.0/go.mod h1:y9zPUI+C42V9xuuJeN+tGhpOjr4gUHz2Pi2RLFVEdZg=
github.com/ncruces/go-sqlite3 v0.15.0 h1:C+SIrcYsAIR5GUYWmCnif6x81n6BS9y75vYcQynuGNU=
github.com/ncruces/go-sqlite3 v0.15.0/go.mod h1:kHHYmFmK4G2VFFoIovEg9BEQ8BP+D81y4ESHXnzJV/w=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/tetratelabs/wazero v1.7.0 h1:jg5qPydno59wqjpGrHph81lbtHzTrWzwwtD4cD88+hQ=
github.com/tetratelabs/wazero v1.7.0/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
github.com/tetratelabs/wazero v1.7.2 h1:1+z5nXJNwMLPAWaTePFi49SSTL0IMx/i3Fg8Yc25GDc=
github.com/tetratelabs/wazero v1.7.2/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
gorm.io/gorm v1.25.9 h1:wct0gxZIELDk8+ZqF/MVnHLkA1rvYlBWUMv2EdsK1g8=
gorm.io/gorm v1.25.9/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
gorm.io/gorm v1.25.10 h1:dQpO+33KalOA+aFYGlK+EfxcI5MbO7EP2yYygwh9h+s=
gorm.io/gorm v1.25.10/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=

View File

@@ -9,6 +9,7 @@ import (
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestDialector(t *testing.T) {

View File

@@ -7,7 +7,7 @@ rm -rf gorm/ tests/
go work use -r .
go test
git clone --branch v1.25.9 --filter=blob:none https://github.com/go-gorm/gorm.git
git clone --branch v1.25.10 --filter=blob:none https://github.com/go-gorm/gorm.git
mv gorm/tests tests
rm -rf gorm/

View File

@@ -0,0 +1,9 @@
//go:build !(unix || windows) || sqlite3_nosys
package util
import "github.com/tetratelabs/wazero/experimental"
func virtualAlloc(cap, max uint64) experimental.LinearMemory {
return sliceAlloc(cap, max)
}

View File

@@ -0,0 +1,25 @@
//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64) || sqlite3_noshm || sqlite3_nosys
package util
import "github.com/tetratelabs/wazero/experimental"
func sliceAlloc(cap, max uint64) experimental.LinearMemory {
return &sliceBuffer{make([]byte, cap), max}
}
type sliceBuffer struct {
buf []byte
max uint64
}
func (b *sliceBuffer) Free() {}
func (b *sliceBuffer) Reallocate(size uint64) []byte {
if cap := uint64(cap(b.buf)); size > cap {
b.buf = append(b.buf[:cap], make([]byte, size-cap)...)
} else {
b.buf = b.buf[:size]
}
return b.buf
}

View File

@@ -1,4 +1,4 @@
//go:build unix
//go:build unix && !sqlite3_nosys
package util
@@ -9,30 +9,24 @@ import (
"golang.org/x/sys/unix"
)
func mmappedAllocator(cap, max uint64) experimental.LinearMemory {
func virtualAlloc(cap, max uint64) experimental.LinearMemory {
// Round up to the page size.
rnd := uint64(unix.Getpagesize() - 1)
max = (max + rnd) &^ rnd
cap = (cap + rnd) &^ rnd
if max > math.MaxInt {
// This ensures int(max) overflows to a negative value,
// and unix.Mmap returns EINVAL.
max = math.MaxUint64
}
// Reserve max bytes of address space, to ensure we won't need to move it.
// A protected, private, anonymous mapping should not commit memory.
b, err := unix.Mmap(-1, 0, int(max), unix.PROT_NONE, unix.MAP_PRIVATE|unix.MAP_ANON)
if err != nil {
panic(err)
}
// Commit the initial cap bytes of memory.
err = unix.Mprotect(b[:cap], unix.PROT_READ|unix.PROT_WRITE)
if err != nil {
unix.Munmap(b)
panic(err)
}
return &mmappedMemory{buf: b[:cap]}
return &mmappedMemory{buf: b[:0]}
}
// The slice covers the entire mmapped memory:
@@ -43,7 +37,9 @@ type mmappedMemory struct {
}
func (m *mmappedMemory) Reallocate(size uint64) []byte {
if com := uint64(len(m.buf)); com < size {
com := uint64(len(m.buf))
res := uint64(cap(m.buf))
if com < size && size < res {
// Round up to the page size.
rnd := uint64(unix.Getpagesize() - 1)
new := (size + rnd) &^ rnd

View File

@@ -0,0 +1,76 @@
//go:build !sqlite3_nosys
package util
import (
"math"
"reflect"
"unsafe"
"github.com/tetratelabs/wazero/experimental"
"golang.org/x/sys/windows"
)
func virtualAlloc(cap, max uint64) experimental.LinearMemory {
// Round up to the page size.
rnd := uint64(windows.Getpagesize() - 1)
max = (max + rnd) &^ rnd
if max > math.MaxInt {
// This ensures uintptr(max) overflows to a large value,
// and windows.VirtualAlloc returns an error.
max = math.MaxUint64
}
// Reserve max bytes of address space, to ensure we won't need to move it.
// This does not commit memory.
r, err := windows.VirtualAlloc(0, uintptr(max), windows.MEM_RESERVE, windows.PAGE_READWRITE)
if err != nil {
panic(err)
}
mem := virtualMemory{addr: r}
// SliceHeader, although deprecated, avoids a go vet warning.
sh := (*reflect.SliceHeader)(unsafe.Pointer(&mem.buf))
sh.Cap = int(max) // Not a bug.
sh.Data = r
return &mem
}
// The slice covers the entire mmapped memory:
// - len(buf) is the already committed memory,
// - cap(buf) is the reserved address space.
type virtualMemory struct {
buf []byte
addr uintptr
}
func (m *virtualMemory) Reallocate(size uint64) []byte {
com := uint64(len(m.buf))
res := uint64(cap(m.buf))
if com < size && size < res {
// Round up to the page size.
rnd := uint64(windows.Getpagesize() - 1)
new := (size + rnd) &^ rnd
// Commit additional memory up to new bytes.
_, err := windows.VirtualAlloc(m.addr, uintptr(new), windows.MEM_COMMIT, windows.PAGE_READWRITE)
if err != nil {
panic(err)
}
// Update committed memory.
m.buf = m.buf[:new]
}
// Limit returned capacity because bytes beyond
// len(m.buf) have not yet been committed.
return m.buf[:size:len(m.buf)]
}
func (m *virtualMemory) Free() {
err := windows.VirtualFree(m.addr, 0, windows.MEM_RELEASE)
if err != nil {
panic(err)
}
m.addr = 0
}

View File

@@ -1,4 +1,4 @@
//go:build (darwin || linux || illumos) && (amd64 || arm64 || riscv64) && !sqlite3_flock && !sqlite3_noshm && !sqlite3_nosys
//go:build (darwin || linux) && (amd64 || arm64 || riscv64) && !(sqlite3_noshm || sqlite3_nosys)
package util
@@ -12,22 +12,13 @@ import (
"golang.org/x/sys/unix"
)
func withAllocator(ctx context.Context) context.Context {
return experimental.WithMemoryAllocator(ctx,
experimental.MemoryAllocatorFunc(virtualAlloc))
}
type mmapState struct {
regions []*MappedRegion
enabled bool
}
func (s *mmapState) init(ctx context.Context, enabled bool) context.Context {
if s.enabled = enabled; enabled {
return experimental.WithMemoryAllocator(ctx,
experimental.MemoryAllocatorFunc(mmappedAllocator))
}
return ctx
}
func CanMap(ctx context.Context) bool {
s := ctx.Value(moduleKey{}).(*moduleState)
return s.mmapState.enabled
}
func (s *mmapState) new(ctx context.Context, mod api.Module, size int32) *MappedRegion {

View File

@@ -1,15 +1,21 @@
//go:build !(darwin || linux || illumos) || !(amd64 || arm64 || riscv64) || sqlite3_flock || sqlite3_noshm || sqlite3_nosys
//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64) || sqlite3_noshm || sqlite3_nosys
package util
import "context"
import (
"context"
"github.com/tetratelabs/wazero/experimental"
)
type mmapState struct{}
func (s *mmapState) init(ctx context.Context, _ bool) context.Context {
return ctx
}
func CanMap(ctx context.Context) bool {
return false
func withAllocator(ctx context.Context) context.Context {
return experimental.WithMemoryAllocator(ctx,
experimental.MemoryAllocatorFunc(func(cap, max uint64) experimental.LinearMemory {
if cap == max {
return virtualAlloc(cap, max)
}
return sliceAlloc(cap, max)
}))
}

View File

@@ -8,14 +8,14 @@ import (
type moduleKey struct{}
type moduleState struct {
handleState
mmapState
handleState
}
func NewContext(ctx context.Context, mappableMemory bool) context.Context {
func NewContext(ctx context.Context) context.Context {
state := new(moduleState)
ctx = context.WithValue(ctx, moduleKey{}, state)
ctx = withAllocator(ctx)
ctx = experimental.WithCloseNotifier(ctx, state)
ctx = state.mmapState.init(ctx, mappableMemory)
ctx = context.WithValue(ctx, moduleKey{}, state)
return ctx
}

View File

@@ -28,6 +28,14 @@ var (
RuntimeConfig wazero.RuntimeConfig
)
// Initialize decodes and compiles the SQLite Wasm binary.
// This is called implicitly when the first connection is openned,
// but is potentially slow, so you may want to call it at a more convenient time.
func Initialize() error {
instance.once.Do(compileSQLite)
return instance.err
}
var instance struct {
runtime wazero.Runtime
compiled wazero.CompiledModule
@@ -79,13 +87,12 @@ type sqlite struct {
}
func instantiateSQLite() (sqlt *sqlite, err error) {
instance.once.Do(compileSQLite)
if instance.err != nil {
return nil, instance.err
if err := Initialize(); err != nil {
return nil, err
}
sqlt = new(sqlite)
sqlt.ctx = util.NewContext(context.Background(), vfs.SupportsSharedMemory)
sqlt.ctx = util.NewContext(context.Background())
sqlt.mod, err = instance.runtime.InstantiateModule(sqlt.ctx,
instance.compiled, wazero.NewModuleConfig().WithName(""))
@@ -289,8 +296,9 @@ func (a *arena) string(s string) uint32 {
}
func exportCallbacks(env wazero.HostModuleBuilder) wazero.HostModuleBuilder {
util.ExportFuncIII(env, "go_busy_handler", busyCallback)
util.ExportFuncII(env, "go_progress_handler", progressCallback)
util.ExportFuncIIII(env, "go_busy_timeout", timeoutCallback)
util.ExportFuncIII(env, "go_busy_handler", busyCallback)
util.ExportFuncII(env, "go_commit_hook", commitCallback)
util.ExportFuncVI(env, "go_rollback_hook", rollbackCallback)
util.ExportFuncVIIIIJ(env, "go_update_hook", updateCallback)

View File

@@ -0,0 +1,13 @@
# Replace sqliteDefaultBusyCallback.
# This patch allows Go to handle (and interrupt) sqlite3_busy_timeout.
--- sqlite3.c.orig
+++ sqlite3.c
@@ -181614,7 +181614,7 @@
if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
#endif
if( ms>0 ){
- sqlite3_busy_handler(db, (int(*)(void*,int))sqliteDefaultBusyCallback,
+ sqlite3_busy_handler(db, (int(*)(void*,int))sqliteBusyCallback,
(void*)db);
db->busyTimeout = ms;
}else{

View File

@@ -1,556 +0,0 @@
# Backport from 3.46.
# https://sqlite.org/draft/releaselog/current.html
--- sqlite3.c.orig
+++ sqlite3.c
@@ -71,13 +71,14 @@ struct DateTime {
int tz; /* Timezone offset in minutes */
double s; /* Seconds */
char validJD; /* True (1) if iJD is valid */
- char rawS; /* Raw numeric value stored in s */
char validYMD; /* True (1) if Y,M,D are valid */
char validHMS; /* True (1) if h,m,s are valid */
- char validTZ; /* True (1) if tz is valid */
- char tzSet; /* Timezone was set explicitly */
- char isError; /* An overflow has occurred */
- char useSubsec; /* Display subsecond precision */
+ char nFloor; /* Days to implement "floor" */
+ unsigned rawS : 1; /* Raw numeric value stored in s */
+ unsigned isError : 1; /* An overflow has occurred */
+ unsigned useSubsec : 1; /* Display subsecond precision */
+ unsigned isUtc : 1; /* Time is known to be UTC */
+ unsigned isLocal : 1; /* Time is known to be localtime */
};
@@ -175,6 +176,8 @@ static int parseTimezone(const char *zDate, DateTime *p){
sgn = +1;
}else if( c=='Z' || c=='z' ){
zDate++;
+ p->isLocal = 0;
+ p->isUtc = 1;
goto zulu_time;
}else{
return c!=0;
@@ -187,7 +190,6 @@ static int parseTimezone(const char *zDate, DateTime *p){
p->tz = sgn*(nMn + nHr*60);
zulu_time:
while( sqlite3Isspace(*zDate) ){ zDate++; }
- p->tzSet = 1;
return *zDate!=0;
}
@@ -231,7 +233,6 @@ static int parseHhMmSs(const char *zDate, DateTime *p){
p->m = m;
p->s = s + ms;
if( parseTimezone(zDate, p) ) return 1;
- p->validTZ = (p->tz!=0)?1:0;
return 0;
}
@@ -278,15 +279,40 @@ static void computeJD(DateTime *p){
p->validJD = 1;
if( p->validHMS ){
p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000 + 0.5);
- if( p->validTZ ){
+ if( p->tz ){
p->iJD -= p->tz*60000;
p->validYMD = 0;
p->validHMS = 0;
- p->validTZ = 0;
+ p->tz = 0;
+ p->isUtc = 1;
+ p->isLocal = 0;
}
}
}
+/*
+** Given the YYYY-MM-DD information current in p, determine if there
+** is day-of-month overflow and set nFloor to the number of days that
+** would need to be subtracted from the date in order to bring the
+** date back to the end of the month.
+*/
+static void computeFloor(DateTime *p){
+ assert( p->validYMD || p->isError );
+ assert( p->D>=0 && p->D<=31 );
+ assert( p->M>=0 && p->M<=12 );
+ if( p->D<=28 ){
+ p->nFloor = 0;
+ }else if( (1<<p->M) & 0x15aa ){
+ p->nFloor = 0;
+ }else if( p->M!=2 ){
+ p->nFloor = (p->D==31);
+ }else if( p->Y%4!=0 || (p->Y%100==0 && p->Y%400!=0) ){
+ p->nFloor = p->D - 28;
+ }else{
+ p->nFloor = p->D - 29;
+ }
+}
+
/*
** Parse dates of the form
**
@@ -325,12 +351,16 @@ static int parseYyyyMmDd(const char *zDate, DateTime *p){
p->Y = neg ? -Y : Y;
p->M = M;
p->D = D;
- if( p->validTZ ){
+ computeFloor(p);
+ if( p->tz ){
computeJD(p);
}
return 0;
}
+
+static void clearYMD_HMS_TZ(DateTime *p); /* Forward declaration */
+
/*
** Set the time to the current time reported by the VFS.
**
@@ -340,6 +370,9 @@ static int setDateTimeToCurrent(sqlite3_context *context, DateTime *p){
p->iJD = sqlite3StmtCurrentTime(context);
if( p->iJD>0 ){
p->validJD = 1;
+ p->isUtc = 1;
+ p->isLocal = 0;
+ clearYMD_HMS_TZ(p);
return 0;
}else{
return 1;
@@ -478,7 +511,7 @@ static void computeYMD_HMS(DateTime *p){
static void clearYMD_HMS_TZ(DateTime *p){
p->validYMD = 0;
p->validHMS = 0;
- p->validTZ = 0;
+ p->tz = 0;
}
#ifndef SQLITE_OMIT_LOCALTIME
@@ -610,7 +643,7 @@ static int toLocaltime(
p->validHMS = 1;
p->validJD = 0;
p->rawS = 0;
- p->validTZ = 0;
+ p->tz = 0;
p->isError = 0;
return SQLITE_OK;
}
@@ -630,12 +663,12 @@ static const struct {
float rLimit; /* Maximum NNN value for this transform */
float rXform; /* Constant used for this transform */
} aXformType[] = {
- { 6, "second", 4.6427e+14, 1.0 },
- { 6, "minute", 7.7379e+12, 60.0 },
- { 4, "hour", 1.2897e+11, 3600.0 },
- { 3, "day", 5373485.0, 86400.0 },
- { 5, "month", 176546.0, 2592000.0 },
- { 4, "year", 14713.0, 31536000.0 },
+ /* 0 */ { 6, "second", 4.6427e+14, 1.0 },
+ /* 1 */ { 6, "minute", 7.7379e+12, 60.0 },
+ /* 2 */ { 4, "hour", 1.2897e+11, 3600.0 },
+ /* 3 */ { 3, "day", 5373485.0, 86400.0 },
+ /* 4 */ { 5, "month", 176546.0, 30.0*86400.0 },
+ /* 5 */ { 4, "year", 14713.0, 365.0*86400.0 },
};
/*
@@ -667,14 +700,20 @@ static void autoAdjustDate(DateTime *p){
** NNN.NNNN seconds
** NNN months
** NNN years
+** +/-YYYY-MM-DD HH:MM:SS.SSS
+** ceiling
+** floor
** start of month
** start of year
** start of week
** start of day
** weekday N
** unixepoch
+** auto
** localtime
** utc
+** subsec
+** subsecond
**
** Return 0 on success and 1 if there is any kind of error. If the error
** is in a system call (i.e. localtime()), then an error message is written
@@ -705,6 +744,37 @@ static int parseModifier(
}
break;
}
+ case 'c': {
+ /*
+ ** ceiling
+ **
+ ** Resolve day-of-month overflow by rolling forward into the next
+ ** month. As this is the default action, this modifier is really
+ ** a no-op that is only included for symmetry. See "floor".
+ */
+ if( sqlite3_stricmp(z, "ceiling")==0 ){
+ computeJD(p);
+ clearYMD_HMS_TZ(p);
+ rc = 0;
+ p->nFloor = 0;
+ }
+ break;
+ }
+ case 'f': {
+ /*
+ ** floor
+ **
+ ** Resolve day-of-month overflow by rolling back to the end of the
+ ** previous month.
+ */
+ if( sqlite3_stricmp(z, "floor")==0 ){
+ computeJD(p);
+ p->iJD -= p->nFloor*86400000;
+ clearYMD_HMS_TZ(p);
+ rc = 0;
+ }
+ break;
+ }
case 'j': {
/*
** julianday
@@ -731,7 +801,9 @@ static int parseModifier(
** show local time.
*/
if( sqlite3_stricmp(z, "localtime")==0 && sqlite3NotPureFunc(pCtx) ){
- rc = toLocaltime(p, pCtx);
+ rc = p->isLocal ? SQLITE_OK : toLocaltime(p, pCtx);
+ p->isUtc = 0;
+ p->isLocal = 1;
}
break;
}
@@ -756,7 +828,7 @@ static int parseModifier(
}
#ifndef SQLITE_OMIT_LOCALTIME
else if( sqlite3_stricmp(z, "utc")==0 && sqlite3NotPureFunc(pCtx) ){
- if( p->tzSet==0 ){
+ if( p->isUtc==0 ){
i64 iOrigJD; /* Original localtime */
i64 iGuess; /* Guess at the corresponding utc time */
int cnt = 0; /* Safety to prevent infinite loop */
@@ -779,7 +851,8 @@ static int parseModifier(
memset(p, 0, sizeof(*p));
p->iJD = iGuess;
p->validJD = 1;
- p->tzSet = 1;
+ p->isUtc = 1;
+ p->isLocal = 0;
}
rc = SQLITE_OK;
}
@@ -799,7 +872,7 @@ static int parseModifier(
&& r>=0.0 && r<7.0 && (n=(int)r)==r ){
sqlite3_int64 Z;
computeYMD_HMS(p);
- p->validTZ = 0;
+ p->tz = 0;
p->validJD = 0;
computeJD(p);
Z = ((p->iJD + 129600000)/86400000) % 7;
@@ -839,7 +912,7 @@ static int parseModifier(
p->h = p->m = 0;
p->s = 0.0;
p->rawS = 0;
- p->validTZ = 0;
+ p->tz = 0;
p->validJD = 0;
if( sqlite3_stricmp(z,"month")==0 ){
p->D = 1;
@@ -910,6 +983,7 @@ static int parseModifier(
x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12;
p->Y += x;
p->M -= x*12;
+ computeFloor(p);
computeJD(p);
p->validHMS = 0;
p->validYMD = 0;
@@ -956,11 +1030,12 @@ static int parseModifier(
z += n;
while( sqlite3Isspace(*z) ) z++;
n = sqlite3Strlen30(z);
- if( n>10 || n<3 ) break;
+ if( n<3 || n>10 ) break;
if( sqlite3UpperToLower[(u8)z[n-1]]=='s' ) n--;
computeJD(p);
assert( rc==1 );
rRounder = r<0 ? -0.5 : +0.5;
+ p->nFloor = 0;
for(i=0; i<ArraySize(aXformType); i++){
if( aXformType[i].nName==n
&& sqlite3_strnicmp(aXformType[i].zName, z, n)==0
@@ -968,21 +1043,24 @@ static int parseModifier(
){
switch( i ){
case 4: { /* Special processing to add months */
- assert( strcmp(aXformType[i].zName,"month")==0 );
+ assert( strcmp(aXformType[4].zName,"month")==0 );
computeYMD_HMS(p);
p->M += (int)r;
x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12;
p->Y += x;
p->M -= x*12;
+ computeFloor(p);
p->validJD = 0;
r -= (int)r;
break;
}
case 5: { /* Special processing to add years */
int y = (int)r;
- assert( strcmp(aXformType[i].zName,"year")==0 );
+ assert( strcmp(aXformType[5].zName,"year")==0 );
computeYMD_HMS(p);
+ assert( p->M>=0 && p->M<=12 );
p->Y += y;
+ computeFloor(p);
p->validJD = 0;
r -= (int)r;
break;
@@ -1236,22 +1314,83 @@ static void dateFunc(
}
}
+/*
+** Compute the number of days after the most recent January 1.
+**
+** In other words, compute the zero-based day number for the
+** current year:
+**
+** Jan01 = 0, Jan02 = 1, ..., Jan31 = 30, Feb01 = 31, ...
+** Dec31 = 364 or 365.
+*/
+static int daysAfterJan01(DateTime *pDate){
+ DateTime jan01 = *pDate;
+ assert( jan01.validYMD );
+ assert( jan01.validHMS );
+ assert( pDate->validJD );
+ jan01.validJD = 0;
+ jan01.M = 1;
+ jan01.D = 1;
+ computeJD(&jan01);
+ return (int)((pDate->iJD-jan01.iJD+43200000)/86400000);
+}
+
+/*
+** Return the number of days after the most recent Monday.
+**
+** In other words, return the day of the week according
+** to this code:
+**
+** 0=Monday, 1=Tuesday, 2=Wednesday, ..., 6=Sunday.
+*/
+static int daysAfterMonday(DateTime *pDate){
+ assert( pDate->validJD );
+ return (int)((pDate->iJD+43200000)/86400000) % 7;
+}
+
+/*
+** Return the number of days after the most recent Sunday.
+**
+** In other words, return the day of the week according
+** to this code:
+**
+** 0=Sunday, 1=Monday, 2=Tues, ..., 6=Saturday
+*/
+static int daysAfterSunday(DateTime *pDate){
+ assert( pDate->validJD );
+ return (int)((pDate->iJD+129600000)/86400000) % 7;
+}
+
/*
** strftime( FORMAT, TIMESTRING, MOD, MOD, ...)
**
** Return a string described by FORMAT. Conversions as follows:
**
-** %d day of month
+** %d day of month 01-31
+** %e day of month 1-31
** %f ** fractional seconds SS.SSS
+** %F ISO date. YYYY-MM-DD
+** %G ISO year corresponding to %V 0000-9999.
+** %g 2-digit ISO year corresponding to %V 00-99
** %H hour 00-24
-** %j day of year 000-366
+** %k hour 0-24 (leading zero converted to space)
+** %I hour 01-12
+** %j day of year 001-366
** %J ** julian day number
+** %l hour 1-12 (leading zero converted to space)
** %m month 01-12
** %M minute 00-59
+** %p "am" or "pm"
+** %P "AM" or "PM"
+** %R time as HH:MM
** %s seconds since 1970-01-01
** %S seconds 00-59
-** %w day of week 0-6 Sunday==0
-** %W week of year 00-53
+** %T time as HH:MM:SS
+** %u day of week 1-7 Monday==1, Sunday==7
+** %w day of week 0-6 Sunday==0, Monday==1
+** %U week of year 00-53 (First Sunday is start of week 01)
+** %V week of year 01-53 (First week containing Thursday is week 01)
+** %W week of year 00-53 (First Monday is start of week 01)
** %Y year 0000-9999
** %% %
*/
@@ -1288,7 +1427,7 @@ static void strftimeFunc(
sqlite3_str_appendf(&sRes, cf=='d' ? "%02d" : "%2d", x.D);
break;
}
- case 'f': {
+ case 'f': { /* Fractional seconds. (Non-standard) */
double s = x.s;
if( s>59.999 ) s = 59.999;
sqlite3_str_appendf(&sRes, "%06.3f", s);
@@ -1298,6 +1437,21 @@ static void strftimeFunc(
sqlite3_str_appendf(&sRes, "%04d-%02d-%02d", x.Y, x.M, x.D);
break;
}
+ case 'G': /* Fall thru */
+ case 'g': {
+ DateTime y = x;
+ assert( y.validJD );
+ /* Move y so that it is the Thursday in the same week as x */
+ y.iJD += (3 - daysAfterMonday(&x))*86400000;
+ y.validYMD = 0;
+ computeYMD(&y);
+ if( cf=='g' ){
+ sqlite3_str_appendf(&sRes, "%02d", y.Y%100);
+ }else{
+ sqlite3_str_appendf(&sRes, "%04d", y.Y);
+ }
+ break;
+ }
case 'H':
case 'k': {
sqlite3_str_appendf(&sRes, cf=='H' ? "%02d" : "%2d", x.h);
@@ -1311,25 +1465,11 @@ static void strftimeFunc(
sqlite3_str_appendf(&sRes, cf=='I' ? "%02d" : "%2d", h);
break;
}
- case 'W': /* Fall thru */
- case 'j': {
- int nDay; /* Number of days since 1st day of year */
- DateTime y = x;
- y.validJD = 0;
- y.M = 1;
- y.D = 1;
- computeJD(&y);
- nDay = (int)((x.iJD-y.iJD+43200000)/86400000);
- if( cf=='W' ){
- int wd; /* 0=Monday, 1=Tuesday, ... 6=Sunday */
- wd = (int)(((x.iJD+43200000)/86400000)%7);
- sqlite3_str_appendf(&sRes,"%02d",(nDay+7-wd)/7);
- }else{
- sqlite3_str_appendf(&sRes,"%03d",nDay+1);
- }
+ case 'j': { /* Day of year. Jan01==1, Jan02==2, and so forth */
+ sqlite3_str_appendf(&sRes,"%03d",daysAfterJan01(&x)+1);
break;
}
- case 'J': {
+ case 'J': { /* Julian day number. (Non-standard) */
sqlite3_str_appendf(&sRes,"%.16g",x.iJD/86400000.0);
break;
}
@@ -1372,13 +1512,33 @@ static void strftimeFunc(
sqlite3_str_appendf(&sRes,"%02d:%02d:%02d", x.h, x.m, (int)x.s);
break;
}
- case 'u': /* Fall thru */
- case 'w': {
- char c = (char)(((x.iJD+129600000)/86400000) % 7) + '0';
+ case 'u': /* Day of week. 1 to 7. Monday==1, Sunday==7 */
+ case 'w': { /* Day of week. 0 to 6. Sunday==0, Monday==1 */
+ char c = (char)daysAfterSunday(&x) + '0';
if( c=='0' && cf=='u' ) c = '7';
sqlite3_str_appendchar(&sRes, 1, c);
break;
}
+ case 'U': { /* Week num. 00-53. First Sun of the year is week 01 */
+ sqlite3_str_appendf(&sRes,"%02d",
+ (daysAfterJan01(&x)-daysAfterSunday(&x)+7)/7);
+ break;
+ }
+ case 'V': { /* Week num. 01-53. First week with a Thur is week 01 */
+ DateTime y = x;
+ /* Adjust y so that is the Thursday in the same week as x */
+ assert( y.validJD );
+ y.iJD += (3 - daysAfterMonday(&x))*86400000;
+ y.validYMD = 0;
+ computeYMD(&y);
+ sqlite3_str_appendf(&sRes,"%02d", daysAfterJan01(&y)/7+1);
+ break;
+ }
+ case 'W': { /* Week num. 00-53. First Mon of the year is week 01 */
+ sqlite3_str_appendf(&sRes,"%02d",
+ (daysAfterJan01(&x)-daysAfterMonday(&x)+7)/7);
+ break;
+ }
case 'Y': {
sqlite3_str_appendf(&sRes,"%04d",x.Y);
break;
@@ -1525,9 +1685,7 @@ static void timediffFunc(
d1.iJD = d2.iJD - d1.iJD;
d1.iJD += (u64)1486995408 * (u64)100000;
}
- d1.validYMD = 0;
- d1.validHMS = 0;
- d1.validTZ = 0;
+ clearYMD_HMS_TZ(&d1);
computeYMD_HMS(&d1);
sqlite3StrAccumInit(&sRes, 0, 0, 0, 100);
sqlite3_str_appendf(&sRes, "%c%04d-%02d-%02d %02d:%02d:%06.3f",
@@ -1596,6 +1754,36 @@ static void currentTimeFunc(
}
#endif
+#if !defined(SQLITE_OMIT_DATETIME_FUNCS) && defined(SQLITE_DEBUG)
+/*
+** datedebug(...)
+**
+** This routine returns JSON that describes the internal DateTime object.
+** Used for debugging and testing only. Subject to change.
+*/
+static void datedebugFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ DateTime x;
+ if( isDate(context, argc, argv, &x)==0 ){
+ char *zJson;
+ zJson = sqlite3_mprintf(
+ "{iJD:%lld,Y:%d,M:%d,D:%d,h:%d,m:%d,tz:%d,"
+ "s:%.3f,validJD:%d,validYMS:%d,validHMS:%d,"
+ "nFloor:%d,rawS:%d,isError:%d,useSubsec:%d,"
+ "isUtc:%d,isLocal:%d}",
+ x.iJD, x.Y, x.M, x.D, x.h, x.m, x.tz,
+ x.s, x.validJD, x.validYMD, x.validHMS,
+ x.nFloor, x.rawS, x.isError, x.useSubsec,
+ x.isUtc, x.isLocal);
+ sqlite3_result_text(context, zJson, -1, sqlite3_free);
+ }
+}
+#endif /* !SQLITE_OMIT_DATETIME_FUNCS && SQLITE_DEBUG */
+
+
/*
** This function registered all of the above C functions as SQL
** functions. This should be the only routine in this file with
@@ -1611,6 +1799,9 @@ void sqlite3RegisterDateTimeFunctions(void){
PURE_DATE(datetime, -1, 0, 0, datetimeFunc ),
PURE_DATE(strftime, -1, 0, 0, strftimeFunc ),
PURE_DATE(timediff, 2, 0, 0, timediffFunc ),
+#ifdef SQLITE_DEBUG
+ PURE_DATE(datedebug, -1, 0, 0, datedebugFunc ),
+#endif
DFUNCTION(current_time, 0, 0, 0, ctimeFunc ),
DFUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc),
DFUNCTION(current_date, 0, 0, 0, cdateFunc ),

View File

@@ -3,7 +3,7 @@ set -euo pipefail
cd -P -- "$(dirname -- "$0")"
curl -#OL "https://sqlite.org/2024/sqlite-amalgamation-3450200.zip"
curl -#OL "https://sqlite.org/2024/sqlite-amalgamation-3460000.zip"
unzip -d . sqlite-amalgamation-*.zip
mv sqlite-amalgamation-*/sqlite3* .
rm -rf sqlite-amalgamation-*
@@ -12,25 +12,25 @@ cat *.patch | patch --no-backup-if-mismatch
mkdir -p ext/
cd ext/
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/ext/misc/anycollseq.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/ext/misc/base64.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/ext/misc/decimal.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/ext/misc/ieee754.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/ext/misc/regexp.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/ext/misc/series.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/ext/misc/uint.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/ext/misc/uuid.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/ext/misc/anycollseq.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/ext/misc/base64.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/ext/misc/decimal.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/ext/misc/ieee754.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/ext/misc/regexp.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/ext/misc/series.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/ext/misc/uint.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/ext/misc/uuid.c"
cd ~-
cd ../vfs/tests/mptest/testdata/
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/mptest/mptest.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/mptest/config01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/mptest/config02.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/mptest/crash01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/mptest/crash02.subtest"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/mptest/multiwrite01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/mptest/mptest.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/mptest/config01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/mptest/config02.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/mptest/crash01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/mptest/crash02.subtest"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/mptest/multiwrite01.test"
cd ~-
cd ../vfs/tests/speedtest1/testdata/
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/test/speedtest1.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/test/speedtest1.c"
cd ~-

View File

@@ -4,6 +4,7 @@
int go_progress_handler(void *);
int go_busy_handler(void *, int);
int go_busy_timeout(void *, int count, int tmout);
int go_commit_hook(void *);
void go_rollback_hook(void *);
@@ -55,4 +56,12 @@ int sqlite3_autovacuum_pages_go(sqlite3 *db, go_handle app) {
int rc = sqlite3_autovacuum_pages(db, go_autovacuum_pages, app, go_destroy);
if (rc) go_destroy(app);
return rc;
}
}
#ifndef sqliteBusyCallback
static int sqliteBusyCallback(sqlite3 *db, int count) {
return go_busy_timeout(db, count, db->busyTimeout);
}
#endif

View File

@@ -1,14 +0,0 @@
# Use exclusive locking mode for WAL databases with v1 VFSes.
--- sqlite3.c.orig
+++ sqlite3.c
@@ -64209,7 +64209,9 @@
SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager){
const sqlite3_io_methods *pMethods = pPager->fd->pMethods;
if( pPager->noLock ) return 0;
- return pPager->exclusiveMode || (pMethods->iVersion>=2 && pMethods->xShmMap);
+ if( pMethods->iVersion>=2 && pMethods->xShmMap ) return 1;
+ pPager->exclusiveMode = 1;
+ return 1;
}
/*

View File

@@ -18,6 +18,8 @@
#define HAVE_STDINT_H 1
#define HAVE_INTTYPES_H 1
#define LONGDOUBLE_TYPE double
#define HAVE_LOG2 1
#define HAVE_LOG10 1
#define HAVE_ISNAN 1
@@ -33,52 +35,14 @@
#define HAVE_MALLOC_H 1
#define HAVE_MALLOC_USABLE_SIZE 1
// Recommended Options
#define SQLITE_DQS 0
#define SQLITE_THREADSAFE 0
#define SQLITE_DEFAULT_MEMSTATUS 0
#define SQLITE_DEFAULT_WAL_SYNCHRONOUS 1
#define SQLITE_LIKE_DOESNT_MATCH_BLOBS
#define SQLITE_MAX_EXPR_DEPTH 0
#define SQLITE_STRICT_SUBTYPE 1
#define SQLITE_USE_ALLOCA
#define SQLITE_OMIT_DEPRECATED
#define SQLITE_OMIT_SHARED_CACHE
#define SQLITE_OMIT_AUTOINIT
// #define SQLITE_OMIT_DECLTYPE
// #define SQLITE_OMIT_PROGRESS_CALLBACK
// Other Options
#define SQLITE_ALLOW_URI_AUTHORITY
#define SQLITE_TRUSTED_SCHEMA 0
#define SQLITE_DEFAULT_FOREIGN_KEYS 1
#define SQLITE_ENABLE_ATOMIC_WRITE
#define SQLITE_ENABLE_BATCH_ATOMIC_WRITE
// Because Wasm does not support shared memory,
// SQLite disables WAL for Wasm builds.
// We patch SQLite to use exclusive locking mode instead.
// https://sqlite.org/wal.html#noshm
#undef SQLITE_OMIT_WAL
// We have our own memdb VFS.
// To avoid interactions between the two,
// omit sqlite3_serialize/sqlite3_deserialize,
// which we also don't wrap.
#define SQLITE_OMIT_DESERIALIZE
// Amalgamated Extensions
#define SQLITE_ENABLE_MATH_FUNCTIONS 1
#define SQLITE_ENABLE_JSON1 1
#define SQLITE_ENABLE_FTS5 1
#define SQLITE_ENABLE_RTREE 1
#define SQLITE_ENABLE_GEOPOLY 1
#define SQLITE_SOUNDEX
#define SQLITE_UNTESTABLE
// Implemented in vfs.c.
int localtime_s(struct tm *const pTm, time_t const *const pTime);
int localtime_s(struct tm *const pTm, time_t const *const pTime);
// Implemented in hooks.c.
#ifndef sqliteBusyCallback
static int sqliteBusyCallback(sqlite3 *, int);
#endif

44
sqlite3/sqlite_opt.h Normal file
View File

@@ -0,0 +1,44 @@
// Recommended Options
#define SQLITE_DQS 0
#define SQLITE_THREADSAFE 0
#define SQLITE_DEFAULT_MEMSTATUS 0
#define SQLITE_DEFAULT_WAL_SYNCHRONOUS 1
#define SQLITE_LIKE_DOESNT_MATCH_BLOBS
#define SQLITE_MAX_EXPR_DEPTH 0
#define SQLITE_STRICT_SUBTYPE 1
#define SQLITE_USE_ALLOCA
#define SQLITE_OMIT_DEPRECATED
#define SQLITE_OMIT_SHARED_CACHE
#define SQLITE_OMIT_AUTOINIT
// We need these:
// #define SQLITE_OMIT_DECLTYPE
// #define SQLITE_OMIT_PROGRESS_CALLBACK
// Other Options
#define SQLITE_ALLOW_URI_AUTHORITY
#define SQLITE_TRUSTED_SCHEMA 0
#define SQLITE_DEFAULT_FOREIGN_KEYS 1
#define SQLITE_ENABLE_ATOMIC_WRITE
#define SQLITE_ENABLE_BATCH_ATOMIC_WRITE
#define SQLITE_ENABLE_COLUMN_METADATA
#define SQLITE_ENABLE_STAT4 1
// We have our own memdb VFS.
// To avoid interactions between the two,
// omit sqlite3_serialize/sqlite3_deserialize,
// which we also don't wrap.
#define SQLITE_OMIT_DESERIALIZE
// Amalgamated Extensions
#define SQLITE_ENABLE_MATH_FUNCTIONS 1
#define SQLITE_ENABLE_JSON1 1
#define SQLITE_ENABLE_FTS5 1
#define SQLITE_ENABLE_RTREE 1
#define SQLITE_ENABLE_GEOPOLY 1
#define SQLITE_SOUNDEX
#define SQLITE_UNTESTABLE

View File

@@ -5,6 +5,8 @@
static int time_collation(void *pArg, int nKey1, const void *pKey1, int nKey2,
const void *pKey2) {
UNUSED_PARAMETER(pArg);
// Remove a Z suffix if one key is no longer than the other.
// A Z suffix collates before any character but after the empty string.
// This avoids making different keys equal.
@@ -29,8 +31,8 @@ static int time_collation(void *pArg, int nKey1, const void *pKey1, int nKey2,
int sqlite3_time_init(sqlite3 *db, char **pzErrMsg,
const sqlite3_api_routines *pApi) {
UNUSED_PARAMETER2(pzErrMsg, pApi);
sqlite3_create_collation_v2(db, "time", SQLITE_UTF8, /*arg=*/NULL,
time_collation,
/*destroy=*/NULL);
time_collation, /*destroy=*/NULL);
return SQLITE_OK;
}

View File

@@ -1,7 +1,8 @@
# Wrap sqlite3_vfs_find.
# This patch allows Go VFSes to be (un)registered.
--- sqlite3.c.orig
+++ sqlite3.c
@@ -26089,7 +26089,7 @@
@@ -26396,7 +26396,7 @@
** Locate a VFS by name. If no name is given, simply return the
** first VFS on the list.
*/

View File

@@ -72,6 +72,8 @@ static void go_mod_destroy(void *pAux) {
static int go_vtab_create_wrapper(sqlite3 *db, void *pAux, int argc,
const char *const *argv,
sqlite3_vtab **ppVTab, char **pzErr) {
UNUSED_PARAMETER(db);
struct go_vtab *vtab = calloc(1, sizeof(struct go_vtab));
if (vtab == NULL) return SQLITE_NOMEM;
*ppVTab = &vtab->base;
@@ -88,6 +90,8 @@ static int go_vtab_create_wrapper(sqlite3 *db, void *pAux, int argc,
static int go_vtab_connect_wrapper(sqlite3 *db, void *pAux, int argc,
const char *const *argv,
sqlite3_vtab **ppVTab, char **pzErr) {
UNUSED_PARAMETER(db);
struct go_vtab *vtab = calloc(1, sizeof(struct go_vtab));
if (vtab == NULL) return SQLITE_NOMEM;
*ppVTab = &vtab->base;

View File

@@ -7,10 +7,12 @@ import (
"testing"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/tetratelabs/wazero"
)
func init() {
Path = "./embed/sqlite3.wasm"
RuntimeConfig = wazero.NewRuntimeConfig().WithMemoryLimitPages(1024)
}
func Test_sqlite_error_OOM(t *testing.T) {

48
stmt.go
View File

@@ -367,12 +367,10 @@ func (s *Stmt) ColumnCount() int {
func (s *Stmt) ColumnName(col int) string {
r := s.c.call("sqlite3_column_name",
uint64(s.handle), uint64(col))
ptr := uint32(r)
if ptr == 0 {
if r == 0 {
panic(util.OOMErr)
}
return util.ReadString(s.c.mod, ptr, _MAX_NAME)
return util.ReadString(s.c.mod, uint32(r), _MAX_NAME)
}
// ColumnType returns the initial [Datatype] of the result column.
@@ -398,6 +396,48 @@ func (s *Stmt) ColumnDeclType(col int) string {
return util.ReadString(s.c.mod, uint32(r), _MAX_NAME)
}
// ColumnDatabaseName returns the name of the database
// that is the origin of a particular result column.
// The leftmost column of the result set has the index 0.
//
// https://sqlite.org/c3ref/column_database_name.html
func (s *Stmt) ColumnDatabaseName(col int) string {
r := s.c.call("sqlite3_column_database_name",
uint64(s.handle), uint64(col))
if r == 0 {
return ""
}
return util.ReadString(s.c.mod, uint32(r), _MAX_NAME)
}
// ColumnTableName returns the name of the table
// that is the origin of a particular result column.
// The leftmost column of the result set has the index 0.
//
// https://sqlite.org/c3ref/column_database_name.html
func (s *Stmt) ColumnTableName(col int) string {
r := s.c.call("sqlite3_column_table_name",
uint64(s.handle), uint64(col))
if r == 0 {
return ""
}
return util.ReadString(s.c.mod, uint32(r), _MAX_NAME)
}
// ColumnOriginName returns the name of the table column
// that is the origin of a particular result column.
// The leftmost column of the result set has the index 0.
//
// https://sqlite.org/c3ref/column_database_name.html
func (s *Stmt) ColumnOriginName(col int) string {
r := s.c.call("sqlite3_column_origin_name",
uint64(s.handle), uint64(col))
if r == 0 {
return ""
}
return util.ReadString(s.c.mod, uint32(r), _MAX_NAME)
}
// ColumnBool returns the value of the result column as a bool.
// The leftmost column of the result set has the index 0.
// SQLite does not have a separate boolean storage class.

View File

@@ -6,9 +6,14 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
"github.com/ncruces/go-sqlite3/vfs"
)
func TestBackup(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
t.Parallel()
backupName := filepath.Join(t.TempDir(), "backup.db")

View File

@@ -11,6 +11,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestBlob(t *testing.T) {

View File

@@ -1,3 +1,5 @@
//go:build (linux || darwin || windows || freebsd || illumos) && !sqlite3_nosys
package bradfitz
// Adapted from: https://github.com/bradfitz/go-sql-test
@@ -12,6 +14,7 @@ import (
_ "github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
type Tester interface {

View File

@@ -11,6 +11,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/vfs/memdb"
)
@@ -138,7 +139,7 @@ func TestConn_SetInterrupt(t *testing.T) {
SELECT 0, 1
UNION ALL
SELECT next, curr + next FROM fibonacci
LIMIT 1e6
LIMIT 1e7
)
SELECT min(curr) FROM fibonacci
`)
@@ -415,6 +416,48 @@ func TestConn_SetLastInsertRowID(t *testing.T) {
}
}
func TestConn_Filename(t *testing.T) {
t.Parallel()
file := filepath.Join(t.TempDir(), "test.db")
db, err := sqlite3.Open(file)
if err != nil {
t.Fatal(err)
}
defer db.Close()
n := db.Filename("")
if n.String() != file {
t.Errorf("got %v", n)
}
if n.Database() != file {
t.Errorf("got %v", n)
}
if n.DatabaseFile() == nil {
t.Errorf("got %v", n)
}
n = db.Filename("xpto")
if n != nil {
t.Errorf("got %v", n)
}
if n.String() != "" {
t.Errorf("got %v", n)
}
if n.Database() != "" {
t.Errorf("got %v", n)
}
if n.Journal() != "" {
t.Errorf("got %v", n)
}
if n.WAL() != "" {
t.Errorf("got %v", n)
}
if n.DatabaseFile() != nil {
t.Errorf("got %v", n)
}
}
func TestConn_ReadOnly(t *testing.T) {
t.Parallel()
@@ -454,7 +497,7 @@ func TestConn_DBName(t *testing.T) {
func TestConn_AutoVacuumPages(t *testing.T) {
t.Parallel()
db, err := sqlite3.Open("file:test.db?vfs=memdb&_pragma=auto_vacuum(FULL)")
db, err := sqlite3.Open("file:test.db?vfs=memdb&_pragma=auto_vacuum(full)")
if err != nil {
t.Fatal(err)
}

View File

@@ -9,14 +9,17 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
"github.com/ncruces/go-sqlite3/vfs"
_ "github.com/ncruces/go-sqlite3/vfs/adiantum"
_ "github.com/ncruces/go-sqlite3/vfs/memdb"
)
//go:embed testdata/wal.db
var waldb []byte
var walDB []byte
//go:embed testdata/utf16be.db
var utf16db []byte
var utf16DB []byte
func TestDB_memory(t *testing.T) {
t.Parallel()
@@ -24,21 +27,22 @@ func TestDB_memory(t *testing.T) {
}
func TestDB_file(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
t.Parallel()
testDB(t, filepath.Join(t.TempDir(), "test.db"))
}
func TestDB_nolock(t *testing.T) {
t.Parallel()
testDB(t, "file:"+
filepath.ToSlash(filepath.Join(t.TempDir(), "test.db"))+
"?nolock=1")
}
func TestDB_wal(t *testing.T) {
if !vfs.SupportsSharedMemory {
t.Skip("skipping without shared memory")
}
t.Parallel()
tmp := filepath.Join(t.TempDir(), "test.db")
err := os.WriteFile(tmp, waldb, 0666)
err := os.WriteFile(tmp, walDB, 0666)
if err != nil {
t.Fatal(err)
}
@@ -46,19 +50,37 @@ func TestDB_wal(t *testing.T) {
}
func TestDB_utf16(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
t.Parallel()
tmp := filepath.Join(t.TempDir(), "test.db")
err := os.WriteFile(tmp, utf16db, 0666)
err := os.WriteFile(tmp, utf16DB, 0666)
if err != nil {
t.Fatal(err)
}
testDB(t, tmp)
}
func TestDB_vfs(t *testing.T) {
func TestDB_memdb(t *testing.T) {
t.Parallel()
testDB(t, "file:test.db?vfs=memdb")
}
func TestDB_adiantum(t *testing.T) {
t.Parallel()
tmp := filepath.Join(t.TempDir(), "test.db")
testDB(t, "file:"+filepath.ToSlash(tmp)+"?nolock=1"+
"&vfs=adiantum&textkey=correct+horse+battery+staple")
}
func TestDB_nolock(t *testing.T) {
t.Parallel()
tmp := filepath.Join(t.TempDir(), "test.db")
testDB(t, "file:"+filepath.ToSlash(tmp)+"?nolock=1")
}
func testDB(t testing.TB, name string) {
db, err := sqlite3.Open(name)
if err != nil {

View File

@@ -6,6 +6,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestDriver(t *testing.T) {

75
tests/endian_test.go Normal file
View File

@@ -0,0 +1,75 @@
package tests
import (
"bytes"
"encoding/binary"
"log"
"strconv"
"testing"
"github.com/ncruces/go-sqlite3"
)
func Test_endianness(t *testing.T) {
big := binary.BigEndian.AppendUint64(nil, 0x1234567890ABCDEF)
little := binary.LittleEndian.AppendUint64(nil, 0x1234567890ABCDEF)
native := binary.NativeEndian.AppendUint64(nil, 0x1234567890ABCDEF)
switch {
case bytes.Equal(big, native):
t.Log("Platform is big endian")
case bytes.Equal(little, native):
t.Log("Platform is little endian")
default:
t.Fatal("Platform is middle endian")
}
db, err := sqlite3.Open(":memory:")
if err != nil {
log.Fatal(err)
}
defer db.Close()
err = db.Exec(`CREATE TABLE test (col)`)
if err != nil {
log.Fatal(err)
}
const value int64 = -9223372036854775808
{
stmt, _, err := db.Prepare(`INSERT INTO test VALUES (?)`)
if err != nil {
t.Fatal(err)
}
defer stmt.Close()
err = stmt.BindInt64(1, value)
if err != nil {
t.Fatal(err)
}
err = stmt.Exec()
if err != nil {
t.Fatal(err)
}
}
{
stmt, _, err := db.Prepare(`SELECT * FROM test`)
if err != nil {
t.Fatal(err)
}
defer stmt.Close()
if stmt.Step() {
if got := stmt.ColumnInt64(0); got != value {
t.Errorf("got %d, want %d", got, value)
}
if got := stmt.ColumnText(0); got != strconv.FormatInt(value, 10) {
t.Errorf("got %s, want %d", got, value)
}
}
if err != nil {
t.Fatal(err)
}
}
}

View File

@@ -5,6 +5,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func Test_base64(t *testing.T) {

View File

@@ -6,6 +6,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestCreateFunction(t *testing.T) {

View File

@@ -10,6 +10,7 @@ import (
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
"github.com/ncruces/julianday"
)

View File

@@ -12,17 +12,17 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
"github.com/ncruces/go-sqlite3/vfs"
_ "github.com/ncruces/go-sqlite3/vfs/adiantum"
"github.com/ncruces/go-sqlite3/vfs/memdb"
"github.com/tetratelabs/wazero"
)
func TestMain(m *testing.M) {
sqlite3.RuntimeConfig = wazero.NewRuntimeConfig().WithMemoryLimitPages(1024)
os.Exit(m.Run())
}
func Test_parallel(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
func TestParallel(t *testing.T) {
var iter int
if testing.Short() {
iter = 1000
@@ -39,7 +39,7 @@ func TestParallel(t *testing.T) {
testIntegrity(t, name)
}
func TestWAL(t *testing.T) {
func Test_wal(t *testing.T) {
if !vfs.SupportsSharedMemory {
t.Skip("skipping without shared memory")
}
@@ -53,7 +53,7 @@ func TestWAL(t *testing.T) {
testIntegrity(t, name)
}
func TestMemory(t *testing.T) {
func Test_memdb(t *testing.T) {
var iter int
if testing.Short() {
iter = 1000
@@ -61,12 +61,37 @@ func TestMemory(t *testing.T) {
iter = 5000
}
memdb.Delete("test.db")
memdb.Create("test.db", nil)
name := "file:/test.db?vfs=memdb"
testParallel(t, name, iter)
testIntegrity(t, name)
}
func Test_adiantum(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
var iter int
if testing.Short() {
iter = 1000
} else {
iter = 5000
}
name := "file:" +
filepath.ToSlash(filepath.Join(t.TempDir(), "test.db")) +
"?vfs=adiantum" +
"&_pragma=hexkey(e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855)"
testParallel(t, name, iter)
testIntegrity(t, name)
}
func TestMultiProcess(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
if testing.Short() {
t.Skip("skipping in short mode")
}
@@ -117,8 +142,12 @@ func TestChildProcess(t *testing.T) {
testParallel(t, name, 1000)
}
func BenchmarkMemory(b *testing.B) {
func Benchmark_memdb(b *testing.B) {
sqlite3.Initialize()
b.ResetTimer()
memdb.Delete("test.db")
memdb.Create("test.db", nil)
name := "file:/test.db?vfs=memdb"
testParallel(b, name, b.N)
}

View File

@@ -8,6 +8,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestStmt(t *testing.T) {
@@ -19,7 +20,7 @@ func TestStmt(t *testing.T) {
}
defer db.Close()
err = db.Exec(`CREATE TABLE test (col)`)
err = db.Exec(`CREATE TABLE test (col ANY) STRICT`)
if err != nil {
t.Fatal(err)
}
@@ -135,7 +136,7 @@ func TestStmt(t *testing.T) {
}
// The table should have: 0, 1, 2, π, NULL, "", "text", "", "blob", NULL, "\0\0\0\0", "true", NULL
stmt, _, err = db.Prepare(`SELECT col FROM test`)
stmt, _, err = db.Prepare(`SELECT col AS c FROM test`)
if err != nil {
t.Fatal(err)
}
@@ -144,6 +145,21 @@ func TestStmt(t *testing.T) {
if got := stmt.ReadOnly(); got != true {
t.Error("got false, want true")
}
if got := stmt.ColumnName(0); got != "c" {
t.Errorf(`got %q, want "c"`, got)
}
if got := stmt.ColumnDeclType(0); got != "ANY" {
t.Errorf(`got %q, want "ANY"`, got)
}
if got := stmt.ColumnOriginName(0); got != "col" {
t.Errorf(`got %q, want "col"`, got)
}
if got := stmt.ColumnTableName(0); got != "test" {
t.Errorf(`got %q, want "test"`, got)
}
if got := stmt.ColumnDatabaseName(0); got != "main" {
t.Errorf(`got %q, want "main"`, got)
}
if stmt.Step() {
if got := stmt.ColumnType(0); got != sqlite3.INTEGER {

24
tests/testcfg/testcfg.go Normal file
View File

@@ -0,0 +1,24 @@
package testcfg
import (
"os"
"path/filepath"
"github.com/ncruces/go-sqlite3"
"github.com/tetratelabs/wazero"
)
func init() {
sqlite3.RuntimeConfig = wazero.NewRuntimeConfig().
WithMemoryCapacityFromMax(true).
WithMemoryLimitPages(1024)
if os.Getenv("CI") != "" {
path := filepath.Join(os.TempDir(), "wazero")
if err := os.MkdirAll(path, 0777); err == nil {
if cache, err := wazero.NewCompilationCacheWithDir(path); err == nil {
sqlite3.RuntimeConfig.WithCompilationCache(cache)
}
}
}
}

View File

@@ -14,7 +14,7 @@ sudo mount -nv -o loop f2fs.img f2fs/
mkdir -p f2fs/tmp/
go test -c "$ROOT/tests" -coverpkg github.com/ncruces/go-sqlite3/...
TMPDIR=f2fs/tmp/ ./tests.test -test.v -test.short -test.coverprofile cover.out
TMPDIR=$PWD/f2fs/tmp/ ./tests.test -test.v -test.short -test.coverprofile cover.out
go tool cover -html cover.out
sudo umount f2fs/

View File

@@ -10,6 +10,7 @@ import (
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
func TestTimeFormat_Encode(t *testing.T) {
@@ -43,7 +44,8 @@ func TestTimeFormat_Encode(t *testing.T) {
func TestTimeFormat_Decode(t *testing.T) {
t.Parallel()
zone := time.FixedZone("", -4*3600)
const offset = -4 * 3600
zone := time.FixedZone("", offset)
reference := time.Date(2013, 10, 7, 4, 23, 19, 120_000_000, zone)
refnodate := time.Date(2000, 01, 1, 4, 23, 19, 120_000_000, zone)
@@ -52,63 +54,63 @@ func TestTimeFormat_Decode(t *testing.T) {
val any
want time.Time
wantDelta time.Duration
wantLoc *time.Location
wantOff int
wantErr bool
}{
{sqlite3.TimeFormatJulianDay, "2456572.849526851851852", reference, 0, time.UTC, false},
{sqlite3.TimeFormatJulianDay, 2456572.849526851851852, reference, time.Millisecond, time.UTC, false},
{sqlite3.TimeFormatJulianDay, int64(2456572), reference, 24 * time.Hour, time.UTC, false},
{sqlite3.TimeFormatJulianDay, false, time.Time{}, 0, nil, true},
{sqlite3.TimeFormatJulianDay, "2456572.849526851851852", reference, 0, 0, false},
{sqlite3.TimeFormatJulianDay, 2456572.849526851851852, reference, time.Millisecond, 0, false},
{sqlite3.TimeFormatJulianDay, int64(2456572), reference, 24 * time.Hour, 0, false},
{sqlite3.TimeFormatJulianDay, false, time.Time{}, 0, 0, true},
{sqlite3.TimeFormatUnix, "1381134199.120", reference, time.Microsecond, time.UTC, false},
{sqlite3.TimeFormatUnix, 1381134199.120, reference, time.Microsecond, time.UTC, false},
{sqlite3.TimeFormatUnix, int64(1381134199), reference, time.Second, time.UTC, false},
{sqlite3.TimeFormatUnix, "abc", time.Time{}, 0, nil, true},
{sqlite3.TimeFormatUnix, false, time.Time{}, 0, nil, true},
{sqlite3.TimeFormatUnix, "1381134199.120", reference, time.Microsecond, 0, false},
{sqlite3.TimeFormatUnix, 1381134199.120, reference, time.Microsecond, 0, false},
{sqlite3.TimeFormatUnix, int64(1381134199), reference, time.Second, 0, false},
{sqlite3.TimeFormatUnix, "abc", time.Time{}, 0, 0, true},
{sqlite3.TimeFormatUnix, false, time.Time{}, 0, 0, true},
{sqlite3.TimeFormatUnixMilli, "1381134199120", reference, 0, time.UTC, false},
{sqlite3.TimeFormatUnixMilli, 1381134199.120e3, reference, 0, time.UTC, false},
{sqlite3.TimeFormatUnixMilli, int64(1381134199_120), reference, 0, time.UTC, false},
{sqlite3.TimeFormatUnixMilli, "abc", time.Time{}, 0, nil, true},
{sqlite3.TimeFormatUnixMilli, false, time.Time{}, 0, nil, true},
{sqlite3.TimeFormatUnixMilli, "1381134199120", reference, 0, 0, false},
{sqlite3.TimeFormatUnixMilli, 1381134199.120e3, reference, 0, 0, false},
{sqlite3.TimeFormatUnixMilli, int64(1381134199_120), reference, 0, 0, false},
{sqlite3.TimeFormatUnixMilli, "abc", time.Time{}, 0, 0, true},
{sqlite3.TimeFormatUnixMilli, false, time.Time{}, 0, 0, true},
{sqlite3.TimeFormatUnixMicro, "1381134199120000", reference, 0, time.UTC, false},
{sqlite3.TimeFormatUnixMicro, 1381134199.120e6, reference, 0, time.UTC, false},
{sqlite3.TimeFormatUnixMicro, int64(1381134199_120000), reference, 0, time.UTC, false},
{sqlite3.TimeFormatUnixMicro, "abc", time.Time{}, 0, nil, true},
{sqlite3.TimeFormatUnixMicro, false, time.Time{}, 0, nil, true},
{sqlite3.TimeFormatUnixMicro, "1381134199120000", reference, 0, 0, false},
{sqlite3.TimeFormatUnixMicro, 1381134199.120e6, reference, 0, 0, false},
{sqlite3.TimeFormatUnixMicro, int64(1381134199_120000), reference, 0, 0, false},
{sqlite3.TimeFormatUnixMicro, "abc", time.Time{}, 0, 0, true},
{sqlite3.TimeFormatUnixMicro, false, time.Time{}, 0, 0, true},
{sqlite3.TimeFormatUnixNano, "1381134199120000000", reference, 0, time.UTC, false},
{sqlite3.TimeFormatUnixNano, 1381134199.120e9, reference, 0, time.UTC, false},
{sqlite3.TimeFormatUnixNano, int64(1381134199_120000000), reference, 0, time.UTC, false},
{sqlite3.TimeFormatUnixNano, "abc", time.Time{}, 0, nil, true},
{sqlite3.TimeFormatUnixNano, false, time.Time{}, 0, nil, true},
{sqlite3.TimeFormatUnixNano, "1381134199120000000", reference, 0, 0, false},
{sqlite3.TimeFormatUnixNano, 1381134199.120e9, reference, 0, 0, false},
{sqlite3.TimeFormatUnixNano, int64(1381134199_120000000), reference, 0, 0, false},
{sqlite3.TimeFormatUnixNano, "abc", time.Time{}, 0, 0, true},
{sqlite3.TimeFormatUnixNano, false, time.Time{}, 0, 0, true},
{sqlite3.TimeFormatAuto, "2456572.849526851851852", reference, time.Millisecond, time.UTC, false},
{sqlite3.TimeFormatAuto, "2456572", reference, 24 * time.Hour, time.UTC, false},
{sqlite3.TimeFormatAuto, "1381134199.120", reference, time.Microsecond, time.UTC, false},
{sqlite3.TimeFormatAuto, "1381134199.120e3", reference, time.Microsecond, time.UTC, false},
{sqlite3.TimeFormatAuto, "1381134199.120e6", reference, time.Microsecond, time.UTC, false},
{sqlite3.TimeFormatAuto, "1381134199.120e9", reference, time.Microsecond, time.UTC, false},
{sqlite3.TimeFormatAuto, "1381134199", reference, time.Second, time.UTC, false},
{sqlite3.TimeFormatAuto, "1381134199120", reference, 0, time.UTC, false},
{sqlite3.TimeFormatAuto, "1381134199120000", reference, 0, time.UTC, false},
{sqlite3.TimeFormatAuto, "1381134199120000000", reference, 0, time.UTC, false},
{sqlite3.TimeFormatAuto, "2013-10-07 04:23:19.12-04:00", reference, 0, zone, false},
{sqlite3.TimeFormatAuto, "04:23:19.12-04:00", refnodate, 0, zone, false},
{sqlite3.TimeFormatAuto, "abc", time.Time{}, 0, nil, true},
{sqlite3.TimeFormatAuto, false, time.Time{}, 0, nil, true},
{sqlite3.TimeFormatAuto, "2456572.849526851851852", reference, time.Millisecond, 0, false},
{sqlite3.TimeFormatAuto, "2456572", reference, 24 * time.Hour, 0, false},
{sqlite3.TimeFormatAuto, "1381134199.120", reference, time.Microsecond, 0, false},
{sqlite3.TimeFormatAuto, "1381134199.120e3", reference, time.Microsecond, 0, false},
{sqlite3.TimeFormatAuto, "1381134199.120e6", reference, time.Microsecond, 0, false},
{sqlite3.TimeFormatAuto, "1381134199.120e9", reference, time.Microsecond, 0, false},
{sqlite3.TimeFormatAuto, "1381134199", reference, time.Second, 0, false},
{sqlite3.TimeFormatAuto, "1381134199120", reference, 0, 0, false},
{sqlite3.TimeFormatAuto, "1381134199120000", reference, 0, 0, false},
{sqlite3.TimeFormatAuto, "1381134199120000000", reference, 0, 0, false},
{sqlite3.TimeFormatAuto, "2013-10-07 04:23:19.12-04:00", reference, 0, offset, false},
{sqlite3.TimeFormatAuto, "04:23:19.12-04:00", refnodate, 0, offset, false},
{sqlite3.TimeFormatAuto, "abc", time.Time{}, 0, 0, true},
{sqlite3.TimeFormatAuto, false, time.Time{}, 0, 0, true},
{sqlite3.TimeFormat3, "2013-10-07 04:23:19.12-04:00", reference, 0, zone, false},
{sqlite3.TimeFormat3, "2013-10-07 08:23:19.12", reference, 0, time.UTC, false},
{sqlite3.TimeFormat9, "04:23:19.12-04:00", refnodate, 0, zone, false},
{sqlite3.TimeFormat9, "08:23:19.12", refnodate, 0, time.UTC, false},
{sqlite3.TimeFormat3, false, time.Time{}, 0, nil, true},
{sqlite3.TimeFormat9, false, time.Time{}, 0, nil, true},
{sqlite3.TimeFormat3, "2013-10-07 04:23:19.12-04:00", reference, 0, offset, false},
{sqlite3.TimeFormat3, "2013-10-07 08:23:19.12", reference, 0, 0, false},
{sqlite3.TimeFormat9, "04:23:19.12-04:00", refnodate, 0, offset, false},
{sqlite3.TimeFormat9, "08:23:19.12", refnodate, 0, 0, false},
{sqlite3.TimeFormat3, false, time.Time{}, 0, 0, true},
{sqlite3.TimeFormat9, false, time.Time{}, 0, 0, true},
{sqlite3.TimeFormatDefault, "2013-10-07T04:23:19.12-04:00", reference, 0, zone, false},
{sqlite3.TimeFormatDefault, "2013-10-07T08:23:19.12Z", reference, 0, time.UTC, false},
{sqlite3.TimeFormatDefault, false, time.Time{}, 0, nil, true},
{sqlite3.TimeFormatDefault, "2013-10-07T04:23:19.12-04:00", reference, 0, offset, false},
{sqlite3.TimeFormatDefault, "2013-10-07T08:23:19.12Z", reference, 0, 0, false},
{sqlite3.TimeFormatDefault, false, time.Time{}, 0, 0, true},
}
for _, tt := range tests {
@@ -121,8 +123,8 @@ func TestTimeFormat_Decode(t *testing.T) {
if got.Sub(tt.want).Abs() > tt.wantDelta {
t.Errorf("%q.Decode(%v) = %v, want %v", tt.fmt, tt.val, got, tt.want)
}
if got.Location().String() != tt.wantLoc.String() {
t.Errorf("%q.Decode(%v) = %v, want %v", tt.fmt, tt.val, got.Location(), tt.wantLoc)
if _, off := got.Zone(); off != tt.wantOff {
t.Errorf("%q.Decode(%v) = %v, want %v", tt.fmt, tt.val, off, tt.wantOff)
}
})
}

View File

@@ -7,6 +7,8 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/vfs/memdb"
)
func TestConn_Transaction_exec(t *testing.T) {
@@ -246,6 +248,51 @@ func TestConn_Transaction_interrupted(t *testing.T) {
}
}
func TestConn_Transaction_busy(t *testing.T) {
t.Parallel()
db1, err := sqlite3.Open("file:/test.db?vfs=memdb")
if err != nil {
t.Fatal(err)
}
defer db1.Close()
db2, err := sqlite3.Open("file:/test.db?vfs=memdb&_pragma=busy_timeout(10000)")
if err != nil {
t.Fatal(err)
}
defer db2.Close()
err = db1.Exec(`CREATE TABLE test (col)`)
if err != nil {
t.Fatal(err)
}
tx, err := db1.BeginImmediate()
if err != nil {
t.Fatal(err)
}
err = db1.Exec(`INSERT INTO test VALUES (1)`)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
db2.SetInterrupt(ctx)
go cancel()
_, err = db2.BeginExclusive()
if !errors.Is(err, sqlite3.BUSY) && !errors.Is(err, sqlite3.INTERRUPT) {
t.Errorf("got %v, want sqlite3.BUSY or sqlite3.INTERRUPT", err)
}
err = nil
tx.End(&err)
if err != nil {
t.Fatal(err)
}
}
func TestConn_Transaction_rollback(t *testing.T) {
t.Parallel()

View File

@@ -6,6 +6,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
"github.com/ncruces/go-sqlite3/vfs/memdb"
"github.com/ncruces/go-sqlite3/vfs/readervfs"
)

View File

@@ -6,10 +6,15 @@ import (
"testing"
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
"github.com/ncruces/go-sqlite3/vfs"
)
func TestWAL_enter_exit(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
t.Parallel()
file := filepath.Join(t.TempDir(), "test.db")
@@ -20,13 +25,20 @@ func TestWAL_enter_exit(t *testing.T) {
}
defer db.Close()
if !vfs.SupportsSharedMemory {
err = db.Exec(`PRAGMA locking_mode=exclusive`)
if err != nil {
t.Fatal(err)
}
}
err = db.Exec(`
CREATE TABLE test (col);
PRAGMA journal_mode=WAL;
PRAGMA journal_mode=wal;
SELECT * FROM test;
PRAGMA journal_mode=DELETE;
PRAGMA journal_mode=delete;
SELECT * FROM test;
PRAGMA journal_mode=WAL;
PRAGMA journal_mode=wal;
SELECT * FROM test;
`)
if err != nil {
@@ -38,11 +50,10 @@ func TestWAL_readonly(t *testing.T) {
if !vfs.SupportsSharedMemory {
t.Skip("skipping without shared memory")
}
t.Parallel()
tmp := filepath.Join(t.TempDir(), "test.db")
err := os.WriteFile(tmp, waldb, 0666)
err := os.WriteFile(tmp, walDB, 0666)
if err != nil {
t.Fatal(err)
}
@@ -65,6 +76,9 @@ func TestWAL_readonly(t *testing.T) {
}
func TestConn_WalCheckpoint(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
t.Parallel()
file := filepath.Join(t.TempDir(), "test.db")
@@ -87,7 +101,8 @@ func TestConn_WalCheckpoint(t *testing.T) {
})
err = db.Exec(`
PRAGMA journal_mode=WAL;
PRAGMA locking_mode=exlusive;
PRAGMA journal_mode=wal;
CREATE TABLE test (col);
`)
if err != nil {

12
time.go
View File

@@ -101,7 +101,7 @@ func (f TimeFormat) Encode(t time.Time) any {
return t.UnixMicro()
case TimeFormatUnixNano:
return t.UnixNano()
// Special formats
// Special formats.
case TimeFormatDefault, TimeFormatAuto:
f = time.RFC3339Nano
// SQLite assumes UTC if unspecified.
@@ -139,7 +139,7 @@ func (f TimeFormat) Encode(t time.Time) any {
// https://sqlite.org/lang_datefunc.html
func (f TimeFormat) Decode(v any) (time.Time, error) {
switch f {
// Numeric formats
// Numeric formats.
case TimeFormatJulianDay:
switch v := v.(type) {
case string:
@@ -183,7 +183,7 @@ func (f TimeFormat) Decode(v any) (time.Time, error) {
case float64:
return time.UnixMilli(int64(math.Floor(v))).UTC(), nil
case int64:
return time.UnixMilli(int64(v)).UTC(), nil
return time.UnixMilli(v).UTC(), nil
default:
return time.Time{}, util.TimeErr
}
@@ -200,7 +200,7 @@ func (f TimeFormat) Decode(v any) (time.Time, error) {
case float64:
return time.UnixMicro(int64(math.Floor(v))).UTC(), nil
case int64:
return time.UnixMicro(int64(v)).UTC(), nil
return time.UnixMicro(v).UTC(), nil
default:
return time.Time{}, util.TimeErr
}
@@ -217,12 +217,12 @@ func (f TimeFormat) Decode(v any) (time.Time, error) {
case float64:
return time.Unix(0, int64(math.Floor(v))).UTC(), nil
case int64:
return time.Unix(0, int64(v)).UTC(), nil
return time.Unix(0, v).UTC(), nil
default:
return time.Time{}, util.TimeErr
}
// Special formats
// Special formats.
case TimeFormatAuto:
switch s := v.(type) {
case string:

View File

@@ -8,6 +8,7 @@ import (
// SeekingReaderAt implements [io.ReaderAt]
// through an underlying [io.ReadSeeker].
type SeekingReaderAt struct {
// +checklocks:l
r io.ReadSeeker
l sync.Mutex
}

View File

@@ -2,79 +2,85 @@
This package implements the SQLite [OS Interface](https://sqlite.org/vfs.html) (aka VFS).
It replaces the default SQLite VFS with a **pure Go** implementation.
It also exposes [interfaces](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs#VFS)
It replaces the default SQLite VFS with a **pure Go** implementation,
and exposes [interfaces](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs#VFS)
that should allow you to implement your own custom VFSes.
Since it is a from scratch reimplementation,
there are naturally some ways it deviates from the original.
The main differences are [file locking](#file-locking) and [WAL mode](write-ahead-logging) support.
The main differences are [file locking](#file-locking) and [WAL mode](#write-ahead-logging) support.
### File Locking
POSIX advisory locks, which SQLite uses on Unix, are
[broken by design](https://sqlite.org/src/artifact/2e8b12?ln=1073-1161).
[broken by design](https://github.com/sqlite/sqlite/blob/b74eb0/src/os_unix.c#L1073-L1161).
On Linux, macOS and illumos, this module uses
On Linux and macOS, this module uses
[OFD locks](https://www.gnu.org/software/libc/manual/html_node/Open-File-Description-Locks.html)
to synchronize access to database files.
OFD locks are fully compatible with POSIX advisory locks.
On BSD Unixes, this module uses
[BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2).
On BSD Unixes, BSD locks are fully compatible with POSIX advisory locks.
However, concurrency is reduced with BSD locks
(`BEGIN IMMEDIATE` behaves the same as `BEGIN EXCLUSIVE`).
This module can also use
[BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2),
albeit with reduced concurrency (`BEGIN IMMEDIATE` behaves like `BEGIN EXCLUSIVE`).
On BSD, macOS, and illumos, BSD locks are fully compatible with POSIX advisory locks;
on Linux and z/OS, they are fully functional, but incompatible;
elsewhere, they are very likely broken.
BSD locks are the default on BSD and illumos,
but you can opt into them with the `sqlite3_flock` build tag.
On Windows, this module uses `LockFileEx` and `UnlockFileEx`,
like SQLite.
On all other platforms, file locking is not supported, and you must use
Otherwise, file locking is not supported, and you must use
[`nolock=1`](https://sqlite.org/uri.html#urinolock)
(or [`immutable=1`](https://sqlite.org/uri.html#uriimmutable))
to open database files.
To use the [`database/sql`](https://pkg.go.dev/database/sql) driver
with `nolock=1` you must disable connection pooling by calling
[`db.SetMaxOpenConns(1)`](https://pkg.go.dev/database/sql#DB.SetMaxOpenConns).
You can use [`vfs.SupportsFileLocking`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs#SupportsFileLocking)
to check if your platform supports file locking.
to check if your build supports file locking.
### Write-Ahead Logging
On 64-bit Linux, macOS and illumos, this module uses `mmap` to implement
On 64-bit Linux and macOS, this module uses `mmap` to implement
[shared-memory for the WAL-index](https://sqlite.org/wal.html#implementation_of_shared_memory_for_the_wal_index),
like SQLite.
To allow `mmap` to work, each connection needs to reserve up to 4GB of address space.\
To limit the amount of address space each connection needs,
use [`WithMemoryLimitPages`](../tests/parallel/parallel_test.go#L21).
To allow `mmap` to work, each connection needs to reserve up to 4GB of address space.
To limit the address space each connection reserves,
use [`WithMemoryLimitPages`](../tests/testcfg/testcfg.go).
On all other platforms, [WAL](https://sqlite.org/wal.html) support is
[limited](https://sqlite.org/wal.html#noshm).
To work around this limitation, SQLite is [patched](sqlite3/locking_mode.patch)
to automatically use `EXCLUSIVE` locking mode for WAL databases on such platforms.
To use the [`database/sql`](https://pkg.go.dev/database/sql) driver
with `EXCLUSIVE` locking mode you should disable connection pooling by calling
Otherwise, [WAL support is limited](https://sqlite.org/wal.html#noshm),
and `EXCLUSIVE` locking mode must be set to create, read, and write WAL databases.
To use `EXCLUSIVE` locking mode with the
[`database/sql`](https://pkg.go.dev/database/sql) driver
you must disable connection pooling by calling
[`db.SetMaxOpenConns(1)`](https://pkg.go.dev/database/sql#DB.SetMaxOpenConns).
You can use [`vfs.SupportsSharedMemory`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs#SupportsSharedMemory)
to check if your platform supports shared memory.
to check if your build supports shared memory.
### Batch-Atomic Write
On 64-bit Linux, this module supports [batch-atomic writes](https://sqlite.org/cgi/src/technote/714)
on the F2FS filesystem.
### Build tags
### Build Tags
The VFS can be customized with a few build tags:
- `sqlite3_flock` forces the use of BSD locks; it can be used on macOS to test the BSD locking implementation.
- `sqlite3_flock` forces the use of BSD locks; it can be used on z/OS to enable locking,
and elsewhere to test BSD locks.
- `sqlite3_nosys` prevents importing [`x/sys`](https://pkg.go.dev/golang.org/x/sys);
disables locking _and_ shared memory on all platforms.
- `sqlite3_noshm` disables shared memory on all platforms.
- `sqlite3_noshm` disables shared memory on all platforms.
> [!IMPORTANT]
> The default configuration of this package is compatible with
> the standard [Unix and Windows SQLite VFSes](https://sqlite.org/vfs.html#multiple_vfses);
> `sqlite3_flock` is compatible with the [`unix-flock` VFS](https://sqlite.org/compile.html#enable_locking_style).
> If incompatible file locking is used, accessing databases concurrently with _other_ SQLite libraries
> will eventually corrupt data.

64
vfs/adiantum/README.md Normal file
View File

@@ -0,0 +1,64 @@
# Go `"adiantum"` SQLite VFS
This package wraps an SQLite VFS to offer encryption at rest.
> [!WARNING]
> This work was not certified by a cryptographer.
> If you need vetted encryption, you should purchase the
> [SQLite Encryption Extension](https://sqlite.org/see),
> and either wrap it, or seek assistance wrapping it.
The `"adiantum"` VFS wraps the default SQLite VFS using the
[Adiantum](https://github.com/lukechampine/adiantum)
tweakable and length-preserving encryption.\
In general, any HBSH construction can be used to wrap any VFS.
The default Adiantum construction uses XChaCha12 for its stream cipher,
AES for its block cipher, and NH and Poly1305 for hashing.\
Additionally, we use [Argon2id](https://pkg.go.dev/golang.org/x/crypto/argon2#hdr-Argon2id)
to derive 256-bit keys from plain text where needed.
File contents are encrypted in 4K blocks, matching the
[default](https://sqlite.org/pgszchng2016.html) SQLite page size.
The VFS encrypts all files _except_
[super journals](https://sqlite.org/tempfiles.html#super_journal_files):
these _never_ contain database data, only filenames,
and padding them to the block size is problematic.
Temporary files _are_ encrypted with **random** keys,
as they _may_ contain database data.
To avoid the overhead of encrypting temporary files,
keep them in memory:
PRAGMA temp_store = memory;
> [!IMPORTANT]
> Adiantum is a cipher composition for disk encryption.
> The standard threat model for disk encryption considers an adversary
> that can read multiple snapshots of a disk.
> The only security property that disk encryption provides
> is that all information such an adversary can obtain
> is whether the data in a sector has or has not changed over time.
The encryption offered by this package is fully deterministic.
This means that an adversary who can get ahold of multiple snapshots
(e.g. backups) of a database file can learn precisely:
which blocks changed, which ones didn't, which got reverted.
This is slightly weaker than other forms of SQLite encryption
that include *some* nondeterminism; with limited nondeterminism,
an adversary can't distinguish between
blocks that actually changed, and blocks that got reverted.
> [!CAUTION]
> This package does not claim protect databases against tampering or forgery.
The major practical consequence of the above point is that,
if you're keeping `"adiantum"` encrypted backups of your database,
and want to protect against forgery, you should sign your backups,
and verify signatures before restoring them.
This is slightly weaker than other forms of SQLite encryption
that include block-level [MACs](https://en.wikipedia.org/wiki/Message_authentication_code).
Block-level MACs can protect against forging individual blocks,
but can't prevent them from being reverted to former versions of themselves.

32
vfs/adiantum/adiantum.go Normal file
View File

@@ -0,0 +1,32 @@
package adiantum
import (
"crypto/rand"
"golang.org/x/crypto/argon2"
"lukechampine.com/adiantum"
"lukechampine.com/adiantum/hbsh"
)
// This variable can be replaced with -ldflags:
//
// go build -ldflags="-X github.com/ncruces/go-sqlite3/vfs/adiantum.pepper=adiantum"
var pepper = "github.com/ncruces/go-sqlite3/vfs/adiantum"
type adiantumCreator struct{}
func (adiantumCreator) HBSH(key []byte) *hbsh.HBSH {
if len(key) != 32 {
return nil
}
return adiantum.New(key)
}
func (adiantumCreator) KDF(text string) []byte {
if text == "" {
key := make([]byte, 32)
n, _ := rand.Read(key)
return key[:n]
}
return argon2.IDKey([]byte(text), []byte(pepper), 3, 64*1024, 4, 32)
}

View File

@@ -0,0 +1,88 @@
package adiantum_test
import (
_ "embed"
"path/filepath"
"strings"
"testing"
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
"github.com/ncruces/go-sqlite3/util/ioutil"
"github.com/ncruces/go-sqlite3/vfs"
"github.com/ncruces/go-sqlite3/vfs/adiantum"
"github.com/ncruces/go-sqlite3/vfs/readervfs"
)
//go:embed testdata/test.db
var testDB string
func Test_fileformat(t *testing.T) {
readervfs.Create("test.db", ioutil.NewSizeReaderAt(strings.NewReader(testDB)))
adiantum.Register("radiantum", vfs.Find("reader"), nil)
db, err := driver.Open("file:test.db?vfs=radiantum", nil)
if err != nil {
t.Fatal(err)
}
defer db.Close()
_, err = db.Exec(`PRAGMA textkey='correct+horse+battery+staple'`)
if err != nil {
t.Fatal(err)
}
var version uint32
err = db.QueryRow(`PRAGMA user_version`).Scan(&version)
if err != nil {
t.Fatal(err)
}
if version != 0xBADDB {
t.Error(version)
}
}
func Benchmark_nokey(b *testing.B) {
tmp := filepath.Join(b.TempDir(), "test.db")
sqlite3.Initialize()
b.ResetTimer()
for n := 0; n < b.N; n++ {
db, err := sqlite3.Open("file:" + filepath.ToSlash(tmp) + "?nolock=1")
if err != nil {
b.Fatal(err)
}
db.Close()
}
}
func Benchmark_hexkey(b *testing.B) {
tmp := filepath.Join(b.TempDir(), "test.db")
sqlite3.Initialize()
b.ResetTimer()
for n := 0; n < b.N; n++ {
db, err := sqlite3.Open("file:" + filepath.ToSlash(tmp) + "?nolock=1" +
"&vfs=adiantum&hexkey=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
if err != nil {
b.Fatal(err)
}
db.Close()
}
}
func Benchmark_textkey(b *testing.B) {
tmp := filepath.Join(b.TempDir(), "test.db")
sqlite3.Initialize()
b.ResetTimer()
for n := 0; n < b.N; n++ {
db, err := sqlite3.Open("file:" + filepath.ToSlash(tmp) + "?nolock=1" +
"&vfs=adiantum&textkey=correct+horse+battery+staple")
if err != nil {
b.Fatal(err)
}
db.Close()
}
}

68
vfs/adiantum/api.go Normal file
View File

@@ -0,0 +1,68 @@
// Package adiantum wraps an SQLite VFS to offer encryption at rest.
//
// The "adiantum" [vfs.VFS] wraps the default VFS using the
// Adiantum tweakable, length-preserving encryption.
//
// Importing package adiantum registers that VFS:
//
// import _ "github.com/ncruces/go-sqlite3/vfs/adiantum"
//
// To open an encrypted database you need to provide key material.
//
// The simplest way to do that is to specify the key through an [URI] parameter:
//
// - key: key material in binary (32 bytes)
// - hexkey: key material in hex (64 hex digits)
// - textkey: key material in text (any length)
//
// However, this makes your key easily accessible to other parts of
// your application (e.g. through [vfs.Filename.URIParameters]).
//
// To avoid this, invoke any of the following PRAGMAs
// immediately after opening a connection:
//
// PRAGMA key='D41d8cD98f00b204e9800998eCf8427e';
// PRAGMA hexkey='e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855';
// PRAGMA textkey='your-secret-key';
//
// For an ATTACH-ed database, you must specify the schema name:
//
// ATTACH DATABASE 'demo.db' AS demo;
// PRAGMA demo.textkey='your-secret-key';
//
// [URI]: https://sqlite.org/uri.html
package adiantum
import (
"github.com/ncruces/go-sqlite3/vfs"
"lukechampine.com/adiantum/hbsh"
)
func init() {
Register("adiantum", vfs.Find(""), nil)
}
// Register registers an encrypting VFS, wrapping a base VFS,
// and possibly using a custom HBSH cipher construction.
// To use the default Adiantum construction, set cipher to nil.
func Register(name string, base vfs.VFS, cipher HBSHCreator) {
if cipher == nil {
cipher = adiantumCreator{}
}
vfs.Register(name, &hbshVFS{
VFS: base,
hbsh: cipher,
})
}
// HBSHCreator creates an [hbsh.HBSH] cipher
// given key material.
type HBSHCreator interface {
// KDF derives an HBSH key from a secret.
// If no secret is given, a random key is generated.
KDF(secret string) (key []byte)
// HBSH creates an HBSH cipher given a key.
// If key is not appropriate, nil is returned.
HBSH(key []byte) *hbsh.HBSH
}

View File

@@ -0,0 +1,52 @@
//go:build (linux || darwin || windows || freebsd || illumos) && !sqlite3_nosys
package adiantum_test
import (
"crypto/rand"
"log"
"os"
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/vfs"
"github.com/ncruces/go-sqlite3/vfs/adiantum"
"golang.org/x/crypto/argon2"
"lukechampine.com/adiantum/hbsh"
"lukechampine.com/adiantum/hpolyc"
)
func ExampleRegister_hpolyc() {
adiantum.Register("hpolyc", vfs.Find(""), hpolycCreator{})
db, err := sqlite3.Open("file:demo.db?vfs=hpolyc" +
"&textkey=correct+horse+battery+staple")
if err != nil {
log.Fatal(err)
}
defer os.Remove("./demo.db")
defer db.Close()
// Output:
}
type hpolycCreator struct{}
// HBSH creates an HBSH cipher given a key.
func (hpolycCreator) HBSH(key []byte) *hbsh.HBSH {
if len(key) != 32 {
// Key is not appropriate, return nil.
return nil
}
return hpolyc.New(key)
}
// KDF gets a key from a secret.
func (hpolycCreator) KDF(secret string) []byte {
if secret == "" {
// No secret is given, generate a random key.
key := make([]byte, 32)
n, _ := rand.Read(key)
return key[:n]
}
// Hash the secret with a KDF.
return argon2.IDKey([]byte(secret), []byte("hpolyc"), 3, 64*1024, 4, 32)
}

278
vfs/adiantum/hbsh.go Normal file
View File

@@ -0,0 +1,278 @@
package adiantum
import (
"encoding/binary"
"encoding/hex"
"io"
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/vfs"
"lukechampine.com/adiantum/hbsh"
)
type hbshVFS struct {
vfs.VFS
hbsh HBSHCreator
}
func (h *hbshVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
return nil, 0, sqlite3.CANTOPEN
}
func (h *hbshVFS) OpenFilename(name *vfs.Filename, flags vfs.OpenFlag) (file vfs.File, _ vfs.OpenFlag, err error) {
if hf, ok := h.VFS.(vfs.VFSFilename); ok {
file, flags, err = hf.OpenFilename(name, flags)
} else {
file, flags, err = h.VFS.Open(name.String(), flags)
}
// Encrypt everything except super journals and memory files.
if err != nil || flags&(vfs.OPEN_SUPER_JOURNAL|vfs.OPEN_MEMORY) != 0 {
return file, flags, err
}
var hbsh *hbsh.HBSH
if f, ok := name.DatabaseFile().(*hbshFile); ok {
hbsh = f.hbsh
} else {
var key []byte
if params := name.URIParameters(); name == nil {
key = h.hbsh.KDF("") // Temporary files get a random key.
} else if t, ok := params["key"]; ok {
key = []byte(t[0])
} else if t, ok := params["hexkey"]; ok {
key, _ = hex.DecodeString(t[0])
} else if t, ok := params["textkey"]; ok {
key = h.hbsh.KDF(t[0])
} else if flags&vfs.OPEN_MAIN_DB != 0 {
// Main datatabases may have their key specified as a PRAGMA.
return &hbshFile{File: file, reset: h.hbsh}, flags, nil
}
hbsh = h.hbsh.HBSH(key)
}
if hbsh == nil {
return nil, flags, sqlite3.CANTOPEN
}
return &hbshFile{File: file, hbsh: hbsh, reset: h.hbsh}, flags, nil
}
const (
tweakSize = 8
blockSize = 4096
)
type hbshFile struct {
vfs.File
hbsh *hbsh.HBSH
reset HBSHCreator
tweak [tweakSize]byte
block [blockSize]byte
}
func (h *hbshFile) Pragma(name string, value string) (string, error) {
var key []byte
switch name {
case "key":
key = []byte(value)
case "hexkey":
key, _ = hex.DecodeString(value)
case "textkey":
key = h.reset.KDF(value)
default:
if f, ok := h.File.(vfs.FilePragma); ok {
return f.Pragma(name, value)
}
return "", sqlite3.NOTFOUND
}
if h.hbsh = h.reset.HBSH(key); h.hbsh != nil {
return "ok", nil
}
return "", sqlite3.CANTOPEN
}
func (h *hbshFile) ReadAt(p []byte, off int64) (n int, err error) {
if h.hbsh == nil {
// Only OPEN_MAIN_DB can have a missing key.
if off == 0 && len(p) == 100 {
// SQLite is trying to read the header of a database file.
// Pretend the file is empty so the key may specified as a PRAGMA.
return 0, io.EOF
}
return 0, sqlite3.CANTOPEN
}
min := (off) &^ (blockSize - 1) // round down
max := (off + int64(len(p)) + (blockSize - 1)) &^ (blockSize - 1) // round up
// Read one block at a time.
for ; min < max; min += blockSize {
m, err := h.File.ReadAt(h.block[:], min)
if m != blockSize {
return n, err
}
binary.LittleEndian.PutUint64(h.tweak[:], uint64(min))
data := h.hbsh.Decrypt(h.block[:], h.tweak[:])
if off > min {
data = data[off-min:]
}
n += copy(p[n:], data)
}
if n != len(p) {
panic(util.AssertErr())
}
return n, nil
}
func (h *hbshFile) WriteAt(p []byte, off int64) (n int, err error) {
if h.hbsh == nil {
return 0, sqlite3.READONLY
}
min := (off) &^ (blockSize - 1) // round down
max := (off + int64(len(p)) + (blockSize - 1)) &^ (blockSize - 1) // round up
// Write one block at a time.
for ; min < max; min += blockSize {
binary.LittleEndian.PutUint64(h.tweak[:], uint64(min))
data := h.block[:]
if off > min || len(p[n:]) < blockSize {
// Partial block write: read-update-write.
m, err := h.File.ReadAt(h.block[:], min)
if m != blockSize {
if err != io.EOF {
return n, err
}
// Writing past the EOF.
// We're either appending an entirely new block,
// or the final block was only partially written.
// A partially written block can't be decrypted,
// and is as good as corrupt.
// Either way, zero pad the file to the next block size.
clear(data)
} else {
data = h.hbsh.Decrypt(h.block[:], h.tweak[:])
}
if off > min {
data = data[off-min:]
}
}
t := copy(data, p[n:])
h.hbsh.Encrypt(h.block[:], h.tweak[:])
m, err := h.File.WriteAt(h.block[:], min)
if m != blockSize {
return n, err
}
n += t
}
if n != len(p) {
panic(util.AssertErr())
}
return n, nil
}
func (h *hbshFile) Truncate(size int64) error {
size = (size + (blockSize - 1)) &^ (blockSize - 1) // round up
return h.File.Truncate(size)
}
func (h *hbshFile) SectorSize() int {
return lcm(h.File.SectorSize(), blockSize)
}
func (h *hbshFile) DeviceCharacteristics() vfs.DeviceCharacteristic {
return h.File.DeviceCharacteristics() & (0 |
// The only safe flags are these:
vfs.IOCAP_UNDELETABLE_WHEN_OPEN |
vfs.IOCAP_IMMUTABLE |
vfs.IOCAP_BATCH_ATOMIC)
}
// Wrap optional methods.
func (h *hbshFile) SharedMemory() vfs.SharedMemory {
if f, ok := h.File.(vfs.FileSharedMemory); ok {
return f.SharedMemory()
}
return nil
}
func (h *hbshFile) ChunkSize(size int) {
if f, ok := h.File.(vfs.FileChunkSize); ok {
size = (size + (blockSize - 1)) &^ (blockSize - 1) // round up
f.ChunkSize(size)
}
}
func (h *hbshFile) SizeHint(size int64) error {
if f, ok := h.File.(vfs.FileSizeHint); ok {
size = (size + (blockSize - 1)) &^ (blockSize - 1) // round up
return f.SizeHint(size)
}
return sqlite3.NOTFOUND
}
func (h *hbshFile) HasMoved() (bool, error) {
if f, ok := h.File.(vfs.FileHasMoved); ok {
return f.HasMoved()
}
return false, sqlite3.NOTFOUND
}
func (h *hbshFile) Overwrite() error {
if f, ok := h.File.(vfs.FileOverwrite); ok {
return f.Overwrite()
}
return sqlite3.NOTFOUND
}
func (h *hbshFile) CommitPhaseTwo() error {
if f, ok := h.File.(vfs.FileCommitPhaseTwo); ok {
return f.CommitPhaseTwo()
}
return sqlite3.NOTFOUND
}
func (h *hbshFile) BeginAtomicWrite() error {
if f, ok := h.File.(vfs.FileBatchAtomicWrite); ok {
return f.BeginAtomicWrite()
}
return sqlite3.NOTFOUND
}
func (h *hbshFile) CommitAtomicWrite() error {
if f, ok := h.File.(vfs.FileBatchAtomicWrite); ok {
return f.CommitAtomicWrite()
}
return sqlite3.NOTFOUND
}
func (h *hbshFile) RollbackAtomicWrite() error {
if f, ok := h.File.(vfs.FileBatchAtomicWrite); ok {
return f.RollbackAtomicWrite()
}
return sqlite3.NOTFOUND
}
func (h *hbshFile) CheckpointDone() error {
if f, ok := h.File.(vfs.FileCheckpoint); ok {
return f.CheckpointDone()
}
return sqlite3.NOTFOUND
}
func (h *hbshFile) CheckpointStart() error {
if f, ok := h.File.(vfs.FileCheckpoint); ok {
return f.CheckpointStart()
}
return sqlite3.NOTFOUND
}

22
vfs/adiantum/math.go Normal file
View File

@@ -0,0 +1,22 @@
package adiantum
func abs(n int) int {
if n < 0 {
return -n
}
return n
}
func gcd(m, n int) int {
for n != 0 {
m, n = n, m%n
}
return abs(m)
}
func lcm(m, n int) int {
if n == 0 {
return 0
}
return abs(n) * (abs(m) / gcd(m, n))
}

82
vfs/adiantum/math_test.go Normal file
View File

@@ -0,0 +1,82 @@
package adiantum
import (
"math"
"testing"
)
func Test_abs(t *testing.T) {
tests := []struct {
arg int
want int
}{
{0, 0},
{1, 1},
{-1, 1},
{math.MaxInt, math.MaxInt},
{math.MinInt, math.MinInt},
}
for _, tt := range tests {
t.Run("", func(t *testing.T) {
if got := abs(tt.arg); got != tt.want {
t.Errorf("abs(%d) = %d, want %d", tt.arg, got, tt.want)
}
})
}
}
func Test_gcd(t *testing.T) {
tests := []struct {
arg1 int
arg2 int
want int
}{
{0, 0, 0},
{0, 1, 1},
{1, 0, 1},
{1, 1, 1},
{2, 3, 1},
{42, 56, 14},
{48, -18, 6},
{1e9, 1e9, 1e9},
{1e9, -1e9, 1e9},
{-1e9, -1e9, 1e9},
{math.MaxInt, math.MaxInt, math.MaxInt},
{math.MinInt, math.MinInt, math.MinInt},
}
for _, tt := range tests {
t.Run("", func(t *testing.T) {
if got := gcd(tt.arg1, tt.arg2); got != tt.want {
t.Errorf("gcd(%d, %d) = %d, want %d", tt.arg1, tt.arg2, got, tt.want)
}
})
}
}
func Test_lcm(t *testing.T) {
tests := []struct {
arg1 int
arg2 int
want int
}{
{0, 0, 0},
{0, 1, 0},
{1, 0, 0},
{1, 1, 1},
{2, 3, 6},
{42, 56, 168},
{48, -18, 144},
{1e9, 1e9, 1e9},
{1e9, -1e9, 1e9},
{-1e9, -1e9, 1e9},
{math.MaxInt, math.MaxInt, math.MaxInt},
{math.MinInt, math.MinInt, math.MinInt},
}
for _, tt := range tests {
t.Run("", func(t *testing.T) {
if got := lcm(tt.arg1, tt.arg2); got != tt.want {
t.Errorf("lcm(%d, %d) = %d, want %d", tt.arg1, tt.arg2, got, tt.want)
}
})
}
}

BIN
vfs/adiantum/testdata/test.db vendored Normal file

Binary file not shown.

View File

@@ -1,7 +1,12 @@
// Package vfs wraps the C SQLite VFS API.
package vfs
import "net/url"
import (
"context"
"io"
"github.com/tetratelabs/wazero/api"
)
// A VFS defines the interface between the SQLite core and the underlying operating system.
//
@@ -15,13 +20,13 @@ type VFS interface {
FullPathname(name string) (string, error)
}
// VFSParams extends VFS with the ability to handle URI parameters
// through the OpenParams method.
// VFSFilename extends VFS with the ability to use Filename
// objects for opening files.
//
// https://sqlite.org/c3ref/uri_boolean.html
type VFSParams interface {
// https://sqlite.org/c3ref/filename.html
type VFSFilename interface {
VFS
OpenParams(name string, flags OpenFlag, params url.Values) (File, OpenFlag, error)
OpenFilename(name *Filename, flags OpenFlag) (File, OpenFlag, error)
}
// A File represents an open file in the OS interface layer.
@@ -53,6 +58,15 @@ type FileLockState interface {
LockState() LockLevel
}
// FileChunkSize extends File to implement the
// SQLITE_FCNTL_CHUNK_SIZE file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlchunksize
type FileChunkSize interface {
File
ChunkSize(size int)
}
// FileSizeHint extends File to implement the
// SQLITE_FCNTL_SIZE_HINT file control opcode.
//
@@ -120,3 +134,42 @@ type FileBatchAtomicWrite interface {
CommitAtomicWrite() error
RollbackAtomicWrite() error
}
// FilePragma extends File to implement the
// SQLITE_FCNTL_PRAGMA file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpragma
type FilePragma interface {
File
Pragma(name, value string) (string, error)
}
// FileCheckpoint extends File to implement the
// SQLITE_FCNTL_CKPT_START and SQLITE_FCNTL_CKPT_DONE
// file control opcodes.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlckptstart
type FileCheckpoint interface {
File
CheckpointDone() error
CheckpointStart() error
}
// FileSharedMemory extends File to possibly implement
// shared-memory for the WAL-index.
// The same shared-memory instance must be returned
// for the entire life of the file.
// It's OK for SharedMemory to return nil.
type FileSharedMemory interface {
File
SharedMemory() SharedMemory
}
// SharedMemory is a shared-memory WAL-index implementation.
// Use [NewSharedMemory] to create a shared-memory.
type SharedMemory interface {
shmMap(context.Context, api.Module, int32, int32, bool) (uint32, error)
shmLock(int32, int32, _ShmFlag) error
shmUnmap(bool)
io.Closer
}

View File

@@ -4,8 +4,11 @@ import "github.com/ncruces/go-sqlite3/internal/util"
const (
_MAX_NAME = 1e6 // Self-imposed limit for most NUL terminated strings.
_MAX_SQL_LENGTH = 1e9
_MAX_PATHNAME = 1024
_DEFAULT_SECTOR_SIZE = 4096
ptrlen = 4
)
// https://sqlite.org/rescode.html
@@ -17,6 +20,7 @@ func (e _ErrorCode) Error() string {
const (
_OK _ErrorCode = util.OK
_ERROR _ErrorCode = util.ERROR
_PERM _ErrorCode = util.PERM
_BUSY _ErrorCode = util.BUSY
_READONLY _ErrorCode = util.READONLY

View File

@@ -1,18 +1,15 @@
package vfs
import (
"context"
"errors"
"io"
"io/fs"
"net/url"
"os"
"path/filepath"
"runtime"
"syscall"
"github.com/ncruces/go-sqlite3/util/osutil"
"github.com/tetratelabs/wazero/api"
)
type vfsOS struct{}
@@ -72,10 +69,10 @@ func (vfsOS) Access(name string, flags AccessFlag) (bool, error) {
}
func (vfsOS) Open(name string, flags OpenFlag) (File, OpenFlag, error) {
return vfsOS{}.OpenParams(name, flags, nil)
return nil, 0, _CANTOPEN
}
func (vfsOS) OpenParams(name string, flags OpenFlag, params url.Values) (File, OpenFlag, error) {
func (vfsOS) OpenFilename(name *Filename, flags OpenFlag) (File, OpenFlag, error) {
var oflags int
if flags&OPEN_EXCLUSIVE != 0 {
oflags |= os.O_EXCL
@@ -92,10 +89,10 @@ func (vfsOS) OpenParams(name string, flags OpenFlag, params url.Values) (File, O
var err error
var f *os.File
if name == "" {
if name == nil {
f, err = os.CreateTemp("", "*.db")
} else {
f, err = osutil.OpenFile(name, oflags, 0666)
f, err = osutil.OpenFile(name.String(), oflags, 0666)
}
if err != nil {
if errors.Is(err, syscall.EISDIR) {
@@ -104,7 +101,7 @@ func (vfsOS) OpenParams(name string, flags OpenFlag, params url.Values) (File, O
return nil, flags, err
}
if modeof := params.Get("modeof"); modeof != "" {
if modeof := name.URIParameter("modeof"); modeof != "" {
if err = osSetMode(f, modeof); err != nil {
f.Close()
return nil, flags, _IOERR_FSTAT
@@ -121,13 +118,14 @@ func (vfsOS) OpenParams(name string, flags OpenFlag, params url.Values) (File, O
syncDir: runtime.GOOS != "windows" &&
flags&(OPEN_CREATE) != 0 &&
flags&(OPEN_MAIN_JOURNAL|OPEN_SUPER_JOURNAL|OPEN_WAL) != 0,
shm: NewSharedMemory(name.String()+"-shm", flags),
}
return &file, flags, nil
}
type vfsFile struct {
*os.File
shm vfsShm
shm SharedMemory
lock LockLevel
readOnly bool
keepWAL bool
@@ -145,7 +143,9 @@ var (
)
func (f *vfsFile) Close() error {
f.shm.Close()
if f.shm != nil {
f.shm.Close()
}
return f.File.Close()
}
@@ -176,7 +176,7 @@ func (f *vfsFile) Size() (int64, error) {
return f.Seek(0, io.SeekEnd)
}
func (*vfsFile) SectorSize() int {
func (f *vfsFile) SectorSize() int {
return _DEFAULT_SECTOR_SIZE
}
@@ -215,9 +215,3 @@ func (f *vfsFile) PowersafeOverwrite() bool { return f.psow }
func (f *vfsFile) PersistentWAL() bool { return f.keepWAL }
func (f *vfsFile) SetPowersafeOverwrite(psow bool) { f.psow = psow }
func (f *vfsFile) SetPersistentWAL(keepWAL bool) { f.keepWAL = keepWAL }
type fileShm interface {
shmMap(context.Context, api.Module, int32, int32, bool) (uint32, error)
shmLock(int32, int32, _ShmFlag) error
shmUnmap(bool)
}

174
vfs/filename.go Normal file
View File

@@ -0,0 +1,174 @@
package vfs
import (
"context"
"net/url"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/tetratelabs/wazero/api"
)
// Filename is used by SQLite to pass filenames
// to the Open method of a VFS.
//
// https://sqlite.org/c3ref/filename.html
type Filename struct {
ctx context.Context
mod api.Module
zPath uint32
flags OpenFlag
stack [2]uint64
}
// OpenFilename is an internal API users should not call directly.
func OpenFilename(ctx context.Context, mod api.Module, id uint32, flags OpenFlag) *Filename {
if id == 0 {
return nil
}
return &Filename{
ctx: ctx,
mod: mod,
zPath: id,
flags: flags,
}
}
// String returns this filename as a string.
func (n *Filename) String() string {
if n == nil || n.zPath == 0 {
return ""
}
return util.ReadString(n.mod, n.zPath, _MAX_PATHNAME)
}
// Database returns the name of the corresponding database file.
//
// https://sqlite.org/c3ref/filename_database.html
func (n *Filename) Database() string {
return n.path("sqlite3_filename_database")
}
// Journal returns the name of the corresponding rollback journal file.
//
// https://sqlite.org/c3ref/filename_database.html
func (n *Filename) Journal() string {
return n.path("sqlite3_filename_journal")
}
// Journal returns the name of the corresponding WAL file.
//
// https://sqlite.org/c3ref/filename_database.html
func (n *Filename) WAL() string {
return n.path("sqlite3_filename_wal")
}
func (n *Filename) path(method string) string {
if n == nil || n.zPath == 0 {
return ""
}
n.stack[0] = uint64(n.zPath)
fn := n.mod.ExportedFunction(method)
if err := fn.CallWithStack(n.ctx, n.stack[:]); err != nil {
panic(err)
}
return util.ReadString(n.mod, uint32(n.stack[0]), _MAX_PATHNAME)
}
// DatabaseFile returns the main database [File] corresponding to a journal.
//
// https://sqlite.org/c3ref/database_file_object.html
func (n *Filename) DatabaseFile() File {
if n == nil || n.zPath == 0 {
return nil
}
if n.flags&(OPEN_MAIN_DB|OPEN_MAIN_JOURNAL|OPEN_WAL) == 0 {
return nil
}
n.stack[0] = uint64(n.zPath)
fn := n.mod.ExportedFunction("sqlite3_database_file_object")
if err := fn.CallWithStack(n.ctx, n.stack[:]); err != nil {
panic(err)
}
file, _ := vfsFileGet(n.ctx, n.mod, uint32(n.stack[0])).(File)
return file
}
// URIParameter returns the value of a URI parameter.
//
// https://sqlite.org/c3ref/uri_boolean.html
func (n *Filename) URIParameter(key string) string {
if n == nil || n.zPath == 0 {
return ""
}
uriKey := n.mod.ExportedFunction("sqlite3_uri_key")
n.stack[0] = uint64(n.zPath)
n.stack[1] = uint64(0)
if err := uriKey.CallWithStack(n.ctx, n.stack[:]); err != nil {
panic(err)
}
ptr := uint32(n.stack[0])
if ptr == 0 {
return ""
}
// Parse the format from:
// https://github.com/sqlite/sqlite/blob/b74eb0/src/pager.c#L4797-L4840
// This avoids having to alloc/free the key just to find a value.
for {
k := util.ReadString(n.mod, ptr, _MAX_NAME)
if k == "" {
return ""
}
ptr += uint32(len(k)) + 1
v := util.ReadString(n.mod, ptr, _MAX_NAME)
if k == key {
return v
}
ptr += uint32(len(v)) + 1
}
}
// URIParameters obtains values for URI parameters.
//
// https://sqlite.org/c3ref/uri_boolean.html
func (n *Filename) URIParameters() url.Values {
if n == nil || n.zPath == 0 {
return nil
}
uriKey := n.mod.ExportedFunction("sqlite3_uri_key")
n.stack[0] = uint64(n.zPath)
n.stack[1] = uint64(0)
if err := uriKey.CallWithStack(n.ctx, n.stack[:]); err != nil {
panic(err)
}
ptr := uint32(n.stack[0])
if ptr == 0 {
return nil
}
var params url.Values
// Parse the format from:
// https://github.com/sqlite/sqlite/blob/b74eb0/src/pager.c#L4797-L4840
// This is the only way to support multiple valued keys.
for {
k := util.ReadString(n.mod, ptr, _MAX_NAME)
if k == "" {
return params
}
ptr += uint32(len(k)) + 1
v := util.ReadString(n.mod, ptr, _MAX_NAME)
if params == nil {
params = url.Values{}
}
params.Add(k, v)
ptr += uint32(len(v)) + 1
}
}

View File

@@ -1,4 +1,4 @@
//go:build (linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos) && !sqlite3_nosys
//go:build (linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && !sqlite3_nosys
package vfs

View File

@@ -1,4 +1,4 @@
//go:build !(linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos) || sqlite3_nosys
//go:build !(linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || sqlite3_nosys
package vfs

View File

@@ -11,6 +11,10 @@ import (
)
func Test_vfsLock(t *testing.T) {
if !SupportsFileLocking {
t.Skip("skipping without locks")
}
name := filepath.Join(t.TempDir(), "test.db")
// Create a temporary file.
@@ -33,7 +37,7 @@ func Test_vfsLock(t *testing.T) {
pOutput = 32
)
mod := wazerotest.NewModule(wazerotest.NewMemory(wazerotest.PageSize))
ctx := util.NewContext(context.TODO(), false)
ctx := util.NewContext(context.TODO())
vfsFileRegister(ctx, mod, pFile1, &vfsFile{File: file1})
vfsFileRegister(ctx, mod, pFile2, &vfsFile{File: file2})
@@ -101,7 +105,7 @@ func Test_vfsLock(t *testing.T) {
t.Fatal("returned", rc)
}
if got := util.ReadUint32(mod, pOutput); got == 0 {
t.Error("file wasn't locked")
t.Log("file wasn't locked, locking is incompatible with SQLite")
}
rc = vfsCheckReservedLock(ctx, mod, pFile2, pOutput)
if rc != _OK {
@@ -128,7 +132,7 @@ func Test_vfsLock(t *testing.T) {
t.Fatal("returned", rc)
}
if got := util.ReadUint32(mod, pOutput); got == 0 {
t.Error("file wasn't locked")
t.Log("file wasn't locked, locking is incompatible with SQLite")
}
rc = vfsCheckReservedLock(ctx, mod, pFile2, pOutput)
if rc != _OK {
@@ -155,7 +159,7 @@ func Test_vfsLock(t *testing.T) {
t.Fatal("returned", rc)
}
if got := util.ReadUint32(mod, pOutput); got == 0 {
t.Error("file wasn't locked")
t.Log("file wasn't locked, locking is incompatible with SQLite")
}
rc = vfsCheckReservedLock(ctx, mod, pFile2, pOutput)
if rc != _OK {

View File

@@ -4,7 +4,7 @@
// among multiple database connections in the same process,
// as long as the database name begins with "/".
//
// Importing package memdb registers the VFS.
// Importing package memdb registers the VFS:
//
// import _ "github.com/ncruces/go-sqlite3/vfs/memdb"
package memdb
@@ -33,8 +33,11 @@ func Create(name string, data []byte) {
memoryMtx.Lock()
defer memoryMtx.Unlock()
db := new(memDB)
db.size = int64(len(data))
db := &memDB{
refs: 1,
name: name,
size: int64(len(data)),
}
// Convert data from WAL to rollback journal.
if len(data) >= 20 && data[18] == 2 && data[19] == 2 {

View File

@@ -3,7 +3,6 @@ package memdb
import (
"io"
"runtime"
"strings"
"sync"
"time"
@@ -11,36 +10,48 @@ import (
"github.com/ncruces/go-sqlite3/vfs"
)
// Must be a multiple of 64K (the largest page size).
const sectorSize = 65536
type memVFS struct{}
func (memVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
// Allowed file types:
// For simplicity, we do not support reading or writing data
// across "sector" boundaries.
//
// This is not a problem for most SQLite file types:
// - databases, which only do page aligned reads/writes;
// - temp journals, used by the sorter, which does the same.
// - temp journals, as used by the sorter, which does the same:
// https://github.com/sqlite/sqlite/blob/b74eb0/src/vdbesort.c#L409-L412
//
// We refuse to open all other file types,
// but returning OPEN_MEMORY means SQLite won't ask us to.
const types = vfs.OPEN_MAIN_DB |
vfs.OPEN_TRANSIENT_DB |
vfs.OPEN_TEMP_DB |
vfs.OPEN_TEMP_JOURNAL
if flags&types == 0 {
return nil, flags, sqlite3.CANTOPEN
}
var db *memDB
// A shared database has a name that begins with "/".
shared := len(name) > 1 && name[0] == '/'
shared := strings.HasPrefix(name, "/")
var db *memDB
if shared {
name = name[1:]
memoryMtx.Lock()
defer memoryMtx.Unlock()
db = memoryDBs[name[1:]]
db = memoryDBs[name]
}
if db == nil {
if flags&vfs.OPEN_CREATE == 0 {
return nil, flags, sqlite3.CANTOPEN
}
db = new(memDB)
db = &memDB{name: name}
}
if shared {
memoryDBs[name[1:]] = db // +checklocksignore: lock is held
db.refs++ // +checklocksforce: memoryMtx is held
memoryDBs[name] = db
}
return &memFile{
@@ -61,10 +72,9 @@ func (memVFS) FullPathname(name string) (string, error) {
return name, nil
}
// Must be a multiple of 64K (the largest page size).
const sectorSize = 65536
type memDB struct {
name string
// +checklocks:lockMtx
pending *memFile
// +checklocks:lockMtx
@@ -79,10 +89,21 @@ type memDB struct {
// +checklocks:lockMtx
shared int
// +checklocks:memoryMtx
refs int
lockMtx sync.Mutex
dataMtx sync.RWMutex
}
func (m *memDB) release() {
memoryMtx.Lock()
defer memoryMtx.Unlock()
if m.refs--; m.refs == 0 && m == memoryDBs[m.name] {
delete(memoryDBs, m.name)
}
}
type memFile struct {
*memDB
lock vfs.LockLevel
@@ -96,6 +117,7 @@ var (
)
func (m *memFile) Close() error {
m.release()
return m.Unlock(vfs.LOCK_NONE)
}
@@ -166,7 +188,7 @@ func (m *memFile) truncate(size int64) error {
return nil
}
func (*memFile) Sync(flag vfs.SyncFlag) error {
func (m *memFile) Sync(flag vfs.SyncFlag) error {
return nil
}
@@ -256,11 +278,11 @@ func (m *memFile) CheckReservedLock() (bool, error) {
return m.reserved != nil, nil
}
func (*memFile) SectorSize() int {
func (m *memFile) SectorSize() int {
return sectorSize
}
func (*memFile) DeviceCharacteristics() vfs.DeviceCharacteristic {
func (m *memFile) DeviceCharacteristics() vfs.DeviceCharacteristic {
return vfs.IOCAP_ATOMIC |
vfs.IOCAP_SEQUENTIAL |
vfs.IOCAP_SAFE_APPEND |

View File

@@ -7,6 +7,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
)
//go:embed testdata/wal.db

View File

@@ -1,4 +1,4 @@
//go:build (freebsd || openbsd || netbsd || dragonfly || sqlite3_flock) && !sqlite3_nosys
//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && !sqlite3_nosys
package vfs

View File

@@ -1,4 +1,4 @@
//go:build !sqlite3_flock && !sqlite3_nosys
//go:build !(sqlite3_flock || sqlite3_nosys)
package vfs

View File

@@ -1,9 +1,11 @@
//go:build !sqlite3_nosys
//go:build !(sqlite3_flock || sqlite3_nosys)
package vfs
import (
"math/rand"
"os"
"time"
"golang.org/x/sys/unix"
)
@@ -19,3 +21,51 @@ func osAllocate(file *os.File, size int64) error {
}
return unix.Fallocate(int(file.Fd()), 0, 0, size)
}
func osUnlock(file *os.File, start, len int64) _ErrorCode {
err := unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &unix.Flock_t{
Type: unix.F_UNLCK,
Start: start,
Len: len,
})
if err != nil {
return _IOERR_UNLOCK
}
return _OK
}
func osLock(file *os.File, typ int16, start, len int64, timeout time.Duration, def _ErrorCode) _ErrorCode {
lock := unix.Flock_t{
Type: typ,
Start: start,
Len: len,
}
var err error
switch {
case timeout == 0:
err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &lock)
case timeout < 0:
err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLKW, &lock)
default:
before := time.Now()
for {
err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &lock)
if errno, _ := err.(unix.Errno); errno != unix.EAGAIN {
break
}
if timeout < time.Since(before) {
break
}
osSleep(time.Duration(rand.Int63n(int64(time.Millisecond))))
}
}
return osLockErrorCode(err, def)
}
func osReadLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode {
return osLock(file, unix.F_RDLCK, start, len, timeout, _IOERR_RDLOCK)
}
func osWriteLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode {
return osLock(file, unix.F_WRLCK, start, len, timeout, _IOERR_LOCK)
}

Some files were not shown because too many files have changed in this diff Show More