Compare commits

...

62 Commits

Author SHA1 Message Date
Nuno Cruces
3484bda553 Attestations. 2024-06-21 15:08:33 +01:00
Nuno Cruces
cf0d56271d Integrity. 2024-06-21 13:59:19 +01:00
Nuno Cruces
a465458255 CSV comments. 2024-06-20 11:02:23 +01:00
Nuno Cruces
65af8065cd Fixes. 2024-06-20 00:16:07 +01:00
Nuno Cruces
5c1c0f03a5 Tweaks. 2024-06-19 14:43:44 +01:00
Nuno Cruces
2d168136f1 Cache bind count. 2024-06-19 13:54:58 +01:00
Nuno Cruces
eb8e716253 Fix CI. 2024-06-19 00:43:12 +01:00
Nuno Cruces
3479e8935a Bloom filter virtual table (#103) 2024-06-18 23:42:20 +01:00
Nuno Cruces
58e91052bb CSV type affinity (#102)
Use sqlite-createtable-parser compiled to Wasm to parse the CREATE TABLE statement.
2024-06-17 23:44:37 +01:00
Nuno Cruces
3719692349 Fix potential BSD locking race. (#98) 2024-06-12 20:41:51 +01:00
Nuno Cruces
f7ac77027c wazero v1.7.2. 2024-06-11 23:50:32 +01:00
Nuno Cruces
ef065b6baa More benchmarks. 2024-06-11 10:52:07 +01:00
Nuno Cruces
e7f8311e2e Fix readonly shared memory (see #94). 2024-06-10 00:24:15 +01:00
Nuno Cruces
35a3bfe2f9 Doc fixes. 2024-06-07 12:10:03 +01:00
Nuno Cruces
7386a52b93 Updated dependencies. 2024-06-07 11:06:15 +01:00
Nuno Cruces
34d0289534 Rename to percentile. 2024-06-06 19:55:32 +01:00
Nuno Cruces
dbf764aaf4 Boolean aggregates. 2024-06-06 19:53:22 +01:00
Nuno Cruces
8fd878afd6 Internal API tweaks. 2024-06-06 12:27:27 +01:00
Nuno Cruces
9b769d94d0 BSD WAL fixes. 2024-06-05 23:12:40 +01:00
Nuno Cruces
79c83cdce5 Windows sleep. 2024-06-05 23:12:39 +01:00
Nuno Cruces
e9ed4c103d BSD WAL support. (#90)
Uses in-memory locks.
Also supports illumos.
2024-06-05 00:43:49 +01:00
Nuno Cruces
d78a53a789 Multiple quantiles. 2024-06-02 13:37:29 +01:00
Nuno Cruces
19bc6e3fac Rename. 2024-06-02 12:34:57 +01:00
Nuno Cruces
3955c226cb Rename. 2024-06-02 10:33:20 +01:00
Nuno Cruces
8a3d454935 More tests. 2024-06-02 10:33:06 +01:00
Nuno Cruces
fa7516ce30 Quantiles. 2024-05-31 17:36:16 +01:00
dependabot[bot]
dbf93b2171 Bump lukechampine.com/adiantum from 1.1.0 to 1.1.1 (#89)
Bumps [lukechampine.com/adiantum](https://github.com/lukechampine/adiantum) from 1.1.0 to 1.1.1.
- [Commits](https://github.com/lukechampine/adiantum/compare/v1.1.0...v1.1.1)

---
updated-dependencies:
- dependency-name: lukechampine.com/adiantum
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2024-05-24 23:55:10 +01:00
Nuno Cruces
f29a999ea7 Updated dependencies. 2024-05-24 12:06:37 +01:00
Nuno Cruces
00d52a873f SQLite 3.46.0. 2024-05-24 11:39:27 +01:00
Nuno Cruces
94fb25e84c Enable sqlite_stat4. 2024-05-24 10:30:14 +01:00
Nuno Cruces
b1f2ff55a0 Remove unnecessary conversions. 2024-05-24 10:30:14 +01:00
Nuno Cruces
53eef1510f Fixed capacity virtual memory. 2024-05-20 14:34:47 +01:00
Nuno Cruces
d23bdcd225 Commiting memory not needed. 2024-05-20 09:41:35 +01:00
Nuno Cruces
321d359663 Ensure benchmarks run. 2024-05-20 01:20:11 +01:00
Nuno Cruces
8f88b687d4 Fix flaky test. 2024-05-20 01:10:13 +01:00
Nuno Cruces
d1075f7dad Fix #87. 2024-05-20 01:04:53 +01:00
Nuno Cruces
ed932ee93b Interrupt busy handlers. 2024-05-19 16:30:09 +01:00
Nuno Cruces
3d30a561f0 Custom allocator. 2024-05-17 17:30:43 +01:00
Nuno Cruces
bdaf77a657 Custom Windows allocator. 2024-05-17 16:57:25 +01:00
Nuno Cruces
323bd6e47e Tweak options early. 2024-05-16 16:24:45 +01:00
Nuno Cruces
5f1c372a65 wazero v1.7.2. 2024-05-13 11:44:34 +01:00
Nuno Cruces
3950be71c1 HPolyC example. 2024-05-12 01:35:40 +01:00
Nuno Cruces
f3dc9bdafc Updated adiantum. 2024-05-10 17:00:35 +01:00
Nuno Cruces
e0720fdb92 Gorm v1.25.10. 2024-05-09 13:24:36 +01:00
Nuno Cruces
5fdcdff7e0 Solaris is flaky. 2024-05-07 17:21:14 +01:00
Nuno Cruces
4d23fc3cee Fix file format. 2024-05-07 16:34:51 +01:00
Nuno Cruces
34882e7c8d Fix z/OS build. 2024-05-07 01:54:13 +01:00
Nuno Cruces
57686a2cf3 Dependencies. 2024-05-06 20:39:37 +01:00
Nuno Cruces
190ca0f0cc NPOT sectors. 2024-05-06 11:57:48 +01:00
Nuno Cruces
1a223fa69f Update README.md 2024-05-06 00:41:56 +01:00
Nuno Cruces
12111a619a Cache LFS. 2024-05-05 23:24:39 +01:00
Nuno Cruces
1c58744f87 Test Solaris. 2024-05-04 15:25:03 +01:00
Nuno Cruces
f0ce3e58eb Docs. 2024-05-04 11:44:54 +01:00
Nuno Cruces
5d5c302ff4 Support for z/OS.
Support is behind sqlite3_flock build tag, and tested through s390x Linux. See #86.
2024-05-04 09:48:50 +01:00
Nuno Cruces
10c494031c EWOULDBLOCK. 2024-05-03 18:09:24 +01:00
Nuno Cruces
d84152dd8d Fix TestTimeFormat. 2024-05-03 18:09:11 +01:00
Nuno Cruces
19209b372c Raise Argon2id iterations. 2024-05-03 14:08:38 +01:00
Nuno Cruces
1e03c6c1fb Add initialize. 2024-05-03 12:39:51 +01:00
Nuno Cruces
bb279cb426 Fixes. 2024-05-02 23:42:31 +01:00
Nuno Cruces
7b646100cb Test endianness. 2024-05-02 23:24:24 +01:00
Nuno Cruces
e0a209908b Enable more tests. 2024-05-02 23:22:59 +01:00
Nuno Cruces
67d859a5b4 Support custom pepper. 2024-05-02 12:09:39 +01:00
148 changed files with 2932 additions and 1153 deletions

23
.github/actions/lfs/action.yml vendored Normal file
View File

@@ -0,0 +1,23 @@
name: Git LFS pull
description: Cached Git LFS pull.
runs:
using: "composite"
steps:
- name: Create LFS file list
shell: bash
run: git lfs ls-files --long | cut -d ' ' -f1 | sort > .lfs-assets-id
- name: Restore LFS cache
uses: actions/cache@v4
with:
path: .git/lfs/objects
key: lfs-${{ hashFiles('.lfs-assets-id') }}
restore-keys: lfs-
enableCrossOsArchive: true
- name: Git LFS pull
shell: bash
run: |
git lfs pull
git lfs prune

View File

@@ -6,6 +6,6 @@ echo 'set -eu' > test.sh
for p in $(go list ./...); do
dir=".${p#github.com/ncruces/go-sqlite3}"
name="$(basename "$p").test"
(cd ${dir}; GOOS=freebsd go test -c)
[ -f "${dir}/${name}" ] && echo "(cd ${dir}; ./${name} -test.v)" >> test.sh
(cd ${dir}; go test -c)
[ -f "${dir}/${name}" ] && echo "(cd ${dir}; ./${name} ${TESTFLAGS})" >> test.sh
done

View File

@@ -16,10 +16,12 @@ echo windows ; GOOS=windows GOARCH=amd64 go build .
echo aix ; GOOS=aix GOARCH=ppc64 go build .
echo js ; GOOS=js GOARCH=wasm go build .
echo wasip1 ; GOOS=wasip1 GOARCH=wasm go build .
echo linux-flock ; GOOS=linux GOARCH=amd64 go build -tags sqlite3_flock .
echo linux-noshm ; GOOS=linux GOARCH=amd64 go build -tags sqlite3_noshm .
echo linux-nosys ; GOOS=linux GOARCH=amd64 go build -tags sqlite3_nosys .
echo darwin-flock ; GOOS=darwin GOARCH=amd64 go build -tags sqlite3_flock .
echo darwin-noshm ; GOOS=darwin GOARCH=amd64 go build -tags sqlite3_noshm .
echo darwin-nosys ; GOOS=darwin GOARCH=amd64 go build -tags sqlite3_nosys .
echo linux-noshm ; GOOS=linux GOARCH=amd64 go build -tags sqlite3_noshm .
echo linux-nosys ; GOOS=linux GOARCH=amd64 go build -tags sqlite3_nosys .
echo windows-nosys ; GOOS=windows GOARCH=amd64 go build -tags sqlite3_nosys .
echo freebsd-nosys ; GOOS=freebsd GOARCH=amd64 go build -tags sqlite3_nosys .
echo freebsd-nosys ; GOOS=freebsd GOARCH=amd64 go build -tags sqlite3_nosys .
echo solaris-flock ; GOOS=solaris GOARCH=amd64 go build -tags sqlite3_flock .

View File

@@ -9,10 +9,8 @@ jobs:
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: stable
with: { go-version: stable }
- name: Build
run: .github/workflows/cross.sh

View File

@@ -1,11 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
echo 'set -eu' > test.sh
for p in $(go list ./...); do
dir=".${p#github.com/ncruces/go-sqlite3}"
name="$(basename "$p").test"
(cd ${dir}; GOOS=illumos go test -c)
[ -f "${dir}/${name}" ] && echo "(cd ${dir}; ./${name} -test.v -test.short)" >> test.sh
done

View File

@@ -18,6 +18,13 @@ mkdir -p tools/
[ -d "tools/binaryen-version"* ] || curl -#L "$BINARYEN" | tar xzC tools &
wait
sqlite3/download.sh # Download SQLite
embed/build.sh # Build Wasm
git diff --exit-code # Check diffs
# Download and build SQLite
sqlite3/download.sh
embed/build.sh
# Download and build sqlite-createtable-parser
util/vtabutil/parse/download.sh
util/vtabutil/parse/build.sh
# Check diffs
git diff --exit-code

View File

@@ -3,6 +3,11 @@ name: Reproducible build
on:
workflow_dispatch:
permissions:
contents: read
id-token: write
attestations: write
jobs:
build:
strategy:
@@ -12,12 +17,15 @@ jobs:
steps:
- uses: actions/checkout@v4
with:
lfs: 'true'
- uses: actions/setup-go@v5
with:
go-version: stable
with: { go-version: stable }
- name: Build
run: .github/workflows/repro.sh
- uses: actions/attest-build-provenance@v1
if: matrix.os == 'ubuntu-latest'
with:
subject-path: |
embed/sqlite3.wasm
util/vtabutil/parse/sql3parse_table.wasm

View File

@@ -16,11 +16,12 @@ jobs:
steps:
- uses: actions/checkout@v4
with: { lfs: 'true' }
- uses: actions/setup-go@v5
with: { go-version: stable }
- name: Git LFS pull
uses: ./.github/actions/lfs
- name: Format
run: gofmt -s -w . && git diff --exit-code
if: matrix.os != 'windows-latest'
@@ -41,7 +42,7 @@ jobs:
run: go build -v ./...
- name: Test
run: go test -v ./...
run: go test -v ./... -bench . -benchtime=1x
- name: Test BSD locks
run: go test -v -tags sqlite3_flock ./...
@@ -66,19 +67,38 @@ jobs:
github.event_name == 'push' &&
matrix.os == 'ubuntu-latest'
test-intel:
runs-on: macos-13
needs: test
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with: { go-version: stable }
- name: Git LFS pull
uses: ./.github/actions/lfs
- name: Test
run: go test -v ./...
test-bsd:
runs-on: ubuntu-latest
needs: test
steps:
- uses: actions/checkout@v4
with: { lfs: 'true' }
- uses: actions/setup-go@v5
with: { go-version: stable }
- name: Git LFS pull
uses: ./.github/actions/lfs
- name: Build
run: .github/workflows/bsd.sh
env:
GOOS: freebsd
TESTFLAGS: '-test.v'
run: .github/workflows/build-test.sh
- name: Test
uses: cross-platform-actions/action@v0.24.0
@@ -89,59 +109,66 @@ jobs:
run: . ./test.sh
sync_files: runner-to-vm
test-illumos:
test-qemu:
runs-on: ubuntu-latest
needs: test
steps:
- uses: docker/setup-qemu-action@v3
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with: { go-version: stable }
- name: Git LFS pull
uses: ./.github/actions/lfs
- name: Test 386 (32-bit)
run: GOARCH=386 go test -v -short ./...
- name: Test arm64 (compiler)
run: GOARCH=arm64 go test -v -short ./...
- name: Test riscv64 (interpreter)
run: GOARCH=riscv64 go test -v -short ./...
- name: Test s390x (big-endian, z/OS demo)
run: GOARCH=s390x go test -v -short -tags sqlite3_flock ./...
test-vm:
runs-on: ubuntu-latest
needs: test
steps:
- uses: actions/checkout@v4
with: { lfs: 'true' }
- uses: actions/setup-go@v5
with: { go-version: stable }
- name: Build
run: .github/workflows/illumos.sh
- name: Git LFS pull
uses: ./.github/actions/lfs
- name: Test
- name: Build illumos
env:
GOOS: illumos
TESTFLAGS: '-test.v -test.short'
run: .github/workflows/build-test.sh
- name: Test illumos
uses: vmactions/omnios-vm@v1
with:
usesh: true
copyback: false
run: . ./test.sh
test-m1:
runs-on: macos-14
needs: test
- name: Build Solaris
env:
GOOS: solaris
TESTFLAGS: '-test.v -test.short'
run: .github/workflows/build-test.sh
steps:
- uses: actions/checkout@v4
with: { lfs: 'true' }
- uses: actions/setup-go@v5
with: { go-version: stable }
- name: Test
run: go test -v ./...
test-qemu:
runs-on: ubuntu-latest
needs: test
steps:
- uses: actions/checkout@v4
with: { lfs: 'true' }
- uses: actions/setup-go@v5
with: { go-version: stable }
- uses: docker/setup-qemu-action@v3
- name: Test 386
run: GOARCH=386 go test -v -short ./...
- name: Test arm64
run: GOARCH=arm64 go test -v -short ./...
- name: Test riscv64
run: GOARCH=riscv64 go test -v -short ./...
- name: Test Solaris
uses: vmactions/solaris-vm@v1
with:
usesh: true
copyback: false
run: . ./test.sh
continue-on-error: true

View File

@@ -33,6 +33,8 @@ Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ run
provides the [`array`](https://sqlite.org/carray.html) table-valued function.
- [`github.com/ncruces/go-sqlite3/ext/blobio`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/blobio)
simplifies [incremental BLOB I/O](https://sqlite.org/c3ref/blob_open.html).
- [`github.com/ncruces/go-sqlite3/ext/bloom`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/bloom)
provides a [Bloom filter](https://github.com/nalgeon/sqlean/issues/27#issuecomment-1002267134) virtual table.
- [`github.com/ncruces/go-sqlite3/ext/csv`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/csv)
reads [comma-separated values](https://sqlite.org/csv.html).
- [`github.com/ncruces/go-sqlite3/ext/fileio`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/fileio)
@@ -51,12 +53,12 @@ Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ run
provides [Unicode aware](https://sqlite.org/src/dir/ext/icu) functions.
- [`github.com/ncruces/go-sqlite3/ext/zorder`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/zorder)
maps multidimensional data to one dimension.
- [`github.com/ncruces/go-sqlite3/vfs/adiantum`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/adiantum)
wraps a VFS to offer encryption at rest.
- [`github.com/ncruces/go-sqlite3/vfs/memdb`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/memdb)
implements an in-memory VFS.
- [`github.com/ncruces/go-sqlite3/vfs/readervfs`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/readervfs)
implements a VFS for immutable databases.
- [`github.com/ncruces/go-sqlite3/vfs/adiantum`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/adiantum)
wraps a VFS to offer encryption at rest.
### Advanced features
@@ -88,7 +90,9 @@ It also benefits greatly from [SQLite's](https://sqlite.org/testing.html) and
[wazero's](https://tetrate.io/blog/introducing-wazero-from-tetrate/#:~:text=Rock%2Dsolid%20test%20approach) thorough testing.
Every commit is [tested](.github/workflows/test.yml) on
Linux (amd64/arm64/386/riscv64), macOS (amd64/arm64), Windows, FreeBSD and illumos.
Linux (amd64/arm64/386/riscv64/s390x), macOS (amd64/arm64),
Windows (amd64), FreeBSD (amd64), illumos (amd64), and Solaris (amd64).
The Go VFS is tested by running SQLite's
[mptest](https://github.com/sqlite/sqlite/blob/master/mptest/mptest.c).

34
conn.go
View File

@@ -346,10 +346,9 @@ func (c *Conn) checkInterrupt() {
}
func progressCallback(ctx context.Context, mod api.Module, pDB uint32) (interrupt uint32) {
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.interrupt != nil {
if c.interrupt.Err() != nil {
interrupt = 1
}
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB &&
c.interrupt != nil && c.interrupt.Err() != nil {
interrupt = 1
}
return interrupt
}
@@ -363,6 +362,30 @@ func (c *Conn) BusyTimeout(timeout time.Duration) error {
return c.error(r)
}
func timeoutCallback(ctx context.Context, mod api.Module, pDB uint32, count, tmout int32) (retry uint32) {
if c, ok := ctx.Value(connKey{}).(*Conn); ok &&
(c.interrupt == nil || c.interrupt.Err() == nil) {
const delays = "\x01\x02\x05\x0a\x0f\x14\x19\x19\x19\x32\x32\x64"
const totals = "\x00\x01\x03\x08\x12\x21\x35\x4e\x67\x80\xb2\xe4"
const ndelay = int32(len(delays) - 1)
var delay, prior int32
if count <= ndelay {
delay = int32(delays[count])
prior = int32(totals[count])
} else {
delay = int32(delays[ndelay])
prior = int32(totals[ndelay]) + delay*(count-ndelay)
}
if delay = min(delay, tmout-prior); delay > 0 {
time.Sleep(time.Duration(delay) * time.Millisecond)
retry = 1
}
}
return retry
}
// BusyHandler registers a callback to handle [BUSY] errors.
//
// https://sqlite.org/c3ref/busy_handler.html
@@ -380,7 +403,8 @@ func (c *Conn) BusyHandler(cb func(count int) (retry bool)) error {
}
func busyCallback(ctx context.Context, mod api.Module, pDB uint32, count int32) (retry uint32) {
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.busy != nil {
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.busy != nil &&
(c.interrupt == nil || c.interrupt.Err() == nil) {
if c.busy(int(count)) {
retry = 1
}

View File

@@ -229,6 +229,7 @@ func (c *conn) Raw() *sqlite3.Conn {
return c.Conn
}
// Deprecated: use BeginTx instead.
func (c *conn) Begin() (driver.Tx, error) {
return c.BeginTx(context.Background(), driver.TxOptions{})
}
@@ -301,7 +302,7 @@ func (c *conn) PrepareContext(ctx context.Context, query string) (driver.Stmt, e
s.Close()
return nil, util.TailErr
}
return &stmt{Stmt: s, tmRead: c.tmRead, tmWrite: c.tmWrite}, nil
return &stmt{Stmt: s, tmRead: c.tmRead, tmWrite: c.tmWrite, inputs: -2}, nil
}
func (c *conn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) {
@@ -335,6 +336,7 @@ type stmt struct {
*sqlite3.Stmt
tmWrite sqlite3.TimeFormat
tmRead sqlite3.TimeFormat
inputs int
}
var (
@@ -345,12 +347,17 @@ var (
)
func (s *stmt) NumInput() int {
if s.inputs >= -1 {
return s.inputs
}
n := s.Stmt.BindCount()
for i := 1; i <= n; i++ {
if s.Stmt.BindName(i) != "" {
s.inputs = -1
return -1
}
}
s.inputs = n
return n
}
@@ -389,12 +396,7 @@ func (s *stmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driv
return &rows{ctx: ctx, stmt: s}, nil
}
func (s *stmt) setupBindings(args []driver.NamedValue) error {
err := s.Stmt.ClearBindings()
if err != nil {
return err
}
func (s *stmt) setupBindings(args []driver.NamedValue) (err error) {
var ids [3]int
for _, arg := range args {
ids := ids[:0]
@@ -558,19 +560,20 @@ func (r *rows) Next(dest []driver.Value) error {
return err
}
func (r *rows) decodeTime(i int, v any) (_ time.Time, _ bool) {
func (r *rows) decodeTime(i int, v any) (_ time.Time, ok bool) {
if r.tmRead == sqlite3.TimeFormatDefault {
return
}
switch r.declType(i) {
case "DATE", "TIME", "DATETIME", "TIMESTAMP":
// maybe
default:
// handled by maybeTime
return
}
switch v.(type) {
case int64, float64, string:
// maybe
// could be a time value
default:
return
}
switch r.declType(i) {
case "DATE", "TIME", "DATETIME", "TIMESTAMP":
// could be a time value
default:
return
}

View File

@@ -1,5 +1,3 @@
//go:build !sqlite3_nosys
package driver
import (
@@ -15,8 +13,9 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
"github.com/ncruces/go-sqlite3/internal/util"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
"github.com/ncruces/go-sqlite3/vfs"
)
func Test_Open_dir(t *testing.T) {
@@ -82,6 +81,9 @@ func Test_Open_pragma_invalid(t *testing.T) {
}
func Test_Open_txLock(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
t.Parallel()
db, err := sql.Open("sqlite3", "file:"+
@@ -128,6 +130,9 @@ func Test_Open_txLock_invalid(t *testing.T) {
}
func Test_BeginTx(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
t.Parallel()
ctx, cancel := context.WithCancel(context.Background())

View File

@@ -1,4 +1,4 @@
//go:build !sqlite3_nosys
//go:build (linux || darwin || windows || freebsd || illumos) && !sqlite3_nosys
package driver_test

View File

@@ -6,7 +6,6 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/vfs/memdb"
)

View File

@@ -1,6 +1,6 @@
# Embeddable Wasm build of SQLite
This folder includes an embeddable Wasm build of SQLite 3.45.3 for use with
This folder includes an embeddable Wasm build of SQLite 3.46.0 for use with
[`github.com/ncruces/go-sqlite3`](https://pkg.go.dev/github.com/ncruces/go-sqlite3).
The following optional features are compiled in:
@@ -10,6 +10,7 @@ The following optional features are compiled in:
- [R*Tree](https://sqlite.org/rtree.html)
- [GeoPoly](https://sqlite.org/geopoly.html)
- [soundex](https://sqlite.org/lang_corefunc.html#soundex)
- [stat4](https://sqlite.org/compile.html#enable_stat4)
- [base64](https://github.com/sqlite/sqlite/blob/master/ext/misc/base64.c)
- [decimal](https://github.com/sqlite/sqlite/blob/master/ext/misc/decimal.c)
- [ieee754](https://github.com/sqlite/sqlite/blob/master/ext/misc/ieee754.c)
@@ -23,4 +24,7 @@ See the [configuration options](../sqlite3/sqlite_cfg.h),
and [patches](../sqlite3) applied.
Built using [`wasi-sdk`](https://github.com/WebAssembly/wasi-sdk),
and [`binaryen`](https://github.com/WebAssembly/binaryen).
and [`binaryen`](https://github.com/WebAssembly/binaryen).
The build is easily reproducible, and verifiable, using
[Artifact Attestations](https://github.com/ncruces/go-sqlite3/attestations).

View File

@@ -8,7 +8,7 @@ BINARYEN="$ROOT/tools/binaryen-version_117/bin"
WASI_SDK="$ROOT/tools/wasi-sdk-22.0/bin"
"$WASI_SDK/clang" --target=wasm32-wasi -std=c17 -flto -g0 -O2 \
-Wall -Wextra -Wno-unused-parameter \
-Wall -Wextra -Wno-unused-parameter -Wno-unused-function \
-o sqlite3.wasm "$ROOT/sqlite3/main.c" \
-I"$ROOT/sqlite3" \
-mexec-model=reactor \
@@ -20,6 +20,7 @@ WASI_SDK="$ROOT/tools/wasi-sdk-22.0/bin"
-Wl,--stack-first \
-Wl,--import-undefined \
-D_HAVE_SQLITE_CONFIG_H \
-DSQLITE_CUSTOM_INCLUDE=sqlite_opt.h \
$(awk '{print "-Wl,--export="$0}' exports.txt)
trap 'rm -f sqlite3.tmp' EXIT

View File

@@ -36,10 +36,13 @@ sqlite3_collation_needed_go
sqlite3_column_blob
sqlite3_column_bytes
sqlite3_column_count
sqlite3_column_database_name
sqlite3_column_decltype
sqlite3_column_double
sqlite3_column_int64
sqlite3_column_name
sqlite3_column_origin_name
sqlite3_column_table_name
sqlite3_column_text
sqlite3_column_type
sqlite3_column_value

Binary file not shown.

View File

@@ -16,7 +16,7 @@ import (
// ints, floats, bools, strings or byte slices,
// using [sqlite3.BindPointer] or [sqlite3.Pointer].
func Register(db *sqlite3.Conn) {
sqlite3.CreateModule[array](db, "array", nil,
sqlite3.CreateModule(db, "array", nil,
func(db *sqlite3.Conn, _, _, _ string, _ ...string) (array, error) {
err := db.DeclareVTab(`CREATE TABLE x(value, array HIDDEN)`)
return array{}, err

View File

@@ -11,7 +11,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/array"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
func Example_driver() {

View File

@@ -12,7 +12,7 @@ import (
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/array"
"github.com/ncruces/go-sqlite3/ext/blobio"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
_ "github.com/ncruces/go-sqlite3/vfs/memdb"
)

340
ext/bloom/bloom.go Normal file
View File

@@ -0,0 +1,340 @@
// Package bloom provides a Bloom filter virtual table.
//
// A Bloom filter is a space-efficient probabilistic data structure
// used to test whether an element is a member of a set.
//
// https://github.com/nalgeon/sqlean/issues/27#issuecomment-1002267134
package bloom
import (
"errors"
"fmt"
"io"
"math"
"strconv"
"github.com/dchest/siphash"
"github.com/ncruces/go-sqlite3"
)
// Register registers the bloom_filter virtual table:
//
// CREATE VIRTUAL TABLE foo USING bloom_filter(nElements, falseProb, kHashes)
func Register(db *sqlite3.Conn) {
sqlite3.CreateModule(db, "bloom_filter", create, connect)
}
type bloom struct {
db *sqlite3.Conn
schema string
storage string
prob float64
bytes int64
hashes int
}
func create(db *sqlite3.Conn, _, schema, table string, arg ...string) (_ *bloom, err error) {
t := bloom{
db: db,
schema: schema,
storage: table + "_storage",
}
var nelem int64
if len(arg) > 0 {
nelem, err = strconv.ParseInt(arg[0], 10, 64)
if err != nil {
return nil, err
}
if nelem <= 0 {
return nil, errors.New("bloom: number of elements in filter must be positive")
}
} else {
nelem = 100
}
if len(arg) > 1 {
t.prob, err = strconv.ParseFloat(arg[1], 64)
if err != nil {
return nil, err
}
if t.prob <= 0 || t.prob >= 1 {
return nil, errors.New("bloom: probability must be in the range (0,1)")
}
} else {
t.prob = 0.01
}
if len(arg) > 2 {
t.hashes, err = strconv.Atoi(arg[2])
if err != nil {
return nil, err
}
if t.hashes <= 0 {
return nil, errors.New("bloom: number of hash functions must be positive")
}
} else {
t.hashes = max(1, numHashes(t.prob))
}
t.bytes = numBytes(nelem, t.prob)
err = db.Exec(fmt.Sprintf(
`CREATE TABLE %s.%s (data BLOB, p REAL, n INTEGER, m INTEGER, k INTEGER)`,
sqlite3.QuoteIdentifier(t.schema), sqlite3.QuoteIdentifier(t.storage)))
if err != nil {
return nil, err
}
id := db.LastInsertRowID()
defer db.SetLastInsertRowID(id)
err = db.Exec(fmt.Sprintf(
`INSERT INTO %s.%s (rowid, data, p, n, m, k)
VALUES (1, zeroblob(%d), %f, %d, %d, %d)`,
sqlite3.QuoteIdentifier(t.schema), sqlite3.QuoteIdentifier(t.storage),
t.bytes, t.prob, nelem, 8*t.bytes, t.hashes))
if err != nil {
return nil, err
}
err = db.DeclareVTab(
`CREATE TABLE x(present, word HIDDEN NOT NULL PRIMARY KEY) WITHOUT ROWID`)
if err != nil {
t.Destroy()
return nil, err
}
return &t, nil
}
func connect(db *sqlite3.Conn, _, schema, table string, arg ...string) (_ *bloom, err error) {
t := bloom{
db: db,
schema: schema,
storage: table + "_storage",
}
err = db.DeclareVTab(
`CREATE TABLE x(present, word HIDDEN NOT NULL PRIMARY KEY) WITHOUT ROWID`)
if err != nil {
return nil, err
}
load, _, err := db.Prepare(fmt.Sprintf(
`SELECT m/8, p, k FROM %s.%s WHERE rowid = 1`,
sqlite3.QuoteIdentifier(t.schema), sqlite3.QuoteIdentifier(t.storage)))
if err != nil {
return nil, err
}
defer load.Close()
if !load.Step() {
if err = load.Err(); err == nil {
err = sqlite3.CORRUPT_VTAB
}
return nil, err
}
t.bytes = load.ColumnInt64(0)
t.prob = load.ColumnFloat(1)
t.hashes = load.ColumnInt(2)
return &t, nil
}
func (b *bloom) Destroy() error {
return b.db.Exec(fmt.Sprintf(`DROP TABLE %s.%s`,
sqlite3.QuoteIdentifier(b.schema),
sqlite3.QuoteIdentifier(b.storage)))
}
func (b *bloom) Rename(new string) error {
new += "_storage"
err := b.db.Exec(fmt.Sprintf(`ALTER TABLE %s.%s RENAME TO %s`,
sqlite3.QuoteIdentifier(b.schema),
sqlite3.QuoteIdentifier(b.storage),
sqlite3.QuoteIdentifier(new),
))
if err == nil {
b.storage = new
}
return err
}
func (t *bloom) ShadowTables() {}
func (t *bloom) Integrity(schema, table string, flags int) error {
load, _, err := t.db.Prepare(fmt.Sprintf(
`SELECT typeof(data), length(data), p, n, m, k FROM %s.%s WHERE rowid = 1`,
sqlite3.QuoteIdentifier(t.schema), sqlite3.QuoteIdentifier(t.storage)))
if err != nil {
return fmt.Errorf("bloom: %v", err) // can't wrap!
}
defer load.Close()
err = errors.New("bloom: invalid parameters")
if !load.Step() {
return err
}
if t := load.ColumnText(0); t != "blob" {
return err
}
if m := load.ColumnInt64(4); m <= 0 || m%8 != 0 {
return err
} else if load.ColumnInt64(1) != m/8 {
return err
}
if p := load.ColumnFloat(2); p <= 0 || p >= 1 {
return err
}
if n := load.ColumnInt64(3); n <= 0 {
return err
}
if k := load.ColumnInt(5); k <= 0 {
return err
}
return nil
}
func (b *bloom) BestIndex(idx *sqlite3.IndexInfo) error {
for n, cst := range idx.Constraint {
if cst.Usable && cst.Column == 1 &&
cst.Op == sqlite3.INDEX_CONSTRAINT_EQ {
idx.ConstraintUsage[n].ArgvIndex = 1
idx.OrderByConsumed = true
idx.EstimatedRows = 1
idx.EstimatedCost = float64(b.hashes)
idx.IdxFlags = sqlite3.INDEX_SCAN_UNIQUE
return nil
}
}
return sqlite3.CONSTRAINT
}
func (b *bloom) Update(arg ...sqlite3.Value) (rowid int64, err error) {
if arg[0].Type() != sqlite3.NULL {
if len(arg) == 1 {
return 0, errors.New("bloom: elements cannot be deleted")
}
return 0, errors.New("bloom: elements cannot be updated")
}
blob := arg[2].RawBlob()
f, err := b.db.OpenBlob(b.schema, b.storage, "data", 1, true)
if err != nil {
return 0, err
}
defer f.Close()
for n := 0; n < b.hashes; n++ {
hash := calcHash(n, blob)
hash %= uint64(b.bytes * 8)
bitpos := byte(hash % 8)
bytepos := int64(hash / 8)
var buf [1]byte
_, err = f.Seek(bytepos, io.SeekStart)
if err != nil {
return 0, err
}
_, err = f.Read(buf[:])
if err != nil {
return 0, err
}
buf[0] |= 1 << bitpos
_, err = f.Seek(bytepos, io.SeekStart)
if err != nil {
return 0, err
}
_, err = f.Write(buf[:])
if err != nil {
return 0, err
}
}
return 0, nil
}
func (b *bloom) Open() (sqlite3.VTabCursor, error) {
return &cursor{bloom: b}, nil
}
type cursor struct {
*bloom
eof bool
arg *sqlite3.Value
}
func (c *cursor) Filter(idxNum int, idxStr string, arg ...sqlite3.Value) error {
if len(arg) != 1 {
return nil
}
c.eof = false
c.arg = &arg[0]
blob := arg[0].RawBlob()
f, err := c.db.OpenBlob(c.schema, c.storage, "data", 1, false)
if err != nil {
return err
}
defer f.Close()
for n := 0; n < c.hashes && !c.eof; n++ {
hash := calcHash(n, blob)
hash %= uint64(c.bytes * 8)
bitpos := byte(hash % 8)
bytepos := int64(hash / 8)
var buf [1]byte
_, err = f.Seek(bytepos, io.SeekStart)
if err != nil {
return err
}
_, err = f.Read(buf[:])
if err != nil {
return err
}
c.eof = buf[0]&(1<<bitpos) == 0
}
return nil
}
func (c *cursor) Column(ctx *sqlite3.Context, n int) error {
switch n {
case 0:
ctx.ResultBool(true)
case 1:
ctx.ResultValue(*c.arg)
}
return nil
}
func (c *cursor) Next() error {
c.eof = true
return nil
}
func (c *cursor) EOF() bool {
return c.eof
}
func (c *cursor) RowID() (int64, error) {
return 0, nil
}
func calcHash(k int, b []byte) uint64 {
return siphash.Hash(^uint64(k), uint64(k), b)
}
func numHashes(p float64) int {
k := math.Round(-math.Log2(p))
return max(1, int(k))
}
func numBytes(n int64, p float64) int64 {
m := math.Ceil(float64(n) * math.Log(p) / -(math.Ln2 * math.Ln2))
return (int64(m) + 7) / 8
}

140
ext/bloom/bloom_test.go Normal file
View File

@@ -0,0 +1,140 @@
package bloom_test
import (
_ "embed"
"os"
"path/filepath"
"testing"
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/bloom"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
func TestRegister(t *testing.T) {
t.Parallel()
db, err := sqlite3.Open(":memory:")
if err != nil {
t.Fatal(err)
}
defer db.Close()
bloom.Register(db)
err = db.Exec(`
CREATE VIRTUAL TABLE sports_cars USING bloom_filter(20);
INSERT INTO sports_cars VALUES ('ferrari'), ('lamborghini'), ('alfa romeo')
`)
if err != nil {
t.Fatal(err)
}
query, _, err := db.Prepare(`SELECT COUNT(*) FROM sports_cars(?)`)
if err != nil {
t.Fatal(err)
}
err = query.BindText(1, "ferrari")
if err != nil {
t.Fatal(err)
}
if !query.Step() {
t.Error("no rows")
}
if !query.ColumnBool(0) {
t.Error("want true")
}
err = query.Reset()
if err != nil {
t.Fatal(err)
}
err = query.BindText(1, "bmw")
if err != nil {
t.Fatal(err)
}
if !query.Step() {
t.Error("no rows")
}
if query.ColumnBool(0) {
t.Error("want false")
}
err = query.Close()
if err != nil {
t.Fatal(err)
}
err = db.Exec(`DROP TABLE sports_cars`)
if err != nil {
t.Fatal(err)
}
}
//go:embed testdata/bloom.db
var testDB []byte
func Test_compatible(t *testing.T) {
t.Parallel()
tmp := filepath.Join(t.TempDir(), "bloom.db")
err := os.WriteFile(tmp, testDB, 0666)
if err != nil {
t.Fatal(err)
}
db, err := sqlite3.Open("file:" + filepath.ToSlash(tmp) + "?nolock=1")
if err != nil {
t.Fatal(err)
}
defer db.Close()
bloom.Register(db)
query, _, err := db.Prepare(`SELECT COUNT(*) FROM plants(?)`)
if err != nil {
t.Fatal(err)
}
defer query.Close()
err = query.BindText(1, "apple")
if err != nil {
t.Fatal(err)
}
if !query.Step() {
t.Error("no rows")
}
if !query.ColumnBool(0) {
t.Error("want true")
}
err = query.Reset()
if err != nil {
t.Fatal(err)
}
err = query.BindText(1, "lemon")
if err != nil {
t.Fatal(err)
}
if !query.Step() {
t.Error("no rows")
}
if query.ColumnBool(0) {
t.Error("want false")
}
err = query.Reset()
if err != nil {
t.Fatal(err)
}
err = db.Exec(`PRAGMA integrity_check`)
if err != nil {
t.Error(err)
}
err = db.Exec(`PRAGMA quick_check`)
if err != nil {
t.Error(err)
}
}

BIN
ext/bloom/testdata/bloom.db vendored Normal file

Binary file not shown.

View File

@@ -40,6 +40,8 @@ func Test_uintArg(t *testing.T) {
}
func Test_boolArg(t *testing.T) {
t.Parallel()
tests := []struct {
arg string
key string
@@ -76,6 +78,8 @@ func Test_boolArg(t *testing.T) {
}
func Test_runeArg(t *testing.T) {
t.Parallel()
tests := []struct {
arg string
key string

View File

@@ -9,9 +9,11 @@ package csv
import (
"bufio"
"encoding/csv"
"errors"
"fmt"
"io"
"io/fs"
"strconv"
"strings"
"github.com/ncruces/go-sqlite3"
@@ -36,6 +38,7 @@ func RegisterFS(db *sqlite3.Conn, fsys fs.FS) {
header bool
columns int = -1
comma rune = ','
comment rune
done = map[string]struct{}{}
)
@@ -58,6 +61,8 @@ func RegisterFS(db *sqlite3.Conn, fsys fs.FS) {
columns, err = uintArg(key, val)
case "comma":
comma, err = runeArg(key, val)
case "comment":
comment, err = runeArg(key, val)
default:
return nil, fmt.Errorf("csv: unknown %q parameter", key)
}
@@ -68,15 +73,16 @@ func RegisterFS(db *sqlite3.Conn, fsys fs.FS) {
}
if (filename == "") == (data == "") {
return nil, fmt.Errorf(`csv: must specify either "filename" or "data" but not both`)
return nil, errors.New(`csv: must specify either "filename" or "data" but not both`)
}
table := &table{
fsys: fsys,
name: filename,
data: data,
comma: comma,
header: header,
fsys: fsys,
name: filename,
data: data,
comma: comma,
comment: comment,
header: header,
}
if schema == "" {
@@ -93,6 +99,12 @@ func RegisterFS(db *sqlite3.Conn, fsys fs.FS) {
}
}
schema = getSchema(header, columns, row)
} else {
defer func() {
if err == nil {
table.typs, err = getColumnAffinities(schema)
}
}()
}
err = db.DeclareVTab(schema)
@@ -110,11 +122,13 @@ func RegisterFS(db *sqlite3.Conn, fsys fs.FS) {
}
type table struct {
fsys fs.FS
name string
data string
comma rune
header bool
fsys fs.FS
name string
data string
typs []affinity
comma rune
comment rune
header bool
}
func (t *table) BestIndex(idx *sqlite3.IndexInfo) error {
@@ -171,6 +185,7 @@ func (t *table) newReader() (*csv.Reader, io.Closer, error) {
csv := csv.NewReader(r)
csv.ReuseRecord = true
csv.Comma = t.comma
csv.Comment = t.comment
return csv, c, nil
}
@@ -226,7 +241,36 @@ func (c *cursor) RowID() (int64, error) {
func (c *cursor) Column(ctx *sqlite3.Context, col int) error {
if col < len(c.row) {
ctx.ResultText(c.row[col])
typ := text
if col < len(c.table.typs) {
typ = c.table.typs[col]
}
txt := c.row[col]
if txt == "" && typ != text {
return nil
}
switch typ {
case numeric, integer:
if strings.TrimLeft(txt, "+-0123456789") == "" {
if i, err := strconv.ParseInt(txt, 10, 64); err == nil {
ctx.ResultInt64(i)
return nil
}
}
fallthrough
case real:
if strings.TrimLeft(txt, "+-.0123456789Ee") == "" {
if f, err := strconv.ParseFloat(txt, 64); err == nil {
ctx.ResultFloat(f)
return nil
}
}
fallthrough
default:
}
ctx.ResultText(txt)
}
return nil
}

View File

@@ -8,7 +8,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/csv"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
func Example() {
@@ -63,14 +63,16 @@ func TestRegister(t *testing.T) {
csv.Register(db)
const data = `
# Comment
"Rob" "Pike" rob
"Ken" Thompson ken
Robert "Griesemer" "gri"`
err = db.Exec(`
CREATE VIRTUAL TABLE temp.users USING csv(
data = ` + sqlite3.Quote(data) + `,
schema = 'CREATE TABLE x(first_name, last_name, username)',
comma = '\t'
data = ` + sqlite3.Quote(data) + `,
schema = 'CREATE TABLE x(first_name, last_name, username)',
comma = '\t',
comment = '#'
)`)
if err != nil {
t.Fatal(err)
@@ -113,6 +115,50 @@ Robert "Griesemer" "gri"`
}
}
func TestAffinity(t *testing.T) {
t.Parallel()
db, err := sqlite3.Open(":memory:")
if err != nil {
t.Fatal(err)
}
defer db.Close()
csv.Register(db)
const data = "01\n0.10\ne"
err = db.Exec(`
CREATE VIRTUAL TABLE temp.nums USING csv(
data = ` + sqlite3.Quote(data) + `,
schema = 'CREATE TABLE x(a numeric)'
)`)
if err != nil {
t.Fatal(err)
}
stmt, _, err := db.Prepare(`SELECT * FROM temp.nums`)
if err != nil {
t.Fatal(err)
}
defer stmt.Close()
if stmt.Step() {
if got := stmt.ColumnText(0); got != "1" {
t.Errorf("got %q want 1", got)
}
}
if stmt.Step() {
if got := stmt.ColumnText(0); got != "0.1" {
t.Errorf("got %q want 0.1", got)
}
}
if stmt.Step() {
if got := stmt.ColumnText(0); got != "e" {
t.Errorf("got %q want e", got)
}
}
}
func TestRegister_errors(t *testing.T) {
t.Parallel()

54
ext/csv/types.go Normal file
View File

@@ -0,0 +1,54 @@
package csv
import (
_ "embed"
"strings"
"github.com/ncruces/go-sqlite3/util/vtabutil"
)
type affinity byte
const (
blob affinity = 0
text affinity = 1
numeric affinity = 2
integer affinity = 3
real affinity = 4
)
func getColumnAffinities(schema string) ([]affinity, error) {
tab, err := vtabutil.Parse(schema)
if err != nil {
return nil, err
}
defer tab.Close()
types := make([]affinity, tab.NumColumns())
for i := range types {
col := tab.Column(i)
types[i] = getAffinity(col.Type())
}
return types, nil
}
func getAffinity(declType string) affinity {
// https://sqlite.org/datatype3.html#determination_of_column_affinity
if declType == "" {
return blob
}
name := strings.ToUpper(declType)
if strings.Contains(name, "INT") {
return integer
}
if strings.Contains(name, "CHAR") || strings.Contains(name, "CLOB") || strings.Contains(name, "TEXT") {
return text
}
if strings.Contains(name, "BLOB") {
return blob
}
if strings.Contains(name, "REAL") || strings.Contains(name, "FLOA") || strings.Contains(name, "DOUB") {
return real
}
return numeric
}

35
ext/csv/types_test.go Normal file
View File

@@ -0,0 +1,35 @@
package csv
import (
_ "embed"
"testing"
)
func Test_getAffinity(t *testing.T) {
tests := []struct {
decl string
want affinity
}{
{"", blob},
{"INTEGER", integer},
{"TINYINT", integer},
{"TEXT", text},
{"CHAR", text},
{"CLOB", text},
{"BLOB", blob},
{"REAL", real},
{"FLOAT", real},
{"DOUBLE", real},
{"NUMERIC", numeric},
{"DECIMAL", numeric},
{"BOOLEAN", numeric},
{"DATETIME", numeric},
}
for _, tt := range tests {
t.Run(tt.decl, func(t *testing.T) {
if got := getAffinity(tt.decl); got != tt.want {
t.Errorf("getAffinity() = %v, want %v", got, tt.want)
}
})
}
}

View File

@@ -11,7 +11,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/fileio"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
func Test_lsmode(t *testing.T) {

View File

@@ -12,7 +12,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/fileio"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
func Test_fsdir(t *testing.T) {

View File

@@ -10,7 +10,7 @@ import (
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
func Test_writefile(t *testing.T) {

View File

@@ -10,7 +10,7 @@ import (
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
_ "golang.org/x/crypto/blake2b"
_ "golang.org/x/crypto/blake2s"
_ "golang.org/x/crypto/md4"

View File

@@ -34,13 +34,13 @@ func Register(db *sqlite3.Conn) {
// The lines_read function reads from a file or an [io.Reader].
// If a filename is specified, fsys is used to open the file.
func RegisterFS(db *sqlite3.Conn, fsys fs.FS) {
sqlite3.CreateModule[lines](db, "lines", nil,
sqlite3.CreateModule(db, "lines", nil,
func(db *sqlite3.Conn, _, _, _ string, _ ...string) (lines, error) {
err := db.DeclareVTab(`CREATE TABLE x(line TEXT, data HIDDEN)`)
db.VTabConfig(sqlite3.VTAB_INNOCUOUS)
return lines{}, err
})
sqlite3.CreateModule[lines](db, "lines_read", nil,
sqlite3.CreateModule(db, "lines_read", nil,
func(db *sqlite3.Conn, _, _, _ string, _ ...string) (lines, error) {
err := db.DeclareVTab(`CREATE TABLE x(line TEXT, data HIDDEN)`)
db.VTabConfig(sqlite3.VTAB_DIRECTONLY)

View File

@@ -14,7 +14,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/lines"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
func Example() {

View File

@@ -65,7 +65,7 @@ func declare(db *sqlite3.Conn, _, _, _ string, arg ...string) (_ *table, err err
}
if stmt.ColumnCount() != 2 {
return nil, fmt.Errorf("pivot: column definition query expects 2 result columns")
return nil, errors.New("pivot: column definition query expects 2 result columns")
}
for stmt.Step() {
name := sqlite3.QuoteIdentifier(stmt.ColumnText(1))
@@ -83,7 +83,7 @@ func declare(db *sqlite3.Conn, _, _, _ string, arg ...string) (_ *table, err err
}
if stmt.ColumnCount() != 1 {
return nil, fmt.Errorf("pivot: cell query expects 1 result columns")
return nil, errors.New("pivot: cell query expects 1 result columns")
}
if stmt.BindCount() != len(table.keys)+1 {
return nil, fmt.Errorf("pivot: cell query expects %d bound parameters", len(table.keys)+1)

View File

@@ -9,7 +9,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/pivot"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
// https://antonz.org/sqlite-pivot-table/

View File

@@ -8,7 +8,7 @@ package statement
import (
"encoding/json"
"fmt"
"errors"
"strconv"
"strings"
"unsafe"
@@ -29,7 +29,7 @@ type table struct {
func declare(db *sqlite3.Conn, _, _, _ string, arg ...string) (*table, error) {
if len(arg) != 1 {
return nil, fmt.Errorf("statement: wrong number of arguments")
return nil, errors.New("statement: wrong number of arguments")
}
sql := "SELECT * FROM\n" + arg[0]

View File

@@ -8,7 +8,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/statement"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
func Example() {

View File

@@ -41,7 +41,16 @@ https://sqlite.org/lang_aggfunc.html
- [X] `RANK() OVER window`
- [X] `DENSE_RANK() OVER window`
- [X] `PERCENT_RANK() OVER window`
- [ ] `PERCENTILE_CONT(percentile) OVER window`
- [ ] `PERCENTILE_DISC(percentile) OVER window`
https://sqlite.org/windowfunctions.html#builtins
https://sqlite.org/windowfunctions.html#builtins
## Boolean aggregates
- [X] `EVERY(boolean)`
- [X] `SOME(boolean)`
## Additional aggregates
- [X] `MEDIAN(expression)`
- [X] `PERCENTILE_CONT(expression, fraction)`
- [X] `PERCENTILE_DISC(expression, fraction)`

46
ext/stats/boolean.go Normal file
View File

@@ -0,0 +1,46 @@
package stats
import "github.com/ncruces/go-sqlite3"
const (
every = iota
some
)
func newBoolean(kind int) func() sqlite3.AggregateFunction {
return func() sqlite3.AggregateFunction { return &boolean{kind: kind} }
}
type boolean struct {
count int
total int
kind int
}
func (b *boolean) Value(ctx sqlite3.Context) {
if b.kind == every {
ctx.ResultBool(b.count == b.total)
} else {
ctx.ResultBool(b.count > 0)
}
}
func (b *boolean) Step(ctx sqlite3.Context, arg ...sqlite3.Value) {
if arg[0].Type() == sqlite3.NULL {
return
}
if arg[0].Bool() {
b.count++
}
b.total++
}
func (b *boolean) Inverse(ctx sqlite3.Context, arg ...sqlite3.Value) {
if arg[0].Type() == sqlite3.NULL {
return
}
if arg[0].Bool() {
b.count--
}
b.total--
}

74
ext/stats/boolean_test.go Normal file
View File

@@ -0,0 +1,74 @@
package stats_test
import (
"testing"
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/stats"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
func TestRegister_boolean(t *testing.T) {
t.Parallel()
db, err := sqlite3.Open(":memory:")
if err != nil {
t.Fatal(err)
}
defer db.Close()
stats.Register(db)
err = db.Exec(`CREATE TABLE data (x)`)
if err != nil {
t.Fatal(err)
}
err = db.Exec(`INSERT INTO data (x) VALUES (4), (7.0), (13), (NULL), (16), (3.14)`)
if err != nil {
t.Fatal(err)
}
stmt, _, err := db.Prepare(`
SELECT
every(x > 0),
every(x > 10),
some(x > 10),
some(x > 20)
FROM data`)
if err != nil {
t.Fatal(err)
}
if stmt.Step() {
if got := stmt.ColumnBool(0); got != true {
t.Errorf("got %v, want true", got)
}
if got := stmt.ColumnBool(1); got != false {
t.Errorf("got %v, want false", got)
}
if got := stmt.ColumnBool(2); got != true {
t.Errorf("got %v, want true", got)
}
if got := stmt.ColumnBool(3); got != false {
t.Errorf("got %v, want false", got)
}
}
stmt.Close()
stmt, _, err = db.Prepare(`SELECT every(x > 10) OVER (ROWS 1 PRECEDING) FROM data`)
if err != nil {
t.Fatal(err)
}
want := [...]bool{false, false, false, true, true, false}
for i := 0; stmt.Step(); i++ {
if got := stmt.ColumnBool(0); got != want[i] {
t.Errorf("got %v, want %v", got, want[i])
}
if got := stmt.ColumnType(0); got != sqlite3.INTEGER {
t.Errorf("got %v, want INTEGER", got)
}
}
stmt.Close()
}

94
ext/stats/percentile.go Normal file
View File

@@ -0,0 +1,94 @@
package stats
import (
"encoding/json"
"fmt"
"math"
"slices"
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/sort/quick"
)
const (
median = iota
percentile_cont
percentile_disc
)
func newPercentile(kind int) func() sqlite3.AggregateFunction {
return func() sqlite3.AggregateFunction { return &percentile{kind: kind} }
}
type percentile struct {
nums []float64
arg1 []byte
kind int
}
func (q *percentile) Step(ctx sqlite3.Context, arg ...sqlite3.Value) {
if a := arg[0]; a.NumericType() != sqlite3.NULL {
q.nums = append(q.nums, a.Float())
}
if q.kind != median {
q.arg1 = arg[1].Blob(q.arg1[:0])
}
}
func (q *percentile) Inverse(ctx sqlite3.Context, arg ...sqlite3.Value) {
// Implementing inverse allows certain queries that don't really need it to succeed.
ctx.ResultError(util.ErrorString("percentile: may not be used as a window function"))
}
func (q *percentile) Value(ctx sqlite3.Context) {
if len(q.nums) == 0 {
return
}
var (
err error
float float64
floats []float64
)
if q.kind == median {
float, err = getPercentile(q.nums, 0.5, false)
ctx.ResultFloat(float)
} else if err = json.Unmarshal(q.arg1, &float); err == nil {
float, err = getPercentile(q.nums, float, q.kind == percentile_disc)
ctx.ResultFloat(float)
} else if err = json.Unmarshal(q.arg1, &floats); err == nil {
err = getPercentiles(q.nums, floats, q.kind == percentile_disc)
ctx.ResultJSON(floats)
}
if err != nil {
ctx.ResultError(fmt.Errorf("percentile: %w", err))
}
}
func getPercentile(nums []float64, pos float64, disc bool) (float64, error) {
if pos < 0 || pos > 1 {
return 0, util.ErrorString("invalid pos")
}
i, f := math.Modf(pos * float64(len(nums)-1))
m0 := quick.Select(nums, int(i))
if f == 0 || disc {
return m0, nil
}
m1 := slices.Min(nums[int(i)+1:])
return math.FMA(f, m1, -math.FMA(f, m0, -m0)), nil
}
func getPercentiles(nums []float64, pos []float64, disc bool) error {
for i := range pos {
v, err := getPercentile(nums, pos[i], disc)
if err != nil {
return err
}
pos[i] = v
}
return nil
}

View File

@@ -0,0 +1,124 @@
package stats_test
import (
"slices"
"testing"
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/stats"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
func TestRegister_percentile(t *testing.T) {
t.Parallel()
db, err := sqlite3.Open(":memory:")
if err != nil {
t.Fatal(err)
}
defer db.Close()
stats.Register(db)
err = db.Exec(`CREATE TABLE data (x)`)
if err != nil {
t.Fatal(err)
}
err = db.Exec(`INSERT INTO data (x) VALUES (4), (7.0), ('13'), (NULL), (16)`)
if err != nil {
t.Fatal(err)
}
stmt, _, err := db.Prepare(`
SELECT
median(x),
percentile_disc(x, 0.5),
percentile_cont(x, '[0.25, 0.5, 0.75]')
FROM data`)
if err != nil {
t.Fatal(err)
}
if stmt.Step() {
if got := stmt.ColumnFloat(0); got != 10 {
t.Errorf("got %v, want 10", got)
}
if got := stmt.ColumnFloat(1); got != 7 {
t.Errorf("got %v, want 7", got)
}
var got []float64
if err := stmt.ColumnJSON(2, &got); err != nil {
t.Error(err)
}
if !slices.Equal(got, []float64{6.25, 10, 13.75}) {
t.Errorf("got %v, want [6.25 10 13.75]", got)
}
}
stmt.Close()
stmt, _, err = db.Prepare(`
SELECT
median(x),
percentile_disc(x, 0.5),
percentile_cont(x, '[0.25, 0.5, 0.75]')
FROM data
WHERE x < 5`)
if err != nil {
t.Fatal(err)
}
if stmt.Step() {
if got := stmt.ColumnFloat(0); got != 4 {
t.Errorf("got %v, want 4", got)
}
if got := stmt.ColumnFloat(1); got != 4 {
t.Errorf("got %v, want 4", got)
}
var got []float64
if err := stmt.ColumnJSON(2, &got); err != nil {
t.Error(err)
}
if !slices.Equal(got, []float64{4, 4, 4}) {
t.Errorf("got %v, want [4 4 4]", got)
}
}
stmt.Close()
stmt, _, err = db.Prepare(`
SELECT
median(x),
percentile_disc(x, 0.5),
percentile_cont(x, '[0.25, 0.5, 0.75]')
FROM data
WHERE x < 0`)
if err != nil {
t.Fatal(err)
}
if stmt.Step() {
if got := stmt.ColumnType(0); got != sqlite3.NULL {
t.Error("want NULL")
}
if got := stmt.ColumnType(1); got != sqlite3.NULL {
t.Error("want NULL")
}
if got := stmt.ColumnType(2); got != sqlite3.NULL {
t.Error("want NULL")
}
}
stmt.Close()
stmt, _, err = db.Prepare(`
SELECT
percentile_disc(x, -2),
percentile_cont(x, +2),
percentile_cont(x, ''),
percentile_cont(x, '[100]')
FROM data`)
if err != nil {
t.Fatal(err)
}
if stmt.Step() {
t.Error("want error")
}
stmt.Close()
}

View File

@@ -18,6 +18,11 @@
// - regr_slope: slope of the least-squares-fit linear equation
// - regr_intercept: y-intercept of the least-squares-fit linear equation
// - regr_json: all regr stats in a JSON object
// - percentile_disc: discrete percentile
// - percentile_cont: continuous percentile
// - median: median value
// - every: boolean and
// - some: boolean or
//
// These join the [Built-in Aggregate Functions]:
// - count: count rows/values
@@ -26,9 +31,16 @@
// - min: minimum value
// - max: maximum value
//
// And the [Built-in Window Functions]:
// - rank: rank of the current row with gaps
// - dense_rank: rank of the current row without gaps
// - percent_rank: relative rank of the row
// - cume_dist: cumulative distribution
//
// See: [ANSI SQL Aggregate Functions]
//
// [Built-in Aggregate Functions]: https://sqlite.org/lang_aggfunc.html
// [Built-in Window Functions]: https://sqlite.org/windowfunctions.html#builtins
// [ANSI SQL Aggregate Functions]: https://www.oreilly.com/library/view/sql-in-a/9780596155322/ch04s02.html
package stats
@@ -54,6 +66,11 @@ func Register(db *sqlite3.Conn) {
db.CreateWindowFunction("regr_intercept", 2, flags, newCovariance(regr_intercept))
db.CreateWindowFunction("regr_count", 2, flags, newCovariance(regr_count))
db.CreateWindowFunction("regr_json", 2, flags, newCovariance(regr_json))
db.CreateWindowFunction("median", 1, flags, newPercentile(median))
db.CreateWindowFunction("percentile_cont", 2, flags, newPercentile(percentile_cont))
db.CreateWindowFunction("percentile_disc", 2, flags, newPercentile(percentile_disc))
db.CreateWindowFunction("every", 1, flags, newBoolean(every))
db.CreateWindowFunction("some", 1, flags, newBoolean(some))
}
const (

View File

@@ -7,7 +7,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/stats"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
func TestRegister_variance(t *testing.T) {
@@ -40,8 +40,6 @@ func TestRegister_variance(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer stmt.Close()
if stmt.Step() {
if got := stmt.ColumnFloat(0); got != 40 {
t.Errorf("got %v, want 40", got)
@@ -62,24 +60,23 @@ func TestRegister_variance(t *testing.T) {
t.Errorf("got %v, want √22.5", got)
}
}
stmt.Close()
{
stmt, _, err := db.Prepare(`SELECT var_samp(x) OVER (ROWS 1 PRECEDING) FROM data`)
if err != nil {
t.Fatal(err)
stmt, _, err = db.Prepare(`SELECT var_samp(x) OVER (ROWS 1 PRECEDING) FROM data`)
if err != nil {
t.Fatal(err)
}
want := [...]float64{0, 4.5, 18, 0, 0}
for i := 0; stmt.Step(); i++ {
if got := stmt.ColumnFloat(0); got != want[i] {
t.Errorf("got %v, want %v", got, want[i])
}
defer stmt.Close()
want := [...]float64{0, 4.5, 18, 0, 0}
for i := 0; stmt.Step(); i++ {
if got := stmt.ColumnFloat(0); got != want[i] {
t.Errorf("got %v, want %v", got, want[i])
}
if got := stmt.ColumnType(0); (got == sqlite3.FLOAT) != (want[i] != 0) {
t.Errorf("got %v, want %v", got, want[i])
}
if got := stmt.ColumnType(0); (got == sqlite3.FLOAT) != (want[i] != 0) {
t.Errorf("got %v, want %v", got, want[i])
}
}
stmt.Close()
}
func TestRegister_covariance(t *testing.T) {
@@ -113,8 +110,6 @@ func TestRegister_covariance(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer stmt.Close()
if stmt.Step() {
if got := stmt.ColumnFloat(0); got != 0.9881049293224639 {
t.Errorf("got %v, want 0.9881049293224639", got)
@@ -159,27 +154,29 @@ func TestRegister_covariance(t *testing.T) {
t.Errorf("got %v, want 5", got)
}
}
stmt.Close()
{
stmt, _, err := db.Prepare(`SELECT covar_samp(y, x) OVER (ROWS 1 PRECEDING) FROM data`)
if err != nil {
t.Fatal(err)
stmt, _, err = db.Prepare(`SELECT covar_samp(y, x) OVER (ROWS 1 PRECEDING) FROM data`)
if err != nil {
t.Fatal(err)
}
want := [...]float64{0, 10, 30, 75, 22.5}
for i := 0; stmt.Step(); i++ {
if got := stmt.ColumnFloat(0); got != want[i] {
t.Errorf("got %v, want %v", got, want[i])
}
defer stmt.Close()
want := [...]float64{0, 10, 30, 75, 22.5}
for i := 0; stmt.Step(); i++ {
if got := stmt.ColumnFloat(0); got != want[i] {
t.Errorf("got %v, want %v", got, want[i])
}
if got := stmt.ColumnType(0); (got == sqlite3.FLOAT) != (want[i] != 0) {
t.Errorf("got %v, want %v", got, want[i])
}
if got := stmt.ColumnType(0); (got == sqlite3.FLOAT) != (want[i] != 0) {
t.Errorf("got %v, want %v", got, want[i])
}
}
stmt.Close()
}
func Benchmark_average(b *testing.B) {
sqlite3.Initialize()
b.ResetTimer()
db, err := sqlite3.Open(":memory:")
if err != nil {
b.Fatal(err)
@@ -211,6 +208,9 @@ func Benchmark_average(b *testing.B) {
}
func Benchmark_variance(b *testing.B) {
sqlite3.Initialize()
b.ResetTimer()
db, err := sqlite3.Open(":memory:")
if err != nil {
b.Fatal(err)

View File

@@ -7,7 +7,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
func TestRegister(t *testing.T) {

View File

@@ -7,7 +7,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/zorder"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
func TestRegister_zorder(t *testing.T) {

View File

@@ -9,6 +9,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/unicode"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
func ExampleConn_CreateCollation() {

14
go.mod
View File

@@ -3,16 +3,16 @@ module github.com/ncruces/go-sqlite3
go 1.21
require (
github.com/dchest/siphash v1.2.3
github.com/ncruces/julianday v1.0.0
github.com/ncruces/sort v0.1.2
github.com/psanford/httpreadat v0.1.0
github.com/tetratelabs/wazero v1.7.1
golang.org/x/crypto v0.22.0
github.com/tetratelabs/wazero v1.7.3
golang.org/x/crypto v0.24.0
golang.org/x/sync v0.7.0
golang.org/x/sys v0.19.0
golang.org/x/text v0.14.0
lukechampine.com/adiantum v1.0.0
golang.org/x/sys v0.21.0
golang.org/x/text v0.16.0
lukechampine.com/adiantum v1.1.1
)
require github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect
retract v0.4.0 // tagged from the wrong branch

33
go.sum
View File

@@ -1,25 +1,20 @@
github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY=
github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA=
github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA=
github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/ncruces/sort v0.1.2 h1:zKQ9CA4fpHPF6xsUhRTfi5EEryspuBpe/QA4VWQOV1U=
github.com/ncruces/sort v0.1.2/go.mod h1:vEJUTBJtebIuCMmXD18GKo5GJGhsay+xZFOoBEIXFmE=
github.com/psanford/httpreadat v0.1.0 h1:VleW1HS2zO7/4c7c7zNl33fO6oYACSagjJIyMIwZLUE=
github.com/psanford/httpreadat v0.1.0/go.mod h1:Zg7P+TlBm3bYbyHTKv/EdtSJZn3qwbPwpfZ/I9GKCRE=
github.com/tetratelabs/wazero v1.7.1 h1:QtSfd6KLc41DIMpDYlJdoMc6k7QTN246DM2+n2Y/Dx8=
github.com/tetratelabs/wazero v1.7.1/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
github.com/tetratelabs/wazero v1.7.3 h1:PBH5KVahrt3S2AHgEjKu4u+LlDbbk+nsGE3KLucy6Rw=
github.com/tetratelabs/wazero v1.7.3/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
lukechampine.com/adiantum v1.0.0 h1:xxLFgKHyno8ES1XiKLbQfU9DGiMaM2xsIJI2czgT7es=
lukechampine.com/adiantum v1.0.0/go.mod h1:kjMpBiZFjVX/FeEYcN81jyt3//7J3XjJgH9OkAXV4n0=
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
lukechampine.com/adiantum v1.1.1 h1:4fp6gTxWCqpEbLy40ExiYDDED3oUNWx5cTqBCtPdZqA=
lukechampine.com/adiantum v1.1.1/go.mod h1:LrAYVnTYLnUtE/yMp5bQr0HstAf060YUF8nM0B6+rUw=

View File

@@ -1,8 +1,7 @@
golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=

View File

@@ -3,14 +3,14 @@ module github.com/ncruces/go-sqlite3/gormlite
go 1.21
require (
github.com/ncruces/go-sqlite3 v0.14.0
gorm.io/gorm v1.25.9
github.com/ncruces/go-sqlite3 v0.16.1
gorm.io/gorm v1.25.10
)
require (
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/ncruces/julianday v1.0.0 // indirect
github.com/tetratelabs/wazero v1.7.1 // indirect
golang.org/x/sys v0.19.0 // indirect
github.com/tetratelabs/wazero v1.7.3 // indirect
golang.org/x/sys v0.21.0 // indirect
)

View File

@@ -2,15 +2,15 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/ncruces/go-sqlite3 v0.14.0 h1:R1Jmnc7o5ECQeZPNzbLHfE8vz1DLewV+bJypHSad354=
github.com/ncruces/go-sqlite3 v0.14.0/go.mod h1:NRmOFatwnQaZq8niw0f3k/j3a0yUVt250qcVeDuyANY=
github.com/ncruces/go-sqlite3 v0.16.1 h1:1wHv7s8y+fWK44UIliotJ42ZV41A5T0sjIAqGmnMrkc=
github.com/ncruces/go-sqlite3 v0.16.1/go.mod h1:feFXbBcbLtxNk6XWG1ROt8MS9+E45yCW3G8o4ixIqZ8=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/tetratelabs/wazero v1.7.1 h1:QtSfd6KLc41DIMpDYlJdoMc6k7QTN246DM2+n2Y/Dx8=
github.com/tetratelabs/wazero v1.7.1/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
gorm.io/gorm v1.25.9 h1:wct0gxZIELDk8+ZqF/MVnHLkA1rvYlBWUMv2EdsK1g8=
gorm.io/gorm v1.25.9/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
github.com/tetratelabs/wazero v1.7.3 h1:PBH5KVahrt3S2AHgEjKu4u+LlDbbk+nsGE3KLucy6Rw=
github.com/tetratelabs/wazero v1.7.3/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
gorm.io/gorm v1.25.10 h1:dQpO+33KalOA+aFYGlK+EfxcI5MbO7EP2yYygwh9h+s=
gorm.io/gorm v1.25.10/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=

View File

@@ -9,7 +9,7 @@ import (
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
func TestDialector(t *testing.T) {

View File

@@ -7,7 +7,7 @@ rm -rf gorm/ tests/
go work use -r .
go test
git clone --branch v1.25.9 --filter=blob:none https://github.com/go-gorm/gorm.git
git clone --branch v1.25.10 --filter=blob:none https://github.com/go-gorm/gorm.git
mv gorm/tests tests
rm -rf gorm/

View File

@@ -0,0 +1,9 @@
//go:build !(unix || windows) || sqlite3_nosys
package alloc
import "github.com/tetratelabs/wazero/experimental"
func Virtual(cap, max uint64) experimental.LinearMemory {
return Slice(cap, max)
}

View File

@@ -0,0 +1,24 @@
//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64) || sqlite3_noshm || sqlite3_nosys
package alloc
import "github.com/tetratelabs/wazero/experimental"
func Slice(cap, _ uint64) experimental.LinearMemory {
return &sliceMemory{make([]byte, 0, cap)}
}
type sliceMemory struct {
buf []byte
}
func (b *sliceMemory) Free() {}
func (b *sliceMemory) Reallocate(size uint64) []byte {
if cap := uint64(cap(b.buf)); size > cap {
b.buf = append(b.buf[:cap], make([]byte, size-cap)...)
} else {
b.buf = b.buf[:size]
}
return b.buf
}

View File

@@ -1,6 +1,6 @@
//go:build unix
//go:build unix && !sqlite3_nosys
package util
package alloc
import (
"math"
@@ -9,30 +9,24 @@ import (
"golang.org/x/sys/unix"
)
func mmappedAllocator(cap, max uint64) experimental.LinearMemory {
func Virtual(_, max uint64) experimental.LinearMemory {
// Round up to the page size.
rnd := uint64(unix.Getpagesize() - 1)
max = (max + rnd) &^ rnd
cap = (cap + rnd) &^ rnd
if max > math.MaxInt {
// This ensures int(max) overflows to a negative value,
// and unix.Mmap returns EINVAL.
max = math.MaxUint64
}
// Reserve max bytes of address space, to ensure we won't need to move it.
// A protected, private, anonymous mapping should not commit memory.
b, err := unix.Mmap(-1, 0, int(max), unix.PROT_NONE, unix.MAP_PRIVATE|unix.MAP_ANON)
if err != nil {
panic(err)
}
// Commit the initial cap bytes of memory.
err = unix.Mprotect(b[:cap], unix.PROT_READ|unix.PROT_WRITE)
if err != nil {
unix.Munmap(b)
panic(err)
}
return &mmappedMemory{buf: b[:cap]}
return &mmappedMemory{buf: b[:0]}
}
// The slice covers the entire mmapped memory:
@@ -43,7 +37,9 @@ type mmappedMemory struct {
}
func (m *mmappedMemory) Reallocate(size uint64) []byte {
if com := uint64(len(m.buf)); com < size {
com := uint64(len(m.buf))
res := uint64(cap(m.buf))
if com < size && size < res {
// Round up to the page size.
rnd := uint64(unix.Getpagesize() - 1)
new := (size + rnd) &^ rnd

View File

@@ -0,0 +1,76 @@
//go:build !sqlite3_nosys
package alloc
import (
"math"
"reflect"
"unsafe"
"github.com/tetratelabs/wazero/experimental"
"golang.org/x/sys/windows"
)
func Virtual(_, max uint64) experimental.LinearMemory {
// Round up to the page size.
rnd := uint64(windows.Getpagesize() - 1)
max = (max + rnd) &^ rnd
if max > math.MaxInt {
// This ensures uintptr(max) overflows to a large value,
// and windows.VirtualAlloc returns an error.
max = math.MaxUint64
}
// Reserve max bytes of address space, to ensure we won't need to move it.
// This does not commit memory.
r, err := windows.VirtualAlloc(0, uintptr(max), windows.MEM_RESERVE, windows.PAGE_READWRITE)
if err != nil {
panic(err)
}
mem := virtualMemory{addr: r}
// SliceHeader, although deprecated, avoids a go vet warning.
sh := (*reflect.SliceHeader)(unsafe.Pointer(&mem.buf))
sh.Cap = int(max)
sh.Data = r
return &mem
}
// The slice covers the entire mmapped memory:
// - len(buf) is the already committed memory,
// - cap(buf) is the reserved address space.
type virtualMemory struct {
buf []byte
addr uintptr
}
func (m *virtualMemory) Reallocate(size uint64) []byte {
com := uint64(len(m.buf))
res := uint64(cap(m.buf))
if com < size && size < res {
// Round up to the page size.
rnd := uint64(windows.Getpagesize() - 1)
new := (size + rnd) &^ rnd
// Commit additional memory up to new bytes.
_, err := windows.VirtualAlloc(m.addr, uintptr(new), windows.MEM_COMMIT, windows.PAGE_READWRITE)
if err != nil {
panic(err)
}
// Update committed memory.
m.buf = m.buf[:new]
}
// Limit returned capacity because bytes beyond
// len(m.buf) have not yet been committed.
return m.buf[:size:len(m.buf)]
}
func (m *virtualMemory) Free() {
err := windows.VirtualFree(m.addr, 0, windows.MEM_RELEASE)
if err != nil {
panic(err)
}
m.addr = 0
}

View File

@@ -0,0 +1,29 @@
package testcfg
import (
"math/bits"
"os"
"path/filepath"
"github.com/ncruces/go-sqlite3"
"github.com/tetratelabs/wazero"
)
func init() {
if bits.UintSize < 64 {
return
}
sqlite3.RuntimeConfig = wazero.NewRuntimeConfig().
WithMemoryCapacityFromMax(true).
WithMemoryLimitPages(1024)
if os.Getenv("CI") != "" {
path := filepath.Join(os.TempDir(), "wazero")
if err := os.MkdirAll(path, 0777); err == nil {
if cache, err := wazero.NewCompilationCacheWithDir(path); err == nil {
sqlite3.RuntimeConfig.WithCompilationCache(cache)
}
}
}
}

View File

@@ -1,6 +1,6 @@
package util
// https://sqlite.com/matrix/rescode.html
// https://sqlite.com/rescode.html
const (
OK = 0 /* Successful result */

View File

@@ -26,7 +26,7 @@ func (j JSON) Scan(value any) error {
buf = v.AppendFormat(buf, time.RFC3339Nano)
buf = append(buf, '"')
case nil:
buf = append(buf, "null"...)
buf = []byte("null")
default:
panic(AssertErr())
}

View File

@@ -1,4 +1,4 @@
//go:build (darwin || linux) && (amd64 || arm64 || riscv64) && !(sqlite3_flock || sqlite3_noshm || sqlite3_nosys)
//go:build unix && (amd64 || arm64 || riscv64) && !(sqlite3_noshm || sqlite3_nosys)
package util
@@ -7,14 +7,15 @@ import (
"os"
"unsafe"
"github.com/ncruces/go-sqlite3/internal/alloc"
"github.com/tetratelabs/wazero/api"
"github.com/tetratelabs/wazero/experimental"
"golang.org/x/sys/unix"
)
func withMmappedAllocator(ctx context.Context) context.Context {
func withAllocator(ctx context.Context) context.Context {
return experimental.WithMemoryAllocator(ctx,
experimental.MemoryAllocatorFunc(mmappedAllocator))
experimental.MemoryAllocatorFunc(alloc.Virtual))
}
type mmapState struct {

View File

@@ -1,11 +1,22 @@
//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64) || sqlite3_flock || sqlite3_noshm || sqlite3_nosys
//go:build !unix || !(amd64 || arm64 || riscv64) || sqlite3_noshm || sqlite3_nosys
package util
import "context"
import (
"context"
"github.com/ncruces/go-sqlite3/internal/alloc"
"github.com/tetratelabs/wazero/experimental"
)
type mmapState struct{}
func withMmappedAllocator(ctx context.Context) context.Context {
return ctx
func withAllocator(ctx context.Context) context.Context {
return experimental.WithMemoryAllocator(ctx,
experimental.MemoryAllocatorFunc(func(cap, max uint64) experimental.LinearMemory {
if cap == max {
return alloc.Virtual(cap, max)
}
return alloc.Slice(cap, max)
}))
}

View File

@@ -14,7 +14,7 @@ type moduleState struct {
func NewContext(ctx context.Context) context.Context {
state := new(moduleState)
ctx = withMmappedAllocator(ctx)
ctx = withAllocator(ctx)
ctx = experimental.WithCloseNotifier(ctx, state)
ctx = context.WithValue(ctx, moduleKey{}, state)
return ctx

View File

@@ -5,7 +5,8 @@ import "github.com/ncruces/go-sqlite3/internal/util"
// JSON returns a value that can be used as an argument to
// [database/sql.DB.Exec], [database/sql.Row.Scan] and similar methods to
// store value as JSON, or decode JSON into value.
// JSON should NOT be used with [BindJSON] or [ResultJSON].
// JSON should NOT be used with [Stmt.BindJSON], [Stmt.ColumnJSON],
// [Value.JSON], or [Context.ResultJSON].
func JSON(value any) any {
return util.JSON{Value: value}
}

View File

@@ -4,7 +4,8 @@ import "github.com/ncruces/go-sqlite3/internal/util"
// Pointer returns a pointer to a value that can be used as an argument to
// [database/sql.DB.Exec] and similar methods.
// Pointer should NOT be used with [BindPointer] or [ResultPointer].
// Pointer should NOT be used with [Stmt.BindPointer],
// [Value.Pointer], or [Context.ResultPointer].
//
// https://sqlite.org/bindptr.html
func Pointer[T any](value T) any {

View File

@@ -28,6 +28,14 @@ var (
RuntimeConfig wazero.RuntimeConfig
)
// Initialize decodes and compiles the SQLite Wasm binary.
// This is called implicitly when the first connection is openned,
// but is potentially slow, so you may want to call it at a more convenient time.
func Initialize() error {
instance.once.Do(compileSQLite)
return instance.err
}
var instance struct {
runtime wazero.Runtime
compiled wazero.CompiledModule
@@ -79,9 +87,8 @@ type sqlite struct {
}
func instantiateSQLite() (sqlt *sqlite, err error) {
instance.once.Do(compileSQLite)
if instance.err != nil {
return nil, instance.err
if err := Initialize(); err != nil {
return nil, err
}
sqlt = new(sqlite)
@@ -289,8 +296,9 @@ func (a *arena) string(s string) uint32 {
}
func exportCallbacks(env wazero.HostModuleBuilder) wazero.HostModuleBuilder {
util.ExportFuncIII(env, "go_busy_handler", busyCallback)
util.ExportFuncII(env, "go_progress_handler", progressCallback)
util.ExportFuncIIII(env, "go_busy_timeout", timeoutCallback)
util.ExportFuncIII(env, "go_busy_handler", busyCallback)
util.ExportFuncII(env, "go_commit_hook", commitCallback)
util.ExportFuncVI(env, "go_rollback_hook", rollbackCallback)
util.ExportFuncVIIIIJ(env, "go_update_hook", updateCallback)

View File

@@ -0,0 +1,13 @@
# Replace sqliteDefaultBusyCallback.
# This patch allows Go to handle (and interrupt) sqlite3_busy_timeout.
--- sqlite3.c.orig
+++ sqlite3.c
@@ -181614,7 +181614,7 @@
if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
#endif
if( ms>0 ){
- sqlite3_busy_handler(db, (int(*)(void*,int))sqliteDefaultBusyCallback,
+ sqlite3_busy_handler(db, (int(*)(void*,int))sqliteBusyCallback,
(void*)db);
db->busyTimeout = ms;
}else{

View File

@@ -1,556 +0,0 @@
# Backport from 3.46.
# https://sqlite.org/draft/releaselog/current.html
--- sqlite3.c.orig
+++ sqlite3.c
@@ -71,13 +71,14 @@ struct DateTime {
int tz; /* Timezone offset in minutes */
double s; /* Seconds */
char validJD; /* True (1) if iJD is valid */
- char rawS; /* Raw numeric value stored in s */
char validYMD; /* True (1) if Y,M,D are valid */
char validHMS; /* True (1) if h,m,s are valid */
- char validTZ; /* True (1) if tz is valid */
- char tzSet; /* Timezone was set explicitly */
- char isError; /* An overflow has occurred */
- char useSubsec; /* Display subsecond precision */
+ char nFloor; /* Days to implement "floor" */
+ unsigned rawS : 1; /* Raw numeric value stored in s */
+ unsigned isError : 1; /* An overflow has occurred */
+ unsigned useSubsec : 1; /* Display subsecond precision */
+ unsigned isUtc : 1; /* Time is known to be UTC */
+ unsigned isLocal : 1; /* Time is known to be localtime */
};
@@ -175,6 +176,8 @@ static int parseTimezone(const char *zDate, DateTime *p){
sgn = +1;
}else if( c=='Z' || c=='z' ){
zDate++;
+ p->isLocal = 0;
+ p->isUtc = 1;
goto zulu_time;
}else{
return c!=0;
@@ -187,7 +190,6 @@ static int parseTimezone(const char *zDate, DateTime *p){
p->tz = sgn*(nMn + nHr*60);
zulu_time:
while( sqlite3Isspace(*zDate) ){ zDate++; }
- p->tzSet = 1;
return *zDate!=0;
}
@@ -231,7 +233,6 @@ static int parseHhMmSs(const char *zDate, DateTime *p){
p->m = m;
p->s = s + ms;
if( parseTimezone(zDate, p) ) return 1;
- p->validTZ = (p->tz!=0)?1:0;
return 0;
}
@@ -278,15 +279,40 @@ static void computeJD(DateTime *p){
p->validJD = 1;
if( p->validHMS ){
p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000 + 0.5);
- if( p->validTZ ){
+ if( p->tz ){
p->iJD -= p->tz*60000;
p->validYMD = 0;
p->validHMS = 0;
- p->validTZ = 0;
+ p->tz = 0;
+ p->isUtc = 1;
+ p->isLocal = 0;
}
}
}
+/*
+** Given the YYYY-MM-DD information current in p, determine if there
+** is day-of-month overflow and set nFloor to the number of days that
+** would need to be subtracted from the date in order to bring the
+** date back to the end of the month.
+*/
+static void computeFloor(DateTime *p){
+ assert( p->validYMD || p->isError );
+ assert( p->D>=0 && p->D<=31 );
+ assert( p->M>=0 && p->M<=12 );
+ if( p->D<=28 ){
+ p->nFloor = 0;
+ }else if( (1<<p->M) & 0x15aa ){
+ p->nFloor = 0;
+ }else if( p->M!=2 ){
+ p->nFloor = (p->D==31);
+ }else if( p->Y%4!=0 || (p->Y%100==0 && p->Y%400!=0) ){
+ p->nFloor = p->D - 28;
+ }else{
+ p->nFloor = p->D - 29;
+ }
+}
+
/*
** Parse dates of the form
**
@@ -325,12 +351,16 @@ static int parseYyyyMmDd(const char *zDate, DateTime *p){
p->Y = neg ? -Y : Y;
p->M = M;
p->D = D;
- if( p->validTZ ){
+ computeFloor(p);
+ if( p->tz ){
computeJD(p);
}
return 0;
}
+
+static void clearYMD_HMS_TZ(DateTime *p); /* Forward declaration */
+
/*
** Set the time to the current time reported by the VFS.
**
@@ -340,6 +370,9 @@ static int setDateTimeToCurrent(sqlite3_context *context, DateTime *p){
p->iJD = sqlite3StmtCurrentTime(context);
if( p->iJD>0 ){
p->validJD = 1;
+ p->isUtc = 1;
+ p->isLocal = 0;
+ clearYMD_HMS_TZ(p);
return 0;
}else{
return 1;
@@ -478,7 +511,7 @@ static void computeYMD_HMS(DateTime *p){
static void clearYMD_HMS_TZ(DateTime *p){
p->validYMD = 0;
p->validHMS = 0;
- p->validTZ = 0;
+ p->tz = 0;
}
#ifndef SQLITE_OMIT_LOCALTIME
@@ -610,7 +643,7 @@ static int toLocaltime(
p->validHMS = 1;
p->validJD = 0;
p->rawS = 0;
- p->validTZ = 0;
+ p->tz = 0;
p->isError = 0;
return SQLITE_OK;
}
@@ -630,12 +663,12 @@ static const struct {
float rLimit; /* Maximum NNN value for this transform */
float rXform; /* Constant used for this transform */
} aXformType[] = {
- { 6, "second", 4.6427e+14, 1.0 },
- { 6, "minute", 7.7379e+12, 60.0 },
- { 4, "hour", 1.2897e+11, 3600.0 },
- { 3, "day", 5373485.0, 86400.0 },
- { 5, "month", 176546.0, 2592000.0 },
- { 4, "year", 14713.0, 31536000.0 },
+ /* 0 */ { 6, "second", 4.6427e+14, 1.0 },
+ /* 1 */ { 6, "minute", 7.7379e+12, 60.0 },
+ /* 2 */ { 4, "hour", 1.2897e+11, 3600.0 },
+ /* 3 */ { 3, "day", 5373485.0, 86400.0 },
+ /* 4 */ { 5, "month", 176546.0, 30.0*86400.0 },
+ /* 5 */ { 4, "year", 14713.0, 365.0*86400.0 },
};
/*
@@ -667,14 +700,20 @@ static void autoAdjustDate(DateTime *p){
** NNN.NNNN seconds
** NNN months
** NNN years
+** +/-YYYY-MM-DD HH:MM:SS.SSS
+** ceiling
+** floor
** start of month
** start of year
** start of week
** start of day
** weekday N
** unixepoch
+** auto
** localtime
** utc
+** subsec
+** subsecond
**
** Return 0 on success and 1 if there is any kind of error. If the error
** is in a system call (i.e. localtime()), then an error message is written
@@ -705,6 +744,37 @@ static int parseModifier(
}
break;
}
+ case 'c': {
+ /*
+ ** ceiling
+ **
+ ** Resolve day-of-month overflow by rolling forward into the next
+ ** month. As this is the default action, this modifier is really
+ ** a no-op that is only included for symmetry. See "floor".
+ */
+ if( sqlite3_stricmp(z, "ceiling")==0 ){
+ computeJD(p);
+ clearYMD_HMS_TZ(p);
+ rc = 0;
+ p->nFloor = 0;
+ }
+ break;
+ }
+ case 'f': {
+ /*
+ ** floor
+ **
+ ** Resolve day-of-month overflow by rolling back to the end of the
+ ** previous month.
+ */
+ if( sqlite3_stricmp(z, "floor")==0 ){
+ computeJD(p);
+ p->iJD -= p->nFloor*86400000;
+ clearYMD_HMS_TZ(p);
+ rc = 0;
+ }
+ break;
+ }
case 'j': {
/*
** julianday
@@ -731,7 +801,9 @@ static int parseModifier(
** show local time.
*/
if( sqlite3_stricmp(z, "localtime")==0 && sqlite3NotPureFunc(pCtx) ){
- rc = toLocaltime(p, pCtx);
+ rc = p->isLocal ? SQLITE_OK : toLocaltime(p, pCtx);
+ p->isUtc = 0;
+ p->isLocal = 1;
}
break;
}
@@ -756,7 +828,7 @@ static int parseModifier(
}
#ifndef SQLITE_OMIT_LOCALTIME
else if( sqlite3_stricmp(z, "utc")==0 && sqlite3NotPureFunc(pCtx) ){
- if( p->tzSet==0 ){
+ if( p->isUtc==0 ){
i64 iOrigJD; /* Original localtime */
i64 iGuess; /* Guess at the corresponding utc time */
int cnt = 0; /* Safety to prevent infinite loop */
@@ -779,7 +851,8 @@ static int parseModifier(
memset(p, 0, sizeof(*p));
p->iJD = iGuess;
p->validJD = 1;
- p->tzSet = 1;
+ p->isUtc = 1;
+ p->isLocal = 0;
}
rc = SQLITE_OK;
}
@@ -799,7 +872,7 @@ static int parseModifier(
&& r>=0.0 && r<7.0 && (n=(int)r)==r ){
sqlite3_int64 Z;
computeYMD_HMS(p);
- p->validTZ = 0;
+ p->tz = 0;
p->validJD = 0;
computeJD(p);
Z = ((p->iJD + 129600000)/86400000) % 7;
@@ -839,7 +912,7 @@ static int parseModifier(
p->h = p->m = 0;
p->s = 0.0;
p->rawS = 0;
- p->validTZ = 0;
+ p->tz = 0;
p->validJD = 0;
if( sqlite3_stricmp(z,"month")==0 ){
p->D = 1;
@@ -910,6 +983,7 @@ static int parseModifier(
x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12;
p->Y += x;
p->M -= x*12;
+ computeFloor(p);
computeJD(p);
p->validHMS = 0;
p->validYMD = 0;
@@ -956,11 +1030,12 @@ static int parseModifier(
z += n;
while( sqlite3Isspace(*z) ) z++;
n = sqlite3Strlen30(z);
- if( n>10 || n<3 ) break;
+ if( n<3 || n>10 ) break;
if( sqlite3UpperToLower[(u8)z[n-1]]=='s' ) n--;
computeJD(p);
assert( rc==1 );
rRounder = r<0 ? -0.5 : +0.5;
+ p->nFloor = 0;
for(i=0; i<ArraySize(aXformType); i++){
if( aXformType[i].nName==n
&& sqlite3_strnicmp(aXformType[i].zName, z, n)==0
@@ -968,21 +1043,24 @@ static int parseModifier(
){
switch( i ){
case 4: { /* Special processing to add months */
- assert( strcmp(aXformType[i].zName,"month")==0 );
+ assert( strcmp(aXformType[4].zName,"month")==0 );
computeYMD_HMS(p);
p->M += (int)r;
x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12;
p->Y += x;
p->M -= x*12;
+ computeFloor(p);
p->validJD = 0;
r -= (int)r;
break;
}
case 5: { /* Special processing to add years */
int y = (int)r;
- assert( strcmp(aXformType[i].zName,"year")==0 );
+ assert( strcmp(aXformType[5].zName,"year")==0 );
computeYMD_HMS(p);
+ assert( p->M>=0 && p->M<=12 );
p->Y += y;
+ computeFloor(p);
p->validJD = 0;
r -= (int)r;
break;
@@ -1236,22 +1314,83 @@ static void dateFunc(
}
}
+/*
+** Compute the number of days after the most recent January 1.
+**
+** In other words, compute the zero-based day number for the
+** current year:
+**
+** Jan01 = 0, Jan02 = 1, ..., Jan31 = 30, Feb01 = 31, ...
+** Dec31 = 364 or 365.
+*/
+static int daysAfterJan01(DateTime *pDate){
+ DateTime jan01 = *pDate;
+ assert( jan01.validYMD );
+ assert( jan01.validHMS );
+ assert( pDate->validJD );
+ jan01.validJD = 0;
+ jan01.M = 1;
+ jan01.D = 1;
+ computeJD(&jan01);
+ return (int)((pDate->iJD-jan01.iJD+43200000)/86400000);
+}
+
+/*
+** Return the number of days after the most recent Monday.
+**
+** In other words, return the day of the week according
+** to this code:
+**
+** 0=Monday, 1=Tuesday, 2=Wednesday, ..., 6=Sunday.
+*/
+static int daysAfterMonday(DateTime *pDate){
+ assert( pDate->validJD );
+ return (int)((pDate->iJD+43200000)/86400000) % 7;
+}
+
+/*
+** Return the number of days after the most recent Sunday.
+**
+** In other words, return the day of the week according
+** to this code:
+**
+** 0=Sunday, 1=Monday, 2=Tues, ..., 6=Saturday
+*/
+static int daysAfterSunday(DateTime *pDate){
+ assert( pDate->validJD );
+ return (int)((pDate->iJD+129600000)/86400000) % 7;
+}
+
/*
** strftime( FORMAT, TIMESTRING, MOD, MOD, ...)
**
** Return a string described by FORMAT. Conversions as follows:
**
-** %d day of month
+** %d day of month 01-31
+** %e day of month 1-31
** %f ** fractional seconds SS.SSS
+** %F ISO date. YYYY-MM-DD
+** %G ISO year corresponding to %V 0000-9999.
+** %g 2-digit ISO year corresponding to %V 00-99
** %H hour 00-24
-** %j day of year 000-366
+** %k hour 0-24 (leading zero converted to space)
+** %I hour 01-12
+** %j day of year 001-366
** %J ** julian day number
+** %l hour 1-12 (leading zero converted to space)
** %m month 01-12
** %M minute 00-59
+** %p "am" or "pm"
+** %P "AM" or "PM"
+** %R time as HH:MM
** %s seconds since 1970-01-01
** %S seconds 00-59
-** %w day of week 0-6 Sunday==0
-** %W week of year 00-53
+** %T time as HH:MM:SS
+** %u day of week 1-7 Monday==1, Sunday==7
+** %w day of week 0-6 Sunday==0, Monday==1
+** %U week of year 00-53 (First Sunday is start of week 01)
+** %V week of year 01-53 (First week containing Thursday is week 01)
+** %W week of year 00-53 (First Monday is start of week 01)
** %Y year 0000-9999
** %% %
*/
@@ -1288,7 +1427,7 @@ static void strftimeFunc(
sqlite3_str_appendf(&sRes, cf=='d' ? "%02d" : "%2d", x.D);
break;
}
- case 'f': {
+ case 'f': { /* Fractional seconds. (Non-standard) */
double s = x.s;
if( s>59.999 ) s = 59.999;
sqlite3_str_appendf(&sRes, "%06.3f", s);
@@ -1298,6 +1437,21 @@ static void strftimeFunc(
sqlite3_str_appendf(&sRes, "%04d-%02d-%02d", x.Y, x.M, x.D);
break;
}
+ case 'G': /* Fall thru */
+ case 'g': {
+ DateTime y = x;
+ assert( y.validJD );
+ /* Move y so that it is the Thursday in the same week as x */
+ y.iJD += (3 - daysAfterMonday(&x))*86400000;
+ y.validYMD = 0;
+ computeYMD(&y);
+ if( cf=='g' ){
+ sqlite3_str_appendf(&sRes, "%02d", y.Y%100);
+ }else{
+ sqlite3_str_appendf(&sRes, "%04d", y.Y);
+ }
+ break;
+ }
case 'H':
case 'k': {
sqlite3_str_appendf(&sRes, cf=='H' ? "%02d" : "%2d", x.h);
@@ -1311,25 +1465,11 @@ static void strftimeFunc(
sqlite3_str_appendf(&sRes, cf=='I' ? "%02d" : "%2d", h);
break;
}
- case 'W': /* Fall thru */
- case 'j': {
- int nDay; /* Number of days since 1st day of year */
- DateTime y = x;
- y.validJD = 0;
- y.M = 1;
- y.D = 1;
- computeJD(&y);
- nDay = (int)((x.iJD-y.iJD+43200000)/86400000);
- if( cf=='W' ){
- int wd; /* 0=Monday, 1=Tuesday, ... 6=Sunday */
- wd = (int)(((x.iJD+43200000)/86400000)%7);
- sqlite3_str_appendf(&sRes,"%02d",(nDay+7-wd)/7);
- }else{
- sqlite3_str_appendf(&sRes,"%03d",nDay+1);
- }
+ case 'j': { /* Day of year. Jan01==1, Jan02==2, and so forth */
+ sqlite3_str_appendf(&sRes,"%03d",daysAfterJan01(&x)+1);
break;
}
- case 'J': {
+ case 'J': { /* Julian day number. (Non-standard) */
sqlite3_str_appendf(&sRes,"%.16g",x.iJD/86400000.0);
break;
}
@@ -1372,13 +1512,33 @@ static void strftimeFunc(
sqlite3_str_appendf(&sRes,"%02d:%02d:%02d", x.h, x.m, (int)x.s);
break;
}
- case 'u': /* Fall thru */
- case 'w': {
- char c = (char)(((x.iJD+129600000)/86400000) % 7) + '0';
+ case 'u': /* Day of week. 1 to 7. Monday==1, Sunday==7 */
+ case 'w': { /* Day of week. 0 to 6. Sunday==0, Monday==1 */
+ char c = (char)daysAfterSunday(&x) + '0';
if( c=='0' && cf=='u' ) c = '7';
sqlite3_str_appendchar(&sRes, 1, c);
break;
}
+ case 'U': { /* Week num. 00-53. First Sun of the year is week 01 */
+ sqlite3_str_appendf(&sRes,"%02d",
+ (daysAfterJan01(&x)-daysAfterSunday(&x)+7)/7);
+ break;
+ }
+ case 'V': { /* Week num. 01-53. First week with a Thur is week 01 */
+ DateTime y = x;
+ /* Adjust y so that is the Thursday in the same week as x */
+ assert( y.validJD );
+ y.iJD += (3 - daysAfterMonday(&x))*86400000;
+ y.validYMD = 0;
+ computeYMD(&y);
+ sqlite3_str_appendf(&sRes,"%02d", daysAfterJan01(&y)/7+1);
+ break;
+ }
+ case 'W': { /* Week num. 00-53. First Mon of the year is week 01 */
+ sqlite3_str_appendf(&sRes,"%02d",
+ (daysAfterJan01(&x)-daysAfterMonday(&x)+7)/7);
+ break;
+ }
case 'Y': {
sqlite3_str_appendf(&sRes,"%04d",x.Y);
break;
@@ -1525,9 +1685,7 @@ static void timediffFunc(
d1.iJD = d2.iJD - d1.iJD;
d1.iJD += (u64)1486995408 * (u64)100000;
}
- d1.validYMD = 0;
- d1.validHMS = 0;
- d1.validTZ = 0;
+ clearYMD_HMS_TZ(&d1);
computeYMD_HMS(&d1);
sqlite3StrAccumInit(&sRes, 0, 0, 0, 100);
sqlite3_str_appendf(&sRes, "%c%04d-%02d-%02d %02d:%02d:%06.3f",
@@ -1596,6 +1754,36 @@ static void currentTimeFunc(
}
#endif
+#if !defined(SQLITE_OMIT_DATETIME_FUNCS) && defined(SQLITE_DEBUG)
+/*
+** datedebug(...)
+**
+** This routine returns JSON that describes the internal DateTime object.
+** Used for debugging and testing only. Subject to change.
+*/
+static void datedebugFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ DateTime x;
+ if( isDate(context, argc, argv, &x)==0 ){
+ char *zJson;
+ zJson = sqlite3_mprintf(
+ "{iJD:%lld,Y:%d,M:%d,D:%d,h:%d,m:%d,tz:%d,"
+ "s:%.3f,validJD:%d,validYMS:%d,validHMS:%d,"
+ "nFloor:%d,rawS:%d,isError:%d,useSubsec:%d,"
+ "isUtc:%d,isLocal:%d}",
+ x.iJD, x.Y, x.M, x.D, x.h, x.m, x.tz,
+ x.s, x.validJD, x.validYMD, x.validHMS,
+ x.nFloor, x.rawS, x.isError, x.useSubsec,
+ x.isUtc, x.isLocal);
+ sqlite3_result_text(context, zJson, -1, sqlite3_free);
+ }
+}
+#endif /* !SQLITE_OMIT_DATETIME_FUNCS && SQLITE_DEBUG */
+
+
/*
** This function registered all of the above C functions as SQL
** functions. This should be the only routine in this file with
@@ -1611,6 +1799,9 @@ void sqlite3RegisterDateTimeFunctions(void){
PURE_DATE(datetime, -1, 0, 0, datetimeFunc ),
PURE_DATE(strftime, -1, 0, 0, strftimeFunc ),
PURE_DATE(timediff, 2, 0, 0, timediffFunc ),
+#ifdef SQLITE_DEBUG
+ PURE_DATE(datedebug, -1, 0, 0, datedebugFunc ),
+#endif
DFUNCTION(current_time, 0, 0, 0, ctimeFunc ),
DFUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc),
DFUNCTION(current_date, 0, 0, 0, cdateFunc ),

View File

@@ -3,7 +3,7 @@ set -euo pipefail
cd -P -- "$(dirname -- "$0")"
curl -#OL "https://sqlite.org/2024/sqlite-amalgamation-3450200.zip"
curl -#OL "https://sqlite.org/2024/sqlite-amalgamation-3460000.zip"
unzip -d . sqlite-amalgamation-*.zip
mv sqlite-amalgamation-*/sqlite3* .
rm -rf sqlite-amalgamation-*
@@ -12,25 +12,25 @@ cat *.patch | patch --no-backup-if-mismatch
mkdir -p ext/
cd ext/
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/ext/misc/anycollseq.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/ext/misc/base64.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/ext/misc/decimal.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/ext/misc/ieee754.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/ext/misc/regexp.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/ext/misc/series.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/ext/misc/uint.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/ext/misc/uuid.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/ext/misc/anycollseq.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/ext/misc/base64.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/ext/misc/decimal.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/ext/misc/ieee754.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/ext/misc/regexp.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/ext/misc/series.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/ext/misc/uint.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/ext/misc/uuid.c"
cd ~-
cd ../vfs/tests/mptest/testdata/
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/mptest/mptest.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/mptest/config01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/mptest/config02.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/mptest/crash01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/mptest/crash02.subtest"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/mptest/multiwrite01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/mptest/mptest.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/mptest/config01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/mptest/config02.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/mptest/crash01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/mptest/crash02.subtest"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/mptest/multiwrite01.test"
cd ~-
cd ../vfs/tests/speedtest1/testdata/
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.45.3/test/speedtest1.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.46.0/test/speedtest1.c"
cd ~-

View File

@@ -4,6 +4,7 @@
int go_progress_handler(void *);
int go_busy_handler(void *, int);
int go_busy_timeout(void *, int count, int tmout);
int go_commit_hook(void *);
void go_rollback_hook(void *);
@@ -55,4 +56,12 @@ int sqlite3_autovacuum_pages_go(sqlite3 *db, go_handle app) {
int rc = sqlite3_autovacuum_pages(db, go_autovacuum_pages, app, go_destroy);
if (rc) go_destroy(app);
return rc;
}
}
#ifndef sqliteBusyCallback
static int sqliteBusyCallback(sqlite3 *db, int count) {
return go_busy_timeout(db, count, db->busyTimeout);
}
#endif

View File

@@ -18,6 +18,8 @@
#define HAVE_STDINT_H 1
#define HAVE_INTTYPES_H 1
#define LONGDOUBLE_TYPE double
#define HAVE_LOG2 1
#define HAVE_LOG10 1
#define HAVE_ISNAN 1
@@ -33,52 +35,14 @@
#define HAVE_MALLOC_H 1
#define HAVE_MALLOC_USABLE_SIZE 1
// Recommended Options
#define SQLITE_DQS 0
#define SQLITE_THREADSAFE 0
#define SQLITE_DEFAULT_MEMSTATUS 0
#define SQLITE_DEFAULT_WAL_SYNCHRONOUS 1
#define SQLITE_LIKE_DOESNT_MATCH_BLOBS
#define SQLITE_MAX_EXPR_DEPTH 0
#define SQLITE_STRICT_SUBTYPE 1
#define SQLITE_USE_ALLOCA
#define SQLITE_OMIT_DEPRECATED
#define SQLITE_OMIT_SHARED_CACHE
#define SQLITE_OMIT_AUTOINIT
// #define SQLITE_OMIT_DECLTYPE
// #define SQLITE_OMIT_PROGRESS_CALLBACK
// Other Options
#define SQLITE_ALLOW_URI_AUTHORITY
#define SQLITE_TRUSTED_SCHEMA 0
#define SQLITE_DEFAULT_FOREIGN_KEYS 1
#define SQLITE_ENABLE_ATOMIC_WRITE
#define SQLITE_ENABLE_BATCH_ATOMIC_WRITE
// Because Wasm does not support shared memory,
// SQLite disables WAL for Wasm builds.
// We patch SQLite to use exclusive locking mode instead.
// https://sqlite.org/wal.html#noshm
#undef SQLITE_OMIT_WAL
// We have our own memdb VFS.
// To avoid interactions between the two,
// omit sqlite3_serialize/sqlite3_deserialize,
// which we also don't wrap.
#define SQLITE_OMIT_DESERIALIZE
// Amalgamated Extensions
#define SQLITE_ENABLE_MATH_FUNCTIONS 1
#define SQLITE_ENABLE_JSON1 1
#define SQLITE_ENABLE_FTS5 1
#define SQLITE_ENABLE_RTREE 1
#define SQLITE_ENABLE_GEOPOLY 1
#define SQLITE_SOUNDEX
#define SQLITE_UNTESTABLE
// Implemented in vfs.c.
int localtime_s(struct tm *const pTm, time_t const *const pTime);
int localtime_s(struct tm *const pTm, time_t const *const pTime);
// Implemented in hooks.c.
#ifndef sqliteBusyCallback
static int sqliteBusyCallback(sqlite3 *, int);
#endif

44
sqlite3/sqlite_opt.h Normal file
View File

@@ -0,0 +1,44 @@
// Recommended Options
#define SQLITE_DQS 0
#define SQLITE_THREADSAFE 0
#define SQLITE_DEFAULT_MEMSTATUS 0
#define SQLITE_DEFAULT_WAL_SYNCHRONOUS 1
#define SQLITE_LIKE_DOESNT_MATCH_BLOBS
#define SQLITE_MAX_EXPR_DEPTH 0
#define SQLITE_STRICT_SUBTYPE 1
#define SQLITE_USE_ALLOCA
#define SQLITE_OMIT_DEPRECATED
#define SQLITE_OMIT_SHARED_CACHE
#define SQLITE_OMIT_AUTOINIT
// We need these:
// #define SQLITE_OMIT_DECLTYPE
// #define SQLITE_OMIT_PROGRESS_CALLBACK
// Other Options
#define SQLITE_ALLOW_URI_AUTHORITY
#define SQLITE_TRUSTED_SCHEMA 0
#define SQLITE_DEFAULT_FOREIGN_KEYS 1
#define SQLITE_ENABLE_ATOMIC_WRITE
#define SQLITE_ENABLE_BATCH_ATOMIC_WRITE
#define SQLITE_ENABLE_COLUMN_METADATA
#define SQLITE_ENABLE_STAT4 1
// We have our own memdb VFS.
// To avoid interactions between the two,
// omit sqlite3_serialize/sqlite3_deserialize,
// which we also don't wrap.
#define SQLITE_OMIT_DESERIALIZE
// Amalgamated Extensions
#define SQLITE_ENABLE_MATH_FUNCTIONS 1
#define SQLITE_ENABLE_JSON1 1
#define SQLITE_ENABLE_FTS5 1
#define SQLITE_ENABLE_RTREE 1
#define SQLITE_ENABLE_GEOPOLY 1
#define SQLITE_SOUNDEX
#define SQLITE_UNTESTABLE

View File

@@ -5,6 +5,8 @@
static int time_collation(void *pArg, int nKey1, const void *pKey1, int nKey2,
const void *pKey2) {
UNUSED_PARAMETER(pArg);
// Remove a Z suffix if one key is no longer than the other.
// A Z suffix collates before any character but after the empty string.
// This avoids making different keys equal.
@@ -29,8 +31,8 @@ static int time_collation(void *pArg, int nKey1, const void *pKey1, int nKey2,
int sqlite3_time_init(sqlite3 *db, char **pzErrMsg,
const sqlite3_api_routines *pApi) {
UNUSED_PARAMETER2(pzErrMsg, pApi);
sqlite3_create_collation_v2(db, "time", SQLITE_UTF8, /*arg=*/NULL,
time_collation,
/*destroy=*/NULL);
time_collation, /*destroy=*/NULL);
return SQLITE_OK;
}

View File

@@ -1,7 +1,8 @@
# Wrap sqlite3_vfs_find.
# This patch allows Go VFSes to be (un)registered.
--- sqlite3.c.orig
+++ sqlite3.c
@@ -26089,7 +26089,7 @@
@@ -26396,7 +26396,7 @@
** Locate a VFS by name. If no name is given, simply return the
** first VFS on the list.
*/

View File

@@ -1,16 +1,18 @@
#include <stdbool.h>
#include <stddef.h>
#include "include.h"
#include "sqlite3.h"
#define SQLITE_VTAB_CREATOR_GO /******/ 0x01
#define SQLITE_VTAB_DESTROYER_GO /****/ 0x02
#define SQLITE_VTAB_UPDATER_GO /******/ 0x04
#define SQLITE_VTAB_RENAMER_GO /******/ 0x08
#define SQLITE_VTAB_OVERLOADER_GO /***/ 0x10
#define SQLITE_VTAB_CHECKER_GO /******/ 0x20
#define SQLITE_VTAB_TXN_GO /**********/ 0x40
#define SQLITE_VTAB_SAVEPOINTER_GO /**/ 0x80
#define SQLITE_VTAB_CREATOR_GO /******/ 0x001
#define SQLITE_VTAB_DESTROYER_GO /****/ 0x002
#define SQLITE_VTAB_UPDATER_GO /******/ 0x004
#define SQLITE_VTAB_RENAMER_GO /******/ 0x008
#define SQLITE_VTAB_OVERLOADER_GO /***/ 0x010
#define SQLITE_VTAB_CHECKER_GO /******/ 0x020
#define SQLITE_VTAB_TXN_GO /**********/ 0x040
#define SQLITE_VTAB_SAVEPOINTER_GO /**/ 0x080
#define SQLITE_VTAB_SHADOWTABS_GO /***/ 0x100
int go_vtab_create(sqlite3_module *, int argc, const char *const *argv,
sqlite3_vtab **, char **pzErr);
@@ -72,6 +74,8 @@ static void go_mod_destroy(void *pAux) {
static int go_vtab_create_wrapper(sqlite3 *db, void *pAux, int argc,
const char *const *argv,
sqlite3_vtab **ppVTab, char **pzErr) {
UNUSED_PARAMETER(db);
struct go_vtab *vtab = calloc(1, sizeof(struct go_vtab));
if (vtab == NULL) return SQLITE_NOMEM;
*ppVTab = &vtab->base;
@@ -88,6 +92,8 @@ static int go_vtab_create_wrapper(sqlite3 *db, void *pAux, int argc,
static int go_vtab_connect_wrapper(sqlite3 *db, void *pAux, int argc,
const char *const *argv,
sqlite3_vtab **ppVTab, char **pzErr) {
UNUSED_PARAMETER(db);
struct go_vtab *vtab = calloc(1, sizeof(struct go_vtab));
if (vtab == NULL) return SQLITE_NOMEM;
*ppVTab = &vtab->base;
@@ -153,6 +159,8 @@ static int go_vtab_integrity_wrapper(sqlite3_vtab *pVTab, const char *zSchema,
return rc;
}
static int go_vtab_shadown_name_wrapper(const char *zName) { return 1; }
int sqlite3_create_module_go(sqlite3 *db, const char *zName, int flags,
go_handle handle) {
struct go_module *mod = malloc(sizeof(struct go_module));
@@ -204,6 +212,9 @@ int sqlite3_create_module_go(sqlite3 *db, const char *zName, int flags,
mod->base.xRelease = go_vtab_release;
mod->base.xRollbackTo = go_vtab_rollback_to;
}
if (flags & SQLITE_VTAB_SHADOWTABS_GO) {
mod->base.xShadowName = go_vtab_shadown_name_wrapper;
}
if (mod->base.xCreate && !mod->base.xDestroy) {
mod->base.xDestroy = mod->base.xDisconnect;
}

54
stmt.go
View File

@@ -367,12 +367,10 @@ func (s *Stmt) ColumnCount() int {
func (s *Stmt) ColumnName(col int) string {
r := s.c.call("sqlite3_column_name",
uint64(s.handle), uint64(col))
ptr := uint32(r)
if ptr == 0 {
if r == 0 {
panic(util.OOMErr)
}
return util.ReadString(s.c.mod, ptr, _MAX_NAME)
return util.ReadString(s.c.mod, uint32(r), _MAX_NAME)
}
// ColumnType returns the initial [Datatype] of the result column.
@@ -398,15 +396,57 @@ func (s *Stmt) ColumnDeclType(col int) string {
return util.ReadString(s.c.mod, uint32(r), _MAX_NAME)
}
// ColumnDatabaseName returns the name of the database
// that is the origin of a particular result column.
// The leftmost column of the result set has the index 0.
//
// https://sqlite.org/c3ref/column_database_name.html
func (s *Stmt) ColumnDatabaseName(col int) string {
r := s.c.call("sqlite3_column_database_name",
uint64(s.handle), uint64(col))
if r == 0 {
return ""
}
return util.ReadString(s.c.mod, uint32(r), _MAX_NAME)
}
// ColumnTableName returns the name of the table
// that is the origin of a particular result column.
// The leftmost column of the result set has the index 0.
//
// https://sqlite.org/c3ref/column_database_name.html
func (s *Stmt) ColumnTableName(col int) string {
r := s.c.call("sqlite3_column_table_name",
uint64(s.handle), uint64(col))
if r == 0 {
return ""
}
return util.ReadString(s.c.mod, uint32(r), _MAX_NAME)
}
// ColumnOriginName returns the name of the table column
// that is the origin of a particular result column.
// The leftmost column of the result set has the index 0.
//
// https://sqlite.org/c3ref/column_database_name.html
func (s *Stmt) ColumnOriginName(col int) string {
r := s.c.call("sqlite3_column_origin_name",
uint64(s.handle), uint64(col))
if r == 0 {
return ""
}
return util.ReadString(s.c.mod, uint32(r), _MAX_NAME)
}
// ColumnBool returns the value of the result column as a bool.
// The leftmost column of the result set has the index 0.
// SQLite does not have a separate boolean storage class.
// Instead, boolean values are retrieved as integers,
// Instead, boolean values are retrieved as numbers,
// with 0 converted to false and any other value to true.
//
// https://sqlite.org/c3ref/column_blob.html
func (s *Stmt) ColumnBool(col int) bool {
return s.ColumnInt64(col) != 0
return s.ColumnFloat(col) != 0
}
// ColumnInt returns the value of the result column as an int.
@@ -524,7 +564,7 @@ func (s *Stmt) ColumnJSON(col int, ptr any) error {
var data []byte
switch s.ColumnType(col) {
case NULL:
data = append(data, "null"...)
data = []byte("null")
case TEXT:
data = s.ColumnRawText(col)
case BLOB:

View File

@@ -1,5 +1,3 @@
//go:build !sqlite3_nosys
package tests
import (
@@ -8,10 +6,14 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
"github.com/ncruces/go-sqlite3/vfs"
)
func TestBackup(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
t.Parallel()
backupName := filepath.Join(t.TempDir(), "backup.db")

View File

@@ -11,7 +11,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
func TestBlob(t *testing.T) {

View File

@@ -1,4 +1,4 @@
//go:build !sqlite3_nosys
//go:build (linux || darwin || windows || freebsd || illumos) && !sqlite3_nosys
package bradfitz
@@ -14,7 +14,7 @@ import (
_ "github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
type Tester interface {

View File

@@ -11,7 +11,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
_ "github.com/ncruces/go-sqlite3/vfs/memdb"
)
@@ -497,7 +497,7 @@ func TestConn_DBName(t *testing.T) {
func TestConn_AutoVacuumPages(t *testing.T) {
t.Parallel()
db, err := sqlite3.Open("file:test.db?vfs=memdb&_pragma=auto_vacuum(FULL)")
db, err := sqlite3.Open("file:test.db?vfs=memdb&_pragma=auto_vacuum(full)")
if err != nil {
t.Fatal(err)
}

View File

@@ -9,17 +9,17 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
"github.com/ncruces/go-sqlite3/vfs"
_ "github.com/ncruces/go-sqlite3/vfs/adiantum"
_ "github.com/ncruces/go-sqlite3/vfs/memdb"
)
//go:embed testdata/wal.db
var waldb []byte
var walDB []byte
//go:embed testdata/utf16be.db
var utf16db []byte
var utf16DB []byte
func TestDB_memory(t *testing.T) {
t.Parallel()
@@ -42,7 +42,7 @@ func TestDB_wal(t *testing.T) {
t.Parallel()
tmp := filepath.Join(t.TempDir(), "test.db")
err := os.WriteFile(tmp, waldb, 0666)
err := os.WriteFile(tmp, walDB, 0666)
if err != nil {
t.Fatal(err)
}
@@ -56,7 +56,7 @@ func TestDB_utf16(t *testing.T) {
t.Parallel()
tmp := filepath.Join(t.TempDir(), "test.db")
err := os.WriteFile(tmp, utf16db, 0666)
err := os.WriteFile(tmp, utf16DB, 0666)
if err != nil {
t.Fatal(err)
}

View File

@@ -6,7 +6,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
func TestDriver(t *testing.T) {

75
tests/endian_test.go Normal file
View File

@@ -0,0 +1,75 @@
package tests
import (
"bytes"
"encoding/binary"
"log"
"strconv"
"testing"
"github.com/ncruces/go-sqlite3"
)
func Test_endianness(t *testing.T) {
big := binary.BigEndian.AppendUint64(nil, 0x1234567890ABCDEF)
little := binary.LittleEndian.AppendUint64(nil, 0x1234567890ABCDEF)
native := binary.NativeEndian.AppendUint64(nil, 0x1234567890ABCDEF)
switch {
case bytes.Equal(big, native):
t.Log("Platform is big endian")
case bytes.Equal(little, native):
t.Log("Platform is little endian")
default:
t.Fatal("Platform is middle endian")
}
db, err := sqlite3.Open(":memory:")
if err != nil {
log.Fatal(err)
}
defer db.Close()
err = db.Exec(`CREATE TABLE test (col)`)
if err != nil {
log.Fatal(err)
}
const value int64 = -9223372036854775808
{
stmt, _, err := db.Prepare(`INSERT INTO test VALUES (?)`)
if err != nil {
t.Fatal(err)
}
defer stmt.Close()
err = stmt.BindInt64(1, value)
if err != nil {
t.Fatal(err)
}
err = stmt.Exec()
if err != nil {
t.Fatal(err)
}
}
{
stmt, _, err := db.Prepare(`SELECT * FROM test`)
if err != nil {
t.Fatal(err)
}
defer stmt.Close()
if stmt.Step() {
if got := stmt.ColumnInt64(0); got != value {
t.Errorf("got %d, want %d", got, value)
}
if got := stmt.ColumnText(0); got != strconv.FormatInt(value, 10) {
t.Errorf("got %s, want %d", got, value)
}
}
if err != nil {
t.Fatal(err)
}
}
}

View File

@@ -5,7 +5,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
func Test_base64(t *testing.T) {

View File

@@ -6,7 +6,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
func TestCreateFunction(t *testing.T) {

View File

@@ -10,7 +10,7 @@ import (
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
"github.com/ncruces/julianday"
)

View File

@@ -1,5 +1,3 @@
//go:build !sqlite3_nosys
package tests
import (
@@ -14,13 +12,17 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
"github.com/ncruces/go-sqlite3/vfs"
_ "github.com/ncruces/go-sqlite3/vfs/adiantum"
"github.com/ncruces/go-sqlite3/vfs/memdb"
)
func Test_parallel(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
var iter int
if testing.Short() {
iter = 1000
@@ -59,12 +61,17 @@ func Test_memdb(t *testing.T) {
iter = 5000
}
memdb.Create("test.db", nil)
name := "file:/test.db?vfs=memdb"
testParallel(t, name, iter)
testIntegrity(t, name)
}
func Test_adiantum(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
var iter int
if testing.Short() {
iter = 1000
@@ -81,6 +88,9 @@ func Test_adiantum(t *testing.T) {
}
func TestMultiProcess(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
if testing.Short() {
t.Skip("skipping in short mode")
}
@@ -131,8 +141,43 @@ func TestChildProcess(t *testing.T) {
testParallel(t, name, 1000)
}
func Benchmark_parallel(b *testing.B) {
if !vfs.SupportsSharedMemory {
b.Skip("skipping without shared memory")
}
sqlite3.Initialize()
b.ResetTimer()
name := "file:" +
filepath.Join(b.TempDir(), "test.db") +
"?_pragma=busy_timeout(10000)" +
"&_pragma=journal_mode(truncate)" +
"&_pragma=synchronous(off)"
testParallel(b, name, b.N)
}
func Benchmark_wal(b *testing.B) {
if !vfs.SupportsSharedMemory {
b.Skip("skipping without shared memory")
}
sqlite3.Initialize()
b.ResetTimer()
name := "file:" +
filepath.Join(b.TempDir(), "test.db") +
"?_pragma=busy_timeout(10000)" +
"&_pragma=journal_mode(wal)" +
"&_pragma=synchronous(off)"
testParallel(b, name, b.N)
}
func Benchmark_memdb(b *testing.B) {
memdb.Delete("test.db")
sqlite3.Initialize()
b.ResetTimer()
memdb.Create("test.db", nil)
name := "file:/test.db?vfs=memdb"
testParallel(b, name, b.N)
}

View File

@@ -3,12 +3,13 @@ package tests
import (
"encoding/json"
"math"
"math/bits"
"testing"
"time"
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
func TestStmt(t *testing.T) {
@@ -20,7 +21,7 @@ func TestStmt(t *testing.T) {
}
defer db.Close()
err = db.Exec(`CREATE TABLE test (col)`)
err = db.Exec(`CREATE TABLE test (col ANY) STRICT`)
if err != nil {
t.Fatal(err)
}
@@ -136,7 +137,7 @@ func TestStmt(t *testing.T) {
}
// The table should have: 0, 1, 2, π, NULL, "", "text", "", "blob", NULL, "\0\0\0\0", "true", NULL
stmt, _, err = db.Prepare(`SELECT col FROM test`)
stmt, _, err = db.Prepare(`SELECT col AS c FROM test`)
if err != nil {
t.Fatal(err)
}
@@ -145,6 +146,21 @@ func TestStmt(t *testing.T) {
if got := stmt.ReadOnly(); got != true {
t.Error("got false, want true")
}
if got := stmt.ColumnName(0); got != "c" {
t.Errorf(`got %q, want "c"`, got)
}
if got := stmt.ColumnDeclType(0); got != "ANY" {
t.Errorf(`got %q, want "ANY"`, got)
}
if got := stmt.ColumnOriginName(0); got != "col" {
t.Errorf(`got %q, want "col"`, got)
}
if got := stmt.ColumnTableName(0); got != "test" {
t.Errorf(`got %q, want "test"`, got)
}
if got := stmt.ColumnDatabaseName(0); got != "main" {
t.Errorf(`got %q, want "main"`, got)
}
if stmt.Step() {
if got := stmt.ColumnType(0); got != sqlite3.INTEGER {
@@ -602,6 +618,9 @@ func TestStmt_ColumnTime(t *testing.T) {
}
func TestStmt_Error(t *testing.T) {
if bits.UintSize < 64 {
t.Skip("skipping on 32-bit")
}
t.Parallel()
db, err := sqlite3.Open(":memory:")

View File

@@ -1,10 +0,0 @@
package testcfg
import (
"github.com/ncruces/go-sqlite3"
"github.com/tetratelabs/wazero"
)
func init() {
sqlite3.RuntimeConfig = wazero.NewRuntimeConfig().WithMemoryLimitPages(1024)
}

View File

@@ -10,7 +10,7 @@ import (
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
func TestTimeFormat_Encode(t *testing.T) {
@@ -44,7 +44,8 @@ func TestTimeFormat_Encode(t *testing.T) {
func TestTimeFormat_Decode(t *testing.T) {
t.Parallel()
zone := time.FixedZone("", -4*3600)
const offset = -4 * 3600
zone := time.FixedZone("", offset)
reference := time.Date(2013, 10, 7, 4, 23, 19, 120_000_000, zone)
refnodate := time.Date(2000, 01, 1, 4, 23, 19, 120_000_000, zone)
@@ -53,63 +54,63 @@ func TestTimeFormat_Decode(t *testing.T) {
val any
want time.Time
wantDelta time.Duration
wantLoc *time.Location
wantOff int
wantErr bool
}{
{sqlite3.TimeFormatJulianDay, "2456572.849526851851852", reference, 0, time.UTC, false},
{sqlite3.TimeFormatJulianDay, 2456572.849526851851852, reference, time.Millisecond, time.UTC, false},
{sqlite3.TimeFormatJulianDay, int64(2456572), reference, 24 * time.Hour, time.UTC, false},
{sqlite3.TimeFormatJulianDay, false, time.Time{}, 0, nil, true},
{sqlite3.TimeFormatJulianDay, "2456572.849526851851852", reference, 0, 0, false},
{sqlite3.TimeFormatJulianDay, 2456572.849526851851852, reference, time.Millisecond, 0, false},
{sqlite3.TimeFormatJulianDay, int64(2456572), reference, 24 * time.Hour, 0, false},
{sqlite3.TimeFormatJulianDay, false, time.Time{}, 0, 0, true},
{sqlite3.TimeFormatUnix, "1381134199.120", reference, time.Microsecond, time.UTC, false},
{sqlite3.TimeFormatUnix, 1381134199.120, reference, time.Microsecond, time.UTC, false},
{sqlite3.TimeFormatUnix, int64(1381134199), reference, time.Second, time.UTC, false},
{sqlite3.TimeFormatUnix, "abc", time.Time{}, 0, nil, true},
{sqlite3.TimeFormatUnix, false, time.Time{}, 0, nil, true},
{sqlite3.TimeFormatUnix, "1381134199.120", reference, time.Microsecond, 0, false},
{sqlite3.TimeFormatUnix, 1381134199.120, reference, time.Microsecond, 0, false},
{sqlite3.TimeFormatUnix, int64(1381134199), reference, time.Second, 0, false},
{sqlite3.TimeFormatUnix, "abc", time.Time{}, 0, 0, true},
{sqlite3.TimeFormatUnix, false, time.Time{}, 0, 0, true},
{sqlite3.TimeFormatUnixMilli, "1381134199120", reference, 0, time.UTC, false},
{sqlite3.TimeFormatUnixMilli, 1381134199.120e3, reference, 0, time.UTC, false},
{sqlite3.TimeFormatUnixMilli, int64(1381134199_120), reference, 0, time.UTC, false},
{sqlite3.TimeFormatUnixMilli, "abc", time.Time{}, 0, nil, true},
{sqlite3.TimeFormatUnixMilli, false, time.Time{}, 0, nil, true},
{sqlite3.TimeFormatUnixMilli, "1381134199120", reference, 0, 0, false},
{sqlite3.TimeFormatUnixMilli, 1381134199.120e3, reference, 0, 0, false},
{sqlite3.TimeFormatUnixMilli, int64(1381134199_120), reference, 0, 0, false},
{sqlite3.TimeFormatUnixMilli, "abc", time.Time{}, 0, 0, true},
{sqlite3.TimeFormatUnixMilli, false, time.Time{}, 0, 0, true},
{sqlite3.TimeFormatUnixMicro, "1381134199120000", reference, 0, time.UTC, false},
{sqlite3.TimeFormatUnixMicro, 1381134199.120e6, reference, 0, time.UTC, false},
{sqlite3.TimeFormatUnixMicro, int64(1381134199_120000), reference, 0, time.UTC, false},
{sqlite3.TimeFormatUnixMicro, "abc", time.Time{}, 0, nil, true},
{sqlite3.TimeFormatUnixMicro, false, time.Time{}, 0, nil, true},
{sqlite3.TimeFormatUnixMicro, "1381134199120000", reference, 0, 0, false},
{sqlite3.TimeFormatUnixMicro, 1381134199.120e6, reference, 0, 0, false},
{sqlite3.TimeFormatUnixMicro, int64(1381134199_120000), reference, 0, 0, false},
{sqlite3.TimeFormatUnixMicro, "abc", time.Time{}, 0, 0, true},
{sqlite3.TimeFormatUnixMicro, false, time.Time{}, 0, 0, true},
{sqlite3.TimeFormatUnixNano, "1381134199120000000", reference, 0, time.UTC, false},
{sqlite3.TimeFormatUnixNano, 1381134199.120e9, reference, 0, time.UTC, false},
{sqlite3.TimeFormatUnixNano, int64(1381134199_120000000), reference, 0, time.UTC, false},
{sqlite3.TimeFormatUnixNano, "abc", time.Time{}, 0, nil, true},
{sqlite3.TimeFormatUnixNano, false, time.Time{}, 0, nil, true},
{sqlite3.TimeFormatUnixNano, "1381134199120000000", reference, 0, 0, false},
{sqlite3.TimeFormatUnixNano, 1381134199.120e9, reference, 0, 0, false},
{sqlite3.TimeFormatUnixNano, int64(1381134199_120000000), reference, 0, 0, false},
{sqlite3.TimeFormatUnixNano, "abc", time.Time{}, 0, 0, true},
{sqlite3.TimeFormatUnixNano, false, time.Time{}, 0, 0, true},
{sqlite3.TimeFormatAuto, "2456572.849526851851852", reference, time.Millisecond, time.UTC, false},
{sqlite3.TimeFormatAuto, "2456572", reference, 24 * time.Hour, time.UTC, false},
{sqlite3.TimeFormatAuto, "1381134199.120", reference, time.Microsecond, time.UTC, false},
{sqlite3.TimeFormatAuto, "1381134199.120e3", reference, time.Microsecond, time.UTC, false},
{sqlite3.TimeFormatAuto, "1381134199.120e6", reference, time.Microsecond, time.UTC, false},
{sqlite3.TimeFormatAuto, "1381134199.120e9", reference, time.Microsecond, time.UTC, false},
{sqlite3.TimeFormatAuto, "1381134199", reference, time.Second, time.UTC, false},
{sqlite3.TimeFormatAuto, "1381134199120", reference, 0, time.UTC, false},
{sqlite3.TimeFormatAuto, "1381134199120000", reference, 0, time.UTC, false},
{sqlite3.TimeFormatAuto, "1381134199120000000", reference, 0, time.UTC, false},
{sqlite3.TimeFormatAuto, "2013-10-07 04:23:19.12-04:00", reference, 0, zone, false},
{sqlite3.TimeFormatAuto, "04:23:19.12-04:00", refnodate, 0, zone, false},
{sqlite3.TimeFormatAuto, "abc", time.Time{}, 0, nil, true},
{sqlite3.TimeFormatAuto, false, time.Time{}, 0, nil, true},
{sqlite3.TimeFormatAuto, "2456572.849526851851852", reference, time.Millisecond, 0, false},
{sqlite3.TimeFormatAuto, "2456572", reference, 24 * time.Hour, 0, false},
{sqlite3.TimeFormatAuto, "1381134199.120", reference, time.Microsecond, 0, false},
{sqlite3.TimeFormatAuto, "1381134199.120e3", reference, time.Microsecond, 0, false},
{sqlite3.TimeFormatAuto, "1381134199.120e6", reference, time.Microsecond, 0, false},
{sqlite3.TimeFormatAuto, "1381134199.120e9", reference, time.Microsecond, 0, false},
{sqlite3.TimeFormatAuto, "1381134199", reference, time.Second, 0, false},
{sqlite3.TimeFormatAuto, "1381134199120", reference, 0, 0, false},
{sqlite3.TimeFormatAuto, "1381134199120000", reference, 0, 0, false},
{sqlite3.TimeFormatAuto, "1381134199120000000", reference, 0, 0, false},
{sqlite3.TimeFormatAuto, "2013-10-07 04:23:19.12-04:00", reference, 0, offset, false},
{sqlite3.TimeFormatAuto, "04:23:19.12-04:00", refnodate, 0, offset, false},
{sqlite3.TimeFormatAuto, "abc", time.Time{}, 0, 0, true},
{sqlite3.TimeFormatAuto, false, time.Time{}, 0, 0, true},
{sqlite3.TimeFormat3, "2013-10-07 04:23:19.12-04:00", reference, 0, zone, false},
{sqlite3.TimeFormat3, "2013-10-07 08:23:19.12", reference, 0, time.UTC, false},
{sqlite3.TimeFormat9, "04:23:19.12-04:00", refnodate, 0, zone, false},
{sqlite3.TimeFormat9, "08:23:19.12", refnodate, 0, time.UTC, false},
{sqlite3.TimeFormat3, false, time.Time{}, 0, nil, true},
{sqlite3.TimeFormat9, false, time.Time{}, 0, nil, true},
{sqlite3.TimeFormat3, "2013-10-07 04:23:19.12-04:00", reference, 0, offset, false},
{sqlite3.TimeFormat3, "2013-10-07 08:23:19.12", reference, 0, 0, false},
{sqlite3.TimeFormat9, "04:23:19.12-04:00", refnodate, 0, offset, false},
{sqlite3.TimeFormat9, "08:23:19.12", refnodate, 0, 0, false},
{sqlite3.TimeFormat3, false, time.Time{}, 0, 0, true},
{sqlite3.TimeFormat9, false, time.Time{}, 0, 0, true},
{sqlite3.TimeFormatDefault, "2013-10-07T04:23:19.12-04:00", reference, 0, zone, false},
{sqlite3.TimeFormatDefault, "2013-10-07T08:23:19.12Z", reference, 0, time.UTC, false},
{sqlite3.TimeFormatDefault, false, time.Time{}, 0, nil, true},
{sqlite3.TimeFormatDefault, "2013-10-07T04:23:19.12-04:00", reference, 0, offset, false},
{sqlite3.TimeFormatDefault, "2013-10-07T08:23:19.12Z", reference, 0, 0, false},
{sqlite3.TimeFormatDefault, false, time.Time{}, 0, 0, true},
}
for _, tt := range tests {
@@ -122,8 +123,8 @@ func TestTimeFormat_Decode(t *testing.T) {
if got.Sub(tt.want).Abs() > tt.wantDelta {
t.Errorf("%q.Decode(%v) = %v, want %v", tt.fmt, tt.val, got, tt.want)
}
if got.Location().String() != tt.wantLoc.String() {
t.Errorf("%q.Decode(%v) = %v, want %v", tt.fmt, tt.val, got.Location(), tt.wantLoc)
if _, off := got.Zone(); off != tt.wantOff {
t.Errorf("%q.Decode(%v) = %v, want %v", tt.fmt, tt.val, off, tt.wantOff)
}
})
}

View File

@@ -7,7 +7,8 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
_ "github.com/ncruces/go-sqlite3/vfs/memdb"
)
func TestConn_Transaction_exec(t *testing.T) {
@@ -247,6 +248,51 @@ func TestConn_Transaction_interrupted(t *testing.T) {
}
}
func TestConn_Transaction_busy(t *testing.T) {
t.Parallel()
db1, err := sqlite3.Open("file:/test.db?vfs=memdb")
if err != nil {
t.Fatal(err)
}
defer db1.Close()
db2, err := sqlite3.Open("file:/test.db?vfs=memdb&_pragma=busy_timeout(10000)")
if err != nil {
t.Fatal(err)
}
defer db2.Close()
err = db1.Exec(`CREATE TABLE test (col)`)
if err != nil {
t.Fatal(err)
}
tx, err := db1.BeginImmediate()
if err != nil {
t.Fatal(err)
}
err = db1.Exec(`INSERT INTO test VALUES (1)`)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
db2.SetInterrupt(ctx)
go cancel()
_, err = db2.BeginExclusive()
if !errors.Is(err, sqlite3.BUSY) && !errors.Is(err, sqlite3.INTERRUPT) {
t.Errorf("got %v, want sqlite3.BUSY or sqlite3.INTERRUPT", err)
}
err = nil
tx.End(&err)
if err != nil {
t.Fatal(err)
}
}
func TestConn_Transaction_rollback(t *testing.T) {
t.Parallel()

View File

@@ -6,7 +6,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
"github.com/ncruces/go-sqlite3/vfs/memdb"
"github.com/ncruces/go-sqlite3/vfs/readervfs"
)

View File

@@ -1,19 +1,20 @@
//go:build !sqlite3_nosys
package tests
import (
"os"
"path/filepath"
"testing"
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/tests/testcfg"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
"github.com/ncruces/go-sqlite3/vfs"
)
func TestWAL_enter_exit(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
t.Parallel()
file := filepath.Join(t.TempDir(), "test.db")
@@ -25,7 +26,7 @@ func TestWAL_enter_exit(t *testing.T) {
defer db.Close()
if !vfs.SupportsSharedMemory {
err = db.Exec(`PRAGMA locking_mode=EXCLUSIVE`)
err = db.Exec(`PRAGMA locking_mode=exclusive`)
if err != nil {
t.Fatal(err)
}
@@ -33,11 +34,11 @@ func TestWAL_enter_exit(t *testing.T) {
err = db.Exec(`
CREATE TABLE test (col);
PRAGMA journal_mode=WAL;
PRAGMA journal_mode=wal;
SELECT * FROM test;
PRAGMA journal_mode=DELETE;
PRAGMA journal_mode=delete;
SELECT * FROM test;
PRAGMA journal_mode=WAL;
PRAGMA journal_mode=wal;
SELECT * FROM test;
`)
if err != nil {
@@ -49,33 +50,64 @@ func TestWAL_readonly(t *testing.T) {
if !vfs.SupportsSharedMemory {
t.Skip("skipping without shared memory")
}
t.Parallel()
tmp := filepath.Join(t.TempDir(), "test.db")
err := os.WriteFile(tmp, waldb, 0666)
tmp := filepath.ToSlash(filepath.Join(t.TempDir(), "test.db"))
db1, err := driver.Open("file:"+tmp+"?_pragma=journal_mode(wal)&_txlock=immediate", nil)
if err != nil {
t.Fatal(err)
}
defer db1.Close()
db2, err := driver.Open("file:"+tmp+"?_pragma=journal_mode(wal)&mode=ro", nil)
if err != nil {
t.Fatal(err)
}
defer db2.Close()
// Create the table using the first (writable) connection.
_, err = db1.Exec(`
CREATE TABLE t(id INTEGER PRIMARY KEY, name TEXT);
INSERT INTO t(name) VALUES('alice');
`)
if err != nil {
t.Fatal(err)
}
db, err := sqlite3.OpenFlags(tmp, sqlite3.OPEN_READONLY)
// Select the data using the second (readonly) connection.
var name string
err = db2.QueryRow("SELECT name FROM t").Scan(&name)
if err != nil {
t.Fatal(err)
}
defer db.Close()
if name != "alice" {
t.Errorf("got %q want alice", name)
}
stmt, _, err := db.Prepare(`SELECT * FROM sqlite_master`)
// Update table.
_, err = db1.Exec(`
DELETE FROM t;
INSERT INTO t(name) VALUES('bob');
`)
if err != nil {
t.Fatal(err)
}
defer stmt.Close()
if stmt.Step() {
t.Error("want no rows")
// Select the data using the second (readonly) connection.
err = db2.QueryRow("SELECT name FROM t").Scan(&name)
if err != nil {
t.Fatal(err)
}
if name != "bob" {
t.Errorf("got %q want bob", name)
}
}
func TestConn_WalCheckpoint(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
t.Parallel()
file := filepath.Join(t.TempDir(), "test.db")
@@ -98,8 +130,8 @@ func TestConn_WalCheckpoint(t *testing.T) {
})
err = db.Exec(`
PRAGMA locking_mode=EXCLUSIVE;
PRAGMA journal_mode=WAL;
PRAGMA locking_mode=exlusive;
PRAGMA journal_mode=wal;
CREATE TABLE test (col);
`)
if err != nil {

12
time.go
View File

@@ -101,7 +101,7 @@ func (f TimeFormat) Encode(t time.Time) any {
return t.UnixMicro()
case TimeFormatUnixNano:
return t.UnixNano()
// Special formats
// Special formats.
case TimeFormatDefault, TimeFormatAuto:
f = time.RFC3339Nano
// SQLite assumes UTC if unspecified.
@@ -139,7 +139,7 @@ func (f TimeFormat) Encode(t time.Time) any {
// https://sqlite.org/lang_datefunc.html
func (f TimeFormat) Decode(v any) (time.Time, error) {
switch f {
// Numeric formats
// Numeric formats.
case TimeFormatJulianDay:
switch v := v.(type) {
case string:
@@ -183,7 +183,7 @@ func (f TimeFormat) Decode(v any) (time.Time, error) {
case float64:
return time.UnixMilli(int64(math.Floor(v))).UTC(), nil
case int64:
return time.UnixMilli(int64(v)).UTC(), nil
return time.UnixMilli(v).UTC(), nil
default:
return time.Time{}, util.TimeErr
}
@@ -200,7 +200,7 @@ func (f TimeFormat) Decode(v any) (time.Time, error) {
case float64:
return time.UnixMicro(int64(math.Floor(v))).UTC(), nil
case int64:
return time.UnixMicro(int64(v)).UTC(), nil
return time.UnixMicro(v).UTC(), nil
default:
return time.Time{}, util.TimeErr
}
@@ -217,12 +217,12 @@ func (f TimeFormat) Decode(v any) (time.Time, error) {
case float64:
return time.Unix(0, int64(math.Floor(v))).UTC(), nil
case int64:
return time.Unix(0, int64(v)).UTC(), nil
return time.Unix(0, v).UTC(), nil
default:
return time.Time{}, util.TimeErr
}
// Special formats
// Special formats.
case TimeFormatAuto:
switch s := v.(type) {
case string:

8
util/vtabutil/README.md Normal file
View File

@@ -0,0 +1,8 @@
# Virtual Table utility functions
This package implements utilities mostly useful to virtual table implementations.
It also wraps a [parser](https://github.com/marcobambini/sqlite-createtable-parser)
for the [`CREATE`](https://sqlite.org/lang_createtable.html) and
[`ALTER TABLE`](https://sqlite.org/lang_altertable.html) commands,
created by [Marco Bambini](https://github.com/marcobambini).

View File

@@ -1,4 +1,3 @@
// Package ioutil implements virtual table utility functions.
package vtabutil
import "strings"

141
util/vtabutil/parse.go Normal file
View File

@@ -0,0 +1,141 @@
package vtabutil
import (
"context"
"sync"
_ "embed"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/api"
)
const (
_NONE = iota
_MEMORY
_SYNTAX
_UNSUPPORTEDSQL
codeptr = 4
baseptr = 8
)
var (
//go:embed parse/sql3parse_table.wasm
binary []byte
ctx context.Context
once sync.Once
runtime wazero.Runtime
module wazero.CompiledModule
)
// Table holds metadata about a table.
type Table struct {
mod api.Module
ptr uint32
sql string
}
// Parse parses a [CREATE] or [ALTER TABLE] command.
//
// [CREATE]: https://sqlite.org/lang_createtable.html
// [ALTER TABLE]: https://sqlite.org/lang_altertable.html
func Parse(sql string) (_ *Table, err error) {
once.Do(func() {
ctx = context.Background()
cfg := wazero.NewRuntimeConfigInterpreter().WithDebugInfoEnabled(false)
runtime = wazero.NewRuntimeWithConfig(ctx, cfg)
module, err = runtime.CompileModule(ctx, binary)
})
if err != nil {
return nil, err
}
mod, err := runtime.InstantiateModule(ctx, module, wazero.NewModuleConfig().WithName(""))
if err != nil {
return nil, err
}
if buf, ok := mod.Memory().Read(baseptr, uint32(len(sql))); ok {
copy(buf, sql)
}
r, err := mod.ExportedFunction("sql3parse_table").Call(ctx, baseptr, uint64(len(sql)), codeptr)
if err != nil {
return nil, err
}
c, _ := mod.Memory().ReadUint32Le(codeptr)
switch c {
case _MEMORY:
panic(util.OOMErr)
case _SYNTAX:
return nil, util.ErrorString("sql3parse: invalid syntax")
case _UNSUPPORTEDSQL:
return nil, util.ErrorString("sql3parse: unsupported SQL")
}
if r[0] == 0 {
return nil, nil
}
return &Table{
sql: sql,
mod: mod,
ptr: uint32(r[0]),
}, nil
}
// Close closes a table handle.
func (t *Table) Close() error {
mod := t.mod
t.mod = nil
return mod.Close(ctx)
}
// NumColumns returns the number of columns of the table.
func (t *Table) NumColumns() int {
r, err := t.mod.ExportedFunction("sql3table_num_columns").Call(ctx, uint64(t.ptr))
if err != nil {
panic(err)
}
return int(int32(r[0]))
}
// Column returns data for the ith column of the table.
//
// https://sqlite.org/lang_createtable.html#column_definitions
func (t *Table) Column(i int) Column {
r, err := t.mod.ExportedFunction("sql3table_get_column").Call(ctx, uint64(t.ptr), uint64(i))
if err != nil {
panic(err)
}
return Column{
tab: t,
ptr: uint32(r[0]),
}
}
func (t *Table) string(ptr uint32) string {
if ptr == 0 {
return ""
}
off, _ := t.mod.Memory().ReadUint32Le(ptr + 0)
len, _ := t.mod.Memory().ReadUint32Le(ptr + 4)
return t.sql[off-baseptr : off+len-baseptr]
}
// Column holds metadata about a column.
type Column struct {
tab *Table
ptr uint32
}
// Type returns the declared type of a column.
//
// https://sqlite.org/lang_createtable.html#column_data_types
func (c Column) Type() string {
r, err := c.tab.mod.ExportedFunction("sql3column_type").Call(ctx, uint64(c.ptr))
if err != nil {
panic(err)
}
return c.tab.string(uint32(r[0]))
}

Some files were not shown because too many files have changed in this diff Show More