Compare commits

..

36 Commits

Author SHA1 Message Date
Nuno Cruces
d9b37307e7 SQLite 3.48.0. 2025-01-14 17:33:53 +00:00
Nuno Cruces
3bae1d7d4b SQLITE_FCNTL_BUSYHANDLER. 2025-01-14 17:09:54 +00:00
Nuno Cruces
8887036c20 SQLITE_FCNTL_SYNC. 2025-01-14 10:05:54 +00:00
Nuno Cruces
ccb3dcd097 SQLITE_FCNTL_PDB. 2025-01-13 13:45:41 +00:00
Nuno Cruces
a9f33cc2b0 New constants. 2025-01-13 12:05:27 +00:00
Nuno Cruces
f025ffb385 Fix naming. 2025-01-13 09:28:47 +00:00
Nuno Cruces
aa4357a78f Ordered-set aggregate syntax. 2025-01-11 19:22:04 +00:00
Nuno Cruces
aef7f051a8 Prevent modification. 2025-01-10 12:38:11 +00:00
Nuno Cruces
a79ee4c2c6 Avoid weird mutex. 2025-01-09 13:44:29 +00:00
Nuno Cruces
7424747338 Update README.md 2025-01-08 23:16:25 +00:00
Nuno Cruces
11830e05a6 Remove legacy. 2025-01-08 18:34:48 +00:00
Nuno Cruces
7dc4520690 Fix #207. 2025-01-08 16:36:41 +00:00
Nuno Cruces
0c09dd89c2 Add wasmtime to CI. (#212) 2025-01-07 16:31:12 +00:00
Nuno Cruces
31c5000875 Updated GORM driver.
Fixes https://github.com/go-gorm/sqlite/issues/192.
2025-01-07 12:29:16 +00:00
Nuno Cruces
8175407754 Enable compiler on ARMv8. (#211) 2025-01-06 18:22:36 +00:00
Nuno Cruces
abfad02d95 Remove LFS test files. (#210) 2025-01-06 11:44:08 +00:00
Nuno Cruces
f7c3fb8062 Lines delimiter. 2025-01-05 19:35:07 +00:00
Nuno Cruces
c3633dda35 Update test.yml 2024-12-21 12:13:45 +00:00
Nuno Cruces
f2d894194d Avoid syscall. 2024-12-21 10:16:23 +00:00
Nuno Cruces
e08c7b3adf Refactor. 2024-12-19 15:14:20 +00:00
Nuno Cruces
66601dd3cb More BCE. 2024-12-19 14:00:46 +00:00
Nuno Cruces
58b66b75f1 Improved assertions. 2024-12-19 13:19:10 +00:00
Nuno Cruces
e0c6086aa9 Fix POSIX locks. 2024-12-18 16:21:24 +00:00
Nuno Cruces
9bc39c5b91 Remove dependency. 2024-12-17 15:43:19 +00:00
Nuno Cruces
12193cedea binaryen-version_121. 2024-12-17 15:25:25 +00:00
Nuno Cruces
71d95bf9d5 Fix #205. 2024-12-17 14:21:56 +00:00
Nuno Cruces
7e23100ff7 Help checklocks. 2024-12-16 13:47:59 +00:00
Nuno Cruces
e32d8401fb Improve Wal locking on BSD (#204) 2024-12-16 13:15:00 +00:00
Nuno Cruces
503db60927 Fix repro.sh. 2024-12-13 16:22:44 +00:00
Nuno Cruces
1227fa7a04 Skip sleeping if blocked. 2024-12-13 16:04:37 +00:00
Nuno Cruces
e455b5f729 Coverage. 2024-12-13 10:30:08 +00:00
Nuno Cruces
2bb1c8c795 Fair retry interval. 2024-12-13 10:23:43 +00:00
Nuno Cruces
844fab4167 Fix fuzzer. 2024-12-12 13:38:46 +00:00
Nuno Cruces
5ed4a6cb9d Fix #201. 2024-12-12 12:57:18 +00:00
Nuno Cruces
37f2145588 Use CancelIoEx. 2024-12-12 10:42:23 +00:00
Nuno Cruces
e17b3ef2c8 wasi-sdk-25. 2024-12-12 09:51:23 +00:00
111 changed files with 1366 additions and 523 deletions

View File

@@ -1,23 +0,0 @@
name: Git LFS pull
description: Cached Git LFS pull.
runs:
using: "composite"
steps:
- name: Create LFS file list
shell: bash
run: git lfs ls-files --long | cut -d ' ' -f1 | sort > .lfs-assets-id
- name: Restore LFS cache
uses: actions/cache@v4
with:
path: .git/lfs/objects
key: lfs-${{ hashFiles('.lfs-assets-id') }}
restore-keys: lfs-
enableCrossOsArchive: true
- name: Git LFS pull
shell: bash
run: |
git lfs pull
git lfs prune

View File

@@ -1,25 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
echo android ; GOOS=android GOARCH=amd64 go build .
echo darwin ; GOOS=darwin GOARCH=amd64 go build .
echo dragonfly ; GOOS=dragonfly GOARCH=amd64 go build .
echo freebsd ; GOOS=freebsd GOARCH=amd64 go build .
echo illumos ; GOOS=illumos GOARCH=amd64 go build .
echo ios ; GOOS=ios GOARCH=amd64 go build .
echo linux ; GOOS=linux GOARCH=amd64 go build .
echo netbsd ; GOOS=netbsd GOARCH=amd64 go build .
echo openbsd ; GOOS=openbsd GOARCH=amd64 go build .
echo plan9 ; GOOS=plan9 GOARCH=amd64 go build .
echo solaris ; GOOS=solaris GOARCH=amd64 go build .
echo windows ; GOOS=windows GOARCH=amd64 go build .
echo aix ; GOOS=aix GOARCH=ppc64 go build .
echo js ; GOOS=js GOARCH=wasm go build .
echo wasip1 ; GOOS=wasip1 GOARCH=wasm go build .
echo linux-flock ; GOOS=linux GOARCH=amd64 go build -tags sqlite3_flock .
echo linux-dotlk ; GOOS=linux GOARCH=amd64 go build -tags sqlite3_dotlk .
echo darwin-flock ; GOOS=darwin GOARCH=amd64 go build -tags sqlite3_flock .
echo darwin-dotlk ; GOOS=darwin GOARCH=amd64 go build -tags sqlite3_dotlk .
echo windows-dotlk ; GOOS=windows GOARCH=amd64 go build -tags sqlite3_dotlk .
echo freebsd-dotlk ; GOOS=freebsd GOARCH=amd64 go build -tags sqlite3_dotlk .
echo solaris-dotlk ; GOOS=solaris GOARCH=amd64 go build -tags sqlite3_dotlk .

View File

@@ -1,16 +0,0 @@
name: Cross compile
on:
workflow_dispatch:
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with: { go-version: stable }
- name: Build
run: .github/workflows/cross.sh

View File

@@ -2,14 +2,14 @@
set -euo pipefail
if [[ "$OSTYPE" == "linux"* ]]; then
WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/wasi-sdk-24.0-x86_64-linux.tar.gz"
BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_120/binaryen-version_120-x86_64-linux.tar.gz"
WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/wasi-sdk-25.0-x86_64-linux.tar.gz"
BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_121/binaryen-version_121-x86_64-linux.tar.gz"
elif [[ "$OSTYPE" == "darwin"* ]]; then
WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/wasi-sdk-24.0-arm64-macos.tar.gz"
BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_120/binaryen-version_120-arm64-macos.tar.gz"
WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/wasi-sdk-25.0-arm64-macos.tar.gz"
BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_121/binaryen-version_121-arm64-macos.tar.gz"
elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then
WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/wasi-sdk-24.0-x86_64-windows.tar.gz"
BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_120/binaryen-version_120-x86_64-windows.tar.gz"
WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/wasi-sdk-25.0-x86_64-windows.tar.gz"
BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_121/binaryen-version_121-x86_64-windows.tar.gz"
fi
# Download tools
@@ -27,8 +27,8 @@ embed/build.sh
embed/bcw2/build.sh
# Download and build sqlite-createtable-parser
util/sql3util/parse/download.sh
util/sql3util/parse/build.sh
util/sql3util/wasm/download.sh
util/sql3util/wasm/build.sh
# Check diffs
git diff --exit-code

View File

@@ -18,8 +18,6 @@ jobs:
steps:
- uses: ilammy/msvc-dev-cmd@v1
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with: { go-version: stable }
- name: Build
shell: bash
@@ -31,4 +29,4 @@ jobs:
subject-path: |
embed/sqlite3.wasm
embed/bcw2/bcw2.wasm
util/sql3util/parse/sql3parse_table.wasm
util/sql3util/wasm/sql3parse_table.wasm

View File

@@ -2,19 +2,17 @@ name: Test
on:
push:
branches: [ "main" ]
branches: [ 'main' ]
paths:
- '**.go'
- '**.mod'
- '**.wasm'
- '**.wasm.bz2'
pull_request:
branches: [ "main" ]
branches: [ 'main' ]
paths:
- '**.go'
- '**.mod'
- '**.wasm'
- '**.wasm.bz2'
workflow_dispatch:
jobs:
@@ -29,9 +27,6 @@ jobs:
- uses: actions/setup-go@v5
with: { go-version: stable }
- name: Git LFS pull
uses: ./.github/actions/lfs
- name: Format
run: gofmt -s -w . && git diff --exit-code
if: matrix.os != 'windows-latest'
@@ -56,7 +51,7 @@ jobs:
- name: Test BSD locks
run: go test -v -tags sqlite3_flock ./...
if: matrix.os == 'macos-latest'
if: matrix.os != 'windows-latest'
- name: Test dot locks
run: go test -v -tags sqlite3_dotlk ./...
@@ -66,6 +61,10 @@ jobs:
shell: bash
run: gormlite/test.sh
- name: Test modules
shell: bash
run: go test -v ./embed/bcw2/...
- name: Collect coverage
run: go run github.com/dave/courtney@latest
if: |
@@ -91,6 +90,14 @@ jobs:
- name: netbsd
version: '10.0'
flags: '-test.v'
- name: freebsd
arch: arm64
version: '14.2'
flags: '-test.v -test.short'
- name: netbsd
arch: arm64
version: '10.0'
flags: '-test.v -test.short'
- name: openbsd
version: '7.6'
flags: '-test.v -test.short'
@@ -100,12 +107,10 @@ jobs:
steps:
- uses: actions/checkout@v4
- name: Git LFS pull
uses: ./.github/actions/lfs
- name: Build
env:
GOOS: ${{ matrix.os.name }}
GOARCH: ${{ matrix.os.arch }}
TESTFLAGS: ${{ matrix.os.flags }}
run: .github/workflows/build-test.sh
@@ -113,6 +118,7 @@ jobs:
uses: cross-platform-actions/action@v0.26.0
with:
operating_system: ${{ matrix.os.name }}
architecture: ${{ matrix.os.arch }}
version: ${{ matrix.os.version }}
shell: bash
run: . ./test.sh
@@ -138,9 +144,6 @@ jobs:
steps:
- uses: actions/checkout@v4
- name: Git LFS pull
uses: ./.github/actions/lfs
- name: Build
env:
GOOS: ${{ matrix.os.name }}
@@ -156,6 +159,27 @@ jobs:
copyback: false
run: . ./test.sh
test-wasip1:
runs-on: ubuntu-latest
needs: test
steps:
- uses: bytecodealliance/actions/wasmtime/setup@v1
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with: { go-version: stable }
- name: Set path
run: echo "$(go env GOROOT)/misc/wasm" >> "$GITHUB_PATH"
- name: Test wasmtime
env:
GOOS: wasip1
GOARCH: wasm
GOWASIRUNTIME: wasmtime
GOWASIRUNTIMEARGS: '--env CI=true'
run: go test -v -short -tags sqlite3_dotlk -skip Example ./...
test-qemu:
runs-on: ubuntu-latest
needs: test
@@ -166,9 +190,6 @@ jobs:
- uses: actions/setup-go@v5
with: { go-version: stable }
- name: Git LFS pull
uses: ./.github/actions/lfs
- name: Test 386 (32-bit)
run: GOARCH=386 go test -v -short ./...
@@ -193,8 +214,5 @@ jobs:
- uses: actions/setup-go@v5
with: { go-version: stable }
- name: Git LFS pull
uses: ./.github/actions/lfs
- name: Test
run: go test -v ./...

View File

@@ -74,9 +74,9 @@ This project aims for [high test coverage](https://github.com/ncruces/go-sqlite3
It also benefits greatly from [SQLite's](https://sqlite.org/testing.html) and
[wazero's](https://tetrate.io/blog/introducing-wazero-from-tetrate/#:~:text=Rock%2Dsolid%20test%20approach) thorough testing.
Every commit is [tested](https://github.com/ncruces/go-sqlite3/wiki/Test-matrix) on
Every commit is [tested](https://github.com/ncruces/go-sqlite3/wiki/Support-matrix) on
Linux (amd64/arm64/386/riscv64/ppc64le/s390x), macOS (amd64/arm64),
Windows (amd64), FreeBSD (amd64), OpenBSD (amd64), NetBSD (amd64),
Windows (amd64), FreeBSD (amd64/arm64), OpenBSD (amd64), NetBSD (amd64/arm64),
DragonFly BSD (amd64), illumos (amd64), and Solaris (amd64).
The Go VFS is tested by running SQLite's

58
conn.go
View File

@@ -4,7 +4,9 @@ import (
"context"
"fmt"
"math"
"math/rand"
"net/url"
"runtime"
"strings"
"time"
@@ -24,7 +26,6 @@ type Conn struct {
interrupt context.Context
pending *Stmt
stmts []*Stmt
timer *time.Timer
busy func(context.Context, int) bool
log func(xErrorCode, string)
collation func(*Conn, string)
@@ -36,7 +37,9 @@ type Conn struct {
rollback func()
arena arena
handle uint32
busy1st time.Time
busylst time.Time
handle uint32
}
// Open calls [OpenFlags] with [OPEN_READWRITE], [OPEN_CREATE] and [OPEN_URI].
@@ -65,7 +68,7 @@ func OpenFlags(filename string, flags OpenFlag) (*Conn, error) {
return newConn(context.Background(), filename, flags)
}
type connKey struct{}
type connKey = util.ConnKey
func newConn(ctx context.Context, filename string, flags OpenFlag) (res *Conn, _ error) {
err := ctx.Err()
@@ -373,8 +376,13 @@ func (c *Conn) checkInterrupt(handle uint32) {
}
func progressCallback(ctx context.Context, mod api.Module, _ uint32) (interrupt uint32) {
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.interrupt.Err() != nil {
interrupt = 1
if c, ok := ctx.Value(connKey{}).(*Conn); ok {
if c.interrupt.Done() != nil {
runtime.Gosched()
}
if c.interrupt.Err() != nil {
interrupt = 1
}
}
return interrupt
}
@@ -389,38 +397,20 @@ func (c *Conn) BusyTimeout(timeout time.Duration) error {
}
func timeoutCallback(ctx context.Context, mod api.Module, count, tmout int32) (retry uint32) {
// https://fractaledmind.github.io/2024/04/15/sqlite-on-rails-the-how-and-why-of-optimal-performance/
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.interrupt.Err() == nil {
const delays = "\x01\x02\x05\x0a\x0f\x14\x19\x19\x19\x32\x32\x64"
const totals = "\x00\x01\x03\x08\x12\x21\x35\x4e\x67\x80\xb2\xe4"
const ndelay = int32(len(delays) - 1)
var delay, prior int32
if count <= ndelay {
delay = int32(delays[count])
prior = int32(totals[count])
} else {
delay = int32(delays[ndelay])
prior = int32(totals[ndelay]) + delay*(count-ndelay)
switch {
case count == 0:
c.busy1st = time.Now()
case time.Since(c.busy1st) >= time.Duration(tmout)*time.Millisecond:
return 0
}
if delay = min(delay, tmout-prior); delay > 0 {
delay := time.Duration(delay) * time.Millisecond
if c.interrupt.Done() == nil {
time.Sleep(delay)
return 1
}
if c.timer == nil {
c.timer = time.NewTimer(delay)
} else {
c.timer.Reset(delay)
}
select {
case <-c.interrupt.Done():
c.timer.Stop()
case <-c.timer.C:
return 1
}
if time.Since(c.busylst) < time.Millisecond {
const sleepIncrement = 2*1024*1024 - 1 // power of two, ~2ms
time.Sleep(time.Duration(rand.Int63() & sleepIncrement))
}
c.busylst = time.Now()
return 1
}
return 0
}

View File

@@ -166,6 +166,7 @@ const (
PREPARE_PERSISTENT PrepareFlag = 0x01
PREPARE_NORMALIZE PrepareFlag = 0x02
PREPARE_NO_VTAB PrepareFlag = 0x04
PREPARE_DONT_LOG PrepareFlag = 0x10
)
// FunctionFlag is a flag that can be passed to
@@ -219,6 +220,7 @@ const (
DBSTATUS_DEFERRED_FKS DBStatus = 10
DBSTATUS_CACHE_USED_SHARED DBStatus = 11
DBSTATUS_CACHE_SPILL DBStatus = 12
// DBSTATUS_MAX DBStatus = 12
)
// DBConfig are the available database connection configuration options.

View File

@@ -274,6 +274,7 @@ func (n *connector) Connect(ctx context.Context) (res driver.Conn, err error) {
// if err != nil {
// log.Fatal(err)
// }
// defer conn.Close()
//
// err = conn.Raw(func(driverConn any) error {
// conn := driverConn.(driver.Conn)
@@ -379,7 +380,7 @@ func (c *conn) PrepareContext(ctx context.Context, query string) (driver.Stmt, e
if err != nil {
return nil, err
}
if tail != "" {
if notWhitespace(tail) {
s.Close()
return nil, util.TailErr
}

View File

@@ -225,8 +225,8 @@ func Test_Prepare(t *testing.T) {
}
_, err = db.Prepare(`SELECT 1; `)
if err.Error() != string(util.TailErr) {
t.Error("want tailErr")
if err != nil {
t.Error(err)
}
_, err = db.Prepare(`SELECT 1; SELECT`)

View File

@@ -6,6 +6,7 @@ import (
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
_ "github.com/ncruces/go-sqlite3/vfs/memdb"
)

View File

@@ -27,12 +27,12 @@ func Fuzz_stringOrTime_1(f *testing.F) {
// Make sure times round-trip to the same string:
// https://pkg.go.dev/database/sql#Rows.Scan
if v.Format(time.RFC3339Nano) != str {
t.Fatalf("did not round-trip: %q", str)
t.Errorf("did not round-trip: %q", str)
}
} else {
date, err := time.Parse(time.RFC3339Nano, str)
if err == nil && date.Format(time.RFC3339Nano) == str {
t.Fatalf("would round-trip: %q", str)
t.Errorf("would round-trip: %q", str)
}
}
})

View File

@@ -12,3 +12,63 @@ func namedValues(args []driver.Value) []driver.NamedValue {
}
return named
}
func notWhitespace(sql string) bool {
const (
code = iota
slash
minus
ccomment
sqlcomment
endcomment
)
state := code
for _, b := range ([]byte)(sql) {
if b == 0 {
break
}
switch state {
case code:
switch b {
case '/':
state = slash
case '-':
state = minus
case ' ', ';', '\t', '\n', '\v', '\f', '\r':
continue
default:
return true
}
case slash:
if b != '*' {
return true
}
state = ccomment
case minus:
if b != '-' {
return true
}
state = sqlcomment
case ccomment:
if b == '*' {
state = endcomment
}
case sqlcomment:
if b == '\n' {
state = code
}
case endcomment:
switch b {
case '/':
state = code
case '*':
state = endcomment
default:
state = ccomment
}
}
}
return state == slash || state == minus
}

View File

@@ -1,9 +1,13 @@
package driver
import (
"context"
"database/sql/driver"
"reflect"
"testing"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
func Test_namedValues(t *testing.T) {
@@ -16,3 +20,67 @@ func Test_namedValues(t *testing.T) {
t.Errorf("got %v, want %v", got, want)
}
}
func Fuzz_notWhitespace(f *testing.F) {
f.Add("")
f.Add(" ")
f.Add(";")
f.Add("0")
f.Add("-")
f.Add("-0")
f.Add("--")
f.Add("--0")
f.Add("--\n")
f.Add("--0\n")
f.Add("/0")
f.Add("/*")
f.Add("/*/")
f.Add("/**")
f.Add("/*0")
f.Add("/**/")
f.Add("/***/")
f.Add("/**0/")
f.Add("\v")
f.Add(" \v")
f.Add("\xf0")
f.Add("\000")
db, err := Open(":memory:")
if err != nil {
f.Fatal(err)
}
defer db.Close()
f.Fuzz(func(t *testing.T, str string) {
if len(str) > 128 {
t.SkipNow()
}
c, err := db.Conn(context.Background())
if err != nil {
t.Fatal(err)
}
defer c.Close()
c.Raw(func(driverConn any) error {
conn := driverConn.(*conn).Conn
stmt, tail, err := conn.Prepare(str)
stmt.Close()
// It's hard to be bug for bug compatible with SQLite.
// We settle for somewhat less:
// - if SQLite reports whitespace, we must too
// - if we report whitespace, SQLite must not parse a statement
if notWhitespace(str) {
if stmt == nil && tail == "" && err == nil {
t.Errorf("was whitespace: %q", str)
}
} else {
if stmt != nil {
t.Errorf("was not whitespace: %q (%v)", str, err)
}
}
return nil
})
})
}

View File

@@ -1,6 +1,6 @@
# Embeddable Wasm build of SQLite
This folder includes an embeddable Wasm build of SQLite 3.47.2 for use with
This folder includes an embeddable Wasm build of SQLite 3.48.0 for use with
[`github.com/ncruces/go-sqlite3`](https://pkg.go.dev/github.com/ncruces/go-sqlite3).
The following optional features are compiled in:

View File

@@ -1,13 +1,19 @@
# Embeddable Wasm build of SQLite
This folder includes an embeddable Wasm build of SQLite, including the experimental
This folder includes an alternative embeddable Wasm build of SQLite,
which includes the experimental
[`BEGIN CONCURRENT`](https://sqlite.org/src/doc/begin-concurrent/doc/begin_concurrent.md) and
[Wal2](https://sqlite.org/cgi/src/doc/wal2/doc/wal2.md) patches.
It also enables the optional
[`UPDATE … ORDER BY … LIMIT`](https://sqlite.org/lang_update.html#optional_limit_and_order_by_clauses) and
[`DELETE … ORDER BY … LIMIT`](https://sqlite.org/lang_delete.html#optional_limit_and_order_by_clauses) clauses,
and the [`WITHIN GROUP ORDER BY`](https://sqlite.org/compile.html#enable_ordered_set_aggregates) aggregate syntax.
> [!IMPORTANT]
> This package is experimental.
> It is built from the `bedrock` branch of SQLite,
> since that is _currently_ the most stable, maintained branch to include both features.
> since that is _currently_ the most stable, maintained branch to include these features.
> [!CAUTION]
> The Wal2 journaling mode creates databases that other versions of SQLite cannot access.

Binary file not shown.

View File

@@ -5,6 +5,7 @@ import (
"testing"
"github.com/ncruces/go-sqlite3/driver"
"github.com/ncruces/go-sqlite3/ext/stats"
"github.com/ncruces/go-sqlite3/vfs"
)
@@ -15,7 +16,7 @@ func Test_bcw2(t *testing.T) {
tmp := filepath.ToSlash(filepath.Join(t.TempDir(), "test.db"))
db, err := driver.Open("file:" + tmp + "?_pragma=journal_mode(wal2)&_txlock=concurrent")
db, err := driver.Open("file:"+tmp+"?_pragma=journal_mode(wal2)&_txlock=concurrent", stats.Register)
if err != nil {
t.Fatal(err)
}
@@ -32,6 +33,16 @@ func Test_bcw2(t *testing.T) {
t.Fatal(err)
}
_, err = tx.Exec(`DELETE FROM test LIMIT 1`)
if err != nil {
t.Fatal(err)
}
_, err = tx.Exec(`SELECT median() WITHIN GROUP (ORDER BY col) FROM test`)
if err != nil {
t.Fatal(err)
}
err = tx.Commit()
if err != nil {
t.Fatal(err)

View File

@@ -13,15 +13,15 @@ mkdir -p build/ext/
cp "$ROOT"/sqlite3/*.[ch] build/
cp "$ROOT"/sqlite3/*.patch build/
# https://sqlite.org/src/info/08cfa7e8b3090151
curl -# https://sqlite.org/src/tarball/sqlite.tar.gz?r=08cfa7e8 | tar xz
# https://sqlite.org/src/info/fab341c829554573
curl -# https://sqlite.org/src/tarball/sqlite.tar.gz?r=fab341c8 | tar xz
cd sqlite
if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then
MSYS_NO_PATHCONV=1 nmake /f makefile.msc sqlite3.c
MSYS_NO_PATHCONV=1 nmake /f makefile.msc sqlite3.c "OPTS=-DSQLITE_ENABLE_UPDATE_DELETE_LIMIT -DSQLITE_ENABLE_ORDERED_SET_AGGREGATES"
else
sh configure
make sqlite3.c
sh configure --enable-update-limit
OPTS=-DSQLITE_ENABLE_ORDERED_SET_AGGREGATES make sqlite3.c
fi
cd ~-
@@ -38,7 +38,7 @@ mv sqlite/ext/misc/spellfix.c build/ext/
mv sqlite/ext/misc/uint.c build/ext/
cd build
cat *.patch | patch --no-backup-if-mismatch
cat *.patch | patch -p0 --no-backup-if-mismatch
cd ~-
"$WASI_SDK/clang" --target=wasm32-wasi -std=c23 -g0 -O2 \
@@ -46,7 +46,7 @@ cd ~-
-o bcw2.wasm "build/main.c" \
-I"build" \
-mexec-model=reactor \
-matomics -msimd128 -mmutable-globals -mmultivalue \
-msimd128 -mmutable-globals -mmultivalue \
-mbulk-memory -mreference-types \
-mnontrapping-fptoint -msign-ext \
-fno-stack-protector -fno-stack-clash-protection \
@@ -54,6 +54,7 @@ cd ~-
-Wl,--import-undefined \
-Wl,--initial-memory=327680 \
-D_HAVE_SQLITE_CONFIG_H \
-DSQLITE_ENABLE_UPDATE_DELETE_LIMIT \
-DSQLITE_CUSTOM_INCLUDE=sqlite_opt.h \
$(awk '{print "-Wl,--export="$0}' ../exports.txt)

View File

@@ -4,10 +4,11 @@ go 1.21
toolchain go1.23.0
require github.com/ncruces/go-sqlite3 v0.21.0
require github.com/ncruces/go-sqlite3 v0.21.3
require (
github.com/ncruces/julianday v1.0.0 // indirect
github.com/ncruces/sort v0.1.2 // indirect
github.com/tetratelabs/wazero v1.8.2 // indirect
golang.org/x/sys v0.28.0 // indirect
golang.org/x/sys v0.29.0 // indirect
)

View File

@@ -1,10 +1,12 @@
github.com/ncruces/go-sqlite3 v0.21.0 h1:EwKFoy1hHEopN4sFZarmi+McXdbCcbTuLixhEayXVbQ=
github.com/ncruces/go-sqlite3 v0.21.0/go.mod h1:zxMOaSG5kFYVFK4xQa0pdwIszqxqJ0W0BxBgwdrNjuA=
github.com/ncruces/go-sqlite3 v0.21.3 h1:hHkfNQLcbnxPJZhC/RGw9SwP3bfkv/Y0xUHWsr1CdMQ=
github.com/ncruces/go-sqlite3 v0.21.3/go.mod h1:zxMOaSG5kFYVFK4xQa0pdwIszqxqJ0W0BxBgwdrNjuA=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/ncruces/sort v0.1.2 h1:zKQ9CA4fpHPF6xsUhRTfi5EEryspuBpe/QA4VWQOV1U=
github.com/ncruces/sort v0.1.2/go.mod h1:vEJUTBJtebIuCMmXD18GKo5GJGhsay+xZFOoBEIXFmE=
github.com/tetratelabs/wazero v1.8.2 h1:yIgLR/b2bN31bjxwXHD8a3d+BogigR952csSDdLYEv4=
github.com/tetratelabs/wazero v1.8.2/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=

View File

@@ -11,13 +11,14 @@ package bcw2
import (
_ "embed"
"unsafe"
"github.com/ncruces/go-sqlite3"
)
//go:embed bcw2.wasm
var binary []byte
var binary string
func init() {
sqlite3.Binary = binary
sqlite3.Binary = unsafe.Slice(unsafe.StringData(binary), len(binary))
}

View File

@@ -14,7 +14,7 @@ trap 'rm -f sqlite3.tmp' EXIT
-o sqlite3.wasm "$ROOT/sqlite3/main.c" \
-I"$ROOT/sqlite3" \
-mexec-model=reactor \
-matomics -msimd128 -mmutable-globals -mmultivalue \
-msimd128 -mmutable-globals -mmultivalue \
-mbulk-memory -mreference-types \
-mnontrapping-fptoint -msign-ext \
-fno-stack-protector -fno-stack-clash-protection \

View File

@@ -77,6 +77,7 @@ sqlite3_get_autocommit
sqlite3_get_auxdata
sqlite3_hard_heap_limit64
sqlite3_interrupt
sqlite3_invoke_busy_handler_go
sqlite3_last_insert_rowid
sqlite3_limit
sqlite3_malloc64

View File

@@ -8,13 +8,14 @@ package embed
import (
_ "embed"
"unsafe"
"github.com/ncruces/go-sqlite3"
)
//go:embed sqlite3.wasm
var binary []byte
var binary string
func init() {
sqlite3.Binary = binary
sqlite3.Binary = unsafe.Slice(unsafe.StringData(binary), len(binary))
}

View File

@@ -19,7 +19,7 @@ func Test_init(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if version != "3.47.2" {
if version != "3.48.0" {
t.Error(version)
}
}

Binary file not shown.

View File

@@ -44,6 +44,8 @@ func Register(db *sqlite3.Conn) error {
type OpenCallback func(*sqlite3.Blob, ...sqlite3.Value) error
func readblob(ctx sqlite3.Context, arg ...sqlite3.Value) {
_ = arg[5] // bounds check
blob, err := getAuxBlob(ctx, arg, false)
if err != nil {
ctx.ResultError(err)
@@ -78,6 +80,8 @@ func readblob(ctx sqlite3.Context, arg ...sqlite3.Value) {
}
func writeblob(ctx sqlite3.Context, arg ...sqlite3.Value) {
_ = arg[5] // bounds check
blob, err := getAuxBlob(ctx, arg, true)
if err != nil {
ctx.ResultError(err)

View File

@@ -84,10 +84,11 @@ func (c *closure) BestIndex(idx *sqlite3.IndexInfo) error {
cost := 1e7
for i, cst := range idx.Constraint {
if !cst.Usable {
switch {
case !cst.Usable:
continue
}
if plan&1 == 0 && cst.Column == _COL_ROOT {
case plan&1 == 0 && cst.Column == _COL_ROOT:
switch cst.Op {
case sqlite3.INDEX_CONSTRAINT_EQ:
plan |= 1
@@ -97,9 +98,8 @@ func (c *closure) BestIndex(idx *sqlite3.IndexInfo) error {
Omit: true,
}
}
continue
}
if plan&0xf0 == 0 && cst.Column == _COL_DEPTH {
case plan&0xf0 == 0 && cst.Column == _COL_DEPTH:
switch cst.Op {
case sqlite3.INDEX_CONSTRAINT_LT, sqlite3.INDEX_CONSTRAINT_LE, sqlite3.INDEX_CONSTRAINT_EQ:
plan |= posi << 4
@@ -110,9 +110,8 @@ func (c *closure) BestIndex(idx *sqlite3.IndexInfo) error {
plan |= 2
}
}
continue
}
if plan&0xf00 == 0 && cst.Column == _COL_TABLENAME {
case plan&0xf00 == 0 && cst.Column == _COL_TABLENAME:
switch cst.Op {
case sqlite3.INDEX_CONSTRAINT_EQ:
plan |= posi << 8
@@ -123,9 +122,8 @@ func (c *closure) BestIndex(idx *sqlite3.IndexInfo) error {
Omit: true,
}
}
continue
}
if plan&0xf000 == 0 && cst.Column == _COL_IDCOLUMN {
case plan&0xf000 == 0 && cst.Column == _COL_IDCOLUMN:
switch cst.Op {
case sqlite3.INDEX_CONSTRAINT_EQ:
plan |= posi << 12
@@ -135,9 +133,8 @@ func (c *closure) BestIndex(idx *sqlite3.IndexInfo) error {
Omit: true,
}
}
continue
}
if plan&0xf0000 == 0 && cst.Column == _COL_PARENTCOLUMN {
case plan&0xf0000 == 0 && cst.Column == _COL_PARENTCOLUMN:
switch cst.Op {
case sqlite3.INDEX_CONSTRAINT_EQ:
plan |= posi << 16
@@ -147,7 +144,6 @@ func (c *closure) BestIndex(idx *sqlite3.IndexInfo) error {
Omit: true,
}
}
continue
}
}

View File

@@ -38,7 +38,7 @@ func RegisterFS(db *sqlite3.Conn, fsys fs.FS) error {
return errors.Join(
sqlite3.CreateModule(db, "lines", nil,
func(db *sqlite3.Conn, _, _, _ string, _ ...string) (lines, error) {
err := db.DeclareVTab(`CREATE TABLE x(line TEXT, data HIDDEN)`)
err := db.DeclareVTab(`CREATE TABLE x(line TEXT, data HIDDEN, delim HIDDEN)`)
if err == nil {
err = db.VTabConfig(sqlite3.VTAB_INNOCUOUS)
}
@@ -46,7 +46,7 @@ func RegisterFS(db *sqlite3.Conn, fsys fs.FS) error {
}),
sqlite3.CreateModule(db, "lines_read", nil,
func(db *sqlite3.Conn, _, _, _ string, _ ...string) (lines, error) {
err := db.DeclareVTab(`CREATE TABLE x(line TEXT, data HIDDEN)`)
err := db.DeclareVTab(`CREATE TABLE x(line TEXT, data HIDDEN, delim HIDDEN)`)
if err == nil {
err = db.VTabConfig(sqlite3.VTAB_DIRECTONLY)
}
@@ -58,19 +58,29 @@ type lines struct {
fsys fs.FS
}
func (l lines) BestIndex(idx *sqlite3.IndexInfo) error {
func (l lines) BestIndex(idx *sqlite3.IndexInfo) (err error) {
err = sqlite3.CONSTRAINT
for i, cst := range idx.Constraint {
if cst.Column == 1 && cst.Op == sqlite3.INDEX_CONSTRAINT_EQ && cst.Usable {
if !cst.Usable || cst.Op != sqlite3.INDEX_CONSTRAINT_EQ {
continue
}
switch cst.Column {
case 1:
idx.ConstraintUsage[i] = sqlite3.IndexConstraintUsage{
Omit: true,
ArgvIndex: 1,
}
idx.EstimatedCost = 1e6
idx.EstimatedRows = 100
return nil
err = nil
case 2:
idx.ConstraintUsage[i] = sqlite3.IndexConstraintUsage{
Omit: true,
ArgvIndex: 2,
}
}
}
return sqlite3.CONSTRAINT
return err
}
func (l lines) Open() (sqlite3.VTabCursor, error) {
@@ -85,6 +95,7 @@ type cursor struct {
line []byte
rowID int64
eof bool
delim byte
}
func (c *cursor) EOF() bool {
@@ -140,6 +151,15 @@ func (c *reader) Filter(idxNum int, idxStr string, arg ...sqlite3.Value) error {
return fmt.Errorf("lines: unsupported argument:%.0w %v", sqlite3.MISMATCH, typ)
}
c.delim = '\n'
if len(arg) > 1 {
b := arg[1].RawText()
if len(b) != 1 {
return fmt.Errorf("lines: delimiter must be a single byte%.0w", sqlite3.MISMATCH)
}
c.delim = b[0]
}
c.reader = bufio.NewReader(r)
c.closer, _ = r.(io.Closer)
c.rowID = 0
@@ -150,7 +170,12 @@ func (c *reader) Next() (err error) {
c.line = c.line[:0]
for more := true; more; {
var line []byte
line, more, err = c.reader.ReadLine()
if c.delim == '\n' {
line, more, err = c.reader.ReadLine()
} else {
line, err = c.reader.ReadSlice(c.delim)
more = err == bufio.ErrBufferFull
}
c.line = append(c.line, line...)
}
if err == io.EOF {
@@ -177,18 +202,27 @@ func (c *buffer) Filter(idxNum int, idxStr string, arg ...sqlite3.Value) error {
return fmt.Errorf("lines: unsupported argument:%.0w %v", sqlite3.MISMATCH, typ)
}
c.delim = '\n'
if len(arg) > 1 {
b := arg[1].RawText()
if len(b) != 1 {
return fmt.Errorf("lines: delimiter must be a single byte%.0w", sqlite3.MISMATCH)
}
c.delim = b[0]
}
c.rowID = 0
return c.Next()
}
func (c *buffer) Next() error {
i := bytes.IndexByte(c.data, '\n')
i := bytes.IndexByte(c.data, c.delim)
j := i + 1
switch {
case i < 0:
i = len(c.data)
j = i
case i > 0 && c.data[i-1] == '\r':
case i > 0 && c.delim == '\n' && c.data[i-1] == '\r':
i--
}
c.eof = len(c.data) == 0

View File

@@ -163,7 +163,7 @@ func Test_lines_test(t *testing.T) {
}
defer db.Close()
rows, err := db.Query(`SELECT rowid, line FROM lines_read(?)`, "lines_test.go")
rows, err := db.Query(`SELECT rowid, line FROM lines_read(?, '}')`, "lines_test.go")
if errors.Is(err, os.ErrNotExist) {
t.Skip(err)
}

View File

@@ -76,6 +76,7 @@ func load(ctx sqlite3.Context, i int, expr string) (*regexp.Regexp, error) {
}
func regex(ctx sqlite3.Context, arg ...sqlite3.Value) {
_ = arg[1] // bounds check
re, err := load(ctx, 0, arg[0].Text())
if err != nil {
ctx.ResultError(err)
@@ -165,6 +166,8 @@ func regexInstr(ctx sqlite3.Context, arg ...sqlite3.Value) {
}
func regexReplace(ctx sqlite3.Context, arg ...sqlite3.Value) {
_ = arg[2] // bounds check
re, err := load(ctx, 1, arg[1].Text())
if err != nil {
ctx.ResultError(err)

136
ext/serdes/serdes.go Normal file
View File

@@ -0,0 +1,136 @@
// Package serdes provides functions to (de)serialize databases.
package serdes
import (
"io"
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/vfs"
)
func init() {
vfs.Register(vfsName, sliceVFS{})
}
// Serialize backs up a database into a byte slice.
//
// https://sqlite.org/c3ref/serialize.html
func Serialize(db *sqlite3.Conn, schema string) ([]byte, error) {
var file sliceFile
fileToOpen <- &file
err := db.Backup(schema, "file:db?vfs="+vfsName)
return file.data, err
}
// Deserialize restores a database from a byte slice,
// DESTROYING any contents previously stored in schema.
//
// To non-destructively open a database from a byte slice,
// consider alternatives like the ["reader"] or ["memdb"] VFSes.
//
// This differs from the similarly named SQLite API
// in that it DOES NOT disconnect from schema
// to reopen as an in-memory database.
//
// https://sqlite.org/c3ref/deserialize.html
//
// ["memdb"]: https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/memdb
// ["reader"]: https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/readervfs
func Deserialize(db *sqlite3.Conn, schema string, data []byte) error {
fileToOpen <- &sliceFile{data}
return db.Restore(schema, "file:db?vfs="+vfsName)
}
var fileToOpen = make(chan *sliceFile, 1)
const vfsName = "github.com/ncruces/go-sqlite3/ext/deserialize.sliceVFS"
type sliceVFS struct{}
func (sliceVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
if flags&vfs.OPEN_MAIN_DB == 0 {
// notest // OPEN_MEMORY
return nil, flags, sqlite3.CANTOPEN
}
return <-fileToOpen, flags | vfs.OPEN_MEMORY, nil
}
func (sliceVFS) Delete(name string, dirSync bool) error {
// notest // OPEN_MEMORY
return sqlite3.IOERR_DELETE
}
func (sliceVFS) Access(name string, flag vfs.AccessFlag) (bool, error) {
return name == "db", nil
}
func (sliceVFS) FullPathname(name string) (string, error) {
return name, nil
}
type sliceFile struct{ data []byte }
func (f *sliceFile) ReadAt(b []byte, off int64) (n int, err error) {
if d := f.data; off < int64(len(d)) {
n = copy(b, d[off:])
}
if n == 0 {
err = io.EOF
}
return
}
func (f *sliceFile) WriteAt(b []byte, off int64) (n int, err error) {
if d := f.data; off > int64(len(d)) {
f.data = append(d, make([]byte, off-int64(len(d)))...)
}
d := append(f.data[:off], b...)
if len(d) > len(f.data) {
f.data = d
}
return len(b), nil
}
func (f *sliceFile) Size() (int64, error) {
return int64(len(f.data)), nil
}
func (f *sliceFile) Truncate(size int64) error {
if d := f.data; size < int64(len(d)) {
f.data = d[:size]
}
return nil
}
func (f *sliceFile) SizeHint(size int64) error {
if d := f.data; size > int64(len(d)) {
f.data = append(d, make([]byte, size-int64(len(d)))...)
}
return nil
}
func (*sliceFile) Close() error { return nil }
func (*sliceFile) Sync(flag vfs.SyncFlag) error { return nil }
func (*sliceFile) Lock(lock vfs.LockLevel) error { return nil }
func (*sliceFile) Unlock(lock vfs.LockLevel) error { return nil }
func (*sliceFile) CheckReservedLock() (bool, error) {
// notest // OPEN_MEMORY
return false, nil
}
func (*sliceFile) SectorSize() int {
// notest // IOCAP_POWERSAFE_OVERWRITE
return 0
}
func (*sliceFile) DeviceCharacteristics() vfs.DeviceCharacteristic {
return vfs.IOCAP_ATOMIC |
vfs.IOCAP_SAFE_APPEND |
vfs.IOCAP_SEQUENTIAL |
vfs.IOCAP_POWERSAFE_OVERWRITE |
vfs.IOCAP_SUBPAGE_READ
}

68
ext/serdes/serdes_test.go Normal file
View File

@@ -0,0 +1,68 @@
package serdes_test
import (
"io"
"net/http"
"testing"
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/ext/serdes"
)
func TestDeserialize(t *testing.T) {
if testing.Short() {
t.Skip("skipping in short mode")
}
input, err := httpGet()
if err != nil {
t.Fatal(err)
}
db, err := sqlite3.Open(":memory:")
if err != nil {
t.Fatal(err)
}
defer db.Close()
err = serdes.Deserialize(db, "temp", input)
if err != nil {
t.Fatal(err)
}
output, err := serdes.Serialize(db, "temp")
if err != nil {
t.Fatal(err)
}
if len(input) != len(output) {
t.Fatal("lengths are different")
}
for i := range input {
// These may be different.
switch {
case 24 <= i && i < 28:
// File change counter.
continue
case 40 <= i && i < 44:
// Schema cookie.
continue
case 92 <= i && i < 100:
// SQLite version that wrote the file.
continue
}
if input[i] != output[i] {
t.Errorf("difference at %d: %d %d", i, input[i], output[i])
}
}
}
func httpGet() ([]byte, error) {
res, err := http.Get("https://raw.githubusercontent.com/jpwhite3/northwind-SQLite3/refs/heads/main/dist/northwind.db")
if err != nil {
return nil, err
}
defer res.Body.Close()
return io.ReadAll(res.Body)
}

View File

@@ -189,6 +189,7 @@ func like(ctx sqlite3.Context, arg ...sqlite3.Value) {
return
}
}
_ = arg[1] // bounds check
type likeData struct {
*regexp.Regexp

4
go.mod
View File

@@ -8,8 +8,8 @@ require (
github.com/ncruces/julianday v1.0.0
github.com/ncruces/sort v0.1.2
github.com/tetratelabs/wazero v1.8.2
golang.org/x/crypto v0.31.0
golang.org/x/sys v0.28.0
golang.org/x/crypto v0.32.0
golang.org/x/sys v0.29.0
)
require (

8
go.sum
View File

@@ -10,12 +10,12 @@ github.com/psanford/httpreadat v0.1.0 h1:VleW1HS2zO7/4c7c7zNl33fO6oYACSagjJIyMIw
github.com/psanford/httpreadat v0.1.0/go.mod h1:Zg7P+TlBm3bYbyHTKv/EdtSJZn3qwbPwpfZ/I9GKCRE=
github.com/tetratelabs/wazero v1.8.2 h1:yIgLR/b2bN31bjxwXHD8a3d+BogigR952csSDdLYEv4=
github.com/tetratelabs/wazero v1.8.2/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
lukechampine.com/adiantum v1.1.1 h1:4fp6gTxWCqpEbLy40ExiYDDED3oUNWx5cTqBCtPdZqA=

View File

@@ -12,5 +12,6 @@ golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8=
golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=

View File

@@ -17,23 +17,11 @@ var (
indexRegexp = regexp.MustCompile(fmt.Sprintf(`(?is)CREATE(?: UNIQUE)? INDEX [%v]?[\w\d-]+[%v]?(?s:.*?)ON (.*)$`, sqliteSeparator, sqliteSeparator))
tableRegexp = regexp.MustCompile(fmt.Sprintf(`(?is)(CREATE TABLE [%v]?[\w\d-]+[%v]?)(?:\s*\((.*)\))?`, sqliteSeparator, sqliteSeparator))
separatorRegexp = regexp.MustCompile(fmt.Sprintf("[%v]", sqliteSeparator))
columnsRegexp = regexp.MustCompile(fmt.Sprintf(`[(,][%v]?(\w+)[%v]?`, sqliteSeparator, sqliteSeparator))
columnRegexp = regexp.MustCompile(fmt.Sprintf(`^[%v]?([\w\d]+)[%v]?\s+([\w\(\)\d]+)(.*)$`, sqliteSeparator, sqliteSeparator))
defaultValueRegexp = regexp.MustCompile(`(?i) DEFAULT \(?(.+)?\)?( |COLLATE|GENERATED|$)`)
regRealDataType = regexp.MustCompile(`[^\d](\d+)[^\d]?`)
)
func getAllColumns(s string) []string {
allMatches := columnsRegexp.FindAllStringSubmatch(s, -1)
columns := make([]string, 0, len(allMatches))
for _, matches := range allMatches {
if len(matches) > 1 {
columns = append(columns, matches[1])
}
}
return columns
}
type ddl struct {
head string
fields []string
@@ -110,9 +98,10 @@ func parseDDL(strs ...string) (*ddl, error) {
if strings.HasPrefix(fUpper, "CONSTRAINT") {
matches := uniqueRegexp.FindStringSubmatch(f)
if len(matches) > 0 {
if columns := getAllColumns(matches[1]); len(columns) == 1 {
cols, err := parseAllColumns(matches[1])
if err == nil && len(cols) == 1 {
for idx, column := range result.columns {
if column.NameValue.String == columns[0] {
if column.NameValue.String == cols[0] {
column.UniqueValue = sql.NullBool{Bool: true, Valid: true}
result.columns[idx] = column
break
@@ -123,12 +112,15 @@ func parseDDL(strs ...string) (*ddl, error) {
continue
}
if strings.HasPrefix(fUpper, "PRIMARY KEY") {
for _, name := range getAllColumns(f) {
for idx, column := range result.columns {
if column.NameValue.String == name {
column.PrimaryKeyValue = sql.NullBool{Bool: true, Valid: true}
result.columns[idx] = column
break
cols, err := parseAllColumns(f)
if err == nil {
for _, name := range cols {
for idx, column := range result.columns {
if column.NameValue.String == name {
column.PrimaryKeyValue = sql.NullBool{Bool: true, Valid: true}
result.columns[idx] = column
break
}
}
}
}

View File

@@ -0,0 +1,117 @@
package gormlite
import (
"errors"
"fmt"
)
type parseAllColumnsState int
const (
parseAllColumnsState_NONE parseAllColumnsState = iota
parseAllColumnsState_Beginning
parseAllColumnsState_ReadingRawName
parseAllColumnsState_ReadingQuotedName
parseAllColumnsState_EndOfName
parseAllColumnsState_State_End
)
func parseAllColumns(in string) ([]string, error) {
s := []rune(in)
columns := make([]string, 0)
state := parseAllColumnsState_NONE
quote := rune(0)
name := make([]rune, 0)
for i := 0; i < len(s); i++ {
switch state {
case parseAllColumnsState_NONE:
if s[i] == '(' {
state = parseAllColumnsState_Beginning
}
case parseAllColumnsState_Beginning:
if isSpace(s[i]) {
continue
}
if isQuote(s[i]) {
state = parseAllColumnsState_ReadingQuotedName
quote = s[i]
continue
}
if s[i] == '[' {
state = parseAllColumnsState_ReadingQuotedName
quote = ']'
continue
} else if s[i] == ')' {
return columns, fmt.Errorf("unexpected token: %s", string(s[i]))
}
state = parseAllColumnsState_ReadingRawName
name = append(name, s[i])
case parseAllColumnsState_ReadingRawName:
if isSeparator(s[i]) {
state = parseAllColumnsState_Beginning
columns = append(columns, string(name))
name = make([]rune, 0)
continue
}
if s[i] == ')' {
state = parseAllColumnsState_State_End
columns = append(columns, string(name))
}
if isQuote(s[i]) {
return nil, fmt.Errorf("unexpected token: %s", string(s[i]))
}
if isSpace(s[i]) {
state = parseAllColumnsState_EndOfName
columns = append(columns, string(name))
name = make([]rune, 0)
continue
}
name = append(name, s[i])
case parseAllColumnsState_ReadingQuotedName:
if s[i] == quote {
// check if quote character is escaped
if i+1 < len(s) && s[i+1] == quote {
name = append(name, quote)
i++
continue
}
state = parseAllColumnsState_EndOfName
columns = append(columns, string(name))
name = make([]rune, 0)
continue
}
name = append(name, s[i])
case parseAllColumnsState_EndOfName:
if isSpace(s[i]) {
continue
}
if isSeparator(s[i]) {
state = parseAllColumnsState_Beginning
continue
}
if s[i] == ')' {
state = parseAllColumnsState_State_End
continue
}
return nil, fmt.Errorf("unexpected token: %s", string(s[i]))
case parseAllColumnsState_State_End:
break
}
}
if state != parseAllColumnsState_State_End {
return nil, errors.New("unexpected end")
}
return columns, nil
}
func isSpace(r rune) bool {
return r == ' ' || r == '\t'
}
func isQuote(r rune) bool {
return r == '`' || r == '"' || r == '\''
}
func isSeparator(r rune) bool {
return r == ','
}

View File

@@ -0,0 +1,48 @@
package gormlite
import "testing"
func TestParseAllColumns(t *testing.T) {
tc := []struct {
name string
input string
expected []string
}{
{
name: "Simple case",
input: "PRIMARY KEY (column1, column2)",
expected: []string{"column1", "column2"},
},
{
name: "Quoted column name",
input: "PRIMARY KEY (`column,xxx`, \"column 2\", \"column)3\", 'column''4', \"column\"\"5\")",
expected: []string{"column,xxx", "column 2", "column)3", "column'4", "column\"5"},
},
{
name: "Japanese column name",
input: "PRIMARY KEY (カラム1, `カラム2`)",
expected: []string{"カラム1", "カラム2"},
},
{
name: "Column name quoted with []",
input: "PRIMARY KEY ([column1], [column2])",
expected: []string{"column1", "column2"},
},
}
for _, tt := range tc {
t.Run(tt.name, func(t *testing.T) {
cols, err := parseAllColumns(tt.input)
if err != nil {
t.Errorf("Failed to parse columns: %s", err)
}
if len(cols) != len(tt.expected) {
t.Errorf("Expected %d columns, got %d", len(tt.expected), len(cols))
}
for i, col := range cols {
if col != tt.expected[i] {
t.Errorf("Expected %s, got %s", tt.expected[i], col)
}
}
})
}
}

View File

@@ -3,11 +3,13 @@ set -euo pipefail
cd -P -- "$(dirname -- "$0")"
curl -#OL "https://github.com/go-gorm/sqlite/raw/v1.5.6/ddlmod.go"
curl -#OL "https://github.com/go-gorm/sqlite/raw/v1.5.6/ddlmod_test.go"
curl -#OL "https://github.com/go-gorm/sqlite/raw/v1.5.6/error_translator.go"
curl -#OL "https://github.com/go-gorm/sqlite/raw/v1.5.6/migrator.go"
curl -#OL "https://github.com/go-gorm/sqlite/raw/v1.5.6/sqlite.go"
curl -#OL "https://github.com/go-gorm/sqlite/raw/v1.5.6/sqlite_test.go"
curl -#OL "https://github.com/go-gorm/sqlite/raw/v1.5.6/sqlite_test.go"
curl -#OL "https://github.com/go-gorm/sqlite/raw/v1.5.7/ddlmod.go"
curl -#OL "https://github.com/go-gorm/sqlite/raw/v1.5.7/ddlmod_test.go"
curl -#OL "https://github.com/go-gorm/sqlite/raw/v1.5.7/ddlmod_parse_all_columns.go"
curl -#OL "https://github.com/go-gorm/sqlite/raw/v1.5.7/ddlmod_parse_all_columns_test.go"
curl -#OL "https://github.com/go-gorm/sqlite/raw/v1.5.7/error_translator.go"
curl -#OL "https://github.com/go-gorm/sqlite/raw/v1.5.7/migrator.go"
curl -#OL "https://github.com/go-gorm/sqlite/raw/v1.5.7/sqlite.go"
curl -#OL "https://github.com/go-gorm/sqlite/raw/v1.5.7/sqlite_test.go"
curl -#OL "https://github.com/go-gorm/sqlite/raw/v1.5.7/sqlite_test.go"
curl -#L "https://github.com/glebarez/sqlite/raw/v1.11.0/sqlite_error_translator_test.go" > error_translator_test.go

View File

@@ -8,6 +8,7 @@ import (
"github.com/ncruces/go-sqlite3"
)
// Translate it will translate the error to native gorm errors.
func (_Dialector) Translate(err error) error {
switch {
case

View File

@@ -5,7 +5,7 @@ go 1.21
toolchain go1.23.0
require (
github.com/ncruces/go-sqlite3 v0.21.0
github.com/ncruces/go-sqlite3 v0.21.3
gorm.io/gorm v1.25.12
)
@@ -14,6 +14,6 @@ require (
github.com/jinzhu/now v1.1.5 // indirect
github.com/ncruces/julianday v1.0.0 // indirect
github.com/tetratelabs/wazero v1.8.2 // indirect
golang.org/x/sys v0.28.0 // indirect
golang.org/x/sys v0.29.0 // indirect
golang.org/x/text v0.21.0 // indirect
)

View File

@@ -2,14 +2,14 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/ncruces/go-sqlite3 v0.21.0 h1:EwKFoy1hHEopN4sFZarmi+McXdbCcbTuLixhEayXVbQ=
github.com/ncruces/go-sqlite3 v0.21.0/go.mod h1:zxMOaSG5kFYVFK4xQa0pdwIszqxqJ0W0BxBgwdrNjuA=
github.com/ncruces/go-sqlite3 v0.21.3 h1:hHkfNQLcbnxPJZhC/RGw9SwP3bfkv/Y0xUHWsr1CdMQ=
github.com/ncruces/go-sqlite3 v0.21.3/go.mod h1:zxMOaSG5kFYVFK4xQa0pdwIszqxqJ0W0BxBgwdrNjuA=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/tetratelabs/wazero v1.8.2 h1:yIgLR/b2bN31bjxwXHD8a3d+BogigR952csSDdLYEv4=
github.com/tetratelabs/wazero v1.8.2/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8=

View File

@@ -14,6 +14,11 @@ import (
"github.com/ncruces/go-sqlite3/driver"
)
type _Dialector struct {
DSN string
Conn gorm.ConnPool
}
// Open opens a GORM dialector from a data source name.
func Open(dsn string) gorm.Dialector {
return &_Dialector{DSN: dsn}
@@ -24,11 +29,6 @@ func OpenDB(db gorm.ConnPool) gorm.Dialector {
return &_Dialector{Conn: db}
}
type _Dialector struct {
DSN string
Conn gorm.ConnPool
}
func (dialector _Dialector) Name() string {
return "sqlite"
}

29
internal/dotlk/dotlk.go Normal file
View File

@@ -0,0 +1,29 @@
package dotlk
import (
"errors"
"io/fs"
"os"
)
// LockShm creates a directory on disk to prevent SQLite
// from using this path for a shared memory file.
func LockShm(name string) error {
err := os.Mkdir(name, 0777)
if errors.Is(err, fs.ErrExist) {
s, err := os.Lstat(name)
if err == nil && s.IsDir() {
return nil
}
}
return err
}
// Unlock removes the lock or shared memory file.
func Unlock(name string) error {
err := os.Remove(name)
if errors.Is(err, fs.ErrNotExist) {
return nil
}
return err
}

View File

@@ -0,0 +1,13 @@
//go:build !unix
package dotlk
import "os"
// TryLock returns nil if it acquired the lock,
// fs.ErrExist if another process has the lock.
func TryLock(name string) error {
f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
f.Close()
return err
}

View File

@@ -0,0 +1,50 @@
//go:build unix
package dotlk
import (
"errors"
"io/fs"
"os"
"strconv"
"golang.org/x/sys/unix"
)
// TryLock returns nil if it acquired the lock,
// fs.ErrExist if another process has the lock.
func TryLock(name string) error {
for retry := true; retry; retry = false {
f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
if err == nil {
f.WriteString(strconv.Itoa(os.Getpid()))
f.Close()
return nil
}
if !errors.Is(err, fs.ErrExist) {
return err
}
if !removeStale(name) {
break
}
}
return fs.ErrExist
}
func removeStale(name string) bool {
buf, err := os.ReadFile(name)
if err != nil {
return errors.Is(err, fs.ErrNotExist)
}
pid, err := strconv.Atoi(string(buf))
if err != nil {
return false
}
if unix.Kill(pid, 0) == nil {
return false
}
err = os.Remove(name)
return err == nil || errors.Is(err, fs.ErrNotExist)
}

View File

@@ -7,14 +7,18 @@ import (
"github.com/tetratelabs/wazero"
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/internal/util"
)
// notest
func init() {
sqlite3.RuntimeConfig = wazero.NewRuntimeConfig().
WithMemoryLimitPages(512)
if util.CompilerSupported() {
sqlite3.RuntimeConfig = wazero.NewRuntimeConfigCompiler()
} else {
sqlite3.RuntimeConfig = wazero.NewRuntimeConfigInterpreter()
}
sqlite3.RuntimeConfig = sqlite3.RuntimeConfig.WithMemoryLimitPages(512)
if os.Getenv("CI") != "" {
path := filepath.Join(os.TempDir(), "wazero")
if err := os.MkdirAll(path, 0777); err == nil {

27
internal/util/compiler.go Normal file
View File

@@ -0,0 +1,27 @@
package util
import (
"runtime"
"golang.org/x/sys/cpu"
)
func CompilerSupported() bool {
switch runtime.GOOS {
case "linux", "android",
"windows", "darwin",
"freebsd", "netbsd", "dragonfly",
"solaris", "illumos":
break
default:
return false
}
switch runtime.GOARCH {
case "amd64":
return cpu.X86.HasSSE41
case "arm64":
return true
default:
return false
}
}

View File

@@ -39,13 +39,13 @@ func (s *mmapState) new(ctx context.Context, mod api.Module, size int32) *Mapped
// Save the newly allocated region.
ptr := uint32(stack[0])
buf := View(mod, ptr, uint64(size))
addr := unsafe.Pointer(&buf[0])
s.regions = append(s.regions, &MappedRegion{
res := &MappedRegion{
Ptr: ptr,
addr: addr,
size: size,
})
return s.regions[len(s.regions)-1]
addr: unsafe.Pointer(&buf[0]),
}
s.regions = append(s.regions, res)
return res
}
type MappedRegion struct {

View File

@@ -8,6 +8,8 @@ import (
"github.com/ncruces/go-sqlite3/internal/alloc"
)
type ConnKey struct{}
type moduleKey struct{}
type moduleState struct {
mmapState

View File

@@ -11,7 +11,6 @@ import (
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/api"
"github.com/tetratelabs/wazero/experimental"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/vfs"
@@ -49,14 +48,18 @@ func compileSQLite() {
ctx := context.Background()
cfg := RuntimeConfig
if cfg == nil {
cfg = wazero.NewRuntimeConfig()
if bits.UintSize >= 64 {
cfg = cfg.WithMemoryLimitPages(4096) // 256MB
if util.CompilerSupported() {
cfg = wazero.NewRuntimeConfigCompiler()
} else {
cfg = wazero.NewRuntimeConfigInterpreter()
}
if bits.UintSize < 64 {
cfg = cfg.WithMemoryLimitPages(512) // 32MB
} else {
cfg = cfg.WithMemoryLimitPages(4096) // 256MB
}
}
cfg = cfg.WithCoreFeatures(api.CoreFeaturesV2 | experimental.CoreFeaturesThreads)
cfg = cfg.WithCoreFeatures(api.CoreFeaturesV2)
instance.runtime = wazero.NewRuntimeWithConfig(ctx, cfg)
@@ -265,10 +268,11 @@ func (a *arena) mark() (reset func()) {
ptrs := len(a.ptrs)
next := a.next
return func() {
for _, ptr := range a.ptrs[ptrs:] {
rest := a.ptrs[ptrs:]
for _, ptr := range a.ptrs[:ptrs] {
a.sqlt.free(ptr)
}
a.ptrs = a.ptrs[:ptrs]
a.ptrs = rest
a.next = next
}
}

11
sqlite3/base64.patch Normal file
View File

@@ -0,0 +1,11 @@
--- ext/base64.c.orig
+++ ext/base64.c
@@ -198,7 +198,7 @@
deliberate_fall_through; /* FALLTHRU */
case 1:
pOut[0] = (qv>>16) & 0xff;
- deliberate_fall_through; /* FALLTHRU */
+ break; /* FALLTHRU */
}
pOut += nbo;
}

View File

@@ -2,7 +2,7 @@
# handle, and interrupt, sqlite3_busy_timeout.
--- sqlite3.c.orig
+++ sqlite3.c
@@ -182928,7 +182928,7 @@
@@ -183054,7 +183054,7 @@
if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
#endif
if( ms>0 ){

View File

@@ -3,7 +3,7 @@ set -euo pipefail
cd -P -- "$(dirname -- "$0")"
curl -#OL "https://sqlite.org/2024/sqlite-amalgamation-3470200.zip"
curl -#OL "https://sqlite.org/2025/sqlite-amalgamation-3480000.zip"
unzip -d . sqlite-amalgamation-*.zip
mv sqlite-amalgamation-*/sqlite3.c .
mv sqlite-amalgamation-*/sqlite3.h .
@@ -17,29 +17,32 @@ rm -rf sqlite-amalgamation-*
# mv sqlite-snapshot-*/sqlite3ext.h .
# rm -rf sqlite-snapshot-*
cat *.patch | patch --no-backup-if-mismatch
mkdir -p ext/
cd ext/
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/ext/misc/anycollseq.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/ext/misc/base64.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/ext/misc/decimal.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/ext/misc/ieee754.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/ext/misc/regexp.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/ext/misc/series.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/ext/misc/spellfix.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/ext/misc/uint.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.48.0/ext/misc/anycollseq.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.48.0/ext/misc/base64.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.48.0/ext/misc/decimal.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.48.0/ext/misc/ieee754.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.48.0/ext/misc/regexp.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.48.0/ext/misc/series.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.48.0/ext/misc/spellfix.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.48.0/ext/misc/uint.c"
cd ~-
cd ../vfs/tests/mptest/testdata/
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/mptest/mptest.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/mptest/config01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/mptest/config02.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/mptest/crash01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/mptest/crash02.subtest"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/mptest/multiwrite01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.48.0/mptest/config01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.48.0/mptest/config02.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.48.0/mptest/crash01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.48.0/mptest/crash02.subtest"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.48.0/mptest/multiwrite01.test"
cd ~-
cd ../vfs/tests/speedtest1/testdata/
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/test/speedtest1.c"
cd ~-
cd ../vfs/tests/mptest/wasm/
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.48.0/mptest/mptest.c"
cd ~-
cd ../vfs/tests/speedtest1/wasm/
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.48.0/test/speedtest1.c"
cd ~-
cat *.patch | patch -p0 --no-backup-if-mismatch

View File

@@ -18,8 +18,6 @@
#define HAVE_STDINT_H 1
#define HAVE_INTTYPES_H 1
#define LONGDOUBLE_TYPE double
#define HAVE_LOG2 1
#define HAVE_LOG10 1
#define HAVE_ISNAN 1
@@ -35,10 +33,6 @@
#define HAVE_MALLOC_H 1
#define HAVE_MALLOC_USABLE_SIZE 1
// Because Wasm does not support shared memory,
// SQLite disables WAL for Wasm builds.
#undef SQLITE_OMIT_WAL
// Implemented in vfs.c.
int localtime_s(struct tm *const pTm, time_t const *const pTime);

View File

@@ -162,5 +162,10 @@ int sqlite3_os_init() {
return SQLITE_OK;
}
int sqlite3_invoke_busy_handler_go(sqlite3_int64 token) {
void **ap = (void **)&token;
return ((int(*)(void *))(ap[0]))(ap[1]);
}
static_assert(offsetof(sqlite3_vfs, zName) == 16, "Unexpected offset");
static_assert(offsetof(struct go_file, handle) == 4, "Unexpected offset");

View File

@@ -1,7 +1,7 @@
# Remove VFS registration. Go handles it.
--- sqlite3.c.orig
+++ sqlite3.c
@@ -26603,7 +26603,7 @@
@@ -26623,7 +26623,7 @@
sqlite3_free(p);
return sqlite3_os_init();
}
@@ -10,7 +10,7 @@
/*
** The list of all registered VFS implementations.
*/
@@ -26700,7 +26700,7 @@
@@ -26720,7 +26720,7 @@
sqlite3_mutex_leave(mutex);
return SQLITE_OK;
}

View File

@@ -130,8 +130,6 @@ func TestConn_SetInterrupt(t *testing.T) {
t.Fatal(err)
}
db.SetInterrupt(context.Background())
stmt, _, err := db.Prepare(`
WITH RECURSIVE
fibonacci (curr, next)
@@ -148,7 +146,6 @@ func TestConn_SetInterrupt(t *testing.T) {
}
defer stmt.Close()
db.SetInterrupt(ctx)
go func() {
time.Sleep(time.Millisecond)
cancel()

View File

@@ -140,7 +140,7 @@ func Test_xts(t *testing.T) {
testIntegrity(t, name)
}
func TestMultiProcess(t *testing.T) {
func Test_MultiProcess_rollback(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
@@ -149,7 +149,7 @@ func TestMultiProcess(t *testing.T) {
}
file := filepath.Join(t.TempDir(), "test.db")
t.Setenv("TestMultiProcess_dbfile", file)
t.Setenv("Test_MultiProcess_dbfile", file)
name := "file:" + filepath.ToSlash(file) +
"?_pragma=busy_timeout(10000)" +
@@ -161,7 +161,7 @@ func TestMultiProcess(t *testing.T) {
t.Fatal(err)
}
cmd := exec.Command(exe, append(os.Args[1:], "-test.v", "-test.run=TestChildProcess")...)
cmd := exec.Command(exe, append(os.Args[1:], "-test.v", "-test.run=Test_ChildProcess_rollback")...)
out, err := cmd.StdoutPipe()
if err != nil {
t.Fatal(err)
@@ -185,8 +185,8 @@ func TestMultiProcess(t *testing.T) {
testIntegrity(t, name)
}
func TestChildProcess(t *testing.T) {
file := os.Getenv("TestMultiProcess_dbfile")
func Test_ChildProcess_rollback(t *testing.T) {
file := os.Getenv("Test_MultiProcess_dbfile")
if file == "" || testing.Short() {
t.SkipNow()
}
@@ -199,6 +199,65 @@ func TestChildProcess(t *testing.T) {
testParallel(t, name, 1000)
}
func Test_MultiProcess_wal(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
if testing.Short() {
t.Skip("skipping in short mode")
}
file := filepath.Join(t.TempDir(), "test.db")
t.Setenv("Test_MultiProcess_dbfile", file)
name := "file:" + filepath.ToSlash(file) +
"?_pragma=busy_timeout(10000)" +
"&_pragma=journal_mode(wal)" +
"&_pragma=synchronous(off)"
exe, err := os.Executable()
if err != nil {
t.Fatal(err)
}
cmd := exec.Command(exe, append(os.Args[1:], "-test.v", "-test.run=Test_ChildProcess_wal")...)
out, err := cmd.StdoutPipe()
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
var buf [3]byte
// Wait for child to start.
if _, err := io.ReadFull(out, buf[:]); err != nil {
t.Fatal(err)
} else if str := string(buf[:]); str != "===" {
t.Fatal(str)
}
testParallel(t, name, 1000)
if err := cmd.Wait(); err != nil {
t.Error(err)
}
testIntegrity(t, name)
}
func Test_ChildProcess_wal(t *testing.T) {
file := os.Getenv("Test_MultiProcess_dbfile")
if file == "" || testing.Short() {
t.SkipNow()
}
name := "file:" + filepath.ToSlash(file) +
"?_pragma=busy_timeout(10000)" +
"&_pragma=journal_mode(wal)" +
"&_pragma=synchronous(off)"
testParallel(t, name, 1000)
}
func Benchmark_parallel(b *testing.B) {
if !vfs.SupportsSharedMemory {
b.Skip("skipping without shared memory")

View File

@@ -17,7 +17,7 @@ const (
)
var (
//go:embed parse/sql3parse_table.wasm
//go:embed wasm/sql3parse_table.wasm
binary []byte
once sync.Once
runtime wazero.Runtime

Binary file not shown.

View File

@@ -38,18 +38,18 @@ func WrapLockState(f vfs.File) vfs.LockLevel {
return vfs.LOCK_EXCLUSIVE + 1 // UNKNOWN_LOCK
}
// WrapPersistentWAL helps wrap [vfs.FilePersistentWAL].
func WrapPersistentWAL(f vfs.File) bool {
if f, ok := f.(vfs.FilePersistentWAL); ok {
return f.PersistentWAL()
// WrapPersistWAL helps wrap [vfs.FilePersistWAL].
func WrapPersistWAL(f vfs.File) bool {
if f, ok := f.(vfs.FilePersistWAL); ok {
return f.PersistWAL()
}
return false
}
// WrapSetPersistentWAL helps wrap [vfs.FilePersistentWAL].
func WrapSetPersistentWAL(f vfs.File, keepWAL bool) {
if f, ok := f.(vfs.FilePersistentWAL); ok {
f.SetPersistentWAL(keepWAL)
// WrapSetPersistWAL helps wrap [vfs.FilePersistWAL].
func WrapSetPersistWAL(f vfs.File, keepWAL bool) {
if f, ok := f.(vfs.FilePersistWAL); ok {
f.SetPersistWAL(keepWAL)
}
}
@@ -99,6 +99,14 @@ func WrapOverwrite(f vfs.File) error {
return sqlite3.NOTFOUND
}
// WrapSyncSuper helps wrap [vfs.FileSync].
func WrapSyncSuper(f vfs.File, super string) error {
if f, ok := f.(vfs.FileSync); ok {
return f.SyncSuper(super)
}
return sqlite3.NOTFOUND
}
// WrapCommitPhaseTwo helps wrap [vfs.FileCommitPhaseTwo].
func WrapCommitPhaseTwo(f vfs.File) error {
if f, ok := f.(vfs.FileCommitPhaseTwo); ok {
@@ -153,6 +161,13 @@ func WrapPragma(f vfs.File, name, value string) (string, error) {
return "", sqlite3.NOTFOUND
}
// WrapBusyHandler helps wrap [vfs.FilePragma].
func WrapBusyHandler(f vfs.File, handler func() bool) {
if f, ok := f.(vfs.FileBusyHandler); ok {
f.BusyHandler(handler)
}
}
// WrapSharedMemory helps wrap [vfs.FileSharedMemory].
func WrapSharedMemory(f vfs.File) vfs.SharedMemory {
if f, ok := f.(vfs.FileSharedMemory); ok {

View File

@@ -48,11 +48,6 @@ On Unix, this package may use `mmap` to implement
[shared-memory for the WAL-index](https://sqlite.org/wal.html#implementation_of_shared_memory_for_the_wal_index),
like SQLite.
With [BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2)
a WAL database can only be accessed by a single proccess.
Other processes that attempt to access a database locked with BSD locks,
will fail with the [`SQLITE_PROTOCOL`](https://sqlite.org/rescode.html#protocol) error code.
On Windows, this package may use `MapViewOfFile`, like SQLite.
You can also opt into a cross-platform, in-process, memory sharing implementation

View File

@@ -238,11 +238,11 @@ func (h *hbshFile) LockState() vfs.LockLevel {
}
func (h *hbshFile) PersistentWAL() bool {
return vfsutil.WrapPersistentWAL(h.File) // notest
return vfsutil.WrapPersistWAL(h.File) // notest
}
func (h *hbshFile) SetPersistentWAL(keepWAL bool) {
vfsutil.WrapSetPersistentWAL(h.File, keepWAL) // notest
vfsutil.WrapSetPersistWAL(h.File, keepWAL) // notest
}
func (h *hbshFile) HasMoved() (bool, error) {
@@ -253,6 +253,10 @@ func (h *hbshFile) Overwrite() error {
return vfsutil.WrapOverwrite(h.File) // notest
}
func (h *hbshFile) SyncSuper(super string) error {
return vfsutil.WrapSyncSuper(h.File, super) // notest
}
func (h *hbshFile) CommitPhaseTwo() error {
return vfsutil.WrapCommitPhaseTwo(h.File) // notest
}
@@ -276,3 +280,7 @@ func (h *hbshFile) CheckpointStart() {
func (h *hbshFile) CheckpointDone() {
vfsutil.WrapCheckpointDone(h.File) // notest
}
func (h *hbshFile) BusyHandler(handler func() bool) {
vfsutil.WrapBusyHandler(h.File, handler) // notest
}

View File

@@ -65,14 +65,14 @@ type FileLockState interface {
LockState() LockLevel
}
// FilePersistentWAL extends File to implement the
// FilePersistWAL extends File to implement the
// SQLITE_FCNTL_PERSIST_WAL file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpersistwal
type FilePersistentWAL interface {
type FilePersistWAL interface {
File
PersistentWAL() bool
SetPersistentWAL(bool)
PersistWAL() bool
SetPersistWAL(bool)
}
// FilePowersafeOverwrite extends File to implement the
@@ -121,6 +121,15 @@ type FileOverwrite interface {
Overwrite() error
}
// FileSync extends File to implement the
// SQLITE_FCNTL_SYNC file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlsync
type FileSync interface {
File
SyncSuper(super string) error
}
// FileCommitPhaseTwo extends File to implement the
// SQLITE_FCNTL_COMMIT_PHASETWO file control opcode.
//
@@ -162,6 +171,15 @@ type FilePragma interface {
Pragma(name, value string) (string, error)
}
// FileBusyHandler extends File to implement the
// SQLITE_FCNTL_BUSYHANDLER file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlbusyhandler
type FileBusyHandler interface {
File
BusyHandler(func() bool)
}
// FileSharedMemory extends File to possibly implement
// shared-memory for the WAL-index.
// The same shared-memory instance must be returned
@@ -191,3 +209,8 @@ type fileControl interface {
File
fileControl(ctx context.Context, mod api.Module, op _FcntlOpcode, pArg uint32) _ErrorCode
}
type filePDB interface {
File
SetDB(any)
}

View File

@@ -225,6 +225,7 @@ const (
_FCNTL_EXTERNAL_READER _FcntlOpcode = 40
_FCNTL_CKSM_FILE _FcntlOpcode = 41
_FCNTL_RESET_CACHE _FcntlOpcode = 42
_FCNTL_NULL_IO _FcntlOpcode = 43
)
// https://sqlite.org/c3ref/c_shm_exclusive.html

View File

@@ -6,7 +6,6 @@ import (
"io/fs"
"os"
"path/filepath"
"runtime"
"syscall"
"github.com/ncruces/go-sqlite3/util/osutil"
@@ -41,7 +40,7 @@ func (vfsOS) Delete(path string, syncDir bool) error {
if err != nil {
return err
}
if runtime.GOOS != "windows" && syncDir {
if canSyncDirs && syncDir {
f, err := os.Open(filepath.Dir(path))
if err != nil {
return _OK
@@ -120,9 +119,9 @@ func (vfsOS) OpenFilename(name *Filename, flags OpenFlag) (File, OpenFlag, error
File: f,
psow: true,
readOnly: flags&OPEN_READONLY != 0,
syncDir: runtime.GOOS != "windows" &&
flags&(OPEN_CREATE) != 0 &&
flags&(OPEN_MAIN_JOURNAL|OPEN_SUPER_JOURNAL|OPEN_WAL) != 0,
syncDir: canSyncDirs &&
flags&(OPEN_MAIN_JOURNAL|OPEN_SUPER_JOURNAL|OPEN_WAL) != 0 &&
flags&(OPEN_CREATE) != 0,
shm: NewSharedMemory(name.String()+"-shm", flags),
}
return &file, flags, nil
@@ -143,7 +142,7 @@ var (
_ FileLockState = &vfsFile{}
_ FileHasMoved = &vfsFile{}
_ FileSizeHint = &vfsFile{}
_ FilePersistentWAL = &vfsFile{}
_ FilePersistWAL = &vfsFile{}
_ FilePowersafeOverwrite = &vfsFile{}
)
@@ -163,7 +162,7 @@ func (f *vfsFile) Sync(flags SyncFlag) error {
if err != nil {
return err
}
if runtime.GOOS != "windows" && f.syncDir {
if canSyncDirs && f.syncDir {
f.syncDir = false
d, err := os.Open(filepath.Dir(f.File.Name()))
if err != nil {
@@ -218,6 +217,6 @@ func (f *vfsFile) HasMoved() (bool, error) {
func (f *vfsFile) LockState() LockLevel { return f.lock }
func (f *vfsFile) PowersafeOverwrite() bool { return f.psow }
func (f *vfsFile) PersistentWAL() bool { return f.keepWAL }
func (f *vfsFile) PersistWAL() bool { return f.keepWAL }
func (f *vfsFile) SetPowersafeOverwrite(psow bool) { f.psow = psow }
func (f *vfsFile) SetPersistentWAL(keepWAL bool) { f.keepWAL = keepWAL }
func (f *vfsFile) SetPersistWAL(keepWAL bool) { f.keepWAL = keepWAL }

View File

@@ -20,12 +20,10 @@ const (
)
func (f *vfsFile) Lock(lock LockLevel) error {
// Argument check. SQLite never explicitly requests a pending lock.
if lock != LOCK_SHARED && lock != LOCK_RESERVED && lock != LOCK_EXCLUSIVE {
panic(util.AssertErr())
}
switch {
case lock != LOCK_SHARED && lock != LOCK_RESERVED && lock != LOCK_EXCLUSIVE:
// Argument check. SQLite never explicitly requests a pending lock.
panic(util.AssertErr())
case f.lock < LOCK_NONE || f.lock > LOCK_EXCLUSIVE:
// Connection state check.
panic(util.AssertErr())
@@ -87,13 +85,12 @@ func (f *vfsFile) Lock(lock LockLevel) error {
}
func (f *vfsFile) Unlock(lock LockLevel) error {
// Argument check.
if lock != LOCK_NONE && lock != LOCK_SHARED {
switch {
case lock != LOCK_NONE && lock != LOCK_SHARED:
// Argument check.
panic(util.AssertErr())
}
// Connection state check.
if f.lock < LOCK_NONE || f.lock > LOCK_EXCLUSIVE {
case f.lock < LOCK_NONE || f.lock > LOCK_EXCLUSIVE:
// Connection state check.
panic(util.AssertErr())
}

View File

@@ -62,11 +62,11 @@ func (memVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, err
}
func (memVFS) Delete(name string, dirSync bool) error {
return sqlite3.IOERR_DELETE
return sqlite3.IOERR_DELETE_NOENT // used to delete journals
}
func (memVFS) Access(name string, flag vfs.AccessFlag) (bool, error) {
return false, nil
return false, nil // used to check for journals
}
func (memVFS) FullPathname(name string) (string, error) {

View File

@@ -9,11 +9,11 @@ import (
)
func osGetSharedLock(file *os.File) _ErrorCode {
return osLock(file, unix.LOCK_SH|unix.LOCK_NB, _IOERR_RDLOCK)
return osFlock(file, unix.LOCK_SH|unix.LOCK_NB, _IOERR_RDLOCK)
}
func osGetReservedLock(file *os.File) _ErrorCode {
rc := osLock(file, unix.LOCK_EX|unix.LOCK_NB, _IOERR_LOCK)
rc := osFlock(file, unix.LOCK_EX|unix.LOCK_NB, _IOERR_LOCK)
if rc == _BUSY {
// The documentation states that a lock is upgraded by
// releasing the previous lock, then acquiring the new lock.
@@ -37,7 +37,7 @@ func osGetExclusiveLock(file *os.File, state *LockLevel) _ErrorCode {
}
func osDowngradeLock(file *os.File, _ LockLevel) _ErrorCode {
rc := osLock(file, unix.LOCK_SH|unix.LOCK_NB, _IOERR_RDLOCK)
rc := osFlock(file, unix.LOCK_SH|unix.LOCK_NB, _IOERR_RDLOCK)
if rc == _BUSY {
// The documentation states that a lock is downgraded by
// releasing the previous lock then acquiring the new lock.
@@ -66,7 +66,36 @@ func osCheckReservedLock(file *os.File) (bool, _ErrorCode) {
return lock == unix.F_WRLCK, rc
}
func osLock(file *os.File, how int, def _ErrorCode) _ErrorCode {
func osFlock(file *os.File, how int, def _ErrorCode) _ErrorCode {
err := unix.Flock(int(file.Fd()), how)
return osLockErrorCode(err, def)
}
func osReadLock(file *os.File, start, len int64) _ErrorCode {
return osLock(file, unix.F_RDLCK, start, len, _IOERR_RDLOCK)
}
func osWriteLock(file *os.File, start, len int64) _ErrorCode {
return osLock(file, unix.F_WRLCK, start, len, _IOERR_LOCK)
}
func osLock(file *os.File, typ int16, start, len int64, def _ErrorCode) _ErrorCode {
err := unix.FcntlFlock(file.Fd(), unix.F_SETLK, &unix.Flock_t{
Type: typ,
Start: start,
Len: len,
})
return osLockErrorCode(err, def)
}
func osUnlock(file *os.File, start, len int64) _ErrorCode {
err := unix.FcntlFlock(file.Fd(), unix.F_SETLK, &unix.Flock_t{
Type: unix.F_UNLCK,
Start: start,
Len: len,
})
if err != nil {
return _IOERR_UNLOCK
}
return _OK
}

View File

@@ -56,16 +56,12 @@ func osAllocate(file *os.File, size int64) error {
return file.Truncate(size)
}
func osUnlock(file *os.File, start, len int64) _ErrorCode {
err := unix.FcntlFlock(file.Fd(), _F_OFD_SETLK, &unix.Flock_t{
Type: unix.F_UNLCK,
Start: start,
Len: len,
})
if err != nil {
return _IOERR_UNLOCK
}
return _OK
func osReadLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode {
return osLock(file, unix.F_RDLCK, start, len, timeout, _IOERR_RDLOCK)
}
func osWriteLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode {
return osLock(file, unix.F_WRLCK, start, len, timeout, _IOERR_LOCK)
}
func osLock(file *os.File, typ int16, start, len int64, timeout time.Duration, def _ErrorCode) _ErrorCode {
@@ -88,10 +84,14 @@ func osLock(file *os.File, typ int16, start, len int64, timeout time.Duration, d
return osLockErrorCode(err, def)
}
func osReadLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode {
return osLock(file, unix.F_RDLCK, start, len, timeout, _IOERR_RDLOCK)
}
func osWriteLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode {
return osLock(file, unix.F_WRLCK, start, len, timeout, _IOERR_LOCK)
func osUnlock(file *os.File, start, len int64) _ErrorCode {
err := unix.FcntlFlock(file.Fd(), _F_OFD_SETLK, &unix.Flock_t{
Type: unix.F_UNLCK,
Start: start,
Len: len,
})
if err != nil {
return _IOERR_UNLOCK
}
return _OK
}

View File

@@ -7,6 +7,8 @@ import (
"io/fs"
"os"
"sync"
"github.com/ncruces/go-sqlite3/internal/dotlk"
)
var (
@@ -28,12 +30,10 @@ func osGetSharedLock(file *os.File) _ErrorCode {
name := file.Name()
locker := vfsDotLocks[name]
if locker == nil {
f, err := os.OpenFile(name+".lock", os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
f.Close()
if errors.Is(err, fs.ErrExist) {
return _BUSY // Another process has the lock.
}
if err != nil {
if err := dotlk.TryLock(name + ".lock"); err != nil {
if errors.Is(err, fs.ErrExist) {
return _BUSY // Another process has the lock.
}
return _IOERR_LOCK
}
locker = &vfsDotLocker{}
@@ -114,8 +114,7 @@ func osReleaseLock(file *os.File, state LockLevel) _ErrorCode {
}
if locker.shared == 1 {
err := os.Remove(name + ".lock")
if err != nil && !errors.Is(err, fs.ErrNotExist) {
if err := dotlk.Unlock(name + ".lock"); err != nil {
return _IOERR_UNLOCK
}
delete(vfsDotLocks, name)

View File

@@ -3,7 +3,6 @@
package vfs
import (
"math/rand"
"os"
"time"
@@ -22,6 +21,30 @@ func osAllocate(file *os.File, size int64) error {
return unix.Fallocate(int(file.Fd()), 0, 0, size)
}
func osReadLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode {
return osLock(file, unix.F_RDLCK, start, len, timeout, _IOERR_RDLOCK)
}
func osWriteLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode {
return osLock(file, unix.F_WRLCK, start, len, timeout, _IOERR_LOCK)
}
func osLock(file *os.File, typ int16, start, len int64, timeout time.Duration, def _ErrorCode) _ErrorCode {
lock := unix.Flock_t{
Type: typ,
Start: start,
Len: len,
}
var err error
switch {
case timeout < 0:
err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLKW, &lock)
default:
err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &lock)
}
return osLockErrorCode(err, def)
}
func osUnlock(file *os.File, start, len int64) _ErrorCode {
err := unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &unix.Flock_t{
Type: unix.F_UNLCK,
@@ -33,40 +56,3 @@ func osUnlock(file *os.File, start, len int64) _ErrorCode {
}
return _OK
}
func osLock(file *os.File, typ int16, start, len int64, timeout time.Duration, def _ErrorCode) _ErrorCode {
lock := unix.Flock_t{
Type: typ,
Start: start,
Len: len,
}
var err error
switch {
case timeout == 0:
err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &lock)
case timeout < 0:
err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLKW, &lock)
default:
before := time.Now()
for {
err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &lock)
if errno, _ := err.(unix.Errno); errno != unix.EAGAIN {
break
}
if time.Since(before) > timeout {
break
}
const sleepIncrement = 1024*1024 - 1 // power of two, ~1ms
time.Sleep(time.Duration(rand.Int63() & sleepIncrement))
}
}
return osLockErrorCode(err, def)
}
func osReadLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode {
return osLock(file, unix.F_RDLCK, start, len, timeout, _IOERR_RDLOCK)
}
func osWriteLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode {
return osLock(file, unix.F_WRLCK, start, len, timeout, _IOERR_LOCK)
}

View File

@@ -7,7 +7,10 @@ import (
"os"
)
const _O_NOFOLLOW = 0
const (
_O_NOFOLLOW = 0
canSyncDirs = false
)
func osAccess(path string, flags AccessFlag) error {
fi, err := os.Stat(path)

View File

@@ -9,7 +9,10 @@ import (
"golang.org/x/sys/unix"
)
const _O_NOFOLLOW = unix.O_NOFOLLOW
const (
_O_NOFOLLOW = unix.O_NOFOLLOW
canSyncDirs = true
)
func osAccess(path string, flags AccessFlag) error {
var access uint32 // unix.F_OK

View File

@@ -45,6 +45,7 @@ func osGetExclusiveLock(file *os.File, state *LockLevel) _ErrorCode {
osUnlock(file, _SHARED_FIRST, _SHARED_SIZE)
// Acquire the EXCLUSIVE lock.
// Can't wait here, because the file is not OVERLAPPED.
rc := osWriteLock(file, _SHARED_FIRST, _SHARED_SIZE, 0)
if rc != _OK {
@@ -106,6 +107,27 @@ func osCheckReservedLock(file *os.File) (bool, _ErrorCode) {
return false, rc
}
func osReadLock(file *os.File, start, len uint32, timeout time.Duration) _ErrorCode {
return osLock(file, 0, start, len, timeout, _IOERR_RDLOCK)
}
func osWriteLock(file *os.File, start, len uint32, timeout time.Duration) _ErrorCode {
return osLock(file, windows.LOCKFILE_EXCLUSIVE_LOCK, start, len, timeout, _IOERR_LOCK)
}
func osLock(file *os.File, flags, start, len uint32, timeout time.Duration, def _ErrorCode) _ErrorCode {
var err error
switch {
case timeout == 0:
err = osLockEx(file, flags|windows.LOCKFILE_FAIL_IMMEDIATELY, start, len)
case timeout < 0:
err = osLockEx(file, flags, start, len)
default:
err = osLockExTimeout(file, flags, start, len, timeout)
}
return osLockErrorCode(err, def)
}
func osUnlock(file *os.File, start, len uint32) _ErrorCode {
err := windows.UnlockFileEx(windows.Handle(file.Fd()),
0, len, 0, &windows.Overlapped{Offset: start})
@@ -118,52 +140,40 @@ func osUnlock(file *os.File, start, len uint32) _ErrorCode {
return _OK
}
func osLock(file *os.File, flags, start, len uint32, timeout time.Duration, def _ErrorCode) _ErrorCode {
var err error
switch {
case timeout == 0:
err = osLockEx(file, flags|windows.LOCKFILE_FAIL_IMMEDIATELY, start, len, 0)
case timeout < 0:
err = osLockEx(file, flags, start, len, 0)
default:
var event windows.Handle
event, err = windows.CreateEvent(nil, 1, 0, nil)
if err != nil {
break
}
defer windows.CloseHandle(event)
err = osLockEx(file, flags, start, len, event)
if err == windows.ERROR_IO_PENDING {
rc, serr := windows.WaitForSingleObject(event, uint32(timeout/time.Millisecond))
if rc == windows.WAIT_OBJECT_0 {
return _OK
}
if serr != nil {
err = serr
} else {
err = windows.Errno(rc)
}
windows.CancelIo(windows.Handle(file.Fd()))
}
}
return osLockErrorCode(err, def)
}
func osLockEx(file *os.File, flags, start, len uint32, event windows.Handle) error {
func osLockEx(file *os.File, flags, start, len uint32) error {
return windows.LockFileEx(windows.Handle(file.Fd()), flags,
0, len, 0, &windows.Overlapped{
Offset: start,
HEvent: event,
})
0, len, 0, &windows.Overlapped{Offset: start})
}
func osReadLock(file *os.File, start, len uint32, timeout time.Duration) _ErrorCode {
return osLock(file, 0, start, len, timeout, _IOERR_RDLOCK)
}
func osLockExTimeout(file *os.File, flags, start, len uint32, timeout time.Duration) error {
event, err := windows.CreateEvent(nil, 1, 0, nil)
if err != nil {
return err
}
defer windows.CloseHandle(event)
func osWriteLock(file *os.File, start, len uint32, timeout time.Duration) _ErrorCode {
return osLock(file, windows.LOCKFILE_EXCLUSIVE_LOCK, start, len, timeout, _IOERR_LOCK)
fd := windows.Handle(file.Fd())
overlapped := &windows.Overlapped{
Offset: start,
HEvent: event,
}
err = windows.LockFileEx(fd, flags, 0, len, 0, overlapped)
if err != windows.ERROR_IO_PENDING {
return err
}
ms := (timeout + time.Millisecond - 1) / time.Millisecond
rc, err := windows.WaitForSingleObject(event, uint32(ms))
if rc == windows.WAIT_OBJECT_0 {
return nil
}
defer windows.CancelIoEx(fd, overlapped)
if err != nil {
return err
}
return windows.Errno(rc)
}
func osLockErrorCode(err error, def _ErrorCode) _ErrorCode {
@@ -175,8 +185,8 @@ func osLockErrorCode(err error, def _ErrorCode) _ErrorCode {
switch errno {
case
windows.ERROR_LOCK_VIOLATION,
windows.ERROR_IO_PENDING,
windows.ERROR_OPERATION_ABORTED,
windows.ERROR_IO_PENDING,
windows.WAIT_TIMEOUT:
return _BUSY
}

View File

@@ -4,7 +4,9 @@ package vfs
import (
"context"
"errors"
"io"
"io/fs"
"os"
"sync"
@@ -20,7 +22,7 @@ type vfsShmParent struct {
refs int // +checklocks:vfsShmListMtx
lock [_SHM_NLOCK]int16 // +checklocks:Mutex
lock [_SHM_NLOCK]int8 // +checklocks:Mutex
sync.Mutex
}
@@ -71,23 +73,21 @@ func (s *vfsShm) shmOpen() _ErrorCode {
return _OK
}
// Always open file read-write, as it will be shared.
f, err := os.OpenFile(s.path,
os.O_RDWR|os.O_CREATE|_O_NOFOLLOW, 0666)
if err != nil {
return _CANTOPEN
}
// Closes file if it's not nil.
var f *os.File
// Close file on error.
// Keep this here to avoid confusing checklocks.
defer func() { f.Close() }()
fi, err := f.Stat()
if err != nil {
return _IOERR_FSTAT
}
vfsShmListMtx.Lock()
defer vfsShmListMtx.Unlock()
// Stat file without opening it.
// Closing it would release all POSIX locks on it.
fi, err := os.Stat(s.path)
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return _IOERR_FSTAT
}
// Find a shared file, increase the reference count.
for _, g := range vfsShmList {
if g != nil && os.SameFile(fi, g.info) {
@@ -97,13 +97,33 @@ func (s *vfsShm) shmOpen() _ErrorCode {
}
}
// Lock and truncate the file.
// The lock is only released by closing the file.
if rc := osLock(f, unix.LOCK_EX|unix.LOCK_NB, _IOERR_LOCK); rc != _OK {
// Always open file read-write, as it will be shared.
f, err = os.OpenFile(s.path,
os.O_RDWR|os.O_CREATE|_O_NOFOLLOW, 0666)
if err != nil {
return _CANTOPEN
}
// Dead man's switch.
if lock, rc := osTestLock(f, _SHM_DMS, 1); rc != _OK {
return _IOERR_LOCK
} else if lock == unix.F_WRLCK {
return _BUSY
} else if lock == unix.F_UNLCK {
if rc := osWriteLock(f, _SHM_DMS, 1); rc != _OK {
return rc
}
if err := f.Truncate(0); err != nil {
return _IOERR_SHMOPEN
}
}
if rc := osReadLock(f, _SHM_DMS, 1); rc != _OK {
return rc
}
if err := f.Truncate(0); err != nil {
return _IOERR_SHMOPEN
fi, err = f.Stat()
if err != nil {
return _IOERR_FSTAT
}
// Add the new shared file.
@@ -157,7 +177,52 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext
func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) _ErrorCode {
s.Lock()
defer s.Unlock()
return s.shmMemLock(offset, n, flags)
// Check if we can obtain/release locks locally.
rc := s.shmMemLock(offset, n, flags)
if rc != _OK {
return rc
}
// Obtain/release the appropriate file locks.
switch {
case flags&_SHM_UNLOCK != 0:
// Relasing a shared lock decrements the counter,
// but may leave parts of the range still locked.
begin, end := offset, offset+n
for i := begin; i < end; i++ {
if s.vfsShmParent.lock[i] != 0 {
if i > begin {
rc |= osUnlock(s.File, _SHM_BASE+int64(begin), int64(i-begin))
}
begin = i + 1
}
}
if end > begin {
rc |= osUnlock(s.File, _SHM_BASE+int64(begin), int64(end-begin))
}
return rc
case flags&_SHM_SHARED != 0:
// Acquiring a new shared lock on the file is only necessary
// if there was a new shared lock in the range.
for i := offset; i < offset+n; i++ {
if s.vfsShmParent.lock[i] == 1 {
rc = osReadLock(s.File, _SHM_BASE+int64(offset), int64(n))
break
}
}
case flags&_SHM_EXCLUSIVE != 0:
// Acquiring an exclusive lock on the file is always necessary.
rc = osWriteLock(s.File, _SHM_BASE+int64(offset), int64(n))
default:
panic(util.AssertErr())
}
// Release the local locks we had acquired.
if rc != _OK {
s.shmMemLock(offset, n, flags^(_SHM_UNLOCK|_SHM_LOCK))
}
return rc
}
func (s *vfsShm) shmUnmap(delete bool) {

View File

@@ -6,11 +6,11 @@ import (
"context"
"errors"
"io/fs"
"os"
"sync"
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3/internal/dotlk"
"github.com/ncruces/go-sqlite3/internal/util"
)
@@ -18,7 +18,7 @@ type vfsShmParent struct {
shared [][_WALINDEX_PGSZ]byte
refs int // +checklocks:vfsShmListMtx
lock [_SHM_NLOCK]int16 // +checklocks:Mutex
lock [_SHM_NLOCK]int8 // +checklocks:Mutex
sync.Mutex
}
@@ -58,8 +58,7 @@ func (s *vfsShm) Close() error {
return nil
}
err := os.Remove(s.path)
if err != nil && !errors.Is(err, fs.ErrNotExist) {
if err := dotlk.Unlock(s.path); err != nil {
return _IOERR_UNLOCK
}
delete(vfsShmList, s.path)
@@ -82,9 +81,8 @@ func (s *vfsShm) shmOpen() _ErrorCode {
return _OK
}
// Create a directory on disk to ensure only this process
// uses this path to register a shared memory.
err := os.Mkdir(s.path, 0777)
// Dead man's switch.
err := dotlk.LockShm(s.path)
if errors.Is(err, fs.ErrExist) {
return _BUSY
}

View File

@@ -10,9 +10,6 @@ func (s *vfsShm) shmMemLock(offset, n int32, flags _ShmFlag) _ErrorCode {
case flags&_SHM_UNLOCK != 0:
for i := offset; i < offset+n; i++ {
if s.lock[i] {
if s.vfsShmParent.lock[i] == 0 {
panic(util.AssertErr())
}
if s.vfsShmParent.lock[i] <= 0 {
s.vfsShmParent.lock[i] = 0
} else {
@@ -23,20 +20,21 @@ func (s *vfsShm) shmMemLock(offset, n int32, flags _ShmFlag) _ErrorCode {
}
case flags&_SHM_SHARED != 0:
for i := offset; i < offset+n; i++ {
if s.lock[i] {
panic(util.AssertErr())
}
if s.vfsShmParent.lock[i]+1 <= 0 {
if !s.lock[i] &&
s.vfsShmParent.lock[i]+1 <= 0 {
return _BUSY
}
}
for i := offset; i < offset+n; i++ {
s.vfsShmParent.lock[i]++
s.lock[i] = true
if !s.lock[i] {
s.vfsShmParent.lock[i]++
s.lock[i] = true
}
}
case flags&_SHM_EXCLUSIVE != 0:
for i := offset; i < offset+n; i++ {
if s.lock[i] {
// SQLite never requests an exclusive lock that it already holds.
panic(util.AssertErr())
}
if s.vfsShmParent.lock[i] != 0 {

View File

@@ -110,7 +110,12 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext
func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) _ErrorCode {
// Argument check.
if n <= 0 || offset < 0 || offset+n > _SHM_NLOCK {
switch {
case n <= 0:
panic(util.AssertErr())
case offset < 0 || offset+n > _SHM_NLOCK:
panic(util.AssertErr())
case n != 1 && flags&_SHM_EXCLUSIVE == 0:
panic(util.AssertErr())
}
switch flags {
@@ -123,9 +128,6 @@ func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) _ErrorCode {
default:
panic(util.AssertErr())
}
if n != 1 && flags&_SHM_EXCLUSIVE == 0 {
panic(util.AssertErr())
}
var timeout time.Duration
if s.blocking {

View File

@@ -2,7 +2,6 @@ package mptest
import (
"bytes"
"compress/bzip2"
"context"
"crypto/rand"
"embed"
@@ -29,14 +28,9 @@ import (
_ "github.com/ncruces/go-sqlite3/vfs/xts"
)
//go:embed testdata/mptest.wasm.bz2
var compressed string
//go:embed testdata/*.*test
//go:embed testdata/*
var scripts embed.FS
const qemuCI = runtime.GOARCH != "386" && runtime.GOARCH != "amd64" && runtime.GOARCH != "arm64"
var (
rt wazero.Runtime
module wazero.CompiledModule
@@ -57,10 +51,7 @@ func TestMain(m *testing.M) {
panic(err)
}
if !strings.HasPrefix(compressed, "BZh") {
panic("Please use Git LFS to clone this repo: https://git-lfs.com/")
}
binary, err := io.ReadAll(bzip2.NewReader(strings.NewReader(compressed)))
binary, err := os.ReadFile("wasm/mptest.wasm")
if err != nil {
panic(err)
}
@@ -162,8 +153,8 @@ func Test_crash01(t *testing.T) {
}
func Test_multiwrite01(t *testing.T) {
if os.Getenv("CI") != "" && qemuCI {
t.Skip("skipping in CI")
if testing.Short() && os.Getenv("CI") != "" {
t.Skip("skipping in slow CI")
}
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
@@ -192,8 +183,8 @@ func Test_config01_memory(t *testing.T) {
}
func Test_multiwrite01_memory(t *testing.T) {
if os.Getenv("CI") != "" && qemuCI {
t.Skip("skipping in CI")
if testing.Short() && os.Getenv("CI") != "" {
t.Skip("skipping in slow CI")
}
memdb.Create("test.db", nil)
@@ -227,8 +218,8 @@ func Test_crash01_wal(t *testing.T) {
}
func Test_multiwrite01_wal(t *testing.T) {
if os.Getenv("CI") != "" && qemuCI {
t.Skip("skipping in CI")
if testing.Short() && os.Getenv("CI") != "" {
t.Skip("skipping in slow CI")
}
if !vfs.SupportsSharedMemory {
t.Skip("skipping without shared memory")

View File

@@ -1,2 +1 @@
mptest.wasm.bz2 filter=lfs diff=lfs merge=lfs -text
*.*test -crlf
* -crlf

View File

@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:da52d62d7ccd9d203eaa6d281a8a61786f99b772b6de3c31c633e6a480e80df9
size 480698

View File

@@ -10,7 +10,7 @@ WASI_SDK="$ROOT/tools/wasi-sdk/bin"
"$WASI_SDK/clang" --target=wasm32-wasi -std=c23 -g0 -O2 \
-o mptest.wasm main.c \
-I"$ROOT/sqlite3" \
-matomics -msimd128 -mmutable-globals -mmultivalue \
-msimd128 -mmutable-globals -mmultivalue \
-mbulk-memory -mreference-types \
-mnontrapping-fptoint -msign-ext \
-fno-stack-protector -fno-stack-clash-protection \
@@ -30,5 +30,4 @@ WASI_SDK="$ROOT/tools/wasi-sdk/bin"
--enable-simd --enable-mutable-globals --enable-multivalue \
--enable-bulk-memory --enable-reference-types \
--enable-nontrapping-float-to-int --enable-sign-ext
mv mptest.tmp mptest.wasm
bzip2 -9f mptest.wasm
mv mptest.tmp mptest.wasm

View File

Binary file not shown.

View File

@@ -2,7 +2,6 @@ package speedtest1
import (
"bytes"
"compress/bzip2"
"context"
"crypto/rand"
_ "embed"
@@ -27,9 +26,6 @@ import (
_ "github.com/ncruces/go-sqlite3/vfs/xts"
)
//go:embed testdata/speedtest1.wasm.bz2
var compressed string
var (
rt wazero.Runtime
module wazero.CompiledModule
@@ -52,10 +48,7 @@ func TestMain(m *testing.M) {
panic(err)
}
if !strings.HasPrefix(compressed, "BZh") {
panic("Please use Git LFS to clone this repo: https://git-lfs.com/")
}
binary, err := io.ReadAll(bzip2.NewReader(strings.NewReader(compressed)))
binary, err := os.ReadFile("wasm/speedtest1.wasm")
if err != nil {
panic(err)
}

View File

@@ -1 +0,0 @@
speedtest1.wasm.bz2 filter=lfs diff=lfs merge=lfs -text

Some files were not shown because too many files have changed in this diff Show More