Compare commits

..

29 Commits

Author SHA1 Message Date
Nuno Cruces
62f69011f1 Updated dependencies. 2025-09-08 13:59:41 +01:00
Nuno Cruces
4f9e3f900b binaryen-version_124. 2025-09-08 12:23:58 +01:00
dependabot[bot]
4e90618350 Bump actions/setup-go from 5 to 6 (#318)
Bumps [actions/setup-go](https://github.com/actions/setup-go) from 5 to 6.
- [Release notes](https://github.com/actions/setup-go/releases)
- [Commits](https://github.com/actions/setup-go/compare/v5...v6)

---
updated-dependencies:
- dependency-name: actions/setup-go
  dependency-version: '6'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-09-05 00:25:35 +02:00
Nuno Cruces
54bb94ce58 Improve WithMemoryCapacityFromMax (#317). 2025-09-03 09:03:59 +02:00
Nuno Cruces
07fec784e1 Grow memory geometrically. (#316) 2025-09-01 12:57:33 +02:00
dependabot[bot]
da4638cbff Bump actions/attest-build-provenance from 2 to 3 (#313)
Bumps [actions/attest-build-provenance](https://github.com/actions/attest-build-provenance) from 2 to 3.
- [Release notes](https://github.com/actions/attest-build-provenance/releases)
- [Changelog](https://github.com/actions/attest-build-provenance/blob/main/RELEASE.md)
- [Commits](https://github.com/actions/attest-build-provenance/compare/v2...v3)

---
updated-dependencies:
- dependency-name: actions/attest-build-provenance
  dependency-version: '3'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-08-29 00:19:53 +02:00
dependabot[bot]
085872c2f3 Bump github.com/ncruces/aa from 0.3.1 to 0.3.2 (#311)
Bumps [github.com/ncruces/aa](https://github.com/ncruces/aa) from 0.3.1 to 0.3.2.
- [Commits](https://github.com/ncruces/aa/compare/v0.3.1...v0.3.2)

---
updated-dependencies:
- dependency-name: github.com/ncruces/aa
  dependency-version: 0.3.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-08-26 17:20:13 +02:00
dependabot[bot]
de49aa2b06 Bump github.com/ncruces/aa from 0.3.0 to 0.3.1 (#310)
Bumps [github.com/ncruces/aa](https://github.com/ncruces/aa) from 0.3.0 to 0.3.1.
- [Commits](https://github.com/ncruces/aa/compare/v0.3.0...v0.3.1)

---
updated-dependencies:
- dependency-name: github.com/ncruces/aa
  dependency-version: 0.3.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-08-23 05:21:11 +01:00
Nuno Cruces
1f3ad0165e SQLite 3.50.4. 2025-08-21 19:05:44 +01:00
Nuno Cruces
0bda48d1d9 Gorm v1.30.1. 2025-08-21 18:56:05 +01:00
Nuno Cruces
0026bc91aa MVCC memory VFS. (#309) 2025-08-21 18:44:40 +01:00
Nuno Cruces
d84ca9d627 Fix #308. 2025-08-16 19:45:10 +01:00
Nuno Cruces
5d14e01f94 Fix #304. 2025-08-16 19:27:00 +01:00
Nuno Cruces
342df983d4 Fix #305. 2025-08-14 23:46:48 +01:00
Nuno Cruces
00476fb1e2 Tests. 2025-08-14 15:04:10 +01:00
Nuno Cruces
8a64ee6eaa Implement RowsColumnScanner. 2025-08-14 01:42:00 +01:00
Nuno Cruces
8f9a8e2752 Learnings from truffle. 2025-08-13 13:10:50 +01:00
Nuno Cruces
d8880e4cee Fixes. 2025-08-13 03:34:21 +01:00
dependabot[bot]
4b154a842c Bump actions/checkout from 4 to 5 (#303)
Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/v4...v5)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-version: '5'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-08-12 08:47:41 +01:00
dependabot[bot]
758a53e9bf Bump golang.org/x/crypto from 0.40.0 to 0.41.0 (#300)
Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.40.0 to 0.41.0.
- [Commits](https://github.com/golang/crypto/compare/v0.40.0...v0.41.0)

---
updated-dependencies:
- dependency-name: golang.org/x/crypto
  dependency-version: 0.41.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-08-08 00:30:25 +01:00
Nuno Cruces
1a42b4c590 Better fuzzing. 2025-08-07 16:27:45 +01:00
Nuno Cruces
7e4ec1df1c Fix #299. 2025-08-07 01:52:01 +01:00
Nuno Cruces
2c582a1d66 VFS improvements. 2025-08-05 14:15:21 +01:00
Nuno Cruces
20a67ca669 WAL mode serdes. 2025-08-02 11:48:37 +01:00
Nuno Cruces
789e2dc136 wasi-sdk-27. 2025-07-29 16:50:07 +01:00
Nuno Cruces
0399f10c06 VFS improvements. 2025-07-23 09:57:53 +01:00
Nuno Cruces
75c6744b5b FreeBSD 14.3. 2025-07-22 23:47:22 +01:00
Nuno Cruces
754e806164 Tests. 2025-07-22 10:34:30 +01:00
Nuno Cruces
2640c9fb54 SQLite 3.50.3. 2025-07-17 19:42:01 +01:00
72 changed files with 1883 additions and 863 deletions

View File

@@ -14,8 +14,8 @@ jobs:
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
- uses: actions/checkout@v5
- uses: actions/setup-go@v6
with: { go-version: stable }
- name: Benchmark

View File

@@ -17,13 +17,13 @@ jobs:
steps:
- uses: ilammy/msvc-dev-cmd@v1
- uses: actions/checkout@v4
- uses: actions/checkout@v5
- name: Build
shell: bash
run: .github/workflows/repro.sh
- uses: actions/attest-build-provenance@v2
- uses: actions/attest-build-provenance@v3
if: matrix.os == 'ubuntu-latest'
with:
subject-path: |

View File

@@ -30,8 +30,8 @@ jobs:
contents: write
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
- uses: actions/checkout@v5
- uses: actions/setup-go@v6
with: { go-version: stable }
- name: Format
@@ -98,14 +98,14 @@ jobs:
matrix:
os:
- name: freebsd
version: '14.2'
version: '14.3'
flags: '-test.v'
- name: netbsd
version: '10.1'
flags: '-test.v'
- name: freebsd
arch: arm64
version: '14.2'
version: '14.3'
flags: '-test.v -test.short'
- name: netbsd
arch: arm64
@@ -118,7 +118,7 @@ jobs:
needs: test
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v5
- name: Build
env:
@@ -128,7 +128,7 @@ jobs:
run: .github/workflows/build-test.sh
- name: Test
uses: cross-platform-actions/action@v0.28.0
uses: cross-platform-actions/action@v0.29.0
with:
operating_system: ${{ matrix.os.name }}
architecture: ${{ matrix.os.arch }}
@@ -155,7 +155,7 @@ jobs:
needs: test
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v5
- name: Build
env:
@@ -174,8 +174,8 @@ jobs:
steps:
- uses: bytecodealliance/actions/wasmtime/setup@v1
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
- uses: actions/checkout@v5
- uses: actions/setup-go@v6
with: { go-version: stable }
- name: Set path
@@ -195,8 +195,8 @@ jobs:
steps:
- uses: docker/setup-qemu-action@v3
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
- uses: actions/checkout@v5
- uses: actions/setup-go@v6
with: { go-version: stable }
- name: Test 386 (32-bit)
@@ -216,8 +216,8 @@ jobs:
needs: test
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
- uses: actions/checkout@v5
- uses: actions/setup-go@v6
with: { go-version: stable }
- name: Test
@@ -228,8 +228,8 @@ jobs:
needs: test
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
- uses: actions/checkout@v5
- uses: actions/setup-go@v6
with: { go-version: stable }
- name: Test

View File

@@ -604,15 +604,6 @@ func (r resultRowsAffected) RowsAffected() (int64, error) {
return int64(r), nil
}
type rows struct {
ctx context.Context
*stmt
names []string
types []string
nulls []bool
scans []scantype
}
type scantype byte
const (
@@ -648,10 +639,20 @@ func scanFromDecl(decl string) scantype {
return _ANY
}
type rows struct {
ctx context.Context
*stmt
names []string
types []string
nulls []bool
scans []scantype
}
var (
// Ensure these interfaces are implemented:
_ driver.RowsColumnTypeDatabaseTypeName = &rows{}
_ driver.RowsColumnTypeNullable = &rows{}
// _ driver.RowsColumnScanner = &rows{}
)
func (r *rows) Close() error {
@@ -740,7 +741,7 @@ func (r *rows) ColumnTypeScanType(index int) (typ reflect.Type) {
switch {
case scan == _TIME && val != _BLOB && val != _NULL:
t := r.Stmt.ColumnTime(index, r.tmRead)
useValType = t == time.Time{}
useValType = t.IsZero()
case scan == _BOOL && val == _INT:
i := r.Stmt.ColumnInt64(index)
useValType = i != 0 && i != 1
@@ -830,3 +831,23 @@ func (r *rows) Next(dest []driver.Value) error {
}
return nil
}
func (r *rows) ScanColumn(dest any, index int) error {
// notest // Go 1.26
var ptr *time.Time
switch d := dest.(type) {
case *time.Time:
ptr = d
case *sql.NullTime:
ptr = &d.Time
case *sql.Null[time.Time]:
ptr = &d.V
default:
return driver.ErrSkip
}
if t := r.Stmt.ColumnTime(index, r.tmRead); !t.IsZero() {
*ptr = t
return nil
}
return driver.ErrSkip
}

View File

@@ -1,6 +1,6 @@
# Embeddable Wasm build of SQLite
This folder includes an embeddable Wasm build of SQLite 3.50.2 for use with
This folder includes an embeddable Wasm build of SQLite 3.50.4 for use with
[`github.com/ncruces/go-sqlite3`](https://pkg.go.dev/github.com/ncruces/go-sqlite3).
The following optional features are compiled in:

Binary file not shown.

View File

@@ -7,16 +7,18 @@ ROOT=../../
BINARYEN="$ROOT/tools/binaryen/bin"
WASI_SDK="$ROOT/tools/wasi-sdk/bin"
trap 'rm -rf build/ sqlite/ bcw2.tmp' EXIT
trap 'rm -rf sqlite/ build/ bcw2.tmp' EXIT
mkdir -p sqlite/
mkdir -p build/ext/
cp "$ROOT"/sqlite3/*.[ch] build/
cp "$ROOT"/sqlite3/*.patch build/
cd sqlite/
# https://sqlite.org/src/info/a6f6fbe6173de8a2
curl -# https://sqlite.org/src/tarball/sqlite.tar.gz?r=a6f6fbe617 | tar xz
# https://sqlite.org/src/info/ba2174bdca7d1d1a
curl -#L https://github.com/sqlite/sqlite/archive/b46738f.tar.gz | tar xz --strip-components=1
# curl -#L https://sqlite.org/src/tarball/sqlite.tar.gz?r=ba2174bdca | tar xz --strip-components=1
cd sqlite
if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then
MSYS_NO_PATHCONV=1 nmake /f makefile.msc sqlite3.c "OPTS=-DSQLITE_ENABLE_UPDATE_DELETE_LIMIT -DSQLITE_ENABLE_ORDERED_SET_AGGREGATES"
else
@@ -49,7 +51,8 @@ cd ~-
-mmutable-globals -mnontrapping-fptoint \
-msimd128 -mbulk-memory -msign-ext \
-mreference-types -mmultivalue \
-fno-stack-protector -fno-stack-clash-protection \
-mno-extended-const \
-fno-stack-protector \
-Wl,--stack-first \
-Wl,--import-undefined \
-Wl,--initial-memory=327680 \
@@ -59,8 +62,9 @@ cd ~-
$(awk '{print "-Wl,--export="$0}' ../exports.txt)
"$BINARYEN/wasm-ctor-eval" -g -c _initialize bcw2.wasm -o bcw2.tmp
"$BINARYEN/wasm-opt" -g --strip --strip-producers -c -O3 \
bcw2.tmp -o bcw2.wasm --low-memory-unused \
"$BINARYEN/wasm-opt" -g bcw2.tmp -o bcw2.wasm \
--low-memory-unused --gufa --generate-global-effects --converge -O3 \
--enable-mutable-globals --enable-nontrapping-float-to-int \
--enable-simd --enable-bulk-memory --enable-sign-ext \
--enable-reference-types --enable-multivalue
--enable-reference-types --enable-multivalue \
--strip --strip-producers

View File

@@ -1,14 +1,12 @@
module github.com/ncruces/go-sqlite3/embed/bcw2
go 1.23.0
go 1.24.0
toolchain go1.24.0
require github.com/ncruces/go-sqlite3 v0.26.3
require github.com/ncruces/go-sqlite3 v0.28.0
require (
github.com/ncruces/julianday v1.0.0 // indirect
github.com/ncruces/sort v0.1.5 // indirect
github.com/tetratelabs/wazero v1.9.0 // indirect
golang.org/x/sys v0.34.0 // indirect
golang.org/x/sys v0.36.0 // indirect
)

View File

@@ -1,12 +1,12 @@
github.com/ncruces/go-sqlite3 v0.26.3 h1:WFkQj4KNMhbqiBPGDrVpK74w1DzcxQu3wYpmdWAvfYM=
github.com/ncruces/go-sqlite3 v0.26.3/go.mod h1:XFTPtFIo1DmGCh+XVP8KGn9b/o2f+z0WZuT09x2N6eo=
github.com/ncruces/go-sqlite3 v0.28.0 h1:AQVTUPgfamONl09LS+4rGFbHmLKM8/QrJJJi1UukjEQ=
github.com/ncruces/go-sqlite3 v0.28.0/go.mod h1:WqvLhYwtEiZzg1H8BIeahUv/DxbmR+3xG5jDHDiBAGk=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/ncruces/sort v0.1.5 h1:fiFWXXAqKI8QckPf/6hu/bGFwcEPrirIOFaJqWujs4k=
github.com/ncruces/sort v0.1.5/go.mod h1:obJToO4rYr6VWP0Uw5FYymgYGt3Br4RXcs/JdKaXAPk=
github.com/tetratelabs/wazero v1.9.0 h1:IcZ56OuxrtaEz8UYNRHBrUa9bYeX9oVY93KspZZBf/I=
github.com/tetratelabs/wazero v1.9.0/go.mod h1:TSbcXCfFP0L2FGkRPxHphadXPjo1T6W+CseNNY7EkjM=
golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=

View File

@@ -17,7 +17,8 @@ trap 'rm -f sqlite3.tmp' EXIT
-mmutable-globals -mnontrapping-fptoint \
-msimd128 -mbulk-memory -msign-ext \
-mreference-types -mmultivalue \
-fno-stack-protector -fno-stack-clash-protection \
-mno-extended-const \
-fno-stack-protector \
-Wl,--stack-first \
-Wl,--import-undefined \
-Wl,--initial-memory=327680 \
@@ -26,8 +27,9 @@ trap 'rm -f sqlite3.tmp' EXIT
$(awk '{print "-Wl,--export="$0}' exports.txt)
"$BINARYEN/wasm-ctor-eval" -g -c _initialize sqlite3.wasm -o sqlite3.tmp
"$BINARYEN/wasm-opt" -g --strip --strip-producers -c -O3 \
sqlite3.tmp -o sqlite3.wasm --low-memory-unused \
"$BINARYEN/wasm-opt" -g sqlite3.tmp -o sqlite3.wasm \
--low-memory-unused --gufa --generate-global-effects --converge -O3 \
--enable-mutable-globals --enable-nontrapping-float-to-int \
--enable-simd --enable-bulk-memory --enable-sign-ext \
--enable-reference-types --enable-multivalue
--enable-reference-types --enable-multivalue \
--strip --strip-producers

View File

@@ -19,7 +19,7 @@ func Test_init(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if version != "3.50.2" {
if version != "3.50.4" {
t.Error(version)
}
}

Binary file not shown.

View File

@@ -38,11 +38,11 @@ you can load into your database connections.
- [`github.com/ncruces/go-sqlite3/ext/zorder`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/zorder)
maps multidimensional data to one dimension.
### Pakages
### Packages
These packages may also be useful to work with SQLite:
- [`github.com/ncruces/decimal`](https://pkg.go.dev/github.com/ncruces/decimal)
decimal arithmetic.
- [`github.com/ncruces/julianday`](https://pkg.go.dev/github.com/ncruces/julianday)
Julian day math.
Julian day math.

View File

@@ -18,7 +18,7 @@ func Register(db *sqlite3.Conn) error {
return RegisterFS(db, nil)
}
// Register registers SQL functions readfile, lsmode,
// RegisterFS registers SQL functions readfile, lsmode,
// and the table-valued function fsdir;
// fsys will be used to read files and list directories.
func RegisterFS(db *sqlite3.Conn, fsys fs.FS) error {

View File

@@ -2,9 +2,8 @@
package serdes
import (
"io"
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/util/vfsutil"
"github.com/ncruces/go-sqlite3/vfs"
)
@@ -14,16 +13,16 @@ func init() {
vfs.Register(vfsName, sliceVFS{})
}
var fileToOpen = make(chan *sliceFile, 1)
var fileToOpen = make(chan *[]byte, 1)
// Serialize backs up a database into a byte slice.
//
// https://sqlite.org/c3ref/serialize.html
func Serialize(db *sqlite3.Conn, schema string) ([]byte, error) {
var file sliceFile
var file []byte
fileToOpen <- &file
err := db.Backup(schema, "file:serdes.db?vfs="+vfsName)
return file.data, err
err := db.Backup(schema, "file:serdes.db?nolock=1&vfs="+vfsName)
return file, err
}
// Deserialize restores a database from a byte slice,
@@ -41,8 +40,8 @@ func Serialize(db *sqlite3.Conn, schema string) ([]byte, error) {
// ["memdb"]: https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/memdb
// ["reader"]: https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/readervfs
func Deserialize(db *sqlite3.Conn, schema string, data []byte) error {
fileToOpen <- &sliceFile{data}
return db.Restore(schema, "file:serdes.db?vfs="+vfsName)
fileToOpen <- &data
return db.Restore(schema, "file:serdes.db?immutable=1&vfs="+vfsName)
}
type sliceVFS struct{}
@@ -53,14 +52,14 @@ func (sliceVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, e
}
select {
case file := <-fileToOpen:
return file, flags | vfs.OPEN_MEMORY, nil
return (*vfsutil.SliceFile)(file), flags | vfs.OPEN_MEMORY, nil
default:
return nil, flags, sqlite3.MISUSE
}
}
func (sliceVFS) Delete(name string, dirSync bool) error {
// notest // OPEN_MEMORY
// notest // no journals to delete
return sqlite3.IOERR_DELETE
}
@@ -71,70 +70,3 @@ func (sliceVFS) Access(name string, flag vfs.AccessFlag) (bool, error) {
func (sliceVFS) FullPathname(name string) (string, error) {
return name, nil
}
type sliceFile struct{ data []byte }
func (f *sliceFile) ReadAt(b []byte, off int64) (n int, err error) {
if d := f.data; off < int64(len(d)) {
n = copy(b, d[off:])
}
if n == 0 {
err = io.EOF
}
return
}
func (f *sliceFile) WriteAt(b []byte, off int64) (n int, err error) {
if d := f.data; off > int64(len(d)) {
f.data = append(d, make([]byte, off-int64(len(d)))...)
}
d := append(f.data[:off], b...)
if len(d) > len(f.data) {
f.data = d
}
return len(b), nil
}
func (f *sliceFile) Size() (int64, error) {
return int64(len(f.data)), nil
}
func (f *sliceFile) Truncate(size int64) error {
if d := f.data; size < int64(len(d)) {
f.data = d[:size]
}
return nil
}
func (f *sliceFile) SizeHint(size int64) error {
if d := f.data; size > int64(len(d)) {
f.data = append(d, make([]byte, size-int64(len(d)))...)
}
return nil
}
func (*sliceFile) Close() error { return nil }
func (*sliceFile) Sync(flag vfs.SyncFlag) error { return nil }
func (*sliceFile) Lock(lock vfs.LockLevel) error { return nil }
func (*sliceFile) Unlock(lock vfs.LockLevel) error { return nil }
func (*sliceFile) CheckReservedLock() (bool, error) {
// notest // OPEN_MEMORY
return false, nil
}
func (*sliceFile) SectorSize() int {
// notest // IOCAP_POWERSAFE_OVERWRITE
return 0
}
func (*sliceFile) DeviceCharacteristics() vfs.DeviceCharacteristic {
return vfs.IOCAP_ATOMIC |
vfs.IOCAP_SAFE_APPEND |
vfs.IOCAP_SEQUENTIAL |
vfs.IOCAP_POWERSAFE_OVERWRITE |
vfs.IOCAP_SUBPAGE_READ
}

View File

@@ -1,6 +1,7 @@
package serdes_test
import (
_ "embed"
"errors"
"io"
"net/http"
@@ -11,7 +12,30 @@ import (
"github.com/ncruces/go-sqlite3/ext/serdes"
)
func TestDeserialize(t *testing.T) {
//go:embed testdata/wal.db
var walDB []byte
func Test_wal(t *testing.T) {
db, err := sqlite3.Open("testdata/wal.db")
if err != nil {
t.Fatal(err)
}
defer db.Close()
data, err := serdes.Serialize(db, "main")
if err != nil {
t.Fatal(err)
}
compareDBs(t, data, walDB)
err = serdes.Deserialize(db, "temp", walDB)
if err != nil {
t.Fatal(err)
}
}
func Test_northwind(t *testing.T) {
if testing.Short() {
t.Skip("skipping in short mode")
}
@@ -37,10 +61,14 @@ func TestDeserialize(t *testing.T) {
t.Fatal(err)
}
if len(input) != len(output) {
compareDBs(t, input, output)
}
func compareDBs(t *testing.T, a, b []byte) {
if len(a) != len(b) {
t.Fatal("lengths are different")
}
for i := range input {
for i := range a {
// These may be different.
switch {
case 24 <= i && i < 28:
@@ -53,14 +81,14 @@ func TestDeserialize(t *testing.T) {
// SQLite version that wrote the file.
continue
}
if input[i] != output[i] {
t.Errorf("difference at %d: %d %d", i, input[i], output[i])
if a[i] != b[i] {
t.Errorf("difference at %d: %d %d", i, a[i], b[i])
}
}
}
func httpGet() ([]byte, error) {
res, err := http.Get("https://raw.githubusercontent.com/jpwhite3/northwind-SQLite3/refs/heads/main/dist/northwind.db")
res, err := http.Get("https://github.com/jpwhite3/northwind-SQLite3/raw/refs/heads/main/dist/northwind.db")
if err != nil {
return nil, err
}

BIN
ext/serdes/testdata/wal.db vendored Normal file

Binary file not shown.

View File

@@ -43,7 +43,7 @@ import (
"github.com/ncruces/go-sqlite3/internal/util"
)
// Set RegisterLike to false to not register a Unicode aware LIKE operator.
// RegisterLike must be set to false to not register a Unicode aware LIKE operator.
// Overriding the built-in LIKE operator disables the [LIKE optimization].
//
// [LIKE optimization]: https://sqlite.org/optoverview.html#the_like_optimization

View File

@@ -19,9 +19,9 @@ func Register(db *sqlite3.Conn) error {
}
func zorder(ctx sqlite3.Context, arg ...sqlite3.Value) {
var x [63]int64
if len(arg) > len(x) {
ctx.ResultError(util.ErrorString("zorder: too many parameters"))
var x [24]int64
if n := len(arg); n < 2 || n > 24 {
ctx.ResultError(util.ErrorString("zorder: needs between 2 and 24 dimensions"))
return
}
for i := range arg {
@@ -29,17 +29,15 @@ func zorder(ctx sqlite3.Context, arg ...sqlite3.Value) {
}
var z int64
if len(arg) > 0 {
for i := range x {
j := i % len(arg)
z |= (x[j] & 1) << i
x[j] >>= 1
}
for i := range 63 {
j := i % len(arg)
z |= (x[j] & 1) << i
x[j] >>= 1
}
for i := range arg {
if x[i] != 0 {
ctx.ResultError(util.ErrorString("zorder: parameter too large"))
ctx.ResultError(util.ErrorString("zorder: argument out of range"))
return
}
}
@@ -51,6 +49,19 @@ func unzorder(ctx sqlite3.Context, arg ...sqlite3.Value) {
n := arg[1].Int64()
z := arg[0].Int64()
if n < 2 || n > 24 {
ctx.ResultError(util.ErrorString("unzorder: needs between 2 and 24 dimensions"))
return
}
if i < 0 || i >= n {
ctx.ResultError(util.ErrorString("unzorder: index out of range"))
return
}
if z < 0 {
ctx.ResultError(util.ErrorString("unzorder: argument out of range"))
return
}
var k int
var x int64
for j := i; j < 63; j += n {

View File

@@ -12,7 +12,7 @@ import (
"github.com/ncruces/go-sqlite3/vfs/memdb"
)
func TestRegister_zorder(t *testing.T) {
func Test_zorder(t *testing.T) {
t.Parallel()
tmp := memdb.TestDB(t)
@@ -57,7 +57,7 @@ func TestRegister_zorder(t *testing.T) {
}
}
func TestRegister_unzorder(t *testing.T) {
func Test_unzorder(t *testing.T) {
t.Parallel()
tmp := memdb.TestDB(t)
@@ -85,7 +85,7 @@ func TestRegister_unzorder(t *testing.T) {
}
}
func TestRegister_error(t *testing.T) {
func Test_zorder_error(t *testing.T) {
t.Parallel()
tmp := memdb.TestDB(t)
@@ -103,7 +103,7 @@ func TestRegister_error(t *testing.T) {
var buf strings.Builder
buf.WriteString("SELECT zorder(0")
for i := 1; i < 80; i++ {
for i := 1; i < 25; i++ {
buf.WriteByte(',')
buf.WriteString(strconv.Itoa(0))
}
@@ -113,3 +113,30 @@ func TestRegister_error(t *testing.T) {
t.Error("want error")
}
}
func Test_unzorder_error(t *testing.T) {
t.Parallel()
tmp := memdb.TestDB(t)
db, err := driver.Open(tmp, zorder.Register)
if err != nil {
t.Fatal(err)
}
defer db.Close()
var got int64
err = db.QueryRow(`SELECT unzorder(-1, 2, 0)`).Scan(&got)
if err == nil {
t.Error("want error")
}
err = db.QueryRow(`SELECT unzorder(0, 2, 2)`).Scan(&got)
if err == nil {
t.Error("want error")
}
err = db.QueryRow(`SELECT unzorder(0, 25, 2)`).Scan(&got)
if err == nil {
t.Error("want error")
}
}

14
func.go
View File

@@ -59,7 +59,7 @@ func (c *Conn) CreateCollation(name string, fn CollatingFunction) error {
return c.error(rc)
}
// Collating function is the type of a collation callback.
// CollatingFunction is the type of a collation callback.
// Implementations must not retain a or b.
type CollatingFunction func(a, b []byte) int
@@ -132,7 +132,7 @@ func (c *Conn) CreateWindowFunction(name string, nArg int, flag FunctionFlag, fn
if win, ok := agg.(WindowFunction); ok {
return win
}
return windowFunc{agg, name}
return agg
}))
}
rc := res_t(c.call("sqlite3_create_window_function_go",
@@ -307,13 +307,3 @@ func (a *aggregateFunc) Close() error {
a.stop()
return nil
}
type windowFunc struct {
AggregateFunction
name string
}
func (w windowFunc) Inverse(ctx Context, arg ...Value) {
// Implementing inverse allows certain queries that don't really need it to succeed.
ctx.ResultError(util.ErrorString(w.name + ": may not be used as a window function"))
}

13
go.mod
View File

@@ -1,23 +1,22 @@
module github.com/ncruces/go-sqlite3
go 1.23.0
toolchain go1.24.0
go 1.24.0
require (
github.com/ncruces/aa v0.3.3
github.com/ncruces/julianday v1.0.0
github.com/ncruces/sort v0.1.5
github.com/tetratelabs/wazero v1.9.0
golang.org/x/crypto v0.40.0
golang.org/x/sys v0.34.0
golang.org/x/crypto v0.41.0
golang.org/x/sys v0.36.0
)
require (
github.com/dchest/siphash v1.2.3 // ext/bloom
github.com/google/uuid v1.6.0 // ext/uuid
github.com/psanford/httpreadat v0.1.0 // example
golang.org/x/sync v0.16.0 // test
golang.org/x/text v0.27.0 // ext/unicode
golang.org/x/sync v0.17.0 // test
golang.org/x/text v0.29.0 // ext/unicode
lukechampine.com/adiantum v1.1.1 // vfs/adiantum
)

18
go.sum
View File

@@ -2,6 +2,8 @@ github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA=
github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/ncruces/aa v0.3.3 h1:f5Y9nQcbKHEaBks7kIcLmUgsDbokRPRCzNCx6XVGEr0=
github.com/ncruces/aa v0.3.3/go.mod h1:ctOw1LVqfuqzqg2S9LlR045bLAiXtaTiPMCL3zzl7Ik=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/ncruces/sort v0.1.5 h1:fiFWXXAqKI8QckPf/6hu/bGFwcEPrirIOFaJqWujs4k=
@@ -10,13 +12,13 @@ github.com/psanford/httpreadat v0.1.0 h1:VleW1HS2zO7/4c7c7zNl33fO6oYACSagjJIyMIw
github.com/psanford/httpreadat v0.1.0/go.mod h1:Zg7P+TlBm3bYbyHTKv/EdtSJZn3qwbPwpfZ/I9GKCRE=
github.com/tetratelabs/wazero v1.9.0 h1:IcZ56OuxrtaEz8UYNRHBrUa9bYeX9oVY93KspZZBf/I=
github.com/tetratelabs/wazero v1.9.0/go.mod h1:TSbcXCfFP0L2FGkRPxHphadXPjo1T6W+CseNNY7EkjM=
golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM=
golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY=
golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4=
golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU=
golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4=
golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc=
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
lukechampine.com/adiantum v1.1.1 h1:4fp6gTxWCqpEbLy40ExiYDDED3oUNWx5cTqBCtPdZqA=
lukechampine.com/adiantum v1.1.1/go.mod h1:LrAYVnTYLnUtE/yMp5bQr0HstAf060YUF8nM0B6+rUw=

View File

@@ -1,12 +1,10 @@
module github.com/ncruces/go-sqlite3/gormlite
go 1.23.0
toolchain go1.24.0
go 1.24.0
require (
github.com/ncruces/go-sqlite3 v0.26.3
gorm.io/gorm v1.30.0
github.com/ncruces/go-sqlite3 v0.28.0
gorm.io/gorm v1.30.5
)
require (
@@ -14,6 +12,6 @@ require (
github.com/jinzhu/now v1.1.5 // indirect
github.com/ncruces/julianday v1.0.0 // indirect
github.com/tetratelabs/wazero v1.9.0 // indirect
golang.org/x/sys v0.34.0 // indirect
golang.org/x/text v0.27.0 // indirect
golang.org/x/sys v0.36.0 // indirect
golang.org/x/text v0.29.0 // indirect
)

View File

@@ -2,15 +2,15 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/ncruces/go-sqlite3 v0.26.3 h1:WFkQj4KNMhbqiBPGDrVpK74w1DzcxQu3wYpmdWAvfYM=
github.com/ncruces/go-sqlite3 v0.26.3/go.mod h1:XFTPtFIo1DmGCh+XVP8KGn9b/o2f+z0WZuT09x2N6eo=
github.com/ncruces/go-sqlite3 v0.28.0 h1:AQVTUPgfamONl09LS+4rGFbHmLKM8/QrJJJi1UukjEQ=
github.com/ncruces/go-sqlite3 v0.28.0/go.mod h1:WqvLhYwtEiZzg1H8BIeahUv/DxbmR+3xG5jDHDiBAGk=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/tetratelabs/wazero v1.9.0 h1:IcZ56OuxrtaEz8UYNRHBrUa9bYeX9oVY93KspZZBf/I=
github.com/tetratelabs/wazero v1.9.0/go.mod h1:TSbcXCfFP0L2FGkRPxHphadXPjo1T6W+CseNNY7EkjM=
golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4=
golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU=
gorm.io/gorm v1.30.0 h1:qbT5aPv1UH8gI99OsRlvDToLxW5zR7FzS9acZDOZcgs=
gorm.io/gorm v1.30.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
gorm.io/gorm v1.30.5 h1:dvEfYwxL+i+xgCNSGGBT1lDjCzfELK8fHZxL3Ee9X0s=
gorm.io/gorm v1.30.5/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=

View File

@@ -7,7 +7,7 @@ rm -rf gorm/ tests/
go work use -r .
go test
git clone --branch v1.30.0 --filter=blob:none https://github.com/go-gorm/gorm.git
git clone --branch v1.30.5 --filter=blob:none https://github.com/go-gorm/gorm.git
mv gorm/tests tests
rm -rf gorm/

View File

@@ -7,13 +7,14 @@ diff --git a/tests/.gitignore b/tests/.gitignore
diff --git a/tests/tests_test.go b/tests/tests_test.go
--- a/tests/tests_test.go
+++ b/tests/tests_test.go
@@ -7,9 +7,11 @@ import (
@@ -8,9 +8,11 @@ import (
"path/filepath"
"time"
+ _ "github.com/ncruces/go-sqlite3/embed"
+ sqlite "github.com/ncruces/go-sqlite3/gormlite"
+
"gorm.io/driver/gaussdb"
"gorm.io/driver/mysql"
"gorm.io/driver/postgres"
- "gorm.io/driver/sqlite"

View File

@@ -9,24 +9,31 @@ import (
"golang.org/x/sys/unix"
)
func NewMemory(_, max uint64) experimental.LinearMemory {
func NewMemory(cap, max uint64) experimental.LinearMemory {
// Round up to the page size.
rnd := uint64(unix.Getpagesize() - 1)
max = (max + rnd) &^ rnd
res := (max + rnd) &^ rnd
if max > math.MaxInt {
// This ensures int(max) overflows to a negative value,
if res > math.MaxInt {
// This ensures int(res) overflows to a negative value,
// and unix.Mmap returns EINVAL.
max = math.MaxUint64
res = math.MaxUint64
}
// Reserve max bytes of address space, to ensure we won't need to move it.
com := res
prot := unix.PROT_READ | unix.PROT_WRITE
if cap < max { // Commit memory only if cap=max.
com = 0
prot = unix.PROT_NONE
}
// Reserve res bytes of address space, to ensure we won't need to move it.
// A protected, private, anonymous mapping should not commit memory.
b, err := unix.Mmap(-1, 0, int(max), unix.PROT_NONE, unix.MAP_PRIVATE|unix.MAP_ANON)
b, err := unix.Mmap(-1, 0, int(res), prot, unix.MAP_PRIVATE|unix.MAP_ANON)
if err != nil {
panic(err)
}
return &mmappedMemory{buf: b[:0]}
return &mmappedMemory{buf: b[:com]}
}
// The slice covers the entire mmapped memory:
@@ -40,9 +47,11 @@ func (m *mmappedMemory) Reallocate(size uint64) []byte {
com := uint64(len(m.buf))
res := uint64(cap(m.buf))
if com < size && size <= res {
// Round up to the page size.
// Grow geometrically, round up to the page size.
rnd := uint64(unix.Getpagesize() - 1)
new := (size + rnd) &^ rnd
new := com + com>>3
new = min(max(size, new), res)
new = (new + rnd) &^ rnd
// Commit additional memory up to new bytes.
err := unix.Mprotect(m.buf[com:new], unix.PROT_READ|unix.PROT_WRITE)
@@ -50,8 +59,7 @@ func (m *mmappedMemory) Reallocate(size uint64) []byte {
return nil
}
// Update committed memory.
m.buf = m.buf[:new]
m.buf = m.buf[:new] // Update committed memory.
}
// Limit returned capacity because bytes beyond
// len(m.buf) have not yet been committed.

View File

@@ -9,20 +9,26 @@ import (
"golang.org/x/sys/windows"
)
func NewMemory(_, max uint64) experimental.LinearMemory {
func NewMemory(cap, max uint64) experimental.LinearMemory {
// Round up to the page size.
rnd := uint64(windows.Getpagesize() - 1)
max = (max + rnd) &^ rnd
res := (max + rnd) &^ rnd
if max > math.MaxInt {
// This ensures uintptr(max) overflows to a large value,
if res > math.MaxInt {
// This ensures uintptr(res) overflows to a large value,
// and windows.VirtualAlloc returns an error.
max = math.MaxUint64
res = math.MaxUint64
}
// Reserve max bytes of address space, to ensure we won't need to move it.
// This does not commit memory.
r, err := windows.VirtualAlloc(0, uintptr(max), windows.MEM_RESERVE, windows.PAGE_READWRITE)
com := res
kind := windows.MEM_COMMIT
if cap < max { // Commit memory only if cap=max.
com = 0
kind = windows.MEM_RESERVE
}
// Reserve res bytes of address space, to ensure we won't need to move it.
r, err := windows.VirtualAlloc(0, uintptr(res), uint32(kind), windows.PAGE_READWRITE)
if err != nil {
panic(err)
}
@@ -30,8 +36,9 @@ func NewMemory(_, max uint64) experimental.LinearMemory {
mem := virtualMemory{addr: r}
// SliceHeader, although deprecated, avoids a go vet warning.
sh := (*reflect.SliceHeader)(unsafe.Pointer(&mem.buf))
sh.Cap = int(max)
sh.Data = r
sh.Len = int(com)
sh.Cap = int(res)
return &mem
}
@@ -47,9 +54,11 @@ func (m *virtualMemory) Reallocate(size uint64) []byte {
com := uint64(len(m.buf))
res := uint64(cap(m.buf))
if com < size && size <= res {
// Round up to the page size.
// Grow geometrically, round up to the page size.
rnd := uint64(windows.Getpagesize() - 1)
new := (size + rnd) &^ rnd
new := com + com>>3
new = min(max(size, new), res)
new = (new + rnd) &^ rnd
// Commit additional memory up to new bytes.
_, err := windows.VirtualAlloc(m.addr, uintptr(new), windows.MEM_COMMIT, windows.PAGE_READWRITE)
@@ -57,8 +66,7 @@ func (m *virtualMemory) Reallocate(size uint64) []byte {
return nil
}
// Update committed memory.
m.buf = m.buf[:new]
m.buf = m.buf[:new] // Update committed memory.
}
// Limit returned capacity because bytes beyond
// len(m.buf) have not yet been committed.

View File

@@ -20,20 +20,6 @@ func ExportFuncVI[T0 i32](mod wazero.HostModuleBuilder, name string, fn func(con
Export(name)
}
type funcVII[T0, T1 i32] func(context.Context, api.Module, T0, T1)
func (fn funcVII[T0, T1]) Call(ctx context.Context, mod api.Module, stack []uint64) {
_ = stack[1] // prevent bounds check on every slice access
fn(ctx, mod, T0(stack[0]), T1(stack[1]))
}
func ExportFuncVII[T0, T1 i32](mod wazero.HostModuleBuilder, name string, fn func(context.Context, api.Module, T0, T1)) {
mod.NewFunctionBuilder().
WithGoModuleFunction(funcVII[T0, T1](fn),
[]api.ValueType{api.ValueTypeI32, api.ValueTypeI32}, nil).
Export(name)
}
type funcVIII[T0, T1, T2 i32] func(context.Context, api.Module, T0, T1, T2)
func (fn funcVIII[T0, T1, T2]) Call(ctx context.Context, mod api.Module, stack []uint64) {

View File

@@ -2,7 +2,7 @@
# handle, and interrupt, sqlite3_busy_timeout.
--- sqlite3.c.orig
+++ sqlite3.c
@@ -184433,7 +184433,7 @@
@@ -184474,7 +184474,7 @@
if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
#endif
if( ms>0 ){

View File

@@ -3,7 +3,7 @@ set -euo pipefail
cd -P -- "$(dirname -- "$0")"
curl -#OL "https://sqlite.org/2025/sqlite-amalgamation-3500200.zip"
curl -#OL "https://sqlite.org/2025/sqlite-amalgamation-3500400.zip"
unzip -d . sqlite-amalgamation-*.zip
mv sqlite-amalgamation-*/sqlite3.c .
mv sqlite-amalgamation-*/sqlite3.h .
@@ -19,30 +19,30 @@ rm -rf sqlite-amalgamation-*
mkdir -p ext/
cd ext/
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.2/ext/misc/anycollseq.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.2/ext/misc/base64.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.2/ext/misc/decimal.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.2/ext/misc/ieee754.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.2/ext/misc/regexp.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.2/ext/misc/series.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.2/ext/misc/spellfix.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.2/ext/misc/uint.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.4/ext/misc/anycollseq.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.4/ext/misc/base64.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.4/ext/misc/decimal.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.4/ext/misc/ieee754.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.4/ext/misc/regexp.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.4/ext/misc/series.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.4/ext/misc/spellfix.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.4/ext/misc/uint.c"
cd ~-
cd ../vfs/tests/mptest/testdata/
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.2/mptest/config01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.2/mptest/config02.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.2/mptest/crash01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.2/mptest/crash02.subtest"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.2/mptest/multiwrite01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.4/mptest/config01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.4/mptest/config02.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.4/mptest/crash01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.4/mptest/crash02.subtest"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.4/mptest/multiwrite01.test"
cd ~-
cd ../vfs/tests/mptest/wasm/
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.2/mptest/mptest.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.4/mptest/mptest.c"
cd ~-
cd ../vfs/tests/speedtest1/wasm/
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.2/test/speedtest1.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.50.4/test/speedtest1.c"
cd ~-
cat *.patch | patch -p0 --no-backup-if-mismatch

View File

@@ -22,7 +22,8 @@ EOF
-mmutable-globals -mnontrapping-fptoint \
-msimd128 -mbulk-memory -msign-ext \
-mreference-types -mmultivalue \
-fno-stack-protector -fno-stack-clash-protection \
-mno-extended-const \
-fno-stack-protector \
-Wl,-z,stack-size=4096 \
-Wl,--stack-first \
-Wl,--import-undefined \
@@ -42,10 +43,11 @@ EOF
-Wl,--export=qsort
"$BINARYEN/wasm-ctor-eval" -g -c _initialize libc.wasm -o libc.tmp
"$BINARYEN/wasm-opt" -g --strip --strip-producers -c -O3 \
libc.tmp -o libc.wasm \
"$BINARYEN/wasm-opt" -g libc.tmp -o libc.wasm \
--low-memory-unused --generate-global-effects --converge -O3 \
--enable-mutable-globals --enable-nontrapping-float-to-int \
--enable-simd --enable-bulk-memory --enable-sign-ext \
--enable-reference-types --enable-multivalue
--enable-reference-types --enable-multivalue \
--strip --strip-debug --strip-producers
"$BINARYEN/wasm-dis" -o libc.wat libc.wasm

Binary file not shown.

File diff suppressed because it is too large Load Diff

View File

@@ -8,7 +8,6 @@ import (
"os"
"strings"
"testing"
"unicode/utf8"
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/api"
@@ -750,15 +749,7 @@ func Fuzz_strspn(f *testing.F) {
s = term(s)
chars = term(chars)
want := strings.IndexFunc(s, func(r rune) bool {
if uint32(r) >= utf8.RuneSelf {
t.Skip()
}
return strings.IndexByte(chars, byte(r)) < 0
})
if want < 0 {
want = len(s)
}
want := indexNotByte(s, chars)
if uint32(got) != uint32(want) {
t.Errorf("strspn(%v, %v) = %d, want %d",
@@ -778,11 +769,6 @@ func Fuzz_strcspn(f *testing.F) {
if len(s) > 128 || len(chars) > 128 {
t.SkipNow()
}
if strings.ContainsFunc(chars, func(r rune) bool {
return uint32(r) >= utf8.RuneSelf
}) {
t.SkipNow()
}
copy(memory[ptr1:], s)
copy(memory[ptr2:], chars)
memory[ptr1+len(s)] = 0
@@ -792,10 +778,7 @@ func Fuzz_strcspn(f *testing.F) {
s = term(s)
chars = term(chars)
want := strings.IndexAny(s, chars)
if want < 0 {
want = len(s)
}
want := indexAnyByte(s, chars)
if uint32(got) != uint32(want) {
t.Errorf("strcspn(%q, %q) = %d, want %d",
@@ -838,3 +821,21 @@ func term1[T interface{ []byte | string }](s T) T {
}
return s
}
func indexNotByte(s, chars string) int {
for i, c := range []byte(s) {
if strings.IndexByte(chars, c) < 0 {
return i
}
}
return len(s)
}
func indexAnyByte(s, chars string) int {
for i, c := range []byte(s) {
if strings.IndexByte(chars, c) >= 0 {
return i
}
}
return len(s)
}

View File

@@ -242,17 +242,21 @@ char *strrchr(const char *s, int c) {
// SIMDized check which bytes are in a set (Geoff Langdale)
// http://0x80.pl/notesen/2018-10-18-simd-byte-lookup.html
// This is the same algorithm as truffle from Hyperscan:
// https://github.com/intel/hyperscan/blob/v5.4.2/src/nfa/truffle.c#L64-L81
// https://github.com/intel/hyperscan/blob/v5.4.2/src/nfa/trufflecompile.cpp
typedef struct {
__u8x16 lo;
__u8x16 hi;
} __wasm_v128_bitmap256_t;
__attribute__((always_inline))
static void __wasm_v128_setbit(__wasm_v128_bitmap256_t *bitmap, int i) {
uint8_t hi_nibble = (uint8_t)i >> 4;
uint8_t lo_nibble = (uint8_t)i & 0xf;
bitmap->lo[lo_nibble] |= (uint8_t)((uint32_t)1 << (hi_nibble - 0));
bitmap->hi[lo_nibble] |= (uint8_t)((uint32_t)1 << (hi_nibble - 8));
static void __wasm_v128_setbit(__wasm_v128_bitmap256_t *bitmap, uint8_t i) {
uint8_t hi_nibble = i >> 4;
uint8_t lo_nibble = i & 0xf;
bitmap->lo[lo_nibble] |= (uint8_t)(1u << (hi_nibble - 0));
bitmap->hi[lo_nibble] |= (uint8_t)(1u << (hi_nibble - 8));
}
#ifndef __wasm_relaxed_simd__
@@ -264,18 +268,17 @@ static void __wasm_v128_setbit(__wasm_v128_bitmap256_t *bitmap, int i) {
__attribute__((always_inline))
static v128_t __wasm_v128_chkbits(__wasm_v128_bitmap256_t bitmap, v128_t v) {
v128_t hi_nibbles = wasm_u8x16_shr(v, 4);
v128_t bitmask_lookup = wasm_u8x16_const(1, 2, 4, 8, 16, 32, 64, 128, //
1, 2, 4, 8, 16, 32, 64, 128);
v128_t bitmask_lookup = wasm_u64x2_const_splat(0x8040201008040201);
v128_t bitmask = wasm_i8x16_relaxed_swizzle(bitmask_lookup, hi_nibbles);
v128_t indices_0_7 = v & wasm_u8x16_const_splat(0x8f);
v128_t indices_8_15 = indices_0_7 ^ wasm_u8x16_const_splat(0x80);
v128_t row_0_7 = wasm_i8x16_swizzle(bitmap.lo, indices_0_7);
v128_t row_8_15 = wasm_i8x16_swizzle(bitmap.hi, indices_8_15);
v128_t row_0_7 = wasm_i8x16_swizzle((v128_t)bitmap.lo, indices_0_7);
v128_t row_8_15 = wasm_i8x16_swizzle((v128_t)bitmap.hi, indices_8_15);
v128_t bitsets = row_0_7 | row_8_15;
return wasm_i8x16_eq(bitsets & bitmask, bitmask);
return bitsets & bitmask;
}
#undef wasm_i8x16_relaxed_swizzle
@@ -317,17 +320,18 @@ size_t strspn(const char *s, const char *c) {
for (; *c; c++) {
// Terminator IS NOT on the bitmap.
__wasm_v128_setbit(&bitmap, *c);
__wasm_v128_setbit(&bitmap, (uint8_t)*c);
}
for (;;) {
v128_t v = *(v128_t *)addr;
v128_t cmp = __wasm_v128_chkbits(bitmap, v);
v128_t found = __wasm_v128_chkbits(bitmap, v);
// Bitmask is slow on AArch64, all_true is much faster.
if (!wasm_i8x16_all_true(cmp)) {
if (!wasm_i8x16_all_true(found)) {
v128_t cmp = wasm_i8x16_eq(found, (v128_t){});
// Clear the bits corresponding to align (little-endian)
// so we can count trailing zeros.
int mask = (uint16_t)~wasm_i8x16_bitmask(cmp) >> align << align;
int mask = wasm_i8x16_bitmask(cmp) >> align << align;
// At least one bit will be set, unless align cleared them.
// Knowing this helps the compiler if it unrolls the loop.
__builtin_assume(mask || align);
@@ -356,17 +360,18 @@ size_t strcspn(const char *s, const char *c) {
do {
// Terminator IS on the bitmap.
__wasm_v128_setbit(&bitmap, *c);
__wasm_v128_setbit(&bitmap, (uint8_t)*c);
} while (*c++);
for (;;) {
v128_t v = *(v128_t *)addr;
v128_t cmp = __wasm_v128_chkbits(bitmap, v);
v128_t found = __wasm_v128_chkbits(bitmap, v);
// Bitmask is slow on AArch64, any_true is much faster.
if (wasm_v128_any_true(cmp)) {
if (wasm_v128_any_true(found)) {
v128_t cmp = wasm_i8x16_eq(found, (v128_t){});
// Clear the bits corresponding to align (little-endian)
// so we can count trailing zeros.
int mask = wasm_i8x16_bitmask(cmp) >> align << align;
int mask = (uint16_t)~wasm_i8x16_bitmask(cmp) >> align << align;
// At least one bit will be set, unless align cleared them.
// Knowing this helps the compiler if it unrolls the loop.
__builtin_assume(mask || align);

View File

@@ -26,8 +26,8 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then
fi
fi
WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/wasi-sdk-25.0-$WASI_SDK.tar.gz"
BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_123/binaryen-version_123-$BINARYEN.tar.gz"
WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-27/wasi-sdk-27.0-$WASI_SDK.tar.gz"
BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_124/binaryen-version_124-$BINARYEN.tar.gz"
# Download tools
mkdir -p "$ROOT/tools"

View File

@@ -1,7 +1,7 @@
# Remove VFS registration. Go handles it.
--- sqlite3.c.orig
+++ sqlite3.c
@@ -26883,7 +26883,7 @@
@@ -26884,7 +26884,7 @@
sqlite3_free(p);
return sqlite3_os_init();
}
@@ -10,7 +10,7 @@
/*
** The list of all registered VFS implementations.
*/
@@ -26980,7 +26980,7 @@
@@ -26981,7 +26981,7 @@
sqlite3_mutex_leave(mutex);
return SQLITE_OK;
}

View File

@@ -35,7 +35,7 @@ type params struct {
*sql.DB
}
func (t params) mustExec(sql string, args ...interface{}) sql.Result {
func (t params) mustExec(sql string, args ...any) sql.Result {
res, err := t.DB.Exec(sql, args...)
if err != nil {
t.Fatalf("Error running %q: %v", sql, err)

View File

@@ -12,7 +12,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
_ "github.com/ncruces/go-sqlite3/vfs/memdb"
"github.com/ncruces/go-sqlite3/vfs/memdb"
)
func TestConn_Open_dir(t *testing.T) {
@@ -112,6 +112,48 @@ func TestConn_Close_BUSY(t *testing.T) {
}
}
func TestConn_BusyHandler(t *testing.T) {
t.Parallel()
dsn := memdb.TestDB(t)
db1, err := sqlite3.Open(dsn)
if err != nil {
t.Fatal(err)
}
defer db1.Close()
db2, err := sqlite3.Open(dsn)
if err != nil {
t.Fatal(err)
}
defer db2.Close()
var called bool
err = db2.BusyHandler(func(ctx context.Context, count int) (retry bool) {
called = true
return count < 1
})
if err != nil {
t.Fatal(err)
}
tx, err := db1.BeginExclusive()
if err != nil {
t.Fatal(err)
}
defer tx.End(&err)
_, err = db2.BeginExclusive()
if !errors.Is(err, sqlite3.BUSY) {
t.Errorf("got %v, want sqlite3.BUSY", err)
}
if !called {
t.Error("busy handler not called")
}
}
func TestConn_SetInterrupt(t *testing.T) {
t.Parallel()

View File

@@ -18,6 +18,7 @@ import (
"github.com/ncruces/go-sqlite3/vfs"
_ "github.com/ncruces/go-sqlite3/vfs/adiantum"
"github.com/ncruces/go-sqlite3/vfs/memdb"
"github.com/ncruces/go-sqlite3/vfs/mvcc"
_ "github.com/ncruces/go-sqlite3/vfs/xts"
)
@@ -98,6 +99,22 @@ func Test_memdb(t *testing.T) {
testIntegrity(t, name)
}
func Test_mvcc(t *testing.T) {
var iter int
if testing.Short() {
iter = 1000
} else {
iter = 5000
}
mvcc.Create("test.db", "")
name := "file:/test.db?vfs=mvcc" +
"&_pragma=busy_timeout(10000)"
createDB(t, name)
testParallel(t, name, iter)
testIntegrity(t, name)
}
func Test_adiantum(t *testing.T) {
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
@@ -312,6 +329,16 @@ func Benchmark_memdb(b *testing.B) {
testParallel(b, name, b.N)
}
func Benchmark_mvcc(b *testing.B) {
mvcc.Create("test.db", "")
name := "file:/test.db?vfs=mvcc" +
"&_pragma=busy_timeout(10000)"
createDB(b, name)
b.ResetTimer()
testParallel(b, name, b.N)
}
func createDB(t testing.TB, name string) {
db, err := sqlite3.Open(name)
if err != nil {

View File

@@ -49,7 +49,7 @@ func (s *SeekingReaderAt) Size() (int64, error) {
return s.r.Seek(0, io.SeekEnd)
}
// ReadAt implements [io.Closer].
// Close implements [io.Closer].
func (s *SeekingReaderAt) Close() error {
s.l.Lock()
defer s.l.Unlock()

View File

@@ -23,7 +23,7 @@ func (FS) Open(name string) (fs.File, error) {
return os.OpenFile(name, os.O_RDONLY, 0)
}
// ReadFileFS implements [fs.StatFS].
// Stat implements [fs.StatFS].
func (FS) Stat(name string) (fs.FileInfo, error) {
return os.Stat(name)
}

View File

@@ -17,14 +17,16 @@ trap 'rm -f sql3parse_table.tmp' EXIT
-mmutable-globals -mnontrapping-fptoint \
-msimd128 -mbulk-memory -msign-ext \
-mreference-types -mmultivalue \
-fno-stack-protector -fno-stack-clash-protection \
-mno-extended-const \
-fno-stack-protector \
-Wl,--stack-first \
-Wl,--import-undefined \
-Wl,--export=sql3parse_table
"$BINARYEN/wasm-ctor-eval" -c _initialize sql3parse_table.wasm -o sql3parse_table.tmp
"$BINARYEN/wasm-opt" --strip --strip-debug --strip-producers -c -Oz \
sql3parse_table.tmp -o sql3parse_table.wasm --low-memory-unused \
"$BINARYEN/wasm-opt" sql3parse_table.tmp -o sql3parse_table.wasm \
--low-memory-unused --gufa --generate-global-effects --converge -Oz \
--enable-mutable-globals --enable-nontrapping-float-to-int \
--enable-simd --enable-bulk-memory --enable-sign-ext \
--enable-reference-types --enable-multivalue
--enable-reference-types --enable-multivalue \
--strip --strip-debug --strip-producers

102
util/vfsutil/slice.go Normal file
View File

@@ -0,0 +1,102 @@
package vfsutil
import (
"io"
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/vfs"
)
// SliceFile implements [vfs.File] with a byte slice.
// It is suitable for temporary files (such as [vfs.OPEN_TEMP_JOURNAL]),
// but not concurrency safe.
type SliceFile []byte
var (
// Ensure these interfaces are implemented:
_ vfs.FileSizeHint = &SliceFile{}
)
// ReadAt implements [io.ReaderAt].
func (f *SliceFile) ReadAt(b []byte, off int64) (n int, err error) {
if d := *f; off < int64(len(d)) {
n = copy(b, d[off:])
}
if n < len(b) {
err = io.EOF
}
return
}
// WriteAt implements [io.WriterAt].
func (f *SliceFile) WriteAt(b []byte, off int64) (n int, err error) {
d := *f
if off > int64(len(d)) {
d = append(d, make([]byte, off-int64(len(d)))...)
}
d = append(d[:off], b...)
if len(d) > len(*f) {
*f = d
}
return len(b), nil
}
// Size implements [vfs.File].
func (f *SliceFile) Size() (int64, error) {
return int64(len(*f)), nil
}
// Truncate implements [vfs.File].
func (f *SliceFile) Truncate(size int64) error {
if d := *f; size < int64(len(d)) {
*f = d[:size]
}
return nil
}
// SizeHint implements [vfs.FileSizeHint].
func (f *SliceFile) SizeHint(size int64) error {
if d := *f; size > int64(len(d)) {
*f = append(d, make([]byte, size-int64(len(d)))...)
}
return nil
}
// Close implements [io.Closer].
func (*SliceFile) Close() error { return nil }
// Sync implements [vfs.File].
func (*SliceFile) Sync(flags vfs.SyncFlag) error { return nil }
// Lock implements [vfs.File].
func (*SliceFile) Lock(lock vfs.LockLevel) error {
// notest // not concurrency safe
return sqlite3.IOERR_LOCK
}
// Unlock implements [vfs.File].
func (*SliceFile) Unlock(lock vfs.LockLevel) error {
// notest // not concurrency safe
return sqlite3.IOERR_UNLOCK
}
// CheckReservedLock implements [vfs.File].
func (*SliceFile) CheckReservedLock() (bool, error) {
// notest // not concurrency safe
return false, sqlite3.IOERR_CHECKRESERVEDLOCK
}
// SectorSize implements [vfs.File].
func (*SliceFile) SectorSize() int {
// notest // safe default
return 0
}
// DeviceCharacteristics implements [vfs.File].
func (*SliceFile) DeviceCharacteristics() vfs.DeviceCharacteristic {
return vfs.IOCAP_ATOMIC |
vfs.IOCAP_SEQUENTIAL |
vfs.IOCAP_SAFE_APPEND |
vfs.IOCAP_POWERSAFE_OVERWRITE |
vfs.IOCAP_SUBPAGE_READ
}

View File

@@ -22,6 +22,14 @@ func UnwrapFile[T vfs.File](f vfs.File) (_ T, _ bool) {
}
}
// WrapOpen helps wrap [vfs.VFS].
func WrapOpen(f vfs.VFS, name string, flags vfs.OpenFlag) (file vfs.File, _ vfs.OpenFlag, err error) {
if f, ok := f.(vfs.VFSFilename); name == "" && ok {
return f.OpenFilename(nil, flags)
}
return f.Open(name, flags)
}
// WrapOpenFilename helps wrap [vfs.VFSFilename].
func WrapOpenFilename(f vfs.VFS, name *vfs.Filename, flags vfs.OpenFlag) (file vfs.File, _ vfs.OpenFlag, err error) {
if f, ok := f.(vfs.VFSFilename); ok {

330
util/vfsutil/wrap_test.go Normal file
View File

@@ -0,0 +1,330 @@
// Package vfsutil implements virtual filesystem utilities.
package vfsutil
import (
"testing"
"github.com/ncruces/go-sqlite3/vfs"
)
func TestWrapOpen(t *testing.T) {
called := 0
WrapOpen(mockVFS{open: func(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
called++
return nil, flags, nil
}}, "", 0)
if called != 1 {
t.Error("open not called")
}
WrapOpenFilename(mockVFS{open: func(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
called++
return nil, flags, nil
}}, nil, 0)
if called != 2 {
t.Error("open not called")
}
}
func TestWrapOpenFilename(t *testing.T) {
called := 0
WrapOpen(mockVFSFilename{openFilename: func(name *vfs.Filename, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
called++
return nil, flags, nil
}}, "", 0)
if called != 1 {
t.Error("openFilename not called")
}
WrapOpenFilename(mockVFSFilename{openFilename: func(name *vfs.Filename, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
called++
return nil, flags, nil
}}, nil, 0)
if called != 2 {
t.Error("openFilename not called")
}
}
func TestWrapLockState(t *testing.T) {
called := 0
WrapLockState(mockFile{lockState: func() vfs.LockLevel {
called++
return 0
}})
if called != 1 {
t.Error("lockState not called")
}
}
func TestWrapPersistWAL(t *testing.T) {
persist := false
WrapSetPersistWAL(mockFile{setPersistWAL: func(b bool) { persist = b }}, true)
if !persist {
t.Error("setPersistWAL not called")
}
called := 0
WrapPersistWAL(mockFile{persistWAL: func() bool { called++; return persist }})
if !persist {
t.Error("persistWAL not called")
}
if called != 1 {
}
}
func TestWrapPowersafeOverwrite(t *testing.T) {
persist := false
WrapSetPowersafeOverwrite(mockFile{setPowersafeOverwrite: func(b bool) { persist = b }}, true)
if !persist {
t.Error("setPowersafeOverwrite not called")
}
called := 0
WrapPowersafeOverwrite(mockFile{powersafeOverwrite: func() bool { called++; return persist }})
if !persist {
t.Error("powersafeOverwrite not called")
}
if called != 1 {
}
}
func TestWrapChunkSize(t *testing.T) {
var chunk int
WrapChunkSize(mockFile{chunkSize: func(size int) {
chunk = size
}}, 5)
if chunk != 5 {
t.Error("chunkSize not called")
}
}
func TestWrapSizeHint(t *testing.T) {
var hint int64
WrapSizeHint(mockFile{sizeHint: func(size int64) error {
hint = size
return nil
}}, 5)
if hint != 5 {
t.Error("sizeHint not called")
}
}
func TestWrapHasMoved(t *testing.T) {
called := 0
WrapHasMoved(mockFile{hasMoved: func() (bool, error) {
called++
return false, nil
}})
if called != 1 {
t.Error("hasMoved not called")
}
}
func TestWrapOverwrite(t *testing.T) {
called := 0
WrapOverwrite(mockFile{overwrite: func() error {
called++
return nil
}})
if called != 1 {
t.Error("overwrite not called")
}
}
func TestWrapSyncSuper(t *testing.T) {
called := 0
WrapSyncSuper(mockFile{syncSuper: func(super string) error {
called++
return nil
}}, "")
if called != 1 {
t.Error("syncSuper not called")
}
}
func TestWrapCommitPhaseTwo(t *testing.T) {
called := 0
WrapCommitPhaseTwo(mockFile{commitPhaseTwo: func() error {
called++
return nil
}})
if called != 1 {
t.Error("commitPhaseTwo not called")
}
}
func TestWrapBatchAtomicWrite(t *testing.T) {
calledBegin := 0
calledCommit := 0
calledRollback := 0
f := mockFile{
begin: func() error { calledBegin++; return nil },
commit: func() error { calledCommit++; return nil },
rollback: func() error { calledRollback++; return nil },
}
WrapBeginAtomicWrite(f)
WrapCommitAtomicWrite(f)
WrapRollbackAtomicWrite(f)
if calledBegin != 1 {
t.Error("beginAtomicWrite not called")
}
if calledCommit != 1 {
t.Error("commitAtomicWrite not called")
}
if calledRollback != 1 {
t.Error("rollbackAtomicWrite not called")
}
}
func TestWrapCheckpoint(t *testing.T) {
calledStart := 0
calledDone := 0
f := mockFile{
ckptStart: func() { calledStart++ },
ckptDone: func() { calledDone++ },
}
WrapCheckpointStart(f)
WrapCheckpointDone(f)
if calledStart != 1 {
t.Error("checkpointStart not called")
}
if calledDone != 1 {
t.Error("checkpointDone not called")
}
}
func TestWrapPragma(t *testing.T) {
called := 0
val, err := WrapPragma(mockFile{
pragma: func(name, value string) (string, error) {
called++
if name != "foo" || value != "bar" {
t.Error("wrong pragma arguments")
}
return "baz", nil
},
}, "foo", "bar")
if called != 1 {
t.Error("pragma not called")
}
if err != nil {
t.Error(err)
}
if val != "baz" {
t.Error("unexpected pragma return value")
}
}
func TestWrapBusyHandler(t *testing.T) {
called := 0
WrapBusyHandler(mockFile{
busyHandler: func(handler func() bool) {
handler()
called++
},
}, func() bool { return true })
if called != 1 {
t.Error("busyHandler not called")
}
}
type mockVFS struct {
open func(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error)
}
func (m mockVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
return m.open(name, flags)
}
func (m mockVFS) Delete(name string, syncDir bool) error { panic("unimplemented") }
func (m mockVFS) FullPathname(name string) (string, error) { panic("unimplemented") }
func (m mockVFS) Access(name string, flags vfs.AccessFlag) (bool, error) { panic("unimplemented") }
type mockVFSFilename struct {
mockVFS
openFilename func(name *vfs.Filename, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error)
}
func (m mockVFSFilename) OpenFilename(name *vfs.Filename, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
return m.openFilename(name, flags)
}
type mockFile struct {
lockState func() vfs.LockLevel
persistWAL func() bool
setPersistWAL func(bool)
powersafeOverwrite func() bool
setPowersafeOverwrite func(bool)
chunkSize func(int)
sizeHint func(int64) error
hasMoved func() (bool, error)
overwrite func() error
syncSuper func(super string) error
commitPhaseTwo func() error
begin func() error
commit func() error
rollback func() error
ckptStart func()
ckptDone func()
busyHandler func(func() bool)
pragma func(name, value string) (string, error)
}
func (m mockFile) LockState() vfs.LockLevel { return m.lockState() }
func (m mockFile) PersistWAL() bool { return m.persistWAL() }
func (m mockFile) SetPersistWAL(v bool) { m.setPersistWAL(v) }
func (m mockFile) PowersafeOverwrite() bool { return m.powersafeOverwrite() }
func (m mockFile) SetPowersafeOverwrite(v bool) { m.setPowersafeOverwrite(v) }
func (m mockFile) ChunkSize(s int) { m.chunkSize(s) }
func (m mockFile) SizeHint(s int64) error { return m.sizeHint(s) }
func (m mockFile) HasMoved() (bool, error) { return m.hasMoved() }
func (m mockFile) Overwrite() error { return m.overwrite() }
func (m mockFile) SyncSuper(s string) error { return m.syncSuper(s) }
func (m mockFile) CommitPhaseTwo() error { return m.commitPhaseTwo() }
func (m mockFile) BeginAtomicWrite() error { return m.begin() }
func (m mockFile) CommitAtomicWrite() error { return m.commit() }
func (m mockFile) RollbackAtomicWrite() error { return m.rollback() }
func (m mockFile) CheckpointStart() { m.ckptStart() }
func (m mockFile) CheckpointDone() { m.ckptDone() }
func (m mockFile) BusyHandler(f func() bool) { m.busyHandler(f) }
func (m mockFile) Pragma(n, v string) (string, error) { return m.pragma(n, v) }
func (m mockFile) Close() error { panic("unimplemented") }
func (m mockFile) ReadAt(p []byte, off int64) (n int, err error) { panic("unimplemented") }
func (m mockFile) WriteAt(p []byte, off int64) (n int, err error) { panic("unimplemented") }
func (m mockFile) Truncate(size int64) error { panic("unimplemented") }
func (m mockFile) Sync(flags vfs.SyncFlag) error { panic("unimplemented") }
func (m mockFile) Size() (int64, error) { panic("unimplemented") }
func (m mockFile) Lock(lock vfs.LockLevel) error { panic("unimplemented") }
func (m mockFile) Unlock(lock vfs.LockLevel) error { panic("unimplemented") }
func (m mockFile) CheckReservedLock() (bool, error) { panic("unimplemented") }
func (m mockFile) SectorSize() int { panic("unimplemented") }
func (m mockFile) DeviceCharacteristics() vfs.DeviceCharacteristic { panic("unimplemented") }

View File

@@ -31,9 +31,9 @@ func (v Value) Dup() *Value {
// Close frees an SQL value previously obtained by [Value.Dup].
//
// https://sqlite.org/c3ref/value_dup.html
func (dup *Value) Close() error {
dup.c.call("sqlite3_value_free", stk_t(dup.handle))
dup.handle = 0
func (v *Value) Close() error {
v.c.call("sqlite3_value_free", stk_t(v.handle))
v.handle = 0
return nil
}

View File

@@ -20,7 +20,10 @@ type hbshVFS struct {
func (h *hbshVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
// notest // OpenFilename is called instead
return nil, 0, sqlite3.CANTOPEN
if name == "" {
return h.OpenFilename(nil, flags)
}
return nil, flags, sqlite3.CANTOPEN
}
func (h *hbshVFS) OpenFilename(name *vfs.Filename, flags vfs.OpenFlag) (file vfs.File, _ vfs.OpenFlag, err error) {
@@ -160,10 +163,11 @@ func (h *hbshFile) WriteAt(p []byte, off int64) (n int, err error) {
if off > min || len(p[n:]) < blockSize {
// Partial block write: read-update-write.
m, err := h.File.ReadAt(h.block[:], min)
if m != blockSize {
if err != io.EOF {
return n, err
}
if m == blockSize {
data = h.hbsh.Decrypt(h.block[:], h.tweak[:])
} else if err != io.EOF {
return n, err
} else {
// Writing past the EOF.
// We're either appending an entirely new block,
// or the final block was only partially written.
@@ -171,8 +175,6 @@ func (h *hbshFile) WriteAt(p []byte, off int64) (n int, err error) {
// and is as good as corrupt.
// Either way, zero pad the file to the next block size.
clear(data)
} else {
data = h.hbsh.Decrypt(h.block[:], h.tweak[:])
}
if off > min {
data = data[off-min:]
@@ -223,16 +225,16 @@ func (h *hbshFile) SizeHint(size int64) error {
return vfsutil.WrapSizeHint(h.File, roundUp(size))
}
// Wrap optional methods.
func (h *hbshFile) Unwrap() vfs.File {
return h.File
return h.File // notest
}
func (h *hbshFile) SharedMemory() vfs.SharedMemory {
return vfsutil.WrapSharedMemory(h.File)
return vfsutil.WrapSharedMemory(h.File) // notest
}
// Wrap optional methods.
func (h *hbshFile) LockState() vfs.LockLevel {
return vfsutil.WrapLockState(h.File) // notest
}

View File

@@ -36,9 +36,9 @@ type VFSFilename interface {
//
// https://sqlite.org/c3ref/io_methods.html
type File interface {
Close() error
ReadAt(p []byte, off int64) (n int, err error)
WriteAt(p []byte, off int64) (n int, err error)
io.Closer
io.ReaderAt
io.WriterAt
Truncate(size int64) error
Sync(flags SyncFlag) error
Size() (int64, error)

View File

@@ -5,7 +5,6 @@ import (
"context"
_ "embed"
"encoding/binary"
"strconv"
"github.com/tetratelabs/wazero/api"
@@ -13,48 +12,30 @@ import (
"github.com/ncruces/go-sqlite3/util/sql3util"
)
func cksmWrapFile(name *Filename, flags OpenFlag, file File) File {
// Checksum only main databases and WALs.
if flags&(OPEN_MAIN_DB|OPEN_WAL) == 0 {
func cksmWrapFile(file File, flags OpenFlag) File {
// Checksum only main databases.
if flags&OPEN_MAIN_DB == 0 {
return file
}
cksm := cksmFile{File: file}
if flags&OPEN_WAL != 0 {
main, _ := name.DatabaseFile().(cksmFile)
cksm.cksmFlags = main.cksmFlags
} else {
cksm.cksmFlags = new(cksmFlags)
cksm.isDB = true
}
return cksm
return &cksmFile{File: file}
}
type cksmFile struct {
File
*cksmFlags
isDB bool
}
type cksmFlags struct {
computeCksm bool
verifyCksm bool
inCkpt bool
pageSize int
computeCksm bool
}
func (c cksmFile) ReadAt(p []byte, off int64) (n int, err error) {
func (c *cksmFile) ReadAt(p []byte, off int64) (n int, err error) {
n, err = c.File.ReadAt(p, off)
p = p[:n]
if isHeader(c.isDB, p, off) {
if isHeader(p, off) {
c.init((*[100]byte)(p))
}
// Verify checksums.
if c.verifyCksm && !c.inCkpt && len(p) == c.pageSize {
if c.verifyCksm && sql3util.ValidPageSize(len(p)) {
cksm1 := cksmCompute(p[:len(p)-8])
cksm2 := *(*[8]byte)(p[len(p)-8:])
if cksm1 != cksm2 {
@@ -64,20 +45,20 @@ func (c cksmFile) ReadAt(p []byte, off int64) (n int, err error) {
return n, err
}
func (c cksmFile) WriteAt(p []byte, off int64) (n int, err error) {
if isHeader(c.isDB, p, off) {
func (c *cksmFile) WriteAt(p []byte, off int64) (n int, err error) {
if isHeader(p, off) {
c.init((*[100]byte)(p))
}
// Compute checksums.
if c.computeCksm && !c.inCkpt && len(p) == c.pageSize {
if c.computeCksm && sql3util.ValidPageSize(len(p)) {
*(*[8]byte)(p[len(p)-8:]) = cksmCompute(p[:len(p)-8])
}
return c.File.WriteAt(p, off)
}
func (c cksmFile) Pragma(name string, value string) (string, error) {
func (c *cksmFile) Pragma(name string, value string) (string, error) {
switch name {
case "checksum_verification":
b, ok := sql3util.ParseBool(value)
@@ -90,15 +71,15 @@ func (c cksmFile) Pragma(name string, value string) (string, error) {
return "1", nil
case "page_size":
if c.computeCksm {
if c.computeCksm && value != "" {
// Do not allow page size changes on a checksum database.
return strconv.Itoa(c.pageSize), nil
return "", nil
}
}
return "", _NOTFOUND
}
func (c cksmFile) DeviceCharacteristics() DeviceCharacteristic {
func (c *cksmFile) DeviceCharacteristics() DeviceCharacteristic {
ret := c.File.DeviceCharacteristics()
if c.verifyCksm {
ret &^= IOCAP_SUBPAGE_READ
@@ -106,13 +87,8 @@ func (c cksmFile) DeviceCharacteristics() DeviceCharacteristic {
return ret
}
func (c cksmFile) fileControl(ctx context.Context, mod api.Module, op _FcntlOpcode, pArg ptr_t) _ErrorCode {
switch op {
case _FCNTL_CKPT_START:
c.inCkpt = true
case _FCNTL_CKPT_DONE:
c.inCkpt = false
case _FCNTL_PRAGMA:
func (c *cksmFile) fileControl(ctx context.Context, mod api.Module, op _FcntlOpcode, pArg ptr_t) _ErrorCode {
if op == _FCNTL_PRAGMA {
rc := vfsFileControlImpl(ctx, mod, c, op, pArg)
if rc != _NOTFOUND {
return rc
@@ -121,24 +97,26 @@ func (c cksmFile) fileControl(ctx context.Context, mod api.Module, op _FcntlOpco
return vfsFileControlImpl(ctx, mod, c.File, op, pArg)
}
func (f *cksmFlags) init(header *[100]byte) {
f.pageSize = 256 * int(binary.LittleEndian.Uint16(header[16:18]))
if r := header[20] == 8; r != f.computeCksm {
f.computeCksm = r
f.verifyCksm = r
}
if !sql3util.ValidPageSize(f.pageSize) {
f.computeCksm = false
f.verifyCksm = false
func (c *cksmFile) init(header *[100]byte) {
if r := header[20] == 8; r != c.computeCksm {
c.computeCksm = r
c.verifyCksm = r
}
}
func isHeader(isDB bool, p []byte, off int64) bool {
check := sql3util.ValidPageSize(len(p))
if isDB {
check = off == 0 && len(p) >= 100
func (c *cksmFile) SharedMemory() SharedMemory {
if f, ok := c.File.(FileSharedMemory); ok {
return f.SharedMemory()
}
return check && bytes.HasPrefix(p, []byte("SQLite format 3\000"))
return nil
}
func (c *cksmFile) Unwrap() File {
return c.File
}
func isHeader(p []byte, off int64) bool {
return off == 0 && len(p) >= 100 && bytes.HasPrefix(p, []byte("SQLite format 3\000"))
}
func cksmCompute(a []byte) (cksm [8]byte) {
@@ -155,14 +133,3 @@ func cksmCompute(a []byte) (cksm [8]byte) {
binary.LittleEndian.PutUint32(cksm[4:8], s2)
return
}
func (c cksmFile) SharedMemory() SharedMemory {
if f, ok := c.File.(FileSharedMemory); ok {
return f.SharedMemory()
}
return nil
}
func (c cksmFile) Unwrap() File {
return c.File
}

View File

@@ -75,6 +75,9 @@ func (vfsOS) Access(name string, flags AccessFlag) (bool, error) {
func (vfsOS) Open(name string, flags OpenFlag) (File, OpenFlag, error) {
// notest // OpenFilename is called instead
if name == "" {
return vfsOS{}.OpenFilename(nil, flags)
}
return nil, 0, _CANTOPEN
}

View File

@@ -56,7 +56,7 @@ func (n *Filename) Journal() string {
return n.path("sqlite3_filename_journal")
}
// Journal returns the name of the corresponding WAL file.
// WAL returns the name of the corresponding WAL file.
//
// https://sqlite.org/c3ref/filename_database.html
func (n *Filename) WAL() string {

View File

@@ -2,40 +2,39 @@ package memdb
import (
"io"
"strings"
"sync"
"time"
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/util/vfsutil"
"github.com/ncruces/go-sqlite3/vfs"
)
const sectorSize = 65536
// Ensure sectorSize is a multiple of 64K (the largest page size).
var _ [0]struct{} = [sectorSize & 65535]struct{}{}
type memVFS struct{}
func (memVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
// For simplicity, we do not support reading or writing data
// across "sector" boundaries.
//
// This is not a problem for most SQLite file types:
// - databases, which only do page aligned reads/writes;
// - temp journals, as used by the sorter, which does the same:
// https://github.com/sqlite/sqlite/blob/b74eb0/src/vdbesort.c#L409-L412
//
// We refuse to open all other file types,
// but returning OPEN_MEMORY means SQLite won't ask us to.
const types = vfs.OPEN_MAIN_DB | vfs.OPEN_TEMP_DB |
vfs.OPEN_TRANSIENT_DB | vfs.OPEN_TEMP_JOURNAL
if flags&types == 0 {
// This is not a problem for SQLite database files.
const databases = vfs.OPEN_MAIN_DB | vfs.OPEN_TEMP_DB | vfs.OPEN_TRANSIENT_DB
// Temp journals, as used by the sorter, use SliceFile.
if flags&vfs.OPEN_TEMP_JOURNAL != 0 {
return &vfsutil.SliceFile{}, flags | vfs.OPEN_MEMORY, nil
}
// Refuse to open all other file types.
// Returning OPEN_MEMORY means SQLite won't ask us to.
if flags&databases == 0 {
// notest // OPEN_MEMORY
return nil, flags, sqlite3.CANTOPEN
}
// A shared database has a name that begins with "/".
shared := len(name) > 1 && name[0] == '/'
shared := strings.HasPrefix(name, "/")
var db *memDB
if shared {
@@ -76,18 +75,16 @@ func (memVFS) FullPathname(name string) (string, error) {
type memDB struct {
name string
// +checklocks:lockMtx
waiter *sync.Cond
// +checklocks:dataMtx
data []*[sectorSize]byte
// +checklocks:dataMtx
size int64
// +checklocks:memoryMtx
refs int32
shared int32 // +checklocks:lockMtx
pending bool // +checklocks:lockMtx
reserved bool // +checklocks:lockMtx
waiter *sync.Cond // +checklocks:lockMtx
size int64 // +checklocks:dataMtx
refs int32 // +checklocks:memoryMtx
shared int32 // +checklocks:lockMtx
pending bool // +checklocks:lockMtx
reserved bool // +checklocks:lockMtx
lockMtx sync.Mutex
dataMtx sync.RWMutex
@@ -129,7 +126,7 @@ func (m *memFile) ReadAt(b []byte, off int64) (n int, err error) {
base := off / sectorSize
rest := off % sectorSize
have := int64(sectorSize)
if base == int64(len(m.data))-1 {
if m.size < off+int64(len(b)) {
have = modRoundUp(m.size, sectorSize)
}
n = copy(b, (*m.data[base])[rest:have])
@@ -150,22 +147,37 @@ func (m *memFile) WriteAt(b []byte, off int64) (n int, err error) {
m.data = append(m.data, new([sectorSize]byte))
}
n = copy((*m.data[base])[rest:], b)
if size := off + int64(n); size > m.size {
m.size = size
}
if n < len(b) {
// notest // assume writes are page aligned
return n, io.ErrShortWrite
}
if size := off + int64(len(b)); size > m.size {
m.size = size
}
return n, nil
}
func (m *memFile) Size() (int64, error) {
m.dataMtx.RLock()
defer m.dataMtx.RUnlock()
return m.size, nil
}
func (m *memFile) Truncate(size int64) error {
m.dataMtx.Lock()
defer m.dataMtx.Unlock()
return m.truncate(size)
}
func (m *memFile) SizeHint(size int64) error {
m.dataMtx.Lock()
defer m.dataMtx.Unlock()
if size > m.size {
return m.truncate(size)
}
return nil
}
// +checklocks:m.dataMtx
func (m *memFile) truncate(size int64) error {
if size < m.size {
@@ -185,16 +197,6 @@ func (m *memFile) truncate(size int64) error {
return nil
}
func (m *memFile) Sync(flag vfs.SyncFlag) error {
return nil
}
func (m *memFile) Size() (int64, error) {
m.dataMtx.RLock()
defer m.dataMtx.RUnlock()
return m.size, nil
}
func (m *memFile) Lock(lock vfs.LockLevel) error {
if m.lock >= lock {
return nil
@@ -278,31 +280,24 @@ func (m *memFile) CheckReservedLock() (bool, error) {
return m.reserved, nil
}
func (m *memFile) SectorSize() int {
func (m *memFile) LockState() vfs.LockLevel {
return m.lock
}
func (*memFile) Sync(flag vfs.SyncFlag) error { return nil }
func (*memFile) SectorSize() int {
// notest // IOCAP_POWERSAFE_OVERWRITE
return sectorSize
}
func (m *memFile) DeviceCharacteristics() vfs.DeviceCharacteristic {
func (*memFile) DeviceCharacteristics() vfs.DeviceCharacteristic {
return vfs.IOCAP_ATOMIC |
vfs.IOCAP_SEQUENTIAL |
vfs.IOCAP_SAFE_APPEND |
vfs.IOCAP_POWERSAFE_OVERWRITE
}
func (m *memFile) SizeHint(size int64) error {
m.dataMtx.Lock()
defer m.dataMtx.Unlock()
if size > m.size {
return m.truncate(size)
}
return nil
}
func (m *memFile) LockState() vfs.LockLevel {
return m.lock
}
func divRoundUp(a, b int64) int64 {
return (a + b - 1) / b
}

9
vfs/mvcc/README.md Normal file
View File

@@ -0,0 +1,9 @@
# Go `mvcc` SQLite VFS
This package implements the **EXPERIMENTAL** `"mvcc"` in-memory SQLite VFS.
It has some benefits over the [`"memdb"`](../memdb/README.md) VFS:
- panics do not corrupt a shared database;
- single-writer not blocked by readers,
- readers never block,
- instant snapshots.

67
vfs/mvcc/api.go Normal file
View File

@@ -0,0 +1,67 @@
// Package mvcc implements the "mvcc" SQLite VFS.
//
// The "mvcc" [vfs.VFS] allows the same in-memory database to be shared
// among multiple database connections in the same process,
// as long as the database name begins with "/".
//
// Importing package mvcc registers the VFS:
//
// import _ "github.com/ncruces/go-sqlite3/vfs/mvcc"
package mvcc
import (
"sync"
"github.com/ncruces/go-sqlite3/vfs"
)
func init() {
vfs.Register("mvcc", mvccVFS{})
}
var (
memoryMtx sync.Mutex
// +checklocks:memoryMtx
memoryDBs = map[string]*mvccDB{}
)
// Create creates a shared memory database,
// using data as its initial contents.
func Create(name string, data string) {
memoryMtx.Lock()
defer memoryMtx.Unlock()
db := &mvccDB{
refs: 1,
name: name,
}
memoryDBs[name] = db
if len(data) == 0 {
return
}
// Convert data from WAL/2 to rollback journal.
if len(data) >= 20 && (false ||
data[18] == 2 && data[19] == 2 ||
data[18] == 3 && data[19] == 3) {
db.data = db.data.
Put(0, data[:18]).
Put(18, "\001\001").
Put(20, data[20:])
} else {
db.data = db.data.Put(0, data)
}
}
// Delete deletes a shared memory database.
func Delete(name string) {
memoryMtx.Lock()
defer memoryMtx.Unlock()
delete(memoryDBs, name)
}
// Snapshot stores a snapshot of database src into dst.
func Snapshot(dst, src string) {
memoryMtx.Lock()
defer memoryMtx.Unlock()
memoryDBs[dst] = memoryDBs[src].fork()
}

50
vfs/mvcc/example_test.go Normal file
View File

@@ -0,0 +1,50 @@
package mvcc_test
import (
"database/sql"
_ "embed"
"fmt"
"log"
_ "github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/vfs/mvcc"
)
//go:embed testdata/test.db
var testDB string
func Example() {
mvcc.Create("test.db", testDB)
db, err := sql.Open("sqlite3", "file:/test.db?vfs=mvcc")
if err != nil {
log.Fatal(err)
}
defer db.Close()
_, err = db.Exec(`INSERT INTO users (id, name) VALUES (3, 'rust')`)
if err != nil {
log.Fatal(err)
}
rows, err := db.Query(`SELECT id, name FROM users`)
if err != nil {
log.Fatal(err)
}
defer rows.Close()
for rows.Next() {
var id, name string
err = rows.Scan(&id, &name)
if err != nil {
log.Fatal(err)
}
fmt.Printf("%s %s\n", id, name)
}
// Output:
// 0 go
// 1 zig
// 2 whatever
// 3 rust
}

333
vfs/mvcc/mvcc.go Normal file
View File

@@ -0,0 +1,333 @@
package mvcc
import (
"io"
"strings"
"sync"
"time"
"github.com/ncruces/aa"
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/util/vfsutil"
"github.com/ncruces/go-sqlite3/vfs"
)
type mvccVFS struct{}
func (mvccVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
// Temporary files use SliceFile.
if name == "" || flags&vfs.OPEN_DELETEONCLOSE != 0 {
return &vfsutil.SliceFile{}, flags | vfs.OPEN_MEMORY, nil
}
// Only main databases benefit from multiversion concurrency control.
// Refuse to open all other file types.
// Returning OPEN_MEMORY means SQLite won't ask us to.
if flags&vfs.OPEN_MAIN_DB == 0 {
// notest // OPEN_MEMORY
return nil, flags, sqlite3.CANTOPEN
}
// A shared database has a name that begins with "/".
shared := strings.HasPrefix(name, "/")
var db *mvccDB
if shared {
name = name[1:]
memoryMtx.Lock()
defer memoryMtx.Unlock()
db = memoryDBs[name]
}
if db == nil {
if flags&vfs.OPEN_CREATE == 0 {
return nil, flags, sqlite3.CANTOPEN
}
db = &mvccDB{name: name}
}
if shared {
db.refs++ // +checklocksforce: memoryMtx is held
memoryDBs[name] = db
}
return &mvccFile{
mvccDB: db,
readOnly: flags&vfs.OPEN_READONLY != 0,
}, flags | vfs.OPEN_MEMORY, nil
}
func (mvccVFS) Delete(name string, dirSync bool) error {
return sqlite3.IOERR_DELETE_NOENT // used to delete journals
}
func (mvccVFS) Access(name string, flag vfs.AccessFlag) (bool, error) {
return false, nil // used to check for journals
}
func (mvccVFS) FullPathname(name string) (string, error) {
return name, nil
}
type mvccDB struct {
data *aa.Tree[int64, string] // +checklocks:mtx
owner *mvccFile // +checklocks:mtx
waiter *sync.Cond // +checklocks:mtx
name string
refs int // +checklocks:memoryMtx
mtx sync.Mutex
}
func (m *mvccDB) release() {
memoryMtx.Lock()
defer memoryMtx.Unlock()
if m.refs--; m.refs == 0 && m == memoryDBs[m.name] {
delete(memoryDBs, m.name)
}
}
func (m *mvccDB) fork() *mvccDB {
m.mtx.Lock()
defer m.mtx.Unlock()
return &mvccDB{
refs: 1,
name: m.name,
data: m.data,
}
}
type mvccFile struct {
*mvccDB
data *aa.Tree[int64, string]
lock vfs.LockLevel
readOnly bool
}
var (
// Ensure these interfaces are implemented:
_ vfs.FileLockState = &mvccFile{}
_ vfs.FileCommitPhaseTwo = &mvccFile{}
)
func (m *mvccFile) Close() error {
// Relase ownership, discard changes.
m.release()
m.data = nil
m.lock = vfs.LOCK_NONE
m.mtx.Lock()
defer m.mtx.Unlock()
if m.owner == m {
m.owner = nil
}
return nil
}
func (m *mvccFile) ReadAt(b []byte, off int64) (n int, err error) {
// If unlocked, use a snapshot of the database.
data := m.data
if m.lock == vfs.LOCK_NONE {
m.mtx.Lock()
defer m.mtx.Unlock()
data = m.mvccDB.data
}
for k, v := range data.AscendFloor(off) {
if i := k - off; i >= 0 {
if +i > int64(n) {
// Missing data.
clear(b[n:])
}
if +i < int64(len(b)) {
// Copy prefix.
n = copy(b[+i:], v) + int(i)
}
} else {
if -i < int64(len(v)) {
// Copy suffix.
n = copy(b, v[-i:])
}
}
if n >= len(b) {
return n, nil
}
}
return n, io.EOF
}
func (m *mvccFile) WriteAt(b []byte, off int64) (n int, err error) {
// If unlocked, take a snapshot of the database.
data := m.data
if m.lock == vfs.LOCK_NONE {
m.mtx.Lock()
defer m.mtx.Unlock()
data = m.mvccDB.data
m.lock = vfs.LOCK_EXCLUSIVE + 1 // UNKNOWN_LOCK
}
next := off + int64(len(b))
for k, v := range data.AscendFloor(off) {
if k >= next {
break
}
switch {
case k > off:
// Delete overlap.
data = data.Delete(k)
case k < off && off < k+int64(len(v)):
// Reinsert prefix.
data = data.Put(k, v[:off-k])
}
if k+int64(len(v)) > next {
// Reinsert suffix.
data = data.Put(next, v[next-k:])
}
}
m.data = data.Put(off, string(b))
return len(b), nil
}
func (m *mvccFile) Size() (int64, error) {
// If unlocked, use a snapshot of the database.
data := m.data
if m.lock == vfs.LOCK_NONE {
m.mtx.Lock()
defer m.mtx.Unlock()
data = m.mvccDB.data
}
if data == nil {
return 0, nil
}
data = data.Max()
return data.Key() + int64(len(data.Value())), nil
}
func (m *mvccFile) Truncate(size int64) error {
// If unlocked, take a snapshot of the database.
data := m.data
if m.lock == vfs.LOCK_NONE {
m.mtx.Lock()
defer m.mtx.Unlock()
data = m.mvccDB.data
m.lock = vfs.LOCK_EXCLUSIVE + 1 // UNKNOWN_LOCK
}
for data != nil && data.Key() >= size {
data = data.Left()
}
for k := range data.AscendCeil(size) {
data = data.Delete(k)
}
m.data = data
return nil
}
func (m *mvccFile) Lock(lock vfs.LockLevel) error {
if m.lock >= lock {
return nil
}
if m.readOnly && lock >= vfs.LOCK_RESERVED {
return sqlite3.IOERR_LOCK
}
m.mtx.Lock()
defer m.mtx.Unlock()
// Take a snapshot of the database.
if lock == vfs.LOCK_SHARED {
m.data = m.mvccDB.data
m.lock = lock
return nil
}
// We are the owners.
if m.owner == m {
m.lock = lock
return nil
}
// Someone else is the owner.
if m.owner != nil {
before := time.Now()
if m.waiter == nil {
m.waiter = sync.NewCond(&m.mtx)
}
defer time.AfterFunc(time.Millisecond, m.waiter.Broadcast).Stop()
for m.owner != nil {
// Our snapshot is invalid.
if m.data != m.mvccDB.data {
return sqlite3.BUSY_SNAPSHOT
}
if time.Since(before) > time.Millisecond {
return sqlite3.BUSY
}
m.waiter.Wait()
}
}
// Our snapshot is invalid.
if m.data != m.mvccDB.data {
return sqlite3.BUSY_SNAPSHOT
}
// Take ownership.
m.lock = lock
m.owner = m
return nil
}
func (m *mvccFile) Unlock(lock vfs.LockLevel) error {
if m.lock <= lock {
return nil
}
m.mtx.Lock()
defer m.mtx.Unlock()
// Relase ownership, commit changes.
if m.owner == m {
m.owner = nil
m.mvccDB.data = m.data
if m.waiter != nil {
m.waiter.Broadcast()
}
}
m.lock = lock
return nil
}
func (m *mvccFile) CheckReservedLock() (bool, error) {
// notest // OPEN_MEMORY
if m.lock >= vfs.LOCK_RESERVED {
return true, nil
}
m.mtx.Lock()
defer m.mtx.Unlock()
return m.owner != nil, nil
}
func (m *mvccFile) CommitPhaseTwo() error {
// Modified without lock, commit changes.
if m.lock > vfs.LOCK_EXCLUSIVE {
m.mtx.Lock()
defer m.mtx.Unlock()
m.mvccDB.data = m.data
}
return nil
}
func (m *mvccFile) LockState() vfs.LockLevel {
return m.lock
}
func (*mvccFile) Sync(flag vfs.SyncFlag) error { return nil }
func (*mvccFile) SectorSize() int {
// notest // safe default
return 0
}
func (*mvccFile) DeviceCharacteristics() vfs.DeviceCharacteristic {
return vfs.IOCAP_ATOMIC |
vfs.IOCAP_SEQUENTIAL |
vfs.IOCAP_SAFE_APPEND |
vfs.IOCAP_POWERSAFE_OVERWRITE |
vfs.IOCAP_SUBPAGE_READ
}

30
vfs/mvcc/mvcc_test.go Normal file
View File

@@ -0,0 +1,30 @@
package mvcc
import (
_ "embed"
"testing"
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
_ "github.com/ncruces/go-sqlite3/internal/testcfg"
)
//go:embed testdata/wal.db
var walDB string
func Test_wal(t *testing.T) {
t.Parallel()
Create("test.db", walDB)
db, err := sqlite3.Open("file:/test.db?vfs=mvcc")
if err != nil {
t.Fatal(err)
}
defer db.Close()
err = db.Exec(`CREATE TABLE users (id INT, name VARCHAR(10))`)
if err != nil {
t.Fatal(err)
}
}

BIN
vfs/mvcc/testdata/test.db vendored Normal file

Binary file not shown.

BIN
vfs/mvcc/testdata/wal.db vendored Normal file

Binary file not shown.

View File

@@ -3,14 +3,19 @@ package readervfs
import (
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/util/ioutil"
"github.com/ncruces/go-sqlite3/util/vfsutil"
"github.com/ncruces/go-sqlite3/vfs"
)
type readerVFS struct{}
func (readerVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
// Temp journals, as used by the sorter, use SliceFile.
if flags&vfs.OPEN_TEMP_JOURNAL != 0 {
return &vfsutil.SliceFile{}, flags | vfs.OPEN_MEMORY, nil
}
// Refuse to open all other file types.
if flags&vfs.OPEN_MAIN_DB == 0 {
// notest
return nil, flags, sqlite3.CANTOPEN
}
readerMtx.RLock()
@@ -22,13 +27,13 @@ func (readerVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag,
}
func (readerVFS) Delete(name string, dirSync bool) error {
// notest
// notest // IOCAP_IMMUTABLE
return sqlite3.IOERR_DELETE
}
func (readerVFS) Access(name string, flag vfs.AccessFlag) (bool, error) {
// notest
return false, nil
// notest // IOCAP_IMMUTABLE
return false, sqlite3.IOERR_ACCESS
}
func (readerVFS) FullPathname(name string) (string, error) {
@@ -42,37 +47,37 @@ func (readerFile) Close() error {
}
func (readerFile) WriteAt(b []byte, off int64) (n int, err error) {
// notest
return 0, sqlite3.READONLY
// notest // IOCAP_IMMUTABLE
return 0, sqlite3.IOERR_WRITE
}
func (readerFile) Truncate(size int64) error {
// notest
return sqlite3.READONLY
// notest // IOCAP_IMMUTABLE
return sqlite3.IOERR_TRUNCATE
}
func (readerFile) Sync(flag vfs.SyncFlag) error {
// notest
return nil
// notest // IOCAP_IMMUTABLE
return sqlite3.IOERR_FSYNC
}
func (readerFile) Lock(lock vfs.LockLevel) error {
// notest
return nil
// notest // IOCAP_IMMUTABLE
return sqlite3.IOERR_LOCK
}
func (readerFile) Unlock(lock vfs.LockLevel) error {
// notest
return nil
// notest // IOCAP_IMMUTABLE
return sqlite3.IOERR_UNLOCK
}
func (readerFile) CheckReservedLock() (bool, error) {
// notest
return false, nil
// notest // IOCAP_IMMUTABLE
return false, sqlite3.IOERR_CHECKRESERVEDLOCK
}
func (readerFile) SectorSize() int {
// notest
// notest // IOCAP_IMMUTABLE
return 0
}

View File

@@ -24,6 +24,7 @@ import (
"github.com/ncruces/go-sqlite3/vfs"
_ "github.com/ncruces/go-sqlite3/vfs/adiantum"
"github.com/ncruces/go-sqlite3/vfs/memdb"
"github.com/ncruces/go-sqlite3/vfs/mvcc"
_ "github.com/ncruces/go-sqlite3/vfs/xts"
)
@@ -195,6 +196,50 @@ func Test_multiwrite01_memory(t *testing.T) {
mod.Close(ctx)
}
func Test_config01_mvcc(t *testing.T) {
mvcc.Create("test.db", "")
ctx := util.NewContext(newContext(t))
cfg := config(ctx).WithArgs("mptest", "/test.db", "config01.test",
"--vfs", "mvcc")
mod, err := rt.InstantiateModule(ctx, module, cfg)
if err != nil {
t.Fatal(err)
}
mod.Close(ctx)
}
func Test_crash01_mvcc(t *testing.T) {
if testing.Short() {
t.Skip("skipping in short mode")
}
mvcc.Create("test.db", "")
ctx := util.NewContext(newContext(t))
cfg := config(ctx).WithArgs("mptest", "/test.db", "crash01.test",
"--vfs", "mvcc")
mod, err := rt.InstantiateModule(ctx, module, cfg)
if err != nil {
t.Fatal(err)
}
mod.Close(ctx)
}
func Test_multiwrite01_mvcc(t *testing.T) {
if testing.Short() && os.Getenv("CI") != "" {
t.Skip("skipping in slow CI")
}
mvcc.Create("test.db", "")
ctx := util.NewContext(newContext(t))
cfg := config(ctx).WithArgs("mptest", "/test.db", "multiwrite01.test",
"--vfs", "mvcc")
mod, err := rt.InstantiateModule(ctx, module, cfg)
if err != nil {
t.Fatal(err)
}
mod.Close(ctx)
}
func Test_crash01_wal(t *testing.T) {
if testing.Short() {
t.Skip("skipping in short mode")

View File

@@ -13,7 +13,8 @@ WASI_SDK="$ROOT/tools/wasi-sdk/bin"
-mmutable-globals -mnontrapping-fptoint \
-msimd128 -mbulk-memory -msign-ext \
-mreference-types -mmultivalue \
-fno-stack-protector -fno-stack-clash-protection \
-mno-extended-const \
-fno-stack-protector \
-Wl,--stack-first \
-Wl,--import-undefined \
-D_HAVE_SQLITE_CONFIG_H -DSQLITE_USE_URI \
@@ -25,9 +26,10 @@ WASI_SDK="$ROOT/tools/wasi-sdk/bin"
-D_WASI_EMULATED_GETPID -lwasi-emulated-getpid \
$(awk '{print "-Wl,--export="$0}' exports.txt)
"$BINARYEN/wasm-opt" -g --strip --strip-producers -c -O3 \
mptest.wasm -o mptest.tmp --low-memory-unused \
"$BINARYEN/wasm-opt" -g mptest.wasm -o mptest.tmp \
--low-memory-unused --gufa --generate-global-effects --converge -O3 \
--enable-mutable-globals --enable-nontrapping-float-to-int \
--enable-simd --enable-bulk-memory --enable-sign-ext \
--enable-reference-types --enable-multivalue
--enable-reference-types --enable-multivalue \
--strip --strip-producers
mv mptest.tmp mptest.wasm

Binary file not shown.

View File

@@ -21,6 +21,7 @@ import (
"github.com/ncruces/go-sqlite3/vfs"
_ "github.com/ncruces/go-sqlite3/vfs/adiantum"
_ "github.com/ncruces/go-sqlite3/vfs/memdb"
_ "github.com/ncruces/go-sqlite3/vfs/mvcc"
_ "github.com/ncruces/go-sqlite3/vfs/xts"
)
@@ -35,7 +36,7 @@ func TestMain(m *testing.M) {
initFlags()
ctx := context.Background()
cfg := wazero.NewRuntimeConfig().WithMemoryLimitPages(512)
cfg := wazero.NewRuntimeConfig().WithMemoryLimitPages(2048)
rt = wazero.NewRuntimeWithConfig(ctx, cfg)
wasi_snapshot_preview1.MustInstantiate(ctx, rt)
env := vfs.ExportHostFunctions(rt.NewHostModuleBuilder("env"))

View File

@@ -13,16 +13,18 @@ WASI_SDK="$ROOT/tools/wasi-sdk/bin"
-mmutable-globals -mnontrapping-fptoint \
-msimd128 -mbulk-memory -msign-ext \
-mreference-types -mmultivalue \
-fno-stack-protector -fno-stack-clash-protection \
-mno-extended-const \
-fno-stack-protector \
-Wl,--stack-first \
-Wl,--import-undefined \
-D_HAVE_SQLITE_CONFIG_H -DSQLITE_USE_URI \
-DSQLITE_CUSTOM_INCLUDE=sqlite_opt.h \
$(awk '{print "-Wl,--export="$0}' exports.txt)
"$BINARYEN/wasm-opt" -g --strip --strip-producers -c -O3 \
speedtest1.wasm -o speedtest1.tmp --low-memory-unused \
"$BINARYEN/wasm-opt" -g speedtest1.wasm -o speedtest1.tmp \
--low-memory-unused --gufa --generate-global-effects --converge -O3 \
--enable-mutable-globals --enable-nontrapping-float-to-int \
--enable-simd --enable-bulk-memory --enable-sign-ext \
--enable-reference-types --enable-multivalue
--enable-reference-types --enable-multivalue \
--strip --strip-producers
mv speedtest1.tmp speedtest1.wasm

View File

@@ -148,7 +148,7 @@ func vfsOpen(ctx context.Context, mod api.Module, pVfs, zPath, pFile ptr_t, flag
if pOutFlags != 0 {
util.Write32(mod, pOutFlags, flags)
}
file = cksmWrapFile(name, flags, file)
file = cksmWrapFile(file, flags)
vfsFileRegister(ctx, mod, pFile, file)
return _OK
}

View File

@@ -19,7 +19,10 @@ type xtsVFS struct {
func (x *xtsVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
// notest // OpenFilename is called instead
return nil, 0, sqlite3.CANTOPEN
if name == "" {
return x.OpenFilename(nil, flags)
}
return nil, flags, sqlite3.CANTOPEN
}
func (x *xtsVFS) OpenFilename(name *vfs.Filename, flags vfs.OpenFlag) (file vfs.File, _ vfs.OpenFlag, err error) {
@@ -124,10 +127,10 @@ func (x *xtsFile) ReadAt(p []byte, off int64) (n int, err error) {
return n, err
}
sectorNum := uint64(min / sectorSize)
x.cipher.Decrypt(x.sector[:], x.sector[:], sectorNum)
data := x.sector[:]
sectorNum := uint64(min / sectorSize)
x.cipher.Decrypt(data, x.sector[:], sectorNum)
if off > min {
data = data[off-min:]
}
@@ -150,16 +153,17 @@ func (x *xtsFile) WriteAt(p []byte, off int64) (n int, err error) {
// Write one block at a time.
for ; min < max; min += sectorSize {
sectorNum := uint64(min / sectorSize)
data := x.sector[:]
sectorNum := uint64(min / sectorSize)
if off > min || len(p[n:]) < sectorSize {
// Partial block write: read-update-write.
m, err := x.File.ReadAt(x.sector[:], min)
if m != sectorSize {
if err != io.EOF {
return n, err
}
if m == sectorSize {
x.cipher.Decrypt(data, x.sector[:], sectorNum)
} else if err != io.EOF {
return n, err
} else {
// Writing past the EOF.
// We're either appending an entirely new block,
// or the final block was only partially written.
@@ -167,8 +171,6 @@ func (x *xtsFile) WriteAt(p []byte, off int64) (n int, err error) {
// and is as good as corrupt.
// Either way, zero pad the file to the next block size.
clear(data)
} else {
x.cipher.Decrypt(data, data, sectorNum)
}
if off > min {
data = data[off-min:]
@@ -219,16 +221,16 @@ func (x *xtsFile) SizeHint(size int64) error {
return vfsutil.WrapSizeHint(x.File, roundUp(size))
}
// Wrap optional methods.
func (x *xtsFile) Unwrap() vfs.File {
return x.File
return x.File // notest
}
func (x *xtsFile) SharedMemory() vfs.SharedMemory {
return vfsutil.WrapSharedMemory(x.File)
return vfsutil.WrapSharedMemory(x.File) // notest
}
// Wrap optional methods.
func (x *xtsFile) LockState() vfs.LockLevel {
return vfsutil.WrapLockState(x.File) // notest
}