Compare commits

..

44 Commits

Author SHA1 Message Date
Nuno Cruces
9f456fecb9 Updated dependencies. 2024-12-11 18:38:57 +00:00
Nuno Cruces
36bbd674c2 Add ColumnTypeScanType to driver (#199). 2024-12-11 18:35:50 +00:00
Nuno Cruces
7f5ea54009 Windows blocking locks. (#200) 2024-12-11 15:05:22 +00:00
Nuno Cruces
5f1d5727cd SQLite 3.47.2. 2024-12-10 18:38:49 +00:00
Nuno Cruces
6fb259e2b9 Fix. 2024-12-10 18:25:49 +00:00
Nuno Cruces
301f6bc2bd Fix. 2024-12-09 19:26:47 +00:00
Nuno Cruces
9e112c54b0 Fix, see #197. 2024-12-08 11:47:11 +00:00
Nuno Cruces
270efcb4af Updated dependencies. 2024-12-05 12:46:46 +00:00
dependabot[bot]
8252198dd2 Bump actions/attest-build-provenance from 1 to 2 (#196)
Bumps [actions/attest-build-provenance](https://github.com/actions/attest-build-provenance) from 1 to 2.
- [Release notes](https://github.com/actions/attest-build-provenance/releases)
- [Changelog](https://github.com/actions/attest-build-provenance/blob/main/RELEASE.md)
- [Commits](https://github.com/actions/attest-build-provenance/compare/v1...v2)

---
updated-dependencies:
- dependency-name: actions/attest-build-provenance
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2024-12-05 13:04:47 +00:00
dependabot[bot]
dff825ae81 Bump golang.org/x/crypto from 0.29.0 to 0.30.0 (#193)
Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.29.0 to 0.30.0.
- [Commits](https://github.com/golang/crypto/compare/v0.29.0...v0.30.0)

---
updated-dependencies:
- dependency-name: golang.org/x/crypto
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2024-12-05 03:03:36 +00:00
Nuno Cruces
aae732e530 API fix. 2024-12-04 17:09:22 +00:00
Nuno Cruces
2e3ba3949e Remove sqlite3_nosys. 2024-11-29 15:51:51 +00:00
Nuno Cruces
a44690035f Refactor. 2024-11-28 00:15:55 +00:00
Nuno Cruces
7e12105b22 Split module. 2024-11-27 14:25:01 +00:00
Nuno Cruces
987db177ad Split module. 2024-11-27 14:22:31 +00:00
Nuno Cruces
1469cb9f1a Imports. 2024-11-27 11:42:25 +00:00
Nuno Cruces
4ede2c7216 Updated dependencies. 2024-11-26 14:18:53 +00:00
Nuno Cruces
cf14f190b2 SQLite 3.47.1. 2024-11-26 11:40:11 +00:00
dependabot[bot]
6ca92b035d Bump github.com/tetratelabs/wazero (#189)
Bumps [github.com/tetratelabs/wazero](https://github.com/tetratelabs/wazero) from 1.8.2-0.20241115151925-0a207958052e to 1.8.2.
- [Release notes](https://github.com/tetratelabs/wazero/releases)
- [Commits](https://github.com/tetratelabs/wazero/commits/v1.8.2)

---
updated-dependencies:
- dependency-name: github.com/tetratelabs/wazero
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2024-11-25 22:34:34 +00:00
Nuno Cruces
2912adf226 Update README.md 2024-11-25 14:17:08 +00:00
Nuno Cruces
b3f83a4392 Enable compiler. 2024-11-18 12:52:48 +00:00
Nuno Cruces
1223c4fc80 Flags. 2024-11-18 12:52:04 +00:00
Nuno Cruces
e9d6509577 Tweak. 2024-11-15 18:41:31 +00:00
Nuno Cruces
81dd786af7 Test more. 2024-11-12 14:14:45 +00:00
Nuno Cruces
a2253558ef Use lock file. 2024-11-11 16:55:37 +00:00
Nuno Cruces
466d14a9e0 Fix, speed. 2024-11-08 13:02:19 +00:00
Nuno Cruces
ada7b3a906 Updated dependencies. 2024-11-08 11:48:45 +00:00
Nuno Cruces
eba73a87d9 Refactor. 2024-11-07 12:18:42 +00:00
Nuno Cruces
38da27b5d1 Fix. 2024-11-06 12:10:48 +00:00
Nuno Cruces
23737a61ba Log. 2024-11-06 12:10:47 +00:00
Nuno Cruces
af473c7519 Update README.md 2024-11-06 01:34:21 +00:00
Nuno Cruces
a946c00f8e Refactor, speed. 2024-11-06 00:30:37 +00:00
Nuno Cruces
32153763a3 Update README.md 2024-11-05 17:39:36 +00:00
Nuno Cruces
a57ce87157 Windows shared memory. (#181) 2024-11-05 17:30:10 +00:00
Nuno Cruces
81e7a94ca4 Optimize regexp. 2024-11-04 19:30:10 +00:00
Nuno Cruces
034b9a3b4d More regexp. 2024-11-04 18:53:03 +00:00
Nuno Cruces
363b12ee4c Fix #178. 2024-11-03 13:14:38 +00:00
Nuno Cruces
90d6ec31b9 Refactor, speed. 2024-11-02 15:32:41 +00:00
Nuno Cruces
17f7840a83 Share memory by copying. (#180) 2024-10-31 15:21:15 +00:00
Nuno Cruces
b2e8636227 Dot file locking. (#179) 2024-10-30 12:20:21 +00:00
Nuno Cruces
1ad1608228 Refactor. 2024-10-30 00:50:28 +00:00
Nuno Cruces
9f284f0b26 Fix tests. 2024-10-30 00:44:50 +00:00
Nuno Cruces
df4e144e89 Prevent overflow. 2024-10-28 14:16:43 +00:00
Nuno Cruces
96074b24bf binaryen-version_120. 2024-10-28 10:56:10 +00:00
85 changed files with 1775 additions and 637 deletions

16
.github/actions/vmactions/template.yml vendored Normal file
View File

@@ -0,0 +1,16 @@
name: VM Actions matrix
description: VM Actions matrix template
inputs:
run:
description: The CI command to run
required: true
runs:
using: composite
steps:
- uses: ${VMACTIONS}
with:
usesh: true
copyback: false
run: ${{inputs.run}}

View File

@@ -6,6 +6,10 @@ echo 'set -eu' > test.sh
for p in $(go list ./...); do
dir=".${p#github.com/ncruces/go-sqlite3}"
name="$(basename "$p").test"
(cd ${dir}; go test -c)
[ -f "${dir}/${name}" ] && echo "(cd ${dir}; ./${name} ${TESTFLAGS})" >> test.sh
done
(cd ${dir}; go test -c ${BUILDFLAGS:-})
[ -f "${dir}/${name}" ] && echo "(cd ${dir}; ./${name} ${TESTFLAGS:-})" >> test.sh
done
if [[ -v VMACTIONS ]]; then
envsubst < .github/actions/vmactions/template.yml > .github/actions/vmactions/action.yml
fi

View File

@@ -17,11 +17,9 @@ echo aix ; GOOS=aix GOARCH=ppc64 go build .
echo js ; GOOS=js GOARCH=wasm go build .
echo wasip1 ; GOOS=wasip1 GOARCH=wasm go build .
echo linux-flock ; GOOS=linux GOARCH=amd64 go build -tags sqlite3_flock .
echo linux-noshm ; GOOS=linux GOARCH=amd64 go build -tags sqlite3_noshm .
echo linux-nosys ; GOOS=linux GOARCH=amd64 go build -tags sqlite3_nosys .
echo linux-dotlk ; GOOS=linux GOARCH=amd64 go build -tags sqlite3_dotlk .
echo darwin-flock ; GOOS=darwin GOARCH=amd64 go build -tags sqlite3_flock .
echo darwin-noshm ; GOOS=darwin GOARCH=amd64 go build -tags sqlite3_noshm .
echo darwin-nosys ; GOOS=darwin GOARCH=amd64 go build -tags sqlite3_nosys .
echo windows-nosys ; GOOS=windows GOARCH=amd64 go build -tags sqlite3_nosys .
echo freebsd-nosys ; GOOS=freebsd GOARCH=amd64 go build -tags sqlite3_nosys .
echo solaris-flock ; GOOS=solaris GOARCH=amd64 go build -tags sqlite3_flock .
echo darwin-dotlk ; GOOS=darwin GOARCH=amd64 go build -tags sqlite3_dotlk .
echo windows-dotlk ; GOOS=windows GOARCH=amd64 go build -tags sqlite3_dotlk .
echo freebsd-dotlk ; GOOS=freebsd GOARCH=amd64 go build -tags sqlite3_dotlk .
echo solaris-dotlk ; GOOS=solaris GOARCH=amd64 go build -tags sqlite3_dotlk .

View File

@@ -3,13 +3,13 @@ set -euo pipefail
if [[ "$OSTYPE" == "linux"* ]]; then
WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/wasi-sdk-24.0-x86_64-linux.tar.gz"
BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_119/binaryen-version_119-x86_64-linux.tar.gz"
BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_120/binaryen-version_120-x86_64-linux.tar.gz"
elif [[ "$OSTYPE" == "darwin"* ]]; then
WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/wasi-sdk-24.0-x86_64-macos.tar.gz"
BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_119/binaryen-version_119-x86_64-macos.tar.gz"
WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/wasi-sdk-24.0-arm64-macos.tar.gz"
BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_120/binaryen-version_120-arm64-macos.tar.gz"
elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then
WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/wasi-sdk-24.0-x86_64-windows.tar.gz"
BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_119/binaryen-version_119-x86_64-windows.tar.gz"
BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_120/binaryen-version_120-x86_64-windows.tar.gz"
fi
# Download tools

View File

@@ -25,7 +25,7 @@ jobs:
shell: bash
run: .github/workflows/repro.sh
- uses: actions/attest-build-provenance@v1
- uses: actions/attest-build-provenance@v2
if: matrix.os == 'ubuntu-latest'
with:
subject-path: |

View File

@@ -5,10 +5,16 @@ on:
branches: [ "main" ]
paths:
- '**.go'
- '**.mod'
- '**.wasm'
- '**.wasm.bz2'
pull_request:
branches: [ "main" ]
paths:
- '**.go'
- '**.mod'
- '**.wasm'
- '**.wasm.bz2'
workflow_dispatch:
jobs:
@@ -52,13 +58,9 @@ jobs:
run: go test -v -tags sqlite3_flock ./...
if: matrix.os == 'macos-latest'
- name: Test no shared memory
run: go test -v -tags sqlite3_noshm ./...
if: matrix.os == 'ubuntu-latest'
- name: Test no locks
run: go test -v -tags sqlite3_nosys ./...
if: matrix.os == 'ubuntu-latest'
- name: Test dot locks
run: go test -v -tags sqlite3_dotlk ./...
if: matrix.os != 'windows-latest'
- name: Test GORM
shell: bash
@@ -79,41 +81,24 @@ jobs:
github.event_name == 'push' &&
matrix.os == 'ubuntu-latest'
test-intel:
runs-on: macos-13
needs: test
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with: { go-version: stable }
- name: Git LFS pull
uses: ./.github/actions/lfs
- name: Test
run: go test -v ./...
test-bsd:
strategy:
matrix:
os:
- name: freebsd
version: '14.1'
version: '14.2'
flags: '-test.v'
- name: openbsd
version: '7.5'
flags: '-test.v -test.short'
- name: netbsd
version: '10.0'
flags: '-test.v'
- name: openbsd
version: '7.6'
flags: '-test.v -test.short'
runs-on: ubuntu-latest
needs: test
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with: { go-version: stable }
- name: Git LFS pull
uses: ./.github/actions/lfs
@@ -125,7 +110,7 @@ jobs:
run: .github/workflows/build-test.sh
- name: Test
uses: cross-platform-actions/action@v0.25.0
uses: cross-platform-actions/action@v0.26.0
with:
operating_system: ${{ matrix.os.name }}
version: ${{ matrix.os.version }}
@@ -133,6 +118,44 @@ jobs:
run: . ./test.sh
sync_files: runner-to-vm
test-vm:
strategy:
matrix:
os:
- name: dragonfly
action: 'vmactions/dragonflybsd-vm@v1'
tflags: '-test.v'
- name: illumos
action: 'vmactions/omnios-vm@v1'
tflags: '-test.v'
- name: solaris
action: 'vmactions/solaris-vm@v1'
bflags: '-tags sqlite3_dotlk'
tflags: '-test.v'
runs-on: ubuntu-latest
needs: test
steps:
- uses: actions/checkout@v4
- name: Git LFS pull
uses: ./.github/actions/lfs
- name: Build
env:
GOOS: ${{ matrix.os.name }}
BUILDFLAGS: ${{ matrix.os.bflags }}
TESTFLAGS: ${{ matrix.os.tflags }}
VMACTIONS: ${{ matrix.os.action }}
run: .github/workflows/build-test.sh
- name: Test
uses: ./.github/actions/vmactions
with:
usesh: true
copyback: false
run: . ./test.sh
test-qemu:
runs-on: ubuntu-latest
needs: test
@@ -158,11 +181,11 @@ jobs:
- name: Test ppc64le (interpreter)
run: GOARCH=ppc64le go test -v -short ./...
- name: Test s390x (big-endian, z/OS like)
run: GOARCH=s390x go test -v -short -tags sqlite3_flock ./...
- name: Test s390x (big-endian)
run: GOARCH=s390x go test -v -short -tags sqlite3_dotlk ./...
test-vm:
runs-on: ubuntu-latest
test-macintel:
runs-on: macos-13
needs: test
steps:
@@ -173,29 +196,5 @@ jobs:
- name: Git LFS pull
uses: ./.github/actions/lfs
- name: Build illumos
env:
GOOS: illumos
TESTFLAGS: '-test.v -test.short'
run: .github/workflows/build-test.sh
- name: Test illumos
uses: vmactions/omnios-vm@v1
with:
usesh: true
copyback: false
run: . ./test.sh
- name: Build Solaris
env:
GOOS: solaris
TESTFLAGS: '-test.v -test.short'
run: .github/workflows/build-test.sh
- name: Test Solaris
uses: vmactions/solaris-vm@v1
with:
usesh: true
copyback: false
run: . ./test.sh
continue-on-error: true
- name: Test
run: go test -v ./...

View File

@@ -10,7 +10,7 @@ as well as direct access to most of the [C SQLite API](https://sqlite.org/cintro
It wraps a [Wasm](https://webassembly.org/) [build](embed/) of SQLite,
and uses [wazero](https://wazero.io/) as the runtime.\
Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ runtime dependencies.
Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ direct dependencies.
### Getting started
@@ -77,7 +77,7 @@ It also benefits greatly from [SQLite's](https://sqlite.org/testing.html) and
Every commit is [tested](https://github.com/ncruces/go-sqlite3/wiki/Test-matrix) on
Linux (amd64/arm64/386/riscv64/ppc64le/s390x), macOS (amd64/arm64),
Windows (amd64), FreeBSD (amd64), OpenBSD (amd64), NetBSD (amd64),
illumos (amd64), and Solaris (amd64).
DragonFly BSD (amd64), illumos (amd64), and Solaris (amd64).
The Go VFS is tested by running SQLite's
[mptest](https://github.com/sqlite/sqlite/blob/master/mptest/mptest.c).
@@ -90,9 +90,20 @@ Perfomance of the [`database/sql`](https://pkg.go.dev/database/sql) driver is
The Wasm and VFS layers are also tested by running SQLite's
[speedtest1](https://github.com/sqlite/sqlite/blob/master/test/speedtest1.c).
### FAQ, issues, new features
For questions, please see [Discussions](https://github.com/ncruces/go-sqlite3/discussions/categories/q-a).
Also, post there if you used this driver for something interesting
([_"Show and tell"_](https://github.com/ncruces/go-sqlite3/discussions/categories/show-and-tell)),
have an [idea](https://github.com/ncruces/go-sqlite3/discussions/categories/ideas)…
The [Issue](https://github.com/ncruces/go-sqlite3/issues) tracker is for bugs we want fixed,
and features we're working on, planning to work on, or asking for help with.
### Alternatives
- [`modernc.org/sqlite`](https://pkg.go.dev/modernc.org/sqlite)
- [`crawshaw.io/sqlite`](https://pkg.go.dev/crawshaw.io/sqlite)
- [`github.com/mattn/go-sqlite3`](https://pkg.go.dev/github.com/mattn/go-sqlite3)
- [`github.com/zombiezen/go-sqlite`](https://pkg.go.dev/github.com/zombiezen/go-sqlite)
- [`github.com/zombiezen/go-sqlite`](https://pkg.go.dev/github.com/zombiezen/go-sqlite)

View File

@@ -2,6 +2,7 @@ package sqlite3
import (
"context"
"fmt"
"strconv"
"github.com/tetratelabs/wazero/api"
@@ -70,6 +71,15 @@ func logCallback(ctx context.Context, mod api.Module, _, iCode, zMsg uint32) {
}
}
// Log writes a message into the error log established by [Conn.ConfigLog].
//
// https://sqlite.org/c3ref/log.html
func (c *Conn) Log(code ExtendedErrorCode, format string, a ...any) {
if c.log != nil {
c.log(code, fmt.Sprintf(format, a...))
}
}
// FileControl allows low-level control of database files.
// Only a subset of opcodes are supported.
//

View File

@@ -501,8 +501,12 @@ func (c *Conn) TableColumnMetadata(schema, table, column string) (declType, coll
uint64(declTypePtr), uint64(collSeqPtr),
uint64(notNullPtr), uint64(primaryKeyPtr), uint64(autoIncPtr))
if err = c.error(r); err == nil && column != "" {
declType = util.ReadString(c.mod, util.ReadUint32(c.mod, declTypePtr), _MAX_NAME)
collSeq = util.ReadString(c.mod, util.ReadUint32(c.mod, collSeqPtr), _MAX_NAME)
if ptr := util.ReadUint32(c.mod, declTypePtr); ptr != 0 {
declType = util.ReadString(c.mod, ptr, _MAX_NAME)
}
if ptr := util.ReadUint32(c.mod, collSeqPtr); ptr != 0 {
collSeq = util.ReadString(c.mod, ptr, _MAX_NAME)
}
notNull = util.ReadUint32(c.mod, notNullPtr) != 0
autoInc = util.ReadUint32(c.mod, autoIncPtr) != 0
primaryKey = util.ReadUint32(c.mod, primaryKeyPtr) != 0

View File

@@ -89,6 +89,7 @@ func (ctx Context) ResultText(value string) {
}
// ResultRawText sets the text result of the function to a []byte.
// Returning a nil slice is the same as calling [Context.ResultNull].
//
// https://sqlite.org/c3ref/result_blob.html
func (ctx Context) ResultRawText(value []byte) {

View File

@@ -81,6 +81,7 @@ import (
"fmt"
"io"
"net/url"
"reflect"
"strings"
"time"
"unsafe"
@@ -107,17 +108,17 @@ func init() {
// The second callback is called before the driver closes a connection.
// The [sqlite3.Conn] can be used to execute queries, register functions, etc.
func Open(dataSourceName string, fn ...func(*sqlite3.Conn) error) (*sql.DB, error) {
var drv SQLite
if len(fn) > 2 {
return nil, sqlite3.MISUSE
}
var init, term func(*sqlite3.Conn) error
if len(fn) > 1 {
drv.term = fn[1]
term = fn[1]
}
if len(fn) > 0 {
drv.init = fn[0]
init = fn[0]
}
c, err := drv.OpenConnector(dataSourceName)
c, err := newConnector(dataSourceName, init, term)
if err != nil {
return nil, err
}
@@ -125,10 +126,7 @@ func Open(dataSourceName string, fn ...func(*sqlite3.Conn) error) (*sql.DB, erro
}
// SQLite implements [database/sql/driver.Driver].
type SQLite struct {
init func(*sqlite3.Conn) error
term func(*sqlite3.Conn) error
}
type SQLite struct{}
var (
// Ensure these interfaces are implemented:
@@ -137,7 +135,7 @@ var (
// Open implements [database/sql/driver.Driver].
func (d *SQLite) Open(name string) (driver.Conn, error) {
c, err := d.newConnector(name)
c, err := newConnector(name, nil, nil)
if err != nil {
return nil, err
}
@@ -146,11 +144,11 @@ func (d *SQLite) Open(name string) (driver.Conn, error) {
// OpenConnector implements [database/sql/driver.DriverContext].
func (d *SQLite) OpenConnector(name string) (driver.Connector, error) {
return d.newConnector(name)
return newConnector(name, nil, nil)
}
func (d *SQLite) newConnector(name string) (*connector, error) {
c := connector{driver: d, name: name}
func newConnector(name string, init, term func(*sqlite3.Conn) error) (*connector, error) {
c := connector{name: name, init: init, term: term}
var txlock, timefmt string
if strings.HasPrefix(name, "file:") {
@@ -190,7 +188,8 @@ func (d *SQLite) newConnector(name string) (*connector, error) {
}
type connector struct {
driver *SQLite
init func(*sqlite3.Conn) error
term func(*sqlite3.Conn) error
name string
txLock string
tmRead sqlite3.TimeFormat
@@ -199,7 +198,7 @@ type connector struct {
}
func (n *connector) Driver() driver.Driver {
return n.driver
return &SQLite{}
}
func (n *connector) Connect(ctx context.Context) (res driver.Conn, err error) {
@@ -228,13 +227,13 @@ func (n *connector) Connect(ctx context.Context) (res driver.Conn, err error) {
return nil, err
}
}
if n.driver.init != nil {
err = n.driver.init(c.Conn)
if n.init != nil {
err = n.init(c.Conn)
if err != nil {
return nil, err
}
}
if n.pragmas || n.driver.init != nil {
if n.pragmas || n.init != nil {
s, _, err := c.Conn.Prepare(`PRAGMA query_only`)
if err != nil {
return nil, err
@@ -250,9 +249,9 @@ func (n *connector) Connect(ctx context.Context) (res driver.Conn, err error) {
return nil, err
}
}
if n.driver.term != nil {
if n.term != nil {
err = c.Conn.Trace(sqlite3.TRACE_CLOSE, func(sqlite3.TraceEvent, any, any) error {
return n.driver.term(c.Conn)
return n.term(c.Conn)
})
if err != nil {
return nil, err
@@ -288,6 +287,8 @@ func (n *connector) Connect(ctx context.Context) (res driver.Conn, err error) {
type Conn interface {
Raw() *sqlite3.Conn
driver.Conn
driver.ConnBeginTx
driver.ConnPrepareContext
}
type conn struct {
@@ -301,10 +302,8 @@ type conn struct {
var (
// Ensure these interfaces are implemented:
_ Conn = &conn{}
_ driver.ConnBeginTx = &conn{}
_ driver.ConnPrepareContext = &conn{}
_ driver.ExecerContext = &conn{}
_ Conn = &conn{}
_ driver.ExecerContext = &conn{}
)
func (c *conn) Raw() *sqlite3.Conn {
@@ -581,8 +580,22 @@ type rows struct {
names []string
types []string
nulls []bool
scans []scantype
}
type scantype byte
const (
_ANY scantype = iota
_INT scantype = scantype(sqlite3.INTEGER)
_REAL scantype = scantype(sqlite3.FLOAT)
_TEXT scantype = scantype(sqlite3.TEXT)
_BLOB scantype = scantype(sqlite3.BLOB)
_NULL scantype = scantype(sqlite3.NULL)
_BOOL scantype = iota
_TIME
)
var (
// Ensure these interfaces are implemented:
_ driver.RowsColumnTypeDatabaseTypeName = &rows{}
@@ -606,21 +619,42 @@ func (r *rows) Columns() []string {
return r.names
}
func (r *rows) loadTypes() {
func (r *rows) loadColumnMetadata() {
if r.nulls == nil {
count := r.Stmt.ColumnCount()
nulls := make([]bool, count)
types := make([]string, count)
scans := make([]scantype, count)
for i := range nulls {
if col := r.Stmt.ColumnOriginName(i); col != "" {
types[i], _, nulls[i], _, _, _ = r.Stmt.Conn().TableColumnMetadata(
r.Stmt.ColumnDatabaseName(i),
r.Stmt.ColumnTableName(i),
col)
types[i] = strings.ToUpper(types[i])
// These types are only used before we have rows,
// and otherwise as type hints.
// The first few ensure STRICT tables are strictly typed.
// The other two are type hints for booleans and time.
switch types[i] {
case "INT", "INTEGER":
scans[i] = _INT
case "REAL":
scans[i] = _REAL
case "TEXT":
scans[i] = _TEXT
case "BLOB":
scans[i] = _BLOB
case "BOOLEAN":
scans[i] = _BOOL
case "DATE", "TIME", "DATETIME", "TIMESTAMP":
scans[i] = _TIME
}
}
}
r.nulls = nulls
r.types = types
r.scans = scans
}
}
@@ -637,7 +671,7 @@ func (r *rows) declType(index int) string {
}
func (r *rows) ColumnTypeDatabaseTypeName(index int) string {
r.loadTypes()
r.loadColumnMetadata()
decltype := r.types[index]
if len := len(decltype); len > 0 && decltype[len-1] == ')' {
if i := strings.LastIndexByte(decltype, '('); i >= 0 {
@@ -648,13 +682,57 @@ func (r *rows) ColumnTypeDatabaseTypeName(index int) string {
}
func (r *rows) ColumnTypeNullable(index int) (nullable, ok bool) {
r.loadTypes()
r.loadColumnMetadata()
if r.nulls[index] {
return false, true
}
return true, false
}
func (r *rows) ColumnTypeScanType(index int) (typ reflect.Type) {
r.loadColumnMetadata()
scan := r.scans[index]
if r.Stmt.Busy() {
// SQLite is dynamically typed and we now have a row.
// Always use the type of the value itself,
// unless the scan type is more specific
// and can scan the actual value.
val := scantype(r.Stmt.ColumnType(index))
useValType := true
switch {
case scan == _TIME && val != _BLOB && val != _NULL:
t := r.Stmt.ColumnTime(index, r.tmRead)
useValType = t == time.Time{}
case scan == _BOOL && val == _INT:
i := r.Stmt.ColumnInt64(index)
useValType = i != 0 && i != 1
case scan == _BLOB && val == _NULL:
useValType = false
}
if useValType {
scan = val
}
}
switch scan {
case _INT:
return reflect.TypeOf(int64(0))
case _REAL:
return reflect.TypeOf(float64(0))
case _TEXT:
return reflect.TypeOf("")
case _BLOB:
return reflect.TypeOf([]byte{})
case _BOOL:
return reflect.TypeOf(false)
case _TIME:
return reflect.TypeOf(time.Time{})
default:
return reflect.TypeOf((*any)(nil)).Elem()
}
}
func (r *rows) Next(dest []driver.Value) error {
old := r.Stmt.Conn().SetInterrupt(r.ctx)
defer r.Stmt.Conn().SetInterrupt(old)
@@ -667,7 +745,7 @@ func (r *rows) Next(dest []driver.Value) error {
}
data := unsafe.Slice((*any)(unsafe.SliceData(dest)), len(dest))
err := r.Stmt.Columns(data)
err := r.Stmt.Columns(data...)
for i := range dest {
if t, ok := r.decodeTime(i, dest[i]); ok {
dest[i] = t

View File

@@ -7,6 +7,7 @@ import (
"errors"
"math"
"net/url"
"reflect"
"testing"
"time"
@@ -365,3 +366,104 @@ func Test_time(t *testing.T) {
})
}
}
func Test_ColumnType_ScanType(t *testing.T) {
var (
INT = reflect.TypeOf(int64(0))
REAL = reflect.TypeOf(float64(0))
TEXT = reflect.TypeOf("")
BLOB = reflect.TypeOf([]byte{})
BOOL = reflect.TypeOf(false)
TIME = reflect.TypeOf(time.Time{})
ANY = reflect.TypeOf((*any)(nil)).Elem()
)
t.Parallel()
tmp := memdb.TestDB(t)
db, err := sql.Open("sqlite3", tmp)
if err != nil {
t.Fatal(err)
}
defer db.Close()
_, err = db.Exec(`
CREATE TABLE test (
col_int INTEGER,
col_real REAL,
col_text TEXT,
col_blob BLOB,
col_bool BOOLEAN,
col_time DATETIME,
col_decimal DECIMAL
);
INSERT INTO test VALUES
(1, 1, 1, 1, 1, 1, 1),
(2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0),
('1', '1', '1', '1', '1', '1', '1'),
('x', 'x', 'x', 'x', 'x', 'x', 'x'),
(x'', x'', x'', x'', x'', x'', x''),
('2006-01-02T15:04:05Z', '2006-01-02T15:04:05Z', '2006-01-02T15:04:05Z', '2006-01-02T15:04:05Z',
'2006-01-02T15:04:05Z', '2006-01-02T15:04:05Z', '2006-01-02T15:04:05Z'),
(TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE),
(NULL, NULL, NULL, NULL, NULL, NULL, NULL);
`)
if err != nil {
t.Fatal(err)
}
rows, err := db.Query(`SELECT * FROM test`)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
cols, err := rows.ColumnTypes()
if err != nil {
t.Fatal(err)
}
want := [][]reflect.Type{
{INT, REAL, TEXT, BLOB, BOOL, TIME, ANY},
{INT, REAL, TEXT, INT, BOOL, TIME, INT},
{INT, REAL, TEXT, REAL, INT, TIME, INT},
{INT, REAL, TEXT, TEXT, BOOL, TIME, INT},
{TEXT, TEXT, TEXT, TEXT, TEXT, TEXT, TEXT},
{BLOB, BLOB, BLOB, BLOB, BLOB, BLOB, BLOB},
{TEXT, TEXT, TEXT, TEXT, TEXT, TIME, TEXT},
{INT, REAL, TEXT, INT, BOOL, TIME, INT},
{ANY, ANY, ANY, BLOB, ANY, ANY, ANY},
}
for j, c := range cols {
got := c.ScanType()
if got != want[0][j] {
t.Errorf("want %v, got %v, at column %d", want[0][j], got, j)
}
}
dest := make([]any, len(cols))
for i := 1; rows.Next(); i++ {
cols, err := rows.ColumnTypes()
if err != nil {
t.Fatal(err)
}
for j, c := range cols {
got := c.ScanType()
if got != want[i][j] {
t.Errorf("want %v, got %v, at row %d column %d", want[i][j], got, i, j)
}
dest[j] = reflect.New(got).Interface()
}
err = rows.Scan(dest...)
if err != nil {
t.Error(err)
}
}
err = rows.Err()
if err != nil {
t.Fatal(err)
}
}

View File

@@ -1,4 +1,4 @@
//go:build (linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && !sqlite3_nosys
//go:build linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock || sqlite3_dotlk
package driver_test

View File

@@ -1,4 +1,4 @@
//go:build (linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && !sqlite3_nosys
//go:build linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock || sqlite3_dotlk
package driver_test

View File

@@ -1,8 +1,6 @@
package driver
import (
"time"
)
import "time"
// Convert a string in [time.RFC3339Nano] format into a [time.Time]
// if it roundtrips back to the same string.

View File

@@ -1,6 +1,6 @@
# Embeddable Wasm build of SQLite
This folder includes an embeddable Wasm build of SQLite 3.47.0 for use with
This folder includes an embeddable Wasm build of SQLite 3.47.2 for use with
[`github.com/ncruces/go-sqlite3`](https://pkg.go.dev/github.com/ncruces/go-sqlite3).
The following optional features are compiled in:

View File

@@ -1,6 +1,6 @@
# Embeddable Wasm build of SQLite
This folder includes an embeddable Wasm build of SQLite 3.47.0, including the experimental
This folder includes an embeddable Wasm build of SQLite, including the experimental
[`BEGIN CONCURRENT`](https://sqlite.org/src/doc/begin-concurrent/doc/begin_concurrent.md) and
[Wal2](https://sqlite.org/cgi/src/doc/wal2/doc/wal2.md) patches.

Binary file not shown.

View File

@@ -42,7 +42,7 @@ func Test_bcw2(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if version != "3.47.0" {
if version != "3.48.0" {
t.Error(version)
}
}

View File

@@ -13,8 +13,8 @@ mkdir -p build/ext/
cp "$ROOT"/sqlite3/*.[ch] build/
cp "$ROOT"/sqlite3/*.patch build/
# https://sqlite.org/src/info/d2d954d43abe20a3
curl -# https://sqlite.org/src/tarball/sqlite.tar.gz?r=d2d954d4 | tar xz
# https://sqlite.org/src/info/08cfa7e8b3090151
curl -# https://sqlite.org/src/tarball/sqlite.tar.gz?r=08cfa7e8 | tar xz
cd sqlite
if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then

13
embed/bcw2/go.mod Normal file
View File

@@ -0,0 +1,13 @@
module github.com/ncruces/go-sqlite3/embed/bcw2
go 1.21
toolchain go1.23.0
require github.com/ncruces/go-sqlite3 v0.20.4-0.20241205130447-8252198dd21c
require (
github.com/ncruces/julianday v1.0.0 // indirect
github.com/tetratelabs/wazero v1.8.2 // indirect
golang.org/x/sys v0.28.0 // indirect
)

10
embed/bcw2/go.sum Normal file
View File

@@ -0,0 +1,10 @@
github.com/ncruces/go-sqlite3 v0.20.4-0.20241205130447-8252198dd21c h1:VDpb7CDIfkEUtPsHgYQwygxST9VocLk8y0fDMQYGPzI=
github.com/ncruces/go-sqlite3 v0.20.4-0.20241205130447-8252198dd21c/go.mod h1:2+txOGfyB5f8fUNOjmos2TuHYA8Xf/oAzI++lHwbHno=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/tetratelabs/wazero v1.8.2 h1:yIgLR/b2bN31bjxwXHD8a3d+BogigR952csSDdLYEv4=
github.com/tetratelabs/wazero v1.8.2/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=

View File

@@ -19,7 +19,7 @@ func Test_init(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if version != "3.47.0" {
if version != "3.47.2" {
t.Error(version)
}
}

Binary file not shown.

View File

@@ -106,6 +106,11 @@ func (e ErrorCode) Temporary() bool {
return e == BUSY
}
// ExtendedCode returns the extended error code for this error.
func (e ErrorCode) ExtendedCode() ExtendedErrorCode {
return ExtendedErrorCode(e)
}
// Error implements the error interface.
func (e ExtendedErrorCode) Error() string {
return util.ErrorCodeString(uint32(e))
@@ -136,6 +141,11 @@ func (e ExtendedErrorCode) Timeout() bool {
return e == BUSY_TIMEOUT
}
// Code returns the primary error code for this error.
func (e ExtendedErrorCode) Code() ErrorCode {
return ErrorCode(e)
}
func errorCode(err error, def ErrorCode) (msg string, code uint32) {
switch code := err.(type) {
case nil:

View File

@@ -1,9 +1,11 @@
// Package regexp provides additional regular expression functions.
//
// It provides the following Unicode aware functions:
// - regexp_like(),
// - regexp_substr(),
// - regexp_replace(),
// - regexp_like(text, pattern),
// - regexp_count(text, pattern [, start]),
// - regexp_instr(text, pattern [, start [, N [, endoption [, subexpr ]]]]),
// - regexp_substr(text, pattern [, start [, N [, subexpr ]]]),
// - regexp_replace(text, pattern, replacement [, start [, N ]]),
// - and a REGEXP operator.
//
// The implementation uses Go [regexp/syntax] for regular expressions.
@@ -14,6 +16,7 @@ package regexp
import (
"errors"
"regexp"
"strings"
"github.com/ncruces/go-sqlite3"
)
@@ -24,8 +27,39 @@ func Register(db *sqlite3.Conn) error {
return errors.Join(
db.CreateFunction("regexp", 2, flags, regex),
db.CreateFunction("regexp_like", 2, flags, regexLike),
db.CreateFunction("regexp_count", 2, flags, regexCount),
db.CreateFunction("regexp_count", 3, flags, regexCount),
db.CreateFunction("regexp_instr", 2, flags, regexInstr),
db.CreateFunction("regexp_instr", 3, flags, regexInstr),
db.CreateFunction("regexp_instr", 4, flags, regexInstr),
db.CreateFunction("regexp_instr", 5, flags, regexInstr),
db.CreateFunction("regexp_instr", 6, flags, regexInstr),
db.CreateFunction("regexp_substr", 2, flags, regexSubstr),
db.CreateFunction("regexp_replace", 3, flags, regexReplace))
db.CreateFunction("regexp_substr", 3, flags, regexSubstr),
db.CreateFunction("regexp_substr", 4, flags, regexSubstr),
db.CreateFunction("regexp_substr", 5, flags, regexSubstr),
db.CreateFunction("regexp_replace", 3, flags, regexReplace),
db.CreateFunction("regexp_replace", 4, flags, regexReplace),
db.CreateFunction("regexp_replace", 5, flags, regexReplace))
}
// GlobPrefix returns a GLOB for a regular expression
// appropriate to take advantage of the [LIKE optimization]
// in a query such as:
//
// SELECT column WHERE column GLOB :glob_prefix AND column REGEXP :regexp
//
// [LIKE optimization]: https://sqlite.org/optoverview.html#the_like_optimization
func GlobPrefix(re *regexp.Regexp) string {
prefix, complete := re.LiteralPrefix()
i := strings.IndexAny(prefix, "*?[")
if i < 0 {
if complete {
return prefix
}
i = len(prefix)
}
return prefix[:i] + "*"
}
func load(ctx sqlite3.Context, i int, expr string) (*regexp.Regexp, error) {
@@ -44,35 +78,163 @@ func load(ctx sqlite3.Context, i int, expr string) (*regexp.Regexp, error) {
func regex(ctx sqlite3.Context, arg ...sqlite3.Value) {
re, err := load(ctx, 0, arg[0].Text())
if err != nil {
ctx.ResultError(err) // notest
} else {
ctx.ResultBool(re.Match(arg[1].RawText()))
ctx.ResultError(err)
return // notest
}
text := arg[1].RawText()
ctx.ResultBool(re.Match(text))
}
func regexLike(ctx sqlite3.Context, arg ...sqlite3.Value) {
re, err := load(ctx, 1, arg[1].Text())
if err != nil {
ctx.ResultError(err) // notest
} else {
ctx.ResultBool(re.Match(arg[0].RawText()))
ctx.ResultError(err)
return // notest
}
text := arg[0].RawText()
ctx.ResultBool(re.Match(text))
}
func regexCount(ctx sqlite3.Context, arg ...sqlite3.Value) {
re, err := load(ctx, 1, arg[1].Text())
if err != nil {
ctx.ResultError(err)
return // notest
}
text := arg[0].RawText()
if len(arg) > 2 {
pos := arg[2].Int()
text = text[skip(text, pos):]
}
ctx.ResultInt(len(re.FindAll(text, -1)))
}
func regexSubstr(ctx sqlite3.Context, arg ...sqlite3.Value) {
re, err := load(ctx, 1, arg[1].Text())
if err != nil {
ctx.ResultError(err) // notest
} else {
ctx.ResultRawText(re.Find(arg[0].RawText()))
ctx.ResultError(err)
return // notest
}
text := arg[0].RawText()
var pos, n, subexpr int
if len(arg) > 2 {
pos = arg[2].Int()
}
if len(arg) > 3 {
n = arg[3].Int()
}
if len(arg) > 4 {
subexpr = arg[4].Int()
}
loc := regexFind(re, text, pos, n, subexpr)
if loc != nil {
ctx.ResultRawText(text[loc[0]:loc[1]])
}
}
func regexInstr(ctx sqlite3.Context, arg ...sqlite3.Value) {
re, err := load(ctx, 1, arg[1].Text())
if err != nil {
ctx.ResultError(err)
return // notest
}
text := arg[0].RawText()
var pos, n, end, subexpr int
if len(arg) > 2 {
pos = arg[2].Int()
}
if len(arg) > 3 {
n = arg[3].Int()
}
if len(arg) > 4 && arg[4].Bool() {
end = 1
}
if len(arg) > 5 {
subexpr = arg[5].Int()
}
loc := regexFind(re, text, pos, n, subexpr)
if loc != nil {
ctx.ResultInt(loc[end] + 1)
}
}
func regexReplace(ctx sqlite3.Context, arg ...sqlite3.Value) {
re, err := load(ctx, 1, arg[1].Text())
if err != nil {
ctx.ResultError(err) // notest
} else {
ctx.ResultRawText(re.ReplaceAll(arg[0].RawText(), arg[2].RawText()))
ctx.ResultError(err)
return // notest
}
text := arg[0].RawText()
repl := arg[2].RawText()
var pos, n int
if len(arg) > 3 {
pos = arg[3].Int()
}
if len(arg) > 4 {
n = arg[4].Int()
}
res := text
pos = skip(text, pos)
if n > 0 {
all := re.FindAllSubmatchIndex(text[pos:], n)
if n <= len(all) {
loc := all[n-1]
res = text[:pos+loc[0]]
res = re.Expand(res, repl, text[pos:], loc)
res = append(res, text[pos+loc[1]:]...)
}
} else {
res = append(text[:pos], re.ReplaceAll(text[pos:], repl)...)
}
ctx.ResultRawText(res)
}
func regexFind(re *regexp.Regexp, text []byte, pos, n, subexpr int) (loc []int) {
pos = skip(text, pos)
text = text[pos:]
if n <= 1 {
if subexpr == 0 {
loc = re.FindIndex(text)
} else {
loc = re.FindSubmatchIndex(text)
}
} else {
if subexpr == 0 {
all := re.FindAllIndex(text, n)
if n <= len(all) {
loc = all[n-1]
}
} else {
all := re.FindAllSubmatchIndex(text, n)
if n <= len(all) {
loc = all[n-1]
}
}
}
if 2+2*subexpr <= len(loc) {
loc = loc[2*subexpr : 2+2*subexpr]
loc[0] += pos
loc[1] += pos
return loc
}
return nil
}
func skip(text []byte, start int) int {
for pos := range string(text) {
if start--; start <= 0 {
return pos
}
}
return len(text)
}

View File

@@ -1,6 +1,8 @@
package regexp
import (
"database/sql"
"regexp"
"testing"
"github.com/ncruces/go-sqlite3/driver"
@@ -29,18 +31,47 @@ func TestRegister(t *testing.T) {
{`regexp_like('Hello', 'elo')`, "0"},
{`regexp_like('Hello', 'ell')`, "1"},
{`regexp_like('Hello', 'el.')`, "1"},
{`regexp_count('Hello', 'l')`, "2"},
{`regexp_instr('Hello', 'el.')`, "2"},
{`regexp_instr('Hello', '.', 6)`, ""},
{`regexp_substr('Hello', 'el.')`, "ell"},
{`regexp_replace('Hello', 'llo', 'll')`, "Hell"},
// https://www.postgresql.org/docs/current/functions-matching.html
{`regexp_count('ABCABCAXYaxy', 'A.')`, "3"},
{`regexp_count('ABCABCAXYaxy', '(?i)A.', 1)`, "4"},
{`regexp_instr('number of your street, town zip, FR', '[^,]+', 1, 2)`, "23"},
{`regexp_instr('ABCDEFGHI', '(?i)(c..)(...)', 1, 1, 0, 2)`, "6"},
{`regexp_substr('number of your street, town zip, FR', '[^,]+', 1, 2)`, " town zip"},
{`regexp_substr('ABCDEFGHI', '(?i)(c..)(...)', 1, 1, 2)`, "FGH"},
{`regexp_replace('foobarbaz', 'b..', 'X', 1, 1)`, "fooXbaz"},
{`regexp_replace('foobarbaz', 'b..', 'X')`, "fooXX"},
{`regexp_replace('foobarbaz', 'b(..)', 'X${1}Y')`, "fooXarYXazY"},
{`regexp_replace('A PostgreSQL function', '(?i)a|e|i|o|u', 'X', 1, 0)`, "X PXstgrXSQL fXnctXXn"},
{`regexp_replace('A PostgreSQL function', '(?i)a|e|i|o|u', 'X', 1, 3)`, "A PostgrXSQL function"},
// https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/REGEXP_COUNT.html
{`regexp_count('123123123123123', '(12)3', 1)`, "5"},
{`regexp_count('123123123123', '123', 3)`, "3"},
{`regexp_instr('500 Oracle Parkway, Redwood Shores, CA', '[^ ]+', 1, 6)`, "37"},
{`regexp_instr('500 Oracle Parkway, Redwood Shores, CA', '(?i)[s|r|p][[:alpha:]]{6}', 3, 2, 1)`, "28"},
{`regexp_instr('1234567890', '(123)(4(56)(78))', 1, 1, 0, 1)`, "1"},
{`regexp_instr('1234567890', '(123)(4(56)(78))', 1, 1, 0, 2)`, "4"},
{`regexp_instr('1234567890', '(123)(4(56)(78))', 1, 1, 0, 4)`, "7"},
{`regexp_substr('500 Oracle Parkway, Redwood Shores, CA', ',[^,]+,')`, ", Redwood Shores,"},
{`regexp_substr('http://www.example.com/products', 'http://([[:alnum:]]+\.?){3,4}/?')`, "http://www.example.com/"},
{`regexp_substr('1234567890', '(123)(4(56)(78))', 1, 1, 1)`, "123"},
{`regexp_substr('1234567890', '(123)(4(56)(78))', 1, 1, 4)`, "78"},
{`regexp_substr('123123123123', '1(.)3', 3, 2, 1)`, "2"},
{`regexp_replace('500 Oracle Parkway, Redwood Shores, CA', '( ){2,}', ' ')`, "500 Oracle Parkway, Redwood Shores, CA"},
}
for _, tt := range tests {
var got string
var got sql.NullString
err := db.QueryRow(`SELECT ` + tt.test).Scan(&got)
if err != nil {
t.Fatal(err)
}
if got != tt.want {
t.Errorf("got %q, want %q", got, tt.want)
if got.String != tt.want {
t.Errorf("got %q, want %q", got.String, tt.want)
}
}
}
@@ -58,6 +89,8 @@ func TestRegister_errors(t *testing.T) {
tests := []string{
`'' REGEXP ?`,
`regexp_like('', ?)`,
`regexp_count('', ?)`,
`regexp_instr('', ?)`,
`regexp_substr('', ?)`,
`regexp_replace('', ?, '')`,
}
@@ -69,3 +102,25 @@ func TestRegister_errors(t *testing.T) {
}
}
}
func TestGlobPrefix(t *testing.T) {
tests := []struct {
re string
want string
}{
{``, ""},
{`a`, "a"},
{`a*`, "*"},
{`a+`, "a*"},
{`ab*`, "a*"},
{`ab+`, "ab*"},
{`a\?b`, "a*"},
}
for _, tt := range tests {
t.Run(tt.re, func(t *testing.T) {
if got := GlobPrefix(regexp.MustCompile(tt.re)); got != tt.want {
t.Errorf("GlobPrefix() = %v, want %v", got, tt.want)
}
})
}
}

21
go.mod
View File

@@ -5,17 +5,20 @@ go 1.21
toolchain go1.23.0
require (
github.com/dchest/siphash v1.2.3
github.com/google/uuid v1.6.0
github.com/ncruces/julianday v1.0.0
github.com/ncruces/sort v0.1.2
github.com/psanford/httpreadat v0.1.0
github.com/tetratelabs/wazero v1.8.1
golang.org/x/crypto v0.28.0
golang.org/x/sync v0.8.0
golang.org/x/sys v0.26.0
golang.org/x/text v0.19.0
lukechampine.com/adiantum v1.1.1
github.com/tetratelabs/wazero v1.8.2
golang.org/x/crypto v0.31.0
golang.org/x/sys v0.28.0
)
require (
github.com/dchest/siphash v1.2.3 // ext/bloom
github.com/google/uuid v1.6.0 // ext/uuid
github.com/psanford/httpreadat v0.1.0 // example
golang.org/x/sync v0.10.0 // test
golang.org/x/text v0.21.0 // ext/unicode
lukechampine.com/adiantum v1.1.1 // vfs/adiantum
)
retract v0.4.0 // tagged from the wrong branch

20
go.sum
View File

@@ -8,15 +8,15 @@ github.com/ncruces/sort v0.1.2 h1:zKQ9CA4fpHPF6xsUhRTfi5EEryspuBpe/QA4VWQOV1U=
github.com/ncruces/sort v0.1.2/go.mod h1:vEJUTBJtebIuCMmXD18GKo5GJGhsay+xZFOoBEIXFmE=
github.com/psanford/httpreadat v0.1.0 h1:VleW1HS2zO7/4c7c7zNl33fO6oYACSagjJIyMIwZLUE=
github.com/psanford/httpreadat v0.1.0/go.mod h1:Zg7P+TlBm3bYbyHTKv/EdtSJZn3qwbPwpfZ/I9GKCRE=
github.com/tetratelabs/wazero v1.8.1 h1:NrcgVbWfkWvVc4UtT4LRLDf91PsOzDzefMdwhLfA550=
github.com/tetratelabs/wazero v1.8.1/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
github.com/tetratelabs/wazero v1.8.2 h1:yIgLR/b2bN31bjxwXHD8a3d+BogigR952csSDdLYEv4=
github.com/tetratelabs/wazero v1.8.2/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
lukechampine.com/adiantum v1.1.1 h1:4fp6gTxWCqpEbLy40ExiYDDED3oUNWx5cTqBCtPdZqA=
lukechampine.com/adiantum v1.1.1/go.mod h1:LrAYVnTYLnUtE/yMp5bQr0HstAf060YUF8nM0B6+rUw=

View File

@@ -3,4 +3,5 @@ go 1.21
use (
.
./gormlite
./embed/bcw2
)

View File

@@ -9,5 +9,7 @@ golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8=
golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=

View File

@@ -5,7 +5,7 @@ go 1.21
toolchain go1.23.0
require (
github.com/ncruces/go-sqlite3 v0.20.0
github.com/ncruces/go-sqlite3 v0.20.3
gorm.io/gorm v1.25.12
)
@@ -13,7 +13,7 @@ require (
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/ncruces/julianday v1.0.0 // indirect
github.com/tetratelabs/wazero v1.8.1 // indirect
golang.org/x/sys v0.26.0 // indirect
golang.org/x/text v0.19.0 // indirect
github.com/tetratelabs/wazero v1.8.2 // indirect
golang.org/x/sys v0.28.0 // indirect
golang.org/x/text v0.21.0 // indirect
)

View File

@@ -2,15 +2,15 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/ncruces/go-sqlite3 v0.20.0 h1:/nBLvYxj7sk9S6y57nmMFvoQ/KJtGo0pNi8J80s8oJU=
github.com/ncruces/go-sqlite3 v0.20.0/go.mod h1:yL4ZNWGsr1/8pcLfpPW1RT1WFdvyeHonrgIwwi4rvkg=
github.com/ncruces/go-sqlite3 v0.20.3 h1:+4G4uEqOeusF0yRuQVUl9fuoEebUolwQSnBUjYBLYIw=
github.com/ncruces/go-sqlite3 v0.20.3/go.mod h1:ojLIAB243gtz68Eo283Ps+k9PyR3dvzS+9/RgId4+AA=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/tetratelabs/wazero v1.8.1 h1:NrcgVbWfkWvVc4UtT4LRLDf91PsOzDzefMdwhLfA550=
github.com/tetratelabs/wazero v1.8.1/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
github.com/tetratelabs/wazero v1.8.2 h1:yIgLR/b2bN31bjxwXHD8a3d+BogigR952csSDdLYEv4=
github.com/tetratelabs/wazero v1.8.2/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8=
gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=

View File

@@ -1,4 +1,4 @@
//go:build !(unix || windows) || sqlite3_nosys
//go:build !unix && !windows
package alloc

View File

@@ -1,4 +1,4 @@
//go:build unix && !sqlite3_nosys
//go:build unix
package alloc

View File

@@ -1,5 +1,3 @@
//go:build !sqlite3_nosys
package alloc
import (

View File

@@ -19,7 +19,8 @@ func init() {
path := filepath.Join(os.TempDir(), "wazero")
if err := os.MkdirAll(path, 0777); err == nil {
if cache, err := wazero.NewCompilationCacheWithDir(path); err == nil {
sqlite3.RuntimeConfig.WithCompilationCache(cache)
sqlite3.RuntimeConfig = sqlite3.RuntimeConfig.
WithCompilationCache(cache)
}
}
}

View File

@@ -1,4 +1,4 @@
//go:build !unix || !(386 || arm || amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys
//go:build !unix
package util

View File

@@ -1,4 +1,4 @@
//go:build unix && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys)
//go:build unix
package util
@@ -55,10 +55,10 @@ type MappedRegion struct {
used bool
}
func MapRegion(ctx context.Context, mod api.Module, f *os.File, offset int64, size int32, prot int) (*MappedRegion, error) {
func MapRegion(ctx context.Context, mod api.Module, f *os.File, offset int64, size int32, readOnly bool) (*MappedRegion, error) {
s := ctx.Value(moduleKey{}).(*moduleState)
r := s.new(ctx, mod, size)
err := r.mmap(f, offset, prot)
err := r.mmap(f, offset, readOnly)
if err != nil {
return nil, err
}
@@ -75,7 +75,11 @@ func (r *MappedRegion) Unmap() error {
return err
}
func (r *MappedRegion) mmap(f *os.File, offset int64, prot int) error {
func (r *MappedRegion) mmap(f *os.File, offset int64, readOnly bool) error {
prot := unix.PROT_READ
if !readOnly {
prot |= unix.PROT_WRITE
}
_, err := unix.MmapPtr(int(f.Fd()), offset, r.addr, uintptr(r.size),
prot, unix.MAP_SHARED|unix.MAP_FIXED)
r.used = err == nil

View File

@@ -0,0 +1,51 @@
package util
import (
"context"
"os"
"reflect"
"unsafe"
"github.com/tetratelabs/wazero/api"
"golang.org/x/sys/windows"
)
type MappedRegion struct {
windows.Handle
Data []byte
addr uintptr
}
func MapRegion(ctx context.Context, mod api.Module, f *os.File, offset int64, size int32) (*MappedRegion, error) {
h, err := windows.CreateFileMapping(windows.Handle(f.Fd()), nil, windows.PAGE_READWRITE, 0, 0, nil)
if h == 0 {
return nil, err
}
a, err := windows.MapViewOfFile(h, windows.FILE_MAP_WRITE,
uint32(offset>>32), uint32(offset), uintptr(size))
if a == 0 {
windows.CloseHandle(h)
return nil, err
}
res := &MappedRegion{Handle: h, addr: a}
// SliceHeader, although deprecated, avoids a go vet warning.
sh := (*reflect.SliceHeader)(unsafe.Pointer(&res.Data))
sh.Len = int(size)
sh.Cap = int(size)
sh.Data = a
return res, nil
}
func (r *MappedRegion) Unmap() error {
if r.Data == nil {
return nil
}
err := windows.UnmapViewOfFile(r.addr)
if err != nil {
return err
}
r.Data = nil
return windows.CloseHandle(r.Handle)
}

View File

@@ -2,7 +2,7 @@
# handle, and interrupt, sqlite3_busy_timeout.
--- sqlite3.c.orig
+++ sqlite3.c
@@ -182908,7 +182908,7 @@
@@ -182928,7 +182928,7 @@
if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
#endif
if( ms>0 ){

View File

@@ -38,6 +38,9 @@ int sqlite3_columns_go(sqlite3_stmt *stmt, int nCol, char *aType,
}
if (ptr == NULL && rc == SQLITE_OK) {
rc = sqlite3_errcode(sqlite3_db_handle(stmt));
if (rc == SQLITE_ROW || rc == SQLITE_DONE) {
rc = SQLITE_OK;
}
}
aData[i].ptr = ptr;
aData[i].len = sqlite3_column_bytes(stmt, i);

View File

@@ -3,7 +3,7 @@ set -euo pipefail
cd -P -- "$(dirname -- "$0")"
curl -#OL "https://sqlite.org/2024/sqlite-amalgamation-3470000.zip"
curl -#OL "https://sqlite.org/2024/sqlite-amalgamation-3470200.zip"
unzip -d . sqlite-amalgamation-*.zip
mv sqlite-amalgamation-*/sqlite3.c .
mv sqlite-amalgamation-*/sqlite3.h .
@@ -21,25 +21,25 @@ cat *.patch | patch --no-backup-if-mismatch
mkdir -p ext/
cd ext/
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.0/ext/misc/anycollseq.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.0/ext/misc/base64.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.0/ext/misc/decimal.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.0/ext/misc/ieee754.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.0/ext/misc/regexp.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.0/ext/misc/series.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.0/ext/misc/spellfix.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.0/ext/misc/uint.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/ext/misc/anycollseq.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/ext/misc/base64.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/ext/misc/decimal.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/ext/misc/ieee754.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/ext/misc/regexp.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/ext/misc/series.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/ext/misc/spellfix.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/ext/misc/uint.c"
cd ~-
cd ../vfs/tests/mptest/testdata/
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.0/mptest/mptest.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.0/mptest/config01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.0/mptest/config02.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.0/mptest/crash01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.0/mptest/crash02.subtest"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.0/mptest/multiwrite01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/mptest/mptest.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/mptest/config01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/mptest/config02.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/mptest/crash01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/mptest/crash02.subtest"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/mptest/multiwrite01.test"
cd ~-
cd ../vfs/tests/speedtest1/testdata/
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.0/test/speedtest1.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.47.2/test/speedtest1.c"
cd ~-

View File

@@ -1,7 +1,7 @@
# Remove VFS registration. Go handles it.
--- sqlite3.c.orig
+++ sqlite3.c
@@ -26594,7 +26594,7 @@
@@ -26603,7 +26603,7 @@
sqlite3_free(p);
return sqlite3_os_init();
}
@@ -10,7 +10,7 @@
/*
** The list of all registered VFS implementations.
*/
@@ -26691,7 +26691,7 @@
@@ -26700,7 +26700,7 @@
sqlite3_mutex_leave(mutex);
return SQLITE_OK;
}

11
stmt.go
View File

@@ -255,6 +255,7 @@ func (s *Stmt) BindText(param int, value string) error {
// BindRawText binds a []byte to the prepared statement as text.
// The leftmost SQL parameter has an index of 1.
// Binding a nil slice is the same as calling [Stmt.BindNull].
//
// https://sqlite.org/c3ref/bind_blob.html
func (s *Stmt) BindRawText(param int, value []byte) error {
@@ -581,7 +582,9 @@ func (s *Stmt) ColumnRawBlob(col int) []byte {
func (s *Stmt) columnRawBytes(col int, ptr uint32) []byte {
if ptr == 0 {
r := s.c.call("sqlite3_errcode", uint64(s.c.handle))
s.err = s.c.error(r)
if r != _ROW && r != _DONE {
s.err = s.c.error(r)
}
return nil
}
@@ -636,7 +639,7 @@ func (s *Stmt) ColumnValue(col int) Value {
// [TEXT] as string, and [BLOB] as []byte.
// Any []byte are owned by SQLite and may be invalidated by
// subsequent calls to [Stmt] methods.
func (s *Stmt) Columns(dest []any) error {
func (s *Stmt) Columns(dest ...any) error {
defer s.c.arena.mark()()
count := uint64(len(dest))
typePtr := s.c.arena.new(count)
@@ -665,6 +668,10 @@ func (s *Stmt) Columns(dest []any) error {
dest[i] = nil
default:
ptr := util.ReadUint32(s.c.mod, dataPtr+0)
if ptr == 0 {
dest[i] = []byte{}
continue
}
len := util.ReadUint32(s.c.mod, dataPtr+4)
buf := util.View(s.c.mod, ptr, uint64(len))
if types[i] == byte(TEXT) {

View File

@@ -1,4 +1,4 @@
//go:build (linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && !sqlite3_nosys
//go:build linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock || sqlite3_dotlk
package bradfitz

View File

@@ -87,9 +87,15 @@ func TestConn_ConfigLog(t *testing.T) {
db.Prepare(`SELECT * FRM sqlite_schema`)
if code != sqlite3.ExtendedErrorCode(sqlite3.ERROR) {
if code != sqlite3.ERROR.ExtendedCode() {
t.Error("want sqlite3.ERROR")
}
db.Log(sqlite3.NOTICE.ExtendedCode(), "")
if code.Code() != sqlite3.NOTICE {
t.Error("want sqlite3.NOTICE")
}
}
func TestConn_FileControl(t *testing.T) {

View File

@@ -101,6 +101,9 @@ func syscallOpen(path string, mode int, perm uint32) (fd Handle, err error) {
const _FILE_FLAG_WRITE_THROUGH = 0x80000000
attrs |= _FILE_FLAG_WRITE_THROUGH
}
if mode&O_NONBLOCK != 0 {
attrs |= FILE_FLAG_OVERLAPPED
}
return CreateFile(pathp, access, sharemode, sa, createmode, attrs, 0)
}

View File

@@ -15,24 +15,22 @@ The main differences are [file locking](#file-locking) and [WAL mode](#write-ahe
POSIX advisory locks, which SQLite uses on Unix, are
[broken by design](https://github.com/sqlite/sqlite/blob/b74eb0/src/os_unix.c#L1073-L1161).
On Linux and macOS, this package uses
Instead, on Linux and macOS, this package uses
[OFD locks](https://www.gnu.org/software/libc/manual/html_node/Open-File-Description-Locks.html)
to synchronize access to database files.
OFD locks are fully compatible with POSIX advisory locks.
This package can also use
[BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2),
albeit with reduced concurrency (`BEGIN IMMEDIATE` behaves like `BEGIN EXCLUSIVE`).
On BSD, macOS, and illumos, BSD locks are fully compatible with POSIX advisory locks;
on Linux and z/OS, they are fully functional, but incompatible;
elsewhere, they are very likely broken.
BSD locks are the default on BSD and illumos,
but you can opt into them with the `sqlite3_flock` build tag.
On Windows, this package uses `LockFileEx` and `UnlockFileEx`,
like SQLite.
You can also opt into a cross-platform locking implementation
with the `sqlite3_dotlk` build tag.
Otherwise, file locking is not supported, and you must use
[`nolock=1`](https://sqlite.org/uri.html#urinolock)
(or [`immutable=1`](https://sqlite.org/uri.html#uriimmutable))
@@ -46,7 +44,7 @@ to check if your build supports file locking.
### Write-Ahead Logging
On little-endian Unix, this package uses `mmap` to implement
On Unix, this package may use `mmap` to implement
[shared-memory for the WAL-index](https://sqlite.org/wal.html#implementation_of_shared_memory_for_the_wal_index),
like SQLite.
@@ -55,6 +53,11 @@ a WAL database can only be accessed by a single proccess.
Other processes that attempt to access a database locked with BSD locks,
will fail with the [`SQLITE_PROTOCOL`](https://sqlite.org/rescode.html#protocol) error code.
On Windows, this package may use `MapViewOfFile`, like SQLite.
You can also opt into a cross-platform, in-process, memory sharing implementation
with the `sqlite3_dotlk` build tag.
Otherwise, [WAL support is limited](https://sqlite.org/wal.html#noshm),
and `EXCLUSIVE` locking mode must be set to create, read, and write WAL databases.
To use `EXCLUSIVE` locking mode with the
@@ -67,7 +70,7 @@ to check if your build supports shared memory.
### Batch-Atomic Write
On 64-bit Linux, this package supports
On Linux, this package may support
[batch-atomic writes](https://sqlite.org/cgi/src/technote/714)
on the F2FS filesystem.
@@ -86,27 +89,26 @@ The implementation is compatible with SQLite's
### Build Tags
The VFS can be customized with a few build tags:
- `sqlite3_flock` forces the use of BSD locks; it can be used on z/OS to enable locking,
and elsewhere to test BSD locks.
- `sqlite3_nosys` prevents importing [`x/sys`](https://pkg.go.dev/golang.org/x/sys);
disables locking _and_ shared memory on all platforms.
- `sqlite3_noshm` disables shared memory on all platforms.
- `sqlite3_flock` forces the use of BSD locks.
- `sqlite3_dotlk` forces the use of dot-file locks.
> [!IMPORTANT]
> The default configuration of this package is compatible with the standard
> [Unix and Windows SQLite VFSes](https://sqlite.org/vfs.html#multiple_vfses);
> `sqlite3_flock` builds are compatible with the
> [`unix-flock` VFS](https://sqlite.org/compile.html#enable_locking_style).
> [`unix-flock` VFS](https://sqlite.org/compile.html#enable_locking_style);
> `sqlite3_dotlk` builds are compatible with the
> [`unix-dotfile` VFS](https://sqlite.org/compile.html#enable_locking_style).
> If incompatible file locking is used, accessing databases concurrently with
> _other_ SQLite libraries will eventually corrupt data.
### Custom VFSes
- [`github.com/ncruces/go-sqlite3/vfs/adiantum`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/adiantum)
wraps a VFS to offer encryption at rest.
- [`github.com/ncruces/go-sqlite3/vfs/memdb`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/memdb)
implements an in-memory VFS.
- [`github.com/ncruces/go-sqlite3/vfs/readervfs`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/readervfs)
implements a VFS for immutable databases.
- [`github.com/ncruces/go-sqlite3/vfs/adiantum`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/adiantum)
wraps a VFS to offer encryption at rest.
- [`github.com/ncruces/go-sqlite3/vfs/xts`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/xts)
wraps a VFS to offer encryption at rest.
wraps a VFS to offer encryption at rest.

View File

@@ -1,4 +1,4 @@
//go:build (linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && !sqlite3_nosys
//go:build linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock || sqlite3_dotlk
package adiantum_test

View File

@@ -198,11 +198,13 @@ func (h *hbshFile) WriteAt(p []byte, off int64) (n int, err error) {
func (h *hbshFile) DeviceCharacteristics() vfs.DeviceCharacteristic {
var _ [0]struct{} = [blockSize - 4096]struct{}{} // Ensure blockSize is 4K.
return h.File.DeviceCharacteristics() & (0 |
// The only safe flags are these:
// These flags are safe:
vfs.IOCAP_ATOMIC4K |
vfs.IOCAP_UNDELETABLE_WHEN_OPEN |
vfs.IOCAP_IMMUTABLE |
vfs.IOCAP_BATCH_ATOMIC)
vfs.IOCAP_SEQUENTIAL |
vfs.IOCAP_SUBPAGE_READ |
vfs.IOCAP_BATCH_ATOMIC |
vfs.IOCAP_UNDELETABLE_WHEN_OPEN)
}
func (h *hbshFile) SectorSize() int {

View File

@@ -101,6 +101,14 @@ func (c cksmFile) Pragma(name string, value string) (string, error) {
return "", _NOTFOUND
}
func (c cksmFile) DeviceCharacteristics() DeviceCharacteristic {
res := c.File.DeviceCharacteristics()
if c.verifyCksm {
res &^= IOCAP_SUBPAGE_READ
}
return res
}
func (c cksmFile) fileControl(ctx context.Context, mod api.Module, op _FcntlOpcode, pArg uint32) _ErrorCode {
switch op {
case _FCNTL_CKPT_START:

View File

@@ -177,6 +177,7 @@ const (
IOCAP_POWERSAFE_OVERWRITE DeviceCharacteristic = 0x00001000
IOCAP_IMMUTABLE DeviceCharacteristic = 0x00002000
IOCAP_BATCH_ATOMIC DeviceCharacteristic = 0x00004000
IOCAP_SUBPAGE_READ DeviceCharacteristic = 0x00008000
)
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html
@@ -234,4 +235,8 @@ const (
_SHM_LOCK _ShmFlag = 2
_SHM_SHARED _ShmFlag = 4
_SHM_EXCLUSIVE _ShmFlag = 8
_SHM_NLOCK = 8
_SHM_BASE = 120
_SHM_DMS = _SHM_BASE + _SHM_NLOCK
)

View File

@@ -35,10 +35,10 @@ func testSymlinks(path string) error {
func (vfsOS) Delete(path string, syncDir bool) error {
err := os.Remove(path)
if errors.Is(err, fs.ErrNotExist) {
return _IOERR_DELETE_NOENT
}
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return _IOERR_DELETE_NOENT
}
return err
}
if runtime.GOOS != "windows" && syncDir {
@@ -151,6 +151,7 @@ func (f *vfsFile) Close() error {
if f.shm != nil {
f.shm.Close()
}
f.Unlock(LOCK_NONE)
return f.File.Close()
}
@@ -186,7 +187,7 @@ func (f *vfsFile) SectorSize() int {
}
func (f *vfsFile) DeviceCharacteristics() DeviceCharacteristic {
var res DeviceCharacteristic
res := IOCAP_SUBPAGE_READ
if osBatchAtomic(f.File) {
res |= IOCAP_BATCH_ATOMIC
}
@@ -206,10 +207,10 @@ func (f *vfsFile) HasMoved() (bool, error) {
return false, err
}
pi, err := os.Stat(f.Name())
if errors.Is(err, fs.ErrNotExist) {
return true, nil
}
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return true, nil
}
return false, err
}
return !os.SameFile(fi, pi), nil

View File

@@ -1,4 +1,4 @@
//go:build (linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && !sqlite3_nosys
//go:build linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock || sqlite3_dotlk
package vfs

View File

@@ -1,4 +1,4 @@
//go:build !(linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || sqlite3_nosys
//go:build !(linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock || sqlite3_dotlk)
package vfs

View File

@@ -78,19 +78,15 @@ type memDB struct {
// +checklocks:dataMtx
data []*[sectorSize]byte
// +checklocks:dataMtx
size int64
// +checklocks:lockMtx
shared int32
// +checklocks:lockMtx
reserved bool
// +checklocks:lockMtx
pending bool
// +checklocks:memoryMtx
refs int
refs int32
shared int32 // +checklocks:lockMtx
pending bool // +checklocks:lockMtx
reserved bool // +checklocks:lockMtx
lockMtx sync.Mutex
dataMtx sync.RWMutex
@@ -253,12 +249,12 @@ func (m *memFile) Unlock(lock vfs.LockLevel) error {
m.lockMtx.Lock()
defer m.lockMtx.Unlock()
if m.pending && m.lock >= vfs.LOCK_PENDING {
m.pending = false
}
if m.reserved && m.lock >= vfs.LOCK_RESERVED {
if m.lock >= vfs.LOCK_RESERVED {
m.reserved = false
}
if m.lock >= vfs.LOCK_PENDING {
m.pending = false
}
if lock < vfs.LOCK_SHARED {
m.shared--
}

View File

@@ -1,4 +1,4 @@
//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && !sqlite3_nosys
//go:build ((freebsd || openbsd || netbsd || dragonfly || illumos) && !sqlite3_dotlk) || sqlite3_flock
package vfs
@@ -15,9 +15,15 @@ func osGetSharedLock(file *os.File) _ErrorCode {
func osGetReservedLock(file *os.File) _ErrorCode {
rc := osLock(file, unix.LOCK_EX|unix.LOCK_NB, _IOERR_LOCK)
if rc == _BUSY {
// The documentation states the lock is upgraded by releasing the previous lock,
// then acquiring the new lock.
// This is a race, so return BUSY_SNAPSHOT to ensure the transaction is aborted.
// The documentation states that a lock is upgraded by
// releasing the previous lock, then acquiring the new lock.
// Going over the source code of various BSDs, though,
// with LOCK_NB, the lock is not released,
// and EAGAIN is returned holding the shared lock.
// Still, if we're already in a transaction, we want to abort it,
// so return BUSY_SNAPSHOT here. If there's no transaction active,
// SQLite will change this back to SQLITE_BUSY,
// and invoke the busy handler if appropriate.
return _BUSY_SNAPSHOT
}
return rc
@@ -33,9 +39,11 @@ func osGetExclusiveLock(file *os.File, state *LockLevel) _ErrorCode {
func osDowngradeLock(file *os.File, _ LockLevel) _ErrorCode {
rc := osLock(file, unix.LOCK_SH|unix.LOCK_NB, _IOERR_RDLOCK)
if rc == _BUSY {
// The documentation states the lock is upgraded by releasing the previous lock,
// then acquiring the new lock.
// This is a race, so return IOERR_RDLOCK to ensure the transaction is aborted.
// The documentation states that a lock is downgraded by
// releasing the previous lock then acquiring the new lock.
// Going over the source code of various BSDs, though,
// with LOCK_SH|LOCK_NB this should never happen.
// Return IOERR_RDLOCK, as BUSY would cause an assert to fail.
return _IOERR_RDLOCK
}
return _OK
@@ -50,7 +58,10 @@ func osReleaseLock(file *os.File, _ LockLevel) _ErrorCode {
}
func osCheckReservedLock(file *os.File) (bool, _ErrorCode) {
// Test the RESERVED lock.
// Test the RESERVED lock with fcntl(F_GETLK).
// This only works on systems where fcntl and flock are compatible.
// However, SQLite only calls this while holding a shared lock,
// so the difference is immaterial.
lock, rc := osTestLock(file, _RESERVED_BYTE, 1)
return lock == unix.F_WRLCK, rc
}

View File

@@ -1,4 +1,4 @@
//go:build !(sqlite3_flock || sqlite3_nosys)
//go:build !sqlite3_flock
package vfs

144
vfs/os_dotlk.go Normal file
View File

@@ -0,0 +1,144 @@
//go:build sqlite3_dotlk
package vfs
import (
"errors"
"io/fs"
"os"
"sync"
)
var (
// +checklocks:vfsDotLocksMtx
vfsDotLocks = map[string]*vfsDotLocker{}
vfsDotLocksMtx sync.Mutex
)
type vfsDotLocker struct {
shared int // +checklocks:vfsDotLocksMtx
pending *os.File // +checklocks:vfsDotLocksMtx
reserved *os.File // +checklocks:vfsDotLocksMtx
}
func osGetSharedLock(file *os.File) _ErrorCode {
vfsDotLocksMtx.Lock()
defer vfsDotLocksMtx.Unlock()
name := file.Name()
locker := vfsDotLocks[name]
if locker == nil {
f, err := os.OpenFile(name+".lock", os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
f.Close()
if errors.Is(err, fs.ErrExist) {
return _BUSY // Another process has the lock.
}
if err != nil {
return _IOERR_LOCK
}
locker = &vfsDotLocker{}
vfsDotLocks[name] = locker
}
if locker.pending != nil {
return _BUSY
}
locker.shared++
return _OK
}
func osGetReservedLock(file *os.File) _ErrorCode {
vfsDotLocksMtx.Lock()
defer vfsDotLocksMtx.Unlock()
name := file.Name()
locker := vfsDotLocks[name]
if locker == nil {
return _IOERR_LOCK
}
if locker.reserved != nil && locker.reserved != file {
return _BUSY
}
locker.reserved = file
return _OK
}
func osGetExclusiveLock(file *os.File, _ *LockLevel) _ErrorCode {
vfsDotLocksMtx.Lock()
defer vfsDotLocksMtx.Unlock()
name := file.Name()
locker := vfsDotLocks[name]
if locker == nil {
return _IOERR_LOCK
}
if locker.pending != nil && locker.pending != file {
return _BUSY
}
locker.pending = file
if locker.shared > 1 {
return _BUSY
}
return _OK
}
func osDowngradeLock(file *os.File, _ LockLevel) _ErrorCode {
vfsDotLocksMtx.Lock()
defer vfsDotLocksMtx.Unlock()
name := file.Name()
locker := vfsDotLocks[name]
if locker == nil {
return _IOERR_UNLOCK
}
if locker.reserved == file {
locker.reserved = nil
}
if locker.pending == file {
locker.pending = nil
}
return _OK
}
func osReleaseLock(file *os.File, state LockLevel) _ErrorCode {
vfsDotLocksMtx.Lock()
defer vfsDotLocksMtx.Unlock()
name := file.Name()
locker := vfsDotLocks[name]
if locker == nil {
return _IOERR_UNLOCK
}
if locker.shared == 1 {
err := os.Remove(name + ".lock")
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return _IOERR_UNLOCK
}
delete(vfsDotLocks, name)
}
if locker.reserved == file {
locker.reserved = nil
}
if locker.pending == file {
locker.pending = nil
}
locker.shared--
return _OK
}
func osCheckReservedLock(file *os.File) (bool, _ErrorCode) {
vfsDotLocksMtx.Lock()
defer vfsDotLocksMtx.Unlock()
name := file.Name()
locker := vfsDotLocks[name]
if locker == nil {
return false, _OK
}
return locker.reserved != nil, _OK
}

View File

@@ -1,4 +1,4 @@
//go:build (amd64 || arm64 || riscv64) && !sqlite3_nosys
//go:build amd64 || arm64 || riscv64
package vfs

View File

@@ -1,4 +1,4 @@
//go:build !(sqlite3_flock || sqlite3_nosys)
//go:build !sqlite3_flock
package vfs

View File

@@ -1,4 +1,4 @@
//go:build (linux || darwin) && !(sqlite3_flock || sqlite3_nosys)
//go:build (linux || darwin) && !(sqlite3_flock || sqlite3_dotlk)
package vfs

View File

@@ -1,4 +1,4 @@
//go:build !unix || sqlite3_nosys
//go:build !unix
package vfs

View File

@@ -1,4 +1,4 @@
//go:build !(linux || darwin) || sqlite3_flock || sqlite3_nosys
//go:build !(linux || darwin) || sqlite3_flock
package vfs

View File

@@ -1,4 +1,4 @@
//go:build !linux || !(amd64 || arm64 || riscv64) || sqlite3_nosys
//go:build !linux || !(amd64 || arm64 || riscv64)
package vfs

View File

@@ -1,4 +1,4 @@
//go:build !(linux || darwin) || sqlite3_flock || sqlite3_nosys
//go:build !(linux || darwin) || sqlite3_flock
package vfs

View File

@@ -1,4 +1,4 @@
//go:build unix && !sqlite3_nosys
//go:build unix
package vfs

View File

@@ -1,4 +1,4 @@
//go:build unix && !(sqlite3_flock || sqlite3_nosys)
//go:build unix && !sqlite3_flock
package vfs

View File

@@ -1,9 +1,8 @@
//go:build !sqlite3_nosys
//go:build !sqlite3_dotlk
package vfs
import (
"math/rand"
"os"
"time"
@@ -46,18 +45,21 @@ func osGetExclusiveLock(file *os.File, state *LockLevel) _ErrorCode {
osUnlock(file, _SHARED_FIRST, _SHARED_SIZE)
// Acquire the EXCLUSIVE lock.
rc := osWriteLock(file, _SHARED_FIRST, _SHARED_SIZE, time.Millisecond)
rc := osWriteLock(file, _SHARED_FIRST, _SHARED_SIZE, 0)
if rc != _OK {
// Reacquire the SHARED lock.
osReadLock(file, _SHARED_FIRST, _SHARED_SIZE, 0)
if rc := osReadLock(file, _SHARED_FIRST, _SHARED_SIZE, 0); rc != _OK {
// notest // this should never happen
return _IOERR_RDLOCK
}
}
return rc
}
func osDowngradeLock(file *os.File, state LockLevel) _ErrorCode {
if state >= LOCK_EXCLUSIVE {
// Release the EXCLUSIVE lock.
// Release the EXCLUSIVE lock while holding the PENDING lock.
osUnlock(file, _SHARED_FIRST, _SHARED_SIZE)
// Reacquire the SHARED lock.
@@ -78,7 +80,7 @@ func osDowngradeLock(file *os.File, state LockLevel) _ErrorCode {
}
func osReleaseLock(file *os.File, state LockLevel) _ErrorCode {
// Release all locks.
// Release all locks, PENDING must be last.
if state >= LOCK_RESERVED {
osUnlock(file, _RESERVED_BYTE, 1)
}
@@ -120,29 +122,40 @@ func osLock(file *os.File, flags, start, len uint32, timeout time.Duration, def
var err error
switch {
case timeout == 0:
err = osLockEx(file, flags|windows.LOCKFILE_FAIL_IMMEDIATELY, start, len)
err = osLockEx(file, flags|windows.LOCKFILE_FAIL_IMMEDIATELY, start, len, 0)
case timeout < 0:
err = osLockEx(file, flags, start, len)
err = osLockEx(file, flags, start, len, 0)
default:
before := time.Now()
for {
err = osLockEx(file, flags|windows.LOCKFILE_FAIL_IMMEDIATELY, start, len)
if errno, _ := err.(windows.Errno); errno != windows.ERROR_LOCK_VIOLATION {
break
var event windows.Handle
event, err = windows.CreateEvent(nil, 1, 0, nil)
if err != nil {
break
}
defer windows.CloseHandle(event)
err = osLockEx(file, flags, start, len, event)
if err == windows.ERROR_IO_PENDING {
rc, serr := windows.WaitForSingleObject(event, uint32(timeout/time.Millisecond))
if rc == windows.WAIT_OBJECT_0 {
return _OK
}
if time.Since(before) > timeout {
break
if serr != nil {
err = serr
} else {
err = windows.Errno(rc)
}
const sleepIncrement = 1024*1024 - 1 // power of two, ~1ms
time.Sleep(time.Duration(rand.Int63() & sleepIncrement))
windows.CancelIo(windows.Handle(file.Fd()))
}
}
return osLockErrorCode(err, def)
}
func osLockEx(file *os.File, flags, start, len uint32) error {
func osLockEx(file *os.File, flags, start, len uint32, event windows.Handle) error {
return windows.LockFileEx(windows.Handle(file.Fd()), flags,
0, len, 0, &windows.Overlapped{Offset: start})
0, len, 0, &windows.Overlapped{
Offset: start,
HEvent: event,
})
}
func osReadLock(file *os.File, start, len uint32, timeout time.Duration) _ErrorCode {
@@ -163,7 +176,8 @@ func osLockErrorCode(err error, def _ErrorCode) _ErrorCode {
case
windows.ERROR_LOCK_VIOLATION,
windows.ERROR_IO_PENDING,
windows.ERROR_OPERATION_ABORTED:
windows.ERROR_OPERATION_ABORTED,
windows.WAIT_TIMEOUT:
return _BUSY
}
}

View File

@@ -77,5 +77,5 @@ func (readerFile) SectorSize() int {
}
func (readerFile) DeviceCharacteristics() vfs.DeviceCharacteristic {
return vfs.IOCAP_IMMUTABLE
return vfs.IOCAP_IMMUTABLE | vfs.IOCAP_SUBPAGE_READ
}

View File

@@ -1,20 +1,7 @@
//go:build (darwin || linux) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_flock || sqlite3_noshm || sqlite3_nosys)
//go:build ((linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le)) || sqlite3_flock || sqlite3_dotlk
package vfs
import (
"context"
"io"
"os"
"sync"
"time"
"github.com/tetratelabs/wazero/api"
"golang.org/x/sys/unix"
"github.com/ncruces/go-sqlite3/internal/util"
)
// SupportsSharedMemory is false on platforms that do not support shared memory.
// To use [WAL without shared-memory], you need to set [EXCLUSIVE locking mode].
//
@@ -22,12 +9,6 @@ import (
// [EXCLUSIVE locking mode]: https://sqlite.org/pragma.html#pragma_locking_mode
const SupportsSharedMemory = true
const (
_SHM_NLOCK = 8
_SHM_BASE = 120
_SHM_DMS = _SHM_BASE + _SHM_NLOCK
)
func (f *vfsFile) SharedMemory() SharedMemory { return f.shm }
// NewSharedMemory returns a shared-memory WAL-index
@@ -41,172 +22,5 @@ func NewSharedMemory(path string, flags OpenFlag) SharedMemory {
if flags&OPEN_MAIN_DB == 0 || flags&(OPEN_DELETEONCLOSE|OPEN_MEMORY) != 0 {
return nil
}
return &vfsShm{
path: path,
readOnly: flags&OPEN_READONLY != 0,
}
}
var _ blockingSharedMemory = &vfsShm{}
type vfsShm struct {
*os.File
path string
regions []*util.MappedRegion
readOnly bool
blocking bool
sync.Mutex
}
func (s *vfsShm) shmOpen() _ErrorCode {
if s.File == nil {
var flag int
if s.readOnly {
flag = unix.O_RDONLY
} else {
flag = unix.O_RDWR
}
f, err := os.OpenFile(s.path,
flag|unix.O_CREAT|unix.O_NOFOLLOW, 0666)
if err != nil {
return _CANTOPEN
}
s.File = f
}
// Dead man's switch.
if lock, rc := osTestLock(s.File, _SHM_DMS, 1); rc != _OK {
return _IOERR_LOCK
} else if lock == unix.F_WRLCK {
return _BUSY
} else if lock == unix.F_UNLCK {
if s.readOnly {
return _READONLY_CANTINIT
}
// Do not use a blocking lock here.
// If the lock cannot be obtained immediately,
// it means some other connection is truncating the file.
// And after it has done so, it will not release its lock,
// but only downgrade it to a shared lock.
// So no point in blocking here.
// The call below to obtain the shared DMS lock may use a blocking lock.
if rc := osWriteLock(s.File, _SHM_DMS, 1, 0); rc != _OK {
return rc
}
if err := s.Truncate(0); err != nil {
return _IOERR_SHMOPEN
}
}
if rc := osReadLock(s.File, _SHM_DMS, 1, time.Millisecond); rc != _OK {
return rc
}
return _OK
}
func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, _ErrorCode) {
// Ensure size is a multiple of the OS page size.
if int(size)&(unix.Getpagesize()-1) != 0 {
return 0, _IOERR_SHMMAP
}
if rc := s.shmOpen(); rc != _OK {
return 0, rc
}
// Check if file is big enough.
o, err := s.Seek(0, io.SeekEnd)
if err != nil {
return 0, _IOERR_SHMSIZE
}
if n := (int64(id) + 1) * int64(size); n > o {
if !extend {
return 0, _OK
}
err := osAllocate(s.File, n)
if err != nil {
return 0, _IOERR_SHMSIZE
}
}
var prot int
if s.readOnly {
prot = unix.PROT_READ
} else {
prot = unix.PROT_READ | unix.PROT_WRITE
}
r, err := util.MapRegion(ctx, mod, s.File, int64(id)*int64(size), size, prot)
if err != nil {
return 0, _IOERR_SHMMAP
}
s.regions = append(s.regions, r)
if s.readOnly {
return r.Ptr, _READONLY
}
return r.Ptr, _OK
}
func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) _ErrorCode {
// Argument check.
if n <= 0 || offset < 0 || offset+n > _SHM_NLOCK {
panic(util.AssertErr())
}
switch flags {
case
_SHM_LOCK | _SHM_SHARED,
_SHM_LOCK | _SHM_EXCLUSIVE,
_SHM_UNLOCK | _SHM_SHARED,
_SHM_UNLOCK | _SHM_EXCLUSIVE:
//
default:
panic(util.AssertErr())
}
if n != 1 && flags&_SHM_EXCLUSIVE == 0 {
panic(util.AssertErr())
}
var timeout time.Duration
if s.blocking {
timeout = time.Millisecond
}
switch {
case flags&_SHM_UNLOCK != 0:
return osUnlock(s.File, _SHM_BASE+int64(offset), int64(n))
case flags&_SHM_SHARED != 0:
return osReadLock(s.File, _SHM_BASE+int64(offset), int64(n), timeout)
case flags&_SHM_EXCLUSIVE != 0:
return osWriteLock(s.File, _SHM_BASE+int64(offset), int64(n), timeout)
default:
panic(util.AssertErr())
}
}
func (s *vfsShm) shmUnmap(delete bool) {
if s.File == nil {
return
}
// Unmap regions.
for _, r := range s.regions {
r.Unmap()
}
clear(s.regions)
s.regions = s.regions[:0]
// Close the file.
if delete {
os.Remove(s.path)
}
s.Close()
s.File = nil
}
func (s *vfsShm) shmBarrier() {
s.Lock()
//lint:ignore SA2001 memory barrier.
s.Unlock()
}
func (s *vfsShm) shmEnableBlocking(block bool) {
s.blocking = block
return &vfsShm{path: path}
}

View File

@@ -1,4 +1,4 @@
//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys)
//go:build ((freebsd || openbsd || netbsd || dragonfly || illumos) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !sqlite3_dotlk) || sqlite3_flock
package vfs
@@ -14,97 +14,66 @@ import (
"github.com/ncruces/go-sqlite3/internal/util"
)
// SupportsSharedMemory is false on platforms that do not support shared memory.
// To use [WAL without shared-memory], you need to set [EXCLUSIVE locking mode].
//
// [WAL without shared-memory]: https://sqlite.org/wal.html#noshm
// [EXCLUSIVE locking mode]: https://sqlite.org/pragma.html#pragma_locking_mode
const SupportsSharedMemory = true
const _SHM_NLOCK = 8
func (f *vfsFile) SharedMemory() SharedMemory { return f.shm }
// NewSharedMemory returns a shared-memory WAL-index
// backed by a file with the given path.
// It will return nil if shared-memory is not supported,
// or not appropriate for the given flags.
// Only [OPEN_MAIN_DB] databases may need a WAL-index.
// You must ensure all concurrent accesses to a database
// use shared-memory instances created with the same path.
func NewSharedMemory(path string, flags OpenFlag) SharedMemory {
if flags&OPEN_MAIN_DB == 0 || flags&(OPEN_DELETEONCLOSE|OPEN_MEMORY) != 0 {
return nil
}
return &vfsShm{
path: path,
readOnly: flags&OPEN_READONLY != 0,
}
}
type vfsShmFile struct {
type vfsShmParent struct {
*os.File
info os.FileInfo
// +checklocks:vfsShmFilesMtx
refs int
refs int // +checklocks:vfsShmListMtx
// +checklocks:lockMtx
lock [_SHM_NLOCK]int16
lockMtx sync.Mutex
lock [_SHM_NLOCK]int16 // +checklocks:Mutex
sync.Mutex
}
var (
// +checklocks:vfsShmFilesMtx
vfsShmFiles []*vfsShmFile
vfsShmFilesMtx sync.Mutex
// +checklocks:vfsShmListMtx
vfsShmList []*vfsShmParent
vfsShmListMtx sync.Mutex
)
type vfsShm struct {
*vfsShmFile
path string
lock [_SHM_NLOCK]bool
regions []*util.MappedRegion
readOnly bool
*vfsShmParent
path string
lock [_SHM_NLOCK]bool
regions []*util.MappedRegion
}
func (s *vfsShm) Close() error {
if s.vfsShmFile == nil {
if s.vfsShmParent == nil {
return nil
}
vfsShmFilesMtx.Lock()
defer vfsShmFilesMtx.Unlock()
vfsShmListMtx.Lock()
defer vfsShmListMtx.Unlock()
// Unlock everything.
s.shmLock(0, _SHM_NLOCK, _SHM_UNLOCK)
// Decrease reference count.
if s.vfsShmFile.refs > 1 {
s.vfsShmFile.refs--
s.vfsShmFile = nil
if s.vfsShmParent.refs > 0 {
s.vfsShmParent.refs--
s.vfsShmParent = nil
return nil
}
err := s.File.Close()
for i, g := range vfsShmFiles {
if g == s.vfsShmFile {
vfsShmFiles[i] = nil
s.vfsShmFile = nil
for i, g := range vfsShmList {
if g == s.vfsShmParent {
vfsShmList[i] = nil
s.vfsShmParent = nil
return err
}
}
panic(util.AssertErr())
}
func (s *vfsShm) shmOpen() (rc _ErrorCode) {
if s.vfsShmFile != nil {
func (s *vfsShm) shmOpen() _ErrorCode {
if s.vfsShmParent != nil {
return _OK
}
// Always open file read-write, as it will be shared.
f, err := os.OpenFile(s.path,
unix.O_RDWR|unix.O_CREAT|unix.O_NOFOLLOW, 0666)
os.O_RDWR|os.O_CREATE|_O_NOFOLLOW, 0666)
if err != nil {
return _CANTOPEN
}
@@ -116,46 +85,41 @@ func (s *vfsShm) shmOpen() (rc _ErrorCode) {
return _IOERR_FSTAT
}
vfsShmFilesMtx.Lock()
defer vfsShmFilesMtx.Unlock()
vfsShmListMtx.Lock()
defer vfsShmListMtx.Unlock()
// Find a shared file, increase the reference count.
for _, g := range vfsShmFiles {
for _, g := range vfsShmList {
if g != nil && os.SameFile(fi, g.info) {
s.vfsShmFile = g
s.vfsShmParent = g
g.refs++
return _OK
}
}
// Lock and truncate the file, if not readonly.
// Lock and truncate the file.
// The lock is only released by closing the file.
if s.readOnly {
rc = _READONLY_CANTINIT
} else {
if rc := osLock(f, unix.LOCK_EX|unix.LOCK_NB, _IOERR_LOCK); rc != _OK {
return rc
}
if err := f.Truncate(0); err != nil {
return _IOERR_SHMOPEN
}
if rc := osLock(f, unix.LOCK_EX|unix.LOCK_NB, _IOERR_LOCK); rc != _OK {
return rc
}
if err := f.Truncate(0); err != nil {
return _IOERR_SHMOPEN
}
// Add the new shared file.
s.vfsShmFile = &vfsShmFile{
s.vfsShmParent = &vfsShmParent{
File: f,
info: fi,
refs: 1,
}
f = nil // Don't close the file.
for i, g := range vfsShmFiles {
for i, g := range vfsShmList {
if g == nil {
vfsShmFiles[i] = s.vfsShmFile
return rc
vfsShmList[i] = s.vfsShmParent
return _OK
}
}
vfsShmFiles = append(vfsShmFiles, s.vfsShmFile)
return rc
vfsShmList = append(vfsShmList, s.vfsShmParent)
return _OK
}
func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, _ErrorCode) {
@@ -177,83 +141,27 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext
if !extend {
return 0, _OK
}
err := osAllocate(s.File, n)
if err != nil {
if osAllocate(s.File, n) != nil {
return 0, _IOERR_SHMSIZE
}
}
var prot int
if s.readOnly {
prot = unix.PROT_READ
} else {
prot = unix.PROT_READ | unix.PROT_WRITE
}
r, err := util.MapRegion(ctx, mod, s.File, int64(id)*int64(size), size, prot)
r, err := util.MapRegion(ctx, mod, s.File, int64(id)*int64(size), size, false)
if err != nil {
return 0, _IOERR_SHMMAP
}
s.regions = append(s.regions, r)
if s.readOnly {
return r.Ptr, _READONLY
}
return r.Ptr, _OK
}
func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) _ErrorCode {
s.lockMtx.Lock()
defer s.lockMtx.Unlock()
switch {
case flags&_SHM_UNLOCK != 0:
for i := offset; i < offset+n; i++ {
if s.lock[i] {
if s.vfsShmFile.lock[i] == 0 {
panic(util.AssertErr())
}
if s.vfsShmFile.lock[i] <= 0 {
s.vfsShmFile.lock[i] = 0
} else {
s.vfsShmFile.lock[i]--
}
s.lock[i] = false
}
}
case flags&_SHM_SHARED != 0:
for i := offset; i < offset+n; i++ {
if s.lock[i] {
panic(util.AssertErr())
}
if s.vfsShmFile.lock[i] < 0 {
return _BUSY
}
}
for i := offset; i < offset+n; i++ {
s.vfsShmFile.lock[i]++
s.lock[i] = true
}
case flags&_SHM_EXCLUSIVE != 0:
for i := offset; i < offset+n; i++ {
if s.lock[i] {
panic(util.AssertErr())
}
if s.vfsShmFile.lock[i] != 0 {
return _BUSY
}
}
for i := offset; i < offset+n; i++ {
s.vfsShmFile.lock[i] = -1
s.lock[i] = true
}
default:
panic(util.AssertErr())
}
return _OK
s.Lock()
defer s.Unlock()
return s.shmMemLock(offset, n, flags)
}
func (s *vfsShm) shmUnmap(delete bool) {
if s.vfsShmFile == nil {
if s.vfsShmParent == nil {
return
}
@@ -261,8 +169,7 @@ func (s *vfsShm) shmUnmap(delete bool) {
for _, r := range s.regions {
r.Unmap()
}
clear(s.regions)
s.regions = s.regions[:0]
s.regions = nil
// Close the file.
if delete {
@@ -272,7 +179,7 @@ func (s *vfsShm) shmUnmap(delete bool) {
}
func (s *vfsShm) shmBarrier() {
s.lockMtx.Lock()
s.Lock()
//lint:ignore SA2001 memory barrier.
s.lockMtx.Unlock()
s.Unlock()
}

87
vfs/shm_copy.go Normal file
View File

@@ -0,0 +1,87 @@
//go:build (windows && (386 || arm || amd64 || arm64 || riscv64 || ppc64le)) || sqlite3_dotlk
package vfs
import (
"unsafe"
"github.com/ncruces/go-sqlite3/internal/util"
)
const (
_WALINDEX_HDR_SIZE = 136
_WALINDEX_PGSZ = 32768
)
// This seems a safe way of keeping the WAL-index in sync.
//
// The WAL-index file starts with a header,
// and the index doesn't meaningfully change if the header doesn't change.
//
// The header starts with two 48 byte, checksummed, copies of the same information,
// which are accessed independently between memory barriers.
// The checkpoint information that follows uses 4 byte aligned words.
//
// Finally, we have the WAL-index hash tables,
// which are only modified holding the exclusive WAL_WRITE_LOCK.
//
// Since all the data is either redundant+checksummed,
// 4 byte aligned, or modified under an exclusive lock,
// the copies below should correctly keep memory in sync.
//
// https://sqlite.org/walformat.html#the_wal_index_file_format
func (s *vfsShm) shmAcquire(ptr *_ErrorCode) {
if ptr != nil && *ptr != _OK {
return
}
if len(s.ptrs) == 0 || shmEqual(s.shadow[0][:], s.shared[0][:]) {
return
}
// Copies modified words from shared to private memory.
for id, p := range s.ptrs {
shared := shmPage(s.shared[id][:])
shadow := shmPage(s.shadow[id][:])
privat := shmPage(util.View(s.mod, p, _WALINDEX_PGSZ))
for i, shared := range shared {
if shadow[i] != shared {
shadow[i] = shared
privat[i] = shared
}
}
}
}
func (s *vfsShm) shmRelease() {
if len(s.ptrs) == 0 || shmEqual(s.shadow[0][:], util.View(s.mod, s.ptrs[0], _WALINDEX_HDR_SIZE)) {
return
}
// Copies modified words from private to shared memory.
for id, p := range s.ptrs {
shared := shmPage(s.shared[id][:])
shadow := shmPage(s.shadow[id][:])
privat := shmPage(util.View(s.mod, p, _WALINDEX_PGSZ))
for i, privat := range privat {
if shadow[i] != privat {
shadow[i] = privat
shared[i] = privat
}
}
}
}
func (s *vfsShm) shmBarrier() {
s.Lock()
s.shmAcquire(nil)
s.shmRelease()
s.Unlock()
}
func shmPage(s []byte) *[_WALINDEX_PGSZ / 4]uint32 {
p := (*uint32)(unsafe.Pointer(unsafe.SliceData(s)))
return (*[_WALINDEX_PGSZ / 4]uint32)(unsafe.Slice(p, _WALINDEX_PGSZ/4))
}
func shmEqual(v1, v2 []byte) bool {
return *(*[_WALINDEX_HDR_SIZE]byte)(v1[:]) == *(*[_WALINDEX_HDR_SIZE]byte)(v2[:])
}

180
vfs/shm_dotlk.go Normal file
View File

@@ -0,0 +1,180 @@
//go:build sqlite3_dotlk
package vfs
import (
"context"
"errors"
"io/fs"
"os"
"sync"
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3/internal/util"
)
type vfsShmParent struct {
shared [][_WALINDEX_PGSZ]byte
refs int // +checklocks:vfsShmListMtx
lock [_SHM_NLOCK]int16 // +checklocks:Mutex
sync.Mutex
}
var (
// +checklocks:vfsShmListMtx
vfsShmList = map[string]*vfsShmParent{}
vfsShmListMtx sync.Mutex
)
type vfsShm struct {
*vfsShmParent
mod api.Module
alloc api.Function
free api.Function
path string
shadow [][_WALINDEX_PGSZ]byte
ptrs []uint32
stack [1]uint64
lock [_SHM_NLOCK]bool
}
func (s *vfsShm) Close() error {
if s.vfsShmParent == nil {
return nil
}
vfsShmListMtx.Lock()
defer vfsShmListMtx.Unlock()
// Unlock everything.
s.shmLock(0, _SHM_NLOCK, _SHM_UNLOCK)
// Decrease reference count.
if s.vfsShmParent.refs > 0 {
s.vfsShmParent.refs--
s.vfsShmParent = nil
return nil
}
err := os.Remove(s.path)
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return _IOERR_UNLOCK
}
delete(vfsShmList, s.path)
s.vfsShmParent = nil
return nil
}
func (s *vfsShm) shmOpen() _ErrorCode {
if s.vfsShmParent != nil {
return _OK
}
vfsShmListMtx.Lock()
defer vfsShmListMtx.Unlock()
// Find a shared buffer, increase the reference count.
if g, ok := vfsShmList[s.path]; ok {
s.vfsShmParent = g
g.refs++
return _OK
}
// Create a directory on disk to ensure only this process
// uses this path to register a shared memory.
err := os.Mkdir(s.path, 0777)
if errors.Is(err, fs.ErrExist) {
return _BUSY
}
if err != nil {
return _IOERR_LOCK
}
// Add the new shared buffer.
s.vfsShmParent = &vfsShmParent{}
vfsShmList[s.path] = s.vfsShmParent
return _OK
}
func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, _ErrorCode) {
if size != _WALINDEX_PGSZ {
return 0, _IOERR_SHMMAP
}
if s.mod == nil {
s.mod = mod
s.free = mod.ExportedFunction("sqlite3_free")
s.alloc = mod.ExportedFunction("sqlite3_malloc64")
}
if rc := s.shmOpen(); rc != _OK {
return 0, rc
}
s.Lock()
defer s.Unlock()
defer s.shmAcquire(nil)
// Extend shared memory.
if int(id) >= len(s.shared) {
if !extend {
return 0, _OK
}
s.shared = append(s.shared, make([][_WALINDEX_PGSZ]byte, int(id)-len(s.shared)+1)...)
}
// Allocate shadow memory.
if int(id) >= len(s.shadow) {
s.shadow = append(s.shadow, make([][_WALINDEX_PGSZ]byte, int(id)-len(s.shadow)+1)...)
}
// Allocate local memory.
for int(id) >= len(s.ptrs) {
s.stack[0] = uint64(size)
if err := s.alloc.CallWithStack(ctx, s.stack[:]); err != nil {
panic(err)
}
if s.stack[0] == 0 {
panic(util.OOMErr)
}
clear(util.View(s.mod, uint32(s.stack[0]), _WALINDEX_PGSZ))
s.ptrs = append(s.ptrs, uint32(s.stack[0]))
}
s.shadow[0][4] = 1
return s.ptrs[id], _OK
}
func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) (rc _ErrorCode) {
s.Lock()
defer s.Unlock()
switch {
case flags&_SHM_LOCK != 0:
defer s.shmAcquire(&rc)
case flags&_SHM_EXCLUSIVE != 0:
s.shmRelease()
}
return s.shmMemLock(offset, n, flags)
}
func (s *vfsShm) shmUnmap(delete bool) {
if s.vfsShmParent == nil {
return
}
defer s.Close()
s.Lock()
s.shmRelease()
defer s.Unlock()
for _, p := range s.ptrs {
s.stack[0] = uint64(p)
if err := s.free.CallWithStack(context.Background(), s.stack[:]); err != nil {
panic(err)
}
}
s.ptrs = nil
s.shadow = nil
}

55
vfs/shm_memlk.go Normal file
View File

@@ -0,0 +1,55 @@
//go:build ((freebsd || openbsd || netbsd || dragonfly || illumos) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le)) || sqlite3_flock || sqlite3_dotlk
package vfs
import "github.com/ncruces/go-sqlite3/internal/util"
// +checklocks:s.Mutex
func (s *vfsShm) shmMemLock(offset, n int32, flags _ShmFlag) _ErrorCode {
switch {
case flags&_SHM_UNLOCK != 0:
for i := offset; i < offset+n; i++ {
if s.lock[i] {
if s.vfsShmParent.lock[i] == 0 {
panic(util.AssertErr())
}
if s.vfsShmParent.lock[i] <= 0 {
s.vfsShmParent.lock[i] = 0
} else {
s.vfsShmParent.lock[i]--
}
s.lock[i] = false
}
}
case flags&_SHM_SHARED != 0:
for i := offset; i < offset+n; i++ {
if s.lock[i] {
panic(util.AssertErr())
}
if s.vfsShmParent.lock[i]+1 <= 0 {
return _BUSY
}
}
for i := offset; i < offset+n; i++ {
s.vfsShmParent.lock[i]++
s.lock[i] = true
}
case flags&_SHM_EXCLUSIVE != 0:
for i := offset; i < offset+n; i++ {
if s.lock[i] {
panic(util.AssertErr())
}
if s.vfsShmParent.lock[i] != 0 {
return _BUSY
}
}
for i := offset; i < offset+n; i++ {
s.vfsShmParent.lock[i] = -1
s.lock[i] = true
}
default:
panic(util.AssertErr())
}
return _OK
}

174
vfs/shm_ofd.go Normal file
View File

@@ -0,0 +1,174 @@
//go:build (linux || darwin) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_flock || sqlite3_dotlk)
package vfs
import (
"context"
"io"
"os"
"sync"
"time"
"github.com/tetratelabs/wazero/api"
"golang.org/x/sys/unix"
"github.com/ncruces/go-sqlite3/internal/util"
)
type vfsShm struct {
*os.File
path string
regions []*util.MappedRegion
readOnly bool
fileLock bool
blocking bool
sync.Mutex
}
var _ blockingSharedMemory = &vfsShm{}
func (s *vfsShm) shmOpen() _ErrorCode {
if s.File == nil {
f, err := os.OpenFile(s.path,
os.O_RDWR|os.O_CREATE|_O_NOFOLLOW, 0666)
if err != nil {
f, err = os.OpenFile(s.path,
os.O_RDONLY|os.O_CREATE|_O_NOFOLLOW, 0666)
s.readOnly = true
}
if err != nil {
return _CANTOPEN
}
s.File = f
}
if s.fileLock {
return _OK
}
// Dead man's switch.
if lock, rc := osTestLock(s.File, _SHM_DMS, 1); rc != _OK {
return _IOERR_LOCK
} else if lock == unix.F_WRLCK {
return _BUSY
} else if lock == unix.F_UNLCK {
if s.readOnly {
return _READONLY_CANTINIT
}
// Do not use a blocking lock here.
// If the lock cannot be obtained immediately,
// it means some other connection is truncating the file.
// And after it has done so, it will not release its lock,
// but only downgrade it to a shared lock.
// So no point in blocking here.
// The call below to obtain the shared DMS lock may use a blocking lock.
if rc := osWriteLock(s.File, _SHM_DMS, 1, 0); rc != _OK {
return rc
}
if err := s.Truncate(0); err != nil {
return _IOERR_SHMOPEN
}
}
rc := osReadLock(s.File, _SHM_DMS, 1, time.Millisecond)
s.fileLock = rc == _OK
return rc
}
func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, _ErrorCode) {
// Ensure size is a multiple of the OS page size.
if int(size)&(unix.Getpagesize()-1) != 0 {
return 0, _IOERR_SHMMAP
}
if rc := s.shmOpen(); rc != _OK {
return 0, rc
}
// Check if file is big enough.
o, err := s.Seek(0, io.SeekEnd)
if err != nil {
return 0, _IOERR_SHMSIZE
}
if n := (int64(id) + 1) * int64(size); n > o {
if !extend {
return 0, _OK
}
if s.readOnly || osAllocate(s.File, n) != nil {
return 0, _IOERR_SHMSIZE
}
}
r, err := util.MapRegion(ctx, mod, s.File, int64(id)*int64(size), size, s.readOnly)
if err != nil {
return 0, _IOERR_SHMMAP
}
s.regions = append(s.regions, r)
if s.readOnly {
return r.Ptr, _READONLY
}
return r.Ptr, _OK
}
func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) _ErrorCode {
// Argument check.
if n <= 0 || offset < 0 || offset+n > _SHM_NLOCK {
panic(util.AssertErr())
}
switch flags {
case
_SHM_LOCK | _SHM_SHARED,
_SHM_LOCK | _SHM_EXCLUSIVE,
_SHM_UNLOCK | _SHM_SHARED,
_SHM_UNLOCK | _SHM_EXCLUSIVE:
//
default:
panic(util.AssertErr())
}
if n != 1 && flags&_SHM_EXCLUSIVE == 0 {
panic(util.AssertErr())
}
var timeout time.Duration
if s.blocking {
timeout = time.Millisecond
}
switch {
case flags&_SHM_UNLOCK != 0:
return osUnlock(s.File, _SHM_BASE+int64(offset), int64(n))
case flags&_SHM_SHARED != 0:
return osReadLock(s.File, _SHM_BASE+int64(offset), int64(n), timeout)
case flags&_SHM_EXCLUSIVE != 0:
return osWriteLock(s.File, _SHM_BASE+int64(offset), int64(n), timeout)
default:
panic(util.AssertErr())
}
}
func (s *vfsShm) shmUnmap(delete bool) {
if s.File == nil {
return
}
// Unmap regions.
for _, r := range s.regions {
r.Unmap()
}
s.regions = nil
// Close the file.
if delete {
os.Remove(s.path)
}
s.Close()
s.File = nil
}
func (s *vfsShm) shmBarrier() {
s.Lock()
//lint:ignore SA2001 memory barrier.
s.Unlock()
}
func (s *vfsShm) shmEnableBlocking(block bool) {
s.blocking = block
}

View File

@@ -1,4 +1,4 @@
//go:build !(darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || !(386 || arm || amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys
//go:build !(((linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le)) || sqlite3_flock || sqlite3_dotlk)
package vfs

190
vfs/shm_windows.go Normal file
View File

@@ -0,0 +1,190 @@
//go:build (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !sqlite3_dotlk
package vfs
import (
"context"
"io"
"os"
"sync"
"syscall"
"time"
"github.com/tetratelabs/wazero/api"
"golang.org/x/sys/windows"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/util/osutil"
)
type vfsShm struct {
*os.File
mod api.Module
alloc api.Function
free api.Function
path string
regions []*util.MappedRegion
shared [][]byte
shadow [][_WALINDEX_PGSZ]byte
ptrs []uint32
stack [1]uint64
fileLock bool
blocking bool
sync.Mutex
}
var _ blockingSharedMemory = &vfsShm{}
func (s *vfsShm) Close() error {
// Unmap regions.
for _, r := range s.regions {
r.Unmap()
}
s.regions = nil
// Close the file.
return s.File.Close()
}
func (s *vfsShm) shmOpen() _ErrorCode {
if s.File == nil {
f, err := osutil.OpenFile(s.path,
os.O_RDWR|os.O_CREATE|syscall.O_NONBLOCK, 0666)
if err != nil {
return _CANTOPEN
}
s.File = f
}
if s.fileLock {
return _OK
}
// Dead man's switch.
if rc := osWriteLock(s.File, _SHM_DMS, 1, 0); rc == _OK {
err := s.Truncate(0)
osUnlock(s.File, _SHM_DMS, 1)
if err != nil {
return _IOERR_SHMOPEN
}
}
rc := osReadLock(s.File, _SHM_DMS, 1, time.Millisecond)
s.fileLock = rc == _OK
return rc
}
func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (_ uint32, rc _ErrorCode) {
// Ensure size is a multiple of the OS page size.
if size != _WALINDEX_PGSZ || (windows.Getpagesize()-1)&_WALINDEX_PGSZ != 0 {
return 0, _IOERR_SHMMAP
}
if s.mod == nil {
s.mod = mod
s.free = mod.ExportedFunction("sqlite3_free")
s.alloc = mod.ExportedFunction("sqlite3_malloc64")
}
if rc := s.shmOpen(); rc != _OK {
return 0, rc
}
defer s.shmAcquire(&rc)
// Check if file is big enough.
o, err := s.Seek(0, io.SeekEnd)
if err != nil {
return 0, _IOERR_SHMSIZE
}
if n := (int64(id) + 1) * int64(size); n > o {
if !extend {
return 0, _OK
}
if osAllocate(s.File, n) != nil {
return 0, _IOERR_SHMSIZE
}
}
// Maps regions into memory.
for int(id) >= len(s.shared) {
r, err := util.MapRegion(ctx, mod, s.File, int64(id)*int64(size), size)
if err != nil {
return 0, _IOERR_SHMMAP
}
s.regions = append(s.regions, r)
s.shared = append(s.shared, r.Data)
}
// Allocate shadow memory.
if int(id) >= len(s.shadow) {
s.shadow = append(s.shadow, make([][_WALINDEX_PGSZ]byte, int(id)-len(s.shadow)+1)...)
}
// Allocate local memory.
for int(id) >= len(s.ptrs) {
s.stack[0] = uint64(size)
if err := s.alloc.CallWithStack(ctx, s.stack[:]); err != nil {
panic(err)
}
if s.stack[0] == 0 {
panic(util.OOMErr)
}
clear(util.View(s.mod, uint32(s.stack[0]), _WALINDEX_PGSZ))
s.ptrs = append(s.ptrs, uint32(s.stack[0]))
}
s.shadow[0][4] = 1
return s.ptrs[id], _OK
}
func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) (rc _ErrorCode) {
var timeout time.Duration
if s.blocking {
timeout = time.Millisecond
}
switch {
case flags&_SHM_LOCK != 0:
defer s.shmAcquire(&rc)
case flags&_SHM_EXCLUSIVE != 0:
s.shmRelease()
}
switch {
case flags&_SHM_UNLOCK != 0:
return osUnlock(s.File, _SHM_BASE+uint32(offset), uint32(n))
case flags&_SHM_SHARED != 0:
return osReadLock(s.File, _SHM_BASE+uint32(offset), uint32(n), timeout)
case flags&_SHM_EXCLUSIVE != 0:
return osWriteLock(s.File, _SHM_BASE+uint32(offset), uint32(n), timeout)
default:
panic(util.AssertErr())
}
}
func (s *vfsShm) shmUnmap(delete bool) {
if s.File == nil {
return
}
s.shmRelease()
// Free local memory.
for _, p := range s.ptrs {
s.stack[0] = uint64(p)
if err := s.free.CallWithStack(context.Background(), s.stack[:]); err != nil {
panic(err)
}
}
s.ptrs = nil
s.shadow = nil
s.shared = nil
// Close the file.
s.Close()
s.File = nil
if delete {
os.Remove(s.path)
}
}
func (s *vfsShm) shmEnableBlocking(block bool) {
s.blocking = block
}

View File

@@ -35,6 +35,8 @@ var compressed string
//go:embed testdata/*.*test
var scripts embed.FS
const qemuCI = runtime.GOARCH != "386" && runtime.GOARCH != "amd64" && runtime.GOARCH != "arm64"
var (
rt wazero.Runtime
module wazero.CompiledModule
@@ -145,9 +147,6 @@ func Test_crash01(t *testing.T) {
if testing.Short() {
t.Skip("skipping in short mode")
}
if os.Getenv("CI") != "" {
t.Skip("skipping in CI")
}
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
}
@@ -163,8 +162,8 @@ func Test_crash01(t *testing.T) {
}
func Test_multiwrite01(t *testing.T) {
if testing.Short() {
t.Skip("skipping in short mode")
if os.Getenv("CI") != "" && qemuCI {
t.Skip("skipping in CI")
}
if !vfs.SupportsFileLocking {
t.Skip("skipping without locks")
@@ -193,8 +192,8 @@ func Test_config01_memory(t *testing.T) {
}
func Test_multiwrite01_memory(t *testing.T) {
if testing.Short() {
t.Skip("skipping in short mode")
if os.Getenv("CI") != "" && qemuCI {
t.Skip("skipping in CI")
}
memdb.Create("test.db", nil)
@@ -212,9 +211,6 @@ func Test_crash01_wal(t *testing.T) {
if testing.Short() {
t.Skip("skipping in short mode")
}
if os.Getenv("CI") != "" {
t.Skip("skipping in CI")
}
if !vfs.SupportsSharedMemory {
t.Skip("skipping without shared memory")
}
@@ -231,8 +227,8 @@ func Test_crash01_wal(t *testing.T) {
}
func Test_multiwrite01_wal(t *testing.T) {
if testing.Short() {
t.Skip("skipping in short mode")
if os.Getenv("CI") != "" && qemuCI {
t.Skip("skipping in CI")
}
if !vfs.SupportsSharedMemory {
t.Skip("skipping without shared memory")

View File

@@ -1,5 +1,5 @@
aligned_alloc
free
malloc
sqlite3_database_file_object
sqlite3_free
sqlite3_malloc64
sqlite3_uri_key

View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:5c69a9b1c88bc735797d81f9ba979a2683f66f8b2fe88106dcbd1cdb0bb26043
size 480731
oid sha256:da52d62d7ccd9d203eaa6d281a8a61786f99b772b6de3c31c633e6a480e80df9
size 480698

View File

@@ -1,5 +1,5 @@
aligned_alloc
free
malloc
sqlite3_database_file_object
sqlite3_free
sqlite3_malloc64
sqlite3_uri_key

View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:a2f414d7540e31733b6c2f1cbfb2f8d39506215487a1b73aa03f80d8a2fe371a
size 493518
oid sha256:1b6dfeb087f4463d8b2e5904721541908436bdcc1f1acf2bf5674bd697e2960f
size 493536

View File

@@ -194,11 +194,13 @@ func (x *xtsFile) WriteAt(p []byte, off int64) (n int, err error) {
func (x *xtsFile) DeviceCharacteristics() vfs.DeviceCharacteristic {
var _ [0]struct{} = [sectorSize - 512]struct{}{} // Ensure sectorSize is 512.
return x.File.DeviceCharacteristics() & (0 |
// The only safe flags are these:
// These flags are safe:
vfs.IOCAP_ATOMIC512 |
vfs.IOCAP_UNDELETABLE_WHEN_OPEN |
vfs.IOCAP_IMMUTABLE |
vfs.IOCAP_BATCH_ATOMIC)
vfs.IOCAP_SEQUENTIAL |
vfs.IOCAP_SUBPAGE_READ |
vfs.IOCAP_BATCH_ATOMIC |
vfs.IOCAP_UNDELETABLE_WHEN_OPEN)
}
func (x *xtsFile) SectorSize() int {