Compare commits

..

1 Commits
main ... dotlk

Author SHA1 Message Date
Nuno Cruces
e6b2d2aef5 Bulk copies. 2025-11-24 13:42:12 +00:00
75 changed files with 2094 additions and 1214 deletions

View File

@@ -68,8 +68,8 @@ jobs:
shell: bash
run: |
go work init .
go work use -r embed/bcw2 gormlite
go test ./embed/bcw2 ./gormlite
go work use -r embed gormlite
go test ./embed/bcw2/...
- name: Test GORM
shell: bash
@@ -93,28 +93,25 @@ jobs:
github.event_name == 'push' &&
matrix.os == 'ubuntu-latest'
test-cross:
test-bsd:
strategy:
matrix:
os:
- name: freebsd
version: '15.0'
version: '14.3'
- name: netbsd
version: '10.1'
- name: illumos
action: omnios
version: 'r151056'
- name: openbsd
version: '7.8'
tflags: '-test.short'
- name: freebsd
arch: arm64
version: '15.0'
version: '14.3'
tflags: '-test.short'
- name: netbsd
arch: arm64
version: '10.1'
tflags: '-test.short'
- name: openbsd
version: '7.8'
tflags: '-test.short'
runs-on: ubuntu-latest
needs: test
@@ -129,9 +126,9 @@ jobs:
run: .github/workflows/build-test.sh
- name: Test
uses: cross-platform-actions/action@v0.32.0
uses: cross-platform-actions/action@v0.30.0
with:
operating_system: ${{ matrix.os.action || matrix.os.name }}
operating_system: ${{ matrix.os.name }}
architecture: ${{ matrix.os.arch }}
version: ${{ matrix.os.version }}
shell: bash
@@ -145,7 +142,7 @@ jobs:
- name: dragonfly
action: 'vmactions/dragonflybsd-vm@v1'
- name: illumos
action: 'vmactions/openindiana-vm@v0'
action: 'vmactions/omnios-vm@v1'
- name: solaris
action: 'vmactions/solaris-vm@v1'
bflags: '-tags sqlite3_dotlk'

View File

@@ -157,20 +157,16 @@ func (c *Conn) FileControl(schema string, op FcntlOpcode, arg ...any) (any, erro
stk_t(op), stk_t(ptr)))
ret = util.Read32[vfs.LockLevel](c.mod, ptr)
case FCNTL_VFSNAME, FCNTL_VFS_POINTER:
case FCNTL_VFS_POINTER:
rc = res_t(c.call("sqlite3_file_control",
stk_t(c.handle), stk_t(schemaPtr),
stk_t(FCNTL_VFS_POINTER), stk_t(ptr)))
stk_t(op), stk_t(ptr)))
if rc == _OK {
const zNameOffset = 16
ptr = util.Read32[ptr_t](c.mod, ptr)
ptr = util.Read32[ptr_t](c.mod, ptr+zNameOffset)
name := util.ReadString(c.mod, ptr, _MAX_NAME)
if op == FCNTL_VFS_POINTER {
ret = vfs.Find(name)
} else {
ret = name
}
ret = vfs.Find(name)
}
case FCNTL_FILE_POINTER, FCNTL_JOURNAL_POINTER:

View File

@@ -173,7 +173,7 @@ const (
// PrepareFlag is a flag that can be passed to [Conn.PrepareFlags].
//
// https://sqlite.org/c3ref/c_prepare_dont_log.html
// https://sqlite.org/c3ref/c_prepare_normalize.html
type PrepareFlag uint32
const (
@@ -181,7 +181,6 @@ const (
PREPARE_NORMALIZE PrepareFlag = 0x02
PREPARE_NO_VTAB PrepareFlag = 0x04
PREPARE_DONT_LOG PrepareFlag = 0x10
PREPARE_FROM_DDL PrepareFlag = 0x20
)
// FunctionFlag is a flag that can be passed to
@@ -281,7 +280,6 @@ const (
FCNTL_CHUNK_SIZE FcntlOpcode = 6
FCNTL_FILE_POINTER FcntlOpcode = 7
FCNTL_PERSIST_WAL FcntlOpcode = 10
FCNTL_VFSNAME FcntlOpcode = 12
FCNTL_POWERSAFE_OVERWRITE FcntlOpcode = 13
FCNTL_VFS_POINTER FcntlOpcode = 27
FCNTL_JOURNAL_POINTER FcntlOpcode = 28
@@ -309,7 +307,6 @@ const (
LIMIT_VARIABLE_NUMBER LimitCategory = 9
LIMIT_TRIGGER_DEPTH LimitCategory = 10
LIMIT_WORKER_THREADS LimitCategory = 11
LIMIT_PARSER_DEPTH LimitCategory = 12
)
// AuthorizerActionCode are the integer action codes

View File

@@ -440,6 +440,22 @@ func (c *conn) CheckNamedValue(arg *driver.NamedValue) error {
return nil
}
// Deprecated: for Litestream use only; may be removed at any time.
func (c *conn) FileControlPersistWAL(schema string, mode int) (int, error) {
// notest
arg := make([]any, 1)
if mode >= 0 {
arg[0] = mode > 0
} else {
arg = arg[:0]
}
res, err := c.Conn.FileControl(schema, sqlite3.FCNTL_PERSIST_WAL, arg...)
if res == true {
return 1, err
}
return 0, err
}
type stmt struct {
*sqlite3.Stmt
tmWrite sqlite3.TimeFormat
@@ -653,12 +669,14 @@ type rows struct {
names []string
types []string
scans []scantype
dest []driver.Value
}
var (
// Ensure these interfaces are implemented:
_ driver.RowsColumnTypeDatabaseTypeName = &rows{}
_ driver.RowsColumnTypeNullable = &rows{}
// _ driver.RowsColumnScanner = &rows{}
)
func (r *rows) Close() error {
@@ -698,23 +716,17 @@ func (r *rows) loadColumnMetadata() {
types := make([]string, count)
scans := make([]scantype, count)
for i := range types {
var declType string
var notNull, autoInc bool
if column := r.Stmt.ColumnOriginName(i); column != "" {
declType, _, notNull, _, autoInc, _ = c.TableColumnMetadata(
var notnull bool
if col := r.Stmt.ColumnOriginName(i); col != "" {
types[i], _, notnull, _, _, _ = c.TableColumnMetadata(
r.Stmt.ColumnDatabaseName(i),
r.Stmt.ColumnTableName(i),
column)
} else {
declType = r.Stmt.ColumnDeclType(i)
}
if declType != "" {
declType = strings.ToUpper(declType)
scans[i] = scanFromDecl(declType)
types[i] = declType
}
if notNull || autoInc {
scans[i] |= _NOT_NULL
col)
types[i] = strings.ToUpper(types[i])
scans[i] = scanFromDecl(types[i])
if notnull {
scans[i] |= _NOT_NULL
}
}
}
r.types = types
@@ -784,6 +796,7 @@ func (r *rows) ColumnTypeScanType(index int) (typ reflect.Type) {
}
func (r *rows) Next(dest []driver.Value) error {
r.dest = nil
c := r.Stmt.Conn()
if old := c.SetInterrupt(r.ctx); old != r.ctx {
defer c.SetInterrupt(old)
@@ -833,5 +846,33 @@ func (r *rows) Next(dest []driver.Value) error {
}
}
}
r.dest = dest
return nil
}
func (r *rows) ScanColumn(dest any, index int) (err error) {
// notest // Go 1.26
var tm *time.Time
var ok *bool
switch d := dest.(type) {
case *time.Time:
tm = d
case *sql.NullTime:
tm = &d.Time
ok = &d.Valid
case *sql.Null[time.Time]:
tm = &d.V
ok = &d.Valid
default:
return driver.ErrSkip
}
value := r.dest[index]
*tm, err = r.tmRead.Decode(value)
if ok != nil {
*ok = err == nil
if value == nil {
return nil
}
}
return err
}

View File

@@ -8,6 +8,7 @@ import (
"math"
"net/url"
"reflect"
"strings"
"testing"
"time"
@@ -520,6 +521,39 @@ func Test_ColumnType_ScanType(t *testing.T) {
}
}
func Test_rows_ScanColumn(t *testing.T) {
t.Parallel()
dsn := memdb.TestDB(t)
db, err := Open(dsn)
if err != nil {
t.Fatal(err)
}
defer db.Close()
var tm time.Time
err = db.QueryRow(`SELECT NULL`).Scan(&tm)
if err == nil {
t.Error("want error")
}
// Go 1.26
err = db.QueryRow(`SELECT datetime()`).Scan(&tm)
if err != nil && !strings.HasPrefix(err.Error(), "sql: Scan error") {
t.Error(err)
}
var nt sql.NullTime
err = db.QueryRow(`SELECT NULL`).Scan(&nt)
if err != nil {
t.Error(err)
}
// Go 1.26
err = db.QueryRow(`SELECT datetime()`).Scan(&nt)
if err != nil && !strings.HasPrefix(err.Error(), "sql: Scan error") {
t.Error(err)
}
}
func Benchmark_loop(b *testing.B) {
db, err := Open(":memory:")
if err != nil {

View File

@@ -1,6 +1,6 @@
# Embeddable Wasm build of SQLite
This folder includes an embeddable Wasm build of SQLite 3.51.1 for use with
This folder includes an embeddable Wasm build of SQLite 3.50.4 for use with
[`github.com/ncruces/go-sqlite3`](https://pkg.go.dev/github.com/ncruces/go-sqlite3).
The following optional features are compiled in:

Binary file not shown.

View File

@@ -15,15 +15,14 @@ cp "$ROOT"/sqlite3/*.[ch] build/
cp "$ROOT"/sqlite3/*.patch build/
cd sqlite/
# https://sqlite.org/src/info/f273f6b8245c5dca
curl -#L https://github.com/sqlite/sqlite/archive/7c126d7.tar.gz | tar xz --strip-components=1
# curl -#L https://sqlite.org/src/tarball/sqlite.tar.gz?r=f273f6b824 | tar xz --strip-components=1
# https://sqlite.org/src/info/352b363a5d727047
curl -#L https://github.com/sqlite/sqlite/archive/dbd613c.tar.gz | tar xz --strip-components=1
# curl -#L https://sqlite.org/src/tarball/sqlite.tar.gz?r=352b363a5d | tar xz --strip-components=1
if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then
MSYS_NO_PATHCONV=1 nmake /f makefile.msc sqlite3.c "OPTS=-DSQLITE_ENABLE_UPDATE_DELETE_LIMIT -DSQLITE_ENABLE_ORDERED_SET_AGGREGATES"
else
sh configure --enable-update-limit
make verify-source
OPTS=-DSQLITE_ENABLE_ORDERED_SET_AGGREGATES make sqlite3.c
fi
cd ~-
@@ -66,7 +65,7 @@ cd ~-
"$BINARYEN/wasm-ctor-eval" -g -c _initialize bcw2.wasm -o bcw2.tmp
"$BINARYEN/wasm-opt" -g bcw2.tmp -o bcw2.wasm \
--gufa --generate-global-effects --low-memory-unused --converge -O3 \
--low-memory-unused --gufa --generate-global-effects --converge -O3 \
--enable-mutable-globals --enable-nontrapping-float-to-int \
--enable-simd --enable-bulk-memory --enable-sign-ext \
--enable-reference-types --enable-multivalue \

View File

@@ -2,11 +2,11 @@ module github.com/ncruces/go-sqlite3/embed/bcw2
go 1.24.0
require github.com/ncruces/go-sqlite3 v0.30.3
require github.com/ncruces/go-sqlite3 v0.30.1
require (
github.com/ncruces/julianday v1.0.0 // indirect
github.com/ncruces/sort v0.1.6 // indirect
github.com/tetratelabs/wazero v1.11.0 // indirect
golang.org/x/sys v0.39.0 // indirect
github.com/tetratelabs/wazero v1.10.1 // indirect
golang.org/x/sys v0.38.0 // indirect
)

View File

@@ -1,12 +1,12 @@
github.com/ncruces/go-sqlite3 v0.30.3 h1:X/CgWW9GzmIAkEPrifhKqf0cC15DuOVxAJaHFTTAURQ=
github.com/ncruces/go-sqlite3 v0.30.3/go.mod h1:AxKu9sRxkludimFocbktlY6LiYSkxiI5gTA8r+os/Nw=
github.com/ncruces/go-sqlite3 v0.30.1 h1:pHC3YsyRdJv4pCMB4MO1Q2BXw/CAa+Hoj7GSaKtVk+g=
github.com/ncruces/go-sqlite3 v0.30.1/go.mod h1:UVsWrQaq1qkcal5/vT5lOJnZCVlR5rsThKdwidjFsKc=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/ncruces/sort v0.1.6 h1:TrsJfGRH1AoWoaeB4/+gCohot9+cA6u/INaH5agIhNk=
github.com/ncruces/sort v0.1.6/go.mod h1:obJToO4rYr6VWP0Uw5FYymgYGt3Br4RXcs/JdKaXAPk=
github.com/tetratelabs/wazero v1.11.0 h1:+gKemEuKCTevU4d7ZTzlsvgd1uaToIDtlQlmNbwqYhA=
github.com/tetratelabs/wazero v1.11.0/go.mod h1:eV28rsN8Q+xwjogd7f4/Pp4xFxO7uOGbLcD/LzB1wiU=
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
github.com/tetratelabs/wazero v1.10.1 h1:2DugeJf6VVk58KTPszlNfeeN8AhhpwcZqkJj2wwFuH8=
github.com/tetratelabs/wazero v1.10.1/go.mod h1:DRm5twOQ5Gr1AoEdSi0CLjDQF1J9ZAuyqFIjl1KKfQU=
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=

View File

@@ -23,14 +23,13 @@ trap 'rm -f sqlite3.tmp' EXIT
-Wl,--import-undefined \
-Wl,--initial-memory=327680 \
-D_HAVE_SQLITE_CONFIG_H \
-DSQLITE_EXPERIMENTAL_PRAGMA_20251114 \
-DSQLITE_CUSTOM_INCLUDE=sqlite_opt.h \
$(awk '{print "-Wl,--export="$0}' exports.txt)
"$BINARYEN/wasm-ctor-eval" -g -c _initialize sqlite3.wasm -o sqlite3.tmp
"$BINARYEN/wasm-opt" -g sqlite3.tmp -o sqlite3.wasm \
--gufa --generate-global-effects --low-memory-unused --converge -O3 \
--low-memory-unused --gufa --generate-global-effects --converge -O3 \
--enable-mutable-globals --enable-nontrapping-float-to-int \
--enable-simd --enable-bulk-memory --enable-sign-ext \
--enable-reference-types --enable-multivalue \
--strip --strip-producers
--strip --strip-producers

View File

@@ -19,7 +19,7 @@ func Test_init(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if version != "3.51.1" {
if version != "3.51.0" {
t.Error(version)
}
}

Binary file not shown.

View File

@@ -16,7 +16,6 @@ import (
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/util/sql3util"
)
// Register registers the bloom_filter virtual table:
@@ -35,8 +34,6 @@ type bloom struct {
hashes int
}
const vtab = `CREATE TABLE x(present, word TEXT HIDDEN NOT NULL PRIMARY KEY) WITHOUT ROWID`
func create(db *sqlite3.Conn, _, schema, table string, arg ...string) (_ *bloom, err error) {
b := bloom{
db: db,
@@ -58,9 +55,11 @@ func create(db *sqlite3.Conn, _, schema, table string, arg ...string) (_ *bloom,
}
if len(arg) > 1 {
var ok bool
b.prob, ok = sql3util.ParseFloat(arg[1])
if !ok || b.prob <= 0 || b.prob >= 1 {
b.prob, err = strconv.ParseFloat(arg[1], 64)
if err != nil {
return nil, err
}
if b.prob <= 0 || b.prob >= 1 {
return nil, util.ErrorString("bloom: probability must be in the range (0,1)")
}
} else {
@@ -81,7 +80,8 @@ func create(db *sqlite3.Conn, _, schema, table string, arg ...string) (_ *bloom,
b.bytes = numBytes(nelem, b.prob)
err = db.DeclareVTab(vtab)
err = db.DeclareVTab(
`CREATE TABLE x(present, word HIDDEN NOT NULL PRIMARY KEY) WITHOUT ROWID`)
if err != nil {
return nil, err
}
@@ -115,15 +115,15 @@ func connect(db *sqlite3.Conn, _, schema, table string, arg ...string) (_ *bloom
storage: table + "_storage",
}
err = db.DeclareVTab(vtab)
err = db.DeclareVTab(
`CREATE TABLE x(present, word HIDDEN NOT NULL PRIMARY KEY) WITHOUT ROWID`)
if err != nil {
return nil, err
}
load, _, err := db.PrepareFlags(fmt.Sprintf(
load, _, err := db.Prepare(fmt.Sprintf(
`SELECT m/8, p, k FROM %s.%s WHERE rowid = 1`,
sqlite3.QuoteIdentifier(b.schema), sqlite3.QuoteIdentifier(b.storage)),
sqlite3.PREPARE_DONT_LOG)
sqlite3.QuoteIdentifier(b.schema), sqlite3.QuoteIdentifier(b.storage)))
if err != nil {
return nil, err
}
@@ -166,10 +166,9 @@ func (t *bloom) ShadowTables() {
}
func (t *bloom) Integrity(schema, table string, flags int) error {
load, _, err := t.db.PrepareFlags(fmt.Sprintf(
load, _, err := t.db.Prepare(fmt.Sprintf(
`SELECT typeof(data), length(data), p, n, m, k FROM %s.%s WHERE rowid = 1`,
sqlite3.QuoteIdentifier(t.schema), sqlite3.QuoteIdentifier(t.storage)),
sqlite3.PREPARE_DONT_LOG)
sqlite3.QuoteIdentifier(t.schema), sqlite3.QuoteIdentifier(t.storage)))
if err != nil {
return fmt.Errorf("bloom: %v", err) // can't wrap!
}

View File

@@ -56,7 +56,7 @@ func Register(db *sqlite3.Conn) error {
done.Add(key)
}
err := db.DeclareVTab(`CREATE TABLE x(id INT,depth INT,root HIDDEN,tablename TEXT HIDDEN,idcolumn TEXT HIDDEN,parentcolumn TEXT HIDDEN)`)
err := db.DeclareVTab(`CREATE TABLE x(id,depth,root HIDDEN,tablename HIDDEN,idcolumn HIDDEN,parentcolumn HIDDEN)`)
if err != nil {
return nil, err
}
@@ -202,7 +202,7 @@ func (c *cursor) Filter(idxNum int, idxStr string, arg ...sqlite3.Value) error {
sqlite3.QuoteIdentifier(column),
sqlite3.QuoteIdentifier(parent),
)
stmt, _, err := c.db.PrepareFlags(sql, sqlite3.PREPARE_DONT_LOG)
stmt, _, err := c.db.Prepare(sql)
if err != nil {
return err
}

View File

@@ -85,8 +85,7 @@ func RegisterFS(db *sqlite3.Conn, fsys fs.FS) error {
header: header,
}
hadSchema := schema != ""
if !hadSchema {
if schema == "" {
var row []string
if header || columns < 0 {
csv, c, err := t.newReader()
@@ -100,14 +99,17 @@ func RegisterFS(db *sqlite3.Conn, fsys fs.FS) error {
}
}
schema = getSchema(header, columns, row)
} else {
t.typs, err = getColumnAffinities(schema)
if err != nil {
return nil, err
}
}
err = db.DeclareVTab(schema)
if err == nil {
err = db.VTabConfig(sqlite3.VTAB_DIRECTONLY)
}
if err == nil && hadSchema {
t.typs, err = getColumnAffinities(schema)
}
if err != nil {
return nil, err
}
@@ -121,7 +123,7 @@ type table struct {
fsys fs.FS
name string
data string
typs []sql3util.Affinity
typs []affinity
comma rune
comment rune
header bool
@@ -240,27 +242,31 @@ func (c *cursor) RowID() (int64, error) {
func (c *cursor) Column(ctx sqlite3.Context, col int) error {
if col < len(c.row) {
typ := sql3util.TEXT
typ := text
if col < len(c.table.typs) {
typ = c.table.typs[col]
}
txt := c.row[col]
if txt == "" && typ != sql3util.TEXT {
if txt == "" && typ != text {
return nil
}
switch typ {
case sql3util.NUMERIC, sql3util.INTEGER:
if i, err := strconv.ParseInt(txt, 10, 64); err == nil {
ctx.ResultInt64(i)
return nil
case numeric, integer:
if strings.TrimLeft(txt, "+-0123456789") == "" {
if i, err := strconv.ParseInt(txt, 10, 64); err == nil {
ctx.ResultInt64(i)
return nil
}
}
fallthrough
case sql3util.REAL:
if f, ok := sql3util.ParseFloat(txt); ok {
ctx.ResultFloat(f)
return nil
case real:
if strings.TrimLeft(txt, "+-.0123456789Ee") == "" {
if f, err := strconv.ParseFloat(txt, 64); err == nil {
ctx.ResultFloat(f)
return nil
}
}
fallthrough
default:

View File

@@ -9,31 +9,31 @@ import (
func getSchema(header bool, columns int, row []string) string {
var sep string
var buf strings.Builder
buf.WriteString("CREATE TABLE x(")
var str strings.Builder
str.WriteString("CREATE TABLE x(")
if 0 <= columns && columns < len(row) {
row = row[:columns]
}
for i, f := range row {
buf.WriteString(sep)
str.WriteString(sep)
if header && f != "" {
buf.WriteString(sqlite3.QuoteIdentifier(f))
str.WriteString(sqlite3.QuoteIdentifier(f))
} else {
buf.WriteString("c")
buf.WriteString(strconv.Itoa(i + 1))
str.WriteString("c")
str.WriteString(strconv.Itoa(i + 1))
}
buf.WriteString(" TEXT")
str.WriteString(" TEXT")
sep = ","
}
for i := len(row); i < columns; i++ {
buf.WriteString(sep)
buf.WriteString("c")
buf.WriteString(strconv.Itoa(i + 1))
buf.WriteString(" TEXT")
str.WriteString(sep)
str.WriteString("c")
str.WriteString(strconv.Itoa(i + 1))
str.WriteString(" TEXT")
sep = ","
}
buf.WriteByte(')')
str.WriteByte(')')
return buf.String()
return str.String()
}

View File

@@ -1,17 +1,52 @@
package csv
import "github.com/ncruces/go-sqlite3/util/sql3util"
import (
"strings"
func getColumnAffinities(schema string) ([]sql3util.Affinity, error) {
"github.com/ncruces/go-sqlite3/util/sql3util"
)
type affinity byte
const (
blob affinity = 0
text affinity = 1
numeric affinity = 2
integer affinity = 3
real affinity = 4
)
func getColumnAffinities(schema string) ([]affinity, error) {
tab, err := sql3util.ParseTable(schema)
if err != nil {
return nil, err
}
columns := tab.Columns
types := make([]sql3util.Affinity, len(columns))
types := make([]affinity, len(columns))
for i, col := range columns {
types[i] = sql3util.GetAffinity(col.Type)
types[i] = getAffinity(col.Type)
}
return types, nil
}
func getAffinity(declType string) affinity {
// https://sqlite.org/datatype3.html#determination_of_column_affinity
if declType == "" {
return blob
}
name := strings.ToUpper(declType)
if strings.Contains(name, "INT") {
return integer
}
if strings.Contains(name, "CHAR") || strings.Contains(name, "CLOB") || strings.Contains(name, "TEXT") {
return text
}
if strings.Contains(name, "BLOB") {
return blob
}
if strings.Contains(name, "REAL") || strings.Contains(name, "FLOA") || strings.Contains(name, "DOUB") {
return real
}
return numeric
}

32
ext/csv/types_test.go Normal file
View File

@@ -0,0 +1,32 @@
package csv
import "testing"
func Test_getAffinity(t *testing.T) {
tests := []struct {
decl string
want affinity
}{
{"", blob},
{"INTEGER", integer},
{"TINYINT", integer},
{"TEXT", text},
{"CHAR", text},
{"CLOB", text},
{"BLOB", blob},
{"REAL", real},
{"FLOAT", real},
{"DOUBLE", real},
{"NUMERIC", numeric},
{"DECIMAL", numeric},
{"BOOLEAN", numeric},
{"DATETIME", numeric},
}
for _, tt := range tests {
t.Run(tt.decl, func(t *testing.T) {
if got := getAffinity(tt.decl); got != tt.want {
t.Errorf("getAffinity() = %v, want %v", got, tt.want)
}
})
}
}

View File

@@ -30,7 +30,7 @@ func RegisterFS(db *sqlite3.Conn, fsys fs.FS) error {
db.CreateFunction("readfile", 1, sqlite3.DIRECTONLY, readfile(fsys)),
db.CreateFunction("lsmode", 1, sqlite3.DETERMINISTIC, lsmode),
sqlite3.CreateModule(db, "fsdir", nil, func(db *sqlite3.Conn, _, _, _ string, _ ...string) (fsdir, error) {
err := db.DeclareVTab(`CREATE TABLE x(name TEXT,mode INT,mtime TIMESTAMP,data BLOB,path HIDDEN,dir HIDDEN)`)
err := db.DeclareVTab(`CREATE TABLE x(name,mode,mtime TIMESTAMP,data,path HIDDEN,dir HIDDEN)`)
if err == nil {
err = db.VTabConfig(sqlite3.VTAB_DIRECTONLY)
}

View File

@@ -43,14 +43,11 @@ func declare(db *sqlite3.Conn, _, _, _ string, arg ...string) (ret *table, err e
// Row key query.
t.scan = "SELECT * FROM\n" + arg[0]
stmt, tail, err := db.PrepareFlags(t.scan, sqlite3.PREPARE_FROM_DDL)
stmt, _, err := db.Prepare(t.scan)
if err != nil {
return nil, err
}
defer stmt.Close()
if tail != "" {
return nil, util.TailErr
}
t.keys = make([]string, stmt.ColumnCount())
for i := range t.keys {
@@ -58,20 +55,15 @@ func declare(db *sqlite3.Conn, _, _, _ string, arg ...string) (ret *table, err e
t.keys[i] = name
create.WriteString(sep)
create.WriteString(name)
create.WriteString(" ")
create.WriteString(stmt.ColumnDeclType(i))
sep = ","
}
stmt.Close()
// Column definition query.
stmt, tail, err = db.PrepareFlags("SELECT * FROM\n"+arg[1], sqlite3.PREPARE_FROM_DDL)
stmt, _, err = db.Prepare("SELECT * FROM\n" + arg[1])
if err != nil {
return nil, err
}
if tail != "" {
return nil, util.TailErr
}
if stmt.ColumnCount() != 2 {
return nil, util.ErrorString("pivot: column definition query expects 2 result columns")
@@ -79,23 +71,17 @@ func declare(db *sqlite3.Conn, _, _, _ string, arg ...string) (ret *table, err e
for stmt.Step() {
name := sqlite3.QuoteIdentifier(stmt.ColumnText(1))
t.cols = append(t.cols, stmt.ColumnValue(0).Dup())
create.WriteString(sep)
create.WriteString(",")
create.WriteString(name)
create.WriteString(" ")
create.WriteString(stmt.ColumnDeclType(1))
sep = ","
}
stmt.Close()
// Pivot cell query.
t.cell = "SELECT * FROM\n" + arg[2]
stmt, tail, err = db.PrepareFlags(t.cell, sqlite3.PREPARE_FROM_DDL)
stmt, _, err = db.Prepare(t.cell)
if err != nil {
return nil, err
}
if tail != "" {
return nil, util.TailErr
}
if stmt.ColumnCount() != 1 {
return nil, util.ErrorString("pivot: cell query expects 1 result columns")
@@ -196,9 +182,7 @@ func (c *cursor) Filter(idxNum int, idxStr string, arg ...sqlite3.Value) error {
return err
}
const prepflags = sqlite3.PREPARE_DONT_LOG | sqlite3.PREPARE_FROM_DDL
c.scan, _, err = c.table.db.PrepareFlags(idxStr, prepflags)
c.scan, _, err = c.table.db.Prepare(idxStr)
if err != nil {
return err
}
@@ -210,7 +194,7 @@ func (c *cursor) Filter(idxNum int, idxStr string, arg ...sqlite3.Value) error {
}
if c.cell == nil {
c.cell, _, err = c.table.db.PrepareFlags(c.table.cell, prepflags)
c.cell, _, err = c.table.db.Prepare(c.table.cell)
if err != nil {
return err
}

View File

@@ -35,45 +35,40 @@ func declare(db *sqlite3.Conn, _, _, _ string, arg ...string) (*table, error) {
sql := "SELECT * FROM\n" + arg[0]
stmt, tail, err := db.PrepareFlags(sql,
sqlite3.PREPARE_PERSISTENT|sqlite3.PREPARE_FROM_DDL)
stmt, _, err := db.PrepareFlags(sql, sqlite3.PREPARE_PERSISTENT)
if err != nil {
return nil, err
}
if tail != "" {
stmt.Close()
return nil, util.TailErr
}
var sep string
var buf strings.Builder
buf.WriteString("CREATE TABLE x(")
var str strings.Builder
str.WriteString("CREATE TABLE x(")
outputs := stmt.ColumnCount()
for i := range outputs {
name := sqlite3.QuoteIdentifier(stmt.ColumnName(i))
buf.WriteString(sep)
buf.WriteString(name)
buf.WriteString(" ")
buf.WriteString(stmt.ColumnDeclType(i))
str.WriteString(sep)
str.WriteString(name)
str.WriteString(" ")
str.WriteString(stmt.ColumnDeclType(i))
sep = ","
}
inputs := stmt.BindCount()
for i := 1; i <= inputs; i++ {
buf.WriteString(sep)
str.WriteString(sep)
name := stmt.BindName(i)
if name == "" {
buf.WriteString("[")
buf.WriteString(strconv.Itoa(i))
buf.WriteString("] HIDDEN")
str.WriteString("[")
str.WriteString(strconv.Itoa(i))
str.WriteString("] HIDDEN")
} else {
buf.WriteString(sqlite3.QuoteIdentifier(name[1:]))
buf.WriteString(" HIDDEN")
str.WriteString(sqlite3.QuoteIdentifier(name[1:]))
str.WriteString(" HIDDEN")
}
sep = ","
}
buf.WriteByte(')')
str.WriteByte(')')
err = db.DeclareVTab(buf.String())
err = db.DeclareVTab(str.String())
if err != nil {
stmt.Close()
return nil, err
@@ -134,8 +129,7 @@ func (t *table) Open() (_ sqlite3.VTabCursor, err error) {
if !t.inuse {
t.inuse = true
} else {
stmt, _, err = t.stmt.Conn().PrepareFlags(t.sql,
sqlite3.PREPARE_DONT_LOG|sqlite3.PREPARE_FROM_DDL)
stmt, _, err = t.stmt.Conn().Prepare(t.sql)
if err != nil {
return nil, err
}

12
go.mod
View File

@@ -5,18 +5,18 @@ go 1.24.0
require (
github.com/ncruces/julianday v1.0.0
github.com/ncruces/sort v0.1.6
github.com/ncruces/wbt v1.0.0
github.com/tetratelabs/wazero v1.11.0
golang.org/x/sys v0.40.0
github.com/ncruces/wbt v0.2.0
github.com/tetratelabs/wazero v1.10.1
golang.org/x/sys v0.38.0
)
require (
github.com/dchest/siphash v1.2.3 // ext/bloom
github.com/google/uuid v1.6.0 // ext/uuid
github.com/psanford/httpreadat v0.1.0 // example
golang.org/x/crypto v0.47.0 // vfs/adiantum vfs/xts
golang.org/x/sync v0.19.0 // test
golang.org/x/text v0.33.0 // ext/unicode
golang.org/x/crypto v0.45.0 // vfs/adiantum vfs/xts
golang.org/x/sync v0.18.0 // test
golang.org/x/text v0.31.0 // ext/unicode
lukechampine.com/adiantum v1.1.1 // vfs/adiantum
)

24
go.sum
View File

@@ -6,19 +6,19 @@ github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/ncruces/sort v0.1.6 h1:TrsJfGRH1AoWoaeB4/+gCohot9+cA6u/INaH5agIhNk=
github.com/ncruces/sort v0.1.6/go.mod h1:obJToO4rYr6VWP0Uw5FYymgYGt3Br4RXcs/JdKaXAPk=
github.com/ncruces/wbt v1.0.0 h1:8iBE7UPjTLUpzu3/FCRjAmuQjWzgxo10RGBgt3ooLSc=
github.com/ncruces/wbt v1.0.0/go.mod h1:DtF92amvMxH69EmBFUSFWRDAlo6hOEfoNQnClxj9C/c=
github.com/ncruces/wbt v0.2.0 h1:Q9zlKOBSZc7Yy/R2cGa35g6RKUUE3BjNIW3tfGC4F04=
github.com/ncruces/wbt v0.2.0/go.mod h1:DtF92amvMxH69EmBFUSFWRDAlo6hOEfoNQnClxj9C/c=
github.com/psanford/httpreadat v0.1.0 h1:VleW1HS2zO7/4c7c7zNl33fO6oYACSagjJIyMIwZLUE=
github.com/psanford/httpreadat v0.1.0/go.mod h1:Zg7P+TlBm3bYbyHTKv/EdtSJZn3qwbPwpfZ/I9GKCRE=
github.com/tetratelabs/wazero v1.11.0 h1:+gKemEuKCTevU4d7ZTzlsvgd1uaToIDtlQlmNbwqYhA=
github.com/tetratelabs/wazero v1.11.0/go.mod h1:eV28rsN8Q+xwjogd7f4/Pp4xFxO7uOGbLcD/LzB1wiU=
golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
github.com/tetratelabs/wazero v1.10.1 h1:2DugeJf6VVk58KTPszlNfeeN8AhhpwcZqkJj2wwFuH8=
github.com/tetratelabs/wazero v1.10.1/go.mod h1:DRm5twOQ5Gr1AoEdSi0CLjDQF1J9ZAuyqFIjl1KKfQU=
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
lukechampine.com/adiantum v1.1.1 h1:4fp6gTxWCqpEbLy40ExiYDDED3oUNWx5cTqBCtPdZqA=
lukechampine.com/adiantum v1.1.1/go.mod h1:LrAYVnTYLnUtE/yMp5bQr0HstAf060YUF8nM0B6+rUw=

View File

@@ -3,7 +3,7 @@ module github.com/ncruces/go-sqlite3/gormlite
go 1.24.0
require (
github.com/ncruces/go-sqlite3 v0.30.3
github.com/ncruces/go-sqlite3 v0.30.1
gorm.io/gorm v1.31.1
)
@@ -11,7 +11,7 @@ require (
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/ncruces/julianday v1.0.0 // indirect
github.com/tetratelabs/wazero v1.11.0 // indirect
golang.org/x/sys v0.39.0 // indirect
golang.org/x/text v0.32.0 // indirect
github.com/tetratelabs/wazero v1.10.1 // indirect
golang.org/x/sys v0.38.0 // indirect
golang.org/x/text v0.31.0 // indirect
)

View File

@@ -2,15 +2,15 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/ncruces/go-sqlite3 v0.30.3 h1:X/CgWW9GzmIAkEPrifhKqf0cC15DuOVxAJaHFTTAURQ=
github.com/ncruces/go-sqlite3 v0.30.3/go.mod h1:AxKu9sRxkludimFocbktlY6LiYSkxiI5gTA8r+os/Nw=
github.com/ncruces/go-sqlite3 v0.30.1 h1:pHC3YsyRdJv4pCMB4MO1Q2BXw/CAa+Hoj7GSaKtVk+g=
github.com/ncruces/go-sqlite3 v0.30.1/go.mod h1:UVsWrQaq1qkcal5/vT5lOJnZCVlR5rsThKdwidjFsKc=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/tetratelabs/wazero v1.11.0 h1:+gKemEuKCTevU4d7ZTzlsvgd1uaToIDtlQlmNbwqYhA=
github.com/tetratelabs/wazero v1.11.0/go.mod h1:eV28rsN8Q+xwjogd7f4/Pp4xFxO7uOGbLcD/LzB1wiU=
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
github.com/tetratelabs/wazero v1.10.1 h1:2DugeJf6VVk58KTPszlNfeeN8AhhpwcZqkJj2wwFuH8=
github.com/tetratelabs/wazero v1.10.1/go.mod h1:DRm5twOQ5Gr1AoEdSi0CLjDQF1J9ZAuyqFIjl1KKfQU=
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg=
gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs=

View File

@@ -1,5 +0,0 @@
package util
func ValidPageSize(s int) bool {
return s&(s-1) == 0 && 512 <= s && s <= 65536
}

View File

@@ -1,142 +0,0 @@
package util
import (
"strconv"
"strings"
"time"
)
func ParseBool(s string) (b, ok bool) {
if len(s) == 0 {
return false, false
}
if s[0] == '0' {
return false, true
}
if '1' <= s[0] && s[0] <= '9' {
return true, true
}
switch strings.ToLower(s) {
case "true", "yes", "on":
return true, true
case "false", "no", "off":
return false, true
}
return false, false
}
func ParseFloat(s string) (f float64, ok bool) {
if strings.TrimLeft(s, "+-.0123456789Ee") != "" {
return
}
f, err := strconv.ParseFloat(s, 64)
return f, err == nil
}
func ParseTimeShift(s string) (years, months, days int, duration time.Duration, ok bool) {
// Sign part: ±
neg := strings.HasPrefix(s, "-")
sign := neg || strings.HasPrefix(s, "+")
if sign {
s = s[1:]
}
if ok = len(s) >= 5; !ok {
return // !ok
}
defer func() {
if neg {
years = -years
months = -months
days = -days
duration = -duration
}
}()
// Date part: YYYY-MM-DD
if s[4] == '-' {
if ok = sign && len(s) >= 10 && s[7] == '-'; !ok {
return // !ok
}
if years, ok = parseInt(s[0:4], 0); !ok {
return // !ok
}
if months, ok = parseInt(s[5:7], 12); !ok {
return // !ok
}
if days, ok = parseInt(s[8:10], 31); !ok {
return // !ok
}
if len(s) == 10 {
return
}
if ok = s[10] == ' '; !ok {
return // !ok
}
s = s[11:]
}
// Time part: HH:MM
if ok = len(s) >= 5 && s[2] == ':'; !ok {
return // !ok
}
var hours, minutes int
if hours, ok = parseInt(s[0:2], 24); !ok {
return
}
if minutes, ok = parseInt(s[3:5], 60); !ok {
return
}
duration = time.Duration(hours)*time.Hour + time.Duration(minutes)*time.Minute
if len(s) == 5 {
return
}
if ok = len(s) >= 8 && s[5] == ':'; !ok {
return // !ok
}
// Seconds part: HH:MM:SS
var seconds int
if seconds, ok = parseInt(s[6:8], 60); !ok {
return
}
duration += time.Duration(seconds) * time.Second
if len(s) == 8 {
return
}
if ok = len(s) >= 10 && s[8] == '.'; !ok {
return // !ok
}
s = s[9:]
// Nanosecond part: HH:MM:SS.SSS
var nanos int
if nanos, ok = parseInt(s[0:min(9, len(s))], 0); !ok {
return
}
for i := len(s); i < 9; i++ {
nanos *= 10
}
duration += time.Duration(nanos)
// Subnanosecond part.
if len(s) > 9 {
_, ok = parseInt(s[9:], 0)
}
return
}
func parseInt(s string, max int) (i int, _ bool) {
for _, r := range []byte(s) {
r -= '0'
if r > 9 {
return
}
i = i*10 + int(r)
}
return i, max == 0 || i < max
}

View File

@@ -1,144 +0,0 @@
package util_test
import (
"testing"
"time"
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/internal/util"
)
func TestParseBool(t *testing.T) {
tests := []struct {
str string
val bool
ok bool
}{
{"", false, false},
{"0", false, true},
{"1", true, true},
{"9", true, true},
{"T", false, false},
{"true", true, true},
{"FALSE", false, true},
{"false?", false, false},
}
for _, tt := range tests {
t.Run(tt.str, func(t *testing.T) {
gotVal, gotOK := util.ParseBool(tt.str)
if gotVal != tt.val || gotOK != tt.ok {
t.Errorf("ParseBool(%q) = (%v, %v) want (%v, %v)", tt.str, gotVal, gotOK, tt.val, tt.ok)
}
})
}
}
func TestParseTimeShift(t *testing.T) {
epoch := time.Unix(0, 0)
tests := []struct {
str string
val time.Time
ok bool
}{
{"", epoch, false},
{"0001-11-30", epoch, false},
{"+_001-11-30", epoch, false},
{"+0001-_1-30", epoch.AddDate(1, 0, 0), false},
{"+0001-11-_0", epoch.AddDate(1, 11, 0), false},
{"+0001-11-30", epoch.AddDate(1, 11, 30), true},
{"-0001-11-30", epoch.AddDate(-1, -11, -30), true},
{"+0001-11-30T", epoch.AddDate(1, 11, 30), false},
{"+0001-11-30 12", epoch.AddDate(1, 11, 30), false},
{"+0001-11-30 _2:30", epoch.AddDate(1, 11, 30), false},
{"+0001-11-30 12:_0", epoch.AddDate(1, 11, 30), false},
{"+0001-11-30 12:30", epoch.AddDate(1, 11, 30).Add(12*time.Hour + 30*time.Minute), true},
{"+0001-11-30 12:30:", epoch.AddDate(1, 11, 30).Add(12*time.Hour + 30*time.Minute), false},
{"+0001-11-30 12:30:_0", epoch.AddDate(1, 11, 30).Add(12*time.Hour + 30*time.Minute), false},
{"+0001-11-30 12:30:59", epoch.AddDate(1, 11, 30).Add(12*time.Hour + 30*time.Minute + 59*time.Second), true},
{"+0001-11-30 12:30:59.", epoch.AddDate(1, 11, 30).Add(12*time.Hour + 30*time.Minute + 59*time.Second), false},
{"+0001-11-30 12:30:59._", epoch.AddDate(1, 11, 30).Add(12*time.Hour + 30*time.Minute + 59*time.Second), false},
{"+0001-11-30 12:30:59.1", epoch.AddDate(1, 11, 30).Add(12*time.Hour + 30*time.Minute + 59*time.Second + 100*time.Millisecond), true},
{"+0001-11-30 12:30:59.123456789_", epoch.AddDate(1, 11, 30).Add(12*time.Hour + 30*time.Minute + 59*time.Second + 123456789), false},
{"+0001-11-30 12:30:59.1234567890", epoch.AddDate(1, 11, 30).Add(12*time.Hour + 30*time.Minute + 59*time.Second + 123456789), true},
{"-12:30:59.1234567890", epoch.Add(-12*time.Hour - 30*time.Minute - 59*time.Second - 123456789), true},
}
for _, tt := range tests {
t.Run(tt.str, func(t *testing.T) {
years, months, days, duration, gotOK := util.ParseTimeShift(tt.str)
gotVal := epoch.AddDate(years, months, days).Add(duration)
if !gotVal.Equal(tt.val) || gotOK != tt.ok {
t.Errorf("ParseTimeShift(%q) = (%v, %v) want (%v, %v)", tt.str, gotVal, gotOK, tt.val, tt.ok)
}
})
}
}
func FuzzParseTimeShift(f *testing.F) {
f.Add("")
f.Add("0001-12-30")
f.Add("+_001-12-30")
f.Add("+0001-_2-30")
f.Add("+0001-12-_0")
f.Add("+0001-12-30")
f.Add("-0001-12-30")
f.Add("+0001-12-30T")
f.Add("+0001-12-30 12")
f.Add("+0001-12-30 _2:30")
f.Add("+0001-12-30 12:_0")
f.Add("+0001-12-30 12:30")
f.Add("+0001-12-30 12:30:")
f.Add("+0001-12-30 12:30:_0")
f.Add("+0001-12-30 12:30:60")
f.Add("+0001-12-30 12:30:60.")
f.Add("+0001-12-30 12:30:60._")
f.Add("+0001-12-30 12:30:60.1")
f.Add("+0001-12-30 12:30:60.123456789_")
f.Add("+0001-12-30 12:30:60.1234567890")
f.Add("-12:30:60.1234567890")
c, err := sqlite3.Open(":memory:")
if err != nil {
f.Fatal(err)
}
defer c.Close()
s, _, err := c.Prepare(`SELECT julianday('00:00', ?)`)
if err != nil {
f.Fatal(err)
}
defer s.Close()
// Default SQLite date.
epoch := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
f.Fuzz(func(t *testing.T, str string) {
years, months, days, duration, ok := util.ParseTimeShift(str)
// Account for a full 400 year cycle.
if years < -200 || years > +200 {
t.Skip()
}
// SQLite only tracks milliseconds.
if duration != duration.Truncate(time.Millisecond) {
t.Skip()
}
if ok {
s.Reset()
s.BindText(1, str)
if !s.Step() {
t.Fail()
}
got := epoch.AddDate(years, months, days).Add(duration)
// Julian day introduces floating point inaccuracy.
want := s.ColumnTime(0, sqlite3.TimeFormatJulianDay)
want = want.Round(time.Millisecond)
if !got.Equal(want) {
t.Fatalf("with %q, got %v, want %v", str, got, want)
}
}
})
}

11
litestream/README.md Normal file
View File

@@ -0,0 +1,11 @@
# Litestream lightweight read-replicas
This package implements the **EXPERIMENTAL** `"litestream"` SQLite VFS
that offers Litestream [lightweight read-replicas](https://fly.io/blog/litestream-revamped/#lightweight-read-replicas).
See the [example](vfs_test.go) for how to use.
To improve performance,
increase `PollInterval` (and `MinLevel`) as much as you can,
and set [`PRAGMA cache_size=N`](https://www.sqlite.org/pragma.html#pragma_cache_size)
(or use `_pragma=cache_size(N)`).

78
litestream/api.go Normal file
View File

@@ -0,0 +1,78 @@
// Package litestream implements a Litestream lightweight read-replica VFS.
package litestream
import (
"log/slog"
"sync"
"time"
"github.com/benbjohnson/litestream"
"github.com/ncruces/go-sqlite3/vfs"
)
const (
// The default poll interval.
DefaultPollInterval = 1 * time.Second
// The default cache size: 10 MiB.
DefaultCacheSize = 10 * 1024 * 1024
)
func init() {
vfs.Register("litestream", liteVFS{})
}
var (
liteMtx sync.RWMutex
// +checklocks:liteMtx
liteDBs = map[string]*liteDB{}
)
// ReplicaOptions represents options for [NewReplica].
type ReplicaOptions struct {
// Where to log error messages. May be nil.
Logger *slog.Logger
// Replica poll interval.
// Should be less than the compaction interval
// used by the replica at MinLevel+1.
PollInterval time.Duration
// Minimum compaction level to track.
MinLevel int
// CacheSize is the maximum size of the page cache in bytes.
// Zero means DefaultCacheSize, negative disables caching.
CacheSize int
}
// NewReplica creates a read-replica from a Litestream client.
func NewReplica(name string, client litestream.ReplicaClient, options ReplicaOptions) {
if options.Logger != nil {
options.Logger = options.Logger.With("name", name)
} else {
options.Logger = slog.New(slog.DiscardHandler)
}
if options.PollInterval <= 0 {
options.PollInterval = DefaultPollInterval
}
if options.CacheSize == 0 {
options.CacheSize = DefaultCacheSize
}
liteMtx.Lock()
defer liteMtx.Unlock()
liteDBs[name] = &liteDB{
client: client,
opts: options,
cache: pageCache{size: options.CacheSize},
}
}
// RemoveReplica removes a replica by name.
func RemoveReplica(name string) {
liteMtx.Lock()
defer liteMtx.Unlock()
delete(liteDBs, name)
}

72
litestream/cache.go Normal file
View File

@@ -0,0 +1,72 @@
package litestream
import (
"encoding/binary"
"sync"
"golang.org/x/sync/singleflight"
"github.com/superfly/ltx"
)
type pageCache struct {
single singleflight.Group
pages map[uint32]cachedPage // +checklocks:mtx
size int
mtx sync.Mutex
}
type cachedPage struct {
data []byte
txid ltx.TXID
}
func (c *pageCache) getOrFetch(pgno uint32, maxTXID ltx.TXID, fetch func() (any, error)) ([]byte, error) {
if c.size >= 0 {
c.mtx.Lock()
if c.pages == nil {
c.pages = map[uint32]cachedPage{}
}
page := c.pages[pgno]
c.mtx.Unlock()
if page.txid == maxTXID {
return page.data, nil
}
}
var key [12]byte
binary.LittleEndian.PutUint32(key[0:], pgno)
binary.LittleEndian.PutUint64(key[4:], uint64(maxTXID))
v, err, _ := c.single.Do(string(key[:]), fetch)
if err != nil {
return nil, err
}
page := cachedPage{v.([]byte), maxTXID}
if c.size >= 0 {
c.mtx.Lock()
c.evict(len(page.data))
c.pages[pgno] = page
c.mtx.Unlock()
}
return page.data, nil
}
// +checklocks:c.mtx
func (c *pageCache) evict(pageSize int) {
// Evict random keys until we're under the maximum size.
// SQLite has its own page cache, which it will use for each connection.
// Since this is a second layer of shared cache,
// random eviction is probably good enough.
if pageSize*len(c.pages) < c.size {
return
}
for key := range c.pages {
delete(c.pages, key)
if pageSize*len(c.pages) < c.size {
return
}
}
}

View File

@@ -0,0 +1,48 @@
package litestream_test
import (
"log"
"time"
"github.com/benbjohnson/litestream/s3"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/litestream"
)
func ExampleNewReplica() {
client := s3.NewReplicaClient()
client.Bucket = "test-bucket"
client.Path = "fruits.db"
litestream.NewReplica("fruits.db", client, litestream.ReplicaOptions{
PollInterval: 5 * time.Second,
})
db, err := driver.Open("file:fruits.db?vfs=litestream")
if err != nil {
log.Fatalln(err)
}
defer db.Close()
for {
time.Sleep(time.Second)
rows, err := db.Query("SELECT * FROM fruits")
if err != nil {
log.Fatalln(err)
}
for rows.Next() {
var name, color string
err := rows.Scan(&name, &color)
if err != nil {
log.Fatalln(err)
}
log.Println(name, color)
}
log.Println("===")
rows.Close()
}
}

63
litestream/go.mod Normal file
View File

@@ -0,0 +1,63 @@
module github.com/ncruces/go-sqlite3/litestream
go 1.24.4
require (
github.com/benbjohnson/litestream v0.5.2
github.com/ncruces/go-sqlite3 v0.30.1
github.com/ncruces/wbt v0.2.0
github.com/superfly/ltx v0.5.0
)
// github.com/ncruces/go-sqlite3
require (
github.com/ncruces/julianday v1.0.0 // indirect
github.com/tetratelabs/wazero v1.10.1 // indirect
golang.org/x/sys v0.38.0 // indirect
)
// github.com/superfly/ltx
require github.com/pierrec/lz4/v4 v4.1.22 // indirect
// github.com/benbjohnson/litestream
require (
filippo.io/age v1.2.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/prometheus/client_golang v1.23.2 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.67.3 // indirect
github.com/prometheus/procfs v0.19.2 // indirect
github.com/psanford/sqlite3vfs v0.0.0-20240315230605-24e1d98cf361 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
golang.org/x/crypto v0.45.0 // indirect
google.golang.org/protobuf v1.36.10 // indirect
modernc.org/sqlite v1.40.1 // indirect
)
// github.com/benbjohnson/litestream/s3
require (
github.com/aws/aws-sdk-go-v2 v1.37.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 // indirect
github.com/aws/aws-sdk-go-v2/config v1.30.2 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.18.2 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.1 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.2 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.1 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.85.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.26.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.31.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.35.1 // indirect
github.com/aws/smithy-go v1.22.5 // indirect
)
replace modernc.org/sqlite => github.com/ncruces/go-sqlite3/litestream/modernc v0.0.0-20251109124432-99b097de3b79

197
litestream/go.sum Normal file
View File

@@ -0,0 +1,197 @@
c2sp.org/CCTV/age v0.0.0-20240306222714-3ec4d716e805 h1:u2qwJeEvnypw+OCPUHmoZE3IqwfuN5kgDfo5MLzpNM0=
c2sp.org/CCTV/age v0.0.0-20240306222714-3ec4d716e805/go.mod h1:FomMrUJ2Lxt5jCLmZkG3FHa72zUprnhd3v/Z18Snm4w=
cloud.google.com/go v0.111.0 h1:YHLKNupSD1KqjDbQ3+LVdQ81h/UJbJyZG203cEfnQgM=
cloud.google.com/go v0.111.0/go.mod h1:0mibmpKP1TyOOFYQY5izo0LnT+ecvOQ0Sg3OdmMiNRU=
cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk=
cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI=
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI=
cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8=
cloud.google.com/go/storage v1.36.0 h1:P0mOkAcaJxhCTvAkMhxMfrTKiNcub4YmmPBtlhAyTr8=
cloud.google.com/go/storage v1.36.0/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8=
filippo.io/age v1.2.1 h1:X0TZjehAZylOIj4DubWYU1vWQxv9bJpo+Uu2/LGhi1o=
filippo.io/age v1.2.1/go.mod h1:JL9ew2lTN+Pyft4RiNGguFfOpewKwSHm5ayKD/A4004=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.2 h1:Hr5FTipp7SL07o2FvoVOX9HRiRH3CR3Mj8pxqCcdD5A=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.2/go.mod h1:QyVsSSN64v5TGltphKLQ2sQxe4OBQg0J1eKRcVBnfgE=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.11.0 h1:MhRfI58HblXzCtWEZCO0feHs8LweePB3s90r7WaR1KU=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.11.0/go.mod h1:okZ+ZURbArNdlJ+ptXoyHNuOETzOl1Oww19rm8I2WLA=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 h1:FwladfywkNirM+FZYLBR2kBz5C8Tg0fw5w5Y7meRXWI=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2/go.mod h1:vv5Ad0RrIoT1lJFdWBZwt4mB1+j+V8DUroixmKDTCdk=
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs=
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
github.com/aws/aws-sdk-go-v2 v1.37.1 h1:SMUxeNz3Z6nqGsXv0JuJXc8w5YMtrQMuIBmDx//bBDY=
github.com/aws/aws-sdk-go-v2 v1.37.1/go.mod h1:9Q0OoGQoboYIAJyslFyF1f5K1Ryddop8gqMhWx/n4Wg=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 h1:6GMWV6CNpA/6fbFHnoAjrv4+LGfyTqZz2LtCHnspgDg=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0/go.mod h1:/mXlTIVG9jbxkqDnr5UQNQxW1HRYxeGklkM9vAFeabg=
github.com/aws/aws-sdk-go-v2/config v1.30.2 h1:YE1BmSc4fFYqFgN1mN8uzrtc7R9x+7oSWeX8ckoltAw=
github.com/aws/aws-sdk-go-v2/config v1.30.2/go.mod h1:UNrLGZ6jfAVjgVJpkIxjLufRJqTXCVYOpkeVf83kwBo=
github.com/aws/aws-sdk-go-v2/credentials v1.18.2 h1:mfm0GKY/PHLhs7KO0sUaOtFnIQ15Qqxt+wXbO/5fIfs=
github.com/aws/aws-sdk-go-v2/credentials v1.18.2/go.mod h1:v0SdJX6ayPeZFQxgXUKw5RhLpAoZUuynxWDfh8+Eknc=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.1 h1:owmNBboeA0kHKDcdF8KiSXmrIuXZustfMGGytv6OMkM=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.1/go.mod h1:Bg1miN59SGxrZqlP8vJZSmXW+1N8Y1MjQDq1OfuNod8=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.2 h1:YFX4DvH1CPQXgQR8935b46Om+L7+6jus4aTdKqyDR84=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.2/go.mod h1:DgMPy7GqxcV0RSyaITnI3rw8HC3lIHB87U3KPQKDxHg=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.1 h1:ksZXBYv80EFTcgc8OJO48aQ8XDWXIQL7gGasPeCoTzI=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.1/go.mod h1:HSksQyyJETVZS7uM54cir0IgxttTD+8aEoJMPGepHBI=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.1 h1:+dn/xF/05utS7tUhjIcndbuaPjfll2LhbH1cCDGLYUQ=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.1/go.mod h1:hyAGz30LHdm5KBZDI58MXx5lDVZ5CUfvfTZvMu4HCZo=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.1 h1:4HbnOGE9491a9zYJ9VpPh1ApgEq6ZlD4Kuv1PJenFpc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.1/go.mod h1:Z6QnHC6TmpJWUxAy8FI4JzA7rTwl6EIANkyK9OR5z5w=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 h1:6+lZi2JeGKtCraAj1rpoZfKqnQ9SptseRZioejfUOLM=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0/go.mod h1:eb3gfbVIxIoGgJsi9pGne19dhCBpK6opTYpQqAmdy44=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.1 h1:ps3nrmBWdWwakZBydGX1CxeYFK80HsQ79JLMwm7Y4/c=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.1/go.mod h1:bAdfrfxENre68Hh2swNaGEVuFYE74o0SaSCAlaG9E74=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.1 h1:ky79ysLMxhwk5rxJtS+ILd3Mc8kC5fhsLBrP27r6h4I=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.1/go.mod h1:+2MmkvFvPYM1vsozBWduoLJUi5maxFk5B7KJFECujhY=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.1 h1:MdVYlN5pcQu1t1OYx4Ajo3fKl1IEhzgdPQbYFCRjYS8=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.1/go.mod h1:iikmNLrvHm2p4a3/4BPeix2S9P+nW8yM1IZW73x8bFA=
github.com/aws/aws-sdk-go-v2/service/s3 v1.85.1 h1:Hsqo8+dFxSdDvv9B2PgIx1AJAnDpqgS0znVI+R+MoGY=
github.com/aws/aws-sdk-go-v2/service/s3 v1.85.1/go.mod h1:8Q0TAPXD68Z8YqlcIGHs/UNIDHsxErV9H4dl4vJEpgw=
github.com/aws/aws-sdk-go-v2/service/sso v1.26.1 h1:uWaz3DoNK9MNhm7i6UGxqufwu3BEuJZm72WlpGwyVtY=
github.com/aws/aws-sdk-go-v2/service/sso v1.26.1/go.mod h1:ILpVNjL0BO+Z3Mm0SbEeUoYS9e0eJWV1BxNppp0fcb8=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.31.1 h1:XdG6/o1/ZDmn3wJU5SRAejHaWgKS4zHv0jBamuKuS2k=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.31.1/go.mod h1:oiotGTKadCOCl3vg/tYh4k45JlDF81Ka8rdumNhEnIQ=
github.com/aws/aws-sdk-go-v2/service/sts v1.35.1 h1:iF4Xxkc0H9c/K2dS0zZw3SCkj0Z7n6AMnUiiyoJND+I=
github.com/aws/aws-sdk-go-v2/service/sts v1.35.1/go.mod h1:0bxIatfN0aLq4mjoLDeBpOjOke68OsFlXPDFJ7V0MYw=
github.com/aws/smithy-go v1.22.5 h1:P9ATCXPMb2mPjYBgueqJNCA5S9UfktsW0tTxi+a7eqw=
github.com/aws/smithy-go v1.22.5/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
github.com/benbjohnson/litestream v0.5.2 h1:uD9I17n6RgUgyCwPM/Sw2YXNmMGixecUB5kmJ4FL08o=
github.com/benbjohnson/litestream v0.5.2/go.mod h1:jSW6AGqbxmJnEXGjMHchlZclGphzbJ6jGrGo5fYIDhU=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo=
github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o=
github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs=
github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas=
github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI=
github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/nats-io/nats.go v1.44.0 h1:ECKVrDLdh/kDPV1g0gAQ+2+m2KprqZK5O/eJAyAnH2M=
github.com/nats-io/nats.go v1.44.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g=
github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0=
github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/ncruces/go-sqlite3 v0.30.1 h1:pHC3YsyRdJv4pCMB4MO1Q2BXw/CAa+Hoj7GSaKtVk+g=
github.com/ncruces/go-sqlite3 v0.30.1/go.mod h1:UVsWrQaq1qkcal5/vT5lOJnZCVlR5rsThKdwidjFsKc=
github.com/ncruces/go-sqlite3/litestream/modernc v0.0.0-20251109124432-99b097de3b79 h1:evpQceUV2vRbOe84U/QhBBchfqFERRHTx1JOadFFMLE=
github.com/ncruces/go-sqlite3/litestream/modernc v0.0.0-20251109124432-99b097de3b79/go.mod h1:GSM2gXEOb9HIFFtsl0IUtnpvpDmVi7Kbp8z5GzwA0Tw=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/ncruces/wbt v0.2.0 h1:Q9zlKOBSZc7Yy/R2cGa35g6RKUUE3BjNIW3tfGC4F04=
github.com/ncruces/wbt v0.2.0/go.mod h1:DtF92amvMxH69EmBFUSFWRDAlo6hOEfoNQnClxj9C/c=
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo=
github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.67.3 h1:shd26MlnwTw5jksTDhC7rTQIteBxy+ZZDr3t7F2xN2Q=
github.com/prometheus/common v0.67.3/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI=
github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
github.com/psanford/sqlite3vfs v0.0.0-20240315230605-24e1d98cf361 h1:vAKifIJuYY306ZJSrwDgKonWcJGELijdaenABqbV03E=
github.com/psanford/sqlite3vfs v0.0.0-20240315230605-24e1d98cf361/go.mod h1:iW4cSew5PAb1sMZiTEkVJAIBNrepaB6jTYjeP47WtI0=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/superfly/ltx v0.5.0 h1:dXNrcT3ZtMb6iKZopIV7z5UBscnapg0b0F02loQsk5o=
github.com/superfly/ltx v0.5.0/go.mod h1:Nf50QAIXU/ET4ua3AuQ2fh31MbgNQZA7r/DYx6Os77s=
github.com/tetratelabs/wazero v1.10.1 h1:2DugeJf6VVk58KTPszlNfeeN8AhhpwcZqkJj2wwFuH8=
github.com/tetratelabs/wazero v1.10.1/go.mod h1:DRm5twOQ5Gr1AoEdSi0CLjDQF1J9ZAuyqFIjl1KKfQU=
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo=
go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc=
go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo=
go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4=
go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM=
go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc=
go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY=
golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.154.0 h1:X7QkVKZBskztmpPKWQXgjJRPA2dJYrL6r+sYPRLj050=
google.golang.org/api v0.154.0/go.mod h1:qhSMkM85hgqiokIYsrRyKxrjfBeIhgl4Z2JmeRkYylc=
google.golang.org/genproto v0.0.0-20231212172506-995d672761c0 h1:YJ5pD9rF8o9Qtta0Cmy9rdBwkSjrTCT6XTiUQVOtIos=
google.golang.org/genproto v0.0.0-20231212172506-995d672761c0/go.mod h1:l/k7rMz0vFTBPy+tFSGvXEd3z+BcoG1k7EHbqm+YBsY=
google.golang.org/genproto/googleapis/api v0.0.0-20231212172506-995d672761c0 h1:s1w3X6gQxwrLEpxnLd/qXTVLgQE2yXwaOaoa6IlY/+o=
google.golang.org/genproto/googleapis/api v0.0.0-20231212172506-995d672761c0/go.mod h1:CAny0tYF+0/9rmDB9fahA9YLzX3+AEVl1qXbv5hhj6c=
google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 h1:/jFB8jK5R3Sq3i/lmeZO0cATSzFfZaJq1J2Euan3XKU=
google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0/go.mod h1:FUoWkonphQm3RhTS+kOEhF8h0iDpm4tdXolVCeZ9KKA=
google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU=
google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM=
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

11
litestream/modernc/go.mod Normal file
View File

@@ -0,0 +1,11 @@
module modernc.org/sqlite
go 1.24.0
require github.com/ncruces/go-sqlite3 v0.30.1
require (
github.com/ncruces/julianday v1.0.0 // indirect
github.com/tetratelabs/wazero v1.10.1 // indirect
golang.org/x/sys v0.38.0 // indirect
)

10
litestream/modernc/go.sum Normal file
View File

@@ -0,0 +1,10 @@
github.com/ncruces/go-sqlite3 v0.30.1 h1:pHC3YsyRdJv4pCMB4MO1Q2BXw/CAa+Hoj7GSaKtVk+g=
github.com/ncruces/go-sqlite3 v0.30.1/go.mod h1:UVsWrQaq1qkcal5/vT5lOJnZCVlR5rsThKdwidjFsKc=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/tetratelabs/wazero v1.10.1 h1:2DugeJf6VVk58KTPszlNfeeN8AhhpwcZqkJj2wwFuH8=
github.com/tetratelabs/wazero v1.10.1/go.mod h1:DRm5twOQ5Gr1AoEdSi0CLjDQF1J9ZAuyqFIjl1KKfQU=
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=

View File

@@ -0,0 +1,20 @@
// Package sqlite provides a shim that allows Litestream to work with the ncruces SQLite driver.
package sqlite
import (
"database/sql"
"slices"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
)
func init() {
if !slices.Contains(sql.Drivers(), "sqlite") {
sql.Register("sqlite", &driver.SQLite{})
}
}
type FileControl interface {
FileControlPersistWAL(string, int) (int, error)
}

309
litestream/vfs.go Normal file
View File

@@ -0,0 +1,309 @@
package litestream
import (
"context"
"encoding/binary"
"errors"
"fmt"
"io"
"sync"
"time"
"github.com/benbjohnson/litestream"
"github.com/superfly/ltx"
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/util/vfsutil"
"github.com/ncruces/go-sqlite3/vfs"
"github.com/ncruces/wbt"
)
type liteVFS struct{}
func (liteVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
// Temp journals, as used by the sorter, use SliceFile.
if flags&vfs.OPEN_TEMP_JOURNAL != 0 {
return &vfsutil.SliceFile{}, flags | vfs.OPEN_MEMORY, nil
}
// Refuse to open all other file types.
if flags&vfs.OPEN_MAIN_DB == 0 {
return nil, flags, sqlite3.CANTOPEN
}
liteMtx.RLock()
defer liteMtx.RUnlock()
if db, ok := liteDBs[name]; ok {
// Build the page index so we can lookup individual pages.
if err := db.buildIndex(context.Background()); err != nil {
db.opts.Logger.Error("build index", "error", err)
return nil, 0, err
}
return &liteFile{db: db}, flags | vfs.OPEN_READONLY, nil
}
return nil, flags, sqlite3.CANTOPEN
}
func (liteVFS) Delete(name string, dirSync bool) error {
// notest // used to delete journals
return sqlite3.IOERR_DELETE_NOENT
}
func (liteVFS) Access(name string, flag vfs.AccessFlag) (bool, error) {
// notest // used to check for journals
return false, nil
}
func (liteVFS) FullPathname(name string) (string, error) {
return name, nil
}
type liteFile struct {
db *liteDB
conn *sqlite3.Conn
pages *pageIndex
txid ltx.TXID
pageSize uint32
}
func (f *liteFile) Close() error { return nil }
func (f *liteFile) ReadAt(p []byte, off int64) (n int, err error) {
ctx := f.context()
pages, txid := f.pages, f.txid
if pages == nil {
pages, txid, err = f.db.pollReplica(ctx)
}
if err != nil {
return 0, err
}
pgno := uint32(1)
if off >= 512 {
pgno += uint32(off / int64(f.pageSize))
}
elem, ok := pages.Get(pgno)
if !ok {
return 0, io.EOF
}
data, err := f.db.cache.getOrFetch(pgno, elem.MaxTXID, func() (any, error) {
_, data, err := litestream.FetchPage(ctx, f.db.client, elem.Level, elem.MinTXID, elem.MaxTXID, elem.Offset, elem.Size)
return data, err
})
if err != nil {
f.db.opts.Logger.Error("fetch page", "error", err)
return 0, err
}
// Update the first page to pretend we are in journal mode,
// load the page size and track changes to the database.
if pgno == 1 && len(data) >= 100 &&
data[18] >= 1 && data[19] >= 1 &&
data[18] <= 3 && data[19] <= 3 {
data[18], data[19] = 0x01, 0x01
binary.BigEndian.PutUint32(data[24:28], uint32(txid))
f.pageSize = uint32(256 * binary.LittleEndian.Uint16(data[16:18]))
}
n = copy(p, data[off%int64(len(data)):])
return n, nil
}
func (f *liteFile) WriteAt(b []byte, off int64) (n int, err error) {
// notest // OPEN_READONLY
return 0, sqlite3.IOERR_WRITE
}
func (f *liteFile) Truncate(size int64) error {
// notest // OPEN_READONLY
return sqlite3.IOERR_TRUNCATE
}
func (f *liteFile) Sync(flag vfs.SyncFlag) error {
// notest // OPEN_READONLY
return sqlite3.IOERR_FSYNC
}
func (f *liteFile) Size() (size int64, err error) {
if max := f.pages.Max(); max != nil {
size = int64(max.Key()) * int64(f.pageSize)
}
return
}
func (f *liteFile) Lock(lock vfs.LockLevel) (err error) {
if lock >= vfs.LOCK_RESERVED {
return sqlite3.IOERR_LOCK
}
f.pages, f.txid, err = f.db.pollReplica(f.context())
return err
}
func (f *liteFile) Unlock(lock vfs.LockLevel) error {
f.pages, f.txid = nil, 0
return nil
}
func (f *liteFile) CheckReservedLock() (bool, error) {
// notest // used to check for hot journals
return false, nil
}
func (f *liteFile) SectorSize() int {
// notest // safe default
return 0
}
func (f *liteFile) DeviceCharacteristics() vfs.DeviceCharacteristic {
// notest // safe default
return 0
}
func (f *liteFile) SetDB(conn any) {
f.conn = conn.(*sqlite3.Conn)
}
func (f *liteFile) context() context.Context {
if f.conn != nil {
return f.conn.GetInterrupt()
}
return context.Background()
}
type liteDB struct {
client litestream.ReplicaClient
opts ReplicaOptions
cache pageCache
pages *pageIndex // +checklocks:mtx
lastPoll time.Time // +checklocks:mtx
txids levelTXIDs // +checklocks:mtx
mtx sync.Mutex
}
func (f *liteDB) buildIndex(ctx context.Context) error {
f.mtx.Lock()
defer f.mtx.Unlock()
// Skip if we already have an index.
if f.pages != nil {
return nil
}
// Build the index from scratch from a Litestream restore plan.
infos, err := litestream.CalcRestorePlan(ctx, f.client, 0, time.Time{}, f.opts.Logger)
if err != nil {
if !errors.Is(err, litestream.ErrTxNotAvailable) {
return fmt.Errorf("calc restore plan: %w", err)
}
return nil
}
for _, info := range infos {
err := f.updateInfo(ctx, info)
if err != nil {
return err
}
}
f.lastPoll = time.Now()
return nil
}
func (f *liteDB) pollReplica(ctx context.Context) (*pageIndex, ltx.TXID, error) {
f.mtx.Lock()
defer f.mtx.Unlock()
// Limit polling interval.
if time.Since(f.lastPoll) < f.opts.PollInterval {
return f.pages, f.txids[0], nil
}
for level := range pollLevels(f.opts.MinLevel) {
if err := f.updateLevel(ctx, level); err != nil {
f.opts.Logger.Error("cannot poll replica", "error", err)
return nil, 0, err
}
}
f.lastPoll = time.Now()
return f.pages, f.txids[0], nil
}
// +checklocks:f.mtx
func (f *liteDB) updateLevel(ctx context.Context, level int) error {
var nextTXID ltx.TXID
// Snapshots must start from scratch,
// other levels can start from where they were left.
if level != litestream.SnapshotLevel {
nextTXID = f.txids[level] + 1
}
// Start reading from the next LTX file after the current position.
itr, err := f.client.LTXFiles(ctx, level, nextTXID, false)
if err != nil {
return fmt.Errorf("ltx files: %w", err)
}
defer itr.Close()
// Build an update across all new LTX files.
for itr.Next() {
info := itr.Item()
// Skip LTX files already fully loaded into the index.
if info.MaxTXID <= f.txids[level] {
continue
}
err := f.updateInfo(ctx, info)
if err != nil {
return err
}
}
if err := itr.Err(); err != nil {
return err
}
return itr.Close()
}
// +checklocks:f.mtx
func (f *liteDB) updateInfo(ctx context.Context, info *ltx.FileInfo) error {
idx, err := litestream.FetchPageIndex(ctx, f.client, info)
if err != nil {
return fmt.Errorf("fetch page index: %w", err)
}
// Replace pages in the index with new pages.
for k, v := range idx {
// Patch avoids mutating the index for an unmodified page.
f.pages = f.pages.Patch(k, func(node *pageIndex) (ltx.PageIndexElem, bool) {
return v, node == nil || v != node.Value()
})
}
// Track the MaxTXID for each level.
maxTXID := &f.txids[info.Level]
*maxTXID = max(*maxTXID, info.MaxTXID)
return nil
}
func pollLevels(minLevel int) (r []int) {
// Updating from lower to upper levels is non-racy,
// since LTX files are compacted into higher levels
// before the lower level LTX files are deleted.
// Also, only level 0 compactions and snapshots delete files,
// so the intermediate levels never need to be updated.
if minLevel <= 0 {
return append(r, 0, 1, litestream.SnapshotLevel)
}
if minLevel >= litestream.SnapshotLevel {
return append(r, litestream.SnapshotLevel)
}
return append(r, minLevel, litestream.SnapshotLevel)
}
// Type aliases; these are a mouthful.
type pageIndex = wbt.Tree[uint32, ltx.PageIndexElem]
type levelTXIDs = [litestream.SnapshotLevel + 1]ltx.TXID

34
litestream/vfs_test.go Normal file
View File

@@ -0,0 +1,34 @@
package litestream
import (
"slices"
"strconv"
"testing"
"github.com/benbjohnson/litestream"
_ "github.com/ncruces/go-sqlite3/embed"
)
func Test_pollLevels(t *testing.T) {
tests := []struct {
minLevel int
want []int
}{
{minLevel: -1, want: []int{0, 1, litestream.SnapshotLevel}},
{minLevel: 0, want: []int{0, 1, litestream.SnapshotLevel}},
{minLevel: 1, want: []int{1, litestream.SnapshotLevel}},
{minLevel: 2, want: []int{2, litestream.SnapshotLevel}},
{minLevel: 3, want: []int{3, litestream.SnapshotLevel}},
{minLevel: litestream.SnapshotLevel, want: []int{litestream.SnapshotLevel}},
{minLevel: litestream.SnapshotLevel + 1, want: []int{litestream.SnapshotLevel}},
}
for _, tt := range tests {
t.Run(strconv.Itoa(tt.minLevel), func(t *testing.T) {
got := pollLevels(tt.minLevel)
if !slices.Equal(got, tt.want) {
t.Errorf("pollLevels() = %v, want %v", got, tt.want)
}
})
}
}

View File

@@ -14,8 +14,8 @@ var (
// https://sqlite.org/c3ref/auto_extension.html
func AutoExtension(entryPoint func(*Conn) error) {
extRegistryMtx.Lock()
defer extRegistryMtx.Unlock()
extRegistry = append(extRegistry, entryPoint)
extRegistryMtx.Unlock()
}
func initExtensions(c *Conn) error {

View File

@@ -3,11 +3,11 @@ set -euo pipefail
cd -P -- "$(dirname -- "$0")"
curl -#OL "https://sqlite.org/2025/sqlite-autoconf-3510100.tar.gz"
curl -#OL "https://sqlite.org/2025/sqlite-autoconf-3510000.tar.gz"
# Verify download.
if hash=$(openssl dgst -sha3-256 sqlite-autoconf-*.tar.gz); then
if ! [[ $hash =~ 9b2b1e73f577def1d5b75c5541555a7f42e6e073ad19f7a9118478389c9bbd9b ]]; then
if ! [[ $hash =~ fa52f9cc74dbca004aa650ae698036a3350611f672649e165078f4eae21d6a2e ]]; then
echo $hash
exit 1
fi
@@ -23,7 +23,7 @@ mv sqlite-*/sqlite3.h .
mv sqlite-*/sqlite3ext.h .
rm -r sqlite-*
GITHUB_TAG="https://github.com/sqlite/sqlite/raw/version-3.51.1"
GITHUB_TAG="https://github.com/sqlite/sqlite/raw/version-3.51.0"
mkdir -p ext/
cd ext/

View File

@@ -39,10 +39,12 @@ EOF
-Wl,--export=strcspn \
-Wl,--export=strlen \
-Wl,--export=strrchr \
-Wl,--export=strspn
-Wl,--export=strspn \
-Wl,--export=qsort
"$BINARYEN/wasm-ctor-eval" -g -c _initialize libc.wasm -o libc.tmp
"$BINARYEN/wasm-opt" -g libc.tmp -o libc.wasm --converge -O3 \
"$BINARYEN/wasm-opt" -g libc.tmp -o libc.wasm \
--low-memory-unused --generate-global-effects --converge -O3 \
--enable-mutable-globals --enable-nontrapping-float-to-int \
--enable-simd --enable-bulk-memory --enable-sign-ext \
--enable-reference-types --enable-multivalue \

Binary file not shown.

View File

@@ -1,11 +1,13 @@
(module $libc.wasm
(type $0 (func (param i32 i32 i32) (result i32)))
(type $1 (func (param i32 i32) (result i32)))
(type $2 (func (param i32) (result i32)))
(type $0 (func (param i32 i32) (result i32)))
(type $1 (func (param i32 i32 i32) (result i32)))
(type $2 (func (param i32 i32 i32 i32)))
(type $3 (func (param i32) (result i32)))
(memory $0 256)
(data $.data (i32.const 4097) "\10\00\00\01\00\00\00\00\00\00\00\0c\10\00\00\0c\10\00\00\0c\10")
(data $.data.1 (i32.const 4157) "\10\00\00\00\10")
(data $0 (i32.const 4096) "\01")
(table $0 1 1 funcref)
(export "memory" (memory $0))
(export "qsort" (func $qsort))
(export "memset" (func $memset))
(export "memcpy" (func $memcpy))
(export "memmove" (func $memcpy))
@@ -18,6 +20,423 @@
(export "strrchr" (func $strrchr))
(export "strspn" (func $strspn))
(export "strcspn" (func $strcspn))
(func $qsort (param $0 i32) (param $1 i32) (param $2 i32) (param $3 i32)
(local $4 i32)
(local $5 i32)
(local $6 i32)
(local $7 i32)
(local $8 i32)
(local $9 i32)
(local $10 i32)
(local $11 i32)
(local $12 i32)
(local $13 i32)
(local $14 i32)
(local $15 i32)
(local $16 i32)
(local $17 i32)
(local $18 i32)
(local $19 i32)
(local $20 v128)
(local $scratch i32)
(if
(i32.ge_u
(local.get $1)
(i32.const 2)
)
(then
(local.set $14
(i32.mul
(local.get $1)
(local.get $2)
)
)
(local.set $15
(i32.and
(local.get $2)
(i32.const 15)
)
)
(local.set $8
(i32.and
(local.get $2)
(i32.const -16)
)
)
(local.set $16
(i32.add
(local.get $0)
(local.get $2)
)
)
(local.set $17
(i32.lt_u
(local.get $2)
(i32.const 16)
)
)
(loop $label5
(local.set $7
(i32.eq
(local.get $1)
(i32.const 2)
)
)
(local.set $18
(i32.add
(local.get $0)
(i32.mul
(local.get $2)
(i32.add
(local.tee $13
(select
(i32.const 1)
(local.tee $1
(i32.wrap_i64
(i64.div_u
(i64.sub
(i64.mul
(i64.extend_i32_u
(local.get $1)
)
(i64.const 5)
)
(i64.const 1)
)
(i64.const 11)
)
)
)
(local.get $7)
)
)
(i32.const 1)
)
)
)
)
(local.set $10
(local.tee $9
(i32.mul
(local.get $2)
(local.get $13)
)
)
)
(loop $label4
(block $block
(br_if $block
(i32.gt_u
(local.tee $4
(i32.sub
(local.get $10)
(local.get $9)
)
)
(local.get $10)
)
)
(loop $label3
(br_if $block
(i32.le_s
(call_indirect $0 (type $0)
(local.tee $5
(i32.add
(local.get $0)
(local.tee $11
(local.get $4)
)
)
)
(local.tee $4
(i32.add
(local.get $5)
(local.get $9)
)
)
(local.get $3)
)
(i32.const 0)
)
)
(block $block2
(block $block3
(block $block1
(br_if $block1
(local.get $17)
)
(br_if $block1
(i32.and
(i32.lt_u
(local.get $4)
(i32.add
(local.get $11)
(local.get $16)
)
)
(i32.gt_u
(i32.add
(local.get $11)
(local.get $18)
)
(local.get $5)
)
)
)
(local.set $4
(i32.add
(local.get $4)
(local.get $8)
)
)
(local.set $6
(i32.add
(local.get $5)
(local.get $8)
)
)
(local.set $7
(local.get $8)
)
(loop $label
(local.set $20
(v128.load align=1
(local.get $5)
)
)
(v128.store align=1
(local.get $5)
(v128.load align=1
(local.tee $12
(i32.add
(local.get $5)
(local.get $9)
)
)
)
)
(v128.store align=1
(local.get $12)
(local.get $20)
)
(local.set $5
(i32.add
(local.get $5)
(i32.const 16)
)
)
(br_if $label
(local.tee $7
(i32.sub
(local.get $7)
(i32.const 16)
)
)
)
)
(local.set $7
(local.get $15)
)
(br_if $block2
(i32.eq
(local.get $2)
(local.get $8)
)
)
(br $block3)
)
(local.set $6
(local.get $5)
)
(local.set $7
(local.get $2)
)
)
(br_if $block2
(i32.lt_u
(block (result i32)
(local.set $scratch
(i32.sub
(local.get $7)
(i32.const 1)
)
)
(if
(local.tee $5
(i32.and
(local.get $7)
(i32.const 3)
)
)
(then
(local.set $7
(i32.and
(local.get $7)
(i32.const -4)
)
)
(loop $label1
(local.set $19
(i32.load8_u
(local.get $6)
)
)
(i32.store8
(local.get $6)
(i32.load8_u
(local.get $4)
)
)
(i32.store8
(local.get $4)
(local.get $19)
)
(local.set $4
(i32.add
(local.get $4)
(i32.const 1)
)
)
(local.set $6
(i32.add
(local.get $6)
(i32.const 1)
)
)
(br_if $label1
(local.tee $5
(i32.sub
(local.get $5)
(i32.const 1)
)
)
)
)
)
)
(local.get $scratch)
)
(i32.const 3)
)
)
(loop $label2
(local.set $5
(i32.load8_u
(local.get $6)
)
)
(i32.store8
(local.get $6)
(i32.load8_u
(local.get $4)
)
)
(i32.store8
(local.get $4)
(local.get $5)
)
(local.set $5
(i32.load8_u offset=1
(local.get $6)
)
)
(i32.store8 offset=1
(local.get $6)
(i32.load8_u offset=1
(local.get $4)
)
)
(i32.store8 offset=1
(local.get $4)
(local.get $5)
)
(local.set $5
(i32.load8_u offset=2
(local.get $6)
)
)
(i32.store8 offset=2
(local.get $6)
(i32.load8_u offset=2
(local.get $4)
)
)
(i32.store8 offset=2
(local.get $4)
(local.get $5)
)
(local.set $5
(i32.load8_u offset=3
(local.get $6)
)
)
(i32.store8 offset=3
(local.get $6)
(i32.load8_u offset=3
(local.get $4)
)
)
(i32.store8 offset=3
(local.get $4)
(local.get $5)
)
(local.set $6
(i32.add
(local.get $6)
(i32.const 4)
)
)
(local.set $4
(i32.add
(local.get $4)
(i32.const 4)
)
)
(br_if $label2
(local.tee $7
(i32.sub
(local.get $7)
(i32.const 4)
)
)
)
)
)
(br_if $label3
(i32.le_u
(local.tee $4
(i32.sub
(local.get $11)
(local.get $9)
)
)
(local.get $11)
)
)
)
)
(br_if $label4
(i32.lt_u
(local.tee $10
(i32.add
(local.get $2)
(local.get $10)
)
)
(local.get $14)
)
)
)
(br_if $label5
(i32.ge_u
(local.get $13)
(i32.const 2)
)
)
)
)
)
)
(func $memset (param $0 i32) (param $1 i32) (param $2 i32) (result i32)
(if
(local.get $2)
@@ -1074,243 +1493,33 @@
(local $2 i32)
(local $3 i32)
(local $4 i32)
(local $5 v128)
(local $5 i32)
(local $6 v128)
(local $7 v128)
(local $8 v128)
(block $block2
(block $block
(br_if $block
(i32.eqz
(local.tee $3
(i32.load8_u
(local.get $1)
)
)
(local $9 v128)
(block $block
(if
(local.tee $3
(i32.load8_u
(local.get $1)
)
)
(br_if $block
(i32.eqz
(then
(br_if $block
(i32.load8_u offset=1
(local.get $1)
)
)
)
(loop $label
(v128.store
(i32.const 4080)
(local.get $6)
)
(i32.store8
(i32.or
(local.tee $3
(i32.and
(local.tee $2
(i32.load8_u
(local.get $1)
)
)
(i32.const 15)
)
)
(i32.const 4080)
)
(i32.or
(i32.load8_u
(i32.or
(local.get $3)
(i32.const 4080)
)
)
(i32.shl
(i32.const 1)
(i32.sub
(local.tee $4
(i32.shr_u
(local.get $2)
(i32.const 4)
)
)
(i32.const 8)
)
)
)
)
(v128.store
(i32.const 4064)
(local.get $5)
)
(i32.store8
(local.tee $3
(i32.or
(local.get $3)
(i32.const 4064)
)
)
(i32.or
(i32.load8_u
(local.get $3)
)
(i32.shl
(i32.const 1)
(local.get $4)
)
)
)
(local.set $1
(i32.add
(local.get $1)
(i32.const 1)
)
)
(local.set $6
(v128.load
(i32.const 4080)
)
)
(local.set $5
(v128.load
(i32.const 4064)
)
)
(br_if $label
(local.get $2)
)
)
(block $block1
(if
(v128.any_true
(local.tee $7
(v128.and
(v128.or
(i8x16.swizzle
(local.get $6)
(v128.xor
(local.tee $8
(v128.and
(local.tee $7
(v128.load
(local.tee $2
(i32.and
(local.get $0)
(i32.const -16)
)
)
)
)
(v128.const i32x4 0x8f8f8f8f 0x8f8f8f8f 0x8f8f8f8f 0x8f8f8f8f)
)
)
(v128.const i32x4 0x80808080 0x80808080 0x80808080 0x80808080)
)
)
(i8x16.swizzle
(local.get $5)
(local.get $8)
)
)
(i8x16.swizzle
(v128.const i32x4 0x08040201 0x80402010 0x08040201 0x80402010)
(i8x16.shr_u
(local.get $7)
(i32.const 4)
)
)
)
)
)
(then
(br_if $block1
(local.tee $1
(i32.and
(i32.xor
(i8x16.bitmask
(i8x16.eq
(local.get $7)
(v128.const i32x4 0x00000000 0x00000000 0x00000000 0x00000000)
)
)
(i32.const 65535)
)
(i32.shl
(i32.const -1)
(i32.and
(local.get $0)
(i32.const 15)
)
)
)
)
)
)
)
(loop $label1
(br_if $label1
(i32.eqz
(v128.any_true
(local.tee $7
(v128.and
(v128.or
(i8x16.swizzle
(local.get $6)
(v128.xor
(local.tee $8
(v128.and
(local.tee $7
(v128.load
(local.tee $2
(i32.add
(local.get $2)
(i32.const 16)
)
)
)
)
(v128.const i32x4 0x8f8f8f8f 0x8f8f8f8f 0x8f8f8f8f 0x8f8f8f8f)
)
)
(v128.const i32x4 0x80808080 0x80808080 0x80808080 0x80808080)
)
)
(i8x16.swizzle
(local.get $5)
(local.get $8)
)
)
(i8x16.swizzle
(v128.const i32x4 0x08040201 0x80402010 0x08040201 0x80402010)
(i8x16.shr_u
(local.get $7)
(i32.const 4)
)
)
)
)
)
)
)
)
(local.set $1
(i32.xor
(i8x16.bitmask
(i8x16.eq
(local.get $7)
(v128.const i32x4 0x00000000 0x00000000 0x00000000 0x00000000)
)
)
(i32.const 65535)
)
)
)
(br $block2)
)
(block $block3
(block $block1
(if
(v128.any_true
(local.tee $5
(local.tee $6
(v128.or
(i8x16.eq
(local.tee $6
(local.tee $7
(v128.load
(local.tee $2
(i32.and
@@ -1323,8 +1532,8 @@
(v128.const i32x4 0x00000000 0x00000000 0x00000000 0x00000000)
)
(i8x16.eq
(local.get $6)
(local.tee $6
(local.get $7)
(local.tee $7
(i8x16.splat
(local.get $3)
)
@@ -1334,11 +1543,11 @@
)
)
(then
(br_if $block3
(br_if $block1
(local.tee $1
(i32.and
(i8x16.bitmask
(local.get $5)
(local.get $6)
)
(i32.shl
(i32.const -1)
@@ -1352,14 +1561,14 @@
)
)
)
(loop $label2
(br_if $label2
(loop $label
(br_if $label
(i32.eqz
(v128.any_true
(local.tee $5
(local.tee $6
(v128.or
(i8x16.eq
(local.tee $5
(local.tee $6
(v128.load
(local.tee $2
(i32.add
@@ -1372,8 +1581,8 @@
(v128.const i32x4 0x00000000 0x00000000 0x00000000 0x00000000)
)
(i8x16.eq
(local.get $5)
(local.get $6)
(local.get $7)
)
)
)
@@ -1383,10 +1592,230 @@
)
(local.set $1
(i8x16.bitmask
(local.get $6)
)
)
)
(return
(i32.add
(i32.ctz
(local.get $1)
)
(i32.sub
(local.get $2)
(local.get $0)
)
)
)
)
(local.set $4
(i32.and
(local.get $0)
(i32.const 15)
)
)
(loop $label1
(v128.store
(i32.const 4080)
(local.get $7)
)
(i32.store8
(i32.or
(local.tee $3
(i32.and
(local.tee $2
(i32.load8_u
(local.get $1)
)
)
(i32.const 15)
)
)
(i32.const 4080)
)
(i32.or
(i32.load8_u
(i32.or
(local.get $3)
(i32.const 4080)
)
)
(i32.shl
(i32.const 1)
(i32.sub
(local.tee $5
(i32.shr_u
(local.get $2)
(i32.const 4)
)
)
(i32.const 8)
)
)
)
)
(v128.store
(i32.const 4064)
(local.get $6)
)
(i32.store8
(local.tee $3
(i32.or
(local.get $3)
(i32.const 4064)
)
)
(i32.or
(i32.load8_u
(local.get $3)
)
(i32.shl
(i32.const 1)
(local.get $5)
)
)
)
(local.set $1
(i32.add
(local.get $1)
(i32.const 1)
)
)
(local.set $7
(v128.load
(i32.const 4080)
)
)
(local.set $6
(v128.load
(i32.const 4064)
)
)
(br_if $label1
(local.get $2)
)
)
(block $block2
(if
(v128.any_true
(local.tee $8
(v128.and
(v128.or
(i8x16.swizzle
(local.get $7)
(v128.xor
(local.tee $9
(v128.and
(local.tee $8
(v128.load
(local.tee $2
(i32.and
(local.get $0)
(i32.const -16)
)
)
)
)
(v128.const i32x4 0x8f8f8f8f 0x8f8f8f8f 0x8f8f8f8f 0x8f8f8f8f)
)
)
(v128.const i32x4 0x80808080 0x80808080 0x80808080 0x80808080)
)
)
(i8x16.swizzle
(local.get $6)
(local.get $9)
)
)
(i8x16.swizzle
(v128.const i32x4 0x08040201 0x80402010 0x08040201 0x80402010)
(i8x16.shr_u
(local.get $8)
(i32.const 4)
)
)
)
)
)
(then
(br_if $block2
(local.tee $1
(i32.and
(i32.xor
(i8x16.bitmask
(i8x16.eq
(local.get $8)
(v128.const i32x4 0x00000000 0x00000000 0x00000000 0x00000000)
)
)
(i32.const 65535)
)
(i32.shl
(i32.const -1)
(local.get $4)
)
)
)
)
)
)
(loop $label2
(br_if $label2
(i32.eqz
(v128.any_true
(local.tee $8
(v128.and
(v128.or
(i8x16.swizzle
(local.get $7)
(v128.xor
(local.tee $9
(v128.and
(local.tee $8
(v128.load
(local.tee $2
(i32.add
(local.get $2)
(i32.const 16)
)
)
)
)
(v128.const i32x4 0x8f8f8f8f 0x8f8f8f8f 0x8f8f8f8f 0x8f8f8f8f)
)
)
(v128.const i32x4 0x80808080 0x80808080 0x80808080 0x80808080)
)
)
(i8x16.swizzle
(local.get $6)
(local.get $9)
)
)
(i8x16.swizzle
(v128.const i32x4 0x08040201 0x80402010 0x08040201 0x80402010)
(i8x16.shr_u
(local.get $8)
(i32.const 4)
)
)
)
)
)
)
)
)
(local.set $1
(i32.xor
(i8x16.bitmask
(i8x16.eq
(local.get $8)
(v128.const i32x4 0x00000000 0x00000000 0x00000000 0x00000000)
)
)
(i32.const 65535)
)
)
)
(i32.add
(i32.ctz

View File

@@ -189,23 +189,13 @@ func TestConn_FileControl(t *testing.T) {
}
})
t.Run("FCNTL_VFSNAME", func(t *testing.T) {
o, err := db.FileControl("", sqlite3.FCNTL_VFSNAME)
if err != nil {
t.Fatal(err)
}
if o != "os" {
t.Errorf(`got %q, want "os"`, o)
}
})
t.Run("FCNTL_VFS_POINTER", func(t *testing.T) {
o, err := db.FileControl("", sqlite3.FCNTL_VFS_POINTER)
if err != nil {
t.Fatal(err)
}
if o != vfs.Find("os") {
t.Errorf(`got %v, want "os"`, o)
t.Errorf("got %v, want os", o)
}
})

14
time.go
View File

@@ -157,13 +157,11 @@ func (f TimeFormat) Decode(v any) (time.Time, error) {
case TimeFormatUnix, TimeFormatUnixFrac:
if s, ok := v.(string); ok {
if i, err := strconv.ParseInt(s, 10, 64); err == nil {
v = i
} else if f, ok := util.ParseFloat(s); ok {
v = f
} else {
return time.Time{}, util.TimeErr
f, err := strconv.ParseFloat(s, 64)
if err != nil {
return time.Time{}, err
}
v = f
}
switch v := v.(type) {
case float64:
@@ -236,8 +234,8 @@ func (f TimeFormat) Decode(v any) (time.Time, error) {
v = i
break
}
f, ok := util.ParseFloat(s)
if ok {
f, err := strconv.ParseFloat(s, 64)
if err == nil {
v = f
break
}

65
util/sql3util/arg.go Normal file
View File

@@ -0,0 +1,65 @@
package sql3util
import "strings"
// NamedArg splits an named arg into a key and value,
// around an equals sign.
// Spaces are trimmed around both key and value.
func NamedArg(arg string) (key, val string) {
key, val, _ = strings.Cut(arg, "=")
key = strings.TrimSpace(key)
val = strings.TrimSpace(val)
return
}
// Unquote unquotes a string.
//
// https://sqlite.org/lang_keywords.html
func Unquote(val string) string {
if len(val) < 2 {
return val
}
fst := val[0]
lst := val[len(val)-1]
rst := val[1 : len(val)-1]
if fst == '[' && lst == ']' {
return rst
}
if fst != lst {
return val
}
var old, new string
switch fst {
default:
return val
case '`':
old, new = "``", "`"
case '"':
old, new = `""`, `"`
case '\'':
old, new = `''`, `'`
}
return strings.ReplaceAll(rst, old, new)
}
// ParseBool parses a boolean.
//
// https://sqlite.org/pragma.html#syntax
func ParseBool(s string) (b, ok bool) {
if len(s) == 0 {
return false, false
}
if s[0] == '0' {
return false, true
}
if '1' <= s[0] && s[0] <= '9' {
return true, true
}
switch strings.ToLower(s) {
case "true", "yes", "on":
return true, true
case "false", "no", "off":
return false, true
}
return false, false
}

55
util/sql3util/arg_test.go Normal file
View File

@@ -0,0 +1,55 @@
package sql3util_test
import (
"testing"
"github.com/ncruces/go-sqlite3/util/sql3util"
)
func TestUnquote(t *testing.T) {
tests := []struct {
val string
want string
}{
{"a", "a"},
{"abc", "abc"},
{"abba", "abba"},
{"`ab``c`", "ab`c"},
{"'ab''c'", "ab'c"},
{"'ab``c'", "ab``c"},
{"[ab``c]", "ab``c"},
{`"ab""c"`, `ab"c`},
}
for _, tt := range tests {
t.Run(tt.val, func(t *testing.T) {
if got := sql3util.Unquote(tt.val); got != tt.want {
t.Errorf("Unquote(%s) = %s, want %s", tt.val, got, tt.want)
}
})
}
}
func TestParseBool(t *testing.T) {
tests := []struct {
str string
val bool
ok bool
}{
{"", false, false},
{"0", false, true},
{"1", true, true},
{"9", true, true},
{"T", false, false},
{"true", true, true},
{"FALSE", false, true},
{"false?", false, false},
}
for _, tt := range tests {
t.Run(tt.str, func(t *testing.T) {
gotVal, gotOK := sql3util.ParseBool(tt.str)
if gotVal != tt.val || gotOK != tt.ok {
t.Errorf("ParseBool(%q) = (%v, %v) want (%v, %v)", tt.str, gotVal, gotOK, tt.val, tt.ok)
}
})
}
}

View File

@@ -49,15 +49,6 @@ const (
DEFTYPE_NOTDEFERRABLE_INITIALLY_IMMEDIATE
)
type ConstraintType uint32
const (
TABLECONSTRAINT_PRIMARYKEY ConstraintType = iota
TABLECONSTRAINT_UNIQUE
TABLECONSTRAINT_CHECK
TABLECONSTRAINT_FOREIGNKEY
)
type StatementType uint32
const (
@@ -68,11 +59,3 @@ const (
ALTER_ADD_COLUMN
ALTER_DROP_COLUMN
)
type GenType uint32
const (
GENTYPE_NONE GenType = iota
GENTYPE_STORED
GENTYPE_VIRTUA
)

View File

@@ -3,13 +3,11 @@ package sql3util
import (
"context"
_ "embed"
"strings"
"sync"
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/internal/util"
)
@@ -31,10 +29,6 @@ var (
// [CREATE]: https://sqlite.org/lang_createtable.html
// [ALTER TABLE]: https://sqlite.org/lang_altertable.html
func ParseTable(sql string) (_ *Table, err error) {
if len(sql) > 8192 {
return nil, sqlite3.TOOBIG
}
once.Do(func() {
ctx := context.Background()
cfg := wazero.NewRuntimeConfigInterpreter()
@@ -87,15 +81,14 @@ type Table struct {
IsWithoutRowID bool
IsStrict bool
Columns []Column
Constraints []TableConstraint
Type StatementType
CurrentName string
NewName string
}
func (t *Table) load(mod api.Module, ptr uint32, sql string) {
t.Name = loadIdentifier(mod, ptr+0, sql)
t.Schema = loadIdentifier(mod, ptr+8, sql)
t.Name = loadString(mod, ptr+0, sql)
t.Schema = loadString(mod, ptr+8, sql)
t.Comment = loadString(mod, ptr+16, sql)
t.IsTemporary = loadBool(mod, ptr+24)
@@ -103,21 +96,14 @@ func (t *Table) load(mod api.Module, ptr uint32, sql string) {
t.IsWithoutRowID = loadBool(mod, ptr+26)
t.IsStrict = loadBool(mod, ptr+27)
t.Columns = loadSlice(mod, ptr+28, func(ptr uint32, ret *Column) uint32 {
t.Columns = loadSlice(mod, ptr+28, func(ptr uint32, ret *Column) {
p, _ := mod.Memory().ReadUint32Le(ptr)
ret.load(mod, p, sql)
return 4
})
t.Constraints = loadSlice(mod, ptr+36, func(ptr uint32, ret *TableConstraint) uint32 {
p, _ := mod.Memory().ReadUint32Le(ptr)
ret.load(mod, p, sql)
return 4
})
t.Type = loadEnum[StatementType](mod, ptr+44)
t.CurrentName = loadIdentifier(mod, ptr+48, sql)
t.NewName = loadIdentifier(mod, ptr+56, sql)
t.CurrentName = loadString(mod, ptr+48, sql)
t.NewName = loadString(mod, ptr+56, sql)
}
// Column holds metadata about a column.
@@ -139,15 +125,13 @@ type Column struct {
DefaultExpr string
CollateName string
ForeignKeyClause *ForeignKey
GeneratedExpr string
GeneratedType GenType
}
func (c *Column) load(mod api.Module, ptr uint32, sql string) {
c.Name = loadIdentifier(mod, ptr+0, sql)
c.Name = loadString(mod, ptr+0, sql)
c.Type = loadString(mod, ptr+8, sql)
c.Length = loadString(mod, ptr+16, sql)
c.ConstraintName = loadIdentifier(mod, ptr+24, sql)
c.ConstraintName = loadString(mod, ptr+24, sql)
c.Comment = loadString(mod, ptr+32, sql)
c.IsPrimaryKey = loadBool(mod, ptr+40)
@@ -162,94 +146,36 @@ func (c *Column) load(mod api.Module, ptr uint32, sql string) {
c.CheckExpr = loadString(mod, ptr+60, sql)
c.DefaultExpr = loadString(mod, ptr+68, sql)
c.CollateName = loadIdentifier(mod, ptr+76, sql)
c.CollateName = loadString(mod, ptr+76, sql)
if ptr, _ := mod.Memory().ReadUint32Le(ptr + 84); ptr != 0 {
c.ForeignKeyClause = &ForeignKey{}
c.ForeignKeyClause.load(mod, ptr, sql)
}
c.GeneratedExpr = loadString(mod, ptr+88, sql)
c.GeneratedType = loadEnum[GenType](mod, ptr+96)
}
// TableConstraint holds metadata about a table key constraint.
type TableConstraint struct {
Type ConstraintType
Name string
// Type is TABLECONSTRAINT_PRIMARYKEY or TABLECONSTRAINT_UNIQUE
IndexedColumns []IdxColumn
ConflictClause ConflictClause
IsAutoIncrement bool
// Type is TABLECONSTRAINT_CHECK
CheckExpr string
// Type is TABLECONSTRAINT_FOREIGNKEY
ForeignKeyNames []string
ForeignKeyClause *ForeignKey
}
func (c *TableConstraint) load(mod api.Module, ptr uint32, sql string) {
c.Type = loadEnum[ConstraintType](mod, ptr+0)
c.Name = loadIdentifier(mod, ptr+4, sql)
switch c.Type {
case TABLECONSTRAINT_PRIMARYKEY, TABLECONSTRAINT_UNIQUE:
c.IndexedColumns = loadSlice(mod, ptr+12, func(ptr uint32, ret *IdxColumn) uint32 {
ret.load(mod, ptr, sql)
return 20
})
c.ConflictClause = loadEnum[ConflictClause](mod, ptr+20)
c.IsAutoIncrement = loadBool(mod, ptr+24)
case TABLECONSTRAINT_CHECK:
c.CheckExpr = loadString(mod, ptr+12, sql)
case TABLECONSTRAINT_FOREIGNKEY:
c.ForeignKeyNames = loadSlice(mod, ptr+12, func(ptr uint32, ret *string) uint32 {
*ret = loadIdentifier(mod, ptr, sql)
return 8
})
if ptr, _ := mod.Memory().ReadUint32Le(ptr + 20); ptr != 0 {
c.ForeignKeyClause = &ForeignKey{}
c.ForeignKeyClause.load(mod, ptr, sql)
}
}
}
// ForeignKey holds metadata about a foreign key constraint.
type ForeignKey struct {
Table string
ColumnNames []string
OnDelete FKAction
OnUpdate FKAction
Match string
Deferrable FKDefType
Table string
Columns []string
OnDelete FKAction
OnUpdate FKAction
Match string
Deferrable FKDefType
}
func (f *ForeignKey) load(mod api.Module, ptr uint32, sql string) {
f.Table = loadIdentifier(mod, ptr+0, sql)
f.Table = loadString(mod, ptr+0, sql)
f.ColumnNames = loadSlice(mod, ptr+8, func(ptr uint32, ret *string) uint32 {
*ret = loadIdentifier(mod, ptr, sql)
return 8
f.Columns = loadSlice(mod, ptr+8, func(ptr uint32, ret *string) {
*ret = loadString(mod, ptr, sql)
})
f.OnDelete = loadEnum[FKAction](mod, ptr+16)
f.OnUpdate = loadEnum[FKAction](mod, ptr+20)
f.Match = loadIdentifier(mod, ptr+24, sql)
f.Match = loadString(mod, ptr+24, sql)
f.Deferrable = loadEnum[FKDefType](mod, ptr+32)
}
// IdxColumn holds metadata about an indexed column.
type IdxColumn struct {
Name string
CollateName string
Order OrderClause
}
func (c *IdxColumn) load(mod api.Module, ptr uint32, sql string) {
c.Name = loadIdentifier(mod, ptr+0, sql)
c.CollateName = loadIdentifier(mod, ptr+8, sql)
c.Order = loadEnum[OrderClause](mod, ptr+16)
}
func loadString(mod api.Module, ptr uint32, sql string) string {
off, _ := mod.Memory().ReadUint32Le(ptr + 0)
if off == 0 {
@@ -259,28 +185,7 @@ func loadString(mod api.Module, ptr uint32, sql string) string {
return sql[off-sqlp : off+len-sqlp]
}
func loadIdentifier(mod api.Module, ptr uint32, sql string) string {
off, _ := mod.Memory().ReadUint32Le(ptr + 0)
if off == 0 {
return ""
}
var old, new string
len, _ := mod.Memory().ReadUint32Le(ptr + 4)
str := sql[off-sqlp : off+len-sqlp]
switch sql[off-sqlp-1] {
default:
return str
case '`':
old, new = "``", "`"
case '"':
old, new = `""`, `"`
case '\'':
old, new = `''`, `'`
}
return strings.ReplaceAll(str, old, new)
}
func loadSlice[T any](mod api.Module, ptr uint32, fn func(uint32, *T) uint32) []T {
func loadSlice[T any](mod api.Module, ptr uint32, fn func(uint32, *T)) []T {
ref, _ := mod.Memory().ReadUint32Le(ptr + 4)
if ref == 0 {
return nil
@@ -288,7 +193,8 @@ func loadSlice[T any](mod api.Module, ptr uint32, fn func(uint32, *T) uint32) []
len, _ := mod.Memory().ReadUint32Le(ptr + 0)
ret := make([]T, len)
for i := range ret {
ref += fn(ref, &ret[i])
fn(ref, &ret[i])
ref += 4
}
return ret
}

View File

@@ -6,8 +6,8 @@ import (
"github.com/ncruces/go-sqlite3/util/sql3util"
)
func TestParse_references(t *testing.T) {
tab, err := sql3util.ParseTable("CREATE TABLE child(`x` INT REFERENCES parent)")
func TestParse(t *testing.T) {
tab, err := sql3util.ParseTable(`CREATE TABLE child(x REFERENCES parent)`)
if err != nil {
t.Fatal(err)
}
@@ -15,94 +15,17 @@ func TestParse_references(t *testing.T) {
if got := tab.Name; got != "child" {
t.Errorf("got %s, want child", got)
}
if got := len(tab.Columns); got != 1 {
t.Errorf("got %d, want 1", got)
}
col := tab.Columns[0]
if got := col.Name; got != "x" {
t.Errorf("got %s, want x", got)
}
if got := col.Type; got != "INT" {
t.Errorf("got %s, want INT", got)
}
fk := col.ForeignKeyClause
if got := fk.Table; got != "parent" {
t.Errorf("got %s, want parent", got)
}
}
func TestParse_constraint(t *testing.T) {
tab, err := sql3util.ParseTable(`CREATE TABLE child('x', 'y', PRIMARY KEY('x', 'y'))`)
if err != nil {
t.Fatal(err)
}
if got := tab.Name; got != "child" {
t.Errorf("got %s, want child", got)
}
if got := len(tab.Columns); got != 2 {
t.Errorf("got %d, want 2", got)
}
if got := tab.Columns[0].Name; got != "x" {
t.Errorf("got %s, want x", got)
}
if got := tab.Columns[1].Name; got != "y" {
t.Errorf("got %s, want y", got)
}
if got := len(tab.Constraints); got != 1 {
t.Errorf("got %d, want 1", got)
}
if got := tab.Constraints[0].Type; got != sql3util.TABLECONSTRAINT_PRIMARYKEY {
t.Errorf("got %d, want primary key", got)
}
if got := len(tab.Constraints[0].IndexedColumns); got != 2 {
t.Errorf("got %d, want 2", got)
}
if got := tab.Constraints[0].IndexedColumns[0].Name; got != "x" {
t.Errorf("got %s, want x", got)
}
if got := tab.Constraints[0].IndexedColumns[1].Name; got != "y" {
t.Errorf("got %s, want y", got)
}
}
func TestParse_foreign(t *testing.T) {
tab, err := sql3util.ParseTable(`CREATE TABLE child(x, y, FOREIGN KEY (x, y) REFERENCES "parent")`)
if err != nil {
t.Fatal(err)
}
if got := tab.Name; got != "child" {
t.Errorf("got %s, want child", got)
}
if got := len(tab.Columns); got != 2 {
t.Errorf("got %d, want 2", got)
}
if got := tab.Columns[0].Name; got != "x" {
t.Errorf("got %s, want x", got)
}
if got := tab.Columns[1].Name; got != "y" {
t.Errorf("got %s, want y", got)
}
if got := len(tab.Constraints); got != 1 {
t.Errorf("got %d, want 1", got)
}
if got := tab.Constraints[0].Type; got != sql3util.TABLECONSTRAINT_FOREIGNKEY {
t.Errorf("got %d, want foreign key", got)
}
if got := len(tab.Constraints[0].ForeignKeyNames); got != 2 {
t.Errorf("got %d, want 2", got)
}
if got := tab.Constraints[0].ForeignKeyNames[0]; got != "x" {
t.Errorf("got %s, want x", got)
}
if got := tab.Constraints[0].ForeignKeyNames[1]; got != "y" {
t.Errorf("got %s, want y", got)
}
}

View File

@@ -0,0 +1,9 @@
// Package sql3util implements SQLite utilities.
package sql3util
// ValidPageSize returns true if s is a valid page size.
//
// https://sqlite.org/fileformat.html#pages
func ValidPageSize(s int) bool {
return s&(s-1) == 0 && 512 <= s && s <= 65536
}

View File

@@ -1,112 +0,0 @@
// Package sql3util implements SQLite utilities.
package sql3util
import (
"strings"
"time"
"github.com/ncruces/go-sqlite3/internal/util"
)
// Unquote unquotes a string.
//
// https://sqlite.org/lang_keywords.html
func Unquote(val string) string {
if len(val) < 2 {
return val
}
fst := val[0]
lst := val[len(val)-1]
rst := val[1 : len(val)-1]
if fst == '[' && lst == ']' {
return rst
}
if fst != lst {
return val
}
var old, new string
switch fst {
default:
return val
case '`':
old, new = "``", "`"
case '"':
old, new = `""`, `"`
case '\'':
old, new = `''`, `'`
}
return strings.ReplaceAll(rst, old, new)
}
// NamedArg splits an named arg into a key and value,
// around an equals sign.
// Spaces are trimmed around both key and value.
func NamedArg(arg string) (key, val string) {
key, val, _ = strings.Cut(arg, "=")
key = strings.TrimSpace(key)
val = strings.TrimSpace(val)
return
}
// ParseBool parses a boolean.
//
// https://sqlite.org/pragma.html#syntax
func ParseBool(s string) (b, ok bool) {
return util.ParseBool(s)
}
// ParseFloat parses a decimal floating point number.
func ParseFloat(s string) (f float64, ok bool) {
return util.ParseFloat(s)
}
// ParseTimeShift parses a time shift modifier,
// also the output of timediff.
//
// https://sqlite.org/lang_datefunc.html
func ParseTimeShift(s string) (years, months, days int, duration time.Duration, ok bool) {
return util.ParseTimeShift(s)
}
// ValidPageSize returns true if s is a valid page size.
//
// https://sqlite.org/fileformat.html#pages
func ValidPageSize(s int) bool {
return util.ValidPageSize(s)
}
// Affinity is the type affinity of a column.
//
// https://sqlite.org/datatype3.html#type_affinity
type Affinity byte
const (
TEXT Affinity = iota
NUMERIC
INTEGER
REAL
BLOB
)
// GetAffinity determines the affinity of a column by the declared type of the column.
//
// https://sqlite.org/datatype3.html#determination_of_column_affinity
func GetAffinity(declType string) Affinity {
if declType == "" {
return BLOB
}
name := strings.ToUpper(declType)
if strings.Contains(name, "INT") {
return INTEGER
}
if strings.Contains(name, "CHAR") || strings.Contains(name, "CLOB") || strings.Contains(name, "TEXT") {
return TEXT
}
if strings.Contains(name, "BLOB") {
return BLOB
}
if strings.Contains(name, "REAL") || strings.Contains(name, "FLOA") || strings.Contains(name, "DOUB") {
return REAL
}
return NUMERIC
}

View File

@@ -1,60 +0,0 @@
package sql3util_test
import (
"testing"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/ncruces/go-sqlite3/util/sql3util"
)
func TestUnquote(t *testing.T) {
tests := []struct {
val string
want string
}{
{"a", "a"},
{"abc", "abc"},
{"abba", "abba"},
{"`ab``c`", "ab`c"},
{"'ab''c'", "ab'c"},
{"'ab``c'", "ab``c"},
{"[ab``c]", "ab``c"},
{`"ab""c"`, `ab"c`},
}
for _, tt := range tests {
t.Run(tt.val, func(t *testing.T) {
if got := sql3util.Unquote(tt.val); got != tt.want {
t.Errorf("Unquote(%s) = %s, want %s", tt.val, got, tt.want)
}
})
}
}
func TestGetAffinity(t *testing.T) {
tests := []struct {
decl string
want sql3util.Affinity
}{
{"", sql3util.BLOB},
{"INTEGER", sql3util.INTEGER},
{"TINYINT", sql3util.INTEGER},
{"TEXT", sql3util.TEXT},
{"CHAR", sql3util.TEXT},
{"CLOB", sql3util.TEXT},
{"BLOB", sql3util.BLOB},
{"REAL", sql3util.REAL},
{"FLOAT", sql3util.REAL},
{"DOUBLE", sql3util.REAL},
{"NUMERIC", sql3util.NUMERIC},
{"DECIMAL", sql3util.NUMERIC},
{"BOOLEAN", sql3util.NUMERIC},
{"DATETIME", sql3util.NUMERIC},
}
for _, tt := range tests {
t.Run(tt.decl, func(t *testing.T) {
if got := sql3util.GetAffinity(tt.decl); got != tt.want {
t.Errorf("GetAffinity() = %v, want %v", got, tt.want)
}
})
}
}

View File

@@ -25,8 +25,8 @@ trap 'rm -f sql3parse_table.tmp' EXIT
"$BINARYEN/wasm-ctor-eval" -c _initialize sql3parse_table.wasm -o sql3parse_table.tmp
"$BINARYEN/wasm-opt" sql3parse_table.tmp -o sql3parse_table.wasm \
--gufa --generate-global-effects --converge -Oz \
--low-memory-unused --gufa --generate-global-effects --converge -Oz \
--enable-mutable-globals --enable-nontrapping-float-to-int \
--enable-simd --enable-bulk-memory --enable-sign-ext \
--enable-reference-types --enable-multivalue \
--strip --strip-producers
--strip --strip-debug --strip-producers

View File

@@ -11,8 +11,6 @@ static_assert(offsetof(sql3table, is_withoutrowid) == 26, "Unexpected offset");
static_assert(offsetof(sql3table, is_strict) == 27, "Unexpected offset");
static_assert(offsetof(sql3table, num_columns) == 28, "Unexpected offset");
static_assert(offsetof(sql3table, columns) == 32, "Unexpected offset");
static_assert(offsetof(sql3table, num_constraint) == 36, "Unexpected offset");
static_assert(offsetof(sql3table, constraints) == 40, "Unexpected offset");
static_assert(offsetof(sql3table, type) == 44, "Unexpected offset");
static_assert(offsetof(sql3table, current_name) == 48, "Unexpected offset");
static_assert(offsetof(sql3table, new_name) == 56, "Unexpected offset");
@@ -34,8 +32,6 @@ static_assert(offsetof(sql3column, check_expr) == 60, "Unexpected offset");
static_assert(offsetof(sql3column, default_expr) == 68, "Unexpected offset");
static_assert(offsetof(sql3column, collate_name) == 76, "Unexpected offset");
static_assert(offsetof(sql3column, foreignkey_clause) == 84, "Unexpected offset");
static_assert(offsetof(sql3column, generated_expr) == 88, "Unexpected offset");
static_assert(offsetof(sql3column, generated_type) == 96, "Unexpected offset");
static_assert(offsetof(sql3foreignkey, table) == 0, "Unexpected offset");
static_assert(offsetof(sql3foreignkey, num_columns) == 8, "Unexpected offset");
@@ -43,20 +39,4 @@ static_assert(offsetof(sql3foreignkey, column_name) == 12, "Unexpected offset");
static_assert(offsetof(sql3foreignkey, on_delete) == 16, "Unexpected offset");
static_assert(offsetof(sql3foreignkey, on_update) == 20, "Unexpected offset");
static_assert(offsetof(sql3foreignkey, match) == 24, "Unexpected offset");
static_assert(offsetof(sql3foreignkey, deferrable) == 32, "Unexpected offset");
static_assert(offsetof(sql3tableconstraint, type) == 0, "Unexpected offset");
static_assert(offsetof(sql3tableconstraint, name) == 4, "Unexpected offset");
static_assert(offsetof(sql3tableconstraint, num_indexed) == 12, "Unexpected offset");
static_assert(offsetof(sql3tableconstraint, indexed_columns) == 16, "Unexpected offset");
static_assert(offsetof(sql3tableconstraint, conflict_clause) == 20, "Unexpected offset");
static_assert(offsetof(sql3tableconstraint, is_autoincrement) == 24, "Unexpected offset");
static_assert(offsetof(sql3tableconstraint, check_expr) == 12, "Unexpected offset");
static_assert(offsetof(sql3tableconstraint, foreignkey_num) == 12, "Unexpected offset");
static_assert(offsetof(sql3tableconstraint, foreignkey_name) == 16, "Unexpected offset");
static_assert(offsetof(sql3tableconstraint, foreignkey_clause) == 20, "Unexpected offset");
static_assert(offsetof(sql3idxcolumn, name) == 0, "Unexpected offset");
static_assert(offsetof(sql3idxcolumn, collate_name) == 8, "Unexpected offset");
static_assert(offsetof(sql3idxcolumn, order) == 16, "Unexpected offset");
static_assert(sizeof(sql3idxcolumn) == 20, "Unexpected size");
static_assert(offsetof(sql3foreignkey, deferrable) == 32, "Unexpected offset");

View File

@@ -125,5 +125,5 @@ The VFS can be customized with a few build tags:
wraps a VFS to offer encryption at rest.
- [`github.com/ncruces/go-sqlite3/vfs/xts`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/xts)
wraps a VFS to offer encryption at rest.
- [`github.com/ncruces/litestream`](https://pkg.go.dev/github.com/ncruces/litestream)
- [`github.com/ncruces/go-sqlite3/litestream`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/litestream)
implements Litestream [lightweight read-replicas](https://fly.io/blog/litestream-revamped/#lightweight-read-replicas).

View File

@@ -1,3 +1,5 @@
//go:build linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock || sqlite3_dotlk
package adiantum_test
import (
@@ -15,7 +17,7 @@ import (
"github.com/ncruces/go-sqlite3/vfs/adiantum"
)
func Example_hPolyC() {
func ExampleRegister_hpolyc() {
vfs.Register("hpolyc", adiantum.Wrap(vfs.Find(""), hpolycCreator{}))
db, err := sqlite3.Open("file:demo.db?vfs=hpolyc" +

View File

@@ -9,6 +9,7 @@ import (
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/util/sql3util"
)
func cksmWrapFile(file File, flags OpenFlag) File {
@@ -34,7 +35,7 @@ func (c *cksmFile) ReadAt(p []byte, off int64) (n int, err error) {
}
// Verify checksums.
if c.verifyCksm && util.ValidPageSize(len(p)) {
if c.verifyCksm && sql3util.ValidPageSize(len(p)) {
cksm1 := cksmCompute(p[:len(p)-8])
cksm2 := *(*[8]byte)(p[len(p)-8:])
if cksm1 != cksm2 {
@@ -50,7 +51,7 @@ func (c *cksmFile) WriteAt(p []byte, off int64) (n int, err error) {
}
// Compute checksums.
if c.computeCksm && util.ValidPageSize(len(p)) {
if c.computeCksm && sql3util.ValidPageSize(len(p)) {
*(*[8]byte)(p[len(p)-8:]) = cksmCompute(p[:len(p)-8])
}
@@ -60,7 +61,7 @@ func (c *cksmFile) WriteAt(p []byte, off int64) (n int, err error) {
func (c *cksmFile) Pragma(name string, value string) (string, error) {
switch name {
case "checksum_verification":
b, ok := util.ParseBool(value)
b, ok := sql3util.ParseBool(value)
if ok {
c.verifyCksm = b && c.computeCksm
}

View File

@@ -239,7 +239,6 @@ const (
_FCNTL_RESET_CACHE _FcntlOpcode = 42
_FCNTL_NULL_IO _FcntlOpcode = 43
_FCNTL_BLOCK_ON_CONNECT _FcntlOpcode = 44
_FCNTL_FILESTAT _FcntlOpcode = 45
)
// https://sqlite.org/c3ref/c_shm_exclusive.html

View File

@@ -34,6 +34,9 @@ var (
// The new database takes ownership of data,
// and the caller should not use data after this call.
func Create(name string, data []byte) {
memoryMtx.Lock()
defer memoryMtx.Unlock()
db := &memDB{
refs: 1,
name: name,
@@ -60,16 +63,14 @@ func Create(name string, data []byte) {
}
}
memoryMtx.Lock()
memoryDBs[name] = db
memoryMtx.Unlock()
}
// Delete deletes a shared memory database.
func Delete(name string) {
memoryMtx.Lock()
defer memoryMtx.Unlock()
delete(memoryDBs, name)
memoryMtx.Unlock()
}
// TestDB creates an empty shared memory database for the test to use.

View File

@@ -92,10 +92,10 @@ type memDB struct {
func (m *memDB) release() {
memoryMtx.Lock()
defer memoryMtx.Unlock()
if m.refs--; m.refs == 0 && m == memoryDBs[m.name] {
delete(memoryDBs, m.name)
}
memoryMtx.Unlock()
}
type memFile struct {
@@ -223,7 +223,7 @@ func (m *memFile) Lock(lock vfs.LockLevel) error {
m.reserved = true
case vfs.LOCK_EXCLUSIVE:
if m.lock == vfs.LOCK_RESERVED {
if m.lock < vfs.LOCK_PENDING {
m.lock = vfs.LOCK_PENDING
m.pending = true
}

View File

@@ -35,12 +35,13 @@ var (
// using a snapshot as its initial contents.
func Create(name string, snapshot Snapshot) {
memoryMtx.Lock()
defer memoryMtx.Unlock()
memoryDBs[name] = &mvccDB{
refs: 1,
name: name,
data: snapshot.Tree,
}
memoryMtx.Unlock()
}
// Delete deletes a shared memory database.
@@ -48,8 +49,8 @@ func Delete(name string) {
name = getName(name)
memoryMtx.Lock()
defer memoryMtx.Unlock()
delete(memoryDBs, name)
memoryMtx.Unlock()
}
// Snapshot represents a database snapshot.
@@ -82,9 +83,8 @@ func TakeSnapshot(name string) Snapshot {
name = getName(name)
memoryMtx.Lock()
defer memoryMtx.Unlock()
db := memoryDBs[name]
memoryMtx.Unlock()
if db == nil {
return Snapshot{}
}

View File

@@ -79,10 +79,10 @@ type mvccDB struct {
func (m *mvccDB) release() {
memoryMtx.Lock()
defer memoryMtx.Unlock()
if m.refs--; m.refs == 0 && m == memoryDBs[m.name] {
delete(memoryDBs, m.name)
}
memoryMtx.Unlock()
}
type mvccFile struct {
@@ -105,10 +105,10 @@ func (m *mvccFile) Close() error {
m.data = nil
m.lock = vfs.LOCK_NONE
m.mtx.Lock()
defer m.mtx.Unlock()
if m.owner == m {
m.owner = nil
}
m.mtx.Unlock()
return nil
}
@@ -252,8 +252,9 @@ func (m *mvccFile) Lock(lock vfs.LockLevel) error {
}
defer time.AfterFunc(time.Millisecond, m.waiter.Broadcast).Stop()
for m.owner != nil {
// Our snapshot is invalid.
if m.data != nil && m.data != m.mvccDB.data {
return sqlite3.BUSY_SNAPSHOT // Our snapshot is invalid.
return sqlite3.BUSY_SNAPSHOT
}
if time.Since(before) > time.Millisecond {
return sqlite3.BUSY
@@ -265,16 +266,17 @@ func (m *mvccFile) Lock(lock vfs.LockLevel) error {
case m.data == nil:
m.data = m.mvccDB.data
case m.data != m.mvccDB.data:
return sqlite3.BUSY_SNAPSHOT // Our snapshot is invalid.
// Our snapshot is invalid.
return sqlite3.BUSY_SNAPSHOT
}
// Take ownership.
m.wrflag = false
m.lock = lock
m.owner = m
return nil
}
func (m *mvccFile) Unlock(lock vfs.LockLevel) error {
m.wrflag = false // SQLite calls unlock even if locking is unsuccessful.
if m.lock <= lock {
return nil
}
@@ -285,9 +287,7 @@ func (m *mvccFile) Unlock(lock vfs.LockLevel) error {
// Relase ownership, commit changes.
if m.owner == m {
m.owner = nil
if m.lock == vfs.LOCK_EXCLUSIVE {
m.mvccDB.data = m.data
}
m.mvccDB.data = m.data
if m.waiter != nil {
m.waiter.Broadcast()
}
@@ -313,10 +313,10 @@ func (m *mvccFile) CommitPhaseTwo() error {
// Modified without lock, commit changes.
if m.lock > vfs.LOCK_EXCLUSIVE {
m.mtx.Lock()
defer m.mtx.Unlock()
m.mvccDB.data = m.data
m.lock = vfs.LOCK_NONE
m.data = nil
m.mtx.Unlock()
}
return nil
}

View File

@@ -30,13 +30,13 @@ var (
// otherwise SQLite might return incorrect query results and/or [sqlite3.CORRUPT] errors.
func Create(name string, reader ioutil.SizeReaderAt) {
readerMtx.Lock()
defer readerMtx.Unlock()
readerDBs[name] = reader
readerMtx.Unlock()
}
// Delete deletes a shared memory database.
func Delete(name string) {
readerMtx.Lock()
defer readerMtx.Unlock()
delete(readerDBs, name)
readerMtx.Unlock()
}

View File

@@ -3,15 +3,16 @@ package readervfs
import (
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/util/ioutil"
"github.com/ncruces/go-sqlite3/util/vfsutil"
"github.com/ncruces/go-sqlite3/vfs"
)
type readerVFS struct{}
func (readerVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
// Temporary files use the default VFS.
if name == "" || flags&vfs.OPEN_DELETEONCLOSE != 0 {
return vfs.Find("").Open(name, flags)
// Temp journals, as used by the sorter, use SliceFile.
if flags&vfs.OPEN_TEMP_JOURNAL != 0 {
return &vfsutil.SliceFile{}, flags | vfs.OPEN_MEMORY, nil
}
// Refuse to open all other file types.
if flags&vfs.OPEN_MAIN_DB == 0 {

View File

@@ -10,17 +10,13 @@ var (
// Find returns a VFS given its name.
// If there is no match, nil is returned.
// If name is empty or "os", the default VFS is returned.
// If name is empty, the default VFS is returned.
//
// https://sqlite.org/c3ref/vfs_find.html
func Find(name string) VFS {
if name == "" || name == "os" {
return vfsOS{}
}
return find(name)
}
func find(name string) VFS {
vfsRegistryMtx.RLock()
defer vfsRegistryMtx.RUnlock()
return vfsRegistry[name]
@@ -35,11 +31,11 @@ func Register(name string, vfs VFS) {
return
}
vfsRegistryMtx.Lock()
defer vfsRegistryMtx.Unlock()
if vfsRegistry == nil {
vfsRegistry = map[string]VFS{}
}
vfsRegistry[name] = vfs
vfsRegistryMtx.Unlock()
}
// Unregister unregisters a VFS.
@@ -47,6 +43,6 @@ func Register(name string, vfs VFS) {
// https://sqlite.org/c3ref/vfs_find.html
func Unregister(name string) {
vfsRegistryMtx.Lock()
defer vfsRegistryMtx.Unlock()
delete(vfsRegistry, name)
vfsRegistryMtx.Unlock()
}

View File

@@ -35,38 +35,44 @@ func (s *vfsShm) shmAcquire(errp *error) {
if errp != nil && *errp != nil {
return
}
if len(s.ptrs) == 0 || shmEqual(s.shadow[0][:], s.shared[0][:]) {
if len(s.ptrs) == 0 {
return
}
// Copies modified words from shared to private memory.
for id, p := range s.ptrs {
shared := shmPage(s.shared[id][:])
shadow := shmPage(s.shadow[id][:])
privat := shmPage(util.View(s.mod, p, _WALINDEX_PGSZ))
for i, shared := range shared {
if shadow[i] != shared {
shadow[i] = shared
privat[i] = shared
}
}
if !shmCopyHeader(
util.View(s.mod, s.ptrs[0], _WALINDEX_HDR_SIZE),
s.shadow[0][:],
s.shared[0][:]) {
return
}
skip := _WALINDEX_HDR_SIZE
for id := range s.ptrs {
shmCopyTables(
util.View(s.mod, s.ptrs[id], _WALINDEX_PGSZ)[skip:],
s.shadow[id][skip:],
s.shared[id][skip:])
skip = 0
}
}
func (s *vfsShm) shmRelease() {
if len(s.ptrs) == 0 || shmEqual(s.shadow[0][:], util.View(s.mod, s.ptrs[0], _WALINDEX_HDR_SIZE)) {
if len(s.ptrs) == 0 {
return
}
// Copies modified words from private to shared memory.
for id, p := range s.ptrs {
shared := shmPage(s.shared[id][:])
shadow := shmPage(s.shadow[id][:])
privat := shmPage(util.View(s.mod, p, _WALINDEX_PGSZ))
for i, privat := range privat {
if shadow[i] != privat {
shadow[i] = privat
shared[i] = privat
}
}
if !shmCopyHeader(
s.shared[0][:],
s.shadow[0][:],
util.View(s.mod, s.ptrs[0], _WALINDEX_HDR_SIZE)) {
return
}
skip := _WALINDEX_HDR_SIZE
for id := range s.ptrs {
shmCopyTables(
s.shared[id][skip:],
s.shadow[id][skip:],
util.View(s.mod, s.ptrs[id], _WALINDEX_PGSZ)[skip:])
skip = 0
}
}
@@ -77,11 +83,40 @@ func (s *vfsShm) shmBarrier() {
s.Unlock()
}
func shmPage(s []byte) *[_WALINDEX_PGSZ / 4]uint32 {
p := (*uint32)(unsafe.Pointer(unsafe.SliceData(s)))
return (*[_WALINDEX_PGSZ / 4]uint32)(unsafe.Slice(p, _WALINDEX_PGSZ/4))
func shmCopyTables(v1, v2, v3 []byte) {
if string(v2) != string(v3) {
copy(v1, v3)
copy(v2, v3)
}
}
func shmEqual(v1, v2 []byte) bool {
return *(*[_WALINDEX_HDR_SIZE]byte)(v1[:]) == *(*[_WALINDEX_HDR_SIZE]byte)(v2[:])
func shmCopyHeader(s1, s2, s3 []byte) (ret bool) {
// First copy of the WAL Index Information.
if string(s2[:48]) != string(s3[:48]) {
copy(s1, s3[:48])
copy(s2, s3[:48])
ret = true
}
// Second copy of the WAL Index Information.
if string(s2[48:][:48]) != string(s3[48:][:48]) {
copy(s1[48:], s3[48:][:48])
copy(s2[48:], s3[48:][:48])
ret = true
}
// Checkpoint Information and Locks.
i1 := shmCheckpointInfo(s1)
i2 := shmCheckpointInfo(s2)
for i, i3 := range shmCheckpointInfo(s3) {
if i2[i] != i3 {
i1[i] = i3
i2[i] = i3
ret = true
}
}
return
}
func shmCheckpointInfo(s []byte) *[10]uint32 {
p := (*uint32)(unsafe.Pointer(&s[96]))
return (*[10]uint32)(unsafe.Slice(p, 10))
}

View File

@@ -27,7 +27,7 @@ WASI_SDK="$ROOT/tools/wasi-sdk/bin"
$(awk '{print "-Wl,--export="$0}' exports.txt)
"$BINARYEN/wasm-opt" -g mptest.wasm -o mptest.tmp \
--gufa --generate-global-effects --low-memory-unused --converge -O3 \
--low-memory-unused --gufa --generate-global-effects --converge -O3 \
--enable-mutable-globals --enable-nontrapping-float-to-int \
--enable-simd --enable-bulk-memory --enable-sign-ext \
--enable-reference-types --enable-multivalue \

Binary file not shown.

View File

@@ -22,7 +22,7 @@ WASI_SDK="$ROOT/tools/wasi-sdk/bin"
$(awk '{print "-Wl,--export="$0}' exports.txt)
"$BINARYEN/wasm-opt" -g speedtest1.wasm -o speedtest1.tmp \
--gufa --generate-global-effects --low-memory-unused --converge -O3 \
--low-memory-unused --gufa --generate-global-effects --converge -O3 \
--enable-mutable-globals --enable-nontrapping-float-to-int \
--enable-simd --enable-bulk-memory --enable-sign-ext \
--enable-reference-types --enable-multivalue \

View File

@@ -12,6 +12,7 @@ import (
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/util/sql3util"
"github.com/ncruces/julianday"
)
@@ -49,7 +50,8 @@ func ExportHostFunctions(env wazero.HostModuleBuilder) wazero.HostModuleBuilder
}
func vfsFind(ctx context.Context, mod api.Module, zVfsName ptr_t) uint32 {
if find(util.ReadString(mod, zVfsName, _MAX_NAME)) != nil {
name := util.ReadString(mod, zVfsName, _MAX_NAME)
if vfs := Find(name); vfs != nil && vfs != (vfsOS{}) {
return 1
}
return 0
@@ -136,7 +138,7 @@ func vfsOpen(ctx context.Context, mod api.Module, pVfs, zPath, pFile ptr_t, flag
}
if file, ok := file.(FilePowersafeOverwrite); ok {
if b, ok := util.ParseBool(name.URIParameter("psow")); ok {
if b, ok := sql3util.ParseBool(name.URIParameter("psow")); ok {
file.SetPowersafeOverwrite(b)
}
}
@@ -327,9 +329,9 @@ func vfsFileControlImpl(ctx context.Context, mod api.Module, file File, op _Fcnt
case _FCNTL_PRAGMA:
if file, ok := file.(FilePragma); ok {
var value string
ptr := util.Read32[ptr_t](mod, pArg+1*ptrlen)
name := util.ReadString(mod, ptr, _MAX_SQL_LENGTH)
var value string
if ptr := util.Read32[ptr_t](mod, pArg+2*ptrlen); ptr != 0 {
value = util.ReadString(mod, ptr, _MAX_SQL_LENGTH)
}