Compare commits

..

5 Commits

Author SHA1 Message Date
Nuno Cruces
7028e3a5b9 SQLite 3.51.1. 2025-11-30 10:24:34 +00:00
Nuno Cruces
03bb20de6e Lock tweaks. 2025-11-29 16:12:29 +00:00
Nuno Cruces
20a51a344e Remove MinLevel. 2025-11-27 17:29:47 +00:00
Nuno Cruces
2dbcc480f7 Initial pragma support. 2025-11-27 16:28:16 +00:00
Nuno Cruces
0f0716c438 Inprocess Litestream primary. 2025-11-26 14:07:44 +00:00
27 changed files with 495 additions and 414 deletions

View File

@@ -1,6 +1,6 @@
# Embeddable Wasm build of SQLite
This folder includes an embeddable Wasm build of SQLite 3.50.4 for use with
This folder includes an embeddable Wasm build of SQLite 3.51.1 for use with
[`github.com/ncruces/go-sqlite3`](https://pkg.go.dev/github.com/ncruces/go-sqlite3).
The following optional features are compiled in:

Binary file not shown.

View File

@@ -15,9 +15,9 @@ cp "$ROOT"/sqlite3/*.[ch] build/
cp "$ROOT"/sqlite3/*.patch build/
cd sqlite/
# https://sqlite.org/src/info/352b363a5d727047
curl -#L https://github.com/sqlite/sqlite/archive/dbd613c.tar.gz | tar xz --strip-components=1
# curl -#L https://sqlite.org/src/tarball/sqlite.tar.gz?r=352b363a5d | tar xz --strip-components=1
# https://sqlite.org/src/info/f273f6b8245c5dca
curl -#L https://github.com/sqlite/sqlite/archive/7c126d7.tar.gz | tar xz --strip-components=1
# curl -#L https://sqlite.org/src/tarball/sqlite.tar.gz?r=f273f6b824 | tar xz --strip-components=1
if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then
MSYS_NO_PATHCONV=1 nmake /f makefile.msc sqlite3.c "OPTS=-DSQLITE_ENABLE_UPDATE_DELETE_LIMIT -DSQLITE_ENABLE_ORDERED_SET_AGGREGATES"

View File

@@ -2,7 +2,7 @@ module github.com/ncruces/go-sqlite3/embed/bcw2
go 1.24.0
require github.com/ncruces/go-sqlite3 v0.30.1
require github.com/ncruces/go-sqlite3 v0.30.2
require (
github.com/ncruces/julianday v1.0.0 // indirect

View File

@@ -1,5 +1,5 @@
github.com/ncruces/go-sqlite3 v0.30.1 h1:pHC3YsyRdJv4pCMB4MO1Q2BXw/CAa+Hoj7GSaKtVk+g=
github.com/ncruces/go-sqlite3 v0.30.1/go.mod h1:UVsWrQaq1qkcal5/vT5lOJnZCVlR5rsThKdwidjFsKc=
github.com/ncruces/go-sqlite3 v0.30.2 h1:1GVbHAkKAOwjJd3JYl8ldrYROudfZUOah7oXPD7VZbQ=
github.com/ncruces/go-sqlite3 v0.30.2/go.mod h1:AxKu9sRxkludimFocbktlY6LiYSkxiI5gTA8r+os/Nw=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/ncruces/sort v0.1.6 h1:TrsJfGRH1AoWoaeB4/+gCohot9+cA6u/INaH5agIhNk=
@@ -8,5 +8,5 @@ github.com/tetratelabs/wazero v1.10.1 h1:2DugeJf6VVk58KTPszlNfeeN8AhhpwcZqkJj2ww
github.com/tetratelabs/wazero v1.10.1/go.mod h1:DRm5twOQ5Gr1AoEdSi0CLjDQF1J9ZAuyqFIjl1KKfQU=
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=

View File

@@ -23,6 +23,7 @@ trap 'rm -f sqlite3.tmp' EXIT
-Wl,--import-undefined \
-Wl,--initial-memory=327680 \
-D_HAVE_SQLITE_CONFIG_H \
-DSQLITE_EXPERIMENTAL_PRAGMA_20251114 \
-DSQLITE_CUSTOM_INCLUDE=sqlite_opt.h \
$(awk '{print "-Wl,--export="$0}' exports.txt)

View File

@@ -19,7 +19,7 @@ func Test_init(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if version != "3.51.0" {
if version != "3.51.1" {
t.Error(version)
}
}

Binary file not shown.

View File

@@ -3,7 +3,7 @@ module github.com/ncruces/go-sqlite3/gormlite
go 1.24.0
require (
github.com/ncruces/go-sqlite3 v0.30.1
github.com/ncruces/go-sqlite3 v0.30.2
gorm.io/gorm v1.31.1
)

View File

@@ -2,8 +2,8 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/ncruces/go-sqlite3 v0.30.1 h1:pHC3YsyRdJv4pCMB4MO1Q2BXw/CAa+Hoj7GSaKtVk+g=
github.com/ncruces/go-sqlite3 v0.30.1/go.mod h1:UVsWrQaq1qkcal5/vT5lOJnZCVlR5rsThKdwidjFsKc=
github.com/ncruces/go-sqlite3 v0.30.2 h1:1GVbHAkKAOwjJd3JYl8ldrYROudfZUOah7oXPD7VZbQ=
github.com/ncruces/go-sqlite3 v0.30.2/go.mod h1:AxKu9sRxkludimFocbktlY6LiYSkxiI5gTA8r+os/Nw=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/tetratelabs/wazero v1.10.1 h1:2DugeJf6VVk58KTPszlNfeeN8AhhpwcZqkJj2wwFuH8=

View File

@@ -1,11 +1,26 @@
# Litestream lightweight read-replicas
# Litestream in-process replication and lightweight read-replicas
This package implements the **EXPERIMENTAL** `"litestream"` SQLite VFS
that offers Litestream [lightweight read-replicas](https://fly.io/blog/litestream-revamped/#lightweight-read-replicas).
This package adds **EXPERIMENTAL** support for in-process [Litestream](https://litestream.io/).
See the [example](vfs_test.go) for how to use.
## Lightweight read-replicas
To improve performance,
increase `PollInterval` (and `MinLevel`) as much as you can,
The `"litestream"` SQLite VFS implements Litestream
[lightweight read-replicas](https://fly.io/blog/litestream-revamped/#lightweight-read-replicas).
See the [example](example_test.go) for how to use.
To improve performance, increase `PollInterval` as much as you can,
and set [`PRAGMA cache_size=N`](https://www.sqlite.org/pragma.html#pragma_cache_size)
(or use `_pragma=cache_size(N)`).
(or use `_pragma=cache_size(N)`).
## In-process replication
For disaster recovery, it is probably best if you run Litestream as a separate background process,
as recommended by the [tutorial](https://litestream.io/getting-started/).
However, running Litestream as a background process requires
compatible locking and cross-process shared memory WAL
(see our [support matrix](https://github.com/ncruces/go-sqlite3/wiki/Support-matrix)).
If your OS lacks locking or shared memory support,
you can use `NewPrimary` with the `sqlite3_dotlk` build tag to setup in-process replication.

View File

@@ -2,6 +2,7 @@
package litestream
import (
"context"
"log/slog"
"sync"
"time"
@@ -39,16 +40,13 @@ type ReplicaOptions struct {
// used by the replica at MinLevel+1.
PollInterval time.Duration
// Minimum compaction level to track.
MinLevel int
// CacheSize is the maximum size of the page cache in bytes.
// Zero means DefaultCacheSize, negative disables caching.
CacheSize int
}
// NewReplica creates a read-replica from a Litestream client.
func NewReplica(name string, client litestream.ReplicaClient, options ReplicaOptions) {
func NewReplica(name string, client ReplicaClient, options ReplicaOptions) {
if options.Logger != nil {
options.Logger = options.Logger.With("name", name)
} else {
@@ -76,3 +74,28 @@ func RemoveReplica(name string) {
defer liteMtx.Unlock()
delete(liteDBs, name)
}
// NewPrimary creates a new primary that replicates through client.
// If restore is not nil, the database is first restored.
func NewPrimary(ctx context.Context, path string, client ReplicaClient, restore *RestoreOptions) (*litestream.DB, error) {
lsdb := litestream.NewDB(path)
lsdb.Replica = litestream.NewReplicaWithClient(lsdb, client)
if restore != nil {
err := lsdb.Replica.Restore(ctx, *restore)
if err != nil {
return nil, err
}
}
err := lsdb.Open()
if err != nil {
return nil, err
}
return lsdb, nil
}
type (
ReplicaClient = litestream.ReplicaClient
RestoreOptions = litestream.RestoreOptions
)

104
litestream/api_test.go Normal file
View File

@@ -0,0 +1,104 @@
package litestream
import (
"path/filepath"
"testing"
"time"
"github.com/benbjohnson/litestream"
"github.com/benbjohnson/litestream/file"
"github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed"
)
func Test_integration(t *testing.T) {
dir := t.TempDir()
dbpath := filepath.Join(dir, "test.db")
backup := filepath.Join(dir, "backup", "test.db")
db, err := driver.Open(dbpath)
if err != nil {
t.Fatal(err)
}
defer db.Close()
client := file.NewReplicaClient(backup)
NewReplica("test.db", client, ReplicaOptions{})
if err := setupPrimary(t, dbpath, client); err != nil {
t.Fatal(err)
}
replica, err := driver.Open("file:test.db?vfs=litestream")
if err != nil {
t.Fatal(err)
}
defer replica.Close()
_, err = db.ExecContext(t.Context(), `CREATE TABLE users (id INT, name VARCHAR(10))`)
if err != nil {
t.Fatal(err)
}
_, err = db.ExecContext(t.Context(),
`INSERT INTO users (id, name) VALUES (0, 'go'), (1, 'zig'), (2, 'whatever')`)
if err != nil {
t.Fatal(err)
}
time.Sleep(DefaultPollInterval + litestream.DefaultMonitorInterval)
rows, err := replica.QueryContext(t.Context(), `SELECT id, name FROM users`)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
row := 0
ids := []int{0, 1, 2}
names := []string{"go", "zig", "whatever"}
for ; rows.Next(); row++ {
var id int
var name string
err := rows.Scan(&id, &name)
if err != nil {
t.Fatal(err)
}
if id != ids[row] {
t.Errorf("got %d, want %d", id, ids[row])
}
if name != names[row] {
t.Errorf("got %q, want %q", name, names[row])
}
}
if row != 3 {
t.Errorf("got %d, want %d", row, len(ids))
}
var lag int
err = replica.QueryRowContext(t.Context(), `PRAGMA litestream_lag`).Scan(&lag)
if err != nil {
t.Fatal(err)
}
if lag < 0 || lag > 2 {
t.Errorf("got %d", lag)
}
var txid string
err = replica.QueryRowContext(t.Context(), `PRAGMA litestream_txid`).Scan(&txid)
if err != nil {
t.Fatal(err)
}
if txid != "0000000000000001" {
t.Errorf("got %q", txid)
}
}
func setupPrimary(tb testing.TB, path string, client ReplicaClient) error {
db, err := NewPrimary(tb.Context(), path, client, nil)
if err == nil {
tb.Cleanup(func() { db.Close(tb.Context()) })
}
return err
}

View File

@@ -3,10 +3,11 @@ module github.com/ncruces/go-sqlite3/litestream
go 1.24.4
require (
github.com/benbjohnson/litestream v0.5.2
github.com/ncruces/go-sqlite3 v0.30.1
github.com/benbjohnson/litestream v0.5.3-0.20251109214555-48dc960260f0
github.com/ncruces/go-sqlite3 v0.30.2
github.com/ncruces/wbt v0.2.0
github.com/superfly/ltx v0.5.0
golang.org/x/sync v0.18.0
)
// github.com/ncruces/go-sqlite3
@@ -28,9 +29,9 @@ require (
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/prometheus/client_golang v1.23.2 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.67.3 // indirect
github.com/prometheus/common v0.67.4 // indirect
github.com/prometheus/procfs v0.19.2 // indirect
github.com/psanford/sqlite3vfs v0.0.0-20240315230605-24e1d98cf361 // indirect
github.com/psanford/sqlite3vfs v0.0.0-20251127171934-4e34e03a991a // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
golang.org/x/crypto v0.45.0 // indirect
google.golang.org/protobuf v1.36.10 // indirect
@@ -60,4 +61,4 @@ require (
github.com/aws/smithy-go v1.22.5 // indirect
)
replace modernc.org/sqlite => github.com/ncruces/go-sqlite3/litestream/modernc v0.0.0-20251109124432-99b097de3b79
replace modernc.org/sqlite => github.com/ncruces/go-sqlite3/litestream/modernc v0.30.1

View File

@@ -60,8 +60,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.35.1 h1:iF4Xxkc0H9c/K2dS0zZw3SCkj0Z7
github.com/aws/aws-sdk-go-v2/service/sts v1.35.1/go.mod h1:0bxIatfN0aLq4mjoLDeBpOjOke68OsFlXPDFJ7V0MYw=
github.com/aws/smithy-go v1.22.5 h1:P9ATCXPMb2mPjYBgueqJNCA5S9UfktsW0tTxi+a7eqw=
github.com/aws/smithy-go v1.22.5/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
github.com/benbjohnson/litestream v0.5.2 h1:uD9I17n6RgUgyCwPM/Sw2YXNmMGixecUB5kmJ4FL08o=
github.com/benbjohnson/litestream v0.5.2/go.mod h1:jSW6AGqbxmJnEXGjMHchlZclGphzbJ6jGrGo5fYIDhU=
github.com/benbjohnson/litestream v0.5.3-0.20251109214555-48dc960260f0 h1:190qM2axbs1p7NyRm5/ygqrDdcIYhc5QXPTK04Cgyno=
github.com/benbjohnson/litestream v0.5.3-0.20251109214555-48dc960260f0/go.mod h1:0gjSLi7Qm5INdtdo6YJMFMsS5e/KM2s3CFYF3OtLmY8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
@@ -113,10 +113,10 @@ github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0=
github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/ncruces/go-sqlite3 v0.30.1 h1:pHC3YsyRdJv4pCMB4MO1Q2BXw/CAa+Hoj7GSaKtVk+g=
github.com/ncruces/go-sqlite3 v0.30.1/go.mod h1:UVsWrQaq1qkcal5/vT5lOJnZCVlR5rsThKdwidjFsKc=
github.com/ncruces/go-sqlite3/litestream/modernc v0.0.0-20251109124432-99b097de3b79 h1:evpQceUV2vRbOe84U/QhBBchfqFERRHTx1JOadFFMLE=
github.com/ncruces/go-sqlite3/litestream/modernc v0.0.0-20251109124432-99b097de3b79/go.mod h1:GSM2gXEOb9HIFFtsl0IUtnpvpDmVi7Kbp8z5GzwA0Tw=
github.com/ncruces/go-sqlite3 v0.30.2 h1:1GVbHAkKAOwjJd3JYl8ldrYROudfZUOah7oXPD7VZbQ=
github.com/ncruces/go-sqlite3 v0.30.2/go.mod h1:AxKu9sRxkludimFocbktlY6LiYSkxiI5gTA8r+os/Nw=
github.com/ncruces/go-sqlite3/litestream/modernc v0.30.1 h1:3SNAOrm+qmLprkZybcvBrVNHyt0QYHljUGxmOXnL+K0=
github.com/ncruces/go-sqlite3/litestream/modernc v0.30.1/go.mod h1:GSM2gXEOb9HIFFtsl0IUtnpvpDmVi7Kbp8z5GzwA0Tw=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/ncruces/wbt v0.2.0 h1:Q9zlKOBSZc7Yy/R2cGa35g6RKUUE3BjNIW3tfGC4F04=
@@ -133,16 +133,18 @@ github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.67.3 h1:shd26MlnwTw5jksTDhC7rTQIteBxy+ZZDr3t7F2xN2Q=
github.com/prometheus/common v0.67.3/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI=
github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc=
github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI=
github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
github.com/psanford/sqlite3vfs v0.0.0-20240315230605-24e1d98cf361 h1:vAKifIJuYY306ZJSrwDgKonWcJGELijdaenABqbV03E=
github.com/psanford/sqlite3vfs v0.0.0-20240315230605-24e1d98cf361/go.mod h1:iW4cSew5PAb1sMZiTEkVJAIBNrepaB6jTYjeP47WtI0=
github.com/psanford/sqlite3vfs v0.0.0-20251127171934-4e34e03a991a h1:r4YWl0uVObCbBFvj1VsIlyHzgZwZOHvY1KdRaQjzzUc=
github.com/psanford/sqlite3vfs v0.0.0-20251127171934-4e34e03a991a/go.mod h1:iW4cSew5PAb1sMZiTEkVJAIBNrepaB6jTYjeP47WtI0=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/studio-b12/gowebdav v0.11.0 h1:qbQzq4USxY28ZYsGJUfO5jR+xkFtcnwWgitp4Zp1irU=
github.com/studio-b12/gowebdav v0.11.0/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE=
github.com/superfly/ltx v0.5.0 h1:dXNrcT3ZtMb6iKZopIV7z5UBscnapg0b0F02loQsk5o=
github.com/superfly/ltx v0.5.0/go.mod h1:Nf50QAIXU/ET4ua3AuQ2fh31MbgNQZA7r/DYx6Os77s=
github.com/tetratelabs/wazero v1.10.1 h1:2DugeJf6VVk58KTPszlNfeeN8AhhpwcZqkJj2wwFuH8=
@@ -169,8 +171,8 @@ golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY=
golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=

View File

@@ -2,7 +2,7 @@ module modernc.org/sqlite
go 1.24.0
require github.com/ncruces/go-sqlite3 v0.30.1
require github.com/ncruces/go-sqlite3 v0.30.2
require (
github.com/ncruces/julianday v1.0.0 // indirect

View File

@@ -1,10 +1,10 @@
github.com/ncruces/go-sqlite3 v0.30.1 h1:pHC3YsyRdJv4pCMB4MO1Q2BXw/CAa+Hoj7GSaKtVk+g=
github.com/ncruces/go-sqlite3 v0.30.1/go.mod h1:UVsWrQaq1qkcal5/vT5lOJnZCVlR5rsThKdwidjFsKc=
github.com/ncruces/go-sqlite3 v0.30.2 h1:1GVbHAkKAOwjJd3JYl8ldrYROudfZUOah7oXPD7VZbQ=
github.com/ncruces/go-sqlite3 v0.30.2/go.mod h1:AxKu9sRxkludimFocbktlY6LiYSkxiI5gTA8r+os/Nw=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/tetratelabs/wazero v1.10.1 h1:2DugeJf6VVk58KTPszlNfeeN8AhhpwcZqkJj2wwFuH8=
github.com/tetratelabs/wazero v1.10.1/go.mod h1:DRm5twOQ5Gr1AoEdSi0CLjDQF1J9ZAuyqFIjl1KKfQU=
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=

View File

@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"io"
"strconv"
"sync"
"time"
@@ -160,6 +161,34 @@ func (f *liteFile) DeviceCharacteristics() vfs.DeviceCharacteristic {
return 0
}
func (f *liteFile) Pragma(name, value string) (string, error) {
switch name {
case "litestream_txid":
txid := f.txid
if txid == 0 {
// Outside transaction.
f.db.mtx.Lock()
txid = f.db.txids[0]
f.db.mtx.Unlock()
}
return txid.String(), nil
case "litestream_lag":
f.db.mtx.Lock()
lastPoll := f.db.lastPoll
f.db.mtx.Unlock()
if lastPoll.IsZero() {
// Never polled successfully.
return "-1", nil
}
lag := time.Since(lastPoll) / time.Second
return strconv.FormatInt(int64(lag), 10), nil
}
return "", sqlite3.NOTFOUND
}
func (f *liteFile) SetDB(conn any) {
f.conn = conn.(*sqlite3.Conn)
}
@@ -219,7 +248,7 @@ func (f *liteDB) pollReplica(ctx context.Context) (*pageIndex, ltx.TXID, error)
return f.pages, f.txids[0], nil
}
for level := range pollLevels(f.opts.MinLevel) {
for level := range []int{0, 1, litestream.SnapshotLevel} {
if err := f.updateLevel(ctx, level); err != nil {
f.opts.Logger.Error("cannot poll replica", "error", err)
return nil, 0, err
@@ -284,26 +313,10 @@ func (f *liteDB) updateInfo(ctx context.Context, info *ltx.FileInfo) error {
// Track the MaxTXID for each level.
maxTXID := &f.txids[info.Level]
*maxTXID = max(*maxTXID, info.MaxTXID)
f.txids[0] = max(f.txids[0], *maxTXID)
return nil
}
func pollLevels(minLevel int) (r []int) {
// Updating from lower to upper levels is non-racy,
// since LTX files are compacted into higher levels
// before the lower level LTX files are deleted.
// Also, only level 0 compactions and snapshots delete files,
// so the intermediate levels never need to be updated.
if minLevel <= 0 {
return append(r, 0, 1, litestream.SnapshotLevel)
}
if minLevel >= litestream.SnapshotLevel {
return append(r, litestream.SnapshotLevel)
}
return append(r, minLevel, litestream.SnapshotLevel)
}
// Type aliases; these are a mouthful.
type pageIndex = wbt.Tree[uint32, ltx.PageIndexElem]
type levelTXIDs = [litestream.SnapshotLevel + 1]ltx.TXID

View File

@@ -1,34 +0,0 @@
package litestream
import (
"slices"
"strconv"
"testing"
"github.com/benbjohnson/litestream"
_ "github.com/ncruces/go-sqlite3/embed"
)
func Test_pollLevels(t *testing.T) {
tests := []struct {
minLevel int
want []int
}{
{minLevel: -1, want: []int{0, 1, litestream.SnapshotLevel}},
{minLevel: 0, want: []int{0, 1, litestream.SnapshotLevel}},
{minLevel: 1, want: []int{1, litestream.SnapshotLevel}},
{minLevel: 2, want: []int{2, litestream.SnapshotLevel}},
{minLevel: 3, want: []int{3, litestream.SnapshotLevel}},
{minLevel: litestream.SnapshotLevel, want: []int{litestream.SnapshotLevel}},
{minLevel: litestream.SnapshotLevel + 1, want: []int{litestream.SnapshotLevel}},
}
for _, tt := range tests {
t.Run(strconv.Itoa(tt.minLevel), func(t *testing.T) {
got := pollLevels(tt.minLevel)
if !slices.Equal(got, tt.want) {
t.Errorf("pollLevels() = %v, want %v", got, tt.want)
}
})
}
}

View File

@@ -3,11 +3,11 @@ set -euo pipefail
cd -P -- "$(dirname -- "$0")"
curl -#OL "https://sqlite.org/2025/sqlite-autoconf-3510000.tar.gz"
curl -#OL "https://sqlite.org/2025/sqlite-autoconf-3510100.tar.gz"
# Verify download.
if hash=$(openssl dgst -sha3-256 sqlite-autoconf-*.tar.gz); then
if ! [[ $hash =~ fa52f9cc74dbca004aa650ae698036a3350611f672649e165078f4eae21d6a2e ]]; then
if ! [[ $hash =~ 9b2b1e73f577def1d5b75c5541555a7f42e6e073ad19f7a9118478389c9bbd9b ]]; then
echo $hash
exit 1
fi
@@ -23,7 +23,7 @@ mv sqlite-*/sqlite3.h .
mv sqlite-*/sqlite3ext.h .
rm -r sqlite-*
GITHUB_TAG="https://github.com/sqlite/sqlite/raw/version-3.51.0"
GITHUB_TAG="https://github.com/sqlite/sqlite/raw/version-3.51.1"
mkdir -p ext/
cd ext/

Binary file not shown.

View File

@@ -4,7 +4,8 @@
(type $2 (func (param i32 i32 i32 i32)))
(type $3 (func (param i32) (result i32)))
(memory $0 256)
(data $0 (i32.const 4096) "\01")
(data $.data (i32.const 4097) "\10\00\00\01\00\00\00\00\00\00\00\0c\10\00\00\0c\10\00\00\0c\10")
(data $.data.1 (i32.const 4157) "\10\00\00\00\10")
(table $0 1 1 funcref)
(export "memory" (memory $0))
(export "qsort" (func $qsort))
@@ -1493,33 +1494,243 @@
(local $2 i32)
(local $3 i32)
(local $4 i32)
(local $5 i32)
(local $5 v128)
(local $6 v128)
(local $7 v128)
(local $8 v128)
(local $9 v128)
(block $block
(if
(local.tee $3
(i32.load8_u
(local.get $1)
(block $block2
(block $block
(br_if $block
(i32.eqz
(local.tee $3
(i32.load8_u
(local.get $1)
)
)
)
)
(then
(br_if $block
(br_if $block
(i32.eqz
(i32.load8_u offset=1
(local.get $1)
)
)
)
(loop $label
(v128.store
(i32.const 4080)
(local.get $6)
)
(i32.store8
(i32.or
(local.tee $3
(i32.and
(local.tee $2
(i32.load8_u
(local.get $1)
)
)
(i32.const 15)
)
)
(i32.const 4080)
)
(i32.or
(i32.load8_u
(i32.or
(local.get $3)
(i32.const 4080)
)
)
(i32.shl
(i32.const 1)
(i32.sub
(local.tee $4
(i32.shr_u
(local.get $2)
(i32.const 4)
)
)
(i32.const 8)
)
)
)
)
(v128.store
(i32.const 4064)
(local.get $5)
)
(i32.store8
(local.tee $3
(i32.or
(local.get $3)
(i32.const 4064)
)
)
(i32.or
(i32.load8_u
(local.get $3)
)
(i32.shl
(i32.const 1)
(local.get $4)
)
)
)
(local.set $1
(i32.add
(local.get $1)
(i32.const 1)
)
)
(local.set $6
(v128.load
(i32.const 4080)
)
)
(local.set $5
(v128.load
(i32.const 4064)
)
)
(br_if $label
(local.get $2)
)
)
(block $block1
(if
(v128.any_true
(local.tee $7
(v128.and
(v128.or
(i8x16.swizzle
(local.get $6)
(v128.xor
(local.tee $8
(v128.and
(local.tee $7
(v128.load
(local.tee $2
(i32.and
(local.get $0)
(i32.const -16)
)
)
)
)
(v128.const i32x4 0x8f8f8f8f 0x8f8f8f8f 0x8f8f8f8f 0x8f8f8f8f)
)
)
(v128.const i32x4 0x80808080 0x80808080 0x80808080 0x80808080)
)
)
(i8x16.swizzle
(local.get $5)
(local.get $8)
)
)
(i8x16.swizzle
(v128.const i32x4 0x08040201 0x80402010 0x08040201 0x80402010)
(i8x16.shr_u
(local.get $7)
(i32.const 4)
)
)
)
)
)
(then
(br_if $block1
(local.tee $1
(i32.and
(i32.xor
(i8x16.bitmask
(i8x16.eq
(local.get $7)
(v128.const i32x4 0x00000000 0x00000000 0x00000000 0x00000000)
)
)
(i32.const 65535)
)
(i32.shl
(i32.const -1)
(i32.and
(local.get $0)
(i32.const 15)
)
)
)
)
)
)
)
(loop $label1
(br_if $label1
(i32.eqz
(v128.any_true
(local.tee $7
(v128.and
(v128.or
(i8x16.swizzle
(local.get $6)
(v128.xor
(local.tee $8
(v128.and
(local.tee $7
(v128.load
(local.tee $2
(i32.add
(local.get $2)
(i32.const 16)
)
)
)
)
(v128.const i32x4 0x8f8f8f8f 0x8f8f8f8f 0x8f8f8f8f 0x8f8f8f8f)
)
)
(v128.const i32x4 0x80808080 0x80808080 0x80808080 0x80808080)
)
)
(i8x16.swizzle
(local.get $5)
(local.get $8)
)
)
(i8x16.swizzle
(v128.const i32x4 0x08040201 0x80402010 0x08040201 0x80402010)
(i8x16.shr_u
(local.get $7)
(i32.const 4)
)
)
)
)
)
)
)
)
(local.set $1
(i32.xor
(i8x16.bitmask
(i8x16.eq
(local.get $7)
(v128.const i32x4 0x00000000 0x00000000 0x00000000 0x00000000)
)
)
(i32.const 65535)
)
)
)
(br $block2)
)
(block $block1
(block $block3
(if
(v128.any_true
(local.tee $6
(local.tee $5
(v128.or
(i8x16.eq
(local.tee $7
(local.tee $6
(v128.load
(local.tee $2
(i32.and
@@ -1532,8 +1743,8 @@
(v128.const i32x4 0x00000000 0x00000000 0x00000000 0x00000000)
)
(i8x16.eq
(local.get $7)
(local.tee $7
(local.get $6)
(local.tee $6
(i8x16.splat
(local.get $3)
)
@@ -1543,11 +1754,11 @@
)
)
(then
(br_if $block1
(br_if $block3
(local.tee $1
(i32.and
(i8x16.bitmask
(local.get $6)
(local.get $5)
)
(i32.shl
(i32.const -1)
@@ -1561,14 +1772,14 @@
)
)
)
(loop $label
(br_if $label
(loop $label2
(br_if $label2
(i32.eqz
(v128.any_true
(local.tee $6
(local.tee $5
(v128.or
(i8x16.eq
(local.tee $6
(local.tee $5
(v128.load
(local.tee $2
(i32.add
@@ -1581,8 +1792,8 @@
(v128.const i32x4 0x00000000 0x00000000 0x00000000 0x00000000)
)
(i8x16.eq
(local.get $5)
(local.get $6)
(local.get $7)
)
)
)
@@ -1592,230 +1803,10 @@
)
(local.set $1
(i8x16.bitmask
(local.get $6)
)
)
)
(return
(i32.add
(i32.ctz
(local.get $1)
)
(i32.sub
(local.get $2)
(local.get $0)
)
)
)
)
(local.set $4
(i32.and
(local.get $0)
(i32.const 15)
)
)
(loop $label1
(v128.store
(i32.const 4080)
(local.get $7)
)
(i32.store8
(i32.or
(local.tee $3
(i32.and
(local.tee $2
(i32.load8_u
(local.get $1)
)
)
(i32.const 15)
)
)
(i32.const 4080)
)
(i32.or
(i32.load8_u
(i32.or
(local.get $3)
(i32.const 4080)
)
)
(i32.shl
(i32.const 1)
(i32.sub
(local.tee $5
(i32.shr_u
(local.get $2)
(i32.const 4)
)
)
(i32.const 8)
)
)
)
)
(v128.store
(i32.const 4064)
(local.get $6)
)
(i32.store8
(local.tee $3
(i32.or
(local.get $3)
(i32.const 4064)
)
)
(i32.or
(i32.load8_u
(local.get $3)
)
(i32.shl
(i32.const 1)
(local.get $5)
)
)
)
(local.set $1
(i32.add
(local.get $1)
(i32.const 1)
)
)
(local.set $7
(v128.load
(i32.const 4080)
)
)
(local.set $6
(v128.load
(i32.const 4064)
)
)
(br_if $label1
(local.get $2)
)
)
(block $block2
(if
(v128.any_true
(local.tee $8
(v128.and
(v128.or
(i8x16.swizzle
(local.get $7)
(v128.xor
(local.tee $9
(v128.and
(local.tee $8
(v128.load
(local.tee $2
(i32.and
(local.get $0)
(i32.const -16)
)
)
)
)
(v128.const i32x4 0x8f8f8f8f 0x8f8f8f8f 0x8f8f8f8f 0x8f8f8f8f)
)
)
(v128.const i32x4 0x80808080 0x80808080 0x80808080 0x80808080)
)
)
(i8x16.swizzle
(local.get $6)
(local.get $9)
)
)
(i8x16.swizzle
(v128.const i32x4 0x08040201 0x80402010 0x08040201 0x80402010)
(i8x16.shr_u
(local.get $8)
(i32.const 4)
)
)
)
)
)
(then
(br_if $block2
(local.tee $1
(i32.and
(i32.xor
(i8x16.bitmask
(i8x16.eq
(local.get $8)
(v128.const i32x4 0x00000000 0x00000000 0x00000000 0x00000000)
)
)
(i32.const 65535)
)
(i32.shl
(i32.const -1)
(local.get $4)
)
)
)
)
)
)
(loop $label2
(br_if $label2
(i32.eqz
(v128.any_true
(local.tee $8
(v128.and
(v128.or
(i8x16.swizzle
(local.get $7)
(v128.xor
(local.tee $9
(v128.and
(local.tee $8
(v128.load
(local.tee $2
(i32.add
(local.get $2)
(i32.const 16)
)
)
)
)
(v128.const i32x4 0x8f8f8f8f 0x8f8f8f8f 0x8f8f8f8f 0x8f8f8f8f)
)
)
(v128.const i32x4 0x80808080 0x80808080 0x80808080 0x80808080)
)
)
(i8x16.swizzle
(local.get $6)
(local.get $9)
)
)
(i8x16.swizzle
(v128.const i32x4 0x08040201 0x80402010 0x08040201 0x80402010)
(i8x16.shr_u
(local.get $8)
(i32.const 4)
)
)
)
)
)
)
)
)
(local.set $1
(i32.xor
(i8x16.bitmask
(i8x16.eq
(local.get $8)
(v128.const i32x4 0x00000000 0x00000000 0x00000000 0x00000000)
)
)
(i32.const 65535)
)
)
)
(i32.add
(i32.ctz

View File

@@ -223,7 +223,7 @@ func (m *memFile) Lock(lock vfs.LockLevel) error {
m.reserved = true
case vfs.LOCK_EXCLUSIVE:
if m.lock < vfs.LOCK_PENDING {
if m.lock == vfs.LOCK_RESERVED {
m.lock = vfs.LOCK_PENDING
m.pending = true
}

View File

@@ -252,9 +252,8 @@ func (m *mvccFile) Lock(lock vfs.LockLevel) error {
}
defer time.AfterFunc(time.Millisecond, m.waiter.Broadcast).Stop()
for m.owner != nil {
// Our snapshot is invalid.
if m.data != nil && m.data != m.mvccDB.data {
return sqlite3.BUSY_SNAPSHOT
return sqlite3.BUSY_SNAPSHOT // Our snapshot is invalid.
}
if time.Since(before) > time.Millisecond {
return sqlite3.BUSY
@@ -266,17 +265,16 @@ func (m *mvccFile) Lock(lock vfs.LockLevel) error {
case m.data == nil:
m.data = m.mvccDB.data
case m.data != m.mvccDB.data:
// Our snapshot is invalid.
return sqlite3.BUSY_SNAPSHOT
return sqlite3.BUSY_SNAPSHOT // Our snapshot is invalid.
}
// Take ownership.
m.wrflag = false
m.lock = lock
m.owner = m
return nil
}
func (m *mvccFile) Unlock(lock vfs.LockLevel) error {
m.wrflag = false // SQLite calls unlock even if locking is unsuccessful.
if m.lock <= lock {
return nil
}
@@ -287,7 +285,9 @@ func (m *mvccFile) Unlock(lock vfs.LockLevel) error {
// Relase ownership, commit changes.
if m.owner == m {
m.owner = nil
m.mvccDB.data = m.data
if m.lock == vfs.LOCK_EXCLUSIVE {
m.mvccDB.data = m.data
}
if m.waiter != nil {
m.waiter.Broadcast()
}

View File

@@ -35,44 +35,38 @@ func (s *vfsShm) shmAcquire(errp *error) {
if errp != nil && *errp != nil {
return
}
if len(s.ptrs) == 0 {
if len(s.ptrs) == 0 || shmEqual(s.shadow[0][:], s.shared[0][:]) {
return
}
if !shmCopyHeader(
util.View(s.mod, s.ptrs[0], _WALINDEX_HDR_SIZE),
s.shadow[0][:],
s.shared[0][:]) {
return
}
skip := _WALINDEX_HDR_SIZE
for id := range s.ptrs {
shmCopyTables(
util.View(s.mod, s.ptrs[id], _WALINDEX_PGSZ)[skip:],
s.shadow[id][skip:],
s.shared[id][skip:])
skip = 0
// Copies modified words from shared to private memory.
for id, p := range s.ptrs {
shared := shmPage(s.shared[id][:])
shadow := shmPage(s.shadow[id][:])
privat := shmPage(util.View(s.mod, p, _WALINDEX_PGSZ))
for i, shared := range shared {
if shadow[i] != shared {
shadow[i] = shared
privat[i] = shared
}
}
}
}
func (s *vfsShm) shmRelease() {
if len(s.ptrs) == 0 {
if len(s.ptrs) == 0 || shmEqual(s.shadow[0][:], util.View(s.mod, s.ptrs[0], _WALINDEX_HDR_SIZE)) {
return
}
if !shmCopyHeader(
s.shared[0][:],
s.shadow[0][:],
util.View(s.mod, s.ptrs[0], _WALINDEX_HDR_SIZE)) {
return
}
skip := _WALINDEX_HDR_SIZE
for id := range s.ptrs {
shmCopyTables(
s.shared[id][skip:],
s.shadow[id][skip:],
util.View(s.mod, s.ptrs[id], _WALINDEX_PGSZ)[skip:])
skip = 0
// Copies modified words from private to shared memory.
for id, p := range s.ptrs {
shared := shmPage(s.shared[id][:])
shadow := shmPage(s.shadow[id][:])
privat := shmPage(util.View(s.mod, p, _WALINDEX_PGSZ))
for i, privat := range privat {
if shadow[i] != privat {
shadow[i] = privat
shared[i] = privat
}
}
}
}
@@ -83,40 +77,11 @@ func (s *vfsShm) shmBarrier() {
s.Unlock()
}
func shmCopyTables(v1, v2, v3 []byte) {
if string(v2) != string(v3) {
copy(v1, v3)
copy(v2, v3)
}
func shmPage(s []byte) *[_WALINDEX_PGSZ / 4]uint32 {
p := (*uint32)(unsafe.Pointer(unsafe.SliceData(s)))
return (*[_WALINDEX_PGSZ / 4]uint32)(unsafe.Slice(p, _WALINDEX_PGSZ/4))
}
func shmCopyHeader(s1, s2, s3 []byte) (ret bool) {
// First copy of the WAL Index Information.
if string(s2[:48]) != string(s3[:48]) {
copy(s1, s3[:48])
copy(s2, s3[:48])
ret = true
}
// Second copy of the WAL Index Information.
if string(s2[48:][:48]) != string(s3[48:][:48]) {
copy(s1[48:], s3[48:][:48])
copy(s2[48:], s3[48:][:48])
ret = true
}
// Checkpoint Information and Locks.
i1 := shmCheckpointInfo(s1)
i2 := shmCheckpointInfo(s2)
for i, i3 := range shmCheckpointInfo(s3) {
if i2[i] != i3 {
i1[i] = i3
i2[i] = i3
ret = true
}
}
return
}
func shmCheckpointInfo(s []byte) *[10]uint32 {
p := (*uint32)(unsafe.Pointer(&s[96]))
return (*[10]uint32)(unsafe.Slice(p, 10))
func shmEqual(v1, v2 []byte) bool {
return *(*[_WALINDEX_HDR_SIZE]byte)(v1[:]) == *(*[_WALINDEX_HDR_SIZE]byte)(v2[:])
}

Binary file not shown.