diff --git a/litestream/README.md b/litestream/README.md index d1fcb41..f19a0e0 100644 --- a/litestream/README.md +++ b/litestream/README.md @@ -1,11 +1,27 @@ -# Litestream lightweight read-replicas +# Litestream in-process replication and lightweight read-replicas -This package implements the **EXPERIMENTAL** `"litestream"` SQLite VFS -that offers Litestream [lightweight read-replicas](https://fly.io/blog/litestream-revamped/#lightweight-read-replicas). +This package adds **EXPERIMENTAL** support for in-process [Litestream](https://litestream.io/). -See the [example](vfs_test.go) for how to use. +## Lightweight read-replicas + +The `"litestream"` SQLite VFS implements Litestream +[lightweight read-replicas](https://fly.io/blog/litestream-revamped/#lightweight-read-replicas). + +See the [example](example_test.go) for how to use. To improve performance, increase `PollInterval` (and `MinLevel`) as much as you can, and set [`PRAGMA cache_size=N`](https://www.sqlite.org/pragma.html#pragma_cache_size) -(or use `_pragma=cache_size(N)`). \ No newline at end of file +(or use `_pragma=cache_size(N)`). + +## In-process replication + +For disaster recovery, it is probably best if you run Litestream as a separate background process, +as recommended by the [tutorial](https://litestream.io/getting-started/). + +However, running Litestream as a background process requires +compatible locking and cross-process shared memory WAL +(see our [support matrix](https://github.com/ncruces/go-sqlite3/wiki/Support-matrix)). + +If your OS lacks locking or shared memory support, +you can use `NewPrimary` with the `sqlite3_dotlk` build tag to setup in-process replication. diff --git a/litestream/api.go b/litestream/api.go index 3b6067a..2ceabf8 100644 --- a/litestream/api.go +++ b/litestream/api.go @@ -2,6 +2,7 @@ package litestream import ( + "context" "log/slog" "sync" "time" @@ -76,3 +77,23 @@ func RemoveReplica(name string) { defer liteMtx.Unlock() delete(liteDBs, name) } + +// NewPrimary creates a new primary that replicates through a client. +// If restore is not nil, the database is first restored. +func NewPrimary(ctx context.Context, path string, client litestream.ReplicaClient, restore *litestream.RestoreOptions) (*litestream.DB, error) { + lsdb := litestream.NewDB(path) + lsdb.Replica = litestream.NewReplicaWithClient(lsdb, client) + + if restore != nil { + err := lsdb.Replica.Restore(ctx, *restore) + if err != nil { + return nil, err + } + } + + err := lsdb.Open() + if err != nil { + return nil, err + } + return lsdb, nil +} diff --git a/litestream/go.mod b/litestream/go.mod index 2958f91..1a1175c 100644 --- a/litestream/go.mod +++ b/litestream/go.mod @@ -7,6 +7,7 @@ require ( github.com/ncruces/go-sqlite3 v0.30.1 github.com/ncruces/wbt v0.2.0 github.com/superfly/ltx v0.5.0 + golang.org/x/sync v0.18.0 ) // github.com/ncruces/go-sqlite3 @@ -28,7 +29,7 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.67.3 // indirect + github.com/prometheus/common v0.67.4 // indirect github.com/prometheus/procfs v0.19.2 // indirect github.com/psanford/sqlite3vfs v0.0.0-20240315230605-24e1d98cf361 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect @@ -60,4 +61,4 @@ require ( github.com/aws/smithy-go v1.22.5 // indirect ) -replace modernc.org/sqlite => github.com/ncruces/go-sqlite3/litestream/modernc v0.0.0-20251109124432-99b097de3b79 +replace modernc.org/sqlite => github.com/ncruces/go-sqlite3/litestream/modernc v0.30.1 diff --git a/litestream/go.sum b/litestream/go.sum index 23fa9e4..ad5acb4 100644 --- a/litestream/go.sum +++ b/litestream/go.sum @@ -115,8 +115,8 @@ github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/ncruces/go-sqlite3 v0.30.1 h1:pHC3YsyRdJv4pCMB4MO1Q2BXw/CAa+Hoj7GSaKtVk+g= github.com/ncruces/go-sqlite3 v0.30.1/go.mod h1:UVsWrQaq1qkcal5/vT5lOJnZCVlR5rsThKdwidjFsKc= -github.com/ncruces/go-sqlite3/litestream/modernc v0.0.0-20251109124432-99b097de3b79 h1:evpQceUV2vRbOe84U/QhBBchfqFERRHTx1JOadFFMLE= -github.com/ncruces/go-sqlite3/litestream/modernc v0.0.0-20251109124432-99b097de3b79/go.mod h1:GSM2gXEOb9HIFFtsl0IUtnpvpDmVi7Kbp8z5GzwA0Tw= +github.com/ncruces/go-sqlite3/litestream/modernc v0.30.1 h1:3SNAOrm+qmLprkZybcvBrVNHyt0QYHljUGxmOXnL+K0= +github.com/ncruces/go-sqlite3/litestream/modernc v0.30.1/go.mod h1:GSM2gXEOb9HIFFtsl0IUtnpvpDmVi7Kbp8z5GzwA0Tw= github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M= github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g= github.com/ncruces/wbt v0.2.0 h1:Q9zlKOBSZc7Yy/R2cGa35g6RKUUE3BjNIW3tfGC4F04= @@ -133,8 +133,8 @@ github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.67.3 h1:shd26MlnwTw5jksTDhC7rTQIteBxy+ZZDr3t7F2xN2Q= -github.com/prometheus/common v0.67.3/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= +github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc= +github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= github.com/psanford/sqlite3vfs v0.0.0-20240315230605-24e1d98cf361 h1:vAKifIJuYY306ZJSrwDgKonWcJGELijdaenABqbV03E= @@ -169,8 +169,8 @@ golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=