Bulk copies.

litestream/README.md

@@ -1,27 +1,11 @@
-# Litestream in-process replication and lightweight read-replicas
+# Litestream lightweight read-replicas
 
-This package adds **EXPERIMENTAL** support for in-process [Litestream](https://litestream.io/).
+This package implements the **EXPERIMENTAL** `"litestream"` SQLite VFS
+that offers Litestream [lightweight read-replicas](https://fly.io/blog/litestream-revamped/#lightweight-read-replicas).
 
-## Lightweight read-replicas
-
-The `"litestream"` SQLite VFS implements Litestream
-[lightweight read-replicas](https://fly.io/blog/litestream-revamped/#lightweight-read-replicas).
-
-See the [example](example_test.go) for how to use.
+See the [example](vfs_test.go) for how to use.
 
 To improve performance,
 increase `PollInterval` (and `MinLevel`) as much as you can,
 and set [`PRAGMA cache_size=N`](https://www.sqlite.org/pragma.html#pragma_cache_size)
-(or use `_pragma=cache_size(N)`).
-
-## In-process replication
-
-For disaster recovery, it is probably best if you run Litestream as a separate background process,
-as recommended by the [tutorial](https://litestream.io/getting-started/).
-
-However, running Litestream as a background process requires
-compatible locking and cross-process shared memory WAL
-(see our [support matrix](https://github.com/ncruces/go-sqlite3/wiki/Support-matrix)).
-
-If your OS lacks locking or shared memory support,
-you can use `NewPrimary` with the `sqlite3_dotlk` build tag to setup in-process replication.
+(or use `_pragma=cache_size(N)`).

litestream/api.go

@@ -2,7 +2,6 @@
 package litestream
 
 import (
-	"context"
 	"log/slog"
 	"sync"
 	"time"
@@ -77,23 +76,3 @@ func RemoveReplica(name string) {
 	defer liteMtx.Unlock()
 	delete(liteDBs, name)
 }
-
-// NewPrimary creates a new primary that replicates through a client.
-// If restore is not nil, the database is first restored.
-func NewPrimary(ctx context.Context, path string, client litestream.ReplicaClient, restore *litestream.RestoreOptions) (*litestream.DB, error) {
-	lsdb := litestream.NewDB(path)
-	lsdb.Replica = litestream.NewReplicaWithClient(lsdb, client)
-
-	if restore != nil {
-		err := lsdb.Replica.Restore(ctx, *restore)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	err := lsdb.Open()
-	if err != nil {
-		return nil, err
-	}
-	return lsdb, nil
-}

litestream/go.mod

@@ -7,7 +7,6 @@ require (
 	github.com/ncruces/go-sqlite3 v0.30.1
 	github.com/ncruces/wbt v0.2.0
 	github.com/superfly/ltx v0.5.0
-	golang.org/x/sync v0.18.0
 )
 
 // github.com/ncruces/go-sqlite3
@@ -29,7 +28,7 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/prometheus/client_golang v1.23.2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
-	github.com/prometheus/common v0.67.4 // indirect
+	github.com/prometheus/common v0.67.3 // indirect
 	github.com/prometheus/procfs v0.19.2 // indirect
 	github.com/psanford/sqlite3vfs v0.0.0-20240315230605-24e1d98cf361 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
@@ -61,4 +60,4 @@ require (
 	github.com/aws/smithy-go v1.22.5 // indirect
 )
 
-replace modernc.org/sqlite => github.com/ncruces/go-sqlite3/litestream/modernc v0.30.1
+replace modernc.org/sqlite => github.com/ncruces/go-sqlite3/litestream/modernc v0.0.0-20251109124432-99b097de3b79

litestream/go.sum

@@ -115,8 +115,8 @@ github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/ncruces/go-sqlite3 v0.30.1 h1:pHC3YsyRdJv4pCMB4MO1Q2BXw/CAa+Hoj7GSaKtVk+g=
 github.com/ncruces/go-sqlite3 v0.30.1/go.mod h1:UVsWrQaq1qkcal5/vT5lOJnZCVlR5rsThKdwidjFsKc=
-github.com/ncruces/go-sqlite3/litestream/modernc v0.30.1 h1:3SNAOrm+qmLprkZybcvBrVNHyt0QYHljUGxmOXnL+K0=
-github.com/ncruces/go-sqlite3/litestream/modernc v0.30.1/go.mod h1:GSM2gXEOb9HIFFtsl0IUtnpvpDmVi7Kbp8z5GzwA0Tw=
+github.com/ncruces/go-sqlite3/litestream/modernc v0.0.0-20251109124432-99b097de3b79 h1:evpQceUV2vRbOe84U/QhBBchfqFERRHTx1JOadFFMLE=
+github.com/ncruces/go-sqlite3/litestream/modernc v0.0.0-20251109124432-99b097de3b79/go.mod h1:GSM2gXEOb9HIFFtsl0IUtnpvpDmVi7Kbp8z5GzwA0Tw=
 github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
 github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
 github.com/ncruces/wbt v0.2.0 h1:Q9zlKOBSZc7Yy/R2cGa35g6RKUUE3BjNIW3tfGC4F04=
@@ -133,8 +133,8 @@ github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h
 github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
 github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
 github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
-github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc=
-github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI=
+github.com/prometheus/common v0.67.3 h1:shd26MlnwTw5jksTDhC7rTQIteBxy+ZZDr3t7F2xN2Q=
+github.com/prometheus/common v0.67.3/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI=
 github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
 github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
 github.com/psanford/sqlite3vfs v0.0.0-20240315230605-24e1d98cf361 h1:vAKifIJuYY306ZJSrwDgKonWcJGELijdaenABqbV03E=
@@ -169,8 +169,8 @@ golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
 golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
 golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY=
 golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
-golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
-golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
+golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
 golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=

vfs/shm_copy.go

@@ -35,38 +35,44 @@ func (s *vfsShm) shmAcquire(errp *error) {
 	if errp != nil && *errp != nil {
 		return
 	}
-	if len(s.ptrs) == 0 || shmEqual(s.shadow[0][:], s.shared[0][:]) {
+	if len(s.ptrs) == 0 {
 		return
 	}
-	// Copies modified words from shared to private memory.
-	for id, p := range s.ptrs {
-		shared := shmPage(s.shared[id][:])
-		shadow := shmPage(s.shadow[id][:])
-		privat := shmPage(util.View(s.mod, p, _WALINDEX_PGSZ))
-		for i, shared := range shared {
-			if shadow[i] != shared {
-				shadow[i] = shared
-				privat[i] = shared
-			}
-		}
+	if !shmCopyHeader(
+		util.View(s.mod, s.ptrs[0], _WALINDEX_HDR_SIZE),
+		s.shadow[0][:],
+		s.shared[0][:]) {
+		return
+	}
+
+	skip := _WALINDEX_HDR_SIZE
+	for id := range s.ptrs {
+		shmCopyTables(
+			util.View(s.mod, s.ptrs[id], _WALINDEX_PGSZ)[skip:],
+			s.shadow[id][skip:],
+			s.shared[id][skip:])
+		skip = 0
 	}
 }
 
 func (s *vfsShm) shmRelease() {
-	if len(s.ptrs) == 0 || shmEqual(s.shadow[0][:], util.View(s.mod, s.ptrs[0], _WALINDEX_HDR_SIZE)) {
+	if len(s.ptrs) == 0 {
 		return
 	}
-	// Copies modified words from private to shared memory.
-	for id, p := range s.ptrs {
-		shared := shmPage(s.shared[id][:])
-		shadow := shmPage(s.shadow[id][:])
-		privat := shmPage(util.View(s.mod, p, _WALINDEX_PGSZ))
-		for i, privat := range privat {
-			if shadow[i] != privat {
-				shadow[i] = privat
-				shared[i] = privat
-			}
-		}
+	if !shmCopyHeader(
+		s.shared[0][:],
+		s.shadow[0][:],
+		util.View(s.mod, s.ptrs[0], _WALINDEX_HDR_SIZE)) {
+		return
+	}
+
+	skip := _WALINDEX_HDR_SIZE
+	for id := range s.ptrs {
+		shmCopyTables(
+			s.shared[id][skip:],
+			s.shadow[id][skip:],
+			util.View(s.mod, s.ptrs[id], _WALINDEX_PGSZ)[skip:])
+		skip = 0
 	}
 }
 
@@ -77,11 +83,40 @@ func (s *vfsShm) shmBarrier() {
 	s.Unlock()
 }
 
-func shmPage(s []byte) *[_WALINDEX_PGSZ / 4]uint32 {
-	p := (*uint32)(unsafe.Pointer(unsafe.SliceData(s)))
-	return (*[_WALINDEX_PGSZ / 4]uint32)(unsafe.Slice(p, _WALINDEX_PGSZ/4))
+func shmCopyTables(v1, v2, v3 []byte) {
+	if string(v2) != string(v3) {
+		copy(v1, v3)
+		copy(v2, v3)
+	}
 }
 
-func shmEqual(v1, v2 []byte) bool {
-	return *(*[_WALINDEX_HDR_SIZE]byte)(v1[:]) == *(*[_WALINDEX_HDR_SIZE]byte)(v2[:])
+func shmCopyHeader(s1, s2, s3 []byte) (ret bool) {
+	// First copy of the WAL Index Information.
+	if string(s2[:48]) != string(s3[:48]) {
+		copy(s1, s3[:48])
+		copy(s2, s3[:48])
+		ret = true
+	}
+	// Second copy of the WAL Index Information.
+	if string(s2[48:][:48]) != string(s3[48:][:48]) {
+		copy(s1[48:], s3[48:][:48])
+		copy(s2[48:], s3[48:][:48])
+		ret = true
+	}
+	// Checkpoint Information and Locks.
+	i1 := shmCheckpointInfo(s1)
+	i2 := shmCheckpointInfo(s2)
+	for i, i3 := range shmCheckpointInfo(s3) {
+		if i2[i] != i3 {
+			i1[i] = i3
+			i2[i] = i3
+			ret = true
+		}
+	}
+	return
+}
+
+func shmCheckpointInfo(s []byte) *[10]uint32 {
+	p := (*uint32)(unsafe.Pointer(&s[96]))
+	return (*[10]uint32)(unsafe.Slice(p, 10))
 }