Refactor reader VFS API.

README.md

@@ -20,9 +20,9 @@ wraps the [C SQLite VFS API](https://www.sqlite.org/vfs.html) and provides a pur
 
 ### Caveats
 
-This module replaces the SQLite [OS Interface](https://www.sqlite.org/vfs.html) (aka VFS)
+This module replaces the SQLite [OS Interface](https://www.sqlite.org/vfs.html)
-with a [pure Go](sqlite3vfs/) implementation.
+(aka VFS) with a [pure Go](sqlite3vfs/) implementation.
-This has numerous benefits, but also comes with some drawbacks.
+This has benefits, but also comes with some drawbacks.
 
 #### Write-Ahead Logging
 
@@ -57,7 +57,7 @@ BSD locks may _not_ be compatible with process-associated POSIX advisory locks.
 
 #### Testing
 
-The pure Go VFS is stress tested by running an unmodified build of SQLite's
+The pure Go VFS is tested by running an unmodified build of SQLite's
 [mptest](https://github.com/sqlite/sqlite/blob/master/mptest/mptest.c)
 on Linux, macOS and Windows.
 Performance is tested by running
@@ -70,12 +70,12 @@ Performance is tested by running
   - [x] incremental BLOB I/O
   - [x] online backup
   - [ ] session extension
-- [ ] custom SQL functions
 - [ ] custom VFSes
   - [x] custom VFS API
   - [x] in-memory VFS
   - [x] read-only VFS, wrapping an [`io.ReaderAt`](https://pkg.go.dev/io#ReaderAt)
   - [ ] cloud-based VFS, based on [Cloud Backed SQLite](https://sqlite.org/cloudsqlite/doc/trunk/www/index.wiki)
+- [ ] custom SQL functions
 
 ### Alternatives
 

embed/README.md

@@ -17,7 +17,8 @@ The following optional features are compiled in:
 - [uuid](https://github.com/sqlite/sqlite/blob/master/ext/misc/uuid.c)
 - [time](../sqlite3/time.c)
 
-See the [configuration options](../sqlite3/sqlite_cfg.h).
+See the [configuration options](../sqlite3/sqlite_cfg.h),
+and [patches](../sqlite3) applied.
 
 Built using [`wasi-sdk`](https://github.com/WebAssembly/wasi-sdk),
 and [`binaryen`](https://github.com/WebAssembly/binaryen).

sqlite3memdb/README.md

@@ -0,0 +1,9 @@
+# Go `"memdb"` SQLite VFS
+
+This package implements the [`"memdb"`](https://www.sqlite.org/src/file/src/memdb.c)
+SQLite VFS in pure Go.
+
+It has some benefits over the C version:
+- the memory backing the database needs not be contiguous,
+- the database can grow/shrink incrementally without copying,
+- reader-writer concurrency is slightly improved.

sqlite3reader/README.md

@@ -0,0 +1,5 @@
+# Go `"reader"` SQLite VFS
+
+This package implements a `"reader"` SQLite VFS
+that allows accessing any [`io.ReaderAt`](https://pkg.go.dev/io#ReaderAt)
+as an immutable SQLite database.

sqlite3reader/api.go

@@ -0,0 +1,83 @@
+// Package sqlite3reader implements an SQLite VFS for immutable databases.
+//
+// The "reader" [sqlite3vfs.VFS] permits accessing any [io.ReaderAt]
+// as an immutable SQLite database.
+//
+// Importing package sqlite3reader registers the VFS.
+//
+//	import _ "github.com/ncruces/go-sqlite3/sqlite3reader"
+package sqlite3reader
+
+import (
+	"io"
+	"io/fs"
+	"sync"
+
+	"github.com/ncruces/go-sqlite3"
+	"github.com/ncruces/go-sqlite3/sqlite3vfs"
+)
+
+func init() {
+	sqlite3vfs.Register("reader", vfs{})
+}
+
+var (
+	readerMtx sync.RWMutex
+	readerDBs = map[string]SizeReaderAt{}
+)
+
+// Create creates an immutable database from reader.
+// The caller should insure that data from reader does not mutate,
+// otherwise SQLite might return incorrect query results and/or [sqlite3.CORRUPT] errors.
+func Create(name string, reader SizeReaderAt) {
+	readerMtx.Lock()
+	defer readerMtx.Unlock()
+	readerDBs[name] = reader
+}
+
+// Delete deletes a shared memory database.
+func Delete(name string) {
+	readerMtx.Lock()
+	defer readerMtx.Unlock()
+	delete(readerDBs, name)
+}
+
+// A SizeReaderAt is a ReaderAt with a Size method.
+// Use [NewSizeReaderAt] to adapt different Size interfaces.
+type SizeReaderAt interface {
+	Size() (int64, error)
+	io.ReaderAt
+}
+
+// NewSizeReaderAt returns a SizeReaderAt given an io.ReaderAt
+// that implements one of:
+//   - Size() (int64, error)
+//   - Size() int64
+//   - Len() int
+//   - Stat() (fs.FileInfo, error)
+//   - Seek(offset int64, whence int) (int64, error)
+func NewSizeReaderAt(r io.ReaderAt) SizeReaderAt {
+	return sizer{r}
+}
+
+type sizer struct{ io.ReaderAt }
+
+func (s sizer) Size() (int64, error) {
+	switch s := s.ReaderAt.(type) {
+	case interface{ Size() (int64, error) }:
+		return s.Size()
+	case interface{ Size() int64 }:
+		return s.Size(), nil
+	case interface{ Len() int }:
+		return int64(s.Len()), nil
+	case interface{ Stat() (fs.FileInfo, error) }:
+		fi, err := s.Stat()
+		if err != nil {
+			return 0, err
+		}
+		return fi.Size(), nil
+	case io.Seeker:
+		return s.Seek(0, io.SeekEnd)
+	}
+	return 0, sqlite3.IOERR_SEEK
+}

sqlite3reader/example_test.go

@@ -1,4 +1,4 @@
-package sqlite3vfs_test
+package sqlite3reader_test
 
 import (
 	"bytes"
@@ -10,19 +10,18 @@ import (
 
 	_ "github.com/ncruces/go-sqlite3/driver"
 	_ "github.com/ncruces/go-sqlite3/embed"
-	"github.com/ncruces/go-sqlite3/sqlite3vfs"
+	"github.com/ncruces/go-sqlite3/sqlite3reader"
 	"github.com/psanford/httpreadat"
 )
 
 //go:embed testdata/test.db
 var testDB []byte
 
-func ExampleReaderVFS_http() {
+func Example_http() {
-	sqlite3vfs.Register("httpvfs", sqlite3vfs.ReaderVFS{
+	sqlite3reader.Create("demo.db", httpreadat.New("https://www.sanford.io/demo.db"))
-		"demo.db": httpreadat.New("https://www.sanford.io/demo.db"),
+	defer sqlite3reader.Delete("demo.db")
-	})
 
-	db, err := sql.Open("sqlite3", "file:demo.db?vfs=httpvfs&mode=ro")
+	db, err := sql.Open("sqlite3", "file:demo.db?vfs=reader")
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -65,12 +64,11 @@ func ExampleReaderVFS_http() {
 	// 2012.06: 18270 million Dollars
 }
 
-func ExampleReaderVFS_embed() {
+func Example_embed() {
-	sqlite3vfs.Register("reader", sqlite3vfs.ReaderVFS{
+	sqlite3reader.Create("test.db", sqlite3reader.NewSizeReaderAt(bytes.NewReader(testDB)))
-		"test.db": sqlite3vfs.NewSizeReaderAt(bytes.NewReader(testDB)),
+	defer sqlite3reader.Delete("test.db")
-	})
 
-	db, err := sql.Open("sqlite3", "file:test.db?vfs=reader&mode=ro")
+	db, err := sql.Open("sqlite3", "file:test.db?vfs=reader")
 	if err != nil {
 		log.Fatal(err)
 	}

sqlite3reader/reader.go

@@ -0,0 +1,79 @@
+package sqlite3reader
+
+import (
+	"io"
+
+	"github.com/ncruces/go-sqlite3"
+	"github.com/ncruces/go-sqlite3/sqlite3vfs"
+)
+
+type vfs struct{}
+
+// Open implements the [sqlite3vfs.VFS] interface.
+func (vfs) Open(name string, flags sqlite3vfs.OpenFlag) (sqlite3vfs.File, sqlite3vfs.OpenFlag, error) {
+	if flags&sqlite3vfs.OPEN_MAIN_DB == 0 {
+		return nil, flags, sqlite3.CANTOPEN
+	}
+	readerMtx.RLock()
+	defer readerMtx.RUnlock()
+	if ra, ok := readerDBs[name]; ok {
+		return readerFile{ra}, flags | sqlite3vfs.OPEN_READONLY, nil
+	}
+	return nil, flags, sqlite3.CANTOPEN
+}
+
+// Delete implements the [sqlite3vfs.VFS] interface.
+func (vfs) Delete(name string, dirSync bool) error {
+	return sqlite3.IOERR_DELETE
+}
+
+// Access implements the [sqlite3vfs.VFS] interface.
+func (vfs) Access(name string, flag sqlite3vfs.AccessFlag) (bool, error) {
+	return false, nil
+}
+
+// FullPathname implements the [sqlite3vfs.VFS] interface.
+func (vfs) FullPathname(name string) (string, error) {
+	return name, nil
+}
+
+type readerFile struct{ SizeReaderAt }
+
+func (r readerFile) Close() error {
+	if c, ok := r.SizeReaderAt.(io.Closer); ok {
+		return c.Close()
+	}
+	return nil
+}
+
+func (readerFile) WriteAt(b []byte, off int64) (n int, err error) {
+	return 0, sqlite3.READONLY
+}
+
+func (readerFile) Truncate(size int64) error {
+	return sqlite3.READONLY
+}
+
+func (readerFile) Sync(flag sqlite3vfs.SyncFlag) error {
+	return nil
+}
+
+func (readerFile) Lock(lock sqlite3vfs.LockLevel) error {
+	return nil
+}
+
+func (readerFile) Unlock(lock sqlite3vfs.LockLevel) error {
+	return nil
+}
+
+func (readerFile) CheckReservedLock() (bool, error) {
+	return false, nil
+}
+
+func (readerFile) SectorSize() int {
+	return 0
+}
+
+func (readerFile) DeviceCharacteristics() sqlite3vfs.DeviceCharacteristic {
+	return sqlite3vfs.IOCAP_IMMUTABLE
+}

sqlite3reader/reader_test.go

@@ -1,4 +1,4 @@
-package sqlite3vfs
+package sqlite3reader
 
 import (
 	"io"

sqlite3vfs/README.md

@@ -0,0 +1,9 @@
+# Go SQLite VFS API
+
+This package implements the SQLite [OS Interface](https://www.sqlite.org/vfs.html) (aka VFS).
+
+It replaces the default VFS with a pure Go implementation,
+that is tested on Linux, macOS and Windows,
+but which should also work on illumos and the various BSDs.
+
+It also exposes interfaces that should allow you to implement your own custom VFSes.

sqlite3vfs/reader.go

@@ -1,118 +0,0 @@
-package sqlite3vfs
-
-import (
-	"io"
-	"io/fs"
-)
-
-// A ReaderVFS is a [VFS] for immutable databases.
-type ReaderVFS map[string]SizeReaderAt
-
-var _ VFS = ReaderVFS{}
-
-// A SizeReaderAt is a ReaderAt with a Size method.
-// Use [NewSizeReaderAt] to adapt different Size interfaces.
-type SizeReaderAt interface {
-	Size() (int64, error)
-	io.ReaderAt
-}
-
-// Open implements the [VFS] interface.
-func (vfs ReaderVFS) Open(name string, flags OpenFlag) (File, OpenFlag, error) {
-	if flags&OPEN_MAIN_DB == 0 {
-		return nil, flags, _CANTOPEN
-	}
-	if ra, ok := vfs[name]; ok {
-		return readerFile{ra}, flags, nil
-	}
-	return nil, flags, _CANTOPEN
-}
-
-// Delete implements the [VFS] interface.
-func (vfs ReaderVFS) Delete(name string, dirSync bool) error {
-	return _IOERR_DELETE
-}
-
-// Access implements the [VFS] interface.
-func (vfs ReaderVFS) Access(name string, flag AccessFlag) (bool, error) {
-	return false, nil
-}
-
-// FullPathname implements the [VFS] interface.
-func (vfs ReaderVFS) FullPathname(name string) (string, error) {
-	return name, nil
-}
-
-type readerFile struct{ SizeReaderAt }
-
-func (r readerFile) Close() error {
-	if c, ok := r.SizeReaderAt.(io.Closer); ok {
-		return c.Close()
-	}
-	return nil
-}
-
-func (readerFile) WriteAt(b []byte, off int64) (n int, err error) {
-	return 0, _READONLY
-}
-
-func (readerFile) Truncate(size int64) error {
-	return _READONLY
-}
-
-func (readerFile) Sync(flag SyncFlag) error {
-	return nil
-}
-
-func (readerFile) Lock(lock LockLevel) error {
-	return nil
-}
-
-func (readerFile) Unlock(lock LockLevel) error {
-	return nil
-}
-
-func (readerFile) CheckReservedLock() (bool, error) {
-	return false, nil
-}
-
-func (readerFile) SectorSize() int {
-	return 0
-}
-
-func (readerFile) DeviceCharacteristics() DeviceCharacteristic {
-	return IOCAP_IMMUTABLE
-}
-
-// NewSizeReaderAt returns a SizeReaderAt given an io.ReaderAt
-// that implements one of:
-//   - Size() (int64, error)
-//   - Size() int64
-//   - Len() int
-//   - Stat() (fs.FileInfo, error)
-//   - Seek(offset int64, whence int) (int64, error)
-func NewSizeReaderAt(r io.ReaderAt) SizeReaderAt {
-	return sizer{r}
-}
-
-type sizer struct{ io.ReaderAt }
-
-func (s sizer) Size() (int64, error) {
-	switch s := s.ReaderAt.(type) {
-	case interface{ Size() (int64, error) }:
-		return s.Size()
-	case interface{ Size() int64 }:
-		return s.Size(), nil
-	case interface{ Len() int }:
-		return int64(s.Len()), nil
-	case interface{ Stat() (fs.FileInfo, error) }:
-		fi, err := s.Stat()
-		if err != nil {
-			return 0, err
-		}
-		return fi.Size(), nil
-	case io.Seeker:
-		return s.Seek(0, io.SeekEnd)
-	}
-	return 0, _IOERR_SEEK
-}

tests/vfs_test.go

@@ -8,7 +8,7 @@ import (
 	_ "github.com/ncruces/go-sqlite3/driver"
 	_ "github.com/ncruces/go-sqlite3/embed"
 	"github.com/ncruces/go-sqlite3/sqlite3memdb"
-	"github.com/ncruces/go-sqlite3/sqlite3vfs"
+	"github.com/ncruces/go-sqlite3/sqlite3reader"
 )
 
 func TestMemoryVFS_Open_notfound(t *testing.T) {
@@ -24,10 +24,9 @@ func TestMemoryVFS_Open_notfound(t *testing.T) {
 }
 
 func TestReaderVFS_Open_notfound(t *testing.T) {
-	sqlite3vfs.Register("reader", sqlite3vfs.ReaderVFS{})
+	sqlite3reader.Delete("demo.db")
-	defer sqlite3vfs.Unregister("reader")
 
-	_, err := sqlite3.Open("file:demo.db?vfs=reader&mode=ro")
+	_, err := sqlite3.Open("file:demo.db?vfs=reader")
 	if err == nil {
 		t.Error("want error")
 	}