diff --git a/README.md b/README.md index 6f170c3..429a33c 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,9 @@ wraps the [C SQLite VFS API](https://www.sqlite.org/vfs.html) and provides a pur ### Caveats -This module replaces the SQLite [OS Interface](https://www.sqlite.org/vfs.html) (aka VFS) -with a [pure Go](sqlite3vfs/) implementation. -This has numerous benefits, but also comes with some drawbacks. +This module replaces the SQLite [OS Interface](https://www.sqlite.org/vfs.html) +(aka VFS) with a [pure Go](sqlite3vfs/) implementation. +This has benefits, but also comes with some drawbacks. #### Write-Ahead Logging @@ -57,7 +57,7 @@ BSD locks may _not_ be compatible with process-associated POSIX advisory locks. #### Testing -The pure Go VFS is stress tested by running an unmodified build of SQLite's +The pure Go VFS is tested by running an unmodified build of SQLite's [mptest](https://github.com/sqlite/sqlite/blob/master/mptest/mptest.c) on Linux, macOS and Windows. Performance is tested by running @@ -70,12 +70,12 @@ Performance is tested by running - [x] incremental BLOB I/O - [x] online backup - [ ] session extension -- [ ] custom SQL functions - [ ] custom VFSes - [x] custom VFS API - [x] in-memory VFS - [x] read-only VFS, wrapping an [`io.ReaderAt`](https://pkg.go.dev/io#ReaderAt) - [ ] cloud-based VFS, based on [Cloud Backed SQLite](https://sqlite.org/cloudsqlite/doc/trunk/www/index.wiki) +- [ ] custom SQL functions ### Alternatives diff --git a/embed/README.md b/embed/README.md index b826b1c..06c4886 100644 --- a/embed/README.md +++ b/embed/README.md @@ -17,7 +17,8 @@ The following optional features are compiled in: - [uuid](https://github.com/sqlite/sqlite/blob/master/ext/misc/uuid.c) - [time](../sqlite3/time.c) -See the [configuration options](../sqlite3/sqlite_cfg.h). +See the [configuration options](../sqlite3/sqlite_cfg.h), +and [patches](../sqlite3) applied. Built using [`wasi-sdk`](https://github.com/WebAssembly/wasi-sdk), and [`binaryen`](https://github.com/WebAssembly/binaryen). \ No newline at end of file diff --git a/sqlite3memdb/README.md b/sqlite3memdb/README.md new file mode 100644 index 0000000..6a2b8a6 --- /dev/null +++ b/sqlite3memdb/README.md @@ -0,0 +1,9 @@ +# Go `"memdb"` SQLite VFS + +This package implements the [`"memdb"`](https://www.sqlite.org/src/file/src/memdb.c) +SQLite VFS in pure Go. + +It has some benefits over the C version: +- the memory backing the database needs not be contiguous, +- the database can grow/shrink incrementally without copying, +- reader-writer concurrency is slightly improved. \ No newline at end of file diff --git a/sqlite3reader/README.md b/sqlite3reader/README.md new file mode 100644 index 0000000..47a5f33 --- /dev/null +++ b/sqlite3reader/README.md @@ -0,0 +1,5 @@ +# Go `"reader"` SQLite VFS + +This package implements a `"reader"` SQLite VFS +that allows accessing any [`io.ReaderAt`](https://pkg.go.dev/io#ReaderAt) +as an immutable SQLite database. \ No newline at end of file diff --git a/sqlite3reader/api.go b/sqlite3reader/api.go new file mode 100644 index 0000000..5e9f8a2 --- /dev/null +++ b/sqlite3reader/api.go @@ -0,0 +1,83 @@ +// Package sqlite3reader implements an SQLite VFS for immutable databases. +// +// The "reader" [sqlite3vfs.VFS] permits accessing any [io.ReaderAt] +// as an immutable SQLite database. +// +// Importing package sqlite3reader registers the VFS. +// +// import _ "github.com/ncruces/go-sqlite3/sqlite3reader" +package sqlite3reader + +import ( + "io" + "io/fs" + "sync" + + "github.com/ncruces/go-sqlite3" + "github.com/ncruces/go-sqlite3/sqlite3vfs" +) + +func init() { + sqlite3vfs.Register("reader", vfs{}) +} + +var ( + readerMtx sync.RWMutex + readerDBs = map[string]SizeReaderAt{} +) + +// Create creates an immutable database from reader. +// The caller should insure that data from reader does not mutate, +// otherwise SQLite might return incorrect query results and/or [sqlite3.CORRUPT] errors. +func Create(name string, reader SizeReaderAt) { + readerMtx.Lock() + defer readerMtx.Unlock() + readerDBs[name] = reader +} + +// Delete deletes a shared memory database. +func Delete(name string) { + readerMtx.Lock() + defer readerMtx.Unlock() + delete(readerDBs, name) +} + +// A SizeReaderAt is a ReaderAt with a Size method. +// Use [NewSizeReaderAt] to adapt different Size interfaces. +type SizeReaderAt interface { + Size() (int64, error) + io.ReaderAt +} + +// NewSizeReaderAt returns a SizeReaderAt given an io.ReaderAt +// that implements one of: +// - Size() (int64, error) +// - Size() int64 +// - Len() int +// - Stat() (fs.FileInfo, error) +// - Seek(offset int64, whence int) (int64, error) +func NewSizeReaderAt(r io.ReaderAt) SizeReaderAt { + return sizer{r} +} + +type sizer struct{ io.ReaderAt } + +func (s sizer) Size() (int64, error) { + switch s := s.ReaderAt.(type) { + case interface{ Size() (int64, error) }: + return s.Size() + case interface{ Size() int64 }: + return s.Size(), nil + case interface{ Len() int }: + return int64(s.Len()), nil + case interface{ Stat() (fs.FileInfo, error) }: + fi, err := s.Stat() + if err != nil { + return 0, err + } + return fi.Size(), nil + case io.Seeker: + return s.Seek(0, io.SeekEnd) + } + return 0, sqlite3.IOERR_SEEK +} diff --git a/sqlite3vfs/example_test.go b/sqlite3reader/example_test.go similarity index 74% rename from sqlite3vfs/example_test.go rename to sqlite3reader/example_test.go index ff4507b..93fc244 100644 --- a/sqlite3vfs/example_test.go +++ b/sqlite3reader/example_test.go @@ -1,4 +1,4 @@ -package sqlite3vfs_test +package sqlite3reader_test import ( "bytes" @@ -10,19 +10,18 @@ import ( _ "github.com/ncruces/go-sqlite3/driver" _ "github.com/ncruces/go-sqlite3/embed" - "github.com/ncruces/go-sqlite3/sqlite3vfs" + "github.com/ncruces/go-sqlite3/sqlite3reader" "github.com/psanford/httpreadat" ) //go:embed testdata/test.db var testDB []byte -func ExampleReaderVFS_http() { - sqlite3vfs.Register("httpvfs", sqlite3vfs.ReaderVFS{ - "demo.db": httpreadat.New("https://www.sanford.io/demo.db"), - }) +func Example_http() { + sqlite3reader.Create("demo.db", httpreadat.New("https://www.sanford.io/demo.db")) + defer sqlite3reader.Delete("demo.db") - db, err := sql.Open("sqlite3", "file:demo.db?vfs=httpvfs&mode=ro") + db, err := sql.Open("sqlite3", "file:demo.db?vfs=reader") if err != nil { log.Fatal(err) } @@ -65,12 +64,11 @@ func ExampleReaderVFS_http() { // 2012.06: 18270 million Dollars } -func ExampleReaderVFS_embed() { - sqlite3vfs.Register("reader", sqlite3vfs.ReaderVFS{ - "test.db": sqlite3vfs.NewSizeReaderAt(bytes.NewReader(testDB)), - }) +func Example_embed() { + sqlite3reader.Create("test.db", sqlite3reader.NewSizeReaderAt(bytes.NewReader(testDB))) + defer sqlite3reader.Delete("test.db") - db, err := sql.Open("sqlite3", "file:test.db?vfs=reader&mode=ro") + db, err := sql.Open("sqlite3", "file:test.db?vfs=reader") if err != nil { log.Fatal(err) } diff --git a/sqlite3reader/reader.go b/sqlite3reader/reader.go new file mode 100644 index 0000000..6688447 --- /dev/null +++ b/sqlite3reader/reader.go @@ -0,0 +1,79 @@ +package sqlite3reader + +import ( + "io" + + "github.com/ncruces/go-sqlite3" + "github.com/ncruces/go-sqlite3/sqlite3vfs" +) + +type vfs struct{} + +// Open implements the [sqlite3vfs.VFS] interface. +func (vfs) Open(name string, flags sqlite3vfs.OpenFlag) (sqlite3vfs.File, sqlite3vfs.OpenFlag, error) { + if flags&sqlite3vfs.OPEN_MAIN_DB == 0 { + return nil, flags, sqlite3.CANTOPEN + } + readerMtx.RLock() + defer readerMtx.RUnlock() + if ra, ok := readerDBs[name]; ok { + return readerFile{ra}, flags | sqlite3vfs.OPEN_READONLY, nil + } + return nil, flags, sqlite3.CANTOPEN +} + +// Delete implements the [sqlite3vfs.VFS] interface. +func (vfs) Delete(name string, dirSync bool) error { + return sqlite3.IOERR_DELETE +} + +// Access implements the [sqlite3vfs.VFS] interface. +func (vfs) Access(name string, flag sqlite3vfs.AccessFlag) (bool, error) { + return false, nil +} + +// FullPathname implements the [sqlite3vfs.VFS] interface. +func (vfs) FullPathname(name string) (string, error) { + return name, nil +} + +type readerFile struct{ SizeReaderAt } + +func (r readerFile) Close() error { + if c, ok := r.SizeReaderAt.(io.Closer); ok { + return c.Close() + } + return nil +} + +func (readerFile) WriteAt(b []byte, off int64) (n int, err error) { + return 0, sqlite3.READONLY +} + +func (readerFile) Truncate(size int64) error { + return sqlite3.READONLY +} + +func (readerFile) Sync(flag sqlite3vfs.SyncFlag) error { + return nil +} + +func (readerFile) Lock(lock sqlite3vfs.LockLevel) error { + return nil +} + +func (readerFile) Unlock(lock sqlite3vfs.LockLevel) error { + return nil +} + +func (readerFile) CheckReservedLock() (bool, error) { + return false, nil +} + +func (readerFile) SectorSize() int { + return 0 +} + +func (readerFile) DeviceCharacteristics() sqlite3vfs.DeviceCharacteristic { + return sqlite3vfs.IOCAP_IMMUTABLE +} diff --git a/sqlite3vfs/reader_test.go b/sqlite3reader/reader_test.go similarity index 98% rename from sqlite3vfs/reader_test.go rename to sqlite3reader/reader_test.go index c94ce51..2ce815e 100644 --- a/sqlite3vfs/reader_test.go +++ b/sqlite3reader/reader_test.go @@ -1,4 +1,4 @@ -package sqlite3vfs +package sqlite3reader import ( "io" diff --git a/sqlite3vfs/testdata/test.db b/sqlite3reader/testdata/test.db similarity index 100% rename from sqlite3vfs/testdata/test.db rename to sqlite3reader/testdata/test.db diff --git a/sqlite3vfs/README.md b/sqlite3vfs/README.md new file mode 100644 index 0000000..f028964 --- /dev/null +++ b/sqlite3vfs/README.md @@ -0,0 +1,9 @@ +# Go SQLite VFS API + +This package implements the SQLite [OS Interface](https://www.sqlite.org/vfs.html) (aka VFS). + +It replaces the default VFS with a pure Go implementation, +that is tested on Linux, macOS and Windows, +but which should also work on illumos and the various BSDs. + +It also exposes interfaces that should allow you to implement your own custom VFSes. \ No newline at end of file diff --git a/sqlite3vfs/reader.go b/sqlite3vfs/reader.go deleted file mode 100644 index 09d1a7a..0000000 --- a/sqlite3vfs/reader.go +++ /dev/null @@ -1,118 +0,0 @@ -package sqlite3vfs - -import ( - "io" - "io/fs" -) - -// A ReaderVFS is a [VFS] for immutable databases. -type ReaderVFS map[string]SizeReaderAt - -var _ VFS = ReaderVFS{} - -// A SizeReaderAt is a ReaderAt with a Size method. -// Use [NewSizeReaderAt] to adapt different Size interfaces. -type SizeReaderAt interface { - Size() (int64, error) - io.ReaderAt -} - -// Open implements the [VFS] interface. -func (vfs ReaderVFS) Open(name string, flags OpenFlag) (File, OpenFlag, error) { - if flags&OPEN_MAIN_DB == 0 { - return nil, flags, _CANTOPEN - } - if ra, ok := vfs[name]; ok { - return readerFile{ra}, flags, nil - } - return nil, flags, _CANTOPEN -} - -// Delete implements the [VFS] interface. -func (vfs ReaderVFS) Delete(name string, dirSync bool) error { - return _IOERR_DELETE -} - -// Access implements the [VFS] interface. -func (vfs ReaderVFS) Access(name string, flag AccessFlag) (bool, error) { - return false, nil -} - -// FullPathname implements the [VFS] interface. -func (vfs ReaderVFS) FullPathname(name string) (string, error) { - return name, nil -} - -type readerFile struct{ SizeReaderAt } - -func (r readerFile) Close() error { - if c, ok := r.SizeReaderAt.(io.Closer); ok { - return c.Close() - } - return nil -} - -func (readerFile) WriteAt(b []byte, off int64) (n int, err error) { - return 0, _READONLY -} - -func (readerFile) Truncate(size int64) error { - return _READONLY -} - -func (readerFile) Sync(flag SyncFlag) error { - return nil -} - -func (readerFile) Lock(lock LockLevel) error { - return nil -} - -func (readerFile) Unlock(lock LockLevel) error { - return nil -} - -func (readerFile) CheckReservedLock() (bool, error) { - return false, nil -} - -func (readerFile) SectorSize() int { - return 0 -} - -func (readerFile) DeviceCharacteristics() DeviceCharacteristic { - return IOCAP_IMMUTABLE -} - -// NewSizeReaderAt returns a SizeReaderAt given an io.ReaderAt -// that implements one of: -// - Size() (int64, error) -// - Size() int64 -// - Len() int -// - Stat() (fs.FileInfo, error) -// - Seek(offset int64, whence int) (int64, error) -func NewSizeReaderAt(r io.ReaderAt) SizeReaderAt { - return sizer{r} -} - -type sizer struct{ io.ReaderAt } - -func (s sizer) Size() (int64, error) { - switch s := s.ReaderAt.(type) { - case interface{ Size() (int64, error) }: - return s.Size() - case interface{ Size() int64 }: - return s.Size(), nil - case interface{ Len() int }: - return int64(s.Len()), nil - case interface{ Stat() (fs.FileInfo, error) }: - fi, err := s.Stat() - if err != nil { - return 0, err - } - return fi.Size(), nil - case io.Seeker: - return s.Seek(0, io.SeekEnd) - } - return 0, _IOERR_SEEK -} diff --git a/tests/vfs_test.go b/tests/vfs_test.go index c6c9e9b..478e717 100644 --- a/tests/vfs_test.go +++ b/tests/vfs_test.go @@ -8,7 +8,7 @@ import ( _ "github.com/ncruces/go-sqlite3/driver" _ "github.com/ncruces/go-sqlite3/embed" "github.com/ncruces/go-sqlite3/sqlite3memdb" - "github.com/ncruces/go-sqlite3/sqlite3vfs" + "github.com/ncruces/go-sqlite3/sqlite3reader" ) func TestMemoryVFS_Open_notfound(t *testing.T) { @@ -24,10 +24,9 @@ func TestMemoryVFS_Open_notfound(t *testing.T) { } func TestReaderVFS_Open_notfound(t *testing.T) { - sqlite3vfs.Register("reader", sqlite3vfs.ReaderVFS{}) - defer sqlite3vfs.Unregister("reader") + sqlite3reader.Delete("demo.db") - _, err := sqlite3.Open("file:demo.db?vfs=reader&mode=ro") + _, err := sqlite3.Open("file:demo.db?vfs=reader") if err == nil { t.Error("want error") }