From 5992403052c123dc61c7d3cf4a17dfae25a30bc3 Mon Sep 17 00:00:00 2001 From: Nuno Cruces Date: Sat, 18 Nov 2023 15:43:39 +0000 Subject: [PATCH] Virtual tables. --- README.md | 16 ++-- ext/blob/blob.go | 5 +- sqlite.go | 20 ++--- vfs/tests/mptest/mptest_test.go | 12 +-- vfs/tests/speedtest1/speedtest1_test.go | 2 +- vtab.go | 115 ++++++++++++++++++++---- vtab_test.go | 4 - 7 files changed, 125 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 2470b79..b965711 100644 --- a/README.md +++ b/README.md @@ -81,21 +81,19 @@ on Linux, macOS, Windows and FreeBSD. Performance is tested by running [speedtest1](https://github.com/sqlite/sqlite/blob/master/test/speedtest1.c). -### Roadmap +### Features -- [ ] advanced SQLite features - - [x] custom functions - - [x] nested transactions +- [x] advanced SQLite features - [x] incremental BLOB I/O + - [x] nested transactions + - [x] custom functions + - [x] virtual tables - [x] online backup - [x] JSON support - - [ ] virtual tables - - [ ] session extension -- [ ] custom VFSes +- [x] custom VFSes - [x] custom VFS API - [x] in-memory VFS - - [x] read-only VFS, wrapping an [`io.ReaderAt`](https://pkg.go.dev/io#ReaderAt) - - [ ] cloud-based VFS, based on [Cloud Backed SQLite](https://sqlite.org/cloudsqlite/doc/trunk/www/index.wiki) + - [x] read-only VFS ### Alternatives diff --git a/ext/blob/blob.go b/ext/blob/blob.go index 9b1b7d6..f12dd13 100644 --- a/ext/blob/blob.go +++ b/ext/blob/blob.go @@ -34,7 +34,7 @@ func openBlob(ctx sqlite3.Context, arg ...sqlite3.Value) { if ok { err = blob.Reopen(row) if errors.Is(err, sqlite3.MISUSE) { - // Blob was closed (db, table or column changed). + // Blob was closed (db, table, column or write changed). ok = false } } @@ -58,10 +58,11 @@ func openBlob(ctx sqlite3.Context, arg ...sqlite3.Value) { return } - // This ensures the blob is closed if db, table or column change. + // This ensures the blob is closed if db, table, column or write change. ctx.SetAuxData(0, blob) ctx.SetAuxData(1, blob) ctx.SetAuxData(2, blob) + ctx.SetAuxData(4, blob) } // OpenCallback is the type for the blob_open callback. diff --git a/sqlite.go b/sqlite.go index d83b401..c3382db 100644 --- a/sqlite.go +++ b/sqlite.go @@ -428,16 +428,16 @@ func exportCallbacks(env wazero.HostModuleBuilder) wazero.HostModuleBuilder { util.ExportFuncII(env, "go_vtab_disconnect", vtabDisconnectCallback) util.ExportFuncII(env, "go_vtab_destroy", vtabDestroyCallback) util.ExportFuncIII(env, "go_vtab_best_index", vtabBestIndexCallback) - util.ExportFuncIIIII(env, "go_vtab_update", vtabCallbackIIII) - util.ExportFuncIII(env, "go_vtab_rename", vtabCallbackII) - util.ExportFuncIIIII(env, "go_vtab_find_function", vtabCallbackIIII) - util.ExportFuncII(env, "go_vtab_begin", vtabCallbackI) - util.ExportFuncII(env, "go_vtab_sync", vtabCallbackI) - util.ExportFuncII(env, "go_vtab_commit", vtabCallbackI) - util.ExportFuncII(env, "go_vtab_rollback", vtabCallbackI) - util.ExportFuncIII(env, "go_vtab_savepoint", vtabCallbackII) - util.ExportFuncIII(env, "go_vtab_release", vtabCallbackII) - util.ExportFuncIII(env, "go_vtab_rollback_to", vtabCallbackII) + util.ExportFuncIIIII(env, "go_vtab_update", vtabUpdateCallback) + util.ExportFuncIII(env, "go_vtab_rename", vtabRenameCallback) + util.ExportFuncIIIII(env, "go_vtab_find_function", vtabFindFuncCallback) + util.ExportFuncII(env, "go_vtab_begin", vtabBeginCallback) + util.ExportFuncII(env, "go_vtab_sync", vtabSyncCallback) + util.ExportFuncII(env, "go_vtab_commit", vtabCommitCallback) + util.ExportFuncII(env, "go_vtab_rollback", vtabRollbackCallback) + util.ExportFuncIII(env, "go_vtab_savepoint", vtabSavepointCallback) + util.ExportFuncIII(env, "go_vtab_release", vtabReleaseCallback) + util.ExportFuncIII(env, "go_vtab_rollback_to", vtabRollbackToCallback) util.ExportFuncIIIIII(env, "go_vtab_integrity", vtabIntegrityCallback) util.ExportFuncIII(env, "go_cur_open", cursorOpenCallback) util.ExportFuncII(env, "go_cur_close", cursorCloseCallback) diff --git a/vfs/tests/mptest/mptest_test.go b/vfs/tests/mptest/mptest_test.go index 2cc2b2d..d7d6918 100644 --- a/vfs/tests/mptest/mptest_test.go +++ b/vfs/tests/mptest/mptest_test.go @@ -102,7 +102,7 @@ func Test_config01(t *testing.T) { cfg := config(ctx).WithArgs("mptest", name, "config01.test") mod, err := rt.InstantiateModule(ctx, module, cfg) if err != nil { - t.Error(err) + t.Fatal(err) } mod.Close(ctx) } @@ -120,7 +120,7 @@ func Test_config02(t *testing.T) { cfg := config(ctx).WithArgs("mptest", name, "config02.test") mod, err := rt.InstantiateModule(ctx, module, cfg) if err != nil { - t.Error(err) + t.Fatal(err) } mod.Close(ctx) } @@ -135,7 +135,7 @@ func Test_crash01(t *testing.T) { cfg := config(ctx).WithArgs("mptest", name, "crash01.test") mod, err := rt.InstantiateModule(ctx, module, cfg) if err != nil { - t.Error(err) + t.Fatal(err) } mod.Close(ctx) } @@ -150,7 +150,7 @@ func Test_multiwrite01(t *testing.T) { cfg := config(ctx).WithArgs("mptest", name, "multiwrite01.test") mod, err := rt.InstantiateModule(ctx, module, cfg) if err != nil { - t.Error(err) + t.Fatal(err) } mod.Close(ctx) } @@ -163,7 +163,7 @@ func Test_config01_memory(t *testing.T) { "--timeout", "1000") mod, err := rt.InstantiateModule(ctx, module, cfg) if err != nil { - t.Error(err) + t.Fatal(err) } mod.Close(ctx) } @@ -180,7 +180,7 @@ func Test_multiwrite01_memory(t *testing.T) { "--timeout", "1000") mod, err := rt.InstantiateModule(ctx, module, cfg) if err != nil { - t.Error(err) + t.Fatal(err) } mod.Close(ctx) } diff --git a/vfs/tests/speedtest1/speedtest1_test.go b/vfs/tests/speedtest1/speedtest1_test.go index 5be9853..efecdaf 100644 --- a/vfs/tests/speedtest1/speedtest1_test.go +++ b/vfs/tests/speedtest1/speedtest1_test.go @@ -92,7 +92,7 @@ func Benchmark_speedtest1(b *testing.B) { WithRandSource(rand.Reader) mod, err := rt.InstantiateModule(ctx, module, cfg) if err != nil { - b.Error(err) + b.Fatal(err) } mod.Close(ctx) } diff --git a/vtab.go b/vtab.go index dce4a88..e26c4e7 100644 --- a/vtab.go +++ b/vtab.go @@ -133,6 +133,11 @@ type VTabOverloader interface { // A VTabChecker allows a virtual table to report errors // to the PRAGMA integrity_check PRAGMA quick_check commands. +// +// Integrity should return an error if it finds problems in the content of the virtual table, +// but should avoid returning a (wrapped) [Error], [ErrorCode] or [ExtendedErrorCode], +// as those indicate the Integrity method itself encountered problems +// while trying to evaluate the virtual table content. type VTabChecker interface { VTab // https://sqlite.org/vtab.html#xintegrity @@ -167,11 +172,11 @@ type VTabSavepointer interface { // A VTabCursor describes cursors that point // into the virtual table and are used // to loop through the virtual table. +// A VTabCursor may optionally implement +// [io.Closer] to free resources. // // http://sqlite.org/c3ref/vtab_cursor.html type VTabCursor interface { - // https://sqlite.org/vtab.html#xclose - Close() error // https://sqlite.org/vtab.html#xfilter Filter(idxNum int, idxStr string, arg ...Value) error // https://sqlite.org/vtab.html#xnext @@ -308,7 +313,7 @@ const ( // IndexScanFlag is a virtual table scan flag. // -// https://www.sqlite.org/c3ref/c_index_scan_unique.html +// https://sqlite.org/c3ref/c_index_scan_unique.html type IndexScanFlag uint32 const ( @@ -334,20 +339,22 @@ func vtabReflectCallback(name string) func(_ context.Context, _ api.Module, _, _ vtabPutHandle(ctx, mod, ppVTab, res[0].Interface()) } - return vtabError(ctx, mod, pzErr, _MODULE_ERROR, err) + return vtabError(ctx, mod, pzErr, _PTR_ERROR, err) } } func vtabDisconnectCallback(ctx context.Context, mod api.Module, pVTab uint32) uint32 { vtab := vtabGetHandle(ctx, mod, pVTab).(VTab) err := vtab.Disconnect() - return vtabError(ctx, mod, pVTab, _VTAB_ERROR, err) + vtabDelHandle(ctx, mod, pVTab) + return errorCode(err, ERROR) } func vtabDestroyCallback(ctx context.Context, mod api.Module, pVTab uint32) uint32 { vtab := vtabGetHandle(ctx, mod, pVTab).(VTabDestroyer) err := vtab.Destroy() - return vtabError(ctx, mod, pVTab, _VTAB_ERROR, err) + vtabDelHandle(ctx, mod, pVTab) + return errorCode(err, ERROR) } func vtabBestIndexCallback(ctx context.Context, mod api.Module, pVTab, pIdxInfo uint32) uint32 { @@ -361,20 +368,89 @@ func vtabBestIndexCallback(ctx context.Context, mod api.Module, pVTab, pIdxInfo return vtabError(ctx, mod, pVTab, _VTAB_ERROR, err) } -func vtabCallbackI(ctx context.Context, mod api.Module, pVTab uint32) uint32 { - return uint32(INTERNAL) +func vtabUpdateCallback(ctx context.Context, mod api.Module, pVTab, argc, argv, pRowID uint32) uint32 { + vtab := vtabGetHandle(ctx, mod, pVTab).(VTabUpdater) + + db := ctx.Value(connKey{}).(*Conn) + args := callbackArgs(db, argc, argv) + rowID, err := vtab.Update(args...) + if err == nil { + util.WriteUint64(mod, pRowID, uint64(rowID)) + } + + return vtabError(ctx, mod, pVTab, _VTAB_ERROR, err) } -func vtabCallbackII(ctx context.Context, mod api.Module, pVTab, _ uint32) uint32 { - return uint32(INTERNAL) +func vtabRenameCallback(ctx context.Context, mod api.Module, pVTab, zNew uint32) uint32 { + vtab := vtabGetHandle(ctx, mod, pVTab).(VTabRenamer) + err := vtab.Rename(util.ReadString(mod, zNew, _MAX_STRING)) + return vtabError(ctx, mod, pVTab, _VTAB_ERROR, err) } -func vtabCallbackIIII(ctx context.Context, mod api.Module, pVTab, _, _, _ uint32) uint32 { - return uint32(INTERNAL) +func vtabFindFuncCallback(ctx context.Context, mod api.Module, pVTab, nArg, zName, pxFunc uint32) uint32 { + vtab := vtabGetHandle(ctx, mod, pVTab).(VTabOverloader) + fn, op := vtab.FindFunction(int(nArg), util.ReadString(mod, zName, _MAX_STRING)) + if fn != nil { + handle := util.AddHandle(ctx, fn) + util.WriteUint32(mod, pxFunc, handle) + if op == 0 { + op = 1 + } + } + return uint32(op) } func vtabIntegrityCallback(ctx context.Context, mod api.Module, pVTab, zSchema, zTabName, mFlags, pzErr uint32) uint32 { - return uint32(INTERNAL) + vtab := vtabGetHandle(ctx, mod, pVTab).(VTabChecker) + schema := util.ReadString(mod, zSchema, _MAX_STRING) + table := util.ReadString(mod, zTabName, _MAX_STRING) + err := vtab.Integrity(schema, table, int(mFlags)) + // xIntegrity should return OK - even if it finds problems in the content of the virtual table. + // https://sqlite.org/vtab.html#xintegrity + vtabError(ctx, mod, pzErr, _PTR_ERROR, err) + return errorCode(err, _OK) +} + +func vtabBeginCallback(ctx context.Context, mod api.Module, pVTab uint32) uint32 { + vtab := vtabGetHandle(ctx, mod, pVTab).(VTabTx) + err := vtab.Begin() + return vtabError(ctx, mod, pVTab, _VTAB_ERROR, err) +} + +func vtabSyncCallback(ctx context.Context, mod api.Module, pVTab uint32) uint32 { + vtab := vtabGetHandle(ctx, mod, pVTab).(VTabTx) + err := vtab.Sync() + return vtabError(ctx, mod, pVTab, _VTAB_ERROR, err) +} + +func vtabCommitCallback(ctx context.Context, mod api.Module, pVTab uint32) uint32 { + vtab := vtabGetHandle(ctx, mod, pVTab).(VTabTx) + err := vtab.Commit() + return vtabError(ctx, mod, pVTab, _VTAB_ERROR, err) +} + +func vtabRollbackCallback(ctx context.Context, mod api.Module, pVTab uint32) uint32 { + vtab := vtabGetHandle(ctx, mod, pVTab).(VTabTx) + err := vtab.Rollback() + return vtabError(ctx, mod, pVTab, _VTAB_ERROR, err) +} + +func vtabSavepointCallback(ctx context.Context, mod api.Module, pVTab, id uint32) uint32 { + vtab := vtabGetHandle(ctx, mod, pVTab).(VTabSavepointer) + err := vtab.Savepoint(int(id)) + return vtabError(ctx, mod, pVTab, _VTAB_ERROR, err) +} + +func vtabReleaseCallback(ctx context.Context, mod api.Module, pVTab, id uint32) uint32 { + vtab := vtabGetHandle(ctx, mod, pVTab).(VTabSavepointer) + err := vtab.Release(int(id)) + return vtabError(ctx, mod, pVTab, _VTAB_ERROR, err) +} + +func vtabRollbackToCallback(ctx context.Context, mod api.Module, pVTab, id uint32) uint32 { + vtab := vtabGetHandle(ctx, mod, pVTab).(VTabSavepointer) + err := vtab.RollbackTo(int(id)) + return vtabError(ctx, mod, pVTab, _VTAB_ERROR, err) } func cursorOpenCallback(ctx context.Context, mod api.Module, pVTab, ppCur uint32) uint32 { @@ -389,9 +465,8 @@ func cursorOpenCallback(ctx context.Context, mod api.Module, pVTab, ppCur uint32 } func cursorCloseCallback(ctx context.Context, mod api.Module, pCur uint32) uint32 { - cursor := vtabGetHandle(ctx, mod, pCur).(VTabCursor) - err := cursor.Close() - return vtabError(ctx, mod, pCur, _CURSOR_ERROR, err) + err := vtabDelHandle(ctx, mod, pCur) + return errorCode(err, ERROR) } func cursorFilterCallback(ctx context.Context, mod api.Module, pCur, idxNum, idxStr, argc, argv uint32) uint32 { @@ -435,7 +510,7 @@ func cursorRowIDCallback(ctx context.Context, mod api.Module, pCur, pRowID uint3 } const ( - _MODULE_ERROR = iota + _PTR_ERROR = iota _VTAB_ERROR _CURSOR_ERROR ) @@ -461,6 +536,12 @@ func vtabGetHandle(ctx context.Context, mod api.Module, ptr uint32) any { return util.GetHandle(ctx, handle) } +func vtabDelHandle(ctx context.Context, mod api.Module, ptr uint32) error { + const handleOffset = 4 + handle := util.ReadUint32(mod, ptr-handleOffset) + return util.DelHandle(ctx, handle) +} + func vtabPutHandle(ctx context.Context, mod api.Module, pptr uint32, val any) { const handleOffset = 4 handle := util.AddHandle(ctx, val) diff --git a/vtab_test.go b/vtab_test.go index 848102e..0404934 100644 --- a/vtab_test.go +++ b/vtab_test.go @@ -83,10 +83,6 @@ type seriesCursor struct { value int64 } -func (*seriesCursor) Close() error { - return nil -} - func (cur *seriesCursor) Filter(idxNum int, idxStr string, arg ...sqlite3.Value) error { switch len(arg) { case 0: