diff --git a/.gitignore b/.gitignore index 722019c..58b5f43 100644 --- a/.gitignore +++ b/.gitignore @@ -16,5 +16,4 @@ tools # Project -demo.db sqlite3/sqlite3* \ No newline at end of file diff --git a/README.md b/README.md index ab486d5..048a93a 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,6 @@ [![Go Report](https://goreportcard.com/badge/github.com/ncruces/go-sqlite3)](https://goreportcard.com/report/github.com/ncruces/go-sqlite3) [![Go Coverage](https://github.com/ncruces/go-sqlite3/wiki/coverage.svg)](https://raw.githack.com/wiki/ncruces/go-sqlite3/coverage.html) -### ⚠️ Work in Progress ⚠️ - Go module `github.com/ncruces/go-sqlite3` wraps a [WASM](https://webassembly.org/) build of [SQLite](https://sqlite.org/), and uses [wazero](https://wazero.io/) to provide `cgo`-free SQLite bindings. @@ -20,6 +18,8 @@ embeds a build of SQLite into your application. ### Caveats +#### Write-Ahead Logging + Because WASM does not support shared memory, [WAL](https://www.sqlite.org/wal.html) support is [limited](https://www.sqlite.org/wal.html#noshm). @@ -33,6 +33,16 @@ Because connection pooling is incompatible with `EXCLUSIVE` locking mode, the `database/sql` driver defaults to `NORMAL` locking mode, and WAL databases are not supported. +#### Open File Description Locks + +On Unix, this module uses [OFD locks](https://www.gnu.org/software/libc/manual/html_node/Open-File-Description-Locks.html) +to synchronize access to database files. + +POSIX advisory locks, which SQLite uses, are [broken by design](https://www.sqlite.org/src/artifact/90c4fa?ln=1073-1161). +OFD locks are fully compatible with process-associated POSIX advisory locks, +and are supported on Linux, macOS and illumos. +As a work around for other Unixes, you can use [`nolock=1`](https://www.sqlite.org/uri.html). + ### Roadmap - [x] build SQLite using `zig cc --target=wasm32-wasi` @@ -50,6 +60,7 @@ and WAL databases are not supported. - [ ] session extension - [ ] resumable bulk update - [ ] shared cache mode + - [ ] unlock-notify - [ ] custom SQL functions - [ ] custom VFSes - [ ] read-only VFS, wrapping an [`io.ReaderAt`](https://pkg.go.dev/io#ReaderAt) diff --git a/api.go b/api.go index 1fbbbf9..987e8fa 100644 --- a/api.go +++ b/api.go @@ -17,7 +17,7 @@ func newConn(ctx context.Context, module api.Module) (_ *Conn, err error) { return f } - getPtr := func(name string) uint32 { + getVal := func(name string) uint32 { global := module.ExportedGlobal(name) if global == nil { err = noGlobalErr + errorString(name) @@ -32,7 +32,7 @@ func newConn(ctx context.Context, module api.Module) (_ *Conn, err error) { api: sqliteAPI{ free: getFun("free"), malloc: getFun("malloc"), - destructor: uint64(getPtr("malloc_destructor")), + destructor: uint64(getVal("malloc_destructor")), errcode: getFun("sqlite3_errcode"), errstr: getFun("sqlite3_errstr"), errmsg: getFun("sqlite3_errmsg"), @@ -65,13 +65,13 @@ func newConn(ctx context.Context, module api.Module) (_ *Conn, err error) { autocommit: getFun("sqlite3_get_autocommit"), lastRowid: getFun("sqlite3_last_insert_rowid"), changes: getFun("sqlite3_changes64"), - interrupt: getFun("sqlite3_interrupt"), blobOpen: getFun("sqlite3_blob_open"), blobClose: getFun("sqlite3_blob_close"), blobReopen: getFun("sqlite3_blob_reopen"), blobBytes: getFun("sqlite3_blob_bytes"), blobRead: getFun("sqlite3_blob_read"), blobWrite: getFun("sqlite3_blob_write"), + interrupt: getVal("sqlite3_interrupt_offset"), }, } if err != nil { @@ -116,11 +116,11 @@ type sqliteAPI struct { autocommit api.Function lastRowid api.Function changes api.Function - interrupt api.Function blobOpen api.Function blobClose api.Function blobReopen api.Function blobBytes api.Function blobRead api.Function blobWrite api.Function + interrupt uint32 } diff --git a/conn.go b/conn.go index 8f5f0bb..1e16d5b 100644 --- a/conn.go +++ b/conn.go @@ -7,7 +7,8 @@ import ( "math" "net/url" "strings" - "sync" + "sync/atomic" + "unsafe" "github.com/tetratelabs/wazero/api" ) @@ -23,7 +24,6 @@ type Conn struct { handle uint32 arena arena - mtx sync.Mutex interrupt context.Context waiter chan struct{} pending *Stmt @@ -263,8 +263,8 @@ func (c *Conn) SetInterrupt(ctx context.Context) (old context.Context) { break case <-ctx.Done(): // Done was closed. - - c.sendInterrupt() + buf := c.mem.view(c.handle+c.api.interrupt, 4) + (*atomic.Uint32)(unsafe.Pointer(&buf[0])).Store(1) // Wait for the next call to SetInterrupt. <-waiter } @@ -279,18 +279,11 @@ func (c *Conn) checkInterrupt() bool { if c.interrupt == nil || c.interrupt.Err() == nil { return false } - c.sendInterrupt() + buf := c.mem.view(c.handle+c.api.interrupt, 4) + (*atomic.Uint32)(unsafe.Pointer(&buf[0])).Store(1) return true } -func (c *Conn) sendInterrupt() { - c.mtx.Lock() - defer c.mtx.Unlock() - // This is safe to call from a goroutine - // because it doesn't touch the C stack. - c.call(c.api.interrupt, uint64(c.handle)) -} - // Pragma executes a PRAGMA statement and returns any results. // // https://www.sqlite.org/pragma.html diff --git a/embed/build.sh b/embed/build.sh index e5f1c00..02a9c38 100755 --- a/embed/build.sh +++ b/embed/build.sh @@ -8,7 +8,7 @@ cd -P -- "$(dirname -- "$0")" # build SQLite zig cc --target=wasm32-wasi -flto -g0 -Os \ - -o sqlite3.wasm ../sqlite3/*.c \ + -o sqlite3.wasm ../sqlite3/amalg.c \ -mmutable-globals \ -mbulk-memory -mreference-types \ -mnontrapping-fptoint -msign-ext \ @@ -54,4 +54,10 @@ zig cc --target=wasm32-wasi -flto -g0 -Os \ -Wl,--export=sqlite3_get_autocommit \ -Wl,--export=sqlite3_last_insert_rowid \ -Wl,--export=sqlite3_changes64 \ - -Wl,--export=sqlite3_interrupt \ \ No newline at end of file + -Wl,--export=sqlite3_unlock_notify \ + -Wl,--export=sqlite3_backup_init \ + -Wl,--export=sqlite3_backup_step \ + -Wl,--export=sqlite3_backup_finish \ + -Wl,--export=sqlite3_backup_remaining \ + -Wl,--export=sqlite3_backup_pagecount \ + -Wl,--export=sqlite3_interrupt_offset \ \ No newline at end of file diff --git a/embed/sqlite3.wasm b/embed/sqlite3.wasm index 8bed1fc..800b058 100755 Binary files a/embed/sqlite3.wasm and b/embed/sqlite3.wasm differ diff --git a/sqlite3/amalg.c b/sqlite3/amalg.c new file mode 100644 index 0000000..96f1630 --- /dev/null +++ b/sqlite3/amalg.c @@ -0,0 +1,9 @@ +#include + +#include "main.c" +#include "os.c" +#include "qsort.c" +#include "sqlite3.c" + +sqlite3_destructor_type malloc_destructor = &free; +size_t sqlite3_interrupt_offset = offsetof(sqlite3, u1.isInterrupted); \ No newline at end of file diff --git a/sqlite3/format.sh b/sqlite3/format.sh new file mode 100755 index 0000000..6b3a857 --- /dev/null +++ b/sqlite3/format.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +cd -P -- "$(dirname -- "$0")" + +clang-format -i \ + main.c \ + os.c \ + qsort.c \ + amalg.c \ No newline at end of file diff --git a/sqlite3/main.c b/sqlite3/main.c new file mode 100644 index 0000000..7bdb885 --- /dev/null +++ b/sqlite3/main.c @@ -0,0 +1,14 @@ +#include + +#include "sqlite3.h" + +int main() { + int rc = sqlite3_initialize(); + if (rc != SQLITE_OK) return 1; +} + +sqlite3_vfs *os_vfs(); + +int sqlite3_os_init() { + return sqlite3_vfs_register(os_vfs(), /*default=*/true); +} diff --git a/sqlite3/os.c b/sqlite3/os.c index 4f35552..9161be6 100644 --- a/sqlite3/os.c +++ b/sqlite3/os.c @@ -1,44 +1,37 @@ -#include -#include #include #include "sqlite3.h" -int main() { - int rc = sqlite3_initialize(); - if (rc != SQLITE_OK) return 1; -} +int os_localtime(sqlite3_int64, struct tm *); -int go_localtime(sqlite3_int64, struct tm *); +int os_randomness(sqlite3_vfs *, int nByte, char *zOut); +int os_sleep(sqlite3_vfs *, int microseconds); +int os_current_time(sqlite3_vfs *, double *); +int os_current_time_64(sqlite3_vfs *, sqlite3_int64 *); -int go_randomness(sqlite3_vfs *, int nByte, char *zOut); -int go_sleep(sqlite3_vfs *, int microseconds); -int go_current_time(sqlite3_vfs *, double *); -int go_current_time_64(sqlite3_vfs *, sqlite3_int64 *); - -int go_open(sqlite3_vfs *, sqlite3_filename zName, sqlite3_file *, int flags, +int os_open(sqlite3_vfs *, sqlite3_filename zName, sqlite3_file *, int flags, int *pOutFlags); -int go_delete(sqlite3_vfs *, const char *zName, int syncDir); -int go_access(sqlite3_vfs *, const char *zName, int flags, int *pResOut); -int go_full_pathname(sqlite3_vfs *, const char *zName, int nOut, char *zOut); +int os_delete(sqlite3_vfs *, const char *zName, int syncDir); +int os_access(sqlite3_vfs *, const char *zName, int flags, int *pResOut); +int os_full_pathname(sqlite3_vfs *, const char *zName, int nOut, char *zOut); -struct go_file { +struct os_file { sqlite3_file base; int id; - int eLock; + int lock; }; -int go_close(sqlite3_file *); -int go_read(sqlite3_file *, void *, int iAmt, sqlite3_int64 iOfst); -int go_write(sqlite3_file *, const void *, int iAmt, sqlite3_int64 iOfst); -int go_truncate(sqlite3_file *, sqlite3_int64 size); -int go_sync(sqlite3_file *, int flags); -int go_file_size(sqlite3_file *, sqlite3_int64 *pSize); -int go_file_control(sqlite3_file *pFile, int op, void *pArg); +int os_close(sqlite3_file *); +int os_read(sqlite3_file *, void *, int iAmt, sqlite3_int64 iOfst); +int os_write(sqlite3_file *, const void *, int iAmt, sqlite3_int64 iOfst); +int os_truncate(sqlite3_file *, sqlite3_int64 size); +int os_sync(sqlite3_file *, int flags); +int os_file_size(sqlite3_file *, sqlite3_int64 *pSize); +int os_file_control(sqlite3_file *pFile, int op, void *pArg); -int go_lock(sqlite3_file *pFile, int eLock); -int go_unlock(sqlite3_file *pFile, int eLock); -int go_check_reserved_lock(sqlite3_file *pFile, int *pResOut); +int os_lock(sqlite3_file *pFile, int eLock); +int os_unlock(sqlite3_file *pFile, int eLock); +int os_check_reserved_lock(sqlite3_file *pFile, int *pResOut); static int no_lock(sqlite3_file *pFile, int eLock) { return SQLITE_OK; } static int no_unlock(sqlite3_file *pFile, int eLock) { return SQLITE_OK; } @@ -54,48 +47,46 @@ static int no_sector_size(sqlite3_file *pFile) { return 0; } static int no_device_characteristics(sqlite3_file *pFile) { return 0; } int localtime_s(struct tm *const pTm, time_t const *const pTime) { - return go_localtime((sqlite3_int64)*pTime, pTm); + return os_localtime((sqlite3_int64)*pTime, pTm); } -static int go_open_c(sqlite3_vfs *vfs, sqlite3_filename zName, +static int os_open_w(sqlite3_vfs *vfs, sqlite3_filename zName, sqlite3_file *file, int flags, int *pOutFlags) { - static const sqlite3_io_methods go_io = { + static const sqlite3_io_methods os_io = { .iVersion = 1, - .xClose = go_close, - .xRead = go_read, - .xWrite = go_write, - .xTruncate = go_truncate, - .xSync = go_sync, - .xFileSize = go_file_size, - .xLock = go_lock, - .xUnlock = go_unlock, - .xCheckReservedLock = go_check_reserved_lock, + .xClose = os_close, + .xRead = os_read, + .xWrite = os_write, + .xTruncate = os_truncate, + .xSync = os_sync, + .xFileSize = os_file_size, + .xLock = os_lock, + .xUnlock = os_unlock, + .xCheckReservedLock = os_check_reserved_lock, .xFileControl = no_file_control, .xDeviceCharacteristics = no_device_characteristics, }; - int rc = go_open(vfs, zName, file, flags, pOutFlags); - file->pMethods = (char)rc == SQLITE_OK ? &go_io : NULL; + int rc = os_open(vfs, zName, file, flags, pOutFlags); + file->pMethods = (char)rc == SQLITE_OK ? &os_io : NULL; return rc; } -int sqlite3_os_init() { - static sqlite3_vfs go_vfs = { +sqlite3_vfs *os_vfs() { + static sqlite3_vfs os_vfs = { .iVersion = 2, - .szOsFile = sizeof(struct go_file), + .szOsFile = sizeof(struct os_file), .mxPathname = 512, - .zName = "go", + .zName = "os", - .xOpen = go_open_c, - .xDelete = go_delete, - .xAccess = go_access, - .xFullPathname = go_full_pathname, + .xOpen = os_open_w, + .xDelete = os_delete, + .xAccess = os_access, + .xFullPathname = os_full_pathname, - .xRandomness = go_randomness, - .xSleep = go_sleep, - .xCurrentTime = go_current_time, - .xCurrentTimeInt64 = go_current_time_64, + .xRandomness = os_randomness, + .xSleep = os_sleep, + .xCurrentTime = os_current_time, + .xCurrentTimeInt64 = os_current_time_64, }; - return sqlite3_vfs_register(&go_vfs, /*default=*/true); + return &os_vfs; } - -sqlite3_destructor_type malloc_destructor = &free; \ No newline at end of file diff --git a/tests/parallel/parallel_test.go b/tests/parallel/parallel_test.go index 24a102f..5ad9129 100644 --- a/tests/parallel/parallel_test.go +++ b/tests/parallel/parallel_test.go @@ -21,7 +21,7 @@ func TestParallel(t *testing.T) { func TestMultiProcess(t *testing.T) { if testing.Short() { - t.Skip() + t.Skip("skipping in short mode") } name := filepath.Join(t.TempDir(), "test.db") diff --git a/vfs.go b/vfs.go index 46f0562..02368ba 100644 --- a/vfs.go +++ b/vfs.go @@ -27,25 +27,25 @@ func vfsInstantiate(ctx context.Context, r wazero.Runtime) { } env := r.NewHostModuleBuilder("env") - env.NewFunctionBuilder().WithFunc(vfsLocaltime).Export("go_localtime") - env.NewFunctionBuilder().WithFunc(vfsRandomness).Export("go_randomness") - env.NewFunctionBuilder().WithFunc(vfsSleep).Export("go_sleep") - env.NewFunctionBuilder().WithFunc(vfsCurrentTime).Export("go_current_time") - env.NewFunctionBuilder().WithFunc(vfsCurrentTime64).Export("go_current_time_64") - env.NewFunctionBuilder().WithFunc(vfsFullPathname).Export("go_full_pathname") - env.NewFunctionBuilder().WithFunc(vfsDelete).Export("go_delete") - env.NewFunctionBuilder().WithFunc(vfsAccess).Export("go_access") - env.NewFunctionBuilder().WithFunc(vfsOpen).Export("go_open") - env.NewFunctionBuilder().WithFunc(vfsClose).Export("go_close") - env.NewFunctionBuilder().WithFunc(vfsRead).Export("go_read") - env.NewFunctionBuilder().WithFunc(vfsWrite).Export("go_write") - env.NewFunctionBuilder().WithFunc(vfsTruncate).Export("go_truncate") - env.NewFunctionBuilder().WithFunc(vfsSync).Export("go_sync") - env.NewFunctionBuilder().WithFunc(vfsFileSize).Export("go_file_size") - env.NewFunctionBuilder().WithFunc(vfsLock).Export("go_lock") - env.NewFunctionBuilder().WithFunc(vfsUnlock).Export("go_unlock") - env.NewFunctionBuilder().WithFunc(vfsCheckReservedLock).Export("go_check_reserved_lock") - env.NewFunctionBuilder().WithFunc(vfsFileControl).Export("go_file_control") + env.NewFunctionBuilder().WithFunc(vfsLocaltime).Export("os_localtime") + env.NewFunctionBuilder().WithFunc(vfsRandomness).Export("os_randomness") + env.NewFunctionBuilder().WithFunc(vfsSleep).Export("os_sleep") + env.NewFunctionBuilder().WithFunc(vfsCurrentTime).Export("os_current_time") + env.NewFunctionBuilder().WithFunc(vfsCurrentTime64).Export("os_current_time_64") + env.NewFunctionBuilder().WithFunc(vfsFullPathname).Export("os_full_pathname") + env.NewFunctionBuilder().WithFunc(vfsDelete).Export("os_delete") + env.NewFunctionBuilder().WithFunc(vfsAccess).Export("os_access") + env.NewFunctionBuilder().WithFunc(vfsOpen).Export("os_open") + env.NewFunctionBuilder().WithFunc(vfsClose).Export("os_close") + env.NewFunctionBuilder().WithFunc(vfsRead).Export("os_read") + env.NewFunctionBuilder().WithFunc(vfsWrite).Export("os_write") + env.NewFunctionBuilder().WithFunc(vfsTruncate).Export("os_truncate") + env.NewFunctionBuilder().WithFunc(vfsSync).Export("os_sync") + env.NewFunctionBuilder().WithFunc(vfsFileSize).Export("os_file_size") + env.NewFunctionBuilder().WithFunc(vfsLock).Export("os_lock") + env.NewFunctionBuilder().WithFunc(vfsUnlock).Export("os_unlock") + env.NewFunctionBuilder().WithFunc(vfsCheckReservedLock).Export("os_check_reserved_lock") + env.NewFunctionBuilder().WithFunc(vfsFileControl).Export("os_file_control") _, err = env.Instantiate(ctx) if err != nil { panic(err) diff --git a/vfs_lock_test.go b/vfs_lock_test.go index cdf467c..631d62d 100644 --- a/vfs_lock_test.go +++ b/vfs_lock_test.go @@ -9,12 +9,11 @@ import ( ) func Test_vfsLock(t *testing.T) { - // Other OSes lack open file descriptors locks. switch runtime.GOOS { case "linux", "darwin", "illumos", "windows": break default: - t.Skip() + t.Skip("OS lacks OFD locks") } name := filepath.Join(t.TempDir(), "test.db")