Refactor native code.

2026-01-12 05:59:14 +00:00 · 2023-03-01 12:16:36 +00:00
parent 35265271aa
commit dbc400eb15
13 changed files with 131 additions and 101 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -16,5 +16,4 @@
 tools

 # Project
-demo.db
 sqlite3/sqlite3*
--- a/README.md
+++ b/README.md
@@ -4,8 +4,6 @@
 [![Go Report](https://goreportcard.com/badge/github.com/ncruces/go-sqlite3)](https://goreportcard.com/report/github.com/ncruces/go-sqlite3)
 [![Go Coverage](https://github.com/ncruces/go-sqlite3/wiki/coverage.svg)](https://raw.githack.com/wiki/ncruces/go-sqlite3/coverage.html)

-### ⚠️ Work in Progress ⚠️
-
 Go module `github.com/ncruces/go-sqlite3` wraps a [WASM](https://webassembly.org/) build of [SQLite](https://sqlite.org/),
 and uses [wazero](https://wazero.io/) to provide `cgo`-free SQLite bindings.

@@ -20,6 +18,8 @@ embeds a build of SQLite into your application.

 ### Caveats

+#### Write-Ahead Logging
+
 Because WASM does not support shared memory,
 [WAL](https://www.sqlite.org/wal.html) support is [limited](https://www.sqlite.org/wal.html#noshm).

@@ -33,6 +33,16 @@ Because connection pooling is incompatible with `EXCLUSIVE` locking mode,
 the `database/sql` driver defaults to `NORMAL` locking mode,
 and WAL databases are not supported.

+#### Open File Description Locks
+
+On Unix, this module uses [OFD locks](https://www.gnu.org/software/libc/manual/html_node/Open-File-Description-Locks.html)
+to synchronize access to database files.
+
+POSIX advisory locks, which SQLite uses, are [broken by design](https://www.sqlite.org/src/artifact/90c4fa?ln=1073-1161).
+OFD locks are fully compatible with process-associated POSIX advisory locks,
+and are supported on Linux, macOS and illumos.
+As a work around for other Unixes, you can use [`nolock=1`](https://www.sqlite.org/uri.html).
+
 ### Roadmap

 - [x] build SQLite using `zig cc --target=wasm32-wasi`
@@ -50,6 +60,7 @@ and WAL databases are not supported.
  - [ ] session extension
  - [ ] resumable bulk update
  - [ ] shared cache mode
+  - [ ] unlock-notify
 - [ ] custom SQL functions
 - [ ] custom VFSes
  - [ ] read-only VFS, wrapping an [`io.ReaderAt`](https://pkg.go.dev/io#ReaderAt)
--- a/api.go
+++ b/api.go
@@ -17,7 +17,7 @@ func newConn(ctx context.Context, module api.Module) (_ *Conn, err error) {
 		return f
 	}

-	getPtr := func(name string) uint32 {
+	getVal := func(name string) uint32 {
 		global := module.ExportedGlobal(name)
 		if global == nil {
 			err = noGlobalErr + errorString(name)
@@ -32,7 +32,7 @@ func newConn(ctx context.Context, module api.Module) (_ *Conn, err error) {
 		api: sqliteAPI{
 			free:          getFun("free"),
 			malloc:        getFun("malloc"),
-			destructor:    uint64(getPtr("malloc_destructor")),
+			destructor:    uint64(getVal("malloc_destructor")),
 			errcode:       getFun("sqlite3_errcode"),
 			errstr:        getFun("sqlite3_errstr"),
 			errmsg:        getFun("sqlite3_errmsg"),
@@ -65,13 +65,13 @@ func newConn(ctx context.Context, module api.Module) (_ *Conn, err error) {
 			autocommit:    getFun("sqlite3_get_autocommit"),
 			lastRowid:     getFun("sqlite3_last_insert_rowid"),
 			changes:       getFun("sqlite3_changes64"),
-			interrupt:     getFun("sqlite3_interrupt"),
 			blobOpen:      getFun("sqlite3_blob_open"),
 			blobClose:     getFun("sqlite3_blob_close"),
 			blobReopen:    getFun("sqlite3_blob_reopen"),
 			blobBytes:     getFun("sqlite3_blob_bytes"),
 			blobRead:      getFun("sqlite3_blob_read"),
 			blobWrite:     getFun("sqlite3_blob_write"),
+			interrupt:     getVal("sqlite3_interrupt_offset"),
 		},
 	}
 	if err != nil {
@@ -116,11 +116,11 @@ type sqliteAPI struct {
 	autocommit    api.Function
 	lastRowid     api.Function
 	changes       api.Function
-	interrupt     api.Function
 	blobOpen      api.Function
 	blobClose     api.Function
 	blobReopen    api.Function
 	blobBytes     api.Function
 	blobRead      api.Function
 	blobWrite     api.Function
+	interrupt     uint32
 }
--- a/conn.go
+++ b/conn.go
@@ -7,7 +7,8 @@ import (
 	"math"
 	"net/url"
 	"strings"
-	"sync"
+	"sync/atomic"
+	"unsafe"

 	"github.com/tetratelabs/wazero/api"
 )
@@ -23,7 +24,6 @@ type Conn struct {
 	handle uint32

 	arena     arena
-	mtx       sync.Mutex
 	interrupt context.Context
 	waiter    chan struct{}
 	pending   *Stmt
@@ -263,8 +263,8 @@ func (c *Conn) SetInterrupt(ctx context.Context) (old context.Context) {
 			break

 		case <-ctx.Done(): // Done was closed.
-
-			c.sendInterrupt()
+			buf := c.mem.view(c.handle+c.api.interrupt, 4)
+			(*atomic.Uint32)(unsafe.Pointer(&buf[0])).Store(1)
 			// Wait for the next call to SetInterrupt.
 			<-waiter
 		}
@@ -279,18 +279,11 @@ func (c *Conn) checkInterrupt() bool {
 	if c.interrupt == nil || c.interrupt.Err() == nil {
 		return false
 	}
-	c.sendInterrupt()
+	buf := c.mem.view(c.handle+c.api.interrupt, 4)
+	(*atomic.Uint32)(unsafe.Pointer(&buf[0])).Store(1)
 	return true
 }

-func (c *Conn) sendInterrupt() {
-	c.mtx.Lock()
-	defer c.mtx.Unlock()
-	// This is safe to call from a goroutine
-	// because it doesn't touch the C stack.
-	c.call(c.api.interrupt, uint64(c.handle))
-}
-
 // Pragma executes a PRAGMA statement and returns any results.
 //
 // https://www.sqlite.org/pragma.html
--- a/embed/build.sh
+++ b/embed/build.sh
@@ -8,7 +8,7 @@ cd -P -- "$(dirname -- "$0")"

 # build SQLite
 zig cc --target=wasm32-wasi -flto -g0 -Os \
-  -o sqlite3.wasm ../sqlite3/*.c \
+  -o sqlite3.wasm ../sqlite3/amalg.c \
 	-mmutable-globals \
 	-mbulk-memory -mreference-types \
 	-mnontrapping-fptoint -msign-ext \
@@ -54,4 +54,10 @@ zig cc --target=wasm32-wasi -flto -g0 -Os \
 	-Wl,--export=sqlite3_get_autocommit \
 	-Wl,--export=sqlite3_last_insert_rowid \
 	-Wl,--export=sqlite3_changes64 \
-	-Wl,--export=sqlite3_interrupt \
+	-Wl,--export=sqlite3_unlock_notify \
+	-Wl,--export=sqlite3_backup_init \
+	-Wl,--export=sqlite3_backup_step \
+	-Wl,--export=sqlite3_backup_finish \
+	-Wl,--export=sqlite3_backup_remaining \
+	-Wl,--export=sqlite3_backup_pagecount \
+	-Wl,--export=sqlite3_interrupt_offset \
--- a/embed/sqlite3.wasm
+++ b/embed/sqlite3.wasm
--- a/sqlite3/amalg.c
+++ b/sqlite3/amalg.c
@@ -0,0 +1,9 @@
+#include <stddef.h>
+
+#include "main.c"
+#include "os.c"
+#include "qsort.c"
+#include "sqlite3.c"
+
+sqlite3_destructor_type malloc_destructor = &free;
+size_t sqlite3_interrupt_offset = offsetof(sqlite3, u1.isInterrupted);
--- a/sqlite3/format.sh
+++ b/sqlite3/format.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+cd -P -- "$(dirname -- "$0")"
+
+clang-format -i \
+	main.c \
+	os.c \
+	qsort.c \
+	amalg.c
--- a/sqlite3/main.c
+++ b/sqlite3/main.c
@@ -0,0 +1,14 @@
+#include <stdbool.h>
+
+#include "sqlite3.h"
+
+int main() {
+  int rc = sqlite3_initialize();
+  if (rc != SQLITE_OK) return 1;
+}
+
+sqlite3_vfs *os_vfs();
+
+int sqlite3_os_init() {
+  return sqlite3_vfs_register(os_vfs(), /*default=*/true);
+}
--- a/sqlite3/os.c
+++ b/sqlite3/os.c
@@ -1,44 +1,37 @@
-#include <stdbool.h>
-#include <stdlib.h>
 #include <time.h>

 #include "sqlite3.h"

-int main() {
-  int rc = sqlite3_initialize();
-  if (rc != SQLITE_OK) return 1;
-}
+int os_localtime(sqlite3_int64, struct tm *);

-int go_localtime(sqlite3_int64, struct tm *);
+int os_randomness(sqlite3_vfs *, int nByte, char *zOut);
+int os_sleep(sqlite3_vfs *, int microseconds);
+int os_current_time(sqlite3_vfs *, double *);
+int os_current_time_64(sqlite3_vfs *, sqlite3_int64 *);

-int go_randomness(sqlite3_vfs *, int nByte, char *zOut);
-int go_sleep(sqlite3_vfs *, int microseconds);
-int go_current_time(sqlite3_vfs *, double *);
-int go_current_time_64(sqlite3_vfs *, sqlite3_int64 *);
-
-int go_open(sqlite3_vfs *, sqlite3_filename zName, sqlite3_file *, int flags,
+int os_open(sqlite3_vfs *, sqlite3_filename zName, sqlite3_file *, int flags,
            int *pOutFlags);
-int go_delete(sqlite3_vfs *, const char *zName, int syncDir);
-int go_access(sqlite3_vfs *, const char *zName, int flags, int *pResOut);
-int go_full_pathname(sqlite3_vfs *, const char *zName, int nOut, char *zOut);
+int os_delete(sqlite3_vfs *, const char *zName, int syncDir);
+int os_access(sqlite3_vfs *, const char *zName, int flags, int *pResOut);
+int os_full_pathname(sqlite3_vfs *, const char *zName, int nOut, char *zOut);

-struct go_file {
+struct os_file {
  sqlite3_file base;
  int id;
-  int eLock;
+  int lock;
 };

-int go_close(sqlite3_file *);
-int go_read(sqlite3_file *, void *, int iAmt, sqlite3_int64 iOfst);
-int go_write(sqlite3_file *, const void *, int iAmt, sqlite3_int64 iOfst);
-int go_truncate(sqlite3_file *, sqlite3_int64 size);
-int go_sync(sqlite3_file *, int flags);
-int go_file_size(sqlite3_file *, sqlite3_int64 *pSize);
-int go_file_control(sqlite3_file *pFile, int op, void *pArg);
+int os_close(sqlite3_file *);
+int os_read(sqlite3_file *, void *, int iAmt, sqlite3_int64 iOfst);
+int os_write(sqlite3_file *, const void *, int iAmt, sqlite3_int64 iOfst);
+int os_truncate(sqlite3_file *, sqlite3_int64 size);
+int os_sync(sqlite3_file *, int flags);
+int os_file_size(sqlite3_file *, sqlite3_int64 *pSize);
+int os_file_control(sqlite3_file *pFile, int op, void *pArg);

-int go_lock(sqlite3_file *pFile, int eLock);
-int go_unlock(sqlite3_file *pFile, int eLock);
-int go_check_reserved_lock(sqlite3_file *pFile, int *pResOut);
+int os_lock(sqlite3_file *pFile, int eLock);
+int os_unlock(sqlite3_file *pFile, int eLock);
+int os_check_reserved_lock(sqlite3_file *pFile, int *pResOut);

 static int no_lock(sqlite3_file *pFile, int eLock) { return SQLITE_OK; }
 static int no_unlock(sqlite3_file *pFile, int eLock) { return SQLITE_OK; }
@@ -54,48 +47,46 @@ static int no_sector_size(sqlite3_file *pFile) { return 0; }
 static int no_device_characteristics(sqlite3_file *pFile) { return 0; }

 int localtime_s(struct tm *const pTm, time_t const *const pTime) {
-  return go_localtime((sqlite3_int64)*pTime, pTm);
+  return os_localtime((sqlite3_int64)*pTime, pTm);
 }

-static int go_open_c(sqlite3_vfs *vfs, sqlite3_filename zName,
+static int os_open_w(sqlite3_vfs *vfs, sqlite3_filename zName,
                     sqlite3_file *file, int flags, int *pOutFlags) {
-  static const sqlite3_io_methods go_io = {
+  static const sqlite3_io_methods os_io = {
      .iVersion = 1,
-      .xClose = go_close,
-      .xRead = go_read,
-      .xWrite = go_write,
-      .xTruncate = go_truncate,
-      .xSync = go_sync,
-      .xFileSize = go_file_size,
-      .xLock = go_lock,
-      .xUnlock = go_unlock,
-      .xCheckReservedLock = go_check_reserved_lock,
+      .xClose = os_close,
+      .xRead = os_read,
+      .xWrite = os_write,
+      .xTruncate = os_truncate,
+      .xSync = os_sync,
+      .xFileSize = os_file_size,
+      .xLock = os_lock,
+      .xUnlock = os_unlock,
+      .xCheckReservedLock = os_check_reserved_lock,
      .xFileControl = no_file_control,
      .xDeviceCharacteristics = no_device_characteristics,
  };
-  int rc = go_open(vfs, zName, file, flags, pOutFlags);
-  file->pMethods = (char)rc == SQLITE_OK ? &go_io : NULL;
+  int rc = os_open(vfs, zName, file, flags, pOutFlags);
+  file->pMethods = (char)rc == SQLITE_OK ? &os_io : NULL;
  return rc;
 }

-int sqlite3_os_init() {
-  static sqlite3_vfs go_vfs = {
+sqlite3_vfs *os_vfs() {
+  static sqlite3_vfs os_vfs = {
      .iVersion = 2,
-      .szOsFile = sizeof(struct go_file),
+      .szOsFile = sizeof(struct os_file),
      .mxPathname = 512,
-      .zName = "go",
+      .zName = "os",

-      .xOpen = go_open_c,
-      .xDelete = go_delete,
-      .xAccess = go_access,
-      .xFullPathname = go_full_pathname,
+      .xOpen = os_open_w,
+      .xDelete = os_delete,
+      .xAccess = os_access,
+      .xFullPathname = os_full_pathname,

-      .xRandomness = go_randomness,
-      .xSleep = go_sleep,
-      .xCurrentTime = go_current_time,
-      .xCurrentTimeInt64 = go_current_time_64,
+      .xRandomness = os_randomness,
+      .xSleep = os_sleep,
+      .xCurrentTime = os_current_time,
+      .xCurrentTimeInt64 = os_current_time_64,
  };
-  return sqlite3_vfs_register(&go_vfs, /*default=*/true);
+  return &os_vfs;
 }
-
-sqlite3_destructor_type malloc_destructor = &free;
--- a/tests/parallel/parallel_test.go
+++ b/tests/parallel/parallel_test.go
@@ -21,7 +21,7 @@ func TestParallel(t *testing.T) {

 func TestMultiProcess(t *testing.T) {
 	if testing.Short() {
-		t.Skip()
+		t.Skip("skipping in short mode")
 	}

 	name := filepath.Join(t.TempDir(), "test.db")
--- a/vfs.go
+++ b/vfs.go
@@ -27,25 +27,25 @@ func vfsInstantiate(ctx context.Context, r wazero.Runtime) {
 	}

 	env := r.NewHostModuleBuilder("env")
-	env.NewFunctionBuilder().WithFunc(vfsLocaltime).Export("go_localtime")
-	env.NewFunctionBuilder().WithFunc(vfsRandomness).Export("go_randomness")
-	env.NewFunctionBuilder().WithFunc(vfsSleep).Export("go_sleep")
-	env.NewFunctionBuilder().WithFunc(vfsCurrentTime).Export("go_current_time")
-	env.NewFunctionBuilder().WithFunc(vfsCurrentTime64).Export("go_current_time_64")
-	env.NewFunctionBuilder().WithFunc(vfsFullPathname).Export("go_full_pathname")
-	env.NewFunctionBuilder().WithFunc(vfsDelete).Export("go_delete")
-	env.NewFunctionBuilder().WithFunc(vfsAccess).Export("go_access")
-	env.NewFunctionBuilder().WithFunc(vfsOpen).Export("go_open")
-	env.NewFunctionBuilder().WithFunc(vfsClose).Export("go_close")
-	env.NewFunctionBuilder().WithFunc(vfsRead).Export("go_read")
-	env.NewFunctionBuilder().WithFunc(vfsWrite).Export("go_write")
-	env.NewFunctionBuilder().WithFunc(vfsTruncate).Export("go_truncate")
-	env.NewFunctionBuilder().WithFunc(vfsSync).Export("go_sync")
-	env.NewFunctionBuilder().WithFunc(vfsFileSize).Export("go_file_size")
-	env.NewFunctionBuilder().WithFunc(vfsLock).Export("go_lock")
-	env.NewFunctionBuilder().WithFunc(vfsUnlock).Export("go_unlock")
-	env.NewFunctionBuilder().WithFunc(vfsCheckReservedLock).Export("go_check_reserved_lock")
-	env.NewFunctionBuilder().WithFunc(vfsFileControl).Export("go_file_control")
+	env.NewFunctionBuilder().WithFunc(vfsLocaltime).Export("os_localtime")
+	env.NewFunctionBuilder().WithFunc(vfsRandomness).Export("os_randomness")
+	env.NewFunctionBuilder().WithFunc(vfsSleep).Export("os_sleep")
+	env.NewFunctionBuilder().WithFunc(vfsCurrentTime).Export("os_current_time")
+	env.NewFunctionBuilder().WithFunc(vfsCurrentTime64).Export("os_current_time_64")
+	env.NewFunctionBuilder().WithFunc(vfsFullPathname).Export("os_full_pathname")
+	env.NewFunctionBuilder().WithFunc(vfsDelete).Export("os_delete")
+	env.NewFunctionBuilder().WithFunc(vfsAccess).Export("os_access")
+	env.NewFunctionBuilder().WithFunc(vfsOpen).Export("os_open")
+	env.NewFunctionBuilder().WithFunc(vfsClose).Export("os_close")
+	env.NewFunctionBuilder().WithFunc(vfsRead).Export("os_read")
+	env.NewFunctionBuilder().WithFunc(vfsWrite).Export("os_write")
+	env.NewFunctionBuilder().WithFunc(vfsTruncate).Export("os_truncate")
+	env.NewFunctionBuilder().WithFunc(vfsSync).Export("os_sync")
+	env.NewFunctionBuilder().WithFunc(vfsFileSize).Export("os_file_size")
+	env.NewFunctionBuilder().WithFunc(vfsLock).Export("os_lock")
+	env.NewFunctionBuilder().WithFunc(vfsUnlock).Export("os_unlock")
+	env.NewFunctionBuilder().WithFunc(vfsCheckReservedLock).Export("os_check_reserved_lock")
+	env.NewFunctionBuilder().WithFunc(vfsFileControl).Export("os_file_control")
 	_, err = env.Instantiate(ctx)
 	if err != nil {
 		panic(err)
--- a/vfs_lock_test.go
+++ b/vfs_lock_test.go
@@ -9,12 +9,11 @@ import (
 )

 func Test_vfsLock(t *testing.T) {
-	// Other OSes lack open file descriptors locks.
 	switch runtime.GOOS {
 	case "linux", "darwin", "illumos", "windows":
 		break
 	default:
-		t.Skip()
+		t.Skip("OS lacks OFD locks")
 	}

 	name := filepath.Join(t.TempDir(), "test.db")