Test ppc64le.

Windows and unix allocators would panic when asked to allocate the maximum size.

.github/workflows/repro.sh

@@ -3,13 +3,13 @@ set -euo pipefail
 
 if [[ "$OSTYPE" == "linux"* ]]; then
   WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/wasi-sdk-24.0-x86_64-linux.tar.gz"
-  BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_118/binaryen-version_118-x86_64-linux.tar.gz"
+  BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_119/binaryen-version_119-x86_64-linux.tar.gz"
 elif [[ "$OSTYPE" == "darwin"* ]]; then
   WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/wasi-sdk-24.0-x86_64-macos.tar.gz"
-  BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_118/binaryen-version_118-x86_64-macos.tar.gz"
+  BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_119/binaryen-version_119-x86_64-macos.tar.gz"
 elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then
   WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/wasi-sdk-24.0-x86_64-windows.tar.gz"
-  BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_118/binaryen-version_118-x86_64-windows.tar.gz"
+  BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_119/binaryen-version_119-x86_64-windows.tar.gz"
 fi
 
 # Download tools

.github/workflows/test.yml

@@ -153,6 +153,9 @@ jobs:
     - name: Test riscv64 (interpreter)
       run: GOARCH=riscv64 go test -v -short ./...
 
+    - name: Test ppc64le (interpreter)
+      run: GOARCH=ppc64le go test -v -short ./...
+
     - name: Test s390x (big-endian, z/OS demo)
       run: GOARCH=s390x go test -v -short -tags sqlite3_flock ./...
 

README.md

@@ -10,7 +10,7 @@ as well as direct access to most of the [C SQLite API](https://sqlite.org/cintro
 
 It wraps a [Wasm](https://webassembly.org/) [build](embed/) of SQLite,
 and uses [wazero](https://wazero.io/) as the runtime.\
-Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ runtime dependencies [^1].
+Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ runtime dependencies.
 
 ### Getting started
 
@@ -49,6 +49,8 @@ db.QueryRow(`SELECT sqlite_version()`).Scan(&version)
   simplifies [incremental BLOB I/O](https://sqlite.org/c3ref/blob_open.html).
 - [`github.com/ncruces/go-sqlite3/ext/bloom`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/bloom)
   provides a [Bloom filter](https://github.com/nalgeon/sqlean/issues/27#issuecomment-1002267134) virtual table.
+- [`github.com/ncruces/go-sqlite3/ext/closure`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/closure)
+  provides a transitive closure virtual table.
 - [`github.com/ncruces/go-sqlite3/ext/csv`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/csv)
   reads [comma-separated values](https://sqlite.org/csv.html).
 - [`github.com/ncruces/go-sqlite3/ext/fileio`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/fileio)
@@ -108,7 +110,7 @@ It also benefits greatly from [SQLite's](https://sqlite.org/testing.html) and
 [wazero's](https://tetrate.io/blog/introducing-wazero-from-tetrate/#:~:text=Rock%2Dsolid%20test%20approach) thorough testing.
 
 Every commit is [tested](https://github.com/ncruces/go-sqlite3/wiki/Test-matrix) on
-Linux (amd64/arm64/386/riscv64/s390x), macOS (amd64/arm64),
+Linux (amd64/arm64/386/riscv64/ppc64le/s390x), macOS (amd64/arm64),
 Windows (amd64), FreeBSD (amd64), OpenBSD (amd64), NetBSD (amd64),
 illumos (amd64), and Solaris (amd64).
 
@@ -128,7 +130,4 @@ The Wasm and VFS layers are also tested by running SQLite's
 - [`modernc.org/sqlite`](https://pkg.go.dev/modernc.org/sqlite)
 - [`crawshaw.io/sqlite`](https://pkg.go.dev/crawshaw.io/sqlite)
 - [`github.com/mattn/go-sqlite3`](https://pkg.go.dev/github.com/mattn/go-sqlite3)
-- [`github.com/zombiezen/go-sqlite`](https://pkg.go.dev/github.com/zombiezen/go-sqlite)
-
-[^1]: anything else you find in `go.mod` is either a test dependency,
-      or needed by one of the extensions.
+- [`github.com/zombiezen/go-sqlite`](https://pkg.go.dev/github.com/zombiezen/go-sqlite)

blob.go

@@ -21,6 +21,8 @@ type Blob struct {
 	bytes  int64
 	offset int64
 	handle uint32
+	bufptr uint32
+	buflen int64
 }
 
 var _ io.ReadWriteSeeker = &Blob{}
@@ -66,7 +68,7 @@ func (b *Blob) Close() error {
 	}
 
 	r := b.c.call("sqlite3_blob_close", uint64(b.handle))
-
+	b.c.free(b.bufptr)
 	b.handle = 0
 	return b.c.error(r)
 }
@@ -86,17 +88,18 @@ func (b *Blob) Read(p []byte) (n int, err error) {
 		return 0, io.EOF
 	}
 
-	avail := b.bytes - b.offset
 	want := int64(len(p))
+	avail := b.bytes - b.offset
 	if want > avail {
 		want = avail
 	}
-
-	defer b.c.arena.mark()()
-	ptr := b.c.arena.new(uint64(want))
+	if want > b.buflen {
+		b.bufptr = b.c.realloc(b.bufptr, uint64(want))
+		b.buflen = want
+	}
 
 	r := b.c.call("sqlite3_blob_read", uint64(b.handle),
-		uint64(ptr), uint64(want), uint64(b.offset))
+		uint64(b.bufptr), uint64(want), uint64(b.offset))
 	err = b.c.error(r)
 	if err != nil {
 		return 0, err
@@ -106,7 +109,7 @@ func (b *Blob) Read(p []byte) (n int, err error) {
 		err = io.EOF
 	}
 
-	copy(p, util.View(b.c.mod, ptr, uint64(want)))
+	copy(p, util.View(b.c.mod, b.bufptr, uint64(want)))
 	return int(want), err
 }
 
@@ -123,19 +126,20 @@ func (b *Blob) WriteTo(w io.Writer) (n int64, err error) {
 	if want > avail {
 		want = avail
 	}
-
-	defer b.c.arena.mark()()
-	ptr := b.c.arena.new(uint64(want))
+	if want > b.buflen {
+		b.bufptr = b.c.realloc(b.bufptr, uint64(want))
+		b.buflen = want
+	}
 
 	for want > 0 {
 		r := b.c.call("sqlite3_blob_read", uint64(b.handle),
-			uint64(ptr), uint64(want), uint64(b.offset))
+			uint64(b.bufptr), uint64(want), uint64(b.offset))
 		err = b.c.error(r)
 		if err != nil {
 			return n, err
 		}
 
-		mem := util.View(b.c.mod, ptr, uint64(want))
+		mem := util.View(b.c.mod, b.bufptr, uint64(want))
 		m, err := w.Write(mem[:want])
 		b.offset += int64(m)
 		n += int64(m)
@@ -159,11 +163,15 @@ func (b *Blob) WriteTo(w io.Writer) (n int64, err error) {
 //
 // https://sqlite.org/c3ref/blob_write.html
 func (b *Blob) Write(p []byte) (n int, err error) {
-	defer b.c.arena.mark()()
-	ptr := b.c.arena.bytes(p)
+	want := int64(len(p))
+	if want > b.buflen {
+		b.bufptr = b.c.realloc(b.bufptr, uint64(want))
+		b.buflen = want
+	}
+	util.WriteBytes(b.c.mod, b.bufptr, p)
 
 	r := b.c.call("sqlite3_blob_write", uint64(b.handle),
-		uint64(ptr), uint64(len(p)), uint64(b.offset))
+		uint64(b.bufptr), uint64(want), uint64(b.offset))
 	err = b.c.error(r)
 	if err != nil {
 		return 0, err
@@ -187,16 +195,17 @@ func (b *Blob) ReadFrom(r io.Reader) (n int64, err error) {
 	if want < 1 {
 		want = 1
 	}
-
-	defer b.c.arena.mark()()
-	ptr := b.c.arena.new(uint64(want))
+	if want > b.buflen {
+		b.bufptr = b.c.realloc(b.bufptr, uint64(want))
+		b.buflen = want
+	}
 
 	for {
-		mem := util.View(b.c.mod, ptr, uint64(want))
+		mem := util.View(b.c.mod, b.bufptr, uint64(want))
 		m, err := r.Read(mem[:want])
 		if m > 0 {
 			r := b.c.call("sqlite3_blob_write", uint64(b.handle),
-				uint64(ptr), uint64(m), uint64(b.offset))
+				uint64(b.bufptr), uint64(m), uint64(b.offset))
 			err := b.c.error(r)
 			if err != nil {
 				return n, err

config.go

@@ -294,3 +294,17 @@ func autoVacuumCallback(ctx context.Context, mod api.Module, pApp, zSchema, nDbP
 	schema := util.ReadString(mod, zSchema, _MAX_NAME)
 	return uint32(fn(schema, uint(nDbPage), uint(nFreePage), uint(nBytePerPage)))
 }
+
+// SoftHeapLimit imposes a soft limit on heap size.
+//
+// https://sqlite.org/c3ref/hard_heap_limit64.html
+func (c *Conn) SoftHeapLimit(n int64) int64 {
+	return int64(c.call("sqlite3_soft_heap_limit64", uint64(n)))
+}
+
+// HardHeapLimit imposes a hard limit on heap size.
+//
+// https://sqlite.org/c3ref/hard_heap_limit64.html
+func (c *Conn) HardHeapLimit(n int64) int64 {
+	return int64(c.call("sqlite3_hard_heap_limit64", uint64(n)))
+}

conn.go

@@ -23,6 +23,7 @@ type Conn struct {
 	interrupt  context.Context
 	pending    *Stmt
 	stmts      []*Stmt
+	timer      *time.Timer
 	busy       func(int) bool
 	log        func(xErrorCode, string)
 	collation  func(*Conn, string)
@@ -389,11 +390,25 @@ func timeoutCallback(ctx context.Context, mod api.Module, pDB uint32, count, tmo
 		}
 
 		if delay = min(delay, tmout-prior); delay > 0 {
-			time.Sleep(time.Duration(delay) * time.Millisecond)
-			retry = 1
+			delay := time.Duration(delay) * time.Millisecond
+			if c.interrupt == nil || c.interrupt.Done() == nil {
+				time.Sleep(delay)
+				return 1
+			}
+			if c.timer == nil {
+				c.timer = time.NewTimer(delay)
+			} else {
+				c.timer.Reset(delay)
+			}
+			select {
+			case <-c.interrupt.Done():
+				c.timer.Stop()
+			case <-c.timer.C:
+				return 1
+			}
 		}
 	}
-	return retry
+	return 0
 }
 
 // BusyHandler registers a callback to handle [BUSY] errors.
@@ -492,9 +507,7 @@ func (c *Conn) stmtsIter(yield func(*Stmt) bool) {
 
 // DriverConn is implemented by the SQLite [database/sql] driver connection.
 //
-// It can be used to access SQLite features like [online backup].
-//
-// [online backup]: https://sqlite.org/backup.html
+// Deprecated: use [github.com/ncruces/go-sqlite3/driver.Conn] instead.
 type DriverConn interface {
 	Raw() *Conn
 }

const.go

@@ -7,13 +7,10 @@ const (
 	_ROW  = 100 /* sqlite3_step() has another row ready */
 	_DONE = 101 /* sqlite3_step() has finished executing */
 
-	_UTF8 = 1
-
-	_MAX_NAME            = 1e6 // Self-imposed limit for most NUL terminated strings.
-	_MAX_LENGTH          = 1e9
-	_MAX_SQL_LENGTH      = 1e9
-	_MAX_ALLOCATION_SIZE = 0x7ffffeff
-	_MAX_FUNCTION_ARG    = 100
+	_MAX_NAME         = 1e6 // Self-imposed limit for most NUL terminated strings.
+	_MAX_LENGTH       = 1e9
+	_MAX_SQL_LENGTH   = 1e9
+	_MAX_FUNCTION_ARG = 100
 
 	ptrlen = 4
 )

context.go

@@ -84,9 +84,8 @@ func (ctx Context) ResultFloat(value float64) {
 // https://sqlite.org/c3ref/result_blob.html
 func (ctx Context) ResultText(value string) {
 	ptr := ctx.c.newString(value)
-	ctx.c.call("sqlite3_result_text64",
-		uint64(ctx.handle), uint64(ptr), uint64(len(value)),
-		uint64(ctx.c.freer), _UTF8)
+	ctx.c.call("sqlite3_result_text_go",
+		uint64(ctx.handle), uint64(ptr), uint64(len(value)))
 }
 
 // ResultRawText sets the text result of the function to a []byte.
@@ -94,9 +93,8 @@ func (ctx Context) ResultText(value string) {
 // https://sqlite.org/c3ref/result_blob.html
 func (ctx Context) ResultRawText(value []byte) {
 	ptr := ctx.c.newBytes(value)
-	ctx.c.call("sqlite3_result_text64",
-		uint64(ctx.handle), uint64(ptr), uint64(len(value)),
-		uint64(ctx.c.freer), _UTF8)
+	ctx.c.call("sqlite3_result_text_go",
+		uint64(ctx.handle), uint64(ptr), uint64(len(value)))
 }
 
 // ResultBlob sets the result of the function to a []byte.
@@ -105,9 +103,8 @@ func (ctx Context) ResultRawText(value []byte) {
 // https://sqlite.org/c3ref/result_blob.html
 func (ctx Context) ResultBlob(value []byte) {
 	ptr := ctx.c.newBytes(value)
-	ctx.c.call("sqlite3_result_blob64",
-		uint64(ctx.handle), uint64(ptr), uint64(len(value)),
-		uint64(ctx.c.freer))
+	ctx.c.call("sqlite3_result_blob_go",
+		uint64(ctx.handle), uint64(ptr), uint64(len(value)))
 }
 
 // ResultZeroBlob sets the result of the function to a zero-filled, length n BLOB.
@@ -154,9 +151,8 @@ func (ctx Context) resultRFC3339Nano(value time.Time) {
 	buf := util.View(ctx.c.mod, ptr, maxlen)
 	buf = value.AppendFormat(buf[:0], time.RFC3339Nano)
 
-	ctx.c.call("sqlite3_result_text64",
-		uint64(ctx.handle), uint64(ptr), uint64(len(buf)),
-		uint64(ctx.c.freer), _UTF8)
+	ctx.c.call("sqlite3_result_text_go",
+		uint64(ctx.handle), uint64(ptr), uint64(len(buf)))
 }
 
 // ResultPointer sets the result of the function to NULL, just like [Context.ResultNull],

driver/driver.go

@@ -130,6 +130,11 @@ type SQLite struct {
 	term func(*sqlite3.Conn) error
 }
 
+var (
+	// Ensure these interfaces are implemented:
+	_ driver.DriverContext = &SQLite{}
+)
+
 // Open implements [database/sql/driver.Driver].
 func (d *SQLite) Open(name string) (driver.Conn, error) {
 	c, err := d.newConnector(name)
@@ -255,6 +260,35 @@ func (n *connector) Connect(ctx context.Context) (_ driver.Conn, err error) {
 	return c, nil
 }
 
+// Conn is implemented by the SQLite [database/sql] driver connections.
+//
+// It can be used to access SQLite features like [online backup]:
+//
+//	db, err := driver.Open("temp.db")
+//	if err != nil {
+//		log.Fatal(err)
+//	}
+//	defer db.Close()
+//
+//	conn, err := db.Conn(context.TODO())
+//	if err != nil {
+//		log.Fatal(err)
+//	}
+//
+//	err = conn.Raw(func(driverConn any) error {
+//		conn := driverConn.(driver.Conn)
+//		return conn.Raw().Backup("main", "backup.db")
+//	})
+//	if err != nil {
+//		log.Fatal(err)
+//	}
+//
+// [online backup]: https://sqlite.org/backup.html
+type Conn interface {
+	Raw() *sqlite3.Conn
+	driver.Conn
+}
+
 type conn struct {
 	*sqlite3.Conn
 	txLock   string
@@ -266,10 +300,10 @@ type conn struct {
 
 var (
 	// Ensure these interfaces are implemented:
+	_ Conn                      = &conn{}
+	_ driver.ConnBeginTx        = &conn{}
 	_ driver.ConnPrepareContext = &conn{}
 	_ driver.ExecerContext      = &conn{}
-	_ driver.ConnBeginTx        = &conn{}
-	_ sqlite3.DriverConn        = &conn{}
 )
 
 func (c *conn) Raw() *sqlite3.Conn {
@@ -544,8 +578,15 @@ type rows struct {
 	*stmt
 	names []string
 	types []string
+	nulls []bool
 }
 
+var (
+	// Ensure these interfaces are implemented:
+	_ driver.RowsColumnTypeDatabaseTypeName = &rows{}
+	_ driver.RowsColumnTypeNullable         = &rows{}
+)
+
 func (r *rows) Close() error {
 	r.Stmt.ClearBindings()
 	return r.Stmt.Reset()
@@ -562,6 +603,22 @@ func (r *rows) Columns() []string {
 	return r.names
 }
 
+func (r *rows) loadTypes() {
+	if r.nulls == nil {
+		count := r.Stmt.ColumnCount()
+		r.nulls = make([]bool, count)
+		r.types = make([]string, count)
+		for i := range r.nulls {
+			if col := r.Stmt.ColumnOriginName(i); col != "" {
+				r.types[i], _, r.nulls[i], _, _, _ = r.Stmt.Conn().TableColumnMetadata(
+					r.Stmt.ColumnDatabaseName(i),
+					r.Stmt.ColumnTableName(i),
+					col)
+			}
+		}
+	}
+}
+
 func (r *rows) declType(index int) string {
 	if r.types == nil {
 		count := r.Stmt.ColumnCount()
@@ -574,7 +631,8 @@ func (r *rows) declType(index int) string {
 }
 
 func (r *rows) ColumnTypeDatabaseTypeName(index int) string {
-	decltype := r.declType(index)
+	r.loadTypes()
+	decltype := r.types[index]
 	if len := len(decltype); len > 0 && decltype[len-1] == ')' {
 		if i := strings.LastIndexByte(decltype, '('); i >= 0 {
 			decltype = decltype[:i]
@@ -583,6 +641,14 @@ func (r *rows) ColumnTypeDatabaseTypeName(index int) string {
 	return strings.TrimSpace(decltype)
 }
 
+func (r *rows) ColumnTypeNullable(index int) (nullable, ok bool) {
+	r.loadTypes()
+	if r.nulls[index] {
+		return false, true
+	}
+	return true, false
+}
+
 func (r *rows) Next(dest []driver.Value) error {
 	old := r.Stmt.Conn().SetInterrupt(r.ctx)
 	defer r.Stmt.Conn().SetInterrupt(old)

embed/bcw2/.gitignore

@@ -0,0 +1,2 @@
+build/
+sqlite/

embed/exports.txt

@@ -1,7 +1,4 @@
 aligned_alloc
-free
-malloc
-malloc_destructor
 sqlite3_anycollseq_init
 sqlite3_autovacuum_pages_go
 sqlite3_backup_finish
@@ -9,7 +6,7 @@ sqlite3_backup_init
 sqlite3_backup_pagecount
 sqlite3_backup_remaining
 sqlite3_backup_step
-sqlite3_bind_blob64
+sqlite3_bind_blob_go
 sqlite3_bind_double
 sqlite3_bind_int64
 sqlite3_bind_null
@@ -17,7 +14,7 @@ sqlite3_bind_parameter_count
 sqlite3_bind_parameter_index
 sqlite3_bind_parameter_name
 sqlite3_bind_pointer_go
-sqlite3_bind_text64
+sqlite3_bind_text_go
 sqlite3_bind_value
 sqlite3_bind_zeroblob64
 sqlite3_blob_bytes
@@ -74,17 +71,21 @@ sqlite3_filename_database
 sqlite3_filename_journal
 sqlite3_filename_wal
 sqlite3_finalize
+sqlite3_free
 sqlite3_get_autocommit
 sqlite3_get_auxdata
+sqlite3_hard_heap_limit64
 sqlite3_interrupt
 sqlite3_last_insert_rowid
 sqlite3_limit
+sqlite3_malloc64
 sqlite3_open_v2
 sqlite3_overload_function
 sqlite3_prepare_v3
 sqlite3_progress_handler_go
+sqlite3_realloc64
 sqlite3_reset
-sqlite3_result_blob64
+sqlite3_result_blob_go
 sqlite3_result_double
 sqlite3_result_error
 sqlite3_result_error_code
@@ -93,13 +94,14 @@ sqlite3_result_error_toobig
 sqlite3_result_int64
 sqlite3_result_null
 sqlite3_result_pointer_go
-sqlite3_result_text64
+sqlite3_result_text_go
 sqlite3_result_value
 sqlite3_result_zeroblob64
 sqlite3_rollback_hook_go
 sqlite3_set_authorizer_go
 sqlite3_set_auxdata_go
 sqlite3_set_last_insert_rowid
+sqlite3_soft_heap_limit64
 sqlite3_step
 sqlite3_stmt_busy
 sqlite3_stmt_readonly

ext/blobio/blob.go

@@ -11,11 +11,11 @@ import (
 
 // Register registers the SQL functions:
 //
-//	readblob(schema, table, column, rowid, offset, n)
+//	readblob(schema, table, column, rowid, offset, n/writer)
 //
 // Reads n bytes of a blob, starting at offset.
 //
-//	writeblob(schema, table, column, rowid, offset, data)
+//	writeblob(schema, table, column, rowid, offset, data/reader)
 //
 // Writes data into a blob, at the given offset.
 //
@@ -27,6 +27,10 @@ import (
 // using [sqlite3.BindPointer] or [sqlite3.Pointer].
 // The optional args will be passed to the callback,
 // along with the [sqlite3.Blob] handle.
+// The [sqlite3.Blob] handle is only valid during
+// the execution of the callback. Callers cannot
+// read or write to the handle after the callback
+// exits.
 //
 // https://sqlite.org/c3ref/blob.html
 func Register(db *sqlite3.Conn) error {
@@ -52,19 +56,24 @@ func readblob(ctx sqlite3.Context, arg ...sqlite3.Value) {
 		return // notest
 	}
 
-	n := arg[5].Int64()
-	if n <= 0 {
-		return
+	if p, ok := arg[5].Pointer().(io.Writer); ok {
+		var n int64
+		n, err = blob.WriteTo(p)
+		ctx.ResultInt64(n)
+	} else {
+		n := arg[5].Int64()
+		if n <= 0 {
+			return
+		}
+		buf := make([]byte, n)
+		_, err = io.ReadFull(blob, buf)
+		ctx.ResultBlob(buf)
 	}
-	buf := make([]byte, n)
-
-	_, err = io.ReadFull(blob, buf)
 	if err != nil {
 		ctx.ResultError(err)
 		return // notest
 	}
 
-	ctx.ResultBlob(buf)
 	setAuxBlob(ctx, blob, false)
 }
 
@@ -82,7 +91,9 @@ func writeblob(ctx sqlite3.Context, arg ...sqlite3.Value) {
 	}
 
 	if p, ok := arg[5].Pointer().(io.Reader); ok {
-		_, err = blob.ReadFrom(p)
+		var n int64
+		n, err = blob.ReadFrom(p)
+		ctx.ResultInt64(n)
 	} else {
 		_, err = blob.Write(arg[5].RawBlob())
 	}

ext/blobio/blob_test.go

@@ -34,23 +34,26 @@ func Example() {
 	const message = "Hello BLOB!"
 
 	// Create the BLOB.
-	_, err = db.Exec(`INSERT INTO test VALUES (?)`, sqlite3.ZeroBlob(len(message)))
+	r, err := db.Exec(`INSERT INTO test VALUES (?)`, sqlite3.ZeroBlob(len(message)))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	id, err := r.LastInsertId()
 	if err != nil {
 		log.Fatal(err)
 	}
 
 	// Write the BLOB.
-	_, err = db.Exec(`SELECT writeblob('main', 'test', 'col', last_insert_rowid(), 0, ?)`, message)
+	_, err = db.Exec(`SELECT writeblob('main', 'test', 'col', ?, 0, ?)`,
+		id, message)
 	if err != nil {
 		log.Fatal(err)
 	}
 
 	// Read the BLOB.
-	_, err = db.Exec(`SELECT openblob('main', 'test', 'col', rowid, false, ?) FROM test`,
-		sqlite3.Pointer[blobio.OpenCallback](func(blob *sqlite3.Blob, _ ...sqlite3.Value) error {
-			_, err = blob.WriteTo(os.Stdout)
-			return err
-		}))
+	_, err = db.Exec(`SELECT readblob('main', 'test', 'col', ?, 0, ?)`,
+		id, sqlite3.Pointer(os.Stdout))
 	if err != nil {
 		log.Fatal(err)
 	}

ext/bloom/bloom.go

@@ -79,6 +79,12 @@ func create(db *sqlite3.Conn, _, schema, table string, arg ...string) (_ *bloom,
 
 	t.bytes = numBytes(nelem, t.prob)
 
+	err = db.DeclareVTab(
+		`CREATE TABLE x(present, word HIDDEN NOT NULL PRIMARY KEY) WITHOUT ROWID`)
+	if err != nil {
+		return nil, err
+	}
+
 	err = db.Exec(fmt.Sprintf(
 		`CREATE TABLE %s.%s (data BLOB, p REAL, n INTEGER, m INTEGER, k INTEGER)`,
 		sqlite3.QuoteIdentifier(t.schema), sqlite3.QuoteIdentifier(t.storage)))
@@ -94,12 +100,6 @@ func create(db *sqlite3.Conn, _, schema, table string, arg ...string) (_ *bloom,
 		 VALUES (1, zeroblob(%d), %f, %d, %d, %d)`,
 		sqlite3.QuoteIdentifier(t.schema), sqlite3.QuoteIdentifier(t.storage),
 		t.bytes, t.prob, nelem, 8*t.bytes, t.hashes))
-	if err != nil {
-		return nil, err
-	}
-
-	err = db.DeclareVTab(
-		`CREATE TABLE x(present, word HIDDEN NOT NULL PRIMARY KEY) WITHOUT ROWID`)
 	if err != nil {
 		t.Destroy()
 		return nil, err
@@ -198,10 +198,10 @@ func (t *bloom) Integrity(schema, table string, flags int) error {
 }
 
 func (b *bloom) BestIndex(idx *sqlite3.IndexInfo) error {
-	for n, cst := range idx.Constraint {
+	for i, cst := range idx.Constraint {
 		if cst.Usable && cst.Column == 1 &&
 			cst.Op == sqlite3.INDEX_CONSTRAINT_EQ {
-			idx.ConstraintUsage[n].ArgvIndex = 1
+			idx.ConstraintUsage[i].ArgvIndex = 1
 			idx.OrderByConsumed = true
 			idx.EstimatedRows = 1
 			idx.EstimatedCost = float64(b.hashes)
@@ -272,10 +272,6 @@ type cursor struct {
 }
 
 func (c *cursor) Filter(idxNum int, idxStr string, arg ...sqlite3.Value) error {
-	if len(arg) != 1 {
-		return nil
-	}
-
 	c.eof = false
 	c.arg = &arg[0]
 	blob := arg[0].RawBlob()

ext/closure/closure.go

@@ -0,0 +1,256 @@
+// Package closure provides a transitive closure virtual table.
+//
+// The transitive_closure virtual table finds the transitive closure of
+// a parent/child relationship in a real table.
+//
+// https://sqlite.org/src/doc/tip/ext/misc/closure.c
+package closure
+
+import (
+	"fmt"
+	"math"
+
+	"github.com/ncruces/go-sqlite3"
+	"github.com/ncruces/go-sqlite3/internal/util"
+	"github.com/ncruces/go-sqlite3/util/vtabutil"
+)
+
+const (
+	_COL_ID           = 0
+	_COL_DEPTH        = 1
+	_COL_ROOT         = 2
+	_COL_TABLENAME    = 3
+	_COL_IDCOLUMN     = 4
+	_COL_PARENTCOLUMN = 5
+)
+
+// Register registers the transitive_closure virtual table:
+//
+//	CREATE VIRTUAL TABLE temp.closure USING transitive_closure;
+func Register(db *sqlite3.Conn) error {
+	return sqlite3.CreateModule(db, "transitive_closure", nil,
+		func(db *sqlite3.Conn, _, _, _ string, arg ...string) (*closure, error) {
+			var (
+				table  string
+				column string
+				parent string
+
+				done = util.Set[string]{}
+			)
+
+			for _, arg := range arg {
+				key, val := vtabutil.NamedArg(arg)
+				if done.Contains(key) {
+					return nil, fmt.Errorf("transitive_closure: more than one %q parameter", key)
+				}
+				switch key {
+				case "tablename":
+					table = vtabutil.Unquote(val)
+				case "idcolumn":
+					column = vtabutil.Unquote(val)
+				case "parentcolumn":
+					parent = vtabutil.Unquote(val)
+				default:
+					return nil, fmt.Errorf("transitive_closure: unknown %q parameter", key)
+				}
+				done.Add(key)
+			}
+
+			err := db.DeclareVTab(`CREATE TABLE x(id,depth,root HIDDEN,tablename HIDDEN,idcolumn HIDDEN,parentcolumn HIDDEN)`)
+			if err != nil {
+				return nil, err
+			}
+			return &closure{
+				db:     db,
+				table:  table,
+				column: column,
+				parent: parent,
+			}, nil
+		})
+}
+
+type closure struct {
+	db     *sqlite3.Conn
+	table  string
+	column string
+	parent string
+}
+
+func (c *closure) Destroy() error { return nil }
+
+func (c *closure) BestIndex(idx *sqlite3.IndexInfo) error {
+	plan := 0
+	posi := 1
+	cost := 1e7
+
+	for i, cst := range idx.Constraint {
+		if !cst.Usable {
+			continue
+		}
+		if plan&1 == 0 && cst.Column == _COL_ROOT {
+			switch cst.Op {
+			case sqlite3.INDEX_CONSTRAINT_EQ:
+				plan |= 1
+				cost /= 100
+				idx.ConstraintUsage[i].ArgvIndex = 1
+				idx.ConstraintUsage[i].Omit = true
+			}
+			continue
+		}
+		if plan&0xf0 == 0 && cst.Column == _COL_DEPTH {
+			switch cst.Op {
+			case sqlite3.INDEX_CONSTRAINT_LT, sqlite3.INDEX_CONSTRAINT_LE, sqlite3.INDEX_CONSTRAINT_EQ:
+				plan |= posi << 4
+				cost /= 5
+				posi += 1
+				idx.ConstraintUsage[i].ArgvIndex = posi
+				if cst.Op == sqlite3.INDEX_CONSTRAINT_LT {
+					plan |= 2
+				}
+			}
+			continue
+		}
+		if plan&0xf00 == 0 && cst.Column == _COL_TABLENAME {
+			switch cst.Op {
+			case sqlite3.INDEX_CONSTRAINT_EQ:
+				plan |= posi << 8
+				cost /= 5
+				posi += 1
+				idx.ConstraintUsage[i].ArgvIndex = posi
+				idx.ConstraintUsage[i].Omit = true
+			}
+			continue
+		}
+		if plan&0xf000 == 0 && cst.Column == _COL_IDCOLUMN {
+			switch cst.Op {
+			case sqlite3.INDEX_CONSTRAINT_EQ:
+				plan |= posi << 12
+				posi += 1
+				idx.ConstraintUsage[i].ArgvIndex = posi
+				idx.ConstraintUsage[i].Omit = true
+			}
+			continue
+		}
+		if plan&0xf0000 == 0 && cst.Column == _COL_PARENTCOLUMN {
+			switch cst.Op {
+			case sqlite3.INDEX_CONSTRAINT_EQ:
+				plan |= posi << 16
+				posi += 1
+				idx.ConstraintUsage[i].ArgvIndex = posi
+				idx.ConstraintUsage[i].Omit = true
+			}
+			continue
+		}
+	}
+
+	if plan&1 == 0 ||
+		c.table == "" && plan&0xf00 == 0 ||
+		c.column == "" && plan&0xf000 == 0 ||
+		c.parent == "" && plan&0xf0000 == 0 {
+		return sqlite3.CONSTRAINT
+	}
+
+	idx.EstimatedCost = cost
+	idx.IdxNum = plan
+	return nil
+}
+
+func (c *closure) Open() (sqlite3.VTabCursor, error) {
+	return &cursor{closure: c}, nil
+}
+
+type cursor struct {
+	*closure
+	nodes []node
+}
+
+type node struct {
+	id    int64
+	depth int
+}
+
+func (c *cursor) Filter(idxNum int, idxStr string, arg ...sqlite3.Value) error {
+	root := arg[0].Int64()
+	maxDepth := math.MaxInt
+	if idxNum&0xf0 != 0 {
+		maxDepth = arg[(idxNum>>4)&0xf].Int()
+		if idxNum&2 != 0 {
+			maxDepth -= 1
+		}
+	}
+	table := c.table
+	if idxNum&0xf00 != 0 {
+		table = arg[(idxNum>>8)&0xf].Text()
+	}
+	column := c.column
+	if idxNum&0xf000 != 0 {
+		column = arg[(idxNum>>12)&0xf].Text()
+	}
+	parent := c.parent
+	if idxNum&0xf0000 != 0 {
+		parent = arg[(idxNum>>16)&0xf].Text()
+	}
+
+	sql := fmt.Sprintf(
+		`SELECT %[1]s.%[2]s FROM %[1]s WHERE %[1]s.%[3]s=?`,
+		sqlite3.QuoteIdentifier(table),
+		sqlite3.QuoteIdentifier(column),
+		sqlite3.QuoteIdentifier(parent),
+	)
+	stmt, _, err := c.db.Prepare(sql)
+	if err != nil {
+		return err
+	}
+	defer stmt.Close()
+
+	c.nodes = []node{{root, 0}}
+	set := util.Set[int64]{}
+	set.Add(root)
+	for i := 0; i < len(c.nodes); i++ {
+		curr := c.nodes[i]
+		if curr.depth >= maxDepth {
+			continue
+		}
+		stmt.BindInt64(1, curr.id)
+		for stmt.Step() {
+			if stmt.ColumnType(0) == sqlite3.INTEGER {
+				next := stmt.ColumnInt64(0)
+				if !set.Contains(next) {
+					set.Add(next)
+					c.nodes = append(c.nodes, node{next, curr.depth + 1})
+				}
+			}
+		}
+		stmt.Reset()
+	}
+	return nil
+}
+
+func (c *cursor) Column(ctx sqlite3.Context, n int) error {
+	switch n {
+	case _COL_ID:
+		ctx.ResultInt64(c.nodes[0].id)
+	case _COL_DEPTH:
+		ctx.ResultInt(c.nodes[0].depth)
+	case _COL_TABLENAME:
+		ctx.ResultText(c.table)
+	case _COL_IDCOLUMN:
+		ctx.ResultText(c.column)
+	case _COL_PARENTCOLUMN:
+		ctx.ResultText(c.parent)
+	}
+	return nil
+}
+
+func (c *cursor) Next() error {
+	c.nodes = c.nodes[1:]
+	return nil
+}
+
+func (c *cursor) EOF() bool {
+	return len(c.nodes) == 0
+}
+
+func (c *cursor) RowID() (int64, error) {
+	return c.nodes[0].id, nil
+}

ext/closure/closure_test.go

@@ -0,0 +1,184 @@
+package closure_test
+
+import (
+	_ "embed"
+	"fmt"
+	"log"
+	"testing"
+
+	"github.com/ncruces/go-sqlite3"
+	_ "github.com/ncruces/go-sqlite3/embed"
+	"github.com/ncruces/go-sqlite3/ext/closure"
+	_ "github.com/ncruces/go-sqlite3/internal/testcfg"
+)
+
+func TestMain(m *testing.M) {
+	sqlite3.AutoExtension(closure.Register)
+	m.Run()
+}
+
+func Example() {
+	db, err := sqlite3.Open(":memory:")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer db.Close()
+
+	closure.Register(db)
+
+	err = db.Exec(`
+		CREATE TABLE employees (
+			id INTEGER PRIMARY KEY,
+			parent_id INTEGER,
+			name TEXT
+		);
+		CREATE INDEX employees_parent_idx ON employees(parent_id);
+		INSERT INTO employees (id, parent_id, name) VALUES
+			(11, NULL, 'Diane'),
+			(12, 11, 'Bob'),
+			(21, 11, 'Emma'),
+			(22, 21, 'Grace'),
+			(23, 21, 'Henry'),
+			(24, 21, 'Irene'),
+			(25, 21, 'Frank'),
+			(31, 11, 'Cindy'),
+			(32, 31, 'Dave'),
+			(33, 31, 'Alice');
+		CREATE VIRTUAL TABLE hierarchy USING transitive_closure(
+			tablename = "employees",
+			idcolumn = "id",
+			parentcolumn = "parent_id"
+		);
+	`)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	stmt, _, err := db.Prepare(`
+		SELECT employees.id, name FROM employees, hierarchy
+		WHERE employees.id = hierarchy.id AND hierarchy.root = 31
+	`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer stmt.Close()
+
+	for stmt.Step() {
+		fmt.Println(stmt.ColumnInt(0), stmt.ColumnText(1))
+	}
+	if err := stmt.Err(); err != nil {
+		log.Fatal(err)
+	}
+
+	err = stmt.Close()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = db.Close()
+	if err != nil {
+		log.Fatal(err)
+	}
+	// Output:
+	// 31 Cindy
+	// 32 Dave
+	// 33 Alice
+}
+
+func TestRegister(t *testing.T) {
+	t.Parallel()
+
+	db, err := sqlite3.Open(":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	err = db.Exec(`
+		CREATE TABLE employees (
+			id INTEGER PRIMARY KEY,
+			parent_id INTEGER,
+			name TEXT
+		);
+		CREATE INDEX employees_parent_idx ON employees(parent_id);
+		INSERT INTO employees (id, parent_id, name) VALUES
+			(11, NULL, 'Diane'),
+			(12, 11, 'Bob'),
+			(21, 11, 'Emma'),
+			(22, 21, 'Grace'),
+			(23, 21, 'Henry'),
+			(24, 21, 'Irene'),
+			(25, 21, 'Frank'),
+			(31, 11, 'Cindy'),
+			(32, 31, 'Dave'),
+			(33, 31, 'Alice');
+		CREATE VIRTUAL TABLE temp.closure USING transitive_closure;
+	`)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	stmt, _, err := db.Prepare(`
+		SELECT employees.id, name FROM employees, closure
+		 WHERE employees.id = closure.id
+		   AND closure.root = 31
+		   AND closure.depth < 1
+		   AND closure.tablename='employees'
+		   AND closure.idcolumn='id'
+		   AND closure.parentcolumn='parent_id'
+	`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer stmt.Close()
+
+	if !stmt.Step() {
+		t.Error("want row")
+	}
+	if stmt.Step() {
+		t.Error("don't want row")
+	}
+	if err := stmt.Err(); err != nil {
+		t.Fatal(err)
+	}
+
+	err = stmt.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = db.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func Test_errors(t *testing.T) {
+	t.Parallel()
+
+	db, err := sqlite3.Open(":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	err = db.Exec(`CREATE VIRTUAL TABLE hierarchy USING transitive_closure(table='employees')`)
+	if err == nil {
+		t.Error("want error")
+	}
+
+	err = db.Exec(`CREATE VIRTUAL TABLE hierarchy USING transitive_closure(tablename='employees', tablename="employees")`)
+	if err == nil {
+		t.Error("want error")
+	}
+
+	err = db.Exec("CREATE VIRTUAL TABLE hierarchy USING transitive_closure(tablename=`employees`)")
+	if err != nil {
+		t.Error(err)
+	}
+
+	err = db.Exec(`SELECT * FROM hierarchy`)
+	if err == nil {
+		t.Error("want error")
+	}
+}

ext/csv/csv.go

@@ -40,12 +40,12 @@ func RegisterFS(db *sqlite3.Conn, fsys fs.FS) error {
 			comma    rune = ','
 			comment  rune
 
-			done = map[string]struct{}{}
+			done = util.Set[string]{}
 		)
 
 		for _, arg := range arg {
 			key, val := vtabutil.NamedArg(arg)
-			if _, ok := done[key]; ok {
+			if done.Contains(key) {
 				return nil, fmt.Errorf("csv: more than one %q parameter", key)
 			}
 			switch key {
@@ -69,7 +69,7 @@ func RegisterFS(db *sqlite3.Conn, fsys fs.FS) error {
 			if err != nil {
 				return nil, err
 			}
-			done[key] = struct{}{}
+			done.Add(key)
 		}
 
 		if (filename == "") == (data == "") {

ext/fileio/fsdir.go

@@ -10,13 +10,22 @@ import (
 	"github.com/ncruces/go-sqlite3"
 )
 
+const (
+	_COL_NAME = 0
+	_COL_MODE = 1
+	_COL_TIME = 2
+	_COL_DATA = 3
+	_COL_ROOT = 4
+	_COL_BASE = 5
+)
+
 type fsdir struct{ fsys fs.FS }
 
 func (d fsdir) BestIndex(idx *sqlite3.IndexInfo) error {
 	var root, base bool
 	for i, cst := range idx.Constraint {
 		switch cst.Column {
-		case 4: // root
+		case _COL_ROOT:
 			if !cst.Usable || cst.Op != sqlite3.INDEX_CONSTRAINT_EQ {
 				return sqlite3.CONSTRAINT
 			}
@@ -25,7 +34,7 @@ func (d fsdir) BestIndex(idx *sqlite3.IndexInfo) error {
 				ArgvIndex: 1,
 			}
 			root = true
-		case 5: // base
+		case _COL_BASE:
 			if !cst.Usable || cst.Op != sqlite3.INDEX_CONSTRAINT_EQ {
 				return sqlite3.CONSTRAINT
 			}
@@ -116,25 +125,25 @@ func (c *cursor) RowID() (int64, error) {
 
 func (c *cursor) Column(ctx sqlite3.Context, n int) error {
 	switch n {
-	case 0: // name
+	case _COL_NAME:
 		name := strings.TrimPrefix(c.curr.path, c.base)
 		ctx.ResultText(name)
 
-	case 1: // mode
+	case _COL_MODE:
 		i, err := c.curr.Info()
 		if err != nil {
 			return err
 		}
 		ctx.ResultInt64(int64(i.Mode()))
 
-	case 2: // mtime
+	case _COL_TIME:
 		i, err := c.curr.Info()
 		if err != nil {
 			return err
 		}
 		ctx.ResultTime(i.ModTime(), sqlite3.TimeFormatUnixFrac)
 
-	case 3: // data
+	case _COL_DATA:
 		switch typ := c.curr.Type(); {
 		case typ.IsRegular():
 			var data []byte

go.mod

@@ -6,17 +6,16 @@ toolchain go1.23.0
 
 require (
 	github.com/dchest/siphash v1.2.3
+	github.com/google/uuid v1.6.0
 	github.com/ncruces/julianday v1.0.0
 	github.com/ncruces/sort v0.1.2
 	github.com/psanford/httpreadat v0.1.0
 	github.com/tetratelabs/wazero v1.8.0
-	golang.org/x/crypto v0.26.0
+	golang.org/x/crypto v0.27.0
 	golang.org/x/sync v0.8.0
-	golang.org/x/sys v0.24.0
-	golang.org/x/text v0.17.0
+	golang.org/x/sys v0.25.0
+	golang.org/x/text v0.18.0
 	lukechampine.com/adiantum v1.1.1
 )
 
-require github.com/google/uuid v1.6.0
-
 retract v0.4.0 // tagged from the wrong branch

go.sum

@@ -10,13 +10,13 @@ github.com/psanford/httpreadat v0.1.0 h1:VleW1HS2zO7/4c7c7zNl33fO6oYACSagjJIyMIw
 github.com/psanford/httpreadat v0.1.0/go.mod h1:Zg7P+TlBm3bYbyHTKv/EdtSJZn3qwbPwpfZ/I9GKCRE=
 github.com/tetratelabs/wazero v1.8.0 h1:iEKu0d4c2Pd+QSRieYbnQC9yiFlMS9D+Jr0LsRmcF4g=
 github.com/tetratelabs/wazero v1.8.0/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
-golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
-golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
+golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
 golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
 golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
-golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
-golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
+golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
+golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
 lukechampine.com/adiantum v1.1.1 h1:4fp6gTxWCqpEbLy40ExiYDDED3oUNWx5cTqBCtPdZqA=
 lukechampine.com/adiantum v1.1.1/go.mod h1:LrAYVnTYLnUtE/yMp5bQr0HstAf060YUF8nM0B6+rUw=

go.work.sum

@@ -5,5 +5,6 @@ golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
 golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
 golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
+golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=

gormlite/error_translator_test.go

@@ -3,20 +3,18 @@ package gormlite
 import (
 	"testing"
 
+	"github.com/ncruces/go-sqlite3/vfs/memdb"
 	"gorm.io/gorm"
 	"gorm.io/gorm/logger"
 )
 
 func TestErrorTranslator(t *testing.T) {
-	// This is the DSN of the in-memory SQLite database for these tests.
-	const InMemoryDSN = "file:testdatabase?mode=memory&cache=shared"
-
 	// This is the example object for testing the unique constraint error
 	type Article struct {
 		ArticleNumber string `gorm:"unique"`
 	}
 
-	db, err := gorm.Open(Open(InMemoryDSN), &gorm.Config{
+	db, err := gorm.Open(Open(memdb.TestDB(t)), &gorm.Config{
 		Logger:         logger.Default.LogMode(logger.Silent),
 		TranslateError: true})
 

gormlite/go.mod

@@ -5,8 +5,8 @@ go 1.21
 toolchain go1.23.0
 
 require (
-	github.com/ncruces/go-sqlite3 v0.18.0
-	gorm.io/gorm v1.25.11
+	github.com/ncruces/go-sqlite3 v0.18.3
+	gorm.io/gorm v1.25.12
 )
 
 require (
@@ -14,6 +14,6 @@ require (
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/ncruces/julianday v1.0.0 // indirect
 	github.com/tetratelabs/wazero v1.8.0 // indirect
-	golang.org/x/sys v0.24.0 // indirect
-	golang.org/x/text v0.17.0 // indirect
+	golang.org/x/sys v0.25.0 // indirect
+	golang.org/x/text v0.18.0 // indirect
 )

gormlite/go.sum

@@ -2,15 +2,15 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
 github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
-github.com/ncruces/go-sqlite3 v0.18.0 h1:aH7WGzOC0CYpUPG1LdFg7JApybiuXgYUE2itzLBwhPM=
-github.com/ncruces/go-sqlite3 v0.18.0/go.mod h1:eEOyZnW1dGTJ+zDpMuzfYamEUBtdFz5zeYhqLBtHxvM=
+github.com/ncruces/go-sqlite3 v0.18.3 h1:tyMa75uh7LcINcfo0WrzOvcTkfz8Hqu0TEPX+KVyes4=
+github.com/ncruces/go-sqlite3 v0.18.3/go.mod h1:HAwOtA+cyEX3iN6YmkpQwfT4vMMgCB7rQRFUdOgEFik=
 github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
 github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
 github.com/tetratelabs/wazero v1.8.0 h1:iEKu0d4c2Pd+QSRieYbnQC9yiFlMS9D+Jr0LsRmcF4g=
 github.com/tetratelabs/wazero v1.8.0/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
-golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
-golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
-golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
-gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg=
-gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=
+golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
+golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
+golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8=
+gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=

gormlite/test.sh

@@ -7,7 +7,7 @@ rm -rf gorm/ tests/
 go work use -r .
 go test
 
-git clone --branch v1.25.11 --filter=blob:none https://github.com/go-gorm/gorm.git
+git clone --branch v1.25.12 --filter=blob:none https://github.com/go-gorm/gorm.git
 mv gorm/tests tests
 rm -rf gorm/
 

internal/alloc/alloc_slice.go

@@ -1,4 +1,4 @@
-//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64) || sqlite3_noshm || sqlite3_nosys
+//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys
 
 package alloc
 

internal/alloc/alloc_unix.go

@@ -39,7 +39,7 @@ type mmappedMemory struct {
 func (m *mmappedMemory) Reallocate(size uint64) []byte {
 	com := uint64(len(m.buf))
 	res := uint64(cap(m.buf))
-	if com < size && size < res {
+	if com < size && size <= res {
 		// Round up to the page size.
 		rnd := uint64(unix.Getpagesize() - 1)
 		new := (size + rnd) &^ rnd

internal/alloc/alloc_windows.go

@@ -48,7 +48,7 @@ type virtualMemory struct {
 func (m *virtualMemory) Reallocate(size uint64) []byte {
 	com := uint64(len(m.buf))
 	res := uint64(cap(m.buf))
-	if com < size && size < res {
+	if com < size && size <= res {
 		// Round up to the page size.
 		rnd := uint64(windows.Getpagesize() - 1)
 		new := (size + rnd) &^ rnd

internal/util/mmap.go

@@ -1,4 +1,4 @@
-//go:build unix && (amd64 || arm64 || riscv64) && !(sqlite3_noshm || sqlite3_nosys)
+//go:build unix && (amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys)
 
 package util
 

internal/util/mmap_other.go

@@ -1,4 +1,4 @@
-//go:build !unix || !(amd64 || arm64 || riscv64) || sqlite3_noshm || sqlite3_nosys
+//go:build !unix || !(amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys
 
 package util
 

internal/util/set.go

@@ -0,0 +1,12 @@
+package util
+
+type Set[E comparable] map[E]struct{}
+
+func (s Set[E]) Add(v E) {
+	s[v] = struct{}{}
+}
+
+func (s Set[E]) Contains(v E) bool {
+	_, ok := s[v]
+	return ok
+}

sqlite.go

@@ -86,7 +86,6 @@ type sqlite struct {
 		mask uint32
 	}
 	stack [9]uint64
-	freer uint32
 }
 
 func instantiateSQLite() (sqlt *sqlite, err error) {
@@ -102,14 +101,7 @@ func instantiateSQLite() (sqlt *sqlite, err error) {
 	if err != nil {
 		return nil, err
 	}
-
-	global := sqlt.mod.ExportedGlobal("malloc_destructor")
-	if global == nil {
-		return nil, util.BadBinaryErr
-	}
-
-	sqlt.freer = util.ReadUint32(sqlt.mod, uint32(global.Get()))
-	if sqlt.freer == 0 {
+	if sqlt.getfn("sqlite3_progress_handler_go") == nil {
 		return nil, util.BadBinaryErr
 	}
 	return sqlt, nil
@@ -196,14 +188,19 @@ func (sqlt *sqlite) free(ptr uint32) {
 	if ptr == 0 {
 		return
 	}
-	sqlt.call("free", uint64(ptr))
+	sqlt.call("sqlite3_free", uint64(ptr))
 }
 
 func (sqlt *sqlite) new(size uint64) uint32 {
-	if size > _MAX_ALLOCATION_SIZE {
+	ptr := uint32(sqlt.call("sqlite3_malloc64", size))
+	if ptr == 0 && size != 0 {
 		panic(util.OOMErr)
 	}
-	ptr := uint32(sqlt.call("malloc", size))
+	return ptr
+}
+
+func (sqlt *sqlite) realloc(ptr uint32, size uint64) uint32 {
+	ptr = uint32(sqlt.call("sqlite3_realloc64", uint64(ptr), size))
 	if ptr == 0 && size != 0 {
 		panic(util.OOMErr)
 	}
@@ -214,7 +211,11 @@ func (sqlt *sqlite) newBytes(b []byte) uint32 {
 	if (*[0]byte)(b) == nil {
 		return 0
 	}
-	ptr := sqlt.new(uint64(len(b)))
+	size := len(b)
+	if size == 0 {
+		size = 1
+	}
+	ptr := sqlt.new(uint64(size))
 	util.WriteBytes(sqlt.mod, ptr, b)
 	return ptr
 }

sqlite3/bind.c

@@ -0,0 +1,13 @@
+#include <stdlib.h>
+
+#include "sqlite3.h"
+
+int sqlite3_bind_text_go(sqlite3_stmt* stmt, int i, const char* zData,
+                         sqlite3_uint64 nData) {
+  return sqlite3_bind_text64(stmt, i, zData, nData, &sqlite3_free, SQLITE_UTF8);
+}
+
+int sqlite3_bind_blob_go(sqlite3_stmt* stmt, int i, const char* zData,
+                         sqlite3_uint64 nData) {
+  return sqlite3_bind_blob64(stmt, i, zData, nData, &sqlite3_free);
+}

sqlite3/main.c

@@ -9,16 +9,16 @@
 #include "ext/series.c"
 #include "ext/uint.c"
 // Bindings
+#include "bind.c"
 #include "column.c"
 #include "func.c"
 #include "hooks.c"
 #include "pointer.c"
+#include "result.c"
 #include "time.c"
 #include "vfs.c"
 #include "vtab.c"
 
-sqlite3_destructor_type malloc_destructor = &free;
-
 __attribute__((constructor)) void init() {
   sqlite3_initialize();
   sqlite3_auto_extension((void (*)(void))sqlite3_base_init);

sqlite3/result.c

@@ -0,0 +1,13 @@
+#include <stdlib.h>
+
+#include "sqlite3.h"
+
+void sqlite3_result_text_go(sqlite3_context* ctx, const char* zData,
+                            sqlite3_uint64 nData) {
+  sqlite3_result_text64(ctx, zData, nData, &sqlite3_free, SQLITE_UTF8);
+}
+
+void sqlite3_result_blob_go(sqlite3_context* ctx, const void* zData,
+                            sqlite3_uint64 nData) {
+  sqlite3_result_blob64(ctx, zData, nData, &sqlite3_free);
+}

sqlite3/sqlite_opt.h

@@ -2,7 +2,6 @@
 
 #define SQLITE_DQS 0
 #define SQLITE_THREADSAFE 0
-#define SQLITE_DEFAULT_MEMSTATUS 0
 #define SQLITE_DEFAULT_WAL_SYNCHRONOUS 1
 #define SQLITE_LIKE_DOESNT_MATCH_BLOBS
 #define SQLITE_MAX_EXPR_DEPTH 0
@@ -13,6 +12,7 @@
 #define SQLITE_OMIT_AUTOINIT
 
 // We need these:
+// #define SQLITE_DEFAULT_MEMSTATUS 0
 // #define SQLITE_OMIT_DECLTYPE
 // #define SQLITE_OMIT_PROGRESS_CALLBACK
 
@@ -24,6 +24,7 @@
 #define SQLITE_ENABLE_ATOMIC_WRITE
 #define SQLITE_ENABLE_BATCH_ATOMIC_WRITE
 #define SQLITE_ENABLE_COLUMN_METADATA
+#define SQLITE_ENABLE_SETLK_TIMEOUT 2
 #define SQLITE_ENABLE_STAT4 1
 
 // We have our own memdb VFS.

sqlite_test.go

@@ -3,7 +3,6 @@ package sqlite3
 import (
 	"bytes"
 	"math"
-	"os"
 	"testing"
 
 	"github.com/ncruces/go-sqlite3/internal/util"
@@ -39,7 +38,7 @@ func Test_sqlite_call_closed(t *testing.T) {
 	sqlite.close()
 
 	defer func() { _ = recover() }()
-	sqlite.call("free")
+	sqlite.call("sqlite3_free")
 	t.Error("want panic")
 }
 
@@ -57,19 +56,6 @@ func Test_sqlite_new(t *testing.T) {
 		sqlite.new(math.MaxUint32)
 		t.Error("want panic")
 	})
-
-	t.Run("_MAX_ALLOCATION_SIZE", func(t *testing.T) {
-		if testing.Short() {
-			t.Skip("skipping in short mode")
-		}
-		if os.Getenv("CI") != "" {
-			t.Skip("skipping in CI")
-		}
-		defer func() { _ = recover() }()
-		sqlite.new(_MAX_ALLOCATION_SIZE)
-		sqlite.new(_MAX_ALLOCATION_SIZE)
-		t.Error("want panic")
-	})
 }
 
 func Test_sqlite_newArena(t *testing.T) {

stmt.go

@@ -246,10 +246,9 @@ func (s *Stmt) BindText(param int, value string) error {
 		return TOOBIG
 	}
 	ptr := s.c.newString(value)
-	r := s.c.call("sqlite3_bind_text64",
+	r := s.c.call("sqlite3_bind_text_go",
 		uint64(s.handle), uint64(param),
-		uint64(ptr), uint64(len(value)),
-		uint64(s.c.freer), _UTF8)
+		uint64(ptr), uint64(len(value)))
 	return s.c.error(r)
 }
 
@@ -262,10 +261,9 @@ func (s *Stmt) BindRawText(param int, value []byte) error {
 		return TOOBIG
 	}
 	ptr := s.c.newBytes(value)
-	r := s.c.call("sqlite3_bind_text64",
+	r := s.c.call("sqlite3_bind_text_go",
 		uint64(s.handle), uint64(param),
-		uint64(ptr), uint64(len(value)),
-		uint64(s.c.freer), _UTF8)
+		uint64(ptr), uint64(len(value)))
 	return s.c.error(r)
 }
 
@@ -279,10 +277,9 @@ func (s *Stmt) BindBlob(param int, value []byte) error {
 		return TOOBIG
 	}
 	ptr := s.c.newBytes(value)
-	r := s.c.call("sqlite3_bind_blob64",
+	r := s.c.call("sqlite3_bind_blob_go",
 		uint64(s.handle), uint64(param),
-		uint64(ptr), uint64(len(value)),
-		uint64(s.c.freer))
+		uint64(ptr), uint64(len(value)))
 	return s.c.error(r)
 }
 
@@ -335,10 +332,9 @@ func (s *Stmt) bindRFC3339Nano(param int, value time.Time) error {
 	buf := util.View(s.c.mod, ptr, maxlen)
 	buf = value.AppendFormat(buf[:0], time.RFC3339Nano)
 
-	r := s.c.call("sqlite3_bind_text64",
+	r := s.c.call("sqlite3_bind_text_go",
 		uint64(s.handle), uint64(param),
-		uint64(ptr), uint64(len(buf)),
-		uint64(s.c.freer), _UTF8)
+		uint64(ptr), uint64(len(buf)))
 	return s.c.error(r)
 }
 

tests/config_test.go

@@ -0,0 +1,427 @@
+package tests
+
+import (
+	"errors"
+	"math"
+	"net/url"
+	"path/filepath"
+	"testing"
+
+	"github.com/ncruces/go-sqlite3"
+	"github.com/ncruces/go-sqlite3/vfs"
+	"github.com/ncruces/go-sqlite3/vfs/memdb"
+)
+
+func TestConn_Config(t *testing.T) {
+	t.Parallel()
+
+	db, err := sqlite3.Open(":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	o, err := db.Config(sqlite3.DBCONFIG_DEFENSIVE)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o != false {
+		t.Error("want false")
+	}
+
+	o, err = db.Config(sqlite3.DBCONFIG_DEFENSIVE, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o != true {
+		t.Error("want true")
+	}
+
+	o, err = db.Config(sqlite3.DBCONFIG_DEFENSIVE)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o != true {
+		t.Error("want true")
+	}
+
+	o, err = db.Config(sqlite3.DBCONFIG_DEFENSIVE, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o != false {
+		t.Error("want false")
+	}
+
+	o, err = db.Config(sqlite3.DBCONFIG_DEFENSIVE)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if o != false {
+		t.Error("want false")
+	}
+}
+
+func TestConn_ConfigLog(t *testing.T) {
+	t.Parallel()
+
+	db, err := sqlite3.Open(":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	var code sqlite3.ExtendedErrorCode
+	err = db.ConfigLog(func(c sqlite3.ExtendedErrorCode, msg string) {
+		t.Log(msg)
+		code = c
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	db.Prepare(`SELECT * FRM sqlite_schema`)
+
+	if code != sqlite3.ExtendedErrorCode(sqlite3.ERROR) {
+		t.Error("want sqlite3.ERROR")
+	}
+}
+
+func TestConn_FileControl(t *testing.T) {
+	t.Parallel()
+
+	file := filepath.Join(t.TempDir(), "test.db")
+	db, err := sqlite3.Open(file)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	t.Run("MISUSE", func(t *testing.T) {
+		_, err := db.FileControl("main", 0)
+		if !errors.Is(err, sqlite3.MISUSE) {
+			t.Errorf("got %v, want MISUSE", err)
+		}
+	})
+	t.Run("FCNTL_RESET_CACHE", func(t *testing.T) {
+		o, err := db.FileControl("", sqlite3.FCNTL_RESET_CACHE)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if o != nil {
+			t.Errorf("got %v, want nil", o)
+		}
+	})
+
+	t.Run("FCNTL_PERSIST_WAL", func(t *testing.T) {
+		o, err := db.FileControl("", sqlite3.FCNTL_PERSIST_WAL)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if o != false {
+			t.Errorf("got %v, want false", o)
+		}
+
+		o, err = db.FileControl("", sqlite3.FCNTL_PERSIST_WAL, true)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if o != true {
+			t.Errorf("got %v, want true", o)
+		}
+
+		o, err = db.FileControl("", sqlite3.FCNTL_PERSIST_WAL)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if o != true {
+			t.Errorf("got %v, want true", o)
+		}
+	})
+
+	t.Run("FCNTL_CHUNK_SIZE", func(t *testing.T) {
+		o, err := db.FileControl("", sqlite3.FCNTL_CHUNK_SIZE, 1024*1024)
+		if !errors.Is(err, sqlite3.NOTFOUND) {
+			t.Errorf("got %v, want NOTFOUND", err)
+		}
+		if o != nil {
+			t.Errorf("got %v, want nil", o)
+		}
+	})
+
+	t.Run("FCNTL_RESERVE_BYTES", func(t *testing.T) {
+		o, err := db.FileControl("", sqlite3.FCNTL_RESERVE_BYTES, 4)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if o != 0 {
+			t.Errorf("got %v, want 0", o)
+		}
+
+		o, err = db.FileControl("", sqlite3.FCNTL_RESERVE_BYTES)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if o != 4 {
+			t.Errorf("got %v, want 4", o)
+		}
+	})
+
+	t.Run("FCNTL_DATA_VERSION", func(t *testing.T) {
+		o, err := db.FileControl("", sqlite3.FCNTL_DATA_VERSION)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if o != uint32(2) {
+			t.Errorf("got %v, want 2", o)
+		}
+	})
+
+	t.Run("FCNTL_VFS_POINTER", func(t *testing.T) {
+		o, err := db.FileControl("", sqlite3.FCNTL_VFS_POINTER)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if o != vfs.Find("os") {
+			t.Errorf("got %v, want os", o)
+		}
+	})
+
+	t.Run("FCNTL_FILE_POINTER", func(t *testing.T) {
+		o, err := db.FileControl("", sqlite3.FCNTL_FILE_POINTER)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if _, ok := o.(vfs.File); !ok {
+			t.Errorf("got %v, want File", o)
+		}
+	})
+
+	t.Run("FCNTL_JOURNAL_POINTER", func(t *testing.T) {
+		o, err := db.FileControl("", sqlite3.FCNTL_JOURNAL_POINTER)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if o != nil {
+			t.Errorf("got %v, want nil", o)
+		}
+	})
+
+	t.Run("FCNTL_LOCKSTATE", func(t *testing.T) {
+		if !vfs.SupportsFileLocking {
+			t.Skip("skipping without locks")
+		}
+
+		txn, err := db.BeginExclusive()
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer txn.End(&err)
+
+		o, err := db.FileControl("", sqlite3.FCNTL_LOCKSTATE)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if o != vfs.LOCK_EXCLUSIVE {
+			t.Errorf("got %v, want LOCK_EXCLUSIVE", o)
+		}
+	})
+}
+
+func TestConn_Limit(t *testing.T) {
+	t.Parallel()
+
+	db, err := sqlite3.Open(":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	l := db.Limit(sqlite3.LIMIT_COLUMN, -1)
+	if l != 2000 {
+		t.Errorf("got %d, want 2000", l)
+	}
+
+	l = db.Limit(sqlite3.LIMIT_COLUMN, 100)
+	if l != 2000 {
+		t.Errorf("got %d, want 2000", l)
+	}
+
+	l = db.Limit(sqlite3.LIMIT_COLUMN, -1)
+	if l != 100 {
+		t.Errorf("got %d, want 100", l)
+	}
+
+	l = db.Limit(math.MaxUint32, -1)
+	if l != -1 {
+		t.Errorf("got %d, want -1", l)
+	}
+}
+
+func TestConn_SetAuthorizer(t *testing.T) {
+	t.Parallel()
+
+	db, err := sqlite3.Open(":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	err = db.SetAuthorizer(func(action sqlite3.AuthorizerActionCode, name3rd, name4th, schema, nameInner string) sqlite3.AuthorizerReturnCode {
+		if action != sqlite3.AUTH_PRAGMA {
+			t.Errorf("got %v, want PRAGMA", action)
+		}
+		if name3rd != "busy_timeout" {
+			t.Errorf("got %q, want busy_timeout", name3rd)
+		}
+		if name4th != "5000" {
+			t.Errorf("got %q, want 5000", name4th)
+		}
+		if schema != "main" {
+			t.Errorf("got %q, want main", schema)
+		}
+		return sqlite3.AUTH_DENY
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = db.Exec(`PRAGMA main.busy_timeout=5000`)
+	if !errors.Is(err, sqlite3.AUTH) {
+		t.Errorf("got %v, want sqlite3.AUTH", err)
+	}
+}
+
+func TestConn_Trace(t *testing.T) {
+	t.Parallel()
+
+	db, err := sqlite3.Open(":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	rows := 0
+	closed := false
+	err = db.Trace(math.MaxUint32, func(evt sqlite3.TraceEvent, a1 any, a2 any) error {
+		switch evt {
+		case sqlite3.TRACE_CLOSE:
+			closed = true
+			_ = a1.(*sqlite3.Conn)
+			return db.Exec(`PRAGMA optimize`)
+		case sqlite3.TRACE_STMT:
+			stmt := a1.(*sqlite3.Stmt)
+			if sql := a2.(string); sql != stmt.SQL() {
+				t.Errorf("got %q, want %q", sql, stmt.SQL())
+			}
+			if sql := stmt.ExpandedSQL(); sql != `SELECT 1` {
+				t.Errorf("got %q", sql)
+			}
+		case sqlite3.TRACE_PROFILE:
+			_ = a1.(*sqlite3.Stmt)
+			if ns := a2.(int64); ns < 0 {
+				t.Errorf("got %d", ns)
+			}
+		case sqlite3.TRACE_ROW:
+			_ = a1.(*sqlite3.Stmt)
+			if a2 != nil {
+				t.Errorf("got %v", a2)
+			}
+			rows++
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	stmt, _, err := db.Prepare(`SELECT ?`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = stmt.BindInt(1, 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = stmt.Exec()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = stmt.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rows != 1 {
+		t.Error("want 1")
+	}
+
+	err = db.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !closed {
+		t.Error("want closed")
+	}
+}
+
+func TestConn_AutoVacuumPages(t *testing.T) {
+	t.Parallel()
+	tmp := memdb.TestDB(t, url.Values{
+		"_pragma": {"auto_vacuum(full)"},
+	})
+
+	db, err := sqlite3.Open(tmp)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	err = db.AutoVacuumPages(func(schema string, dbPages, freePages, bytesPerPage uint) uint {
+		return freePages
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = db.Exec(`CREATE TABLE test (col)`)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = db.Exec(`INSERT INTO test VALUES (zeroblob(1024*1024))`)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = db.Exec(`DROP TABLE test`)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestConn_memoryLimit(t *testing.T) {
+	t.Parallel()
+
+	db, err := sqlite3.Open(":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	n := db.HardHeapLimit(-1)
+	if n != 0 {
+		t.Fatal("want zero")
+	}
+
+	const limit = 64 * 1024 * 1024
+
+	n = db.SoftHeapLimit(limit)
+	if n != 0 {
+		t.Fatal("want zero")
+	}
+
+	n = db.SoftHeapLimit(-1)
+	if n != limit {
+		t.Fatal("want", limit)
+	}
+}

tests/conn_test.go

@@ -3,8 +3,6 @@ package tests
 import (
 	"context"
 	"errors"
-	"math"
-	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
@@ -13,8 +11,6 @@ import (
 	"github.com/ncruces/go-sqlite3"
 	_ "github.com/ncruces/go-sqlite3/embed"
 	_ "github.com/ncruces/go-sqlite3/internal/testcfg"
-	"github.com/ncruces/go-sqlite3/vfs"
-	"github.com/ncruces/go-sqlite3/vfs/memdb"
 	_ "github.com/ncruces/go-sqlite3/vfs/memdb"
 )
 
@@ -265,358 +261,6 @@ func TestConn_Prepare_invalid(t *testing.T) {
 	}
 }
 
-func TestConn_Config(t *testing.T) {
-	t.Parallel()
-
-	db, err := sqlite3.Open(":memory:")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer db.Close()
-
-	o, err := db.Config(sqlite3.DBCONFIG_DEFENSIVE)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if o != false {
-		t.Error("want false")
-	}
-
-	o, err = db.Config(sqlite3.DBCONFIG_DEFENSIVE, true)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if o != true {
-		t.Error("want true")
-	}
-
-	o, err = db.Config(sqlite3.DBCONFIG_DEFENSIVE)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if o != true {
-		t.Error("want true")
-	}
-
-	o, err = db.Config(sqlite3.DBCONFIG_DEFENSIVE, false)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if o != false {
-		t.Error("want false")
-	}
-
-	o, err = db.Config(sqlite3.DBCONFIG_DEFENSIVE)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if o != false {
-		t.Error("want false")
-	}
-}
-
-func TestConn_ConfigLog(t *testing.T) {
-	t.Parallel()
-
-	db, err := sqlite3.Open(":memory:")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer db.Close()
-
-	var code sqlite3.ExtendedErrorCode
-	err = db.ConfigLog(func(c sqlite3.ExtendedErrorCode, msg string) {
-		t.Log(msg)
-		code = c
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	db.Prepare(`SELECT * FRM sqlite_schema`)
-
-	if code != sqlite3.ExtendedErrorCode(sqlite3.ERROR) {
-		t.Error("want sqlite3.ERROR")
-	}
-}
-
-func TestConn_FileControl(t *testing.T) {
-	t.Parallel()
-
-	file := filepath.Join(t.TempDir(), "test.db")
-	db, err := sqlite3.Open(file)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer db.Close()
-
-	t.Run("MISUSE", func(t *testing.T) {
-		_, err := db.FileControl("main", 0)
-		if !errors.Is(err, sqlite3.MISUSE) {
-			t.Errorf("got %v, want MISUSE", err)
-		}
-	})
-	t.Run("FCNTL_RESET_CACHE", func(t *testing.T) {
-		o, err := db.FileControl("", sqlite3.FCNTL_RESET_CACHE)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if o != nil {
-			t.Errorf("got %v, want nil", o)
-		}
-	})
-
-	t.Run("FCNTL_PERSIST_WAL", func(t *testing.T) {
-		o, err := db.FileControl("", sqlite3.FCNTL_PERSIST_WAL)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if o != false {
-			t.Errorf("got %v, want false", o)
-		}
-
-		o, err = db.FileControl("", sqlite3.FCNTL_PERSIST_WAL, true)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if o != true {
-			t.Errorf("got %v, want true", o)
-		}
-
-		o, err = db.FileControl("", sqlite3.FCNTL_PERSIST_WAL)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if o != true {
-			t.Errorf("got %v, want true", o)
-		}
-	})
-
-	t.Run("FCNTL_CHUNK_SIZE", func(t *testing.T) {
-		o, err := db.FileControl("", sqlite3.FCNTL_CHUNK_SIZE, 1024*1024)
-		if !errors.Is(err, sqlite3.NOTFOUND) {
-			t.Errorf("got %v, want NOTFOUND", err)
-		}
-		if o != nil {
-			t.Errorf("got %v, want nil", o)
-		}
-	})
-
-	t.Run("FCNTL_RESERVE_BYTES", func(t *testing.T) {
-		o, err := db.FileControl("", sqlite3.FCNTL_RESERVE_BYTES, 4)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if o != 0 {
-			t.Errorf("got %v, want 0", o)
-		}
-
-		o, err = db.FileControl("", sqlite3.FCNTL_RESERVE_BYTES)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if o != 4 {
-			t.Errorf("got %v, want 4", o)
-		}
-	})
-
-	t.Run("FCNTL_DATA_VERSION", func(t *testing.T) {
-		o, err := db.FileControl("", sqlite3.FCNTL_DATA_VERSION)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if o != uint32(2) {
-			t.Errorf("got %v, want 2", o)
-		}
-	})
-
-	t.Run("FCNTL_VFS_POINTER", func(t *testing.T) {
-		o, err := db.FileControl("", sqlite3.FCNTL_VFS_POINTER)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if o != vfs.Find("os") {
-			t.Errorf("got %v, want os", o)
-		}
-	})
-
-	t.Run("FCNTL_FILE_POINTER", func(t *testing.T) {
-		o, err := db.FileControl("", sqlite3.FCNTL_FILE_POINTER)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if _, ok := o.(vfs.File); !ok {
-			t.Errorf("got %v, want File", o)
-		}
-	})
-
-	t.Run("FCNTL_JOURNAL_POINTER", func(t *testing.T) {
-		o, err := db.FileControl("", sqlite3.FCNTL_JOURNAL_POINTER)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if o != nil {
-			t.Errorf("got %v, want nil", o)
-		}
-	})
-
-	t.Run("FCNTL_LOCKSTATE", func(t *testing.T) {
-		if !vfs.SupportsFileLocking {
-			t.Skip("skipping without locks")
-		}
-
-		txn, err := db.BeginExclusive()
-		if err != nil {
-			t.Fatal(err)
-		}
-		defer txn.End(&err)
-
-		o, err := db.FileControl("", sqlite3.FCNTL_LOCKSTATE)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if o != vfs.LOCK_EXCLUSIVE {
-			t.Errorf("got %v, want LOCK_EXCLUSIVE", o)
-		}
-	})
-}
-
-func TestConn_Limit(t *testing.T) {
-	t.Parallel()
-
-	db, err := sqlite3.Open(":memory:")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer db.Close()
-
-	l := db.Limit(sqlite3.LIMIT_COLUMN, -1)
-	if l != 2000 {
-		t.Errorf("got %d, want 2000", l)
-	}
-
-	l = db.Limit(sqlite3.LIMIT_COLUMN, 100)
-	if l != 2000 {
-		t.Errorf("got %d, want 2000", l)
-	}
-
-	l = db.Limit(sqlite3.LIMIT_COLUMN, -1)
-	if l != 100 {
-		t.Errorf("got %d, want 100", l)
-	}
-
-	l = db.Limit(math.MaxUint32, -1)
-	if l != -1 {
-		t.Errorf("got %d, want -1", l)
-	}
-}
-
-func TestConn_SetAuthorizer(t *testing.T) {
-	t.Parallel()
-
-	db, err := sqlite3.Open(":memory:")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer db.Close()
-
-	err = db.SetAuthorizer(func(action sqlite3.AuthorizerActionCode, name3rd, name4th, schema, nameInner string) sqlite3.AuthorizerReturnCode {
-		if action != sqlite3.AUTH_PRAGMA {
-			t.Errorf("got %v, want PRAGMA", action)
-		}
-		if name3rd != "busy_timeout" {
-			t.Errorf("got %q, want busy_timeout", name3rd)
-		}
-		if name4th != "5000" {
-			t.Errorf("got %q, want 5000", name4th)
-		}
-		if schema != "main" {
-			t.Errorf("got %q, want main", schema)
-		}
-		return sqlite3.AUTH_DENY
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = db.Exec(`PRAGMA main.busy_timeout=5000`)
-	if !errors.Is(err, sqlite3.AUTH) {
-		t.Errorf("got %v, want sqlite3.AUTH", err)
-	}
-}
-
-func TestConn_Trace(t *testing.T) {
-	t.Parallel()
-
-	db, err := sqlite3.Open(":memory:")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer db.Close()
-
-	rows := 0
-	closed := false
-	err = db.Trace(math.MaxUint32, func(evt sqlite3.TraceEvent, a1 any, a2 any) error {
-		switch evt {
-		case sqlite3.TRACE_CLOSE:
-			closed = true
-			_ = a1.(*sqlite3.Conn)
-			return db.Exec(`PRAGMA optimize`)
-		case sqlite3.TRACE_STMT:
-			stmt := a1.(*sqlite3.Stmt)
-			if sql := a2.(string); sql != stmt.SQL() {
-				t.Errorf("got %q, want %q", sql, stmt.SQL())
-			}
-			if sql := stmt.ExpandedSQL(); sql != `SELECT 1` {
-				t.Errorf("got %q", sql)
-			}
-		case sqlite3.TRACE_PROFILE:
-			_ = a1.(*sqlite3.Stmt)
-			if ns := a2.(int64); ns < 0 {
-				t.Errorf("got %d", ns)
-			}
-		case sqlite3.TRACE_ROW:
-			_ = a1.(*sqlite3.Stmt)
-			if a2 != nil {
-				t.Errorf("got %v", a2)
-			}
-			rows++
-		}
-		return nil
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	stmt, _, err := db.Prepare(`SELECT ?`)
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = stmt.BindInt(1, 1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = stmt.Exec()
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = stmt.Close()
-	if err != nil {
-		t.Fatal(err)
-	}
-	if rows != 1 {
-		t.Error("want 1")
-	}
-
-	err = db.Close()
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !closed {
-		t.Error("want closed")
-	}
-}
-
 func TestConn_ReleaseMemory(t *testing.T) {
 	t.Parallel()
 
@@ -727,41 +371,6 @@ func TestConn_DBName(t *testing.T) {
 	}
 }
 
-func TestConn_AutoVacuumPages(t *testing.T) {
-	t.Parallel()
-	tmp := memdb.TestDB(t, url.Values{
-		"_pragma": {"auto_vacuum(full)"},
-	})
-
-	db, err := sqlite3.Open(tmp)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer db.Close()
-
-	err = db.AutoVacuumPages(func(schema string, dbPages, freePages, bytesPerPage uint) uint {
-		return freePages
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = db.Exec(`CREATE TABLE test (col)`)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = db.Exec(`INSERT INTO test VALUES (zeroblob(1024*1024))`)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = db.Exec(`DROP TABLE test`)
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
 func TestConn_Status(t *testing.T) {
 	t.Parallel()
 

tests/driver_test.go

@@ -33,7 +33,7 @@ func TestDriver(t *testing.T) {
 	defer conn.Close()
 
 	res, err := conn.ExecContext(ctx,
-		`CREATE TABLE users (id INT, name VARCHAR(10))`)
+		`CREATE TABLE users (id INTEGER PRIMARY KEY NOT NULL, name VARCHAR(10))`)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -82,11 +82,17 @@ func TestDriver(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if got := typs[0].DatabaseTypeName(); got != "INT" {
-		t.Errorf("got %s, want INT", got)
+	if got := typs[0].DatabaseTypeName(); got != "INTEGER" {
+		t.Errorf("got %s, want INTEGER", got)
 	}
 	if got := typs[1].DatabaseTypeName(); got != "VARCHAR" {
-		t.Errorf("got %s, want INT", got)
+		t.Errorf("got %s, want VARCHAR", got)
+	}
+	if got, ok := typs[0].Nullable(); got || !ok {
+		t.Errorf("got %v/%v, want false/true", got, ok)
+	}
+	if got, ok := typs[1].Nullable(); !got || ok {
+		t.Errorf("got %v/%v, want true/false", got, ok)
 	}
 
 	row := 0

tests/parallel/parallel_test.go

@@ -131,7 +131,12 @@ func TestMultiProcess(t *testing.T) {
 		"&_pragma=journal_mode(truncate)" +
 		"&_pragma=synchronous(off)"
 
-	cmd := exec.Command(os.Args[0], append(os.Args[1:], "-test.v", "-test.run=TestChildProcess")...)
+	exe, err := os.Executable()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cmd := exec.Command(exe, append(os.Args[1:], "-test.v", "-test.run=TestChildProcess")...)
 	out, err := cmd.StdoutPipe()
 	if err != nil {
 		t.Fatal(err)

util/vtabutil/arg.go

@@ -26,6 +26,8 @@ func Unquote(val string) string {
 		return val
 	case '"':
 		old, new = `""`, `"`
+	case '`':
+		old, new = "``", "`"
 	case '\'':
 		old, new = `''`, `'`
 	}

vfs/README.md

@@ -46,7 +46,7 @@ to check if your build supports file locking.
 
 ### Write-Ahead Logging
 
-On 64-bit Unix, this module uses `mmap` to implement
+On 64-bit little-endian Unix, this module uses `mmap` to implement
 [shared-memory for the WAL-index](https://sqlite.org/wal.html#implementation_of_shared_memory_for_the_wal_index),
 like SQLite.
 

vfs/lock.go

@@ -75,20 +75,7 @@ func (f *vfsFile) Lock(lock LockLevel) error {
 		if f.lock <= LOCK_NONE || f.lock >= LOCK_EXCLUSIVE {
 			panic(util.AssertErr())
 		}
-		reserved := f.lock == LOCK_RESERVED
-		// A PENDING lock is needed before acquiring an EXCLUSIVE lock.
-		if f.lock < LOCK_PENDING {
-			// If we're already RESERVED, we can block indefinitely,
-			// since only new readers may briefly hold the PENDING lock.
-			if rc := osGetPendingLock(f.File, reserved /* block */); rc != _OK {
-				return rc
-			}
-			f.lock = LOCK_PENDING
-		}
-		// We already have PENDING, so we're just waiting for readers to leave.
-		// If we were RESERVED, we can wait for a little while, before invoking
-		// the busy handler; we will only do this once.
-		if rc := osGetExclusiveLock(f.File, reserved /* wait */); rc != _OK {
+		if rc := osGetExclusiveLock(f.File, &f.lock); rc != _OK {
 			return rc
 		}
 		f.lock = LOCK_EXCLUSIVE

vfs/os_bsd.go

@@ -4,31 +4,15 @@ package vfs
 
 import (
 	"os"
-	"time"
 
 	"golang.org/x/sys/unix"
 )
 
-func osUnlock(file *os.File, start, len int64) _ErrorCode {
-	if start == 0 && len == 0 {
-		err := unix.Flock(int(file.Fd()), unix.LOCK_UN)
-		if err != nil {
-			return _IOERR_UNLOCK
-		}
-	}
-	return _OK
-}
-
-func osLock(file *os.File, how int, def _ErrorCode) _ErrorCode {
-	err := unix.Flock(int(file.Fd()), how)
-	return osLockErrorCode(err, def)
-}
-
-func osReadLock(file *os.File, _ /*start*/, _ /*len*/ int64, _ /*timeout*/ time.Duration) _ErrorCode {
+func osGetSharedLock(file *os.File) _ErrorCode {
 	return osLock(file, unix.LOCK_SH|unix.LOCK_NB, _IOERR_RDLOCK)
 }
 
-func osWriteLock(file *os.File, _ /*start*/, _ /*len*/ int64, _ /*timeout*/ time.Duration) _ErrorCode {
+func osGetReservedLock(file *os.File) _ErrorCode {
 	rc := osLock(file, unix.LOCK_EX|unix.LOCK_NB, _IOERR_LOCK)
 	if rc == _BUSY {
 		// The documentation states the lock is upgraded by releasing the previous lock,
@@ -38,3 +22,40 @@ func osWriteLock(file *os.File, _ /*start*/, _ /*len*/ int64, _ /*timeout*/ time
 	}
 	return rc
 }
+
+func osGetExclusiveLock(file *os.File, state *LockLevel) _ErrorCode {
+	if *state >= LOCK_RESERVED {
+		return _OK
+	}
+	return osGetReservedLock(file)
+}
+
+func osDowngradeLock(file *os.File, _ LockLevel) _ErrorCode {
+	rc := osLock(file, unix.LOCK_SH|unix.LOCK_NB, _IOERR_RDLOCK)
+	if rc == _BUSY {
+		// The documentation states the lock is upgraded by releasing the previous lock,
+		// then acquiring the new lock.
+		// This is a race, so return IOERR_RDLOCK to ensure the transaction is aborted.
+		return _IOERR_RDLOCK
+	}
+	return _OK
+}
+
+func osReleaseLock(file *os.File, _ LockLevel) _ErrorCode {
+	err := unix.Flock(int(file.Fd()), unix.LOCK_UN)
+	if err != nil {
+		return _IOERR_UNLOCK
+	}
+	return _OK
+}
+
+func osCheckReservedLock(file *os.File) (bool, _ErrorCode) {
+	// Test the RESERVED lock.
+	lock, rc := osTestLock(file, _RESERVED_BYTE, 1)
+	return lock == unix.F_WRLCK, rc
+}
+
+func osLock(file *os.File, how int, def _ErrorCode) _ErrorCode {
+	err := unix.Flock(int(file.Fd()), how)
+	return osLockErrorCode(err, def)
+}

vfs/os_f2fs_linux.go

@@ -1,4 +1,4 @@
-//go:build (amd64 || arm64 || riscv64) && !sqlite3_nosys
+//go:build (amd64 || arm64 || riscv64 || ppc64le) && !sqlite3_nosys
 
 package vfs
 
@@ -9,6 +9,7 @@ import (
 )
 
 const (
+	// https://godbolt.org/z/1PcK5vea3
 	_F2FS_IOC_START_ATOMIC_WRITE  = 62721
 	_F2FS_IOC_COMMIT_ATOMIC_WRITE = 62722
 	_F2FS_IOC_ABORT_ATOMIC_WRITE  = 62725

vfs/os_linux.go

@@ -53,10 +53,11 @@ func osLock(file *os.File, typ int16, start, len int64, timeout time.Duration, d
 			if errno, _ := err.(unix.Errno); errno != unix.EAGAIN {
 				break
 			}
-			if timeout < time.Since(before) {
+			if time.Since(before) > timeout {
 				break
 			}
-			time.Sleep(time.Duration(rand.Int63n(int64(time.Millisecond))))
+			const sleepIncrement = 1024*1024 - 1 // power of two, ~1ms
+			time.Sleep(time.Duration(rand.Int63() & sleepIncrement))
 		}
 	}
 	return osLockErrorCode(err, def)

vfs/os_ofd.go

@@ -0,0 +1,59 @@
+//go:build (linux || darwin) && !(sqlite3_flock || sqlite3_nosys)
+
+package vfs
+
+import (
+	"os"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+func osGetSharedLock(file *os.File) _ErrorCode {
+	// Test the PENDING lock before acquiring a new SHARED lock.
+	if lock, _ := osTestLock(file, _PENDING_BYTE, 1); lock == unix.F_WRLCK {
+		return _BUSY
+	}
+	// Acquire the SHARED lock.
+	return osReadLock(file, _SHARED_FIRST, _SHARED_SIZE, 0)
+}
+
+func osGetReservedLock(file *os.File) _ErrorCode {
+	// Acquire the RESERVED lock.
+	return osWriteLock(file, _RESERVED_BYTE, 1, 0)
+}
+
+func osGetExclusiveLock(file *os.File, state *LockLevel) _ErrorCode {
+	if *state == LOCK_RESERVED {
+		// A PENDING lock is needed before acquiring an EXCLUSIVE lock.
+		if rc := osWriteLock(file, _PENDING_BYTE, 1, -1); rc != _OK {
+			return rc
+		}
+		*state = LOCK_PENDING
+	}
+	// Acquire the EXCLUSIVE lock.
+	return osWriteLock(file, _SHARED_FIRST, _SHARED_SIZE, time.Millisecond)
+}
+
+func osDowngradeLock(file *os.File, state LockLevel) _ErrorCode {
+	if state >= LOCK_EXCLUSIVE {
+		// Downgrade to a SHARED lock.
+		if rc := osReadLock(file, _SHARED_FIRST, _SHARED_SIZE, 0); rc != _OK {
+			// notest // this should never happen
+			return _IOERR_RDLOCK
+		}
+	}
+	// Release the PENDING and RESERVED locks.
+	return osUnlock(file, _PENDING_BYTE, 2)
+}
+
+func osReleaseLock(file *os.File, _ LockLevel) _ErrorCode {
+	// Release all locks.
+	return osUnlock(file, 0, 0)
+}
+
+func osCheckReservedLock(file *os.File) (bool, _ErrorCode) {
+	// Test the RESERVED lock.
+	lock, rc := osTestLock(file, _RESERVED_BYTE, 1)
+	return lock == unix.F_WRLCK, rc
+}

vfs/os_std_atomic.go

@@ -1,4 +1,4 @@
-//go:build !linux || !(amd64 || arm64 || riscv64) || sqlite3_nosys
+//go:build !linux || !(amd64 || arm64 || riscv64 || ppc64le) || sqlite3_nosys
 
 package vfs
 

vfs/os_unix.go

@@ -31,3 +31,41 @@ func osSetMode(file *os.File, modeof string) error {
 	}
 	return nil
 }
+
+func osTestLock(file *os.File, start, len int64) (int16, _ErrorCode) {
+	lock := unix.Flock_t{
+		Type:  unix.F_WRLCK,
+		Start: start,
+		Len:   len,
+	}
+	if unix.FcntlFlock(file.Fd(), unix.F_GETLK, &lock) != nil {
+		return 0, _IOERR_CHECKRESERVEDLOCK
+	}
+	return lock.Type, _OK
+}
+
+func osLockErrorCode(err error, def _ErrorCode) _ErrorCode {
+	if err == nil {
+		return _OK
+	}
+	if errno, ok := err.(unix.Errno); ok {
+		switch errno {
+		case
+			unix.EACCES,
+			unix.EAGAIN,
+			unix.EBUSY,
+			unix.EINTR,
+			unix.ENOLCK,
+			unix.EDEADLK,
+			unix.ETIMEDOUT:
+			return _BUSY
+		case unix.EPERM:
+			return _PERM
+		}
+		// notest // usually EWOULDBLOCK == EAGAIN
+		if errno == unix.EWOULDBLOCK && unix.EWOULDBLOCK != unix.EAGAIN {
+			return _BUSY
+		}
+	}
+	return def
+}

vfs/os_unix_lock.go

@@ -1,108 +0,0 @@
-//go:build (linux || darwin || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && !sqlite3_nosys
-
-package vfs
-
-import (
-	"os"
-	"time"
-
-	"golang.org/x/sys/unix"
-)
-
-func osGetSharedLock(file *os.File) _ErrorCode {
-	// Test the PENDING lock before acquiring a new SHARED lock.
-	if lock, _ := osGetLock(file, _PENDING_BYTE, 1); lock == unix.F_WRLCK {
-		return _BUSY
-	}
-	// Acquire the SHARED lock.
-	return osReadLock(file, _SHARED_FIRST, _SHARED_SIZE, 0)
-}
-
-func osGetReservedLock(file *os.File) _ErrorCode {
-	// Acquire the RESERVED lock.
-	return osWriteLock(file, _RESERVED_BYTE, 1, 0)
-}
-
-func osGetPendingLock(file *os.File, block bool) _ErrorCode {
-	var timeout time.Duration
-	if block {
-		timeout = -1
-	}
-	// Acquire the PENDING lock.
-	return osWriteLock(file, _PENDING_BYTE, 1, timeout)
-}
-
-func osGetExclusiveLock(file *os.File, wait bool) _ErrorCode {
-	var timeout time.Duration
-	if wait {
-		timeout = time.Millisecond
-	}
-	// Acquire the EXCLUSIVE lock.
-	return osWriteLock(file, _SHARED_FIRST, _SHARED_SIZE, timeout)
-}
-
-func osDowngradeLock(file *os.File, state LockLevel) _ErrorCode {
-	if state >= LOCK_EXCLUSIVE {
-		// Downgrade to a SHARED lock.
-		if rc := osReadLock(file, _SHARED_FIRST, _SHARED_SIZE, 0); rc != _OK {
-			// In theory, the downgrade to a SHARED cannot fail because another
-			// process is holding an incompatible lock. If it does, this
-			// indicates that the other process is not following the locking
-			// protocol. If this happens, return IOERR_RDLOCK. Returning
-			// BUSY would confuse the upper layer.
-			// notest
-			return _IOERR_RDLOCK
-		}
-	}
-	// Release the PENDING and RESERVED locks.
-	return osUnlock(file, _PENDING_BYTE, 2)
-}
-
-func osReleaseLock(file *os.File, _ LockLevel) _ErrorCode {
-	// Release all locks.
-	return osUnlock(file, 0, 0)
-}
-
-func osCheckReservedLock(file *os.File) (bool, _ErrorCode) {
-	// Test the RESERVED lock.
-	lock, rc := osGetLock(file, _RESERVED_BYTE, 1)
-	return lock == unix.F_WRLCK, rc
-}
-
-func osGetLock(file *os.File, start, len int64) (int16, _ErrorCode) {
-	lock := unix.Flock_t{
-		Type:  unix.F_WRLCK,
-		Start: start,
-		Len:   len,
-	}
-	if unix.FcntlFlock(file.Fd(), unix.F_GETLK, &lock) != nil {
-		return 0, _IOERR_CHECKRESERVEDLOCK
-	}
-	return lock.Type, _OK
-}
-
-func osLockErrorCode(err error, def _ErrorCode) _ErrorCode {
-	if err == nil {
-		return _OK
-	}
-	if errno, ok := err.(unix.Errno); ok {
-		switch errno {
-		case
-			unix.EACCES,
-			unix.EAGAIN,
-			unix.EBUSY,
-			unix.EINTR,
-			unix.ENOLCK,
-			unix.EDEADLK,
-			unix.ETIMEDOUT:
-			return _BUSY
-		case unix.EPERM:
-			return _PERM
-		}
-		// notest // usually EWOULDBLOCK == EAGAIN
-		if errno == unix.EWOULDBLOCK && unix.EWOULDBLOCK != unix.EAGAIN {
-			return _BUSY
-		}
-	}
-	return def
-}

vfs/os_windows.go

@@ -28,27 +28,25 @@ func osGetReservedLock(file *os.File) _ErrorCode {
 	return osWriteLock(file, _RESERVED_BYTE, 1, 0)
 }
 
-func osGetPendingLock(file *os.File, block bool) _ErrorCode {
-	var timeout time.Duration
-	if block {
-		timeout = -1
-	}
-
-	// Acquire the PENDING lock.
-	return osWriteLock(file, _PENDING_BYTE, 1, timeout)
-}
-
-func osGetExclusiveLock(file *os.File, wait bool) _ErrorCode {
-	var timeout time.Duration
-	if wait {
-		timeout = time.Millisecond
+func osGetExclusiveLock(file *os.File, state *LockLevel) _ErrorCode {
+	// A PENDING lock is needed before releasing the SHARED lock.
+	if *state < LOCK_PENDING {
+		// If we were RESERVED, we can block indefinitely.
+		var timeout time.Duration
+		if *state == LOCK_RESERVED {
+			timeout = -1
+		}
+		if rc := osWriteLock(file, _PENDING_BYTE, 1, timeout); rc != _OK {
+			return rc
+		}
+		*state = LOCK_PENDING
 	}
 
 	// Release the SHARED lock.
 	osUnlock(file, _SHARED_FIRST, _SHARED_SIZE)
 
 	// Acquire the EXCLUSIVE lock.
-	rc := osWriteLock(file, _SHARED_FIRST, _SHARED_SIZE, timeout)
+	rc := osWriteLock(file, _SHARED_FIRST, _SHARED_SIZE, time.Millisecond)
 
 	if rc != _OK {
 		// Reacquire the SHARED lock.
@@ -64,9 +62,7 @@ func osDowngradeLock(file *os.File, state LockLevel) _ErrorCode {
 
 		// Reacquire the SHARED lock.
 		if rc := osReadLock(file, _SHARED_FIRST, _SHARED_SIZE, 0); rc != _OK {
-			// This should never happen.
-			// We should always be able to reacquire the read lock.
-			// notest
+			// notest // this should never happen
 			return _IOERR_RDLOCK
 		}
 	}
@@ -134,10 +130,11 @@ func osLock(file *os.File, flags, start, len uint32, timeout time.Duration, def
 			if errno, _ := err.(windows.Errno); errno != windows.ERROR_LOCK_VIOLATION {
 				break
 			}
-			if timeout < time.Since(before) {
+			if time.Since(before) > timeout {
 				break
 			}
-			time.Sleep(time.Duration(rand.Int63n(int64(time.Millisecond))))
+			const sleepIncrement = 1024*1024 - 1 // power of two, ~1ms
+			time.Sleep(time.Duration(rand.Int63() & sleepIncrement))
 		}
 	}
 	return osLockErrorCode(err, def)

vfs/shm.go

@@ -1,4 +1,4 @@
-//go:build (darwin || linux) && (amd64 || arm64 || riscv64) && !(sqlite3_flock || sqlite3_noshm || sqlite3_nosys)
+//go:build (darwin || linux) && (amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_flock || sqlite3_noshm || sqlite3_nosys)
 
 package vfs
 
@@ -6,6 +6,7 @@ import (
 	"context"
 	"io"
 	"os"
+	"time"
 
 	"github.com/ncruces/go-sqlite3/internal/util"
 	"github.com/tetratelabs/wazero/api"
@@ -49,6 +50,7 @@ type vfsShm struct {
 	path     string
 	regions  []*util.MappedRegion
 	readOnly bool
+	blocking bool
 }
 
 func (s *vfsShm) shmOpen() _ErrorCode {
@@ -68,7 +70,7 @@ func (s *vfsShm) shmOpen() _ErrorCode {
 	}
 
 	// Dead man's switch.
-	if lock, rc := osGetLock(s.File, _SHM_DMS, 1); rc != _OK {
+	if lock, rc := osTestLock(s.File, _SHM_DMS, 1); rc != _OK {
 		return _IOERR_LOCK
 	} else if lock == unix.F_WRLCK {
 		return _BUSY
@@ -76,6 +78,13 @@ func (s *vfsShm) shmOpen() _ErrorCode {
 		if s.readOnly {
 			return _READONLY_CANTINIT
 		}
+		// Do not use a blocking lock here.
+		// If the lock cannot be obtained immediately,
+		// it means some other connection is truncating the file.
+		// And after it has done so, it will not release its lock,
+		// but only downgrade it to a shared lock.
+		// So no point in blocking here.
+		// The call below to obtain the shared DMS lock may use a blocking lock.
 		if rc := osWriteLock(s.File, _SHM_DMS, 1, 0); rc != _OK {
 			return rc
 		}
@@ -83,7 +92,7 @@ func (s *vfsShm) shmOpen() _ErrorCode {
 			return _IOERR_SHMOPEN
 		}
 	}
-	if rc := osReadLock(s.File, _SHM_DMS, 1, 0); rc != _OK {
+	if rc := osReadLock(s.File, _SHM_DMS, 1, time.Millisecond); rc != _OK {
 		return rc
 	}
 	return _OK
@@ -150,13 +159,18 @@ func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) _ErrorCode {
 		panic(util.AssertErr())
 	}
 
+	var timeout time.Duration
+	if s.blocking {
+		timeout = time.Millisecond
+	}
+
 	switch {
 	case flags&_SHM_UNLOCK != 0:
 		return osUnlock(s.File, _SHM_BASE+int64(offset), int64(n))
 	case flags&_SHM_SHARED != 0:
-		return osReadLock(s.File, _SHM_BASE+int64(offset), int64(n), 0)
+		return osReadLock(s.File, _SHM_BASE+int64(offset), int64(n), timeout)
 	case flags&_SHM_EXCLUSIVE != 0:
-		return osWriteLock(s.File, _SHM_BASE+int64(offset), int64(n), 0)
+		return osWriteLock(s.File, _SHM_BASE+int64(offset), int64(n), timeout)
 	default:
 		panic(util.AssertErr())
 	}
@@ -181,3 +195,7 @@ func (s *vfsShm) shmUnmap(delete bool) {
 	s.Close()
 	s.File = nil
 }
+
+func (s *vfsShm) shmEnableBlocking(block bool) {
+	s.blocking = block
+}

vfs/shm_bsd.go

@@ -1,4 +1,4 @@
-//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (amd64 || arm64 || riscv64) && !(sqlite3_noshm || sqlite3_nosys)
+//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && (amd64 || arm64 || riscv64 || ppc64le) && !(sqlite3_noshm || sqlite3_nosys)
 
 package vfs
 

vfs/shm_other.go

@@ -1,4 +1,4 @@
-//go:build !(darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || !(amd64 || arm64 || riscv64) || sqlite3_noshm || sqlite3_nosys
+//go:build !(darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || !(amd64 || arm64 || riscv64 || ppc64le) || sqlite3_noshm || sqlite3_nosys
 
 package vfs
 

vfs/tests/mptest/testdata/mptest.wasm.bz2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4d58c92d45fb60dc2eea461b0e7c1d512cb1dd00deafdfaab3e24b943f714f69
-size 475960
+oid sha256:4b2ca8d3b914990e5c3486e039a6a4160b36c8d6db6c95e2c6c6619d69d26e1d
+size 476258

vfs/tests/speedtest1/testdata/speedtest1.wasm.bz2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:90eb053e2a17bd73d8fb17f82c60e595b71018a7880c7b04d7f4b6aae187f5a5
-size 489132
+oid sha256:259a3d0311302a0ea1d577a8209425f0dbf1ec3a77e214dbafa0e2c1a30723d3
+size 489479

vfs/vfs.go

@@ -243,6 +243,15 @@ func vfsFileControl(ctx context.Context, mod api.Module, pFile uint32, op _Fcntl
 			return _OK
 		}
 
+	case _FCNTL_LOCK_TIMEOUT:
+		if file, ok := file.(FileSharedMemory); ok {
+			if iface, ok := file.SharedMemory().(interface{ shmEnableBlocking(bool) }); ok {
+				if i := util.ReadUint32(mod, pArg); i == 0 || i == 1 {
+					iface.shmEnableBlocking(i != 0)
+				}
+			}
+		}
+
 	case _FCNTL_PERSIST_WAL:
 		if file, ok := file.(FilePersistentWAL); ok {
 			if i := util.ReadUint32(mod, pArg); int32(i) >= 0 {
@@ -347,7 +356,7 @@ func vfsFileControl(ctx context.Context, mod api.Module, pFile uint32, op _Fcntl
 				out = err.Error()
 			}
 			if out != "" {
-				fn := mod.ExportedFunction("malloc")
+				fn := mod.ExportedFunction("sqlite3_malloc64")
 				stack := [...]uint64{uint64(len(out) + 1)}
 				if err := fn.CallWithStack(ctx, stack[:]); err != nil {
 					panic(err)