wasi-sdk-21.

Mozilla is just wrong.
https://searchfox.org/mozilla-central/source/xpcom/glue/FileUtils.cpp

.github/dependabot.yml

@@ -9,3 +9,7 @@ updates:
     directory: "/" # Location of package manifests
     schedule:
       interval: "daily"
+  - package-ecosystem: "github-actions" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "daily"

.github/workflows/bsd.yml

@@ -12,18 +12,13 @@ jobs:
       with:
         lfs: 'true'
 
-    - name: Set up
-      uses: actions/setup-go@v4
-      with:
-        go-version: stable
-
-    - name: Build
-      run: GOOS=freebsd go test -c ./...
-
     - name: Test
       uses: cross-platform-actions/action@v0.21.1
       with:
         operating_system: freebsd
         version: '13.2'
+        memory: 8G
         sync_files: runner-to-vm
-        run: find . -name '*.test' -maxdepth 1 -exec {} -test.v \;
+        run: |
+          sudo pkg install -y go121
+          go121 test -v ./...

.github/workflows/cpu.yml

@@ -13,7 +13,7 @@ jobs:
         lfs: 'true'
 
     - name: Set up
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: stable
 
@@ -29,7 +29,7 @@ jobs:
         lfs: 'true'
 
     - name: Set up
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: stable
 

.github/workflows/cross.yml

@@ -11,7 +11,7 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Set up
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: stable
 

.github/workflows/go.yml

@@ -20,7 +20,7 @@ jobs:
         lfs: 'true'
 
     - name: Set up
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: stable
 
@@ -58,7 +58,6 @@ jobs:
       with:
         chart: true
         amend: true
-        reuse-go: true
       if: |
         github.event_name == 'push' &&
         matrix.os == 'ubuntu-latest'

.github/workflows/repro.sh

@@ -2,13 +2,13 @@
 set -euo pipefail
 
 if [[ "$OSTYPE" == "linux"* ]]; then
-  WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-linux.tar.gz"
+  WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-21/wasi-sdk-21.0-linux.tar.gz"
   BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_116/binaryen-version_116-x86_64-linux.tar.gz"
 elif [[ "$OSTYPE" == "darwin"* ]]; then
-  WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-macos.tar.gz"
+  WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-21/wasi-sdk-21.0-macos.tar.gz"
   BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_116/binaryen-version_116-x86_64-macos.tar.gz"
 elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then
-  WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0.m-mingw.tar.gz"
+  WASI_SDK="https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-21/wasi-sdk-21.0.m-mingw.tar.gz"
   BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/version_116/binaryen-version_116-x86_64-windows.tar.gz"
 fi
 

.github/workflows/repro.yml

@@ -16,7 +16,7 @@ jobs:
         lfs: 'true'
 
     - name: Set up
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: stable
 

README.md

@@ -4,8 +4,14 @@
 [![Go Report](https://goreportcard.com/badge/github.com/ncruces/go-sqlite3)](https://goreportcard.com/report/github.com/ncruces/go-sqlite3)
 [![Go Coverage](https://github.com/ncruces/go-sqlite3/wiki/coverage.svg)](https://github.com/ncruces/go-sqlite3/wiki/Test-coverage-report)
 
-Go module `github.com/ncruces/go-sqlite3` wraps a [WASM](https://webassembly.org/) build of [SQLite](https://sqlite.org/),
-and uses [wazero](https://wazero.io/) to provide `cgo`-free SQLite bindings.
+Go module `github.com/ncruces/go-sqlite3` is `cgo`-free [SQLite](https://sqlite.org/) wrapper.\
+It provides a [`database/sql`](https://pkg.go.dev/database/sql) compatible driver,
+as well as direct access to most of the [C SQLite API](https://sqlite.org/cintro.html).
+
+It wraps a [WASM](https://webassembly.org/) build of SQLite, and uses [wazero](https://wazero.io/) as the runtime.\
+Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ runtime dependencies.
+
+### Packages
 
 - [`github.com/ncruces/go-sqlite3`](https://pkg.go.dev/github.com/ncruces/go-sqlite3)
   wraps the [C SQLite API](https://sqlite.org/cintro.html)
@@ -20,22 +26,24 @@ and uses [wazero](https://wazero.io/) to provide `cgo`-free SQLite bindings.
 - [`github.com/ncruces/go-sqlite3/gormlite`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/gormlite)
   provides a [GORM](https://gorm.io) driver.
 
-### Loadable extensions
+### Extensions
 
-- [`github.com/ncruces/go-sqlite3/ext/array`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/blob)
+- [`github.com/ncruces/go-sqlite3/ext/array`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/array)
   provides the [`array`](https://sqlite.org/carray.html) table-valued function.
-- [`github.com/ncruces/go-sqlite3/ext/blob`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/blob)
+- [`github.com/ncruces/go-sqlite3/ext/blobio`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/blobio)
   simplifies [incremental BLOB I/O](https://sqlite.org/c3ref/blob_open.html).
 - [`github.com/ncruces/go-sqlite3/ext/csv`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/csv)
   reads [comma-separated values](https://sqlite.org/csv.html).
+- [`github.com/ncruces/go-sqlite3/ext/fileio`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/fileio)
+  reads, writes and lists files.
 - [`github.com/ncruces/go-sqlite3/ext/lines`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/lines)
-  reads files [line-by-line](https://github.com/asg017/sqlite-lines).
+  reads data [line-by-line](https://github.com/asg017/sqlite-lines).
 - [`github.com/ncruces/go-sqlite3/ext/pivot`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/pivot)
   creates [pivot tables](https://github.com/jakethaw/pivot_vtab).
 - [`github.com/ncruces/go-sqlite3/ext/statement`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/statement)
-  creates [table-valued functions with SQL](https://github.com/0x09/sqlite-statement-vtab).
+  creates [parameterized views](https://github.com/0x09/sqlite-statement-vtab).
 - [`github.com/ncruces/go-sqlite3/ext/stats`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/stats)
-  provides [statistics functions](https://www.oreilly.com/library/view/sql-in-a/9780596155322/ch04s02.html).
+  provides [statistics](https://www.oreilly.com/library/view/sql-in-a/9780596155322/ch04s02.html) functions.
 - [`github.com/ncruces/go-sqlite3/ext/unicode`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/unicode)
   provides [Unicode aware](https://sqlite.org/src/dir/ext/icu) functions.
 - [`github.com/ncruces/go-sqlite3/vfs/memdb`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/memdb)

conn.go

@@ -103,7 +103,7 @@ func (c *Conn) openDB(filename string, flags OpenFlag) (uint32, error) {
 			return 0, err
 		}
 	}
-
+	c.call("sqlite3_progress_handler_go", uint64(handle), 100)
 	return handle, nil
 }
 
@@ -240,8 +240,8 @@ func (c *Conn) SetInterrupt(ctx context.Context) (old context.Context) {
 		return ctx
 	}
 
-	// An uncompleted SQL statement prevents SQLite from ignoring
-	// an interrupt that comes before any other statements are started.
+	// A busy SQL statement prevents SQLite from ignoring an interrupt
+	// that comes before any other statements are started.
 	if c.pending == nil {
 		c.pending, _, _ = c.Prepare(`SELECT 1 UNION ALL SELECT 2`)
 	} else {
@@ -250,14 +250,11 @@ func (c *Conn) SetInterrupt(ctx context.Context) (old context.Context) {
 
 	old = c.interrupt
 	c.interrupt = ctx
-	// Remove the handler if the context can't be canceled.
 	if ctx == nil || ctx.Done() == nil {
-		c.call("sqlite3_progress_handler_go", uint64(c.handle), 0)
 		return old
 	}
 
 	c.pending.Step()
-	c.call("sqlite3_progress_handler_go", uint64(c.handle), 100)
 	return old
 }
 

const.go

@@ -176,10 +176,11 @@ const (
 type FunctionFlag uint32
 
 const (
-	DETERMINISTIC FunctionFlag = 0x000000800
-	DIRECTONLY    FunctionFlag = 0x000080000
-	SUBTYPE       FunctionFlag = 0x000100000
-	INNOCUOUS     FunctionFlag = 0x000200000
+	DETERMINISTIC  FunctionFlag = 0x000000800
+	DIRECTONLY     FunctionFlag = 0x000080000
+	SUBTYPE        FunctionFlag = 0x000100000
+	INNOCUOUS      FunctionFlag = 0x000200000
+	RESULT_SUBTYPE FunctionFlag = 0x001000000
 )
 
 // StmtStatus name counter values associated with the [Stmt.Status] method.

driver/driver.go

@@ -377,7 +377,7 @@ func (s *stmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driv
 	if err != nil {
 		return nil, err
 	}
-	return &rows{s, ctx}, nil
+	return &rows{ctx: ctx, stmt: s}, nil
 }
 
 func (s *stmt) setupBindings(args []driver.NamedValue) error {
@@ -479,8 +479,10 @@ func (r resultRowsAffected) RowsAffected() (int64, error) {
 }
 
 type rows struct {
-	*stmt
 	ctx context.Context
+	*stmt
+	names []string
+	types []string
 }
 
 func (r *rows) Close() error {
@@ -489,22 +491,35 @@ func (r *rows) Close() error {
 }
 
 func (r *rows) Columns() []string {
-	count := r.Stmt.ColumnCount()
-	columns := make([]string, count)
-	for i := range columns {
-		columns[i] = r.Stmt.ColumnName(i)
+	if r.names == nil {
+		count := r.Stmt.ColumnCount()
+		r.names = make([]string, count)
+		for i := range r.names {
+			r.names[i] = r.Stmt.ColumnName(i)
+		}
 	}
-	return columns
+	return r.names
+}
+
+func (r *rows) declType(index int) string {
+	if r.types == nil {
+		count := r.Stmt.ColumnCount()
+		r.types = make([]string, count)
+		for i := range r.types {
+			r.types[i] = strings.ToUpper(r.Stmt.ColumnDeclType(i))
+		}
+	}
+	return r.types[index]
 }
 
 func (r *rows) ColumnTypeDatabaseTypeName(index int) string {
-	decltype := r.Stmt.ColumnDeclType(index)
+	decltype := r.declType(index)
 	if len := len(decltype); len > 0 && decltype[len-1] == ')' {
 		if i := strings.LastIndexByte(decltype, '('); i >= 0 {
 			decltype = decltype[:i]
 		}
 	}
-	return strings.ToUpper(strings.TrimSpace(decltype))
+	return strings.TrimSpace(decltype)
 }
 
 func (r *rows) Next(dest []driver.Value) error {
@@ -519,11 +534,12 @@ func (r *rows) Next(dest []driver.Value) error {
 	}
 
 	for i := range dest {
-		if t, ok := r.decodeTime(i); ok {
-			dest[i] = t
+		t := r.Stmt.ColumnType(i)
+		if tm, ok := r.decodeTime(i, t); ok {
+			dest[i] = tm
 			continue
 		}
-		switch r.Stmt.ColumnType(i) {
+		switch t {
 		case sqlite3.INTEGER:
 			dest[i] = r.Stmt.ColumnInt64(i)
 		case sqlite3.FLOAT:
@@ -531,7 +547,7 @@ func (r *rows) Next(dest []driver.Value) error {
 		case sqlite3.BLOB:
 			dest[i] = r.Stmt.ColumnRawBlob(i)
 		case sqlite3.TEXT:
-			dest[i] = stringOrTime(r.Stmt.ColumnRawText(i))
+			dest[i] = stringOrTime(r.Stmt.ColumnText(i))
 		case sqlite3.NULL:
 			dest[i] = nil
 		default:
@@ -542,21 +558,21 @@ func (r *rows) Next(dest []driver.Value) error {
 	return r.Stmt.Err()
 }
 
-func (s *stmt) decodeTime(i int) (_ time.Time, _ bool) {
-	if s.tmRead == "" {
+func (r *rows) decodeTime(i int, typ sqlite3.Datatype) (_ time.Time, _ bool) {
+	if r.tmRead == sqlite3.TimeFormatDefault {
 		return
 	}
-	switch s.Stmt.ColumnType(i) {
+	switch typ {
 	case sqlite3.INTEGER, sqlite3.FLOAT, sqlite3.TEXT:
 		// maybe
 	default:
 		return
 	}
-	switch strings.ToUpper(s.Stmt.ColumnDeclType(i)) {
+	switch r.declType(i) {
 	case "DATE", "TIME", "DATETIME", "TIMESTAMP":
 		// maybe
 	default:
 		return
 	}
-	return s.Stmt.ColumnTime(i, s.tmRead), s.Stmt.Err() == nil
+	return r.Stmt.ColumnTime(i, r.tmRead), r.Stmt.Err() == nil
 }

driver/driver_test.go

@@ -6,6 +6,7 @@ import (
 	"database/sql"
 	"errors"
 	"math"
+	"net/url"
 	"path/filepath"
 	"testing"
 	"time"
@@ -295,3 +296,39 @@ func Test_QueryRow_blob_null(t *testing.T) {
 		}
 	}
 }
+
+func Test_time(t *testing.T) {
+	t.Parallel()
+
+	for _, fmt := range []string{"auto", "sqlite", "rfc3339", time.ANSIC} {
+		t.Run(fmt, func(t *testing.T) {
+			db, err := sql.Open("sqlite3", "file::memory:?_timefmt="+url.QueryEscape(fmt))
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer db.Close()
+
+			twosday := time.Date(2022, 2, 22, 22, 22, 22, 0, time.UTC)
+
+			_, err = db.Exec(`CREATE TABLE IF NOT EXISTS test (at DATETIME)`)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			_, err = db.Exec(`INSERT INTO test VALUES (?)`, twosday)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			var got time.Time
+			err = db.QueryRow(`SELECT * FROM test`).Scan(&got)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if !got.Equal(twosday) {
+				t.Errorf("got: %v", got)
+			}
+		})
+	}
+}

driver/time.go

@@ -9,23 +9,23 @@ import (
 // if it roundtrips back to the same string.
 // This way times can be persisted to, and recovered from, the database,
 // but if a string is needed, [database/sql] will recover the same string.
-func stringOrTime(text []byte) driver.Value {
+func stringOrTime(text string) driver.Value {
 	// Weed out (some) values that can't possibly be
 	// [time.RFC3339Nano] timestamps.
 	if len(text) < len("2006-01-02T15:04:05Z") {
-		return string(text)
+		return text
 	}
 	if len(text) > len(time.RFC3339Nano) {
-		return string(text)
+		return text
 	}
 	if text[4] != '-' || text[10] != 'T' || text[16] != ':' {
-		return string(text)
+		return text
 	}
 
 	// Slow path.
-	date, err := time.Parse(time.RFC3339Nano, string(text))
-	if err == nil && date.Format(time.RFC3339Nano) == string(text) {
+	date, err := time.Parse(time.RFC3339Nano, text)
+	if err == nil && date.Format(time.RFC3339Nano) == text {
 		return date
 	}
-	return string(text)
+	return text
 }

driver/time_test.go

@@ -22,7 +22,7 @@ func Fuzz_stringOrTime_1(f *testing.F) {
 	f.Add("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.")
 
 	f.Fuzz(func(t *testing.T, str string) {
-		value := stringOrTime([]byte(str))
+		value := stringOrTime(str)
 
 		switch v := value.(type) {
 		case time.Time:
@@ -59,7 +59,7 @@ func Fuzz_stringOrTime_2(f *testing.F) {
 	f.Add(int64(-763421161058), int64(222_222_222)) // twosday, year 22222BC
 
 	checkTime := func(t testing.TB, date time.Time) {
-		value := stringOrTime([]byte(date.Format(time.RFC3339Nano)))
+		value := stringOrTime(date.Format(time.RFC3339Nano))
 
 		switch v := value.(type) {
 		case time.Time:

embed/build.sh

@@ -5,7 +5,7 @@ cd -P -- "$(dirname -- "$0")"
 
 ROOT=../
 BINARYEN="$ROOT/tools/binaryen-version_116/bin"
-WASI_SDK="$ROOT/tools/wasi-sdk-20.0/bin"
+WASI_SDK="$ROOT/tools/wasi-sdk-21.0/bin"
 
 "$WASI_SDK/clang" --target=wasm32-wasi -flto -g0 -O2 \
 	-o sqlite3.wasm "$ROOT/sqlite3/main.c" \

ext/array/array.go

@@ -1,4 +1,6 @@
 // Package array provides the array table-valued SQL function.
+//
+// https://sqlite.org/carray.html
 package array
 
 import (
@@ -11,8 +13,6 @@ import (
 // Register registers the array single-argument, table-valued SQL function.
 // The argument must be an [sqlite3.Pointer] to a Go slice or array
 // of ints, floats, bools, strings or blobs.
-//
-// https://sqlite.org/carray.html
 func Register(db *sqlite3.Conn) {
 	sqlite3.CreateModule[array](db, "array", nil,
 		func(db *sqlite3.Conn, _, _, _ string, _ ...string) (array, error) {
@@ -131,5 +131,5 @@ func indexable(v reflect.Value) (reflect.Value, error) {
 			return v, nil
 		}
 	}
-	return v, fmt.Errorf("array: unsupported argument:%.0w %v", sqlite3.MISMATCH, v.Type())
+	return v, fmt.Errorf("array: unsupported argument:%.0w %v", sqlite3.MISMATCH, v)
 }

ext/array/array_test.go

@@ -92,3 +92,29 @@ func Test_cursor_Column(t *testing.T) {
 		log.Fatal(err)
 	}
 }
+
+func Test_array_errors(t *testing.T) {
+	t.Parallel()
+
+	db, err := sqlite3.Open(":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	array.Register(db)
+
+	err = db.Exec(`SELECT * FROM array()`)
+	if err == nil {
+		t.Fatal("want error")
+	} else {
+		t.Log(err)
+	}
+
+	err = db.Exec(`SELECT * FROM array(?)`)
+	if err == nil {
+		t.Fatal("want error")
+	} else {
+		t.Log(err)
+	}
+}

ext/blob/blob.go

@@ -1,70 +0,0 @@
-// Package blob provides an alternative interface to incremental BLOB I/O.
-package blob
-
-import (
-	"errors"
-
-	"github.com/ncruces/go-sqlite3"
-	"github.com/ncruces/go-sqlite3/internal/util"
-)
-
-// Register registers the blob_open SQL function:
-//
-//	blob_open(schema, table, column, rowid, flags, callback, args...)
-//
-// The callback must be an [sqlite3.Pointer] to an [OpenCallback].
-// Any optional args will be passed to the callback,
-// along with the [sqlite3.Blob] handle.
-//
-// https://sqlite.org/c3ref/blob.html
-func Register(db *sqlite3.Conn) {
-	db.CreateFunction("blob_open", -1,
-		sqlite3.DETERMINISTIC|sqlite3.DIRECTONLY, openBlob)
-}
-
-func openBlob(ctx sqlite3.Context, arg ...sqlite3.Value) {
-	if len(arg) < 6 {
-		ctx.ResultError(util.ErrorString("blob_open: wrong number of arguments"))
-		return
-	}
-
-	row := arg[3].Int64()
-
-	var err error
-	blob, ok := ctx.GetAuxData(0).(*sqlite3.Blob)
-	if ok {
-		err = blob.Reopen(row)
-		if errors.Is(err, sqlite3.MISUSE) {
-			// Blob was closed (db, table, column or write changed).
-			ok = false
-		}
-	}
-
-	if !ok {
-		db := arg[0].Text()
-		table := arg[1].Text()
-		column := arg[2].Text()
-		write := arg[4].Bool()
-		blob, err = ctx.Conn().OpenBlob(db, table, column, row, write)
-	}
-	if err != nil {
-		ctx.ResultError(err)
-		return
-	}
-
-	fn := arg[5].Pointer().(OpenCallback)
-	err = fn(blob, arg[6:]...)
-	if err != nil {
-		ctx.ResultError(err)
-		return
-	}
-
-	// This ensures the blob is closed if db, table, column or write change.
-	ctx.SetAuxData(0, blob) // db
-	ctx.SetAuxData(1, blob) // table
-	ctx.SetAuxData(2, blob) // column
-	ctx.SetAuxData(4, blob) // write
-}
-
-// OpenCallback is the type for the blob_open callback.
-type OpenCallback func(*sqlite3.Blob, ...sqlite3.Value) error

ext/blobio/blob.go

@@ -0,0 +1,138 @@
+// Package blobio provides an SQL interface to incremental BLOB I/O.
+package blobio
+
+import (
+	"errors"
+	"io"
+
+	"github.com/ncruces/go-sqlite3"
+	"github.com/ncruces/go-sqlite3/internal/util"
+)
+
+// Register registers the SQL functions:
+//
+//	readblob(schema, table, column, rowid, offset, n)
+//
+// Reads n bytes of a blob, starting at offset.
+//
+//	writeblob(schema, table, column, rowid, offset, data)
+//
+// Writes data into a blob, at the given offset.
+//
+//	openblob(schema, table, column, rowid, write, callback, args...)
+//
+// Opens blobs for reading or writing.
+// The callback is invoked for each open blob,
+// and must be an [sqlite3.Pointer] to an [OpenCallback].
+// The optional args will be passed to the callback,
+// along with the [sqlite3.Blob] handle.
+//
+// https://sqlite.org/c3ref/blob.html
+func Register(db *sqlite3.Conn) {
+	db.CreateFunction("readblob", 6, sqlite3.DIRECTONLY, readblob)
+	db.CreateFunction("writeblob", 6, sqlite3.DIRECTONLY, writeblob)
+	db.CreateFunction("openblob", -1, sqlite3.DIRECTONLY, openblob)
+}
+
+// OpenCallback is the type for the openblob callback.
+type OpenCallback func(*sqlite3.Blob, ...sqlite3.Value) error
+
+func readblob(ctx sqlite3.Context, arg ...sqlite3.Value) {
+	blob, err := getAuxBlob(ctx, arg, false)
+	if err != nil {
+		ctx.ResultError(err)
+		return
+	}
+
+	_, err = blob.Seek(arg[4].Int64(), io.SeekStart)
+	if err != nil {
+		ctx.ResultError(err)
+		return
+	}
+
+	n := arg[5].Int64()
+	if n <= 0 {
+		return
+	}
+	buf := make([]byte, n)
+
+	_, err = io.ReadFull(blob, buf)
+	if err != nil {
+		ctx.ResultError(err)
+		return
+	}
+
+	ctx.ResultBlob(buf)
+	setAuxBlob(ctx, blob, false)
+}
+
+func writeblob(ctx sqlite3.Context, arg ...sqlite3.Value) {
+	blob, err := getAuxBlob(ctx, arg, true)
+	if err != nil {
+		ctx.ResultError(err)
+		return
+	}
+
+	_, err = blob.Seek(arg[4].Int64(), io.SeekStart)
+	if err != nil {
+		ctx.ResultError(err)
+		return
+	}
+
+	_, err = blob.Write(arg[5].RawBlob())
+	if err != nil {
+		ctx.ResultError(err)
+		return
+	}
+
+	setAuxBlob(ctx, blob, false)
+}
+
+func openblob(ctx sqlite3.Context, arg ...sqlite3.Value) {
+	if len(arg) < 6 {
+		ctx.ResultError(util.ErrorString("openblob: wrong number of arguments"))
+		return
+	}
+
+	blob, err := getAuxBlob(ctx, arg, arg[4].Bool())
+	if err != nil {
+		ctx.ResultError(err)
+		return
+	}
+
+	fn := arg[5].Pointer().(OpenCallback)
+	err = fn(blob, arg[6:]...)
+	if err != nil {
+		ctx.ResultError(err)
+		return
+	}
+
+	setAuxBlob(ctx, blob, true)
+}
+
+func getAuxBlob(ctx sqlite3.Context, arg []sqlite3.Value, write bool) (*sqlite3.Blob, error) {
+	row := arg[3].Int64()
+
+	if blob, ok := ctx.GetAuxData(0).(*sqlite3.Blob); ok {
+		if err := blob.Reopen(row); errors.Is(err, sqlite3.MISUSE) {
+			// Blob was closed (db, table, column or write changed).
+		} else {
+			return blob, err
+		}
+	}
+
+	db := arg[0].Text()
+	table := arg[1].Text()
+	column := arg[2].Text()
+	return ctx.Conn().OpenBlob(db, table, column, row, write)
+}
+
+func setAuxBlob(ctx sqlite3.Context, blob *sqlite3.Blob, writer bool) {
+	// This ensures the blob is closed if db, table, column or write change.
+	ctx.SetAuxData(0, blob) // db
+	ctx.SetAuxData(1, blob) // table
+	ctx.SetAuxData(2, blob) // column
+	if writer {
+		ctx.SetAuxData(4, blob) // write
+	}
+}

ext/blobio/blob_test.go

@@ -1,4 +1,4 @@
-package blob_test
+package blobio_test
 
 import (
 	"io"
@@ -11,14 +11,14 @@ import (
 	"github.com/ncruces/go-sqlite3/driver"
 	_ "github.com/ncruces/go-sqlite3/embed"
 	"github.com/ncruces/go-sqlite3/ext/array"
-	"github.com/ncruces/go-sqlite3/ext/blob"
+	"github.com/ncruces/go-sqlite3/ext/blobio"
 	_ "github.com/ncruces/go-sqlite3/vfs/memdb"
 )
 
 func Example() {
 	// Open the database, registering the extension.
 	db, err := driver.Open("file:/test.db?vfs=memdb", func(conn *sqlite3.Conn) error {
-		blob.Register(conn)
+		blobio.Register(conn)
 		return nil
 	})
 
@@ -41,18 +41,14 @@ func Example() {
 	}
 
 	// Write the BLOB.
-	_, err = db.Exec(`SELECT blob_open('main', 'test', 'col', last_insert_rowid(), true, ?)`,
-		sqlite3.Pointer[blob.OpenCallback](func(blob *sqlite3.Blob, _ ...sqlite3.Value) error {
-			_, err = io.WriteString(blob, message)
-			return err
-		}))
+	_, err = db.Exec(`SELECT writeblob('main', 'test', 'col', last_insert_rowid(), 0, ?)`, message)
 	if err != nil {
 		log.Fatal(err)
 	}
 
 	// Read the BLOB.
-	_, err = db.Exec(`SELECT blob_open('main', 'test', 'col', rowid, false, ?) FROM test`,
-		sqlite3.Pointer[blob.OpenCallback](func(blob *sqlite3.Blob, _ ...sqlite3.Value) error {
+	_, err = db.Exec(`SELECT openblob('main', 'test', 'col', rowid, false, ?) FROM test`,
+		sqlite3.Pointer[blobio.OpenCallback](func(blob *sqlite3.Blob, _ ...sqlite3.Value) error {
 			_, err = io.Copy(os.Stdout, blob)
 			return err
 		}))
@@ -63,7 +59,7 @@ func Example() {
 	// Hello BLOB!
 }
 
-func TestRegister(t *testing.T) {
+func Test_readblob(t *testing.T) {
 	t.Parallel()
 
 	db, err := sqlite3.Open(":memory:")
@@ -72,10 +68,10 @@ func TestRegister(t *testing.T) {
 	}
 	defer db.Close()
 
-	blob.Register(db)
+	blobio.Register(db)
 	array.Register(db)
 
-	err = db.Exec(`SELECT blob_open()`)
+	err = db.Exec(`SELECT readblob()`)
 	if err == nil {
 		t.Fatal("want error")
 	} else {
@@ -92,14 +88,74 @@ func TestRegister(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	stmt, _, err := db.Prepare(`SELECT blob_open('main', value, 'col', 1, false, ?) FROM array(?)`)
+	stmt, _, err := db.Prepare(`SELECT readblob('main', value, 'col', 1, 1, 1) FROM array(?)`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer stmt.Close()
+
+	err = stmt.BindPointer(1, []string{"test1", "test2"})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if stmt.Step() {
+		got := stmt.ColumnText(0)
+		if got != "\xfe" {
+			t.Errorf("got %q", got)
+		}
+	}
+
+	if stmt.Step() {
+		got := stmt.ColumnText(0)
+		if got != "\xbe" {
+			t.Errorf("got %q", got)
+		}
+	}
+
+	err = stmt.Err()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func Test_openblob(t *testing.T) {
+	t.Parallel()
+
+	db, err := sqlite3.Open(":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	blobio.Register(db)
+	array.Register(db)
+
+	err = db.Exec(`SELECT openblob()`)
+	if err == nil {
+		t.Fatal("want error")
+	} else {
+		t.Log(err)
+	}
+
+	err = db.Exec(`
+		CREATE TABLE IF NOT EXISTS test1 (col);
+		CREATE TABLE IF NOT EXISTS test2 (col);
+		INSERT INTO test1 VALUES (x'cafe');
+		INSERT INTO test2 VALUES (x'babe');
+	`)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	stmt, _, err := db.Prepare(`SELECT openblob('main', value, 'col', 1, false, ?) FROM array(?)`)
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer stmt.Close()
 
 	var got []string
-	err = stmt.BindPointer(1, blob.OpenCallback(func(b *sqlite3.Blob, _ ...sqlite3.Value) error {
+	err = stmt.BindPointer(1, blobio.OpenCallback(func(b *sqlite3.Blob, _ ...sqlite3.Value) error {
 		d, err := io.ReadAll(b)
 		if err != nil {
 			return err

ext/csv/csv.go

@@ -7,10 +7,11 @@
 package csv
 
 import (
+	"bufio"
 	"encoding/csv"
 	"fmt"
 	"io"
-	"math"
+	"io/fs"
 	"os"
 	"strings"
 
@@ -18,16 +19,14 @@ import (
 )
 
 // Register registers the CSV virtual table.
-// If a filename is specified, `os.Open` is used to read it from disk.
+// If a filename is specified, `os.Open` is used to open the file.
 func Register(db *sqlite3.Conn) {
-	RegisterOpen(db, func(name string) (io.ReaderAt, error) {
-		return os.Open(name)
-	})
+	RegisterOpen(db, osfs{})
 }
 
 // RegisterOpen registers the CSV virtual table.
-// If a filename is specified, open is used to open the file.
-func RegisterOpen(db *sqlite3.Conn, open func(name string) (io.ReaderAt, error)) {
+// If a filename is specified, fsys is used to open the file.
+func RegisterOpen(db *sqlite3.Conn, fsys fs.FS) {
 	declare := func(db *sqlite3.Conn, _, _, _ string, arg ...string) (_ *table, err error) {
 		var (
 			filename string
@@ -71,32 +70,23 @@ func RegisterOpen(db *sqlite3.Conn, open func(name string) (io.ReaderAt, error))
 			return nil, fmt.Errorf(`csv: must specify either "filename" or "data" but not both`)
 		}
 
-		var r io.ReaderAt
-		if filename != "" {
-			r, err = open(filename)
-		} else {
-			r = strings.NewReader(data)
-		}
-		if err != nil {
-			return nil, err
-		}
-
 		table := &table{
-			r:      r,
+			fsys:   fsys,
+			name:   filename,
+			data:   data,
 			comma:  comma,
 			header: header,
-			bom:    -1,
 		}
-		defer func() {
-			if err != nil {
-				table.Close()
-			}
-		}()
 
 		if schema == "" {
 			var row []string
 			if header || columns < 0 {
-				row, err = table.newReader().Read()
+				csv, close, err := table.newReader()
+				defer close.Close()
+				if err != nil {
+					return nil, err
+				}
+				row, err = csv.Read()
 				if err != nil {
 					return nil, err
 				}
@@ -118,20 +108,18 @@ func RegisterOpen(db *sqlite3.Conn, open func(name string) (io.ReaderAt, error))
 	sqlite3.CreateModule(db, "csv", declare, declare)
 }
 
-type table struct {
-	r      io.ReaderAt
-	comma  rune
-	header bool
-	bom    int8
+type osfs struct{}
+
+func (osfs) Open(name string) (fs.File, error) {
+	return os.Open(name)
 }
 
-func (t *table) Close() error {
-	if c, ok := t.r.(io.Closer); ok {
-		err := c.Close()
-		t.r = nil
-		return err
-	}
-	return nil
+type table struct {
+	fsys   fs.FS
+	name   string
+	data   string
+	comma  rune
+	header bool
 }
 
 func (t *table) BestIndex(idx *sqlite3.IndexInfo) error {
@@ -147,38 +135,70 @@ func (t *table) Rename(new string) error {
 	return nil
 }
 
-func (t *table) Integrity(schema, table string, flags int) (err error) {
-	if flags&1 == 0 {
-		_, err = t.newReader().ReadAll()
+func (t *table) Integrity(schema, table string, flags int) error {
+	if flags&1 != 0 {
+		return nil
 	}
+	csv, close, err := t.newReader()
+	if err != nil {
+		return err
+	}
+	if close != nil {
+		defer close.Close()
+	}
+	_, err = csv.ReadAll()
 	return err
 }
 
-func (t *table) newReader() *csv.Reader {
-	if t.bom < 0 {
-		var bom [3]byte
-		t.r.ReadAt(bom[:], 0)
-		if string(bom[:]) == "\xEF\xBB\xBF" {
-			t.bom = 3
-		} else {
-			t.bom = 0
+func (t *table) newReader() (*csv.Reader, io.Closer, error) {
+	var r io.Reader
+	var c io.Closer
+	if t.name != "" {
+		f, err := t.fsys.Open(t.name)
+		if err != nil {
+			return nil, f, err
 		}
+
+		buf := bufio.NewReader(f)
+		bom, err := buf.Peek(3)
+		if err != nil {
+			return nil, f, err
+		}
+		if string(bom) == "\xEF\xBB\xBF" {
+			buf.Discard(3)
+		}
+
+		r = buf
+		c = f
+	} else {
+		r = strings.NewReader(t.data)
+		c = io.NopCloser(r)
 	}
-	csv := csv.NewReader(io.NewSectionReader(t.r, int64(t.bom), math.MaxInt64))
+
+	csv := csv.NewReader(r)
 	csv.ReuseRecord = true
 	csv.Comma = t.comma
-	return csv
+	return csv, c, nil
 }
 
 type cursor struct {
 	table *table
+	close io.Closer
 	csv   *csv.Reader
 	row   []string
 	rowID int64
 }
 
+func (c *cursor) Close() error {
+	return c.close.Close()
+}
+
 func (c *cursor) Filter(idxNum int, idxStr string, arg ...sqlite3.Value) error {
-	c.csv = c.table.newReader()
+	var err error
+	c.csv, c.close, err = c.table.newReader()
+	if err != nil {
+		return err
+	}
 	if c.table.header {
 		c.Next() // skip header
 	}

ext/csv/csv_test.go

@@ -61,7 +61,7 @@ func TestRegister(t *testing.T) {
 
 	csv.Register(db)
 
-	const data = "\xEF\xBB\xBF" + `
+	const data = `
 "Rob"	"Pike"	rob
 "Ken"	Thompson	ken
 Robert	"Griesemer"	"gri"`
@@ -84,8 +84,8 @@ Robert	"Griesemer"	"gri"`
 	if !stmt.Step() {
 		t.Fatal("no rows")
 	}
-	if got := stmt.ColumnText(1); got != "Pike" {
-		t.Errorf("got %q want Pike", got)
+	if got := stmt.ColumnText(0); got != "Rob" {
+		t.Errorf("got %q want Rob", got)
 	}
 	if stmt.Step() {
 		t.Fatal("more rows")
@@ -98,12 +98,17 @@ Robert	"Griesemer"	"gri"`
 
 	err = db.Exec(`PRAGMA integrity_check`)
 	if err != nil {
-		t.Fatal(err)
+		t.Error(err)
+	}
+
+	err = db.Exec(`PRAGMA quick_check`)
+	if err != nil {
+		t.Error(err)
 	}
 
 	err = db.Exec(`DROP TABLE temp.csv`)
 	if err != nil {
-		log.Fatal(err)
+		t.Error(err)
 	}
 }
 

ext/csv/testdata/eurofxref.csv

@@ -1,4 +1,4 @@
-Date,USD,JPY,BGN,CYP,CZK,DKK,EEK,GBP,HUF,LTL,LVL,MTL,PLN,ROL,RON,SEK,SIT,SKK,CHF,ISK,NOK,HRK,RUB,TRL,TRY,AUD,BRL,CAD,CNY,HKD,IDR,ILS,INR,KRW,MXN,MYR,NZD,PHP,SGD,THB,ZAR,
+Date,USD,JPY,BGN,CYP,CZK,DKK,EEK,GBP,HUF,LTL,LVL,MTL,PLN,ROL,RON,SEK,SIT,SKK,CHF,ISK,NOK,HRK,RUB,TRL,TRY,AUD,BRL,CAD,CNY,HKD,IDR,ILS,INR,KRW,MXN,MYR,NZD,PHP,SGD,THB,ZAR,
 2022-12-30,1.0666,140.66,1.9558,N/A,24.116,7.4365,N/A,0.88693,400.87,N/A,N/A,N/A,4.6808,N/A,4.9495,11.1218,N/A,N/A,0.9847,151.5,10.5138,7.5365,N/A,N/A,19.9649,1.5693,5.6386,1.444,7.3582,8.3163,16519.82,3.7554,88.171,1344.09,20.856,4.6984,1.6798,59.32,1.43,36.835,18.0986,
 2022-12-29,1.0649,142.24,1.9558,N/A,24.191,7.4365,N/A,0.88549,399.6,N/A,N/A,N/A,4.6855,N/A,4.9493,11.158,N/A,N/A,0.984,152.5,10.55,7.5365,N/A,N/A,19.934,1.5859,5.5351,1.4475,7.4151,8.2994,16680.38,3.7575,88.2295,1350.18,20.651,4.7106,1.6887,59.367,1.436,36.877,18.1967,
 2022-12-28,1.064,142.21,1.9558,N/A,24.252,7.4365,N/A,0.88058,403.3,N/A,N/A,N/A,4.7008,N/A,4.946,11.1038,N/A,N/A,0.9863,151.9,10.4495,7.5365,N/A,N/A,19.9144,1.566,5.6109,1.4361,7.4224,8.2931,16765.93,3.7526,88.0943,1348.59,20.6856,4.7055,1.6772,59.613,1.4323,36.953,18.289,

ext/fileio/fileio.go

@@ -0,0 +1,59 @@
+// Package fileio provides SQL functions to read, write and list files.
+//
+// https://sqlite.org/src/doc/tip/ext/misc/fileio.c
+package fileio
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+
+	"github.com/ncruces/go-sqlite3"
+)
+
+// Register registers SQL functions readfile, writefile, lsmode,
+// and the eponymous virtual table fsdir.
+func Register(db *sqlite3.Conn) {
+	RegisterFS(db, nil)
+}
+
+// Register registers SQL functions readfile, lsmode,
+// and the eponymous virtual table fsdir;
+// fsys will be used to read files and list directories.
+func RegisterFS(db *sqlite3.Conn, fsys fs.FS) {
+	db.CreateFunction("lsmode", 1, 0, lsmode)
+	db.CreateFunction("readfile", 1, sqlite3.DIRECTONLY, readfile(fsys))
+	if fsys == nil {
+		db.CreateFunction("writefile", -1, sqlite3.DIRECTONLY, writefile)
+	}
+	sqlite3.CreateModule(db, "fsdir", nil, func(db *sqlite3.Conn, module, schema, table string, arg ...string) (fsdir, error) {
+		err := db.DeclareVtab(`CREATE TABLE x(name,mode,mtime,data,path HIDDEN,dir HIDDEN)`)
+		db.VtabConfig(sqlite3.VTAB_DIRECTONLY)
+		return fsdir{fsys}, err
+	})
+}
+
+func lsmode(ctx sqlite3.Context, arg ...sqlite3.Value) {
+	ctx.ResultText(fs.FileMode(arg[0].Int()).String())
+}
+
+func readfile(fsys fs.FS) func(ctx sqlite3.Context, arg ...sqlite3.Value) {
+	return func(ctx sqlite3.Context, arg ...sqlite3.Value) {
+		var err error
+		var data []byte
+
+		if fsys != nil {
+			data, err = fs.ReadFile(fsys, arg[0].Text())
+		} else {
+			data, err = os.ReadFile(arg[0].Text())
+		}
+
+		switch {
+		case err == nil:
+			ctx.ResultBlob(data)
+		case !errors.Is(err, fs.ErrNotExist):
+			ctx.ResultError(fmt.Errorf("readfile: %w", err))
+		}
+	}
+}

ext/fileio/fileio_test.go

@@ -0,0 +1,80 @@
+package fileio_test
+
+import (
+	"bytes"
+	"database/sql"
+	"io/fs"
+	"os"
+	"testing"
+
+	"github.com/ncruces/go-sqlite3"
+	"github.com/ncruces/go-sqlite3/driver"
+	_ "github.com/ncruces/go-sqlite3/embed"
+	"github.com/ncruces/go-sqlite3/ext/fileio"
+)
+
+func Test_lsmode(t *testing.T) {
+	t.Parallel()
+
+	db, err := driver.Open(":memory:", func(c *sqlite3.Conn) error {
+		fileio.Register(c)
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	d, err := os.Getwd()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s, err := os.Stat(d)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var mode string
+	err = db.QueryRow(`SELECT lsmode(?)`, s.Mode()).Scan(&mode)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(mode) != 10 || mode[0] != 'd' {
+		t.Errorf("got %s", mode)
+	} else {
+		t.Logf("got %s", mode)
+	}
+}
+
+func Test_readfile(t *testing.T) {
+	t.Parallel()
+
+	for _, fsys := range []fs.FS{nil, os.DirFS(".")} {
+		t.Run("", func(t *testing.T) {
+			db, err := driver.Open(":memory:", func(c *sqlite3.Conn) error {
+				fileio.RegisterFS(c, fsys)
+				return nil
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer db.Close()
+
+			rows, err := db.Query(`SELECT readfile('fileio_test.go')`)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if rows.Next() {
+				var data sql.RawBytes
+				rows.Scan(&data)
+
+				if !bytes.HasPrefix(data, []byte("package fileio_test")) {
+					t.Errorf("got %s", data[:min(64, len(data))])
+				}
+			}
+		})
+	}
+}

ext/fileio/fsdir.go

@@ -0,0 +1,186 @@
+package fileio
+
+import (
+	"io/fs"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/ncruces/go-sqlite3"
+)
+
+type fsdir struct{ fsys fs.FS }
+
+func (d fsdir) BestIndex(idx *sqlite3.IndexInfo) error {
+	var root, base bool
+	for i, cst := range idx.Constraint {
+		switch cst.Column {
+		case 4: // root
+			if !cst.Usable || cst.Op != sqlite3.INDEX_CONSTRAINT_EQ {
+				return sqlite3.CONSTRAINT
+			}
+			idx.ConstraintUsage[i] = sqlite3.IndexConstraintUsage{
+				Omit:      true,
+				ArgvIndex: 1,
+			}
+			root = true
+		case 5: // base
+			if !cst.Usable || cst.Op != sqlite3.INDEX_CONSTRAINT_EQ {
+				return sqlite3.CONSTRAINT
+			}
+			idx.ConstraintUsage[i] = sqlite3.IndexConstraintUsage{
+				Omit:      true,
+				ArgvIndex: 2,
+			}
+			base = true
+		}
+	}
+	if !root {
+		return sqlite3.CONSTRAINT
+	}
+	if base {
+		idx.EstimatedCost = 10
+	} else {
+		idx.EstimatedCost = 100
+	}
+	return nil
+}
+
+func (d fsdir) Open() (sqlite3.VTabCursor, error) {
+	return &cursor{fsys: d.fsys}, nil
+}
+
+type cursor struct {
+	fsys  fs.FS
+	base  string
+	rowID int64
+	eof   bool
+	curr  entry
+	next  chan entry
+	done  chan struct{}
+}
+
+type entry struct {
+	path string
+	fs.DirEntry
+	err error
+}
+
+func (c *cursor) Close() error {
+	if c.done != nil {
+		close(c.done)
+		s := <-c.next
+		c.done = nil
+		c.next = nil
+		return s.err
+	}
+	return nil
+}
+
+func (c *cursor) Filter(idxNum int, idxStr string, arg ...sqlite3.Value) error {
+	if err := c.Close(); err != nil {
+		return err
+	}
+
+	root := arg[0].Text()
+	if len(arg) > 1 {
+		base := arg[1].Text()
+		if c.fsys != nil {
+			root = path.Join(base, root)
+			base = path.Clean(base) + "/"
+		} else {
+			root = filepath.Join(base, root)
+			base = filepath.Clean(base) + string(filepath.Separator)
+		}
+		c.base = base
+	}
+
+	c.rowID = 0
+	c.eof = false
+	c.next = make(chan entry)
+	c.done = make(chan struct{})
+	go c.WalkDir(root)
+	return c.Next()
+}
+
+func (c *cursor) Next() error {
+	curr, ok := <-c.next
+	c.curr = curr
+	c.eof = !ok
+	c.rowID++
+	return c.curr.err
+}
+
+func (c *cursor) EOF() bool {
+	return c.eof
+}
+
+func (c *cursor) RowID() (int64, error) {
+	return c.rowID, nil
+}
+
+func (c *cursor) Column(ctx *sqlite3.Context, n int) error {
+	switch n {
+	case 0: // name
+		name := strings.TrimPrefix(c.curr.path, c.base)
+		ctx.ResultText(name)
+
+	case 1: // mode
+		i, err := c.curr.Info()
+		if err != nil {
+			return err
+		}
+		ctx.ResultInt64(int64(i.Mode()))
+
+	case 2: // mtime
+		i, err := c.curr.Info()
+		if err != nil {
+			return err
+		}
+		ctx.ResultTime(i.ModTime(), sqlite3.TimeFormatUnixFrac)
+
+	case 3: // data
+		switch typ := c.curr.Type(); {
+		case typ.IsRegular():
+			var data []byte
+			var err error
+			if c.fsys != nil {
+				data, err = fs.ReadFile(c.fsys, c.curr.path)
+			} else {
+				data, err = os.ReadFile(c.curr.path)
+			}
+			if err != nil {
+				return err
+			}
+			ctx.ResultBlob(data)
+
+		case typ&fs.ModeSymlink != 0 && c.fsys == nil:
+			t, err := os.Readlink(c.curr.path)
+			if err != nil {
+				return err
+			}
+			ctx.ResultText(t)
+		}
+	}
+	return nil
+}
+
+func (c *cursor) WalkDir(path string) {
+	defer close(c.next)
+
+	if c.fsys != nil {
+		fs.WalkDir(c.fsys, path, c.WalkDirFunc)
+	} else {
+		filepath.WalkDir(path, c.WalkDirFunc)
+	}
+}
+
+func (c *cursor) WalkDirFunc(path string, d fs.DirEntry, err error) error {
+	select {
+	case <-c.done:
+		return fs.SkipAll
+	case c.next <- entry{path, d, err}:
+		return nil
+	}
+}

ext/fileio/fsdir_test.go

@@ -0,0 +1,78 @@
+package fileio_test
+
+import (
+	"bytes"
+	"database/sql"
+	"io/fs"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/ncruces/go-sqlite3"
+	"github.com/ncruces/go-sqlite3/driver"
+	_ "github.com/ncruces/go-sqlite3/embed"
+	"github.com/ncruces/go-sqlite3/ext/fileio"
+)
+
+func Test_fsdir(t *testing.T) {
+	t.Parallel()
+
+	for _, fsys := range []fs.FS{nil, os.DirFS(".")} {
+		t.Run("", func(t *testing.T) {
+			db, err := driver.Open(":memory:", func(c *sqlite3.Conn) error {
+				fileio.RegisterFS(c, fsys)
+				return nil
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer db.Close()
+
+			rows, err := db.Query(`SELECT * FROM fsdir('.', '.') LIMIT 4`)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			for rows.Next() {
+				var name string
+				var mode fs.FileMode
+				var mtime time.Time
+				var data sql.RawBytes
+				err := rows.Scan(&name, &mode, sqlite3.TimeFormatUnixFrac.Scanner(&mtime), &data)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if mode.Perm() == 0 {
+					t.Errorf("got: %v", mode)
+				}
+				if mtime.Before(time.Unix(0, 0)) {
+					t.Errorf("got: %v", mtime)
+				}
+				if name == "fsdir_test.go" {
+					if !bytes.HasPrefix(data, []byte("package fileio_test")) {
+						t.Errorf("got: %s", data[:min(64, len(data))])
+					}
+				}
+			}
+		})
+	}
+}
+
+func Test_fsdir_errors(t *testing.T) {
+	t.Parallel()
+
+	db, err := sqlite3.Open(":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	fileio.Register(db)
+
+	err = db.Exec(`SELECT name FROM fsdir()`)
+	if err == nil {
+		t.Fatal("want error")
+	} else {
+		t.Log(err)
+	}
+}

ext/fileio/write.go

@@ -0,0 +1,130 @@
+package fileio
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/ncruces/go-sqlite3"
+	"github.com/ncruces/go-sqlite3/internal/util"
+)
+
+func writefile(ctx sqlite3.Context, arg ...sqlite3.Value) {
+	if len(arg) < 2 || len(arg) > 4 {
+		ctx.ResultError(util.ErrorString("writefile: wrong number of arguments"))
+		return
+	}
+
+	file := arg[0].Text()
+
+	var mode fs.FileMode
+	if len(arg) > 2 {
+		mode = fixMode(fs.FileMode(arg[2].Int()))
+	}
+
+	n, err := createFileAndDir(file, mode, arg[1])
+	if err != nil {
+		if len(arg) > 2 {
+			ctx.ResultError(fmt.Errorf("writefile: %w", err))
+		}
+		return
+	}
+
+	if mode&fs.ModeSymlink == 0 {
+		if len(arg) > 2 {
+			err := os.Chmod(file, mode.Perm())
+			if err != nil {
+				ctx.ResultError(fmt.Errorf("writefile: %w", err))
+				return
+			}
+		}
+
+		if len(arg) > 3 {
+			mtime := arg[3].Time(sqlite3.TimeFormatUnixFrac)
+			err := os.Chtimes(file, time.Time{}, mtime)
+			if err != nil {
+				ctx.ResultError(fmt.Errorf("writefile: %w", err))
+				return
+			}
+		}
+	}
+
+	if mode.IsRegular() {
+		ctx.ResultInt(n)
+	}
+}
+
+func createFileAndDir(path string, mode fs.FileMode, data sqlite3.Value) (int, error) {
+	n, err := createFile(path, mode, data)
+	if errors.Is(err, fs.ErrNotExist) {
+		if err := os.MkdirAll(filepath.Dir(path), 0777); err == nil {
+			return createFile(path, mode, data)
+		}
+	}
+	return n, err
+}
+
+func createFile(path string, mode fs.FileMode, data sqlite3.Value) (int, error) {
+	if mode.IsRegular() {
+		blob := data.RawBlob()
+		return len(blob), os.WriteFile(path, blob, fixPerm(mode, 0666))
+	}
+	if mode.IsDir() {
+		err := os.Mkdir(path, fixPerm(mode, 0777))
+		if errors.Is(err, fs.ErrExist) {
+			s, err := os.Lstat(path)
+			if err == nil && s.IsDir() {
+				return 0, nil
+			}
+		}
+		return 0, err
+	}
+	if mode&fs.ModeSymlink != 0 {
+		return 0, os.Symlink(data.Text(), path)
+	}
+	return 0, fmt.Errorf("invalid mode: %v", mode)
+}
+
+func fixMode(mode fs.FileMode) fs.FileMode {
+	const (
+		S_IFMT   fs.FileMode = 0170000
+		S_IFIFO  fs.FileMode = 0010000
+		S_IFCHR  fs.FileMode = 0020000
+		S_IFDIR  fs.FileMode = 0040000
+		S_IFBLK  fs.FileMode = 0060000
+		S_IFREG  fs.FileMode = 0100000
+		S_IFLNK  fs.FileMode = 0120000
+		S_IFSOCK fs.FileMode = 0140000
+	)
+
+	switch mode & S_IFMT {
+	case S_IFDIR:
+		mode |= fs.ModeDir
+	case S_IFLNK:
+		mode |= fs.ModeSymlink
+	case S_IFBLK:
+		mode |= fs.ModeDevice
+	case S_IFCHR:
+		mode |= fs.ModeCharDevice | fs.ModeDevice
+	case S_IFIFO:
+		mode |= fs.ModeNamedPipe
+	case S_IFSOCK:
+		mode |= fs.ModeSocket
+	case S_IFREG, 0:
+		//
+	default:
+		mode |= fs.ModeIrregular
+	}
+
+	return mode &^ S_IFMT
+}
+
+func fixPerm(mode fs.FileMode, def fs.FileMode) fs.FileMode {
+	if mode.Perm() == 0 {
+		return def
+	}
+	return mode.Perm()
+}

ext/fileio/write_test.go

@@ -0,0 +1,115 @@
+package fileio
+
+import (
+	"database/sql"
+	"io/fs"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/ncruces/go-sqlite3"
+	"github.com/ncruces/go-sqlite3/driver"
+	_ "github.com/ncruces/go-sqlite3/embed"
+)
+
+func Test_writefile(t *testing.T) {
+	t.Parallel()
+
+	db, err := driver.Open(":memory:", func(c *sqlite3.Conn) error {
+		Register(c)
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	dir := t.TempDir()
+	link := filepath.Join(dir, "link")
+	file := filepath.Join(dir, "test.txt")
+	nest := filepath.Join(dir, "tmp", "test.txt")
+	sock := filepath.Join(dir, "sock")
+	twosday := time.Date(2022, 2, 22, 22, 22, 22, 0, time.UTC)
+
+	_, err = db.Exec(`SELECT writefile(?, 'Hello world!')`, file)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = db.Exec(`SELECT writefile(?, ?, ?)`, link, "test.txt", fs.ModeSymlink)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = db.Exec(`SELECT writefile(?, ?, ?, ?)`, dir, nil, 0040700, twosday.Unix())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rows, err := db.Query(`SELECT * FROM fsdir('.', ?)`, dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for rows.Next() {
+		var name string
+		var mode fs.FileMode
+		var mtime time.Time
+		var data sql.NullString
+		err := rows.Scan(&name, &mode, sqlite3.TimeFormatUnixFrac.Scanner(&mtime), &data)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if mode.IsDir() && !mtime.Equal(twosday) {
+			t.Errorf("got: %v", mtime)
+		}
+		if mode.IsRegular() && data.String != "Hello world!" {
+			t.Errorf("got: %v", data)
+		}
+		if mode&fs.ModeSymlink != 0 && data.String != "test.txt" {
+			t.Errorf("got: %v", data)
+		}
+	}
+
+	_, err = db.Exec(`SELECT writefile(?, 'Hello world!')`, nest)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = db.Exec(`SELECT writefile(?, ?, ?)`, sock, nil, fs.ModeSocket)
+	if err == nil {
+		t.Fatal("want error")
+	} else {
+		t.Log(err)
+	}
+
+	_, err = db.Exec(`SELECT writefile()`)
+	if err == nil {
+		t.Fatal("want error")
+	} else {
+		t.Log(err)
+	}
+}
+
+func Test_fixMode(t *testing.T) {
+	tests := []struct {
+		mode fs.FileMode
+		want fs.FileMode
+	}{
+		{0010754, 0754 | fs.ModeNamedPipe},
+		{0020754, 0754 | fs.ModeCharDevice | fs.ModeDevice},
+		{0040754, 0754 | fs.ModeDir},
+		{0060754, 0754 | fs.ModeDevice},
+		{0100754, 0754},
+		{0120754, 0754 | fs.ModeSymlink},
+		{0140754, 0754 | fs.ModeSocket},
+		{0170754, 0754 | fs.ModeIrregular},
+	}
+	for _, tt := range tests {
+		t.Run(tt.mode.String(), func(t *testing.T) {
+			if got := fixMode(tt.mode); got != tt.want {
+				t.Errorf("fixMode() = %o, want %o", got, tt.want)
+			}
+		})
+	}
+}

ext/lines/lines.go

@@ -1,4 +1,13 @@
-// Package lines provides a virtual table to read large files line-by-line.
+// Package lines provides a virtual table to read data line-by-line.
+//
+// It is particularly useful for line-oriented datasets,
+// like [ndjson] or [JSON Lines],
+// when paired with SQLite's JSON support.
+//
+// https://github.com/asg017/sqlite-lines
+//
+// [ndjson]: https://ndjson.org/
+// [JSON Lines]: https://jsonlines.org/
 package lines
 
 import (
@@ -6,7 +15,6 @@ import (
 	"bytes"
 	"fmt"
 	"io"
-	"math"
 	"os"
 
 	"github.com/ncruces/go-sqlite3"
@@ -14,7 +22,7 @@ import (
 
 // Register registers the lines and lines_read virtual tables.
 // The lines virtual table reads from a database blob or text.
-// The lines_read virtual table reads from a file or an [io.ReaderAt].
+// The lines_read virtual table reads from a file or an [io.Reader].
 func Register(db *sqlite3.Conn) {
 	sqlite3.CreateModule[lines](db, "lines", nil,
 		func(db *sqlite3.Conn, _, _, _ string, _ ...string) (lines, error) {
@@ -48,81 +56,125 @@ func (l lines) BestIndex(idx *sqlite3.IndexInfo) error {
 }
 
 func (l lines) Open() (sqlite3.VTabCursor, error) {
-	return &cursor{reader: bool(l)}, nil
+	if l {
+		return &reader{}, nil
+	} else {
+		return &buffer{}, nil
+	}
 }
 
 type cursor struct {
-	scanner *bufio.Scanner
-	closer  io.Closer
-	rowID   int64
-	eof     bool
-	reader  bool
-}
-
-func (c *cursor) Close() (err error) {
-	if c.closer != nil {
-		err = c.closer.Close()
-		c.closer = nil
-	}
-	return err
+	line  []byte
+	rowID int64
+	eof   bool
 }
 
 func (c *cursor) EOF() bool {
 	return c.eof
 }
 
-func (c *cursor) Next() error {
-	c.rowID++
-	c.eof = !c.scanner.Scan()
-	return c.scanner.Err()
-}
-
 func (c *cursor) RowID() (int64, error) {
 	return c.rowID, nil
 }
 
 func (c *cursor) Column(ctx *sqlite3.Context, n int) error {
 	if n == 0 {
-		ctx.ResultRawText(c.scanner.Bytes())
+		ctx.ResultRawText(c.line)
 	}
 	return nil
 }
 
-func (c *cursor) Filter(idxNum int, idxStr string, arg ...sqlite3.Value) error {
+type reader struct {
+	reader *bufio.Reader
+	closer io.Closer
+	cursor
+}
+
+func (c *reader) Close() (err error) {
+	if c.closer != nil {
+		err = c.closer.Close()
+		c.closer = nil
+	}
+	return err
+}
+
+func (c *reader) Filter(idxNum int, idxStr string, arg ...sqlite3.Value) error {
 	if err := c.Close(); err != nil {
 		return err
 	}
 
 	var r io.Reader
-	data := arg[0]
-	typ := data.Type()
-	if c.reader {
-		switch typ {
-		case sqlite3.NULL:
-			if p, ok := data.Pointer().(io.ReaderAt); ok {
-				r = io.NewSectionReader(p, 0, math.MaxInt64)
-			}
-		case sqlite3.TEXT:
-			f, err := os.Open(data.Text())
-			if err != nil {
-				return err
-			}
-			c.closer = f
-			r = f
+	typ := arg[0].Type()
+	switch typ {
+	case sqlite3.NULL:
+		if p, ok := arg[0].Pointer().(io.Reader); ok {
+			r = p
 		}
-	} else {
-		switch typ {
-		case sqlite3.TEXT:
-			r = bytes.NewReader(data.RawText())
-		case sqlite3.BLOB:
-			r = bytes.NewReader(data.RawBlob())
+	case sqlite3.TEXT:
+		f, err := os.Open(arg[0].Text())
+		if err != nil {
+			return err
 		}
+		r = f
 	}
-
 	if r == nil {
 		return fmt.Errorf("lines: unsupported argument:%.0w %v", sqlite3.MISMATCH, typ)
 	}
-	c.scanner = bufio.NewScanner(r)
+
+	c.reader = bufio.NewReader(r)
+	c.closer, _ = r.(io.Closer)
 	c.rowID = 0
 	return c.Next()
 }
+
+func (c *reader) Next() (err error) {
+	c.line = c.line[:0]
+	for more := true; more; {
+		var line []byte
+		line, more, err = c.reader.ReadLine()
+		c.line = append(c.line, line...)
+	}
+	if err == io.EOF {
+		c.eof = true
+		err = nil
+	}
+	c.rowID++
+	return err
+}
+
+type buffer struct {
+	data []byte
+	cursor
+}
+
+func (c *buffer) Filter(idxNum int, idxStr string, arg ...sqlite3.Value) error {
+	typ := arg[0].Type()
+	switch typ {
+	case sqlite3.TEXT:
+		c.data = arg[0].RawText()
+	case sqlite3.BLOB:
+		c.data = arg[0].RawBlob()
+	default:
+		return fmt.Errorf("lines: unsupported argument:%.0w %v", sqlite3.MISMATCH, typ)
+	}
+
+	c.rowID = 0
+	return c.Next()
+}
+
+func (c *buffer) Next() error {
+	i := bytes.IndexByte(c.data, '\n')
+	j := i + 1
+	switch {
+	case i < 0:
+		i = len(c.data)
+		j = i
+	case i > 0 && c.data[i-1] == '\r':
+		i--
+	}
+	c.eof = len(c.data) == 0
+	c.line = c.data[:i]
+	c.data = c.data[j:]
+	c.rowID++
+	return nil
+}

ext/lines/lines_test.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"net/http"
 	"os"
 	"strings"
 	"testing"
@@ -25,12 +26,11 @@ func Example() {
 	}
 	defer db.Close()
 
-	// https://storage.googleapis.com/quickdraw_dataset/full/simplified/calendar.ndjson
-	f, err := os.Open("calendar.ndjson")
+	res, err := http.Get("https://storage.googleapis.com/quickdraw_dataset/full/simplified/calendar.ndjson")
 	if err != nil {
 		log.Fatal(err)
 	}
-	defer f.Close()
+	defer res.Body.Close()
 
 	rows, err := db.Query(`
 		SELECT
@@ -40,7 +40,7 @@ func Example() {
   		GROUP BY 1
   		ORDER BY 2 DESC
   		LIMIT 5`,
-		sqlite3.Pointer(f))
+		sqlite3.Pointer(res.Body))
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -58,7 +58,7 @@ func Example() {
 	if err := rows.Err(); err != nil {
 		log.Fatal(err)
 	}
-	// Sample output:
+	// Output:
 	// US: 141001
 	// GB: 22560
 	// CA: 11759
@@ -78,7 +78,7 @@ func Test_lines(t *testing.T) {
 	}
 	defer db.Close()
 
-	const data = "line 1\nline 2\nline 3"
+	const data = "line 1\nline 2\r\nline 3\n"
 
 	rows, err := db.Query(`SELECT rowid, line FROM lines(?)`, data)
 	if err != nil {
@@ -93,6 +93,9 @@ func Test_lines(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
+		if want := fmt.Sprintf("line %d", id); line != want {
+			t.Errorf("got %q, want %q", line, want)
+		}
 	}
 }
 
@@ -135,7 +138,7 @@ func Test_lines_read(t *testing.T) {
 	}
 	defer db.Close()
 
-	const data = "line 1\nline 2\nline 3"
+	const data = "line 1\nline 2\r\nline 3\n"
 
 	rows, err := db.Query(`SELECT rowid, line FROM lines_read(?)`,
 		sqlite3.Pointer(strings.NewReader(data)))
@@ -151,6 +154,9 @@ func Test_lines_read(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
+		if want := fmt.Sprintf("line %d", id); line != want {
+			t.Errorf("got %q, want %q", line, want)
+		}
 	}
 }
 

ext/pivot/op_test.go

@@ -0,0 +1,34 @@
+package pivot
+
+import (
+	"testing"
+
+	"github.com/ncruces/go-sqlite3"
+)
+
+func Test_operator(t *testing.T) {
+	tests := []struct {
+		op   sqlite3.IndexConstraintOp
+		want string
+	}{
+		{sqlite3.INDEX_CONSTRAINT_EQ, "="},
+		{sqlite3.INDEX_CONSTRAINT_LT, "<"},
+		{sqlite3.INDEX_CONSTRAINT_GT, ">"},
+		{sqlite3.INDEX_CONSTRAINT_LE, "<="},
+		{sqlite3.INDEX_CONSTRAINT_GE, ">="},
+		{sqlite3.INDEX_CONSTRAINT_NE, "<>"},
+		{sqlite3.INDEX_CONSTRAINT_IS, "IS"},
+		{sqlite3.INDEX_CONSTRAINT_ISNOT, "IS NOT"},
+		{sqlite3.INDEX_CONSTRAINT_REGEXP, "REGEXP"},
+		{sqlite3.INDEX_CONSTRAINT_MATCH, "MATCH"},
+		{sqlite3.INDEX_CONSTRAINT_GLOB, "GLOB"},
+		{sqlite3.INDEX_CONSTRAINT_LIKE, "LIKE"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.want, func(t *testing.T) {
+			if got := operator(tt.op); got != tt.want {
+				t.Errorf("operator() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

ext/pivot/pivot.go

@@ -114,34 +114,8 @@ func (t *table) BestIndex(idx *sqlite3.IndexInfo) error {
 		if !cst.Usable || !(0 <= cst.Column && cst.Column < len(t.keys)) {
 			continue
 		}
-
-		var op string
-		switch cst.Op {
-		case sqlite3.INDEX_CONSTRAINT_EQ:
-			op = "="
-		case sqlite3.INDEX_CONSTRAINT_LT:
-			op = "<"
-		case sqlite3.INDEX_CONSTRAINT_GT:
-			op = ">"
-		case sqlite3.INDEX_CONSTRAINT_LE:
-			op = "<="
-		case sqlite3.INDEX_CONSTRAINT_GE:
-			op = ">="
-		case sqlite3.INDEX_CONSTRAINT_NE:
-			op = "<>"
-		case sqlite3.INDEX_CONSTRAINT_MATCH:
-			op = "MATCH"
-		case sqlite3.INDEX_CONSTRAINT_LIKE:
-			op = "LIKE"
-		case sqlite3.INDEX_CONSTRAINT_GLOB:
-			op = "GLOB"
-		case sqlite3.INDEX_CONSTRAINT_REGEXP:
-			op = "REGEXP"
-		case sqlite3.INDEX_CONSTRAINT_IS, sqlite3.INDEX_CONSTRAINT_ISNULL:
-			op = "IS"
-		case sqlite3.INDEX_CONSTRAINT_ISNOT, sqlite3.INDEX_CONSTRAINT_ISNOTNULL:
-			op = "IS NOT"
-		default:
+		op := operator(cst.Op)
+		if op == "" {
 			continue
 		}
 
@@ -265,3 +239,34 @@ func (c *cursor) Column(ctx *sqlite3.Context, col int) error {
 	}
 	return c.cell.Reset()
 }
+
+func operator(op sqlite3.IndexConstraintOp) string {
+	switch op {
+	case sqlite3.INDEX_CONSTRAINT_EQ:
+		return "="
+	case sqlite3.INDEX_CONSTRAINT_LT:
+		return "<"
+	case sqlite3.INDEX_CONSTRAINT_GT:
+		return ">"
+	case sqlite3.INDEX_CONSTRAINT_LE:
+		return "<="
+	case sqlite3.INDEX_CONSTRAINT_GE:
+		return ">="
+	case sqlite3.INDEX_CONSTRAINT_NE:
+		return "<>"
+	case sqlite3.INDEX_CONSTRAINT_MATCH:
+		return "MATCH"
+	case sqlite3.INDEX_CONSTRAINT_LIKE:
+		return "LIKE"
+	case sqlite3.INDEX_CONSTRAINT_GLOB:
+		return "GLOB"
+	case sqlite3.INDEX_CONSTRAINT_REGEXP:
+		return "REGEXP"
+	case sqlite3.INDEX_CONSTRAINT_IS, sqlite3.INDEX_CONSTRAINT_ISNULL:
+		return "IS"
+	case sqlite3.INDEX_CONSTRAINT_ISNOT, sqlite3.INDEX_CONSTRAINT_ISNOTNULL:
+		return "IS NOT"
+	default:
+		return ""
+	}
+}

ext/statement/stmt.go

@@ -1,4 +1,7 @@
-// Package statement defines table-valued functions natively using SQL.
+// Package statement defines table-valued functions using SQL.
+//
+// It can be used to create "parametrized views":
+// pre-packaged queries that can be parametrized at query execution time.
 //
 // https://github.com/0x09/sqlite-statement-vtab
 package statement

ext/stats/stats_test.go

@@ -1,4 +1,4 @@
-package stats
+package stats_test
 
 import (
 	"math"
@@ -6,6 +6,7 @@ import (
 
 	"github.com/ncruces/go-sqlite3"
 	_ "github.com/ncruces/go-sqlite3/embed"
+	"github.com/ncruces/go-sqlite3/ext/stats"
 )
 
 func TestRegister_variance(t *testing.T) {
@@ -17,7 +18,7 @@ func TestRegister_variance(t *testing.T) {
 	}
 	defer db.Close()
 
-	Register(db)
+	stats.Register(db)
 
 	err = db.Exec(`CREATE TABLE IF NOT EXISTS data (x)`)
 	if err != nil {
@@ -89,7 +90,7 @@ func TestRegister_covariance(t *testing.T) {
 	}
 	defer db.Close()
 
-	Register(db)
+	stats.Register(db)
 
 	err = db.Exec(`CREATE TABLE IF NOT EXISTS data (x, y)`)
 	if err != nil {

ext/todo.txt

@@ -0,0 +1,16 @@
+sha3
+sha3_query
+
+ieee754
+ieee754_exponent
+ieee754_from_blob
+ieee754_inc
+ieee754_mantissa
+ieee754_to_blob
+
+zipfile
+zipfile_cds
+sqlar_compress
+sqlar_uncompress
+
+remember

gormlite/go.mod

@@ -3,7 +3,7 @@ module github.com/ncruces/go-sqlite3/gormlite
 go 1.21
 
 require (
-	github.com/ncruces/go-sqlite3 v0.10.5
+	github.com/ncruces/go-sqlite3 v0.11.0
 	gorm.io/gorm v1.25.5
 )
 
@@ -12,5 +12,5 @@ require (
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/ncruces/julianday v1.0.0 // indirect
 	github.com/tetratelabs/wazero v1.5.0 // indirect
-	golang.org/x/sys v0.14.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
 )

gormlite/go.sum

@@ -2,14 +2,14 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
 github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
-github.com/ncruces/go-sqlite3 v0.10.5 h1:SPnFFYajDfhTuJNjeNwdOhwVCRSAqB1PdSHsGrdfYjw=
-github.com/ncruces/go-sqlite3 v0.10.5/go.mod h1:8aGu9/G8lLZbvO6TXA0FXTP2liIefFmbpeXuhG4nJLw=
+github.com/ncruces/go-sqlite3 v0.11.0 h1:PDjs8Ve2Z0GWmHyKQHGUyG78grCXKhiHCUZQI8CqXO8=
+github.com/ncruces/go-sqlite3 v0.11.0/go.mod h1:zaYJ6xP+EQiWJCa3nd3h28cD8DuSIcIqh+LrJMrBN9k=
 github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
 github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
 github.com/tetratelabs/wazero v1.5.0 h1:Yz3fZHivfDiZFUXnWMPUoiW7s8tC1sjdBtlJn08qYa0=
 github.com/tetratelabs/wazero v1.5.0/go.mod h1:0U0G41+ochRKoPKCJlh0jMg1CHkyfK8kDqiirMmKY8A=
-golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
-golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls=

gormlite/test.sh

@@ -3,6 +3,8 @@ set -euo pipefail
 
 cd -P -- "$(dirname -- "$0")"
 
+go test
+
 rm -rf gorm/ tests/
 git clone --filter=blob:none https://github.com/go-gorm/gorm.git
 mv gorm/tests tests

sqlite.go

@@ -4,8 +4,10 @@ package sqlite3
 import (
 	"context"
 	"math"
+	"math/bits"
 	"os"
 	"sync"
+	"unsafe"
 
 	"github.com/ncruces/go-sqlite3/internal/util"
 	"github.com/ncruces/go-sqlite3/vfs"
@@ -67,7 +69,11 @@ func compileSQLite() {
 type sqlite struct {
 	ctx   context.Context
 	mod   api.Module
-	funcs [8]api.Function
+	funcs struct {
+		fn   [32]api.Function
+		id   [32]*byte
+		mask uint32
+	}
 	stack [8]uint64
 	freer uint32
 }
@@ -137,33 +143,42 @@ func (sqlt *sqlite) error(rc uint64, handle uint32, sql ...string) error {
 	return &err
 }
 
-func (sqlt *sqlite) getfn(name string) (api.Function, uint32) {
-	// https://cr.yp.to/cdb/cdb.txt
-	hash := func(s string) uint32 {
-		var hash uint32 = 5381
-		for _, b := range []byte(s) {
-			hash = (hash<<5 + hash) ^ uint32(b)
+func (sqlt *sqlite) getfn(name string) api.Function {
+	c := &sqlt.funcs
+	p := unsafe.StringData(name)
+	for i := range c.id {
+		if c.id[i] == p {
+			c.id[i] = nil
+			c.mask &^= uint32(1) << i
+			return c.fn[i]
 		}
-		return hash
-	}(name) % uint32(len(sqlt.funcs))
-
-	fn := sqlt.funcs[hash]
-	if fn == nil || name != fn.Definition().Name() {
-		fn = sqlt.mod.ExportedFunction(name)
-	} else {
-		sqlt.funcs[hash] = nil
 	}
-	return fn, hash
+	return sqlt.mod.ExportedFunction(name)
+}
+
+func (sqlt *sqlite) putfn(name string, fn api.Function) {
+	c := &sqlt.funcs
+	p := unsafe.StringData(name)
+	i := bits.TrailingZeros32(^c.mask)
+	if i < 32 {
+		c.id[i] = p
+		c.fn[i] = fn
+		c.mask |= uint32(1) << i
+	} else {
+		c.id[0] = p
+		c.fn[0] = fn
+		c.mask = uint32(1)
+	}
 }
 
 func (sqlt *sqlite) call(name string, params ...uint64) uint64 {
 	copy(sqlt.stack[:], params)
-	fn, hash := sqlt.getfn(name)
+	fn := sqlt.getfn(name)
 	err := fn.CallWithStack(sqlt.ctx, sqlt.stack[:])
 	if err != nil {
 		panic(err)
 	}
-	sqlt.funcs[hash] = fn
+	sqlt.putfn(name, fn)
 	return sqlt.stack[0]
 }
 

tests/parallel/parallel_test.go

@@ -60,7 +60,7 @@ func TestMultiProcess(t *testing.T) {
 		"&_pragma=journal_mode(truncate)" +
 		"&_pragma=synchronous(off)"
 
-	cmd := exec.Command("go", "test", "-v", "-run", "TestChildProcess")
+	cmd := exec.Command(os.Args[0], append(os.Args[1:], "-test.v", "-test.run=TestChildProcess")...)
 	out, err := cmd.StdoutPipe()
 	if err != nil {
 		t.Fatal(err)
@@ -71,8 +71,10 @@ func TestMultiProcess(t *testing.T) {
 
 	var buf [3]byte
 	// Wait for child to start.
-	if _, err := io.ReadFull(out, buf[:]); err != nil || string(buf[:]) != "===" {
+	if _, err := io.ReadFull(out, buf[:]); err != nil {
 		t.Fatal(err)
+	} else if str := string(buf[:]); str != "===" {
+		t.Fatal(str)
 	}
 
 	testParallel(t, name, 1000)

vfs/memdb/README.md

@@ -1,6 +1,6 @@
 # Go `"memdb"` SQLite VFS
 
-This package implements the [`"memdb"`](https://sqlite.org/src/file/src/memdb.c)
+This package implements the [`"memdb"`](https://sqlite.org/src/doc/tip/src/memdb.c)
 SQLite VFS in pure Go.
 
 It has some benefits over the C version:

vfs/os_darwin.go

@@ -39,12 +39,11 @@ func osAllocate(file *os.File, size int64) error {
 		return nil
 	}
 
-	// https://stackoverflow.com/a/11497568/867786
 	store := unix.Fstore_t{
-		Flags:   unix.F_ALLOCATECONTIG,
+		Flags:   unix.F_ALLOCATEALL | unix.F_ALLOCATECONTIG,
 		Posmode: unix.F_PEOFPOSMODE,
 		Offset:  0,
-		Length:  size,
+		Length:  size - off,
 	}
 
 	// Try to get a continuous chunk of disk space.

vfs/os_unix_test.go

@@ -0,0 +1,52 @@
+//go:build unix && !sqlite3_flock && !sqlite3_nosys
+
+package vfs
+
+import (
+	"crypto/rand"
+	"io"
+	"os"
+	"runtime"
+	"testing"
+
+	"golang.org/x/sys/unix"
+)
+
+func Test_osAllocate(t *testing.T) {
+	if runtime.GOOS != "linux" && runtime.GOOS != "darwin" {
+		t.Skip()
+	}
+
+	f, err := os.CreateTemp(t.TempDir(), "file")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = io.CopyN(f, rand.Reader, 1024*1024)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	n, err := f.Seek(0, io.SeekEnd)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n != 1024*1024 {
+		t.Fatalf("got %d, want %d", n, 1024*1024)
+	}
+
+	err = osAllocate(f, 16*1024*1024)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var stat unix.Stat_t
+	err = unix.Stat(f.Name(), &stat)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if stat.Blocks*512 != 16*1024*1024 {
+		t.Fatalf("got %d, want %d", stat.Blocks*512, 16*1024*1024)
+	}
+}

vfs/tests/mptest/testdata/build.sh

@@ -5,7 +5,7 @@ cd -P -- "$(dirname -- "$0")"
 
 ROOT=../../../../
 BINARYEN="$ROOT/tools/binaryen-version_116/bin"
-WASI_SDK="$ROOT/tools/wasi-sdk-20.0/bin"
+WASI_SDK="$ROOT/tools/wasi-sdk-21.0/bin"
 
 "$WASI_SDK/clang" --target=wasm32-wasi -flto -g0 -O2 \
   -o mptest.wasm main.c \

vfs/tests/mptest/testdata/mptest.wasm.bz2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8d26f7fa2d68ba43e7d36433cfb875586f2bf7b4244ec5f97d031d042688b28d
-size 512278
+oid sha256:91dade19dcd4509f47cb09f16af0cdf7eb2b09cef0e43a0e1ee380629a8c9778
+size 512403

vfs/tests/speedtest1/testdata/build.sh

@@ -5,7 +5,7 @@ cd -P -- "$(dirname -- "$0")"
 
 ROOT=../../../../
 BINARYEN="$ROOT/tools/binaryen-version_116/bin"
-WASI_SDK="$ROOT/tools/wasi-sdk-20.0/bin"
+WASI_SDK="$ROOT/tools/wasi-sdk-21.0/bin"
 
 "$WASI_SDK/clang" --target=wasm32-wasi -flto -g0 -O2 \
 	-o speedtest1.wasm main.c \

vfs/tests/speedtest1/testdata/speedtest1.wasm.bz2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3c581db6af92832b16476c0b0d8bbdd18211f11712dcf5b7b728f24397fadfea
-size 526471
+oid sha256:ff3c4ed5a8579206a2e48c4f8e72f483288f8b113b4664e92cb308a977556a42
+size 526453