Fix WAL.

README.md

@@ -31,16 +31,11 @@ This has benefits, but also comes with some drawbacks.
 Because WASM does not support shared memory,
 [WAL](https://www.sqlite.org/wal.html) support is [limited](https://www.sqlite.org/wal.html#noshm).
 
-To work around this limitation, SQLite is compiled with
-[`SQLITE_DEFAULT_LOCKING_MODE=1`](https://www.sqlite.org/compile.html#default_locking_mode),
-making `EXCLUSIVE` the default locking mode.
-For non-WAL databases, `NORMAL` locking mode can be activated with
-[`PRAGMA locking_mode=NORMAL`](https://www.sqlite.org/pragma.html#pragma_locking_mode).
+To work around this limitation, SQLite is [patched](sqlite3/locking_mode.patch)
+to always use `EXCLUSIVE` locking mode for WAL databases.
 
 Because connection pooling is incompatible with `EXCLUSIVE` locking mode,
-the `database/sql` driver defaults to `NORMAL` locking mode.
-To open WAL databases, or use `EXCLUSIVE` locking mode,
-disable connection pooling by calling
+to open WAL databases you should disable connection pooling by calling
 [`db.SetMaxOpenConns(1)`](https://pkg.go.dev/database/sql#DB.SetMaxOpenConns).
 
 #### POSIX Advisory Locks

conn.go

@@ -39,7 +39,7 @@ func Open(filename string) (*Conn, error) {
 // If none of the required flags is used, a combination of [OPEN_READWRITE] and [OPEN_CREATE] is used.
 // If a URI filename is used, PRAGMA statements to execute can be specified using "_pragma":
 //
-//	sqlite3.Open("file:demo.db?_pragma=busy_timeout(10000)&_pragma=locking_mode(normal)")
+//	sqlite3.Open("file:demo.db?_pragma=busy_timeout(10000)")
 //
 // https://www.sqlite.org/c3ref/open.html
 func OpenFlags(filename string, flags OpenFlag) (*Conn, error) {
@@ -278,7 +278,7 @@ func (c *Conn) SetInterrupt(ctx context.Context) (old context.Context) {
 			break
 
 		case <-ctx.Done(): // Done was closed.
-			const isInterruptedOffset = 280
+			const isInterruptedOffset = 288
 			buf := util.View(c.mod, c.handle+isInterruptedOffset, 4)
 			(*atomic.Uint32)(unsafe.Pointer(&buf[0])).Store(1)
 			// Wait for the next call to SetInterrupt.
@@ -295,7 +295,7 @@ func (c *Conn) checkInterrupt() bool {
 	if c.interrupt == nil || c.interrupt.Err() == nil {
 		return false
 	}
-	const isInterruptedOffset = 280
+	const isInterruptedOffset = 288
 	buf := util.View(c.mod, c.handle+isInterruptedOffset, 4)
 	(*atomic.Uint32)(unsafe.Pointer(&buf[0])).Store(1)
 	return true

driver/driver.go

@@ -14,10 +14,9 @@
 //
 // [PRAGMA] statements can be specified using "_pragma":
 //
-//	sql.Open("sqlite3", "file:demo.db?_pragma=busy_timeout(10000)&_pragma=locking_mode(normal)")
+//	sql.Open("sqlite3", "file:demo.db?_pragma=busy_timeout(10000)")
 //
-// If no PRAGMAs are specified, a busy timeout of 1 minute
-// and normal locking mode are used.
+// If no PRAGMAs are specified, a busy timeout of 1 minute is set.
 //
 // Order matters:
 // busy timeout and locking mode should be the first PRAGMAs set, in that order.
@@ -54,8 +53,8 @@ func (sqlite) Open(name string) (_ driver.Conn, err error) {
 		return nil, err
 	}
 
+	var pragmas bool
 	c.txBegin = "BEGIN"
-	var pragmas []string
 	if strings.HasPrefix(name, "file:") {
 		if _, after, ok := strings.Cut(name, "?"); ok {
 			query, _ := url.ParseQuery(after)
@@ -70,14 +69,11 @@ func (sqlite) Open(name string) (_ driver.Conn, err error) {
 				return nil, fmt.Errorf("sqlite3: invalid _txlock: %s", s)
 			}
 
-			pragmas = query["_pragma"]
+			pragmas = len(query["_pragma"]) > 0
 		}
 	}
-	if len(pragmas) == 0 {
-		err := c.Conn.Exec(`
-			PRAGMA busy_timeout=60000;
-			PRAGMA locking_mode=normal;
-		`)
+	if !pragmas {
+		err := c.Conn.Exec(`PRAGMA busy_timeout=60000`)
 		if err != nil {
 			c.Close()
 			return nil, err

embed/build.sh

@@ -11,17 +11,20 @@ WASI_SDK="$ROOT/tools/wasi-sdk-20.0/bin"
 	-o sqlite3.wasm "$ROOT/sqlite3/main.c" \
 	-I"$ROOT/sqlite3" \
 	-mexec-model=reactor \
-	-mmutable-globals \
+	-msimd128 -mmutable-globals \
 	-mbulk-memory -mreference-types \
 	-mnontrapping-fptoint -msign-ext \
+	-fno-stack-protector -fno-stack-clash-protection \
 	-Wl,--initial-memory=327680 \
 	-Wl,--stack-first \
 	-Wl,--import-undefined \
+	-D_HAVE_SQLITE_CONFIG_H \
 	$(awk '{print "-Wl,--export="$0}' exports.txt)
 
 trap 'rm -f sqlite3.tmp' EXIT
 "$BINARYEN/wasm-ctor-eval" -g -c _initialize sqlite3.wasm -o sqlite3.tmp
-"$BINARYEN/wasm-opt" -g -O2 sqlite3.tmp -o sqlite3.wasm \
-	--enable-multivalue --enable-mutable-globals \
+"$BINARYEN/wasm-opt" -g --strip -c -O3 \
+	sqlite3.tmp -o sqlite3.wasm \
+	--enable-simd --enable-mutable-globals --enable-multivalue \
 	--enable-bulk-memory --enable-reference-types \
 	--enable-nontrapping-float-to-int --enable-sign-ext

gormlite/README.md

@@ -21,6 +21,6 @@ Checkout [https://gorm.io](https://gorm.io) for details.
 Foreign-key constraint is disabled by default in SQLite. To activate it, use connection URL parameter:
 ```go
 db, err := gorm.Open(gormlite.Open(
-	"file:gorm.db?_pragma=busy_timeout(10000)&_pragma=locking_mode(normal)&_pragma=foreign_keys(1)"),
+	"file:gorm.db?_pragma=busy_timeout(10000)&_pragma=foreign_keys(1)"),
 	&gorm.Config{})
 ```

gormlite/go.mod

@@ -3,7 +3,7 @@ module github.com/ncruces/go-sqlite3/gormlite
 go 1.19
 
 require (
-	github.com/ncruces/go-sqlite3 v0.7.2
+	github.com/ncruces/go-sqlite3 v0.8.0
 	gorm.io/gorm v1.25.1
 )
 

gormlite/go.sum

@@ -2,8 +2,8 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
 github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
-github.com/ncruces/go-sqlite3 v0.7.2 h1:K7jU4rnUxFdUsbEL+B0Xc+VexLTEwGSO6Qh91Qh4hYc=
-github.com/ncruces/go-sqlite3 v0.7.2/go.mod h1:t3dP4AP9rJddU+ffFv0h6fWyeOCEhjxrYc1nsYG7aQI=
+github.com/ncruces/go-sqlite3 v0.8.0 h1:9FsLTTPqHE/GyW42dGqe9KUXDt4QHRaKf67Lk7jvdIQ=
+github.com/ncruces/go-sqlite3 v0.8.0/go.mod h1:EhHe1qvG6Zc/8ffYMzre8n//rTRs1YNN5dUD1f1mEGc=
 github.com/ncruces/julianday v0.1.5 h1:hDJ9ejiMp3DHsoZ5KW4c1lwfMjbARS7u/gbYcd0FBZk=
 github.com/ncruces/julianday v0.1.5/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
 github.com/tetratelabs/wazero v1.2.1 h1:J4X2hrGzJvt+wqltuvcSjHQ7ujQxA9gb6PeMs4qlUWs=

module.go

@@ -23,7 +23,6 @@ import (
 var (
 	Binary []byte // WASM binary to load.
 	Path   string // Path to load the binary from.
-	Debug  bool   // Whether to enable SQLite debug stack traces.
 )
 
 var sqlite3 struct {
@@ -52,7 +51,7 @@ func instantiateModule() (*module, error) {
 
 func compileModule() {
 	ctx := context.Background()
-	sqlite3.runtime = wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfig().WithDebugInfoEnabled(Debug))
+	sqlite3.runtime = wazero.NewRuntime(ctx)
 
 	env := vfs.ExportHostFunctions(sqlite3.runtime.NewHostModuleBuilder("env"))
 	_, sqlite3.err = env.Instantiate(ctx)

sqlite3/download.sh

@@ -9,7 +9,8 @@ mv sqlite-amalgamation-*/sqlite3* .
 rm -rf sqlite-amalgamation-*
 
 patch < vfs_find.patch
-patch < deserialize.patch
+patch < open_memory.patch
+patch < locking_mode.patch
 
 cd ext/
 curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.42.0/ext/misc/decimal.c"

sqlite3/locking_mode.patch

@@ -0,0 +1,14 @@
+# Use exclusive locking mode for WAL databases with v1 VFSes.
+--- sqlite3.c.orig
++++ sqlite3.c
+@@ -63210,7 +63210,9 @@
+ SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager){
+   const sqlite3_io_methods *pMethods = pPager->fd->pMethods;
+   if( pPager->noLock ) return 0;
+-  return pPager->exclusiveMode || (pMethods->iVersion>=2 && pMethods->xShmMap);
++  if( pMethods->iVersion>=2 && pMethods->xShmMap ) return 1;
++  pPager->exclusiveMode = 1; 
++  return 1;
+ }
+ 
+ /*

sqlite3/main.c

@@ -1,8 +1,6 @@
 #include <stdbool.h>
 #include <stddef.h>
 
-// Configuration
-#include "sqlite_cfg.h"
 // Amalgamation
 #include "sqlite3.c"
 // VFS

sqlite3/open_memory.patch

@@ -1,20 +1,26 @@
+# Allow the VFS to force memory journal mode
+# regardless of SQLITE_OMIT_DESERIALIZE.
 --- sqlite3.c.orig
 +++ sqlite3.c
-@@ -60425,7 +60425,7 @@
+@@ -60425,11 +60425,7 @@
    int rc = SQLITE_OK;      /* Return code */
    int tempFile = 0;        /* True for temp files (incl. in-memory files) */
    int memDb = 0;           /* True if this is an in-memory file */
 -#ifndef SQLITE_OMIT_DESERIALIZE
-+#if 1
    int memJM = 0;           /* Memory journal mode */
- #else
- # define memJM 0
-@@ -60628,7 +60628,7 @@
+-#else
+-# define memJM 0
+-#endif
+   int readOnly = 0;        /* True if this is a read-only file */
+   int journalFileSize;     /* Bytes to allocate for each journal fd */
+   char *zPathname = 0;     /* Full path to database file */
+@@ -60628,9 +60624,7 @@
      int fout = 0;                    /* VFS flags returned by xOpen() */
      rc = sqlite3OsOpen(pVfs, pPager->zFilename, pPager->fd, vfsFlags, &fout);
      assert( !memDb );
 -#ifndef SQLITE_OMIT_DESERIALIZE
-+#if 1
      pPager->memVfs = memJM = (fout&SQLITE_OPEN_MEMORY)!=0;
- #endif
+-#endif
      readOnly = (fout&SQLITE_OPEN_READONLY)!=0;
+ 
+     /* If the file was successfully opened for read/write access,

sqlite3/sqlite_cfg.h

@@ -36,12 +36,9 @@
 
 // Because WASM does not support shared memory,
 // SQLite disables WAL for WASM builds.
-// We set the default locking mode to EXCLUSIVE instead.
+// We patch SQLite to use exclusive locking mode instead.
 // https://www.sqlite.org/wal.html#noshm
 #undef SQLITE_OMIT_WAL
-#ifndef SQLITE_DEFAULT_LOCKING_MODE
-#define SQLITE_DEFAULT_LOCKING_MODE 1
-#endif
 
 // Amalgamated Extensions
 

sqlite3/vfs.c

@@ -134,4 +134,4 @@ sqlite3_vfs *sqlite3_vfs_find(const char *zVfsName) {
 
 static_assert(offsetof(struct go_file, handle) == 4, "Unexpected offset");
 static_assert(offsetof(sqlite3_vfs, zName) == 16, "Unexpected offset");
-static_assert(offsetof(sqlite3, u1.isInterrupted) == 280, "Unexpected offset");
+static_assert(offsetof(sqlite3, u1.isInterrupted) == 288, "Unexpected offset");

sqlite3/vfs_find.patch

@@ -1,3 +1,4 @@
+# Wrap sqlite3_vfs_find.
 --- sqlite3.c.orig
 +++ sqlite3.c
 @@ -25394,7 +25394,7 @@

tests/bradfitz/sql_test.go

@@ -43,7 +43,7 @@ func (t params) mustExec(sql string, args ...interface{}) sql.Result {
 func (sqliteDB) RunTest(t *testing.T, fn func(params)) {
 	db, err := sql.Open("sqlite3", "file:"+
 		filepath.Join(t.TempDir(), "foo.db")+
-		"?_pragma=busy_timeout(10000)&_pragma=locking_mode(normal)&_pragma=synchronous(off)")
+		"?_pragma=busy_timeout(10000)&_pragma=synchronous(off)")
 	if err != nil {
 		t.Fatalf("foo.db open fail: %v", err)
 	}

tests/db_test.go

@@ -1,14 +1,20 @@
 package tests
 
 import (
+	"os"
 	"path/filepath"
 	"testing"
 
+	_ "embed"
+
 	"github.com/ncruces/go-sqlite3"
 	_ "github.com/ncruces/go-sqlite3/embed"
 	_ "github.com/ncruces/go-sqlite3/vfs/memdb"
 )
 
+//go:embed testdata/wal.db
+var waldb []byte
+
 func TestDB_memory(t *testing.T) {
 	t.Parallel()
 	testDB(t, ":memory:")
@@ -19,6 +25,16 @@ func TestDB_file(t *testing.T) {
 	testDB(t, filepath.Join(t.TempDir(), "test.db"))
 }
 
+func TestDB_wal(t *testing.T) {
+	t.Parallel()
+	wal := filepath.Join(t.TempDir(), "test.db")
+	err := os.WriteFile(wal, waldb, 0666)
+	if err != nil {
+		t.Fatal(err)
+	}
+	testDB(t, wal)
+}
+
 func TestDB_vfs(t *testing.T) {
 	testDB(t, "file:test.db?vfs=memdb")
 }

tests/parallel/parallel_test.go

@@ -25,7 +25,6 @@ func TestParallel(t *testing.T) {
 	name := "file:" +
 		filepath.Join(t.TempDir(), "test.db") +
 		"?_pragma=busy_timeout(10000)" +
-		"&_pragma=locking_mode(normal)" +
 		"&_pragma=journal_mode(truncate)" +
 		"&_pragma=synchronous(off)"
 	testParallel(t, name, iter)
@@ -42,7 +41,6 @@ func TestMemory(t *testing.T) {
 
 	name := "file:/test.db?vfs=memdb" +
 		"&_pragma=busy_timeout(10000)" +
-		"&_pragma=locking_mode(normal)" +
 		"&_pragma=journal_mode(memory)" +
 		"&_pragma=synchronous(off)"
 	testParallel(t, name, iter)
@@ -59,7 +57,6 @@ func TestMultiProcess(t *testing.T) {
 
 	name := "file:" + file +
 		"?_pragma=busy_timeout(10000)" +
-		"&_pragma=locking_mode(normal)" +
 		"&_pragma=journal_mode(truncate)" +
 		"&_pragma=synchronous(off)"
 
@@ -93,7 +90,6 @@ func TestChildProcess(t *testing.T) {
 
 	name := "file:" + file +
 		"?_pragma=busy_timeout(10000)" +
-		"&_pragma=locking_mode(normal)" +
 		"&_pragma=journal_mode(truncate)" +
 		"&_pragma=synchronous(off)"
 
@@ -128,10 +124,7 @@ func testParallel(t *testing.T, name string, n int) {
 		}
 		defer db.Close()
 
-		err = db.Exec(`
-			PRAGMA busy_timeout=10000;
-			PRAGMA locking_mode=normal;
-		`)
+		err = db.Exec(`PRAGMA busy_timeout=10000`)
 		if err != nil {
 			return err
 		}

vfs/tests/mptest/mptest_test.go

@@ -37,7 +37,7 @@ var (
 
 func TestMain(m *testing.M) {
 	ctx := context.Background()
-	rt = wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfig().WithDebugInfoEnabled(false))
+	rt = wazero.NewRuntime(ctx)
 	wasi_snapshot_preview1.MustInstantiate(ctx, rt)
 
 	env := vfs.ExportHostFunctions(rt.NewHostModuleBuilder("env"))

vfs/tests/mptest/testdata/build.sh

@@ -10,19 +10,22 @@ WASI_SDK="$ROOT/tools/wasi-sdk-20.0/bin"
 "$WASI_SDK/clang" --target=wasm32-wasi -flto -g0 -O2 \
   -o mptest.wasm main.c \
 	-I"$ROOT/sqlite3" \
-	-mmutable-globals \
+	-msimd128 -mmutable-globals \
 	-mbulk-memory -mreference-types \
 	-mnontrapping-fptoint -msign-ext \
+	-fno-stack-protector -fno-stack-clash-protection \
 	-Wl,--stack-first \
 	-Wl,--import-undefined \
+	-D_HAVE_SQLITE_CONFIG_H \
 	-DSQLITE_DEFAULT_SYNCHRONOUS=0 \
 	-DSQLITE_DEFAULT_LOCKING_MODE=0 \
 	-DHAVE_USLEEP -DSQLITE_NO_SYNC \
 	-DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 	-D_WASI_EMULATED_GETPID -lwasi-emulated-getpid
 
-"$BINARYEN/wasm-opt" -g -O2 mptest.wasm -o mptest.tmp \
-	--enable-multivalue --enable-mutable-globals \
+"$BINARYEN/wasm-opt" -g --strip -c -O3 \
+	mptest.wasm -o mptest.tmp \
+	--enable-simd --enable-mutable-globals --enable-multivalue \
 	--enable-bulk-memory --enable-reference-types \
 	--enable-nontrapping-float-to-int --enable-sign-ext
 mv mptest.tmp mptest.wasm

vfs/tests/mptest/testdata/main.c

@@ -1,8 +1,6 @@
 #include <stdbool.h>
 #include <stddef.h>
 
-// Configuration
-#include "sqlite_cfg.h"
 // Amalgamation
 #include "sqlite3.c"
 // VFS

vfs/tests/mptest/testdata/mptest.wasm

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:53bb204be0ddd312c23976e7b627ad62fb384d220557f4e3f723d89273d78a01
-size 1477009
+oid sha256:d23b37d507077cdcbb616852185370a227278b599187dc134200ed274a7a3a02
+size 1441194

vfs/tests/speedtest1/speedtest1_test.go

@@ -36,7 +36,7 @@ func TestMain(m *testing.M) {
 	initFlags()
 
 	ctx := context.Background()
-	rt = wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfig().WithDebugInfoEnabled(false))
+	rt = wazero.NewRuntime(ctx)
 	wasi_snapshot_preview1.MustInstantiate(ctx, rt)
 	env := vfs.ExportHostFunctions(rt.NewHostModuleBuilder("env"))
 	_, err := env.Instantiate(ctx)

vfs/tests/speedtest1/testdata/build.sh

@@ -10,14 +10,17 @@ WASI_SDK="$ROOT/tools/wasi-sdk-20.0/bin"
 "$WASI_SDK/clang" --target=wasm32-wasi -flto -g0 -O2 \
 	-o speedtest1.wasm main.c \
 	-I"$ROOT/sqlite3" \
-	-mmutable-globals \
+	-msimd128 -mmutable-globals \
 	-mbulk-memory -mreference-types \
 	-mnontrapping-fptoint -msign-ext \
+	-fno-stack-protector -fno-stack-clash-protection \
 	-Wl,--stack-first \
-	-Wl,--import-undefined
+	-Wl,--import-undefined \
+	-D_HAVE_SQLITE_CONFIG_H
 
-"$BINARYEN/wasm-opt" -g -O2 speedtest1.wasm -o speedtest1.tmp \
-	--enable-multivalue --enable-mutable-globals \
+"$BINARYEN/wasm-opt" -g --strip -c -O3 \
+	speedtest1.wasm -o speedtest1.tmp \
+	--enable-simd --enable-mutable-globals --enable-multivalue \
 	--enable-bulk-memory --enable-reference-types \
 	--enable-nontrapping-float-to-int --enable-sign-ext
 mv speedtest1.tmp speedtest1.wasm

vfs/tests/speedtest1/testdata/main.c

@@ -1,8 +1,6 @@
 #include <stdbool.h>
 #include <stddef.h>
 
-// Configuration
-#include "sqlite_cfg.h"
 // Amalgamation
 #include "sqlite3.c"
 // VFS

vfs/tests/speedtest1/testdata/speedtest1.wasm

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:23ac5877bc34cf5ff06ed2cf1c5ac148e5d364d6d01c85bd0f99d14f0a681830
-size 1513516
+oid sha256:83d67feda51cc974634e245ac2b072f9587c607c7ad97321f2de9dde2188e63a
+size 1481348