diff --git a/README.md b/README.md index 8a06a28..819fb9d 100644 --- a/README.md +++ b/README.md @@ -31,16 +31,11 @@ This has benefits, but also comes with some drawbacks. Because WASM does not support shared memory, [WAL](https://www.sqlite.org/wal.html) support is [limited](https://www.sqlite.org/wal.html#noshm). -To work around this limitation, SQLite is compiled with -[`SQLITE_DEFAULT_LOCKING_MODE=1`](https://www.sqlite.org/compile.html#default_locking_mode), -making `EXCLUSIVE` the default locking mode. -For non-WAL databases, `NORMAL` locking mode can be activated with -[`PRAGMA locking_mode=NORMAL`](https://www.sqlite.org/pragma.html#pragma_locking_mode). +To work around this limitation, SQLite is [patched](sqlite3/locking_mode.patch) +to always use `EXCLUSIVE` locking mode for WAL databases. Because connection pooling is incompatible with `EXCLUSIVE` locking mode, -the `database/sql` driver defaults to `NORMAL` locking mode. -To open WAL databases, or use `EXCLUSIVE` locking mode, -disable connection pooling by calling +to open WAL databases you should disable connection pooling by calling [`db.SetMaxOpenConns(1)`](https://pkg.go.dev/database/sql#DB.SetMaxOpenConns). #### POSIX Advisory Locks diff --git a/conn.go b/conn.go index be10c71..ec168aa 100644 --- a/conn.go +++ b/conn.go @@ -39,7 +39,7 @@ func Open(filename string) (*Conn, error) { // If none of the required flags is used, a combination of [OPEN_READWRITE] and [OPEN_CREATE] is used. // If a URI filename is used, PRAGMA statements to execute can be specified using "_pragma": // -// sqlite3.Open("file:demo.db?_pragma=busy_timeout(10000)&_pragma=locking_mode(normal)") +// sqlite3.Open("file:demo.db?_pragma=busy_timeout(10000)") // // https://www.sqlite.org/c3ref/open.html func OpenFlags(filename string, flags OpenFlag) (*Conn, error) { @@ -278,7 +278,7 @@ func (c *Conn) SetInterrupt(ctx context.Context) (old context.Context) { break case <-ctx.Done(): // Done was closed. - const isInterruptedOffset = 280 + const isInterruptedOffset = 288 buf := util.View(c.mod, c.handle+isInterruptedOffset, 4) (*atomic.Uint32)(unsafe.Pointer(&buf[0])).Store(1) // Wait for the next call to SetInterrupt. @@ -295,7 +295,7 @@ func (c *Conn) checkInterrupt() bool { if c.interrupt == nil || c.interrupt.Err() == nil { return false } - const isInterruptedOffset = 280 + const isInterruptedOffset = 288 buf := util.View(c.mod, c.handle+isInterruptedOffset, 4) (*atomic.Uint32)(unsafe.Pointer(&buf[0])).Store(1) return true diff --git a/driver/driver.go b/driver/driver.go index f027c4e..1675dd2 100644 --- a/driver/driver.go +++ b/driver/driver.go @@ -14,10 +14,9 @@ // // [PRAGMA] statements can be specified using "_pragma": // -// sql.Open("sqlite3", "file:demo.db?_pragma=busy_timeout(10000)&_pragma=locking_mode(normal)") +// sql.Open("sqlite3", "file:demo.db?_pragma=busy_timeout(10000)") // -// If no PRAGMAs are specified, a busy timeout of 1 minute -// and normal locking mode are used. +// If no PRAGMAs are specified, a busy timeout of 1 minute is set. // // Order matters: // busy timeout and locking mode should be the first PRAGMAs set, in that order. @@ -54,8 +53,8 @@ func (sqlite) Open(name string) (_ driver.Conn, err error) { return nil, err } + var pragmas bool c.txBegin = "BEGIN" - var pragmas []string if strings.HasPrefix(name, "file:") { if _, after, ok := strings.Cut(name, "?"); ok { query, _ := url.ParseQuery(after) @@ -70,14 +69,11 @@ func (sqlite) Open(name string) (_ driver.Conn, err error) { return nil, fmt.Errorf("sqlite3: invalid _txlock: %s", s) } - pragmas = query["_pragma"] + pragmas = len(query["_pragma"]) > 0 } } - if len(pragmas) == 0 { - err := c.Conn.Exec(` - PRAGMA busy_timeout=60000; - PRAGMA locking_mode=normal; - `) + if !pragmas { + err := c.Conn.Exec(`PRAGMA busy_timeout=60000`) if err != nil { c.Close() return nil, err diff --git a/embed/build.sh b/embed/build.sh index 6e15d2b..105c216 100755 --- a/embed/build.sh +++ b/embed/build.sh @@ -18,6 +18,7 @@ WASI_SDK="$ROOT/tools/wasi-sdk-20.0/bin" -Wl,--initial-memory=327680 \ -Wl,--stack-first \ -Wl,--import-undefined \ + -D_HAVE_SQLITE_CONFIG_H \ $(awk '{print "-Wl,--export="$0}' exports.txt) trap 'rm -f sqlite3.tmp' EXIT diff --git a/embed/sqlite3.wasm b/embed/sqlite3.wasm index f37c9f6..6a7c852 100755 Binary files a/embed/sqlite3.wasm and b/embed/sqlite3.wasm differ diff --git a/gormlite/README.md b/gormlite/README.md index 62ce8ea..768a5b4 100644 --- a/gormlite/README.md +++ b/gormlite/README.md @@ -21,6 +21,6 @@ Checkout [https://gorm.io](https://gorm.io) for details. Foreign-key constraint is disabled by default in SQLite. To activate it, use connection URL parameter: ```go db, err := gorm.Open(gormlite.Open( - "file:gorm.db?_pragma=busy_timeout(10000)&_pragma=locking_mode(normal)&_pragma=foreign_keys(1)"), + "file:gorm.db?_pragma=busy_timeout(10000)&_pragma=foreign_keys(1)"), &gorm.Config{}) ``` \ No newline at end of file diff --git a/sqlite3/download.sh b/sqlite3/download.sh index ff8473a..9850ffb 100755 --- a/sqlite3/download.sh +++ b/sqlite3/download.sh @@ -9,7 +9,8 @@ mv sqlite-amalgamation-*/sqlite3* . rm -rf sqlite-amalgamation-* patch < vfs_find.patch -patch < deserialize.patch +patch < open_memory.patch +patch < locking_mode.patch cd ext/ curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.42.0/ext/misc/decimal.c" diff --git a/sqlite3/locking_mode.patch b/sqlite3/locking_mode.patch new file mode 100644 index 0000000..27042b7 --- /dev/null +++ b/sqlite3/locking_mode.patch @@ -0,0 +1,14 @@ +# Use exclusive locking mode for WAL databases with v1 VFSes. +--- sqlite3.c.orig ++++ sqlite3.c +@@ -63210,7 +63210,9 @@ + SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager){ + const sqlite3_io_methods *pMethods = pPager->fd->pMethods; + if( pPager->noLock ) return 0; +- return pPager->exclusiveMode || (pMethods->iVersion>=2 && pMethods->xShmMap); ++ if( pMethods->iVersion>=2 && pMethods->xShmMap ) return 1; ++ pPager->exclusiveMode = 1; ++ return 1; + } + + /* diff --git a/sqlite3/main.c b/sqlite3/main.c index 756202f..d4b6c2d 100644 --- a/sqlite3/main.c +++ b/sqlite3/main.c @@ -1,8 +1,6 @@ #include #include -// Configuration -#include "sqlite_cfg.h" // Amalgamation #include "sqlite3.c" // VFS diff --git a/sqlite3/deserialize.patch b/sqlite3/open_memory.patch similarity index 58% rename from sqlite3/deserialize.patch rename to sqlite3/open_memory.patch index a384614..892ebe4 100644 --- a/sqlite3/deserialize.patch +++ b/sqlite3/open_memory.patch @@ -1,20 +1,26 @@ +# Allow the VFS to force memory journal mode +# regardless of SQLITE_OMIT_DESERIALIZE. --- sqlite3.c.orig +++ sqlite3.c -@@ -60425,7 +60425,7 @@ +@@ -60425,11 +60425,7 @@ int rc = SQLITE_OK; /* Return code */ int tempFile = 0; /* True for temp files (incl. in-memory files) */ int memDb = 0; /* True if this is an in-memory file */ -#ifndef SQLITE_OMIT_DESERIALIZE -+#if 1 int memJM = 0; /* Memory journal mode */ - #else - # define memJM 0 -@@ -60628,7 +60628,7 @@ +-#else +-# define memJM 0 +-#endif + int readOnly = 0; /* True if this is a read-only file */ + int journalFileSize; /* Bytes to allocate for each journal fd */ + char *zPathname = 0; /* Full path to database file */ +@@ -60628,9 +60624,7 @@ int fout = 0; /* VFS flags returned by xOpen() */ rc = sqlite3OsOpen(pVfs, pPager->zFilename, pPager->fd, vfsFlags, &fout); assert( !memDb ); -#ifndef SQLITE_OMIT_DESERIALIZE -+#if 1 pPager->memVfs = memJM = (fout&SQLITE_OPEN_MEMORY)!=0; - #endif +-#endif readOnly = (fout&SQLITE_OPEN_READONLY)!=0; + + /* If the file was successfully opened for read/write access, diff --git a/sqlite3/sqlite_cfg.h b/sqlite3/sqlite_cfg.h index e86e0e5..0ee4e41 100644 --- a/sqlite3/sqlite_cfg.h +++ b/sqlite3/sqlite_cfg.h @@ -36,12 +36,9 @@ // Because WASM does not support shared memory, // SQLite disables WAL for WASM builds. -// We set the default locking mode to EXCLUSIVE instead. +// We patch SQLite to use exclusive locking mode instead. // https://www.sqlite.org/wal.html#noshm #undef SQLITE_OMIT_WAL -#ifndef SQLITE_DEFAULT_LOCKING_MODE -#define SQLITE_DEFAULT_LOCKING_MODE 1 -#endif // Amalgamated Extensions diff --git a/sqlite3/vfs.c b/sqlite3/vfs.c index 73b0a26..0819de3 100644 --- a/sqlite3/vfs.c +++ b/sqlite3/vfs.c @@ -134,4 +134,4 @@ sqlite3_vfs *sqlite3_vfs_find(const char *zVfsName) { static_assert(offsetof(struct go_file, handle) == 4, "Unexpected offset"); static_assert(offsetof(sqlite3_vfs, zName) == 16, "Unexpected offset"); -static_assert(offsetof(sqlite3, u1.isInterrupted) == 280, "Unexpected offset"); \ No newline at end of file +static_assert(offsetof(sqlite3, u1.isInterrupted) == 288, "Unexpected offset"); \ No newline at end of file diff --git a/sqlite3/vfs_find.patch b/sqlite3/vfs_find.patch index 8b4d318..8a2d680 100644 --- a/sqlite3/vfs_find.patch +++ b/sqlite3/vfs_find.patch @@ -1,3 +1,4 @@ +# Wrap sqlite3_vfs_find. --- sqlite3.c.orig +++ sqlite3.c @@ -25394,7 +25394,7 @@ diff --git a/tests/bradfitz/sql_test.go b/tests/bradfitz/sql_test.go index 7b4c278..d76083c 100644 --- a/tests/bradfitz/sql_test.go +++ b/tests/bradfitz/sql_test.go @@ -43,7 +43,7 @@ func (t params) mustExec(sql string, args ...interface{}) sql.Result { func (sqliteDB) RunTest(t *testing.T, fn func(params)) { db, err := sql.Open("sqlite3", "file:"+ filepath.Join(t.TempDir(), "foo.db")+ - "?_pragma=busy_timeout(10000)&_pragma=locking_mode(normal)&_pragma=synchronous(off)") + "?_pragma=busy_timeout(10000)&_pragma=synchronous(off)") if err != nil { t.Fatalf("foo.db open fail: %v", err) } diff --git a/tests/db_test.go b/tests/db_test.go index 551a467..0fe005f 100644 --- a/tests/db_test.go +++ b/tests/db_test.go @@ -1,14 +1,20 @@ package tests import ( + "os" "path/filepath" "testing" + _ "embed" + "github.com/ncruces/go-sqlite3" _ "github.com/ncruces/go-sqlite3/embed" _ "github.com/ncruces/go-sqlite3/vfs/memdb" ) +//go:embed testdata/wal.db +var waldb []byte + func TestDB_memory(t *testing.T) { t.Parallel() testDB(t, ":memory:") @@ -19,6 +25,16 @@ func TestDB_file(t *testing.T) { testDB(t, filepath.Join(t.TempDir(), "test.db")) } +func TestDB_wal(t *testing.T) { + t.Parallel() + wal := filepath.Join(t.TempDir(), "test.db") + err := os.WriteFile(wal, waldb, 0666) + if err != nil { + t.Fatal(err) + } + testDB(t, wal) +} + func TestDB_vfs(t *testing.T) { testDB(t, "file:test.db?vfs=memdb") } diff --git a/tests/parallel/parallel_test.go b/tests/parallel/parallel_test.go index f30a433..42df69f 100644 --- a/tests/parallel/parallel_test.go +++ b/tests/parallel/parallel_test.go @@ -25,7 +25,6 @@ func TestParallel(t *testing.T) { name := "file:" + filepath.Join(t.TempDir(), "test.db") + "?_pragma=busy_timeout(10000)" + - "&_pragma=locking_mode(normal)" + "&_pragma=journal_mode(truncate)" + "&_pragma=synchronous(off)" testParallel(t, name, iter) @@ -42,7 +41,6 @@ func TestMemory(t *testing.T) { name := "file:/test.db?vfs=memdb" + "&_pragma=busy_timeout(10000)" + - "&_pragma=locking_mode(normal)" + "&_pragma=journal_mode(memory)" + "&_pragma=synchronous(off)" testParallel(t, name, iter) @@ -59,7 +57,6 @@ func TestMultiProcess(t *testing.T) { name := "file:" + file + "?_pragma=busy_timeout(10000)" + - "&_pragma=locking_mode(normal)" + "&_pragma=journal_mode(truncate)" + "&_pragma=synchronous(off)" @@ -93,7 +90,6 @@ func TestChildProcess(t *testing.T) { name := "file:" + file + "?_pragma=busy_timeout(10000)" + - "&_pragma=locking_mode(normal)" + "&_pragma=journal_mode(truncate)" + "&_pragma=synchronous(off)" @@ -128,10 +124,7 @@ func testParallel(t *testing.T, name string, n int) { } defer db.Close() - err = db.Exec(` - PRAGMA busy_timeout=10000; - PRAGMA locking_mode=normal; - `) + err = db.Exec(`PRAGMA busy_timeout=10000`) if err != nil { return err } diff --git a/tests/testdata/wal.db b/tests/testdata/wal.db new file mode 100644 index 0000000..e113317 Binary files /dev/null and b/tests/testdata/wal.db differ diff --git a/vfs/tests/mptest/testdata/build.sh b/vfs/tests/mptest/testdata/build.sh index 19e1d55..fc2f31f 100755 --- a/vfs/tests/mptest/testdata/build.sh +++ b/vfs/tests/mptest/testdata/build.sh @@ -16,6 +16,7 @@ WASI_SDK="$ROOT/tools/wasi-sdk-20.0/bin" -fno-stack-protector -fno-stack-clash-protection \ -Wl,--stack-first \ -Wl,--import-undefined \ + -D_HAVE_SQLITE_CONFIG_H \ -DSQLITE_DEFAULT_SYNCHRONOUS=0 \ -DSQLITE_DEFAULT_LOCKING_MODE=0 \ -DHAVE_USLEEP -DSQLITE_NO_SYNC \ diff --git a/vfs/tests/mptest/testdata/main.c b/vfs/tests/mptest/testdata/main.c index a4295c2..b6ed6e1 100644 --- a/vfs/tests/mptest/testdata/main.c +++ b/vfs/tests/mptest/testdata/main.c @@ -1,8 +1,6 @@ #include #include -// Configuration -#include "sqlite_cfg.h" // Amalgamation #include "sqlite3.c" // VFS diff --git a/vfs/tests/mptest/testdata/mptest.wasm b/vfs/tests/mptest/testdata/mptest.wasm index 06f9ed5..5405715 100644 --- a/vfs/tests/mptest/testdata/mptest.wasm +++ b/vfs/tests/mptest/testdata/mptest.wasm @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:26d10a9ba084caf25512803607466bb7224515f616af9e1eb3bf5d234898597e -size 1417255 +oid sha256:d23b37d507077cdcbb616852185370a227278b599187dc134200ed274a7a3a02 +size 1441194 diff --git a/vfs/tests/speedtest1/testdata/build.sh b/vfs/tests/speedtest1/testdata/build.sh index 60e06e5..0ef94e6 100755 --- a/vfs/tests/speedtest1/testdata/build.sh +++ b/vfs/tests/speedtest1/testdata/build.sh @@ -15,7 +15,8 @@ WASI_SDK="$ROOT/tools/wasi-sdk-20.0/bin" -mnontrapping-fptoint -msign-ext \ -fno-stack-protector -fno-stack-clash-protection \ -Wl,--stack-first \ - -Wl,--import-undefined + -Wl,--import-undefined \ + -D_HAVE_SQLITE_CONFIG_H "$BINARYEN/wasm-opt" -g --strip -c -O3 \ speedtest1.wasm -o speedtest1.tmp \ diff --git a/vfs/tests/speedtest1/testdata/main.c b/vfs/tests/speedtest1/testdata/main.c index 2998839..f208fdc 100644 --- a/vfs/tests/speedtest1/testdata/main.c +++ b/vfs/tests/speedtest1/testdata/main.c @@ -1,8 +1,6 @@ #include #include -// Configuration -#include "sqlite_cfg.h" // Amalgamation #include "sqlite3.c" // VFS diff --git a/vfs/tests/speedtest1/testdata/speedtest1.wasm b/vfs/tests/speedtest1/testdata/speedtest1.wasm index 0eca77f..b1e7c99 100644 --- a/vfs/tests/speedtest1/testdata/speedtest1.wasm +++ b/vfs/tests/speedtest1/testdata/speedtest1.wasm @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e7b58aeeecd691b546c1da531bfea0a67bb33e1c835f06a94d3bd8723ab275d2 -size 1457505 +oid sha256:83d67feda51cc974634e245ac2b072f9587c607c7ad97321f2de9dde2188e63a +size 1481348