Compare commits

...

24 Commits

Author SHA1 Message Date
Nuno Cruces
37a3ff37e8 wazero 1.0. 2023-03-24 21:17:30 +00:00
Nuno Cruces
d880d6842c Refactor VFS. 2023-03-23 13:29:26 +00:00
Nuno Cruces
bef46e7954 Locking improvements (windows). 2023-03-23 12:40:55 +00:00
Nuno Cruces
4e72b4d117 Locking fix. 2023-03-23 11:26:19 +00:00
Nuno Cruces
3b08d02a83 Lock refactoring. 2023-03-23 01:55:54 +00:00
Nuno Cruces
b19c12c4c7 SQLite 3.41.2, prefer speed over size. 2023-03-23 00:44:43 +00:00
Nuno Cruces
859a21ef4e CI improvements. 2023-03-22 12:08:33 +00:00
Nuno Cruces
8ff0ee752f Use flock. 2023-03-22 03:15:54 +00:00
Nuno Cruces
589ad86f76 Extensions. 2023-03-21 00:13:12 +00:00
Nuno Cruces
1a3a1be1f6 Fix test. 2023-03-20 14:26:25 +00:00
Nuno Cruces
222c217bc8 Scripts. 2023-03-20 13:06:31 +00:00
Nuno Cruces
c1dc716391 VFS performance. 2023-03-20 11:02:34 +00:00
Nuno Cruces
71e1e5a8ee Avoid some copies. 2023-03-20 02:16:42 +00:00
Nuno Cruces
e4efb20c71 Generate coverage chart. 2023-03-18 03:51:05 +00:00
Nuno Cruces
2c9459d907 Add SQLite speedtest1. 2023-03-18 03:03:11 +00:00
Nuno Cruces
d0875e5fab Lock timeouts. 2023-03-18 01:13:31 +00:00
Nuno Cruces
15dec13f15 FCNTL_SIZE_HINT, refactor. 2023-03-17 17:13:03 +00:00
Nuno Cruces
f38e36109a FCNTL_HAS_MOVED. 2023-03-17 14:11:09 +00:00
Nuno Cruces
4cb65ccbd9 xFileControl, xDeviceCharacteristics, PSOW. 2023-03-17 13:39:19 +00:00
Nuno Cruces
f789c2fb8b OPEN_NOFOLLOW. 2023-03-16 12:27:44 +00:00
Nuno Cruces
c6a2617dfc Locking fixes. 2023-03-16 02:52:22 +00:00
Nuno Cruces
6fc0afcd12 Towards lock timeouts. 2023-03-15 13:58:16 +00:00
Nuno Cruces
77088962f5 SQLite 3.41.1. 2023-03-15 13:29:09 +00:00
Nuno Cruces
71da34861b Fix time collation. 2023-03-13 04:19:58 +00:00
50 changed files with 1234 additions and 496 deletions

View File

@@ -18,11 +18,27 @@ jobs:
with:
lfs: 'true'
- name: Set up Go
uses: actions/setup-go@v3
- name: Set up
uses: actions/setup-go@v4
with:
go-version: stable
cache: true
- name: Format
run: gofmt -s -w . && git diff --exit-code
if: matrix.os != 'windows-latest'
- name: Tidy
run: go mod tidy && git diff --exit-code
- name: Download
run: go mod download
- name: Verify
run: go mod verify
- name: Vet
run: go vet ./...
continue-on-error: true
- name: Build
run: go build -v ./...
@@ -34,8 +50,15 @@ jobs:
run: go test -v -race ./...
if: matrix.os == 'ubuntu-latest'
- name: Update coverage report
- name: Test BSD locks
run: go test -v -tags sqlite3_bsd ./...
if: matrix.os == 'macos-latest'
- name: Coverage report
uses: ncruces/go-coverage-report@main
with:
chart: 'true'
amend: 'true'
if: |
matrix.os == 'ubuntu-latest' &&
github.event_name == 'push'

View File

@@ -20,7 +20,7 @@ embeds a build of SQLite into your application.
This module replaces the SQLite [OS Interface](https://www.sqlite.org/vfs.html) (aka VFS)
with a pure Go implementation.
This has numerous benefits, but also comes with some caveats.
This has numerous benefits, but also comes with some drawbacks.
#### Write-Ahead Logging
@@ -39,21 +39,27 @@ To open WAL databases, or use `EXCLUSIVE` locking mode,
disable connection pooling by calling
[`db.SetMaxOpenConns(1)`](https://pkg.go.dev/database/sql#DB.SetMaxOpenConns).
#### Open File Description Locks
#### POSIX Advisory Locks
On Unix, this module uses [OFD locks](https://www.gnu.org/software/libc/manual/html_node/Open-File-Description-Locks.html)
POSIX advisory locks, which SQLite uses, are
[broken by design](https://www.sqlite.org/src/artifact/90c4fa?ln=1073-1161).
On Linux, macOS and illumos, this module uses
[OFD locks](https://www.gnu.org/software/libc/manual/html_node/Open-File-Description-Locks.html)
to synchronize access to database files.
OFD locks are fully compatible with process-associated POSIX advisory locks.
POSIX advisory locks, which SQLite uses, are [broken by design](https://www.sqlite.org/src/artifact/90c4fa?ln=1073-1161).
OFD locks are fully compatible with process-associated POSIX advisory locks,
and are supported on Linux, macOS and illumos.
As a work around for other Unixes, you can use [`nolock=1`](https://www.sqlite.org/uri.html).
On BSD Unixes, this module uses
[BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2).
BSD locks may _not_ be compatible with process-associated POSIX advisory locks.
#### Testing
The pure Go VFS is stress tested by running an unmodified build of SQLite's
[mptest](https://github.com/sqlite/sqlite/blob/master/mptest/mptest.c)
on Linux, macOS and Windows.
Performance is tested by running
[speedtest1](https://github.com/sqlite/sqlite/blob/master/test/speedtest1.c).
### Roadmap
@@ -61,18 +67,14 @@ on Linux, macOS and Windows.
- [x] nested transactions
- [x] incremental BLOB I/O
- [x] online backup
- [ ] snapshots
- [ ] session extension
- [ ] resumable bulk update
- [ ] shared-cache mode
- [ ] unlock-notify
- [ ] custom SQL functions
- [ ] custom VFSes
- [ ] in-memory VFS
- [ ] read-only VFS, wrapping an [`io.ReaderAt`](https://pkg.go.dev/io#ReaderAt)
- [ ] in-memory VFS, wrapping a [`bytes.Buffer`](https://pkg.go.dev/bytes#Buffer)
- [ ] cloud-based VFS, based on [Cloud Backed SQLite](https://sqlite.org/cloudsqlite/doc/trunk/www/index.wiki)
- [ ] custom VFS API
### Alternatives
- [`modernc.org/sqlite`](https://pkg.go.dev/modernc.org/sqlite)

View File

@@ -33,7 +33,7 @@ func (src *Conn) Backup(srcDB, dstURI string) error {
//
// https://www.sqlite.org/backup.html
func (dst *Conn) Restore(dstDB, srcURI string) error {
src, err := dst.openDB(srcURI, OPEN_READWRITE|OPEN_CREATE|OPEN_URI)
src, err := dst.openDB(srcURI, OPEN_READONLY|OPEN_URI)
if err != nil {
return err
}

10
conn.go
View File

@@ -26,19 +26,23 @@ type Conn struct {
pending *Stmt
}
// Open calls [OpenFlags] with [OPEN_READWRITE], [OPEN_CREATE] and [OPEN_URI].
// Open calls [OpenFlags] with [OPEN_READWRITE], [OPEN_CREATE], [OPEN_URI] and [OPEN_NOFOLLOW].
func Open(filename string) (*Conn, error) {
return newConn(filename, OPEN_READWRITE|OPEN_CREATE|OPEN_URI)
return newConn(filename, OPEN_READWRITE|OPEN_CREATE|OPEN_URI|OPEN_NOFOLLOW)
}
// OpenFlags opens an SQLite database file as specified by the filename argument.
//
// If none of the required flags is used, a combination of [OPEN_READWRITE] and [OPEN_CREATE] is used.
// If a URI filename is used, PRAGMA statements to execute can be specified using "_pragma":
//
// sqlite3.Open("file:demo.db?_pragma=busy_timeout(10000)&_pragma=locking_mode(normal)")
//
// https://www.sqlite.org/c3ref/open.html
func OpenFlags(filename string, flags OpenFlag) (*Conn, error) {
if flags&(OPEN_READONLY|OPEN_READWRITE|OPEN_CREATE) == 0 {
flags |= OPEN_READWRITE | OPEN_CREATE
}
return newConn(filename, flags)
}
@@ -69,6 +73,7 @@ func (c *Conn) openDB(filename string, flags OpenFlag) (uint32, error) {
connPtr := c.arena.new(ptrlen)
namePtr := c.arena.string(filename)
flags |= OPEN_EXRESCODE
r := c.call(c.api.open, uint64(namePtr), uint64(connPtr), uint64(flags), 0)
handle := c.mem.readUint32(connPtr)
@@ -100,7 +105,6 @@ func (c *Conn) openDB(filename string, flags OpenFlag) (uint32, error) {
}
}
c.call(c.api.timeCollation, uint64(handle))
return handle, nil
}

View File

@@ -7,6 +7,8 @@ const (
_ROW = 100 /* sqlite3_step() has another row ready */
_DONE = 101 /* sqlite3_step() has finished executing */
_OK_SYMLINK = (_OK | (2 << 8)) /* internal use only */
_UTF8 = 1
_MAX_STRING = 512 // Used for short strings: names, error messages…
@@ -225,3 +227,49 @@ const (
_SYNC_FULL _SyncFlag = 0x00003
_SYNC_DATAONLY _SyncFlag = 0x00010
)
type _FcntlOpcode uint32
const (
_FCNTL_LOCKSTATE = 1
_FCNTL_GET_LOCKPROXYFILE = 2
_FCNTL_SET_LOCKPROXYFILE = 3
_FCNTL_LAST_ERRNO = 4
_FCNTL_SIZE_HINT = 5
_FCNTL_CHUNK_SIZE = 6
_FCNTL_FILE_POINTER = 7
_FCNTL_SYNC_OMITTED = 8
_FCNTL_WIN32_AV_RETRY = 9
_FCNTL_PERSIST_WAL = 10
_FCNTL_OVERWRITE = 11
_FCNTL_VFSNAME = 12
_FCNTL_POWERSAFE_OVERWRITE = 13
_FCNTL_PRAGMA = 14
_FCNTL_BUSYHANDLER = 15
_FCNTL_TEMPFILENAME = 16
_FCNTL_MMAP_SIZE = 18
_FCNTL_TRACE = 19
_FCNTL_HAS_MOVED = 20
_FCNTL_SYNC = 21
_FCNTL_COMMIT_PHASETWO = 22
_FCNTL_WIN32_SET_HANDLE = 23
_FCNTL_WAL_BLOCK = 24
_FCNTL_ZIPVFS = 25
_FCNTL_RBU = 26
_FCNTL_VFS_POINTER = 27
_FCNTL_JOURNAL_POINTER = 28
_FCNTL_WIN32_GET_HANDLE = 29
_FCNTL_PDB = 30
_FCNTL_BEGIN_ATOMIC_WRITE = 31
_FCNTL_COMMIT_ATOMIC_WRITE = 32
_FCNTL_ROLLBACK_ATOMIC_WRITE = 33
_FCNTL_LOCK_TIMEOUT = 34
_FCNTL_DATA_VERSION = 35
_FCNTL_SIZE_LIMIT = 36
_FCNTL_CKPT_DONE = 37
_FCNTL_RESERVE_BYTES = 38
_FCNTL_CKPT_START = 39
_FCNTL_EXTERNAL_READER = 40
_FCNTL_CKSM_FILE = 41
_FCNTL_RESET_CACHE = 42
)

View File

@@ -44,7 +44,7 @@ func init() {
type sqlite struct{}
func (sqlite) Open(name string) (_ driver.Conn, err error) {
c, err := sqlite3.OpenFlags(name, sqlite3.OPEN_READWRITE|sqlite3.OPEN_CREATE|sqlite3.OPEN_URI|sqlite3.OPEN_EXRESCODE)
c, err := sqlite3.Open(name)
if err != nil {
return nil, err
}
@@ -386,8 +386,7 @@ func (r rows) Next(dest []driver.Value) error {
case sqlite3.TEXT:
dest[i] = maybeTime(r.stmt.ColumnText(i))
case sqlite3.BLOB:
buf, _ := dest[i].([]byte)
dest[i] = r.stmt.ColumnBlob(i, buf)
dest[i] = r.stmt.ColumnRawBlob(i)
case sqlite3.NULL:
if buf, ok := dest[i].([]byte); ok {
dest[i] = buf[0:0]

View File

@@ -308,7 +308,7 @@ func Test_QueryRow_blob_null(t *testing.T) {
want := [][]byte{nil, {0xca, 0xfe}, {0xba, 0xbe}, nil}
for i := 0; rows.Next(); i++ {
var buf []byte
var buf sql.RawBytes
err = rows.Scan(&buf)
if err != nil {
t.Fatal(err)

View File

@@ -1,14 +1,21 @@
# Embeddable WASM build of SQLite
This folder includes an embeddable WASM build of SQLite 3.41.0 for use with
This folder includes an embeddable WASM build of SQLite 3.41.2 for use with
[`github.com/ncruces/go-sqlite3`](https://pkg.go.dev/github.com/ncruces/go-sqlite3).
The following optional features are compiled in:
- math functions
- FTS3/4/5
- JSON
- R*Tree
- GeoPoly
- [math functions](https://www.sqlite.org/lang_mathfunc.html)
- [FTS3/4](https://www.sqlite.org/fts3.html)/[5](https://www.sqlite.org/fts5.html)
- [JSON](https://www.sqlite.org/json1.html)
- [R*Tree](https://www.sqlite.org/rtree.html)
- [GeoPoly](https://www.sqlite.org/geopoly.html)
- [base64](https://github.com/sqlite/sqlite/blob/master/ext/misc/base64.c)
- [decimal](https://github.com/sqlite/sqlite/blob/master/ext/misc/decimal.c)
- [regexp](https://github.com/sqlite/sqlite/blob/master/ext/misc/regexp.c)
- [series](https://github.com/sqlite/sqlite/blob/master/ext/misc/series.c)
- [uint](https://github.com/sqlite/sqlite/blob/master/ext/misc/uint.c)
- [uuid](https://github.com/sqlite/sqlite/blob/master/ext/misc/uuid.c)
- [time](../sqlite3/time.c)
See the [configuration options](../sqlite3/sqlite_cfg.h).

View File

@@ -3,14 +3,13 @@ set -eo pipefail
cd -P -- "$(dirname -- "$0")"
# download SQLite
../sqlite3/download.sh
# build SQLite
zig cc --target=wasm32-wasi -flto -g0 -Os \
-o sqlite3.wasm ../sqlite3/amalg.c \
zig cc --target=wasm32-wasi -flto -g0 -O2 \
-o sqlite3.wasm ../sqlite3/main.c \
-I../sqlite3/ \
-mmutable-globals \
-mbulk-memory -mreference-types \
-mnontrapping-fptoint -msign-ext \
-mexec-model=reactor \
-D_HAVE_SQLITE_CONFIG_H \
$(awk '{print "-Wl,--export="$0}' exports.txt)

View File

@@ -40,11 +40,9 @@ sqlite3_blob_reopen
sqlite3_get_autocommit
sqlite3_last_insert_rowid
sqlite3_changes64
sqlite3_unlock_notify
sqlite3_backup_init
sqlite3_backup_step
sqlite3_backup_finish
sqlite3_backup_remaining
sqlite3_backup_pagecount
sqlite3_time_collation
sqlite3_interrupt_offset

Binary file not shown.

View File

@@ -202,7 +202,6 @@ const (
noFuncErr = errorString("sqlite3: could not find function: ")
binaryErr = errorString("sqlite3: no SQLite binary embed/set/loaded")
timeErr = errorString("sqlite3: invalid time value")
notImplErr = errorString("sqlite3: not implemented")
whenceErr = errorString("sqlite3: invalid whence")
offsetErr = errorString("sqlite3: invalid offset")
)

2
go.mod
View File

@@ -4,7 +4,7 @@ go 1.19
require (
github.com/ncruces/julianday v0.1.5
github.com/tetratelabs/wazero v1.0.0-rc.1
github.com/tetratelabs/wazero v1.0.0
golang.org/x/sync v0.1.0
golang.org/x/sys v0.6.0
)

4
go.sum
View File

@@ -1,7 +1,7 @@
github.com/ncruces/julianday v0.1.5 h1:hDJ9ejiMp3DHsoZ5KW4c1lwfMjbARS7u/gbYcd0FBZk=
github.com/ncruces/julianday v0.1.5/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/tetratelabs/wazero v1.0.0-rc.1 h1:ytecMV5Ue0BwezjKh/cM5yv1Mo49ep2R2snSsQUyToc=
github.com/tetratelabs/wazero v1.0.0-rc.1/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ=
github.com/tetratelabs/wazero v1.0.0 h1:sCE9+mjFex95Ki6hdqwvhyF25x5WslADjDKIFU5BXzI=
github.com/tetratelabs/wazero v1.0.0/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=

23
mem.go
View File

@@ -25,6 +25,27 @@ func (m memory) view(ptr uint32, size uint64) []byte {
return buf
}
func (m memory) readUint8(ptr uint32) uint8 {
if ptr == 0 {
panic(nilErr)
}
v, ok := m.mod.Memory().ReadByte(ptr)
if !ok {
panic(rangeErr)
}
return v
}
func (m memory) writeUint8(ptr uint32, v uint8) {
if ptr == 0 {
panic(nilErr)
}
ok := m.mod.Memory().WriteByte(ptr, v)
if !ok {
panic(rangeErr)
}
}
func (m memory) readUint32(ptr uint32) uint32 {
if ptr == 0 {
panic(nilErr)
@@ -36,7 +57,7 @@ func (m memory) readUint32(ptr uint32) uint32 {
return v
}
func (m memory) writeUint32(ptr, v uint32) {
func (m memory) writeUint32(ptr uint32, v uint32) {
if ptr == 0 {
panic(nilErr)
}

View File

@@ -3,14 +3,10 @@ package sqlite3
import (
"context"
"crypto/rand"
"io"
"math"
"os"
"runtime"
"strconv"
"sync"
"sync/atomic"
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/api"
@@ -28,11 +24,10 @@ var (
)
var sqlite3 struct {
once sync.Once
runtime wazero.Runtime
compiled wazero.CompiledModule
instances atomic.Uint64
err error
once sync.Once
runtime wazero.Runtime
compiled wazero.CompiledModule
err error
}
func instantiateModule() (*module, error) {
@@ -43,12 +38,7 @@ func instantiateModule() (*module, error) {
return nil, sqlite3.err
}
name := "sqlite3-" + strconv.FormatUint(sqlite3.instances.Add(1), 10)
cfg := wazero.NewModuleConfig().WithName(name).
WithSysWalltime().WithSysNanotime().WithSysNanosleep().
WithOsyield(runtime.Gosched).
WithRandSource(rand.Reader)
cfg := wazero.NewModuleConfig().WithStartFunctions("_initialize")
mod, err := sqlite3.runtime.InstantiateModule(ctx, sqlite3.compiled, cfg)
if err != nil {
@@ -155,7 +145,6 @@ func newModule(mod api.Module) (m *module, err error) {
backupFinish: getFun("sqlite3_backup_finish"),
backupRemaining: getFun("sqlite3_backup_remaining"),
backupPageCount: getFun("sqlite3_backup_pagecount"),
timeCollation: getFun("sqlite3_time_collation"),
interrupt: getVal("sqlite3_interrupt_offset"),
}
if err != nil {
@@ -349,6 +338,5 @@ type sqliteAPI struct {
backupFinish api.Function
backupRemaining api.Function
backupPageCount api.Function
timeCollation api.Function
interrupt uint32
}

View File

@@ -3,11 +3,29 @@ set -eo pipefail
cd -P -- "$(dirname -- "$0")"
if [ ! -f "sqlite3.c" ]; then
url="https://sqlite.org/2023/sqlite-amalgamation-3410000.zip"
curl "$url" > sqlite.zip
unzip -d . sqlite.zip
mv sqlite-amalgamation-*/sqlite3* .
rm -rf sqlite-amalgamation-*
rm sqlite.zip
fi
curl -#OL "https://sqlite.org/2023/sqlite-amalgamation-3410200.zip"
unzip -d . sqlite-amalgamation-*.zip
mv sqlite-amalgamation-*/sqlite3* .
rm -rf sqlite-amalgamation-*
cd ext/
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.41.2/ext/misc/decimal.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.41.2/ext/misc/uint.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.41.2/ext/misc/uuid.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.41.2/ext/misc/base64.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.41.2/ext/misc/regexp.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.41.2/ext/misc/series.c"
cd ~-
cd ../tests/mptest/testdata/
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.41.2/mptest/mptest.c"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.41.2/mptest/config01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.41.2/mptest/config02.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.41.2/mptest/crash01.test"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.41.2/mptest/crash02.subtest"
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.41.2/mptest/multiwrite01.test"
cd ~-
cd ../tests/speedtest1/testdata/
curl -#OL "https://github.com/sqlite/sqlite/raw/version-3.41.2/test/speedtest1.c"
cd ~-

1
sqlite3/ext/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
*.c

View File

@@ -1,8 +1,5 @@
#!/usr/bin/env bash
cd -P -- "$(dirname -- "$0")"
clang-format -i \
main.c \
os.c \
qsort.c \
amalg.c
shopt -s extglob
clang-format -i !(sqlite3*).@(c|h)

View File

@@ -1,14 +1,33 @@
#include <stdbool.h>
#include <stddef.h>
#include "sqlite3.h"
#include "sqlite3.c"
//
#include "os.c"
#include "qsort.c"
//
#include "ext/base64.c"
#include "ext/decimal.c"
#include "ext/regexp.c"
#include "ext/series.c"
#include "ext/uint.c"
#include "ext/uuid.c"
#include "time.c"
int main() {
int rc = sqlite3_initialize();
if (rc != SQLITE_OK) return 1;
}
sqlite3_vfs *os_vfs();
sqlite3_destructor_type malloc_destructor = &free;
size_t sqlite3_interrupt_offset = offsetof(sqlite3, u1.isInterrupted);
int sqlite3_os_init() {
return sqlite3_vfs_register(os_vfs(), /*default=*/true);
}
__attribute__((constructor)) void premain() {
sqlite3_initialize();
sqlite3_auto_extension((void (*)(void))sqlite3_base_init);
sqlite3_auto_extension((void (*)(void))sqlite3_decimal_init);
sqlite3_auto_extension((void (*)(void))sqlite3_regexp_init);
sqlite3_auto_extension((void (*)(void))sqlite3_series_init);
sqlite3_auto_extension((void (*)(void))sqlite3_uint_init);
sqlite3_auto_extension((void (*)(void))sqlite3_uuid_init);
sqlite3_auto_extension((void (*)(void))sqlite3_time_init);
}

View File

@@ -2,7 +2,7 @@
#include "sqlite3.h"
int os_localtime(sqlite3_int64, struct tm *);
int os_localtime(struct tm *, sqlite3_int64);
int os_randomness(sqlite3_vfs *, int nByte, char *zOut);
int os_sleep(sqlite3_vfs *, int microseconds);
@@ -18,36 +18,73 @@ int os_full_pathname(sqlite3_vfs *, const char *zName, int nOut, char *zOut);
struct os_file {
sqlite3_file base;
int id;
int lock;
char lock;
char psow;
int lockTimeout;
};
static_assert(offsetof(struct os_file, id) == 4, "Unexpected offset");
static_assert(offsetof(struct os_file, lock) == 8, "Unexpected offset");
static_assert(offsetof(struct os_file, psow) == 9, "Unexpected offset");
static_assert(offsetof(struct os_file, lockTimeout) == 12, "Unexpected offset");
int os_close(sqlite3_file *);
int os_read(sqlite3_file *, void *, int iAmt, sqlite3_int64 iOfst);
int os_write(sqlite3_file *, const void *, int iAmt, sqlite3_int64 iOfst);
int os_truncate(sqlite3_file *, sqlite3_int64 size);
int os_sync(sqlite3_file *, int flags);
int os_file_size(sqlite3_file *, sqlite3_int64 *pSize);
int os_file_control(sqlite3_file *pFile, int op, void *pArg);
int os_file_control(sqlite3_file *, int op, void *pArg);
int os_lock(sqlite3_file *pFile, int eLock);
int os_unlock(sqlite3_file *pFile, int eLock);
int os_check_reserved_lock(sqlite3_file *pFile, int *pResOut);
int os_lock(sqlite3_file *, int eLock);
int os_unlock(sqlite3_file *, int eLock);
int os_check_reserved_lock(sqlite3_file *, int *pResOut);
static int no_lock(sqlite3_file *pFile, int eLock) { return SQLITE_OK; }
static int no_unlock(sqlite3_file *pFile, int eLock) { return SQLITE_OK; }
static int no_check_reserved_lock(sqlite3_file *pFile, int *pResOut) {
*pResOut = 0;
return SQLITE_OK;
}
static int no_file_control(sqlite3_file *pFile, int op, void *pArg) {
static int os_file_control_w(sqlite3_file *file, int op, void *pArg) {
struct os_file *pFile = (struct os_file *)file;
switch (op) {
case SQLITE_FCNTL_VFSNAME: {
*(char **)pArg = sqlite3_mprintf("%s", "os");
return SQLITE_OK;
}
case SQLITE_FCNTL_LOCKSTATE: {
*(int *)pArg = pFile->lock;
return SQLITE_OK;
}
case SQLITE_FCNTL_LOCK_TIMEOUT: {
int iOld = pFile->lockTimeout;
pFile->lockTimeout = *(int *)pArg;
*(int *)pArg = iOld;
return SQLITE_OK;
}
case SQLITE_FCNTL_POWERSAFE_OVERWRITE: {
if (*(int *)pArg < 0) {
*(int *)pArg = pFile->psow;
} else {
pFile->psow = *(int *)pArg;
}
return SQLITE_OK;
}
case SQLITE_FCNTL_SIZE_HINT:
case SQLITE_FCNTL_HAS_MOVED:
return os_file_control(file, op, pArg);
}
// Consider also implementing these opcodes (in use by SQLite):
// SQLITE_FCNTL_BUSYHANDLER
// SQLITE_FCNTL_COMMIT_PHASETWO
// SQLITE_FCNTL_PDB
// SQLITE_FCNTL_PRAGMA
// SQLITE_FCNTL_SYNC
return SQLITE_NOTFOUND;
}
static int no_sector_size(sqlite3_file *pFile) { return 0; }
static int no_device_characteristics(sqlite3_file *pFile) { return 0; }
int localtime_s(struct tm *const pTm, time_t const *const pTime) {
return os_localtime((sqlite3_int64)*pTime, pTm);
static int os_sector_size(sqlite3_file *file) {
return SQLITE_DEFAULT_SECTOR_SIZE;
}
static int os_device_characteristics(sqlite3_file *file) {
struct os_file *pFile = (struct os_file *)file;
return pFile->psow ? SQLITE_IOCAP_POWERSAFE_OVERWRITE : 0;
}
static int os_open_w(sqlite3_vfs *vfs, sqlite3_filename zName,
@@ -63,12 +100,23 @@ static int os_open_w(sqlite3_vfs *vfs, sqlite3_filename zName,
.xLock = os_lock,
.xUnlock = os_unlock,
.xCheckReservedLock = os_check_reserved_lock,
.xFileControl = no_file_control,
.xDeviceCharacteristics = no_device_characteristics,
.xFileControl = os_file_control_w,
.xSectorSize = os_sector_size,
.xDeviceCharacteristics = os_device_characteristics,
};
memset(file, 0, sizeof(struct os_file));
int rc = os_open(vfs, zName, file, flags, pOutFlags);
file->pMethods = (char)rc == SQLITE_OK ? &os_io : NULL;
return rc;
if (rc) {
return rc;
}
struct os_file *pFile = (struct os_file *)file;
pFile->base.pMethods = &os_io;
if (flags & SQLITE_OPEN_MAIN_DB) {
pFile->psow =
sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE);
}
return SQLITE_OK;
}
sqlite3_vfs *os_vfs() {
@@ -90,3 +138,7 @@ sqlite3_vfs *os_vfs() {
};
return &os_vfs;
}
int localtime_s(struct tm *const pTm, time_t const *const pTime) {
return os_localtime(pTm, (sqlite3_int64)*pTime);
}

View File

@@ -4,15 +4,18 @@
static int time_collation(void *pArg, int nKey1, const void *pKey1, int nKey2,
const void *pKey2) {
// If keys are of different length, and both terminated by a Z,
// ignore the Z for collation purposes.
if (nKey1 && nKey2 && nKey1 != nKey2) {
const char *pK1 = (const char *)pKey1;
const char *pK2 = (const char *)pKey2;
if (pK1[nKey1 - 1] == 'Z' && pK2[nKey2 - 1] == 'Z') {
nKey1--;
nKey2--;
}
// Remove a Z suffix if one key is no longer than the other.
// A Z suffix collates before any character but after the empty string.
// This avoids making different keys equal.
const int nK1 = nKey1;
const int nK2 = nKey2;
const char *pK1 = (const char *)pKey1;
const char *pK2 = (const char *)pKey2;
if (nK1 && nK1 <= nK2 && pK1[nK1 - 1] == 'Z') {
nKey1--;
}
if (nK2 && nK2 <= nK1 && pK2[nK2 - 1] == 'Z') {
nKey2--;
}
int n = nKey1 < nKey2 ? nKey1 : nKey2;
@@ -23,6 +26,7 @@ static int time_collation(void *pArg, int nKey1, const void *pKey1, int nKey2,
return rc;
}
int sqlite3_time_collation(sqlite3 *db) {
return sqlite3_create_collation(db, "TIME", SQLITE_UTF8, 0, time_collation);
int sqlite3_time_init(sqlite3 *db, char **pzErrMsg,
const sqlite3_api_routines *pApi) {
return sqlite3_create_collation(db, "time", SQLITE_UTF8, 0, time_collation);
}

54
stmt.go
View File

@@ -334,21 +334,7 @@ func (s *Stmt) ColumnTime(col int, format TimeFormat) time.Time {
//
// https://www.sqlite.org/c3ref/column_blob.html
func (s *Stmt) ColumnText(col int) string {
r := s.c.call(s.c.api.columnText,
uint64(s.handle), uint64(col))
ptr := uint32(r[0])
if ptr == 0 {
r = s.c.call(s.c.api.errcode, uint64(s.c.handle))
s.err = s.c.error(r[0])
return ""
}
r = s.c.call(s.c.api.columnBytes,
uint64(s.handle), uint64(col))
mem := s.c.mem.view(ptr, r[0])
return string(mem)
return string(s.ColumnRawText(col))
}
// ColumnBlob appends to buf and returns
@@ -357,6 +343,39 @@ func (s *Stmt) ColumnText(col int) string {
//
// https://www.sqlite.org/c3ref/column_blob.html
func (s *Stmt) ColumnBlob(col int, buf []byte) []byte {
return append(buf, s.ColumnRawBlob(col)...)
}
// ColumnRawText returns the value of the result column as a []byte.
// The []byte is owned by SQLite and may be invalidated by
// subsequent calls to [Stmt] methods.
// The leftmost column of the result set has the index 0.
//
// https://www.sqlite.org/c3ref/column_blob.html
func (s *Stmt) ColumnRawText(col int) []byte {
r := s.c.call(s.c.api.columnText,
uint64(s.handle), uint64(col))
ptr := uint32(r[0])
if ptr == 0 {
r = s.c.call(s.c.api.errcode, uint64(s.c.handle))
s.err = s.c.error(r[0])
return nil
}
r = s.c.call(s.c.api.columnBytes,
uint64(s.handle), uint64(col))
return s.c.mem.view(ptr, r[0])
}
// ColumnRawBlob returns the value of the result column as a []byte.
// The []byte is owned by SQLite and may be invalidated by
// subsequent calls to [Stmt] methods.
// The leftmost column of the result set has the index 0.
//
// https://www.sqlite.org/c3ref/column_blob.html
func (s *Stmt) ColumnRawBlob(col int) []byte {
r := s.c.call(s.c.api.columnBlob,
uint64(s.handle), uint64(col))
@@ -364,14 +383,13 @@ func (s *Stmt) ColumnBlob(col int, buf []byte) []byte {
if ptr == 0 {
r = s.c.call(s.c.api.errcode, uint64(s.c.handle))
s.err = s.c.error(r[0])
return buf[0:0]
return nil
}
r = s.c.call(s.c.api.columnBytes,
uint64(s.handle), uint64(col))
mem := s.c.mem.view(ptr, r[0])
return append(buf[0:0], mem...)
return s.c.mem.view(ptr, r[0])
}
// Return true if stmt is an empty SQL statement.

View File

@@ -126,7 +126,7 @@ func testTxQuery(t params) {
if r.Err() != nil {
t.Fatal(err)
}
t.Fatal("expected one rows")
t.Fatal("expected one row")
}
var name string

View File

@@ -13,7 +13,7 @@ import (
func TestConn_Open_dir(t *testing.T) {
t.Parallel()
_, err := sqlite3.Open(".")
_, err := sqlite3.OpenFlags(".", 0)
if err == nil {
t.Fatal("want error")
}

77
tests/ext_test.go Normal file
View File

@@ -0,0 +1,77 @@
package tests
import (
"testing"
"github.com/ncruces/go-sqlite3"
)
func Test_base64(t *testing.T) {
t.Parallel()
db, err := sqlite3.Open(":memory:")
if err != nil {
t.Fatal(err)
}
defer db.Close()
// base64
stmt, _, err := db.Prepare(`SELECT base64('TWFueSBoYW5kcyBtYWtlIGxpZ2h0IHdvcmsu')`)
if err != nil {
t.Error(err)
}
defer stmt.Close()
if !stmt.Step() {
t.Fatal("expected one row")
}
if got := stmt.ColumnText(0); got != "Many hands make light work." {
t.Errorf("got %q", got)
}
}
func Test_decimal(t *testing.T) {
t.Parallel()
db, err := sqlite3.Open(":memory:")
if err != nil {
t.Fatal(err)
}
defer db.Close()
stmt, _, err := db.Prepare(`SELECT decimal_add(decimal('0.1'), decimal('0.2')) = decimal('0.3')`)
if err != nil {
t.Error(err)
}
defer stmt.Close()
if !stmt.Step() {
t.Fatal("expected one row")
}
if !stmt.ColumnBool(0) {
t.Error("want true")
}
}
func Test_uint(t *testing.T) {
t.Parallel()
db, err := sqlite3.Open(":memory:")
if err != nil {
t.Fatal(err)
}
defer db.Close()
stmt, _, err := db.Prepare(`SELECT 'z2' < 'z11' COLLATE UINT`)
if err != nil {
t.Error(err)
}
defer stmt.Close()
if !stmt.Step() {
t.Fatal("expected one row")
}
if !stmt.ColumnBool(0) {
t.Error("want true")
}
}

View File

@@ -99,11 +99,12 @@ func Test_config01(t *testing.T) {
ctx, vfs := vfsContext(newContext(t))
name := filepath.Join(t.TempDir(), "test.db")
cfg := config(ctx).WithArgs("mptest", name, "config01.test")
_, err := rt.InstantiateModule(ctx, module, cfg)
mod, err := rt.InstantiateModule(ctx, module, cfg)
if err != nil {
t.Error(err)
}
vfs.Close()
mod.Close(ctx)
}
func Test_config02(t *testing.T) {
@@ -117,11 +118,12 @@ func Test_config02(t *testing.T) {
ctx, vfs := vfsContext(newContext(t))
name := filepath.Join(t.TempDir(), "test.db")
cfg := config(ctx).WithArgs("mptest", name, "config02.test")
_, err := rt.InstantiateModule(ctx, module, cfg)
mod, err := rt.InstantiateModule(ctx, module, cfg)
if err != nil {
t.Error(err)
}
vfs.Close()
mod.Close(ctx)
}
func Test_crash01(t *testing.T) {
@@ -132,11 +134,12 @@ func Test_crash01(t *testing.T) {
ctx, vfs := vfsContext(newContext(t))
name := filepath.Join(t.TempDir(), "test.db")
cfg := config(ctx).WithArgs("mptest", name, "crash01.test")
_, err := rt.InstantiateModule(ctx, module, cfg)
mod, err := rt.InstantiateModule(ctx, module, cfg)
if err != nil {
t.Error(err)
}
vfs.Close()
mod.Close(ctx)
}
func Test_multiwrite01(t *testing.T) {
@@ -147,11 +150,12 @@ func Test_multiwrite01(t *testing.T) {
ctx, vfs := vfsContext(newContext(t))
name := filepath.Join(t.TempDir(), "test.db")
cfg := config(ctx).WithArgs("mptest", name, "multiwrite01.test")
_, err := rt.InstantiateModule(ctx, module, cfg)
mod, err := rt.InstantiateModule(ctx, module, cfg)
if err != nil {
t.Error(err)
}
vfs.Close()
mod.Close(ctx)
}
func newContext(t *testing.T) context.Context {

View File

@@ -3,18 +3,9 @@ set -eo pipefail
cd -P -- "$(dirname -- "$0")"
if [ ! -f "mptest.c" ]; then
curl -sOL "https://github.com/sqlite/sqlite/raw/master/mptest/mptest.c"
curl -sOL "https://github.com/sqlite/sqlite/raw/master/mptest/config01.test"
curl -sOL "https://github.com/sqlite/sqlite/raw/master/mptest/config02.test"
curl -sOL "https://github.com/sqlite/sqlite/raw/master/mptest/crash01.test"
curl -sOL "https://github.com/sqlite/sqlite/raw/master/mptest/crash02.subtest"
curl -sOL "https://github.com/sqlite/sqlite/raw/master/mptest/multiwrite01.test"
fi
zig cc --target=wasm32-wasi -flto -g0 -Os \
-o mptest.wasm main.c test.c \
-I../../../sqlite3 \
zig cc --target=wasm32-wasi -flto -g0 -O2 \
-o mptest.wasm main.c \
-I../../../sqlite3/ \
-mmutable-globals \
-mbulk-memory -mreference-types \
-mnontrapping-fptoint -msign-ext \

View File

@@ -1,22 +1,23 @@
#include <stdbool.h>
#include <stddef.h>
#include "sqlite3.c"
//
#include "os.c"
#include "qsort.c"
#include "sqlite3.c"
sqlite3_destructor_type malloc_destructor = &free;
size_t sqlite3_interrupt_offset = offsetof(sqlite3, u1.isInterrupted);
void __attribute__((constructor)) premain() { sqlite3_initialize(); }
int sqlite3_enable_load_extension(sqlite3 *db, int onoff) { return SQLITE_OK; }
void *sqlite3_trace(sqlite3 *db, void (*xTrace)(void *, const char *),
void *pArg) {
return NULL;
}
int sqlite3_os_init() {
return sqlite3_vfs_register(os_vfs(), /*default=*/true);
}
}
__attribute__((constructor)) void premain() { sqlite3_initialize(); }
static int dont_unlink(const char *pathname) { return 0; }
#define sqlite3_enable_load_extension(...)
#define sqlite3_trace(...)
#define unlink dont_unlink
#undef UNUSED_PARAMETER
#include "mptest.c"

View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:3960b873a7dab969a66f7859d491cec0dd4e6c0c9f83eab449fb15ec5ebdfd8f
size 1077281
oid sha256:f81ce390812d944d1fa9b2cc607a3629febab0bc0e4473dad3170134509c1751
size 1630826

View File

@@ -1,5 +0,0 @@
#define unlink dont_unlink
#include "mptest.c"
int dont_unlink(const char *pathname) { return 0; }

View File

@@ -0,0 +1,92 @@
package speedtest1
import (
"bytes"
"context"
"crypto/rand"
"io"
"os"
"path/filepath"
"runtime"
"strconv"
"strings"
"testing"
_ "embed"
_ "unsafe"
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
_ "github.com/ncruces/go-sqlite3"
)
//go:embed testdata/speedtest1.wasm
var binary []byte
//go:linkname vfsNewEnvModuleBuilder github.com/ncruces/go-sqlite3.vfsNewEnvModuleBuilder
func vfsNewEnvModuleBuilder(r wazero.Runtime) wazero.HostModuleBuilder
//go:linkname vfsContext github.com/ncruces/go-sqlite3.vfsContext
func vfsContext(ctx context.Context) (context.Context, io.Closer)
var (
rt wazero.Runtime
module wazero.CompiledModule
output bytes.Buffer
options []string
)
func init() {
ctx := context.TODO()
rt = wazero.NewRuntime(ctx)
wasi_snapshot_preview1.MustInstantiate(ctx, rt)
env := vfsNewEnvModuleBuilder(rt)
_, err := env.Instantiate(ctx)
if err != nil {
panic(err)
}
module, err = rt.CompileModule(ctx, binary)
if err != nil {
panic(err)
}
}
func TestMain(m *testing.M) {
i := 1
options = append(options, "speedtest1")
for _, arg := range os.Args[1:] {
if strings.HasPrefix(arg, "-test.") {
os.Args[i] = arg
i++
} else {
options = append(options, arg)
}
}
os.Args = os.Args[:i]
code := m.Run()
io.Copy(os.Stderr, &output)
os.Exit(code)
}
func Benchmark_speedtest1(b *testing.B) {
output.Reset()
ctx, vfs := vfsContext(context.Background())
name := filepath.Join(b.TempDir(), "test.db")
args := append(options, "--size", strconv.Itoa(b.N), name)
cfg := wazero.NewModuleConfig().
WithArgs(args...).WithName("speedtest1").
WithStdout(&output).WithStderr(&output).
WithSysWalltime().WithSysNanotime().WithSysNanosleep().
WithOsyield(runtime.Gosched).
WithRandSource(rand.Reader)
mod, err := rt.InstantiateModule(ctx, module, cfg)
if err != nil {
b.Error(err)
}
vfs.Close()
mod.Close(ctx)
}

View File

@@ -0,0 +1 @@
speedtest1.wasm filter=lfs diff=lfs merge=lfs -text

1
tests/speedtest1/testdata/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
speedtest1.c

12
tests/speedtest1/testdata/build.sh vendored Executable file
View File

@@ -0,0 +1,12 @@
#!/usr/bin/env bash
set -eo pipefail
cd -P -- "$(dirname -- "$0")"
zig cc --target=wasm32-wasi -flto -g0 -O2 \
-o speedtest1.wasm main.c \
-I../../../sqlite3/ \
-mmutable-globals \
-mbulk-memory -mreference-types \
-mnontrapping-fptoint -msign-ext \
-D_HAVE_SQLITE_CONFIG_H

View File

@@ -1,11 +1,17 @@
#include <stdbool.h>
#include <stddef.h>
#include "main.c"
#include "sqlite3.c"
//
#include "os.c"
#include "qsort.c"
#include "time.c"
#include "sqlite3.c"
sqlite3_destructor_type malloc_destructor = &free;
size_t sqlite3_interrupt_offset = offsetof(sqlite3, u1.isInterrupted);
size_t sqlite3_interrupt_offset = offsetof(sqlite3, u1.isInterrupted);
int sqlite3_os_init() {
return sqlite3_vfs_register(os_vfs(), /*default=*/true);
}
#define randomFunc(args...) randomFunc2(args)
#include "speedtest1.c"

3
tests/speedtest1/testdata/speedtest1.wasm vendored Executable file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:0e02a26b86832a4703cd2a86c98ff8041d9d889b865a61f324b512a15d2d361e
size 1676129

216
vfs.go
View File

@@ -14,19 +14,11 @@ import (
"github.com/ncruces/julianday"
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/api"
"github.com/tetratelabs/wazero/sys"
)
func vfsInstantiate(ctx context.Context, r wazero.Runtime) {
wasi := r.NewHostModuleBuilder("wasi_snapshot_preview1")
wasi.NewFunctionBuilder().WithFunc(vfsExit).Export("proc_exit")
_, err := wasi.Instantiate(ctx)
if err != nil {
panic(err)
}
env := vfsNewEnvModuleBuilder(r)
_, err = env.Instantiate(ctx)
_, err := env.Instantiate(ctx)
if err != nil {
panic(err)
}
@@ -34,25 +26,25 @@ func vfsInstantiate(ctx context.Context, r wazero.Runtime) {
func vfsNewEnvModuleBuilder(r wazero.Runtime) wazero.HostModuleBuilder {
env := r.NewHostModuleBuilder("env")
env.NewFunctionBuilder().WithFunc(vfsLocaltime).Export("os_localtime")
env.NewFunctionBuilder().WithFunc(vfsRandomness).Export("os_randomness")
env.NewFunctionBuilder().WithFunc(vfsSleep).Export("os_sleep")
env.NewFunctionBuilder().WithFunc(vfsCurrentTime).Export("os_current_time")
env.NewFunctionBuilder().WithFunc(vfsCurrentTime64).Export("os_current_time_64")
env.NewFunctionBuilder().WithFunc(vfsFullPathname).Export("os_full_pathname")
env.NewFunctionBuilder().WithFunc(vfsDelete).Export("os_delete")
env.NewFunctionBuilder().WithFunc(vfsAccess).Export("os_access")
env.NewFunctionBuilder().WithFunc(vfsOpen).Export("os_open")
env.NewFunctionBuilder().WithFunc(vfsClose).Export("os_close")
env.NewFunctionBuilder().WithFunc(vfsRead).Export("os_read")
env.NewFunctionBuilder().WithFunc(vfsWrite).Export("os_write")
env.NewFunctionBuilder().WithFunc(vfsTruncate).Export("os_truncate")
env.NewFunctionBuilder().WithFunc(vfsSync).Export("os_sync")
env.NewFunctionBuilder().WithFunc(vfsFileSize).Export("os_file_size")
env.NewFunctionBuilder().WithFunc(vfsLock).Export("os_lock")
env.NewFunctionBuilder().WithFunc(vfsUnlock).Export("os_unlock")
env.NewFunctionBuilder().WithFunc(vfsCheckReservedLock).Export("os_check_reserved_lock")
env.NewFunctionBuilder().WithFunc(vfsFileControl).Export("os_file_control")
vfsRegisterFuncT(env, "os_localtime", vfsLocaltime)
vfsRegisterFunc3(env, "os_randomness", vfsRandomness)
vfsRegisterFunc2(env, "os_sleep", vfsSleep)
vfsRegisterFunc2(env, "os_current_time", vfsCurrentTime)
vfsRegisterFunc2(env, "os_current_time_64", vfsCurrentTime64)
vfsRegisterFunc4(env, "os_full_pathname", vfsFullPathname)
vfsRegisterFunc3(env, "os_delete", vfsDelete)
vfsRegisterFunc4(env, "os_access", vfsAccess)
vfsRegisterFunc5(env, "os_open", vfsOpen)
vfsRegisterFunc1(env, "os_close", vfsClose)
vfsRegisterFuncRW(env, "os_read", vfsRead)
vfsRegisterFuncRW(env, "os_write", vfsWrite)
vfsRegisterFuncT(env, "os_truncate", vfsTruncate)
vfsRegisterFunc2(env, "os_sync", vfsSync)
vfsRegisterFunc2(env, "os_file_size", vfsFileSize)
vfsRegisterFunc2(env, "os_lock", vfsLock)
vfsRegisterFunc2(env, "os_unlock", vfsUnlock)
vfsRegisterFunc2(env, "os_check_reserved_lock", vfsCheckReservedLock)
vfsRegisterFunc3(env, "os_file_control", vfsFileControl)
return env
}
@@ -87,14 +79,7 @@ func (vfs *vfsState) Close() error {
return nil
}
func vfsExit(ctx context.Context, mod api.Module, exitCode uint32) {
// Ensure other callers see the exit code.
_ = mod.CloseWithExitCode(ctx, exitCode)
// Prevent any code from executing after this function.
panic(sys.NewExitError(mod.Name(), exitCode))
}
func vfsLocaltime(ctx context.Context, mod api.Module, t uint64, pTm uint32) uint32 {
func vfsLocaltime(ctx context.Context, mod api.Module, pTm uint32, t uint64) uint32 {
tm := time.Unix(int64(t), 0)
var isdst int
if tm.IsDST() {
@@ -121,7 +106,7 @@ func vfsRandomness(ctx context.Context, mod api.Module, pVfs, nByte, zByte uint3
return uint32(n)
}
func vfsSleep(ctx context.Context, pVfs, nMicro uint32) uint32 {
func vfsSleep(ctx context.Context, mod api.Module, pVfs, nMicro uint32) uint32 {
time.Sleep(time.Duration(nMicro) * time.Microsecond)
return _OK
}
@@ -143,22 +128,26 @@ func vfsFullPathname(ctx context.Context, mod api.Module, pVfs, zRelative, nFull
rel := memory{mod}.readString(zRelative, _MAX_PATHNAME)
abs, err := filepath.Abs(rel)
if err != nil {
return uint32(IOERR)
return uint32(CANTOPEN_FULLPATH)
}
// Consider either using [filepath.EvalSymlinks] to canonicalize the path (as the Unix VFS does).
// Or using [os.Readlink] to resolve a symbolic link (as the Unix VFS did).
// This might be buggy on Windows (the Windows VFS doesn't try).
size := uint64(len(abs) + 1)
if size > uint64(nFull) {
return uint32(CANTOPEN_FULLPATH)
}
mem := memory{mod}.view(zFull, size)
mem[len(abs)] = 0
copy(mem, abs)
return _OK
if fi, err := os.Lstat(abs); err == nil {
if fi.Mode()&fs.ModeSymlink != 0 {
return _OK_SYMLINK
}
return _OK
} else if errors.Is(err, fs.ErrNotExist) {
return _OK
}
return uint32(CANTOPEN_FULLPATH)
}
func vfsDelete(ctx context.Context, mod api.Module, pVfs, zPath, syncDir uint32) uint32 {
@@ -186,11 +175,30 @@ func vfsDelete(ctx context.Context, mod api.Module, pVfs, zPath, syncDir uint32)
func vfsAccess(ctx context.Context, mod api.Module, pVfs, zPath uint32, flags _AccessFlag, pResOut uint32) uint32 {
path := memory{mod}.readString(zPath, _MAX_PATHNAME)
ok, rc := vfsOS.Access(path, flags)
err := vfsOS.Access(path, flags)
var res uint32
if ok {
res = 1
var rc xErrorCode
if flags == _ACCESS_EXISTS {
switch {
case err == nil:
res = 1
case errors.Is(err, fs.ErrNotExist):
res = 0
default:
rc = IOERR_ACCESS
}
} else {
switch {
case err == nil:
res = 1
case errors.Is(err, fs.ErrPermission):
res = 0
default:
rc = IOERR_ACCESS
}
}
memory{mod}.writeUint32(pResOut, res)
return uint32(rc)
}
@@ -301,14 +309,110 @@ func vfsFileSize(ctx context.Context, mod api.Module, pFile, pSize uint32) uint3
return _OK
}
func vfsFileControl(ctx context.Context, pFile, op, pArg uint32) uint32 {
// SQLite calls vfsFileControl with these opcodes:
// SQLITE_FCNTL_SIZE_HINT
// SQLITE_FCNTL_PRAGMA
// SQLITE_FCNTL_BUSYHANDLER
// SQLITE_FCNTL_HAS_MOVED
// SQLITE_FCNTL_SYNC
// SQLITE_FCNTL_COMMIT_PHASETWO
// SQLITE_FCNTL_PDB
func vfsFileControl(ctx context.Context, mod api.Module, pFile uint32, op _FcntlOpcode, pArg uint32) uint32 {
switch op {
case _FCNTL_SIZE_HINT:
return vfsSizeHint(ctx, mod, pFile, pArg)
case _FCNTL_HAS_MOVED:
return vfsFileMoved(ctx, mod, pFile, pArg)
}
return uint32(NOTFOUND)
}
func vfsSizeHint(ctx context.Context, mod api.Module, pFile, pArg uint32) uint32 {
file := vfsFile.GetOS(ctx, mod, pFile)
size := memory{mod}.readUint64(pArg)
err := vfsOS.Allocate(file, int64(size))
if err != nil {
return uint32(IOERR_TRUNCATE)
}
return _OK
}
func vfsFileMoved(ctx context.Context, mod api.Module, pFile, pResOut uint32) uint32 {
file := vfsFile.GetOS(ctx, mod, pFile)
fi, err := file.Stat()
if err != nil {
return uint32(IOERR_FSTAT)
}
pi, err := os.Stat(file.Name())
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return uint32(IOERR_FSTAT)
}
var res uint32
if !os.SameFile(fi, pi) {
res = 1
}
memory{mod}.writeUint32(pResOut, res)
return _OK
}
func vfsRegisterFunc1(mod wazero.HostModuleBuilder, name string, fn func(ctx context.Context, mod api.Module, _ uint32) uint32) {
mod.NewFunctionBuilder().
WithGoModuleFunction(api.GoModuleFunc(
func(ctx context.Context, mod api.Module, stack []uint64) {
stack[0] = uint64(fn(ctx, mod, uint32(stack[0])))
}),
[]api.ValueType{api.ValueTypeI32}, []api.ValueType{api.ValueTypeI32}).
Export(name)
}
func vfsRegisterFunc2[T0, T1 ~uint32](mod wazero.HostModuleBuilder, name string, fn func(ctx context.Context, mod api.Module, _ T0, _ T1) uint32) {
mod.NewFunctionBuilder().
WithGoModuleFunction(api.GoModuleFunc(
func(ctx context.Context, mod api.Module, stack []uint64) {
stack[0] = uint64(fn(ctx, mod, T0(stack[0]), T1(stack[1])))
}),
[]api.ValueType{api.ValueTypeI32, api.ValueTypeI32}, []api.ValueType{api.ValueTypeI32}).
Export(name)
}
func vfsRegisterFunc3[T0, T1, T2 ~uint32](mod wazero.HostModuleBuilder, name string, fn func(ctx context.Context, mod api.Module, _ T0, _ T1, _ T2) uint32) {
mod.NewFunctionBuilder().
WithGoModuleFunction(api.GoModuleFunc(
func(ctx context.Context, mod api.Module, stack []uint64) {
stack[0] = uint64(fn(ctx, mod, T0(stack[0]), T1(stack[1]), T2(stack[2])))
}),
[]api.ValueType{api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32}, []api.ValueType{api.ValueTypeI32}).
Export(name)
}
func vfsRegisterFunc4[T0, T1, T2, T3 ~uint32](mod wazero.HostModuleBuilder, name string, fn func(ctx context.Context, mod api.Module, _ T0, _ T1, _ T2, _ T3) uint32) {
mod.NewFunctionBuilder().
WithGoModuleFunction(api.GoModuleFunc(
func(ctx context.Context, mod api.Module, stack []uint64) {
stack[0] = uint64(fn(ctx, mod, T0(stack[0]), T1(stack[1]), T2(stack[2]), T3(stack[3])))
}),
[]api.ValueType{api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32}, []api.ValueType{api.ValueTypeI32}).
Export(name)
}
func vfsRegisterFunc5[T0, T1, T2, T3, T4 ~uint32](mod wazero.HostModuleBuilder, name string, fn func(ctx context.Context, mod api.Module, _ T0, _ T1, _ T2, _ T3, _ T4) uint32) {
mod.NewFunctionBuilder().
WithGoModuleFunction(api.GoModuleFunc(
func(ctx context.Context, mod api.Module, stack []uint64) {
stack[0] = uint64(fn(ctx, mod, T0(stack[0]), T1(stack[1]), T2(stack[2]), T3(stack[3]), T4(stack[4])))
}),
[]api.ValueType{api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32}, []api.ValueType{api.ValueTypeI32}).
Export(name)
}
func vfsRegisterFuncRW(mod wazero.HostModuleBuilder, name string, fn func(ctx context.Context, mod api.Module, _, _, _ uint32, _ uint64) uint32) {
mod.NewFunctionBuilder().
WithGoModuleFunction(api.GoModuleFunc(
func(ctx context.Context, mod api.Module, stack []uint64) {
stack[0] = uint64(fn(ctx, mod, uint32(stack[0]), uint32(stack[1]), uint32(stack[2]), stack[3]))
}),
[]api.ValueType{api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI64}, []api.ValueType{api.ValueTypeI32}).
Export(name)
}
func vfsRegisterFuncT(mod wazero.HostModuleBuilder, name string, fn func(ctx context.Context, mod api.Module, _ uint32, _ uint64) uint32) {
mod.NewFunctionBuilder().
WithGoModuleFunction(api.GoModuleFunc(
func(ctx context.Context, mod api.Module, stack []uint64) {
stack[0] = uint64(fn(ctx, mod, uint32(stack[0]), stack[1]))
}),
[]api.ValueType{api.ValueTypeI32, api.ValueTypeI64}, []api.ValueType{api.ValueTypeI32}).
Export(name)
}

View File

@@ -3,10 +3,18 @@ package sqlite3
import (
"context"
"os"
"time"
"github.com/tetratelabs/wazero/api"
)
const (
// These need to match the offsets asserted in os.c
vfsFileIDOffset = 4
vfsFileLockOffset = 8
vfsFileLockTimeoutOffset = 12
)
func (vfsFileMethods) NewID(ctx context.Context, file *os.File) uint32 {
vfs := ctx.Value(vfsKey{}).(*vfsState)
@@ -26,13 +34,12 @@ func (vfsFileMethods) NewID(ctx context.Context, file *os.File) uint32 {
func (vfsFileMethods) Open(ctx context.Context, mod api.Module, pFile uint32, file *os.File) {
mem := memory{mod}
id := vfsFile.NewID(ctx, file)
mem.writeUint32(pFile+ptrlen, id)
mem.writeUint32(pFile+2*ptrlen, _NO_LOCK)
mem.writeUint32(pFile+vfsFileIDOffset, id)
}
func (vfsFileMethods) Close(ctx context.Context, mod api.Module, pFile uint32) error {
mem := memory{mod}
id := mem.readUint32(pFile + ptrlen)
id := mem.readUint32(pFile + vfsFileIDOffset)
vfs := ctx.Value(vfsKey{}).(*vfsState)
file := vfs.files[id]
vfs.files[id] = nil
@@ -41,17 +48,22 @@ func (vfsFileMethods) Close(ctx context.Context, mod api.Module, pFile uint32) e
func (vfsFileMethods) GetOS(ctx context.Context, mod api.Module, pFile uint32) *os.File {
mem := memory{mod}
id := mem.readUint32(pFile + ptrlen)
id := mem.readUint32(pFile + vfsFileIDOffset)
vfs := ctx.Value(vfsKey{}).(*vfsState)
return vfs.files[id]
}
func (vfsFileMethods) GetLock(ctx context.Context, mod api.Module, pFile uint32) vfsLockState {
mem := memory{mod}
return vfsLockState(mem.readUint32(pFile + 2*ptrlen))
return vfsLockState(mem.readUint8(pFile + vfsFileLockOffset))
}
func (vfsFileMethods) SetLock(ctx context.Context, mod api.Module, pFile uint32, lock vfsLockState) {
mem := memory{mod}
mem.writeUint32(pFile+2*ptrlen, uint32(lock))
mem.writeUint8(pFile+vfsFileLockOffset, uint8(lock))
}
func (vfsFileMethods) GetLockTimeout(ctx context.Context, mod api.Module, pFile uint32) time.Duration {
mem := memory{mod}
return time.Duration(mem.readUint32(pFile+vfsFileLockTimeoutOffset)) * time.Millisecond
}

View File

@@ -3,6 +3,7 @@ package sqlite3
import (
"context"
"os"
"time"
"github.com/tetratelabs/wazero/api"
)
@@ -56,13 +57,14 @@ const (
type vfsLockState uint32
func vfsLock(ctx context.Context, mod api.Module, pFile uint32, eLock vfsLockState) uint32 {
// Argument check. SQLite never explicitly requests a pendig lock.
// Argument check. SQLite never explicitly requests a pending lock.
if eLock != _SHARED_LOCK && eLock != _RESERVED_LOCK && eLock != _EXCLUSIVE_LOCK {
panic(assertErr())
}
file := vfsFile.GetOS(ctx, mod, pFile)
cLock := vfsFile.GetLock(ctx, mod, pFile)
timeout := vfsFile.GetLockTimeout(ctx, mod, pFile)
switch {
case cLock < _NO_LOCK || cLock > _EXCLUSIVE_LOCK:
@@ -87,11 +89,7 @@ func vfsLock(ctx context.Context, mod api.Module, pFile uint32, eLock vfsLockSta
if cLock != _NO_LOCK {
panic(assertErr())
}
// Test the PENDING lock before acquiring a new SHARED lock.
if locked, _ := vfsOS.CheckPendingLock(file); locked {
return uint32(BUSY)
}
if rc := vfsOS.GetSharedLock(file); rc != _OK {
if rc := vfsOS.GetSharedLock(file, timeout); rc != _OK {
return uint32(rc)
}
vfsFile.SetLock(ctx, mod, pFile, _SHARED_LOCK)
@@ -102,7 +100,7 @@ func vfsLock(ctx context.Context, mod api.Module, pFile uint32, eLock vfsLockSta
if cLock != _SHARED_LOCK {
panic(assertErr())
}
if rc := vfsOS.GetReservedLock(file); rc != _OK {
if rc := vfsOS.GetReservedLock(file, timeout); rc != _OK {
return uint32(rc)
}
vfsFile.SetLock(ctx, mod, pFile, _RESERVED_LOCK)
@@ -114,13 +112,13 @@ func vfsLock(ctx context.Context, mod api.Module, pFile uint32, eLock vfsLockSta
panic(assertErr())
}
// A PENDING lock is needed before acquiring an EXCLUSIVE lock.
if cLock == _RESERVED_LOCK {
if cLock < _PENDING_LOCK {
if rc := vfsOS.GetPendingLock(file); rc != _OK {
return uint32(rc)
}
vfsFile.SetLock(ctx, mod, pFile, _PENDING_LOCK)
}
if rc := vfsOS.GetExclusiveLock(file); rc != _OK {
if rc := vfsOS.GetExclusiveLock(file, timeout); rc != _OK {
return uint32(rc)
}
vfsFile.SetLock(ctx, mod, pFile, _EXCLUSIVE_LOCK)
@@ -169,15 +167,22 @@ func vfsUnlock(ctx context.Context, mod api.Module, pFile uint32, eLock vfsLockS
}
func vfsCheckReservedLock(ctx context.Context, mod api.Module, pFile, pResOut uint32) uint32 {
file := vfsFile.GetOS(ctx, mod, pFile)
cLock := vfsFile.GetLock(ctx, mod, pFile)
if cLock > _SHARED_LOCK {
// Connection state check.
if cLock < _NO_LOCK || cLock > _EXCLUSIVE_LOCK {
panic(assertErr())
}
file := vfsFile.GetOS(ctx, mod, pFile)
var locked bool
var rc xErrorCode
if cLock >= _RESERVED_LOCK {
locked = true
} else {
locked, rc = vfsOS.CheckReservedLock(file)
}
locked, rc := vfsOS.CheckReservedLock(file)
var res uint32
if locked {
res = 1
@@ -186,27 +191,17 @@ func vfsCheckReservedLock(ctx context.Context, mod api.Module, pFile, pResOut ui
return uint32(rc)
}
func (vfsOSMethods) GetSharedLock(file *os.File) xErrorCode {
// Acquire the SHARED lock.
return vfsOS.readLock(file, _SHARED_FIRST, _SHARED_SIZE)
}
func (vfsOSMethods) GetReservedLock(file *os.File) xErrorCode {
func (vfsOSMethods) GetReservedLock(file *os.File, timeout time.Duration) xErrorCode {
// Acquire the RESERVED lock.
return vfsOS.writeLock(file, _RESERVED_BYTE, 1)
return vfsOS.writeLock(file, _RESERVED_BYTE, 1, timeout)
}
func (vfsOSMethods) GetPendingLock(file *os.File) xErrorCode {
// Acquire the PENDING lock.
return vfsOS.writeLock(file, _PENDING_BYTE, 1)
return vfsOS.writeLock(file, _PENDING_BYTE, 1, 0)
}
func (vfsOSMethods) CheckReservedLock(file *os.File) (bool, xErrorCode) {
// Test the RESERVED lock.
return vfsOS.checkLock(file, _RESERVED_BYTE, 1)
}
func (vfsOSMethods) CheckPendingLock(file *os.File) (bool, xErrorCode) {
// Test the PENDING lock.
return vfsOS.checkLock(file, _PENDING_BYTE, 1)
}

View File

@@ -10,7 +10,7 @@ import (
func Test_vfsLock(t *testing.T) {
switch runtime.GOOS {
case "linux", "darwin", "illumos", "windows":
case "linux", "darwin", "windows":
break
default:
t.Skip("OS lacks OFD locks")
@@ -51,6 +51,13 @@ func Test_vfsLock(t *testing.T) {
if got := mem.readUint32(pOutput); got != 0 {
t.Error("file was locked")
}
rc = vfsCheckReservedLock(ctx, mem.mod, pFile2, pOutput)
if rc != _OK {
t.Fatal("returned", rc)
}
if got := mem.readUint32(pOutput); got != 0 {
t.Error("file was locked")
}
rc = vfsLock(ctx, mem.mod, pFile2, _SHARED_LOCK)
if rc != _OK {
@@ -64,6 +71,13 @@ func Test_vfsLock(t *testing.T) {
if got := mem.readUint32(pOutput); got != 0 {
t.Error("file was locked")
}
rc = vfsCheckReservedLock(ctx, mem.mod, pFile2, pOutput)
if rc != _OK {
t.Fatal("returned", rc)
}
if got := mem.readUint32(pOutput); got != 0 {
t.Error("file was locked")
}
rc = vfsLock(ctx, mem.mod, pFile2, _RESERVED_LOCK)
if rc != _OK {
@@ -81,6 +95,13 @@ func Test_vfsLock(t *testing.T) {
if got := mem.readUint32(pOutput); got == 0 {
t.Error("file wasn't locked")
}
rc = vfsCheckReservedLock(ctx, mem.mod, pFile2, pOutput)
if rc != _OK {
t.Fatal("returned", rc)
}
if got := mem.readUint32(pOutput); got == 0 {
t.Error("file wasn't locked")
}
rc = vfsLock(ctx, mem.mod, pFile2, _EXCLUSIVE_LOCK)
if rc != _OK {
@@ -94,6 +115,13 @@ func Test_vfsLock(t *testing.T) {
if got := mem.readUint32(pOutput); got == 0 {
t.Error("file wasn't locked")
}
rc = vfsCheckReservedLock(ctx, mem.mod, pFile2, pOutput)
if rc != _OK {
t.Fatal("returned", rc)
}
if got := mem.readUint32(pOutput); got == 0 {
t.Error("file wasn't locked")
}
rc = vfsLock(ctx, mem.mod, pFile1, _SHARED_LOCK)
if rc == _OK {
@@ -107,6 +135,13 @@ func Test_vfsLock(t *testing.T) {
if got := mem.readUint32(pOutput); got == 0 {
t.Error("file wasn't locked")
}
rc = vfsCheckReservedLock(ctx, mem.mod, pFile2, pOutput)
if rc != _OK {
t.Fatal("returned", rc)
}
if got := mem.readUint32(pOutput); got == 0 {
t.Error("file wasn't locked")
}
rc = vfsUnlock(ctx, mem.mod, pFile2, _SHARED_LOCK)
if rc != _OK {
@@ -120,6 +155,13 @@ func Test_vfsLock(t *testing.T) {
if got := mem.readUint32(pOutput); got != 0 {
t.Error("file was locked")
}
rc = vfsCheckReservedLock(ctx, mem.mod, pFile2, pOutput)
if rc != _OK {
t.Fatal("returned", rc)
}
if got := mem.readUint32(pOutput); got != 0 {
t.Error("file was locked")
}
rc = vfsLock(ctx, mem.mod, pFile1, _SHARED_LOCK)
if rc != _OK {

56
vfs_os_bsd.go Normal file
View File

@@ -0,0 +1,56 @@
//go:build freebsd || openbsd || netbsd || dragonfly || (darwin && sqlite3_bsd)
package sqlite3
import (
"os"
"time"
"golang.org/x/sys/unix"
)
func (vfsOSMethods) unlock(file *os.File, start, len int64) xErrorCode {
if start == 0 && len == 0 {
err := unix.Flock(int(file.Fd()), unix.LOCK_UN)
if err != nil {
return IOERR_UNLOCK
}
}
return _OK
}
func (vfsOSMethods) lock(file *os.File, how int, timeout time.Duration, def xErrorCode) xErrorCode {
var err error
for {
err = unix.Flock(int(file.Fd()), how)
if errno, _ := err.(unix.Errno); errno != unix.EAGAIN {
break
}
if timeout < time.Millisecond {
break
}
timeout -= time.Millisecond
time.Sleep(time.Millisecond)
}
return vfsOS.lockErrorCode(err, def)
}
func (vfsOSMethods) readLock(file *os.File, start, len int64, timeout time.Duration) xErrorCode {
return vfsOS.lock(file, unix.LOCK_SH|unix.LOCK_NB, timeout, IOERR_RDLOCK)
}
func (vfsOSMethods) writeLock(file *os.File, start, len int64, timeout time.Duration) xErrorCode {
return vfsOS.lock(file, unix.LOCK_EX|unix.LOCK_NB, timeout, IOERR_LOCK)
}
func (vfsOSMethods) checkLock(file *os.File, start, len int64) (bool, xErrorCode) {
lock := unix.Flock_t{
Type: unix.F_RDLCK,
Start: start,
Len: len,
}
if unix.FcntlFlock(file.Fd(), unix.F_GETLK, &lock) != nil {
return false, IOERR_CHECKRESERVEDLOCK
}
return lock.Type != unix.F_UNLCK, _OK
}

106
vfs_os_darwin.go Normal file
View File

@@ -0,0 +1,106 @@
//go:build !sqlite3_bsd
package sqlite3
import (
"io"
"os"
"time"
"golang.org/x/sys/unix"
)
const (
// https://github.com/apple/darwin-xnu/blob/main/bsd/sys/fcntl.h
_F_OFD_SETLK = 90
_F_OFD_SETLKW = 91
_F_OFD_GETLK = 92
_F_OFD_SETLKWTIMEOUT = 93
)
type flocktimeout_t struct {
fl unix.Flock_t
timeout unix.Timespec
}
func (vfsOSMethods) Sync(file *os.File, fullsync, dataonly bool) error {
if fullsync {
return file.Sync()
}
return unix.Fsync(int(file.Fd()))
}
func (vfsOSMethods) Allocate(file *os.File, size int64) error {
off, err := file.Seek(0, io.SeekEnd)
if err != nil {
return err
}
if size <= off {
return nil
}
// https://stackoverflow.com/a/11497568/867786
store := unix.Fstore_t{
Flags: unix.F_ALLOCATECONTIG,
Posmode: unix.F_PEOFPOSMODE,
Offset: 0,
Length: size,
}
// Try to get a continous chunk of disk space.
err = unix.FcntlFstore(file.Fd(), unix.F_PREALLOCATE, &store)
if err != nil {
// OK, perhaps we are too fragmented, allocate non-continuous.
store.Flags = unix.F_ALLOCATEALL
unix.FcntlFstore(file.Fd(), unix.F_PREALLOCATE, &store)
}
return file.Truncate(size)
}
func (vfsOSMethods) unlock(file *os.File, start, len int64) xErrorCode {
err := unix.FcntlFlock(file.Fd(), _F_OFD_SETLK, &unix.Flock_t{
Type: unix.F_UNLCK,
Start: start,
Len: len,
})
if err != nil {
return IOERR_UNLOCK
}
return _OK
}
func (vfsOSMethods) lock(file *os.File, typ int16, start, len int64, timeout time.Duration, def xErrorCode) xErrorCode {
lock := flocktimeout_t{fl: unix.Flock_t{
Type: typ,
Start: start,
Len: len,
}}
var err error
if timeout == 0 {
err = unix.FcntlFlock(file.Fd(), _F_OFD_SETLK, &lock.fl)
} else {
lock.timeout = unix.NsecToTimespec(int64(timeout / time.Nanosecond))
err = unix.FcntlFlock(file.Fd(), _F_OFD_SETLKWTIMEOUT, &lock.fl)
}
return vfsOS.lockErrorCode(err, def)
}
func (vfsOSMethods) readLock(file *os.File, start, len int64, timeout time.Duration) xErrorCode {
return vfsOS.lock(file, unix.F_RDLCK, start, len, timeout, IOERR_RDLOCK)
}
func (vfsOSMethods) writeLock(file *os.File, start, len int64, timeout time.Duration) xErrorCode {
return vfsOS.lock(file, unix.F_WRLCK, start, len, timeout, IOERR_LOCK)
}
func (vfsOSMethods) checkLock(file *os.File, start, len int64) (bool, xErrorCode) {
lock := unix.Flock_t{
Type: unix.F_RDLCK,
Start: start,
Len: len,
}
if unix.FcntlFlock(file.Fd(), _F_OFD_GETLK, &lock) != nil {
return false, IOERR_CHECKRESERVEDLOCK
}
return lock.Type != unix.F_UNLCK, _OK
}

25
vfs_os_linux.go Normal file
View File

@@ -0,0 +1,25 @@
package sqlite3
import (
"os"
"golang.org/x/sys/unix"
)
func (vfsOSMethods) Sync(file *os.File, fullsync, dataonly bool) error {
if dataonly {
_, _, err := unix.Syscall(unix.SYS_FDATASYNC, file.Fd(), 0, 0)
if err != 0 {
return err
}
return nil
}
return file.Sync()
}
func (vfsOSMethods) Allocate(file *os.File, size int64) error {
if size == 0 {
return nil
}
return unix.Fallocate(int(file.Fd()), 0, 0, size)
}

63
vfs_os_ofd.go Normal file
View File

@@ -0,0 +1,63 @@
//go:build linux || illumos
package sqlite3
import (
"os"
"time"
"golang.org/x/sys/unix"
)
func (vfsOSMethods) unlock(file *os.File, start, len int64) xErrorCode {
err := unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &unix.Flock_t{
Type: unix.F_UNLCK,
Start: start,
Len: len,
})
if err != nil {
return IOERR_UNLOCK
}
return _OK
}
func (vfsOSMethods) lock(file *os.File, typ int16, start, len int64, timeout time.Duration, def xErrorCode) xErrorCode {
lock := unix.Flock_t{
Type: typ,
Start: start,
Len: len,
}
var err error
for {
err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &lock)
if errno, _ := err.(unix.Errno); errno != unix.EAGAIN {
break
}
if timeout < time.Millisecond {
break
}
timeout -= time.Millisecond
time.Sleep(time.Millisecond)
}
return vfsOS.lockErrorCode(err, def)
}
func (vfsOSMethods) readLock(file *os.File, start, len int64, timeout time.Duration) xErrorCode {
return vfsOS.lock(file, unix.F_RDLCK, start, len, timeout, IOERR_RDLOCK)
}
func (vfsOSMethods) writeLock(file *os.File, start, len int64, timeout time.Duration) xErrorCode {
return vfsOS.lock(file, unix.F_WRLCK, start, len, timeout, IOERR_LOCK)
}
func (vfsOSMethods) checkLock(file *os.File, start, len int64) (bool, xErrorCode) {
lock := unix.Flock_t{
Type: unix.F_RDLCK,
Start: start,
Len: len,
}
if unix.FcntlFlock(file.Fd(), unix.F_OFD_GETLK, &lock) != nil {
return false, IOERR_CHECKRESERVEDLOCK
}
return lock.Type != unix.F_UNLCK, _OK
}

23
vfs_os_other.go Normal file
View File

@@ -0,0 +1,23 @@
//go:build !linux && (!darwin || sqlite3_bsd)
package sqlite3
import (
"io"
"os"
)
func (vfsOSMethods) Sync(file *os.File, fullsync, dataonly bool) error {
return file.Sync()
}
func (vfsOSMethods) Allocate(file *os.File, size int64) error {
off, err := file.Seek(0, io.SeekEnd)
if err != nil {
return err
}
if size <= off {
return nil
}
return file.Truncate(size)
}

87
vfs_os_unix.go Normal file
View File

@@ -0,0 +1,87 @@
//go:build unix
package sqlite3
import (
"io/fs"
"os"
"time"
"golang.org/x/sys/unix"
)
func (vfsOSMethods) OpenFile(name string, flag int, perm fs.FileMode) (*os.File, error) {
return os.OpenFile(name, flag, perm)
}
func (vfsOSMethods) Access(path string, flags _AccessFlag) error {
var access uint32 // unix.F_OK
switch flags {
case _ACCESS_READWRITE:
access = unix.R_OK | unix.W_OK
case _ACCESS_READ:
access = unix.R_OK
}
return unix.Access(path, access)
}
func (vfsOSMethods) GetSharedLock(file *os.File, timeout time.Duration) xErrorCode {
// Test the PENDING lock before acquiring a new SHARED lock.
if pending, _ := vfsOS.checkLock(file, _PENDING_BYTE, 1); pending {
return xErrorCode(BUSY)
}
// Acquire the SHARED lock.
return vfsOS.readLock(file, _SHARED_FIRST, _SHARED_SIZE, timeout)
}
func (vfsOSMethods) GetExclusiveLock(file *os.File, timeout time.Duration) xErrorCode {
if timeout == 0 {
timeout = time.Millisecond
}
// Acquire the EXCLUSIVE lock.
return vfsOS.writeLock(file, _SHARED_FIRST, _SHARED_SIZE, timeout)
}
func (vfsOSMethods) DowngradeLock(file *os.File, state vfsLockState) xErrorCode {
if state >= _EXCLUSIVE_LOCK {
// Downgrade to a SHARED lock.
if rc := vfsOS.readLock(file, _SHARED_FIRST, _SHARED_SIZE, 0); rc != _OK {
// In theory, the downgrade to a SHARED cannot fail because another
// process is holding an incompatible lock. If it does, this
// indicates that the other process is not following the locking
// protocol. If this happens, return IOERR_RDLOCK. Returning
// BUSY would confuse the upper layer.
return IOERR_RDLOCK
}
}
// Release the PENDING and RESERVED locks.
return vfsOS.unlock(file, _PENDING_BYTE, 2)
}
func (vfsOSMethods) ReleaseLock(file *os.File, _ vfsLockState) xErrorCode {
// Release all locks.
return vfsOS.unlock(file, 0, 0)
}
func (vfsOSMethods) lockErrorCode(err error, def xErrorCode) xErrorCode {
if err == nil {
return _OK
}
if errno, ok := err.(unix.Errno); ok {
switch errno {
case
unix.EACCES,
unix.EAGAIN,
unix.EBUSY,
unix.EINTR,
unix.ENOLCK,
unix.EDEADLK,
unix.ETIMEDOUT:
return xErrorCode(BUSY)
case unix.EPERM:
return xErrorCode(PERM)
}
}
return def
}

View File

@@ -1,10 +1,10 @@
package sqlite3
import (
"errors"
"io/fs"
"os"
"syscall"
"time"
"golang.org/x/sys/windows"
)
@@ -25,56 +25,56 @@ func (vfsOSMethods) OpenFile(name string, flag int, perm fs.FileMode) (*os.File,
return os.NewFile(uintptr(r), name), nil
}
func (vfsOSMethods) Sync(file *os.File, fullsync, dataonly bool) error {
return file.Sync()
}
func (vfsOSMethods) Access(path string, flags _AccessFlag) (bool, xErrorCode) {
func (vfsOSMethods) Access(path string, flags _AccessFlag) error {
fi, err := os.Stat(path)
switch {
case flags == _ACCESS_EXISTS:
switch {
case err == nil:
return true, _OK
case errors.Is(err, fs.ErrNotExist):
return false, _OK
default:
return false, IOERR_ACCESS
}
case err == nil:
var want fs.FileMode = syscall.S_IRUSR
if flags == _ACCESS_READWRITE {
want |= syscall.S_IWUSR
}
if fi.IsDir() {
want |= syscall.S_IXUSR
}
if fi.Mode()&want == want {
return true, _OK
} else {
return false, _OK
}
case errors.Is(err, fs.ErrPermission):
return false, _OK
default:
return false, IOERR_ACCESS
if err != nil {
return err
}
if flags == _ACCESS_EXISTS {
return nil
}
var want fs.FileMode = windows.S_IRUSR
if flags == _ACCESS_READWRITE {
want |= windows.S_IWUSR
}
if fi.IsDir() {
want |= windows.S_IXUSR
}
if fi.Mode()&want != want {
return fs.ErrPermission
}
return nil
}
func (vfsOSMethods) GetExclusiveLock(file *os.File) xErrorCode {
func (vfsOSMethods) GetSharedLock(file *os.File, timeout time.Duration) xErrorCode {
// Acquire the PENDING lock temporarily before acquiring a new SHARED lock.
rc := vfsOS.readLock(file, _PENDING_BYTE, 1, timeout)
if rc == _OK {
// Acquire the SHARED lock.
rc = vfsOS.readLock(file, _SHARED_FIRST, _SHARED_SIZE, 0)
// Release the PENDING lock.
vfsOS.unlock(file, _PENDING_BYTE, 1)
}
return rc
}
func (vfsOSMethods) GetExclusiveLock(file *os.File, timeout time.Duration) xErrorCode {
if timeout == 0 {
timeout = time.Millisecond
}
// Release the SHARED lock.
vfsOS.unlock(file, _SHARED_FIRST, _SHARED_SIZE)
// Acquire the EXCLUSIVE lock.
rc := vfsOS.writeLock(file, _SHARED_FIRST, _SHARED_SIZE)
rc := vfsOS.writeLock(file, _SHARED_FIRST, _SHARED_SIZE, timeout)
// Reacquire the SHARED lock.
if rc != _OK {
vfsOS.readLock(file, _SHARED_FIRST, _SHARED_SIZE)
// Reacquire the SHARED lock.
vfsOS.readLock(file, _SHARED_FIRST, _SHARED_SIZE, 0)
}
return rc
}
@@ -85,7 +85,7 @@ func (vfsOSMethods) DowngradeLock(file *os.File, state vfsLockState) xErrorCode
vfsOS.unlock(file, _SHARED_FIRST, _SHARED_SIZE)
// Reacquire the SHARED lock.
if rc := vfsOS.readLock(file, _SHARED_FIRST, _SHARED_SIZE); rc != _OK {
if rc := vfsOS.readLock(file, _SHARED_FIRST, _SHARED_SIZE, 0); rc != _OK {
// This should never happen.
// We should always be able to reacquire the read lock.
return IOERR_RDLOCK
@@ -128,38 +128,59 @@ func (vfsOSMethods) unlock(file *os.File, start, len uint32) xErrorCode {
return _OK
}
func (vfsOSMethods) readLock(file *os.File, start, len uint32) xErrorCode {
return vfsOS.lockErrorCode(windows.LockFileEx(windows.Handle(file.Fd()),
windows.LOCKFILE_FAIL_IMMEDIATELY,
0, len, 0, &windows.Overlapped{Offset: start}),
IOERR_RDLOCK)
func (vfsOSMethods) lock(file *os.File, flags, start, len uint32, timeout time.Duration, def xErrorCode) xErrorCode {
var err error
for {
err = windows.LockFileEx(windows.Handle(file.Fd()), flags,
0, len, 0, &windows.Overlapped{Offset: start})
if errno, _ := err.(windows.Errno); errno != windows.ERROR_LOCK_VIOLATION {
break
}
if timeout < time.Millisecond {
break
}
timeout -= time.Millisecond
time.Sleep(time.Millisecond)
}
return vfsOS.lockErrorCode(err, def)
}
func (vfsOSMethods) writeLock(file *os.File, start, len uint32) xErrorCode {
return vfsOS.lockErrorCode(windows.LockFileEx(windows.Handle(file.Fd()),
func (vfsOSMethods) readLock(file *os.File, start, len uint32, timeout time.Duration) xErrorCode {
return vfsOS.lock(file,
windows.LOCKFILE_FAIL_IMMEDIATELY,
start, len, timeout, IOERR_RDLOCK)
}
func (vfsOSMethods) writeLock(file *os.File, start, len uint32, timeout time.Duration) xErrorCode {
return vfsOS.lock(file,
windows.LOCKFILE_FAIL_IMMEDIATELY|windows.LOCKFILE_EXCLUSIVE_LOCK,
0, len, 0, &windows.Overlapped{Offset: start}),
IOERR_LOCK)
start, len, timeout, IOERR_LOCK)
}
func (vfsOSMethods) checkLock(file *os.File, start, len uint32) (bool, xErrorCode) {
rc := vfsOS.readLock(file, start, len)
rc := vfsOS.lock(file,
windows.LOCKFILE_FAIL_IMMEDIATELY,
start, len, 0, IOERR_CHECKRESERVEDLOCK)
if rc == xErrorCode(BUSY) {
return true, _OK
}
if rc == _OK {
vfsOS.unlock(file, start, len)
}
return rc != _OK, _OK
return false, rc
}
func (vfsOSMethods) lockErrorCode(err error, def xErrorCode) xErrorCode {
if err == nil {
return _OK
}
if errno, ok := err.(syscall.Errno); ok {
if errno, ok := err.(windows.Errno); ok {
// https://devblogs.microsoft.com/oldnewthing/20140905-00/?p=63
switch errno {
case
windows.ERROR_LOCK_VIOLATION,
windows.ERROR_IO_PENDING:
windows.ERROR_IO_PENDING,
windows.ERROR_OPERATION_ABORTED:
return xErrorCode(BUSY)
}
}

View File

@@ -14,19 +14,11 @@ import (
"github.com/ncruces/julianday"
)
func Test_vfsExit(t *testing.T) {
mem := newMemory(128)
ctx := context.TODO()
defer func() { _ = recover() }()
vfsExit(ctx, mem.mod, 1)
t.Error("want panic")
}
func Test_vfsLocaltime(t *testing.T) {
mem := newMemory(128)
ctx := context.TODO()
rc := vfsLocaltime(ctx, mem.mod, 0, 4)
rc := vfsLocaltime(ctx, mem.mod, 4, 0)
if rc != 0 {
t.Fatal("returned", rc)
}
@@ -60,8 +52,9 @@ func Test_vfsLocaltime(t *testing.T) {
func Test_vfsRandomness(t *testing.T) {
mem := newMemory(128)
ctx := context.TODO()
rc := vfsRandomness(context.TODO(), mem.mod, 0, 16, 4)
rc := vfsRandomness(ctx, mem.mod, 0, 16, 4)
if rc != 16 {
t.Fatal("returned", rc)
}
@@ -73,10 +66,11 @@ func Test_vfsRandomness(t *testing.T) {
}
func Test_vfsSleep(t *testing.T) {
mem := newMemory(128)
ctx := context.TODO()
now := time.Now()
rc := vfsSleep(ctx, 0, 123456)
rc := vfsSleep(ctx, mem.mod, 0, 123456)
if rc != 0 {
t.Fatal("returned", rc)
}

View File

@@ -1,170 +0,0 @@
//go:build unix
package sqlite3
import (
"io/fs"
"os"
"runtime"
"golang.org/x/sys/unix"
)
func (vfsOSMethods) OpenFile(name string, flag int, perm fs.FileMode) (*os.File, error) {
return os.OpenFile(name, flag, perm)
}
func (vfsOSMethods) Access(path string, flags _AccessFlag) (bool, xErrorCode) {
var access uint32 = unix.F_OK
switch flags {
case _ACCESS_READWRITE:
access = unix.R_OK | unix.W_OK
case _ACCESS_READ:
access = unix.R_OK
}
err := unix.Access(path, access)
if err == nil {
return true, _OK
}
return false, _OK
}
func (vfsOSMethods) Sync(file *os.File, fullsync, dataonly bool) error {
if runtime.GOOS == "darwin" && !fullsync {
return unix.Fsync(int(file.Fd()))
}
if runtime.GOOS == "linux" && dataonly {
//lint:ignore SA1019 OK on linux
_, _, err := unix.Syscall(unix.SYS_FDATASYNC, file.Fd(), 0, 0)
if err != 0 {
return err
}
return nil
}
return file.Sync()
}
func (vfsOSMethods) GetExclusiveLock(file *os.File) xErrorCode {
// Acquire the EXCLUSIVE lock.
return vfsOS.writeLock(file, _SHARED_FIRST, _SHARED_SIZE)
}
func (vfsOSMethods) DowngradeLock(file *os.File, state vfsLockState) xErrorCode {
if state >= _EXCLUSIVE_LOCK {
// Downgrade to a SHARED lock.
if rc := vfsOS.readLock(file, _SHARED_FIRST, _SHARED_SIZE); rc != _OK {
// In theory, the downgrade to a SHARED cannot fail because another
// process is holding an incompatible lock. If it does, this
// indicates that the other process is not following the locking
// protocol. If this happens, return IOERR_RDLOCK. Returning
// BUSY would confuse the upper layer.
return IOERR_RDLOCK
}
}
// Release the PENDING and RESERVED locks.
return vfsOS.unlock(file, _PENDING_BYTE, 2)
}
func (vfsOSMethods) ReleaseLock(file *os.File, _ vfsLockState) xErrorCode {
// Release all locks.
return vfsOS.unlock(file, 0, 0)
}
func (vfsOSMethods) unlock(file *os.File, start, len int64) xErrorCode {
err := vfsOS.fcntlSetLock(file, &unix.Flock_t{
Type: unix.F_UNLCK,
Start: start,
Len: len,
})
if err != nil {
return IOERR_UNLOCK
}
return _OK
}
func (vfsOSMethods) readLock(file *os.File, start, len int64) xErrorCode {
return vfsOS.lockErrorCode(vfsOS.fcntlSetLock(file, &unix.Flock_t{
Type: unix.F_RDLCK,
Start: start,
Len: len,
}), IOERR_RDLOCK)
}
func (vfsOSMethods) writeLock(file *os.File, start, len int64) xErrorCode {
return vfsOS.lockErrorCode(vfsOS.fcntlSetLock(file, &unix.Flock_t{
Type: unix.F_WRLCK,
Start: start,
Len: len,
}), IOERR_LOCK)
}
func (vfsOSMethods) checkLock(file *os.File, start, len int64) (bool, xErrorCode) {
lock := unix.Flock_t{
Type: unix.F_RDLCK,
Start: start,
Len: len,
}
if vfsOS.fcntlGetLock(file, &lock) != nil {
return false, IOERR_CHECKRESERVEDLOCK
}
return lock.Type != unix.F_UNLCK, _OK
}
func (vfsOSMethods) fcntlGetLock(file *os.File, lock *unix.Flock_t) error {
var F_OFD_GETLK int
switch runtime.GOOS {
case "linux":
// https://github.com/torvalds/linux/blob/master/include/uapi/asm-generic/fcntl.h
F_OFD_GETLK = 36
case "darwin":
// https://github.com/apple/darwin-xnu/blob/main/bsd/sys/fcntl.h
F_OFD_GETLK = 92
case "illumos":
// https://github.com/illumos/illumos-gate/blob/master/usr/src/uts/common/sys/fcntl.h
F_OFD_GETLK = 47
default:
return notImplErr
}
return unix.FcntlFlock(file.Fd(), F_OFD_GETLK, lock)
}
func (vfsOSMethods) fcntlSetLock(file *os.File, lock *unix.Flock_t) error {
var F_OFD_SETLK int
switch runtime.GOOS {
case "linux":
// https://github.com/torvalds/linux/blob/master/include/uapi/asm-generic/fcntl.h
F_OFD_SETLK = 37
case "darwin":
// https://github.com/apple/darwin-xnu/blob/main/bsd/sys/fcntl.h
F_OFD_SETLK = 90
case "illumos":
// https://github.com/illumos/illumos-gate/blob/master/usr/src/uts/common/sys/fcntl.h
F_OFD_SETLK = 48
default:
return notImplErr
}
return unix.FcntlFlock(file.Fd(), F_OFD_SETLK, lock)
}
func (vfsOSMethods) lockErrorCode(err error, def xErrorCode) xErrorCode {
if err == nil {
return _OK
}
if errno, ok := err.(unix.Errno); ok {
switch errno {
case
unix.EACCES,
unix.EAGAIN,
unix.EBUSY,
unix.EINTR,
unix.ENOLCK,
unix.EDEADLK,
unix.ETIMEDOUT:
return xErrorCode(BUSY)
case unix.EPERM:
return xErrorCode(PERM)
}
}
return def
}