Towards incremental blobs.

api.go

@@ -30,8 +30,8 @@ func newConn(ctx context.Context, module api.Module) (_ *Conn, err error) {
 		ctx: ctx,
 		mem: memory{module},
 		api: sqliteAPI{
-			malloc:        getFun("malloc"),
 			free:          getFun("free"),
+			malloc:        getFun("malloc"),
 			destructor:    uint64(getPtr("malloc_destructor")),
 			errcode:       getFun("sqlite3_errcode"),
 			errstr:        getFun("sqlite3_errstr"),
@@ -66,6 +66,12 @@ func newConn(ctx context.Context, module api.Module) (_ *Conn, err error) {
 			lastRowid:     getFun("sqlite3_last_insert_rowid"),
 			changes:       getFun("sqlite3_changes64"),
 			interrupt:     getFun("sqlite3_interrupt"),
+			blobOpen:      getFun("sqlite3_blob_open"),
+			blobClose:     getFun("sqlite3_blob_close"),
+			blobReopen:    getFun("sqlite3_blob_reopen"),
+			blobBytes:     getFun("sqlite3_blob_bytes"),
+			blobRead:      getFun("sqlite3_blob_read"),
+			blobWrite:     getFun("sqlite3_blob_write"),
 		},
 	}
 	if err != nil {
@@ -75,8 +81,8 @@ func newConn(ctx context.Context, module api.Module) (_ *Conn, err error) {
 }
 
 type sqliteAPI struct {
-	malloc        api.Function
 	free          api.Function
+	malloc        api.Function
 	destructor    uint64
 	errcode       api.Function
 	errstr        api.Function
@@ -111,4 +117,10 @@ type sqliteAPI struct {
 	lastRowid     api.Function
 	changes       api.Function
 	interrupt     api.Function
+	blobOpen      api.Function
+	blobClose     api.Function
+	blobReopen    api.Function
+	blobBytes     api.Function
+	blobRead      api.Function
+	blobWrite     api.Function
 }

conn.go

@@ -333,8 +333,11 @@ func (c *Conn) free(ptr uint32) {
 	c.call(c.api.free, uint64(ptr))
 }
 
-func (c *Conn) new(size uint32) uint32 {
-	r := c.call(c.api.malloc, uint64(size))
+func (c *Conn) new(size uint64) uint32 {
+	if size > _MAX_ALLOCATION_SIZE {
+		panic(oomErr)
+	}
+	r := c.call(c.api.malloc, size)
 	ptr := uint32(r[0])
 	if ptr == 0 && size != 0 {
 		panic(oomErr)
@@ -346,22 +349,22 @@ func (c *Conn) newBytes(b []byte) uint32 {
 	if b == nil {
 		return 0
 	}
-	ptr := c.new(uint32(len(b)))
+	ptr := c.new(uint64(len(b)))
 	c.mem.writeBytes(ptr, b)
 	return ptr
 }
 
 func (c *Conn) newString(s string) uint32 {
-	ptr := c.new(uint32(len(s) + 1))
+	ptr := c.new(uint64(len(s) + 1))
 	c.mem.writeString(ptr, s)
 	return ptr
 }
 
-func (c *Conn) newArena(size uint32) arena {
+func (c *Conn) newArena(size uint64) arena {
 	return arena{
 		c:    c,
-		size: size,
 		base: c.new(size),
+		size: uint32(size),
 	}
 }
 
@@ -390,10 +393,10 @@ func (a *arena) reset() {
 	a.next = 0
 }
 
-func (a *arena) new(size uint32) uint32 {
-	if a.next+size <= a.size {
+func (a *arena) new(size uint64) uint32 {
+	if size <= uint64(a.size-a.next) {
 		ptr := a.base + a.next
-		a.next += size
+		a.next += uint32(size)
 		return ptr
 	}
 	ptr := a.c.new(size)
@@ -402,7 +405,7 @@ func (a *arena) new(size uint32) uint32 {
 }
 
 func (a *arena) string(s string) uint32 {
-	ptr := a.new(uint32(len(s) + 1))
+	ptr := a.new(uint64(len(s) + 1))
 	a.c.mem.writeString(ptr, s)
 	return ptr
 }

conn_test.go

@@ -6,6 +6,20 @@ import (
 	"testing"
 )
 
+func TestConn_error_OOM(t *testing.T) {
+	t.Parallel()
+
+	db, err := Open(":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	defer func() { _ = recover() }()
+	db.error(uint64(NOMEM))
+	t.Error("want panic")
+}
+
 func TestConn_call_nil(t *testing.T) {
 	t.Parallel()
 
@@ -29,9 +43,14 @@ func TestConn_new(t *testing.T) {
 	}
 	defer db.Close()
 
-	defer func() { _ = recover() }()
-	db.new(math.MaxUint32)
-	t.Error("want panic")
+	testOOM := func(size uint64) {
+		defer func() { _ = recover() }()
+		db.new(size)
+		t.Error("want panic")
+	}
+
+	testOOM(math.MaxUint32)
+	testOOM(_MAX_ALLOCATION_SIZE)
 }
 
 func TestConn_newArena(t *testing.T) {
@@ -88,7 +107,7 @@ func TestConn_newBytes(t *testing.T) {
 	}
 
 	want := buf
-	if got := db.mem.view(ptr, uint32(len(want))); !bytes.Equal(got, want) {
+	if got := db.mem.view(ptr, uint64(len(want))); !bytes.Equal(got, want) {
 		t.Errorf("got %q, want %q", got, want)
 	}
 }
@@ -114,7 +133,7 @@ func TestConn_newString(t *testing.T) {
 	}
 
 	want := str + "\000"
-	if got := db.mem.view(ptr, uint32(len(want))); string(got) != want {
+	if got := db.mem.view(ptr, uint64(len(want))); string(got) != want {
 		t.Errorf("got %q, want %q", got, want)
 	}
 }

const.go

@@ -12,6 +12,8 @@ const (
 	_MAX_STRING   = 512 // Used for short strings: names, error messages…
 	_MAX_PATHNAME = 512
 
+	_MAX_ALLOCATION_SIZE = 0x7ffffeff
+
 	ptrlen = 4
 )
 

embed/build.sh

@@ -13,8 +13,8 @@ zig cc --target=wasm32-wasi -flto -g0 -Os \
 	-mbulk-memory -mreference-types \
 	-mnontrapping-fptoint -msign-ext \
 	-D_HAVE_SQLITE_CONFIG_H \
-	-Wl,--export=malloc \
 	-Wl,--export=free \
+	-Wl,--export=malloc \
 	-Wl,--export=malloc_destructor \
 	-Wl,--export=sqlite3_errcode \
 	-Wl,--export=sqlite3_errstr \
@@ -45,7 +45,13 @@ zig cc --target=wasm32-wasi -flto -g0 -Os \
 	-Wl,--export=sqlite3_column_text \
 	-Wl,--export=sqlite3_column_blob \
 	-Wl,--export=sqlite3_column_bytes \
+	-Wl,--export=sqlite3_blob_open \
+	-Wl,--export=sqlite3_blob_close \
+	-Wl,--export=sqlite3_blob_bytes \
+	-Wl,--export=sqlite3_blob_read \
+	-Wl,--export=sqlite3_blob_write \
+	-Wl,--export=sqlite3_blob_reopen \
 	-Wl,--export=sqlite3_get_autocommit \
 	-Wl,--export=sqlite3_last_insert_rowid \
 	-Wl,--export=sqlite3_changes64 \
-	-Wl,--export=sqlite3_interrupt \
+	-Wl,--export=sqlite3_interrupt \

mem.go

@@ -11,11 +11,14 @@ type memory struct {
 	mod api.Module
 }
 
-func (m memory) view(ptr, size uint32) []byte {
+func (m memory) view(ptr uint32, size uint64) []byte {
 	if ptr == 0 {
 		panic(nilErr)
 	}
-	buf, ok := m.mod.Memory().Read(ptr, size)
+	if size > math.MaxUint32 {
+		panic(rangeErr)
+	}
+	buf, ok := m.mod.Memory().Read(ptr, uint32(size))
 	if !ok {
 		panic(rangeErr)
 	}
@@ -100,12 +103,12 @@ func (m memory) readString(ptr, maxlen uint32) string {
 }
 
 func (m memory) writeBytes(ptr uint32, b []byte) {
-	buf := m.view(ptr, uint32(len(b)))
+	buf := m.view(ptr, uint64(len(b)))
 	copy(buf, b)
 }
 
 func (m memory) writeString(ptr uint32, s string) {
-	buf := m.view(ptr, uint32(len(s)+1))
+	buf := m.view(ptr, uint64(len(s)+1))
 	buf[len(s)] = 0
 	copy(buf, s)
 }

mem_test.go

@@ -19,6 +19,13 @@ func Test_memory_view_range(t *testing.T) {
 	t.Error("want panic")
 }
 
+func Test_memory_view_overflow(t *testing.T) {
+	defer func() { _ = recover() }()
+	mem := newMemory(128)
+	mem.view(1, math.MaxInt64)
+	t.Error("want panic")
+}
+
 func Test_memory_readUint32_nil(t *testing.T) {
 	defer func() { _ = recover() }()
 	mem := newMemory(128)

stmt.go

@@ -347,7 +347,7 @@ func (s *Stmt) ColumnText(col int) string {
 	r = s.c.call(s.c.api.columnBytes,
 		uint64(s.handle), uint64(col))
 
-	mem := s.c.mem.view(ptr, uint32(r[0]))
+	mem := s.c.mem.view(ptr, r[0])
 	return string(mem)
 }
 
@@ -370,7 +370,7 @@ func (s *Stmt) ColumnBlob(col int, buf []byte) []byte {
 	r = s.c.call(s.c.api.columnBytes,
 		uint64(s.handle), uint64(col))
 
-	mem := s.c.mem.view(ptr, uint32(r[0]))
+	mem := s.c.mem.view(ptr, r[0])
 	return append(buf[0:0], mem...)
 }
 

vfs.go

@@ -85,7 +85,7 @@ func vfsLocaltime(ctx context.Context, mod api.Module, t uint64, pTm uint32) uin
 }
 
 func vfsRandomness(ctx context.Context, mod api.Module, pVfs, nByte, zByte uint32) uint32 {
-	mem := memory{mod}.view(zByte, nByte)
+	mem := memory{mod}.view(zByte, uint64(nByte))
 	n, _ := rand.Reader.Read(mem)
 	return uint32(n)
 }
@@ -119,8 +119,8 @@ func vfsFullPathname(ctx context.Context, mod api.Module, pVfs, zRelative, nFull
 	// Or using [os.Readlink] to resolve a symbolic link (as the Unix VFS did).
 	// This might be buggy on Windows (the Windows VFS doesn't try).
 
-	size := uint32(len(abs) + 1)
-	if size > nFull {
+	size := uint64(len(abs) + 1)
+	if size > uint64(nFull) {
 		return uint32(CANTOPEN_FULLPATH)
 	}
 	mem := memory{mod}.view(zFull, size)
@@ -246,7 +246,7 @@ func vfsClose(ctx context.Context, mod api.Module, pFile uint32) uint32 {
 }
 
 func vfsRead(ctx context.Context, mod api.Module, pFile, zBuf, iAmt uint32, iOfst uint64) uint32 {
-	buf := memory{mod}.view(zBuf, iAmt)
+	buf := memory{mod}.view(zBuf, uint64(iAmt))
 
 	file := vfsFilePtr{mod, pFile}.OSFile()
 	n, err := file.ReadAt(buf, int64(iOfst))
@@ -263,7 +263,7 @@ func vfsRead(ctx context.Context, mod api.Module, pFile, zBuf, iAmt uint32, iOfs
 }
 
 func vfsWrite(ctx context.Context, mod api.Module, pFile, zBuf, iAmt uint32, iOfst uint64) uint32 {
-	buf := memory{mod}.view(zBuf, iAmt)
+	buf := memory{mod}.view(zBuf, uint64(iAmt))
 
 	file := vfsFilePtr{mod, pFile}.OSFile()
 	_, err := file.WriteAt(buf, int64(iOfst))