From c1838fc0bc88938fbca34a320f02ffc9e5703b64 Mon Sep 17 00:00:00 2001 From: Nuno Cruces Date: Mon, 27 Nov 2023 14:57:04 +0000 Subject: [PATCH] Fix encoding issues. --- context.go | 2 ++ embed/exports.txt | 6 ++++-- ext/lines/lines.go | 17 ++++++++++++----- ext/unicode/unicode.go | 6 +++--- func_test.go | 2 +- sqlite3/sqlite_cfg.h | 3 +-- tests/db_test.go | 19 ++++++++++++++++--- tests/testdata/utf16be.db | Bin 0 -> 1024 bytes 8 files changed, 39 insertions(+), 16 deletions(-) create mode 100644 tests/testdata/utf16be.db diff --git a/context.go b/context.go index 2d5bfbd..2264c67 100644 --- a/context.go +++ b/context.go @@ -175,6 +175,7 @@ func (ctx Context) ResultJSON(value any) { data, err := json.Marshal(value) if err != nil { ctx.ResultError(err) + return } ctx.ResultRawText(data) } @@ -185,6 +186,7 @@ func (ctx Context) ResultJSON(value any) { func (ctx Context) ResultValue(value Value) { if value.sqlite != ctx.c.sqlite { ctx.ResultError(MISUSE) + return } ctx.c.call(ctx.c.api.resultValue, uint64(ctx.handle), uint64(value.handle)) diff --git a/embed/exports.txt b/embed/exports.txt index 1233f1b..1d20aae 100644 --- a/embed/exports.txt +++ b/embed/exports.txt @@ -50,11 +50,13 @@ sqlite3_uri_key sqlite3_changes64 sqlite3_last_insert_rowid sqlite3_get_autocommit -sqlite3_anycollseq_init sqlite3_create_collation_go sqlite3_create_function_go sqlite3_create_aggregate_function_go sqlite3_create_window_function_go +sqlite3_create_module_go +sqlite3_overload_function +sqlite3_anycollseq_init sqlite3_aggregate_context sqlite3_user_data sqlite3_set_auxdata_go @@ -66,6 +68,7 @@ sqlite3_value_text sqlite3_value_blob sqlite3_value_bytes sqlite3_value_pointer_go +sqlite3_value_nochange sqlite3_result_null sqlite3_result_int64 sqlite3_result_double @@ -78,7 +81,6 @@ sqlite3_result_error sqlite3_result_error_code sqlite3_result_error_nomem sqlite3_result_error_toobig -sqlite3_create_module_go sqlite3_declare_vtab sqlite3_vtab_config_go sqlite3_vtab_collation diff --git a/ext/lines/lines.go b/ext/lines/lines.go index f2c9bb3..b598cf7 100644 --- a/ext/lines/lines.go +++ b/ext/lines/lines.go @@ -95,12 +95,14 @@ func (c *cursor) Filter(idxNum int, idxStr string, arg ...sqlite3.Value) error { var r io.Reader data := arg[0] + typ := data.Type() if c.reader { - if data.Type() == sqlite3.NULL { + switch typ { + case sqlite3.NULL: if p, ok := data.Pointer().(io.ReaderAt); ok { r = io.NewSectionReader(p, 0, math.MaxInt64) } - } else { + case sqlite3.TEXT: f, err := os.Open(data.Text()) if err != nil { return err @@ -108,12 +110,17 @@ func (c *cursor) Filter(idxNum int, idxStr string, arg ...sqlite3.Value) error { c.closer = f r = f } - } else if data.Type() != sqlite3.NULL { - r = bytes.NewReader(data.RawBlob()) + } else { + switch typ { + case sqlite3.TEXT: + r = bytes.NewReader(data.RawText()) + case sqlite3.BLOB: + r = bytes.NewReader(data.RawBlob()) + } } if r == nil { - return fmt.Errorf("lines: unsupported argument:%.0w %v", sqlite3.MISMATCH, data.Type()) + return fmt.Errorf("lines: unsupported argument:%.0w %v", sqlite3.MISMATCH, typ) } c.scanner = bufio.NewScanner(r) c.rowID = 0 diff --git a/ext/unicode/unicode.go b/ext/unicode/unicode.go index 778e8f5..22784d5 100644 --- a/ext/unicode/unicode.go +++ b/ext/unicode/unicode.go @@ -113,14 +113,14 @@ func regex(ctx sqlite3.Context, arg ...sqlite3.Value) { re = r ctx.SetAuxData(0, re) } - ctx.ResultBool(re.Match(arg[1].RawBlob())) + ctx.ResultBool(re.Match(arg[1].RawText())) } func like(ctx sqlite3.Context, arg ...sqlite3.Value) { escape := rune(-1) if len(arg) == 3 { var size int - b := arg[2].RawBlob() + b := arg[2].RawText() escape, size = utf8.DecodeRune(b) if size != len(b) { ctx.ResultError(util.ErrorString("ESCAPE expression must be a single character")) @@ -141,7 +141,7 @@ func like(ctx sqlite3.Context, arg ...sqlite3.Value) { } ctx.SetAuxData(0, re) } - ctx.ResultBool(re.Match(arg[1].RawBlob())) + ctx.ResultBool(re.Match(arg[1].RawText())) } func like2regex(pattern string, escape rune) string { diff --git a/func_test.go b/func_test.go index f2b4cf2..c27eabd 100644 --- a/func_test.go +++ b/func_test.go @@ -129,7 +129,7 @@ func ExampleContext_SetAuxData() { ctx.SetAuxData(0, r) re = r } - ctx.ResultBool(re.Match(arg[1].RawBlob())) + ctx.ResultBool(re.Match(arg[1].RawText())) }) if err != nil { log.Fatal(err) diff --git a/sqlite3/sqlite_cfg.h b/sqlite3/sqlite_cfg.h index 97705bb..252755d 100644 --- a/sqlite3/sqlite_cfg.h +++ b/sqlite3/sqlite_cfg.h @@ -46,9 +46,8 @@ // Other Options #define SQLITE_ALLOW_URI_AUTHORITY -#define SQLITE_ENABLE_BATCH_ATOMIC_WRITE #define SQLITE_ENABLE_ATOMIC_WRITE -#define SQLITE_OMIT_DESERIALIZE +#define SQLITE_ENABLE_BATCH_ATOMIC_WRITE // Because WASM does not support shared memory, // SQLite disables WAL for WASM builds. diff --git a/tests/db_test.go b/tests/db_test.go index 54acbde..27f3475 100644 --- a/tests/db_test.go +++ b/tests/db_test.go @@ -15,6 +15,9 @@ import ( //go:embed testdata/wal.db var waldb []byte +//go:embed testdata/utf16be.db +var utf16db []byte + func TestDB_memory(t *testing.T) { t.Parallel() testDB(t, ":memory:") @@ -34,12 +37,22 @@ func TestDB_nolock(t *testing.T) { func TestDB_wal(t *testing.T) { t.Parallel() - wal := filepath.Join(t.TempDir(), "test.db") - err := os.WriteFile(wal, waldb, 0666) + tmp := filepath.Join(t.TempDir(), "test.db") + err := os.WriteFile(tmp, waldb, 0666) if err != nil { t.Fatal(err) } - testDB(t, wal) + testDB(t, tmp) +} + +func TestDB_utf16(t *testing.T) { + t.Parallel() + tmp := filepath.Join(t.TempDir(), "test.db") + err := os.WriteFile(tmp, utf16db, 0666) + if err != nil { + t.Fatal(err) + } + testDB(t, tmp) } func TestDB_vfs(t *testing.T) { diff --git a/tests/testdata/utf16be.db b/tests/testdata/utf16be.db new file mode 100644 index 0000000000000000000000000000000000000000..08dc812ccfa16560828dbfa65e1b8df8bff49446 GIT binary patch literal 1024 zcmWFz^vNtqRY=P(%1ta$FlJz4U}WTRP*7lCU|Iq`kIG{Yfk5oSqoSiBKw$^~04LT6bN~PV literal 0 HcmV?d00001