diff --git a/ext/csv/arg_test.go b/ext/csv/arg_test.go index 50d4c47..6d009b8 100644 --- a/ext/csv/arg_test.go +++ b/ext/csv/arg_test.go @@ -40,6 +40,8 @@ func Test_uintArg(t *testing.T) { } func Test_boolArg(t *testing.T) { + t.Parallel() + tests := []struct { arg string key string @@ -76,6 +78,8 @@ func Test_boolArg(t *testing.T) { } func Test_runeArg(t *testing.T) { + t.Parallel() + tests := []struct { arg string key string diff --git a/ext/csv/csv.go b/ext/csv/csv.go index eb931c1..70517e1 100644 --- a/ext/csv/csv.go +++ b/ext/csv/csv.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "io/fs" + "strconv" "strings" "github.com/ncruces/go-sqlite3" @@ -93,6 +94,8 @@ func RegisterFS(db *sqlite3.Conn, fsys fs.FS) { } } schema = getSchema(header, columns, row) + } else { + table.typs = getColumnAffinities(schema) } err = db.DeclareVTab(schema) @@ -113,6 +116,7 @@ type table struct { fsys fs.FS name string data string + typs []affinity comma rune header bool } @@ -226,7 +230,40 @@ func (c *cursor) RowID() (int64, error) { func (c *cursor) Column(ctx *sqlite3.Context, col int) error { if col < len(c.row) { - ctx.ResultText(c.row[col]) + var typ affinity + if col < len(c.table.typs) { + typ = c.table.typs[col] + } + + txt := c.row[col] + if typ == blob { + ctx.ResultText(txt) + return nil + } + if txt == "" { + return nil + } + + switch typ { + case numeric, integer: + if strings.TrimLeft(txt, "+-0123456789") == "" { + if i, err := strconv.ParseInt(txt, 10, 64); err == nil { + ctx.ResultInt64(i) + return nil + } + } + fallthrough + case real: + if strings.TrimLeft(txt, "+-.0123456789Ee") == "" { + if f, err := strconv.ParseFloat(txt, 64); err == nil { + ctx.ResultFloat(f) + return nil + } + } + fallthrough + case text: + ctx.ResultText(c.row[col]) + } } return nil } diff --git a/ext/csv/csv_test.go b/ext/csv/csv_test.go index 5f64343..5ae91e2 100644 --- a/ext/csv/csv_test.go +++ b/ext/csv/csv_test.go @@ -113,6 +113,50 @@ Robert "Griesemer" "gri"` } } +func TestAffinity(t *testing.T) { + t.Parallel() + + db, err := sqlite3.Open(":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + + csv.Register(db) + + const data = "01\n0.10\ne" + err = db.Exec(` + CREATE VIRTUAL TABLE temp.nums USING csv( + data = ` + sqlite3.Quote(data) + `, + schema = 'CREATE TABLE x(a numeric)' + )`) + if err != nil { + t.Fatal(err) + } + + stmt, _, err := db.Prepare(`SELECT * FROM temp.nums`) + if err != nil { + t.Fatal(err) + } + defer stmt.Close() + + if stmt.Step() { + if got := stmt.ColumnText(0); got != "1" { + t.Errorf("got %q want 1", got) + } + } + if stmt.Step() { + if got := stmt.ColumnText(0); got != "0.1" { + t.Errorf("got %q want 0.1", got) + } + } + if stmt.Step() { + if got := stmt.ColumnText(0); got != "e" { + t.Errorf("got %q want e", got) + } + } +} + func TestRegister_errors(t *testing.T) { t.Parallel() diff --git a/ext/csv/types.go b/ext/csv/types.go new file mode 100644 index 0000000..967b4e9 --- /dev/null +++ b/ext/csv/types.go @@ -0,0 +1,54 @@ +package csv + +import ( + _ "embed" + "strings" + + "github.com/ncruces/go-sqlite3/util/vtabutil" +) + +type affinity byte + +const ( + blob affinity = 0 + text affinity = 1 + numeric affinity = 2 + integer affinity = 3 + real affinity = 4 +) + +func getColumnAffinities(schema string) []affinity { + tab, err := vtabutil.Parse(schema) + if err != nil { + return nil + } + defer tab.Close() + + types := make([]affinity, tab.NumColumns()) + for i := range types { + col := tab.Column(i) + types[i] = getAffinity(col.Type()) + } + return types +} + +func getAffinity(declType string) affinity { + // https://sqlite.org/datatype3.html#determination_of_column_affinity + if declType == "" { + return blob + } + name := strings.ToUpper(declType) + if strings.Contains(name, "INT") { + return integer + } + if strings.Contains(name, "CHAR") || strings.Contains(name, "CLOB") || strings.Contains(name, "TEXT") { + return text + } + if strings.Contains(name, "BLOB") { + return blob + } + if strings.Contains(name, "REAL") || strings.Contains(name, "FLOA") || strings.Contains(name, "DOUB") { + return real + } + return numeric +} diff --git a/ext/csv/types_test.go b/ext/csv/types_test.go new file mode 100644 index 0000000..b88c351 --- /dev/null +++ b/ext/csv/types_test.go @@ -0,0 +1,35 @@ +package csv + +import ( + _ "embed" + "testing" +) + +func Test_getAffinity(t *testing.T) { + tests := []struct { + decl string + want affinity + }{ + {"", blob}, + {"INTEGER", integer}, + {"TINYINT", integer}, + {"TEXT", text}, + {"CHAR", text}, + {"CLOB", text}, + {"BLOB", blob}, + {"REAL", real}, + {"FLOAT", real}, + {"DOUBLE", real}, + {"NUMERIC", numeric}, + {"DECIMAL", numeric}, + {"BOOLEAN", numeric}, + {"DATETIME", numeric}, + } + for _, tt := range tests { + t.Run(tt.decl, func(t *testing.T) { + if got := getAffinity(tt.decl); got != tt.want { + t.Errorf("getAffinity() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/util/vtabutil/README.md b/util/vtabutil/README.md new file mode 100644 index 0000000..7653ae0 --- /dev/null +++ b/util/vtabutil/README.md @@ -0,0 +1,8 @@ +# Virtual Table utility functions + +This package implements utilities mostly useful to virtual table implementations. + +It also wraps a [parser](https://github.com/marcobambini/sqlite-createtable-parser) +for the [`CREATE`](https://sqlite.org/lang_createtable.html) and +[`ALTER TABLE`](https://sqlite.org/lang_altertable.html) commands, +created by [Marco Bambini](https://github.com/marcobambini). \ No newline at end of file diff --git a/util/vtabutil/arg.go b/util/vtabutil/arg.go index 15e2b74..7f8fac9 100644 --- a/util/vtabutil/arg.go +++ b/util/vtabutil/arg.go @@ -1,4 +1,3 @@ -// Package ioutil implements virtual table utility functions. package vtabutil import "strings" diff --git a/util/vtabutil/parse.go b/util/vtabutil/parse.go new file mode 100644 index 0000000..56154ca --- /dev/null +++ b/util/vtabutil/parse.go @@ -0,0 +1,145 @@ +package vtabutil + +import ( + "context" + "sync" + + _ "embed" + + "github.com/ncruces/go-sqlite3/internal/util" + "github.com/tetratelabs/wazero" + "github.com/tetratelabs/wazero/api" +) + +const ( + code = 4 + base = 8 +) + +var ( + //go:embed parse/sql3parse_table.wasm + binary []byte + ctx context.Context + once sync.Once + runtime wazero.Runtime +) + +// Table holds metadata about a table. +type Table struct { + mod api.Module + ptr uint32 + sql string +} + +// Parse parses a [CREATE] or [ALTER TABLE] command. +// +// [CREATE]: https://sqlite.org/lang_createtable.html +// [ALTER TABLE]: https://sqlite.org/lang_altertable.html +func Parse(sql string) (*Table, error) { + once.Do(func() { + ctx = context.Background() + cfg := wazero.NewRuntimeConfigInterpreter().WithDebugInfoEnabled(false) + runtime = wazero.NewRuntimeWithConfig(ctx, cfg) + }) + + mod, err := runtime.InstantiateWithConfig(ctx, binary, wazero.NewModuleConfig().WithName("")) + if err != nil { + return nil, err + } + + if buf, ok := mod.Memory().Read(base, uint32(len(sql))); ok { + copy(buf, sql) + } + r, err := mod.ExportedFunction("sql3parse_table").Call(ctx, base, uint64(len(sql)), code) + if err != nil { + return nil, err + } + + c, _ := mod.Memory().ReadUint32Le(code) + if c == uint32(_MEMORY) { + panic(util.OOMErr) + } + if c != uint32(_NONE) { + return nil, ecode(c) + } + if r[0] == 0 { + return nil, nil + } + return &Table{ + sql: sql, + mod: mod, + ptr: uint32(r[0]), + }, nil +} + +// Close closes a table handle. +func (t *Table) Close() error { + mod := t.mod + t.mod = nil + return mod.Close(ctx) +} + +// NumColumns returns the number of columns of the table. +func (t *Table) NumColumns() int { + r, err := t.mod.ExportedFunction("sql3table_num_columns").Call(ctx, uint64(t.ptr)) + if err != nil { + panic(err) + } + return int(int32(r[0])) +} + +// Column returns data for the ith column of the table. +// +// https://sqlite.org/lang_createtable.html#column_definitions +func (t *Table) Column(i int) Column { + r, err := t.mod.ExportedFunction("sql3table_get_column").Call(ctx, uint64(t.ptr), uint64(i)) + if err != nil { + panic(err) + } + return Column{ + tab: t, + ptr: uint32(r[0]), + } +} + +// Column holds metadata about a column. +type Column struct { + tab *Table + ptr uint32 +} + +// Type returns the declared type of a column. +// +// https://sqlite.org/lang_createtable.html#column_data_types +func (c Column) Type() string { + r, err := c.tab.mod.ExportedFunction("sql3column_type").Call(ctx, uint64(c.ptr)) + if err != nil { + panic(err) + } + if r[0] == 0 { + return "" + } + off, _ := c.tab.mod.Memory().ReadUint32Le(uint32(r[0]) + 0) + len, _ := c.tab.mod.Memory().ReadUint32Le(uint32(r[0]) + 4) + return c.tab.sql[off-base : off+len-base] +} + +type ecode uint32 + +const ( + _NONE ecode = iota + _MEMORY + _SYNTAX + _UNSUPPORTEDSQL +) + +func (e ecode) Error() string { + switch e { + case _SYNTAX: + return "sql3parse: invalid syntax" + case _UNSUPPORTEDSQL: + return "sql3parse: unsupported SQL" + default: + panic(util.AssertErr()) + } +} diff --git a/util/vtabutil/parse/.gitignore b/util/vtabutil/parse/.gitignore new file mode 100644 index 0000000..2b0defe --- /dev/null +++ b/util/vtabutil/parse/.gitignore @@ -0,0 +1,2 @@ +sql3parse_table.c +sql3parse_table.h \ No newline at end of file diff --git a/util/vtabutil/parse/build.sh b/util/vtabutil/parse/build.sh new file mode 100755 index 0000000..a21b01f --- /dev/null +++ b/util/vtabutil/parse/build.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd -P -- "$(dirname -- "$0")" + +ROOT=../../../ +BINARYEN="$ROOT/tools/binaryen-version_117/bin" +WASI_SDK="$ROOT/tools/wasi-sdk-22.0/bin" + +"$WASI_SDK/clang" --target=wasm32-wasi -std=c17 -flto -g0 -Oz \ + -Wall -Wextra -Wno-unused-parameter -Wno-unused-function \ + -o sql3parse_table.wasm sql3parse_table.c \ + -mexec-model=reactor \ + -msimd128 -mmutable-globals -mmultivalue \ + -mbulk-memory -mreference-types \ + -mnontrapping-fptoint -msign-ext \ + -fno-stack-protector -fno-stack-clash-protection \ + -Wl,--stack-first \ + -Wl,--import-undefined \ + $(awk '{print "-Wl,--export="$0}' exports.txt) + +trap 'rm -f sql3parse_table.tmp' EXIT +"$BINARYEN/wasm-ctor-eval" -g -c _initialize sql3parse_table.wasm -o sql3parse_table.tmp +"$BINARYEN/wasm-opt" -g --strip --strip-producers -c -Oz \ + sql3parse_table.tmp -o sql3parse_table.wasm \ + --enable-simd --enable-mutable-globals --enable-multivalue \ + --enable-bulk-memory --enable-reference-types \ + --enable-nontrapping-float-to-int --enable-sign-ext \ No newline at end of file diff --git a/util/vtabutil/parse/download.sh b/util/vtabutil/parse/download.sh new file mode 100755 index 0000000..4ef99a7 --- /dev/null +++ b/util/vtabutil/parse/download.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd -P -- "$(dirname -- "$0")" + +curl -#OL "https://github.com/ncruces/sqlite-createtable-parser/raw/master/sql3parse_table.c" +curl -#OL "https://github.com/ncruces/sqlite-createtable-parser/raw/master/sql3parse_table.h" \ No newline at end of file diff --git a/util/vtabutil/parse/exports.txt b/util/vtabutil/parse/exports.txt new file mode 100644 index 0000000..6bc38bc --- /dev/null +++ b/util/vtabutil/parse/exports.txt @@ -0,0 +1,4 @@ +sql3parse_table +sql3table_get_column +sql3table_num_columns +sql3column_type \ No newline at end of file diff --git a/util/vtabutil/parse/sql3parse_table.wasm b/util/vtabutil/parse/sql3parse_table.wasm new file mode 100755 index 0000000..bf1efd6 Binary files /dev/null and b/util/vtabutil/parse/sql3parse_table.wasm differ diff --git a/util/vtabutil/vtabutil.go b/util/vtabutil/vtabutil.go new file mode 100644 index 0000000..eb79778 --- /dev/null +++ b/util/vtabutil/vtabutil.go @@ -0,0 +1,2 @@ +// Package vtabutil implements virtual table utility functions. +package vtabutil