diff --git a/README.md b/README.md index 8f0b869..f20b704 100644 --- a/README.md +++ b/README.md @@ -22,22 +22,22 @@ and uses [wazero](https://wazero.io/) to provide `cgo`-free SQLite bindings. ### Loadable extensions -- [`github.com/ncruces/go-sqlite3/ext/array`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/blob) +- [`github.com/ncruces/go-sqlite3/ext/array`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/array) provides the [`array`](https://sqlite.org/carray.html) table-valued function. -- [`github.com/ncruces/go-sqlite3/ext/blob`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/blob) +- [`github.com/ncruces/go-sqlite3/ext/blobio`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/blobio) simplifies [incremental BLOB I/O](https://sqlite.org/c3ref/blob_open.html). - [`github.com/ncruces/go-sqlite3/ext/csv`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/csv) reads [comma-separated values](https://sqlite.org/csv.html). - [`github.com/ncruces/go-sqlite3/ext/fileio`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/fileio) - reads and writes files. + reads, writes and lists files. - [`github.com/ncruces/go-sqlite3/ext/lines`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/lines) - reads files [line-by-line](https://github.com/asg017/sqlite-lines). + reads data [line-by-line](https://github.com/asg017/sqlite-lines). - [`github.com/ncruces/go-sqlite3/ext/pivot`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/pivot) creates [pivot tables](https://github.com/jakethaw/pivot_vtab). - [`github.com/ncruces/go-sqlite3/ext/statement`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/statement) - creates [table-valued functions with SQL](https://github.com/0x09/sqlite-statement-vtab). + creates [parameterized views](https://github.com/0x09/sqlite-statement-vtab). - [`github.com/ncruces/go-sqlite3/ext/stats`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/stats) - provides [statistics functions](https://www.oreilly.com/library/view/sql-in-a/9780596155322/ch04s02.html). + provides [statistics](https://www.oreilly.com/library/view/sql-in-a/9780596155322/ch04s02.html) functions. - [`github.com/ncruces/go-sqlite3/ext/unicode`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/unicode) provides [Unicode aware](https://sqlite.org/src/dir/ext/icu) functions. - [`github.com/ncruces/go-sqlite3/vfs/memdb`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/memdb) diff --git a/ext/blob/blob.go b/ext/blob/blob.go deleted file mode 100644 index a19916e..0000000 --- a/ext/blob/blob.go +++ /dev/null @@ -1,69 +0,0 @@ -// Package blob provides an alternative interface to incremental BLOB I/O. -package blob - -import ( - "errors" - - "github.com/ncruces/go-sqlite3" - "github.com/ncruces/go-sqlite3/internal/util" -) - -// Register registers the blob_open SQL function: -// -// blob_open(schema, table, column, rowid, flags, callback, args...) -// -// The callback must be an [sqlite3.Pointer] to an [OpenCallback]. -// Any optional args will be passed to the callback, -// along with the [sqlite3.Blob] handle. -// -// https://sqlite.org/c3ref/blob.html -func Register(db *sqlite3.Conn) { - db.CreateFunction("blob_open", -1, sqlite3.DIRECTONLY, openBlob) -} - -func openBlob(ctx sqlite3.Context, arg ...sqlite3.Value) { - if len(arg) < 6 { - ctx.ResultError(util.ErrorString("blob_open: wrong number of arguments")) - return - } - - row := arg[3].Int64() - - var err error - blob, ok := ctx.GetAuxData(0).(*sqlite3.Blob) - if ok { - err = blob.Reopen(row) - if errors.Is(err, sqlite3.MISUSE) { - // Blob was closed (db, table, column or write changed). - ok = false - } - } - - if !ok { - db := arg[0].Text() - table := arg[1].Text() - column := arg[2].Text() - write := arg[4].Bool() - blob, err = ctx.Conn().OpenBlob(db, table, column, row, write) - } - if err != nil { - ctx.ResultError(err) - return - } - - fn := arg[5].Pointer().(OpenCallback) - err = fn(blob, arg[6:]...) - if err != nil { - ctx.ResultError(err) - return - } - - // This ensures the blob is closed if db, table, column or write change. - ctx.SetAuxData(0, blob) // db - ctx.SetAuxData(1, blob) // table - ctx.SetAuxData(2, blob) // column - ctx.SetAuxData(4, blob) // write -} - -// OpenCallback is the type for the blob_open callback. -type OpenCallback func(*sqlite3.Blob, ...sqlite3.Value) error diff --git a/ext/blobio/blob.go b/ext/blobio/blob.go new file mode 100644 index 0000000..1a06d2d --- /dev/null +++ b/ext/blobio/blob.go @@ -0,0 +1,138 @@ +// Package blobio provides an SQL interface to incremental BLOB I/O. +package blobio + +import ( + "errors" + "io" + + "github.com/ncruces/go-sqlite3" + "github.com/ncruces/go-sqlite3/internal/util" +) + +// Register registers the SQL functions: +// +// readblob(schema, table, column, rowid, offset, n) +// +// Reads n bytes of a blob, starting at offset. +// +// writeblob(schema, table, column, rowid, offset, data) +// +// Writes data into a blob, at the given offset. +// +// openblob(schema, table, column, rowid, write, callback, args...) +// +// Opens blobs for reading or writing. +// The callback is invoked for each open blob, +// and must be an [sqlite3.Pointer] to an [OpenCallback]. +// The optional args will be passed to the callback, +// along with the [sqlite3.Blob] handle. +// +// https://sqlite.org/c3ref/blob.html +func Register(db *sqlite3.Conn) { + db.CreateFunction("readblob", 6, sqlite3.DIRECTONLY, readblob) + db.CreateFunction("writeblob", 6, sqlite3.DIRECTONLY, writeblob) + db.CreateFunction("openblob", -1, sqlite3.DIRECTONLY, openblob) +} + +// OpenCallback is the type for the openblob callback. +type OpenCallback func(*sqlite3.Blob, ...sqlite3.Value) error + +func readblob(ctx sqlite3.Context, arg ...sqlite3.Value) { + blob, err := getAuxBlob(ctx, arg, false) + if err != nil { + ctx.ResultError(err) + return + } + + _, err = blob.Seek(arg[4].Int64(), io.SeekStart) + if err != nil { + ctx.ResultError(err) + return + } + + n := arg[5].Int64() + if n <= 0 { + return + } + buf := make([]byte, n) + + _, err = io.ReadFull(blob, buf) + if err != nil { + ctx.ResultError(err) + return + } + + ctx.ResultBlob(buf) + setAuxBlob(ctx, blob, false) +} + +func writeblob(ctx sqlite3.Context, arg ...sqlite3.Value) { + blob, err := getAuxBlob(ctx, arg, true) + if err != nil { + ctx.ResultError(err) + return + } + + _, err = blob.Seek(arg[4].Int64(), io.SeekStart) + if err != nil { + ctx.ResultError(err) + return + } + + _, err = blob.Write(arg[5].RawBlob()) + if err != nil { + ctx.ResultError(err) + return + } + + setAuxBlob(ctx, blob, false) +} + +func openblob(ctx sqlite3.Context, arg ...sqlite3.Value) { + if len(arg) < 6 { + ctx.ResultError(util.ErrorString("openblob: wrong number of arguments")) + return + } + + blob, err := getAuxBlob(ctx, arg, arg[4].Bool()) + if err != nil { + ctx.ResultError(err) + return + } + + fn := arg[5].Pointer().(OpenCallback) + err = fn(blob, arg[6:]...) + if err != nil { + ctx.ResultError(err) + return + } + + setAuxBlob(ctx, blob, true) +} + +func getAuxBlob(ctx sqlite3.Context, arg []sqlite3.Value, write bool) (*sqlite3.Blob, error) { + row := arg[3].Int64() + + if blob, ok := ctx.GetAuxData(0).(*sqlite3.Blob); ok { + if err := blob.Reopen(row); errors.Is(err, sqlite3.MISUSE) { + // Blob was closed (db, table, column or write changed). + } else { + return blob, err + } + } + + db := arg[0].Text() + table := arg[1].Text() + column := arg[2].Text() + return ctx.Conn().OpenBlob(db, table, column, row, write) +} + +func setAuxBlob(ctx sqlite3.Context, blob *sqlite3.Blob, writer bool) { + // This ensures the blob is closed if db, table, column or write change. + ctx.SetAuxData(0, blob) // db + ctx.SetAuxData(1, blob) // table + ctx.SetAuxData(2, blob) // column + if writer { + ctx.SetAuxData(4, blob) // write + } +} diff --git a/ext/blob/blob_test.go b/ext/blobio/blob_test.go similarity index 52% rename from ext/blob/blob_test.go rename to ext/blobio/blob_test.go index 68fdcc5..73edd4c 100644 --- a/ext/blob/blob_test.go +++ b/ext/blobio/blob_test.go @@ -1,4 +1,4 @@ -package blob_test +package blobio_test import ( "io" @@ -11,14 +11,14 @@ import ( "github.com/ncruces/go-sqlite3/driver" _ "github.com/ncruces/go-sqlite3/embed" "github.com/ncruces/go-sqlite3/ext/array" - "github.com/ncruces/go-sqlite3/ext/blob" + "github.com/ncruces/go-sqlite3/ext/blobio" _ "github.com/ncruces/go-sqlite3/vfs/memdb" ) func Example() { // Open the database, registering the extension. db, err := driver.Open("file:/test.db?vfs=memdb", func(conn *sqlite3.Conn) error { - blob.Register(conn) + blobio.Register(conn) return nil }) @@ -41,18 +41,14 @@ func Example() { } // Write the BLOB. - _, err = db.Exec(`SELECT blob_open('main', 'test', 'col', last_insert_rowid(), true, ?)`, - sqlite3.Pointer[blob.OpenCallback](func(blob *sqlite3.Blob, _ ...sqlite3.Value) error { - _, err = io.WriteString(blob, message) - return err - })) + _, err = db.Exec(`SELECT writeblob('main', 'test', 'col', last_insert_rowid(), 0, ?)`, message) if err != nil { log.Fatal(err) } // Read the BLOB. - _, err = db.Exec(`SELECT blob_open('main', 'test', 'col', rowid, false, ?) FROM test`, - sqlite3.Pointer[blob.OpenCallback](func(blob *sqlite3.Blob, _ ...sqlite3.Value) error { + _, err = db.Exec(`SELECT openblob('main', 'test', 'col', rowid, false, ?) FROM test`, + sqlite3.Pointer[blobio.OpenCallback](func(blob *sqlite3.Blob, _ ...sqlite3.Value) error { _, err = io.Copy(os.Stdout, blob) return err })) @@ -63,7 +59,7 @@ func Example() { // Hello BLOB! } -func TestRegister(t *testing.T) { +func Test_readblob(t *testing.T) { t.Parallel() db, err := sqlite3.Open(":memory:") @@ -72,10 +68,10 @@ func TestRegister(t *testing.T) { } defer db.Close() - blob.Register(db) + blobio.Register(db) array.Register(db) - err = db.Exec(`SELECT blob_open()`) + err = db.Exec(`SELECT readblob()`) if err == nil { t.Fatal("want error") } else { @@ -92,14 +88,74 @@ func TestRegister(t *testing.T) { t.Fatal(err) } - stmt, _, err := db.Prepare(`SELECT blob_open('main', value, 'col', 1, false, ?) FROM array(?)`) + stmt, _, err := db.Prepare(`SELECT readblob('main', value, 'col', 1, 1, 1) FROM array(?)`) + if err != nil { + t.Fatal(err) + } + defer stmt.Close() + + err = stmt.BindPointer(1, []string{"test1", "test2"}) + if err != nil { + t.Fatal(err) + } + + if stmt.Step() { + got := stmt.ColumnText(0) + if got != "\xfe" { + t.Errorf("got %q", got) + } + } + + if stmt.Step() { + got := stmt.ColumnText(0) + if got != "\xbe" { + t.Errorf("got %q", got) + } + } + + err = stmt.Err() + if err != nil { + t.Fatal(err) + } +} + +func Test_openblob(t *testing.T) { + t.Parallel() + + db, err := sqlite3.Open(":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + + blobio.Register(db) + array.Register(db) + + err = db.Exec(`SELECT openblob()`) + if err == nil { + t.Fatal("want error") + } else { + t.Log(err) + } + + err = db.Exec(` + CREATE TABLE IF NOT EXISTS test1 (col); + CREATE TABLE IF NOT EXISTS test2 (col); + INSERT INTO test1 VALUES (x'cafe'); + INSERT INTO test2 VALUES (x'babe'); + `) + if err != nil { + t.Fatal(err) + } + + stmt, _, err := db.Prepare(`SELECT openblob('main', value, 'col', 1, false, ?) FROM array(?)`) if err != nil { t.Fatal(err) } defer stmt.Close() var got []string - err = stmt.BindPointer(1, blob.OpenCallback(func(b *sqlite3.Blob, _ ...sqlite3.Value) error { + err = stmt.BindPointer(1, blobio.OpenCallback(func(b *sqlite3.Blob, _ ...sqlite3.Value) error { d, err := io.ReadAll(b) if err != nil { return err diff --git a/ext/fileio/fileio.go b/ext/fileio/fileio.go index 78b0330..bb7523a 100644 --- a/ext/fileio/fileio.go +++ b/ext/fileio/fileio.go @@ -1,4 +1,4 @@ -// Package fileio provides SQL functions to read and write files. +// Package fileio provides SQL functions to read, write and list files. // // https://sqlite.org/src/doc/tip/ext/misc/fileio.c package fileio diff --git a/ext/lines/lines.go b/ext/lines/lines.go index 1f7e360..41a8578 100644 --- a/ext/lines/lines.go +++ b/ext/lines/lines.go @@ -1,4 +1,13 @@ -// Package lines provides a virtual table to read large files line-by-line. +// Package lines provides a virtual table to read data line-by-line. +// +// It is particularly useful for line-oriented datasets, +// like [ndjson] or [JSON Lines], +// when paired with SQLite's JSON support. +// +// https://github.com/asg017/sqlite-lines +// +// [ndjson]: https://ndjson.org/ +// [JSON Lines]: https://jsonlines.org/ package lines import ( diff --git a/ext/statement/stmt.go b/ext/statement/stmt.go index 77289c0..7049043 100644 --- a/ext/statement/stmt.go +++ b/ext/statement/stmt.go @@ -1,4 +1,7 @@ -// Package statement defines table-valued functions natively using SQL. +// Package statement defines table-valued functions using SQL. +// +// It can be used to create "parametrized views": +// pre-packaged queries that can be parametrized at query execution time. // // https://github.com/0x09/sqlite-statement-vtab package statement diff --git a/ext/todo.txt b/ext/todo.txt index 9ca5ae1..038ed97 100644 --- a/ext/todo.txt +++ b/ext/todo.txt @@ -11,4 +11,6 @@ ieee754_to_blob zipfile zipfile_cds sqlar_compress -sqlar_uncompress \ No newline at end of file +sqlar_uncompress + +remember \ No newline at end of file