From c99fbcea6f45bf3bad04aeaf2897b60018f6883e Mon Sep 17 00:00:00 2001 From: Nuno Cruces Date: Mon, 11 Dec 2023 14:48:15 +0000 Subject: [PATCH] Towards fileio extension. --- ext/blob/blob.go | 3 +- ext/fileio/fileio.go | 37 +++++++ ext/fileio/fileio_test.go | 46 +++++++++ ext/fileio/fsdir.go | 198 ++++++++++++++++++++++++++++++++++++++ ext/fileio/fsdir_test.go | 47 +++++++++ ext/fileio/write.go | 130 +++++++++++++++++++++++++ ext/stats/stats_test.go | 7 +- ext/todo.txt | 14 +++ 8 files changed, 477 insertions(+), 5 deletions(-) create mode 100644 ext/fileio/fileio.go create mode 100644 ext/fileio/fileio_test.go create mode 100644 ext/fileio/fsdir.go create mode 100644 ext/fileio/fsdir_test.go create mode 100644 ext/fileio/write.go create mode 100644 ext/todo.txt diff --git a/ext/blob/blob.go b/ext/blob/blob.go index 82bd78d..a19916e 100644 --- a/ext/blob/blob.go +++ b/ext/blob/blob.go @@ -18,8 +18,7 @@ import ( // // https://sqlite.org/c3ref/blob.html func Register(db *sqlite3.Conn) { - db.CreateFunction("blob_open", -1, - sqlite3.DETERMINISTIC|sqlite3.DIRECTONLY, openBlob) + db.CreateFunction("blob_open", -1, sqlite3.DIRECTONLY, openBlob) } func openBlob(ctx sqlite3.Context, arg ...sqlite3.Value) { diff --git a/ext/fileio/fileio.go b/ext/fileio/fileio.go new file mode 100644 index 0000000..652d22c --- /dev/null +++ b/ext/fileio/fileio.go @@ -0,0 +1,37 @@ +package fileio + +import ( + "errors" + "fmt" + "io/fs" + "os" + + "github.com/ncruces/go-sqlite3" +) + +func Register(db *sqlite3.Conn) { + db.CreateFunction("lsmode", 1, 0, lsmode) + db.CreateFunction("readfile", 1, sqlite3.DIRECTONLY, readfile) + db.CreateFunction("writefile", -1, sqlite3.DIRECTONLY, writefile) + sqlite3.CreateModule(db, "fsdir", nil, func(db *sqlite3.Conn, module, schema, table string, arg ...string) (fsdir, error) { + err := db.DeclareVtab(`CREATE TABLE x(name,mode,mtime,data,path HIDDEN,dir HIDDEN)`) + db.VtabConfig(sqlite3.VTAB_DIRECTONLY) + return fsdir{}, err + }) +} + +func lsmode(ctx sqlite3.Context, arg ...sqlite3.Value) { + ctx.ResultText(fs.FileMode(arg[0].Int()).String()) +} + +func readfile(ctx sqlite3.Context, arg ...sqlite3.Value) { + d, err := os.ReadFile(arg[0].Text()) + if errors.Is(err, fs.ErrNotExist) { + return + } + if err != nil { + ctx.ResultError(fmt.Errorf("readfile: %w", err)) + } else { + ctx.ResultBlob(d) + } +} diff --git a/ext/fileio/fileio_test.go b/ext/fileio/fileio_test.go new file mode 100644 index 0000000..9b50049 --- /dev/null +++ b/ext/fileio/fileio_test.go @@ -0,0 +1,46 @@ +package fileio_test + +import ( + "os" + "testing" + + "github.com/ncruces/go-sqlite3" + "github.com/ncruces/go-sqlite3/driver" + _ "github.com/ncruces/go-sqlite3/embed" + "github.com/ncruces/go-sqlite3/ext/fileio" +) + +func Test_lsmode(t *testing.T) { + t.Parallel() + + db, err := driver.Open(":memory:", func(c *sqlite3.Conn) error { + fileio.Register(c) + return nil + }) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + d, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + s, err := os.Stat(d) + if err != nil { + t.Fatal(err) + } + + var mode string + err = db.QueryRow(`SELECT lsmode(?)`, s.Mode()).Scan(&mode) + if err != nil { + t.Fatal(err) + } + + if len(mode) != 10 || mode[0] != 'd' { + t.Errorf("got %s", mode) + } else { + t.Logf("got %s", mode) + } +} diff --git a/ext/fileio/fsdir.go b/ext/fileio/fsdir.go new file mode 100644 index 0000000..e4e331b --- /dev/null +++ b/ext/fileio/fsdir.go @@ -0,0 +1,198 @@ +package fileio + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/ncruces/go-sqlite3" +) + +type fsdir struct{ fs.FS } + +func (d fsdir) BestIndex(idx *sqlite3.IndexInfo) error { + var path, dir bool + for i, cst := range idx.Constraint { + switch cst.Column { + case 4: // path + if !cst.Usable || cst.Op != sqlite3.INDEX_CONSTRAINT_EQ { + return sqlite3.CONSTRAINT + } + idx.ConstraintUsage[i] = sqlite3.IndexConstraintUsage{ + Omit: true, + ArgvIndex: 1, + } + path = true + case 5: // dir + if !cst.Usable || cst.Op != sqlite3.INDEX_CONSTRAINT_EQ { + return sqlite3.CONSTRAINT + } + idx.ConstraintUsage[i] = sqlite3.IndexConstraintUsage{ + Omit: true, + ArgvIndex: 2, + } + dir = true + } + } + if path { + idx.EstimatedCost = 100 + } + if dir { + idx.EstimatedCost = 10 + } + return nil +} + +func (d fsdir) Open() (sqlite3.VTabCursor, error) { + return &cursor{fs: d.FS}, nil +} + +type cursor struct { + fs fs.FS + dir string + rowID int64 + eof bool + curr entry + next chan entry + done chan struct{} +} + +type entry struct { + path string + entry fs.DirEntry + err error +} + +func (c *cursor) Close() error { + if c.done != nil { + close(c.done) + s := <-c.next + c.done = nil + c.next = nil + return s.err + } + return nil +} + +func (c *cursor) Filter(idxNum int, idxStr string, arg ...sqlite3.Value) error { + if err := c.Close(); err != nil { + return err + } + if len(arg) == 0 { + return fmt.Errorf("fsdir: wrong number of arguments") + } + + path := arg[0].Text() + if len(arg) > 1 { + if dir := arg[1].RawText(); c.fs != nil { + c.dir = string(dir) + "/" + } else { + c.dir = string(dir) + string(filepath.Separator) + } + path = c.dir + path + } + + c.rowID = 0 + c.eof = false + c.next = make(chan entry) + c.done = make(chan struct{}) + go c.WalkDir(path) + return c.Next() +} + +func (c *cursor) Next() error { + curr, ok := <-c.next + c.curr = curr + c.eof = !ok + c.rowID++ + return c.curr.err +} + +func (c *cursor) EOF() bool { + return c.eof +} + +func (c *cursor) RowID() (int64, error) { + return c.rowID, nil +} + +func (c *cursor) Column(ctx *sqlite3.Context, n int) error { + switch n { + case 0: // name + name := strings.TrimPrefix(c.curr.path, c.dir) + ctx.ResultText(name) + + case 1: // mode + i, err := c.curr.entry.Info() + if err != nil { + return err + } + ctx.ResultInt64(int64(i.Mode())) + + case 2: // mtime + i, err := c.curr.entry.Info() + if err != nil { + return err + } + ctx.ResultTime(i.ModTime(), sqlite3.TimeFormatUnixFrac) + + case 3: // data + switch typ := c.curr.entry.Type(); { + case typ.IsRegular(): + var data []byte + var err error + if name := c.curr.entry.Name(); c.fs != nil { + data, err = fs.ReadFile(c.fs, name) + } else { + data, err = os.ReadFile(name) + } + if err != nil { + return err + } + ctx.ResultBlob(data) + + case typ&fs.ModeSymlink != 0 && c.fs == nil: + t, err := os.Readlink(c.curr.entry.Name()) + if err != nil { + return err + } + ctx.ResultText(t) + } + } + return nil +} + +func (c *cursor) WalkDir(path string) { + var err error + + defer func() { + if p := recover(); p != nil { + if perr, ok := p.(error); ok { + err = fmt.Errorf("panic: %w", perr) + } else { + err = fmt.Errorf("panic: %v", p) + } + } + if err != nil { + c.next <- entry{err: err} + } + close(c.next) + }() + + if c.fs != nil { + err = fs.WalkDir(c.fs, path, c.WalkDirFunc) + } else { + err = filepath.WalkDir(path, c.WalkDirFunc) + } +} + +func (c *cursor) WalkDirFunc(path string, de fs.DirEntry, err error) error { + select { + case <-c.done: + return fs.SkipAll + case c.next <- entry{path, de, err}: + return nil + } +} diff --git a/ext/fileio/fsdir_test.go b/ext/fileio/fsdir_test.go new file mode 100644 index 0000000..5e37bf7 --- /dev/null +++ b/ext/fileio/fsdir_test.go @@ -0,0 +1,47 @@ +package fileio_test + +import ( + "io/fs" + "testing" + "time" + + "github.com/ncruces/go-sqlite3" + "github.com/ncruces/go-sqlite3/driver" + _ "github.com/ncruces/go-sqlite3/embed" + "github.com/ncruces/go-sqlite3/ext/fileio" +) + +func Test_fsdir(t *testing.T) { + t.Parallel() + + db, err := driver.Open(":memory:", func(c *sqlite3.Conn) error { + fileio.Register(c) + return nil + }) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + rows, err := db.Query(`SELECT name, mode, mtime FROM fsdir('.')`) + if err != nil { + t.Fatal(err) + } + + for rows.Next() { + var name string + var mode fs.FileMode + var mtime time.Time + err := rows.Scan(&name, &mode, sqlite3.TimeFormatUnixFrac.Scanner(&mtime)) + if err != nil { + t.Fatal(err) + } + if mode.Perm() == 0 { + t.Errorf("mode %v", mode) + } + if mtime.Before(time.Unix(0, 0)) { + t.Errorf("mtime %v", mtime) + } + t.Log(name) + } +} diff --git a/ext/fileio/write.go b/ext/fileio/write.go new file mode 100644 index 0000000..3aaae6a --- /dev/null +++ b/ext/fileio/write.go @@ -0,0 +1,130 @@ +package fileio + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "time" + + "github.com/ncruces/go-sqlite3" + "github.com/ncruces/go-sqlite3/internal/util" +) + +func writefile(ctx sqlite3.Context, arg ...sqlite3.Value) { + if len(arg) < 2 || len(arg) > 4 { + ctx.ResultError(util.ErrorString("writefile: wrong number of arguments")) + return + } + + file := arg[0].Text() + + var mode fs.FileMode + if len(arg) > 2 { + mode = fixMode(fs.FileMode(arg[2].Int())) + } + + n, err := createFileAndDir(file, mode, arg[1]) + if err != nil { + if len(arg) > 2 { + ctx.ResultError(fmt.Errorf("writefile: %w", err)) + } + return + } + + if mode&fs.ModeSymlink == 0 { + if len(arg) > 2 { + err := os.Chmod(file, mode.Perm()) + if err != nil { + ctx.ResultError(fmt.Errorf("writefile: %w", err)) + return + } + } + + if len(arg) > 3 { + mtime := arg[3].Time(sqlite3.TimeFormatUnixFrac) + err := os.Chtimes(file, time.Time{}, mtime) + if err != nil { + ctx.ResultError(fmt.Errorf("writefile: %w", err)) + return + } + } + } + + if mode.IsRegular() { + ctx.ResultInt(n) + } +} + +func createFileAndDir(path string, mode fs.FileMode, data sqlite3.Value) (int, error) { + n, err := createFile(path, mode, data) + if errors.Is(err, fs.ErrNotExist) { + if err := os.MkdirAll(filepath.Dir(path), 0777); err == nil { + return createFile(path, mode, data) + } + } + return n, err +} + +func createFile(path string, mode fs.FileMode, data sqlite3.Value) (int, error) { + if mode.IsRegular() { + blob := data.RawBlob() + return len(blob), os.WriteFile(path, blob, fixPerm(mode, 0666)) + } + if mode.IsDir() { + err := os.Mkdir(path, fixPerm(mode, 0777)) + if errors.Is(err, fs.ErrExist) { + s, err := os.Lstat(path) + if err == nil && s.IsDir() { + return 0, nil + } + } + return 0, err + } + if mode&fs.ModeSymlink != 0 { + return 0, os.Symlink(data.Text(), path) + } + return 0, fmt.Errorf("invalid mode: %v", mode) +} + +func fixMode(mode fs.FileMode) fs.FileMode { + const ( + S_IFMT fs.FileMode = 0170000 + S_IFIFO fs.FileMode = 0010000 + S_IFCHR fs.FileMode = 0020000 + S_IFDIR fs.FileMode = 0040000 + S_IFBLK fs.FileMode = 0060000 + S_IFREG fs.FileMode = 0100000 + S_IFLNK fs.FileMode = 0120000 + S_IFSOCK fs.FileMode = 0140000 + ) + + switch mode & S_IFMT { + case S_IFDIR: + mode |= fs.ModeDir + case S_IFLNK: + mode |= fs.ModeSymlink + case S_IFBLK: + mode |= fs.ModeDevice + case S_IFCHR: + mode |= fs.ModeCharDevice | fs.ModeDevice + case S_IFIFO: + mode |= fs.ModeNamedPipe + case S_IFSOCK: + mode |= fs.ModeSocket + case S_IFREG, 0: + // + default: + mode |= fs.ModeIrregular + } + + return mode &^ S_IFMT +} + +func fixPerm(mode fs.FileMode, def fs.FileMode) fs.FileMode { + if mode.Perm() == 0 { + return def + } + return mode.Perm() +} diff --git a/ext/stats/stats_test.go b/ext/stats/stats_test.go index cdd9a39..5af1d84 100644 --- a/ext/stats/stats_test.go +++ b/ext/stats/stats_test.go @@ -1,4 +1,4 @@ -package stats +package stats_test import ( "math" @@ -6,6 +6,7 @@ import ( "github.com/ncruces/go-sqlite3" _ "github.com/ncruces/go-sqlite3/embed" + "github.com/ncruces/go-sqlite3/ext/stats" ) func TestRegister_variance(t *testing.T) { @@ -17,7 +18,7 @@ func TestRegister_variance(t *testing.T) { } defer db.Close() - Register(db) + stats.Register(db) err = db.Exec(`CREATE TABLE IF NOT EXISTS data (x)`) if err != nil { @@ -89,7 +90,7 @@ func TestRegister_covariance(t *testing.T) { } defer db.Close() - Register(db) + stats.Register(db) err = db.Exec(`CREATE TABLE IF NOT EXISTS data (x, y)`) if err != nil { diff --git a/ext/todo.txt b/ext/todo.txt new file mode 100644 index 0000000..9ca5ae1 --- /dev/null +++ b/ext/todo.txt @@ -0,0 +1,14 @@ +sha3 +sha3_query + +ieee754 +ieee754_exponent +ieee754_from_blob +ieee754_inc +ieee754_mantissa +ieee754_to_blob + +zipfile +zipfile_cds +sqlar_compress +sqlar_uncompress \ No newline at end of file