From a33b828e134740455d8dfad71347dcb92cb49b38 Mon Sep 17 00:00:00 2001 From: Nuno Cruces Date: Tue, 30 May 2023 11:21:14 +0100 Subject: [PATCH] Examples, tests, max size. --- sqlite3vfs/const.go | 1 + sqlite3vfs/example_test.go | 134 ++++++++++++++++++++++++++++++ sqlite3vfs/memory.go | 31 +++++++ sqlite3vfs/reader_test.go | 141 +++----------------------------- sqlite3vfs/vfs.go | 6 +- tests/db_test.go | 5 +- tests/parallel/parallel_test.go | 1 + tests/testdata/test.db | Bin 0 -> 1024 bytes tests/vfs_test.go | 94 +++++++++++++++++++++ 9 files changed, 279 insertions(+), 134 deletions(-) create mode 100644 sqlite3vfs/example_test.go create mode 100644 tests/testdata/test.db create mode 100644 tests/vfs_test.go diff --git a/sqlite3vfs/const.go b/sqlite3vfs/const.go index 756be70..02cab90 100644 --- a/sqlite3vfs/const.go +++ b/sqlite3vfs/const.go @@ -22,6 +22,7 @@ const ( _READONLY _ErrorCode = util.READONLY _IOERR _ErrorCode = util.IOERR _NOTFOUND _ErrorCode = util.NOTFOUND + _FULL _ErrorCode = util.FULL _CANTOPEN _ErrorCode = util.CANTOPEN _IOERR_READ _ErrorCode = util.IOERR_READ _IOERR_SHORT_READ _ErrorCode = util.IOERR_SHORT_READ diff --git a/sqlite3vfs/example_test.go b/sqlite3vfs/example_test.go new file mode 100644 index 0000000..3c9efec --- /dev/null +++ b/sqlite3vfs/example_test.go @@ -0,0 +1,134 @@ +package sqlite3vfs_test + +import ( + "bytes" + "database/sql" + "fmt" + "log" + + _ "embed" + + _ "github.com/ncruces/go-sqlite3/driver" + _ "github.com/ncruces/go-sqlite3/embed" + "github.com/ncruces/go-sqlite3/sqlite3vfs" + "github.com/psanford/httpreadat" +) + +//go:embed testdata/test.db +var testDB []byte + +func ExampleMemoryVFS_embed() { + sqlite3vfs.Register("memory", sqlite3vfs.MemoryVFS{ + "test.db": sqlite3vfs.NewMemoryDB(testDB), + }) + + db, err := sql.Open("sqlite3", "file:test.db?vfs=memory&_pragma=journal_mode(memory)") + if err != nil { + log.Fatal(err) + } + defer db.Close() + + _, err = db.Exec(`INSERT INTO users (id, name) VALUES (3, 'rust')`) + if err != nil { + log.Fatal(err) + } + + rows, err := db.Query(`SELECT id, name FROM users`) + if err != nil { + log.Fatal(err) + } + defer rows.Close() + + for rows.Next() { + var id, name string + err = rows.Scan(&id, &name) + if err != nil { + log.Fatal(err) + } + fmt.Printf("%s %s\n", id, name) + } + // Output: + // 0 go + // 1 zig + // 2 whatever + // 3 rust +} + +func ExampleReaderVFS_http() { + sqlite3vfs.Register("httpvfs", sqlite3vfs.ReaderVFS{ + "demo.db": httpreadat.New("https://www.sanford.io/demo.db"), + }) + + db, err := sql.Open("sqlite3", "file:demo.db?vfs=httpvfs&mode=ro") + if err != nil { + log.Fatal(err) + } + defer db.Close() + + magname := map[int]string{ + 3: "thousand", + 6: "million", + 9: "billion", + } + rows, err := db.Query(` + SELECT period, data_value, magntude, units FROM csv + WHERE period > '2010' + LIMIT 10`) + if err != nil { + log.Fatal(err) + } + defer rows.Close() + + for rows.Next() { + var period, units string + var value int64 + var mag int + err = rows.Scan(&period, &value, &mag, &units) + if err != nil { + log.Fatal(err) + } + fmt.Printf("%s: %d %s %s\n", period, value, magname[mag], units) + } + // Output: + // 2010.03: 17463 million Dollars + // 2010.06: 17260 million Dollars + // 2010.09: 15419 million Dollars + // 2010.12: 17088 million Dollars + // 2011.03: 18516 million Dollars + // 2011.06: 18835 million Dollars + // 2011.09: 16390 million Dollars + // 2011.12: 18748 million Dollars + // 2012.03: 18477 million Dollars + // 2012.06: 18270 million Dollars +} + +func ExampleReaderVFS_embed() { + sqlite3vfs.Register("reader", sqlite3vfs.ReaderVFS{ + "test.db": sqlite3vfs.NewSizeReaderAt(bytes.NewReader(testDB)), + }) + + db, err := sql.Open("sqlite3", "file:test.db?vfs=reader&mode=ro") + if err != nil { + log.Fatal(err) + } + defer db.Close() + + rows, err := db.Query(`SELECT id, name FROM users`) + if err != nil { + log.Fatal(err) + } + defer rows.Close() + + for rows.Next() { + var id, name string + err = rows.Scan(&id, &name) + if err != nil { + log.Fatal(err) + } + fmt.Printf("%s %s\n", id, name) + } + // Output: + // 0 go + // 1 zig + // 2 whatever +} diff --git a/sqlite3vfs/memory.go b/sqlite3vfs/memory.go index fcc50b6..3f7b745 100644 --- a/sqlite3vfs/memory.go +++ b/sqlite3vfs/memory.go @@ -46,7 +46,10 @@ const memSectorSize = 65536 // A MemoryDB is a [MemoryVFS] database. // // A MemoryDB is safe to access concurrently from multiple SQLite connections. +// It requires journal mode MEMORY or OFF. type MemoryDB struct { + MaxSize int64 + mtx sync.RWMutex size int64 data []*[memSectorSize]byte @@ -57,6 +60,27 @@ type MemoryDB struct { shared int } +// NewMemoryDB creates a new MemoryDB using buf as its initial contents. +// The new MemoryDB takes ownership of buf, and the caller should not use buf after this call. +func NewMemoryDB(buf []byte) *MemoryDB { + m := new(MemoryDB) + m.size = int64(len(buf)) + + sectors := divRoundUp(m.size, memSectorSize) + m.data = make([]*[memSectorSize]byte, sectors) + for i := range m.data { + sector := buf[i*memSectorSize:] + if len(sector) >= memSectorSize { + m.data[i] = (*[memSectorSize]byte)(sector) + } else { + m.data[i] = new([memSectorSize]byte) + copy((*m.data[i])[:], sector) + } + } + + return m +} + type memoryFile struct { *MemoryDB lock LockLevel @@ -96,6 +120,10 @@ func (m *memoryFile) WriteAt(b []byte, off int64) (n int, err error) { m.mtx.Lock() defer m.mtx.Unlock() + if m.MaxSize > 0 && off+int64(len(b)) > m.MaxSize { + return 0, _FULL + } + base := off / memSectorSize rest := off % memSectorSize for base >= int64(len(m.data)) { @@ -117,6 +145,9 @@ func (m *memoryFile) Truncate(size int64) error { } func (m *memoryFile) truncate(size int64) error { + if m.MaxSize > 0 && size > m.MaxSize { + return _FULL + } if size < m.size { base := size / memSectorSize rest := size % memSectorSize diff --git a/sqlite3vfs/reader_test.go b/sqlite3vfs/reader_test.go index 34e856f..c94ce51 100644 --- a/sqlite3vfs/reader_test.go +++ b/sqlite3vfs/reader_test.go @@ -1,130 +1,13 @@ -package sqlite3vfs_test +package sqlite3vfs import ( - "database/sql" - "fmt" "io" - "log" "os" "path/filepath" "strings" "testing" - - _ "embed" - - "github.com/ncruces/go-sqlite3" - _ "github.com/ncruces/go-sqlite3/driver" - _ "github.com/ncruces/go-sqlite3/embed" - "github.com/ncruces/go-sqlite3/sqlite3vfs" - "github.com/psanford/httpreadat" ) -func ExampleReaderVFS() { - sqlite3vfs.Register("httpvfs", sqlite3vfs.ReaderVFS{ - "demo.db": httpreadat.New("https://www.sanford.io/demo.db"), - }) - - db, err := sql.Open("sqlite3", "file:demo.db?vfs=httpvfs&mode=ro") - if err != nil { - log.Fatal(err) - } - defer db.Close() - - magname := map[int]string{ - 3: "thousand", - 6: "million", - 9: "billion", - } - rows, err := db.Query(` - SELECT period, data_value, magntude, units FROM csv - WHERE period > '2010' - LIMIT 10`) - if err != nil { - log.Fatal(err) - } - defer rows.Close() - - for rows.Next() { - var period, units string - var value int64 - var mag int - err = rows.Scan(&period, &value, &mag, &units) - if err != nil { - log.Fatal(err) - } - fmt.Printf("%s: %d %s %s\n", period, value, magname[mag], units) - } - // Output: - // 2010.03: 17463 million Dollars - // 2010.06: 17260 million Dollars - // 2010.09: 15419 million Dollars - // 2010.12: 17088 million Dollars - // 2011.03: 18516 million Dollars - // 2011.06: 18835 million Dollars - // 2011.09: 16390 million Dollars - // 2011.12: 18748 million Dollars - // 2012.03: 18477 million Dollars - // 2012.06: 18270 million Dollars -} - -//go:embed testdata/test.db -var testDB string - -func TestReaderVFS_Open(t *testing.T) { - sqlite3vfs.Register("reader", sqlite3vfs.ReaderVFS{ - "test.db": sqlite3vfs.NewSizeReaderAt(strings.NewReader(testDB)), - }) - - _, err := sqlite3.Open("file:demo.db?vfs=reader&mode=ro") - if err == nil { - t.Error("want error") - } - - db, err := sqlite3.Open("file:test.db?vfs=reader&mode=ro") - if err != nil { - t.Error(err) - } - defer db.Close() - - stmt, _, err := db.Prepare(`SELECT id, name FROM users`) - if err != nil { - t.Fatal(err) - } - defer stmt.Close() - - row := 0 - ids := []int{0, 1, 2} - names := []string{"go", "zig", "whatever"} - for ; stmt.Step(); row++ { - id := stmt.ColumnInt(0) - name := stmt.ColumnText(1) - - if id != ids[row] { - t.Errorf("got %d, want %d", id, ids[row]) - } - if name != names[row] { - t.Errorf("got %q, want %q", name, names[row]) - } - } - if row != 3 { - t.Errorf("got %d, want %d", row, len(ids)) - } - - if err := stmt.Err(); err != nil { - t.Fatal(err) - } - - err = stmt.Close() - if err != nil { - t.Fatal(err) - } - - err = db.Close() - if err != nil { - t.Fatal(err) - } -} - func TestNewSizeReaderAt(t *testing.T) { f, err := os.Create(filepath.Join(t.TempDir(), "abc.txt")) if err != nil { @@ -132,7 +15,7 @@ func TestNewSizeReaderAt(t *testing.T) { } defer f.Close() - n, err := sqlite3vfs.NewSizeReaderAt(f).Size() + n, err := NewSizeReaderAt(f).Size() if err != nil { t.Fatal(err) } @@ -142,7 +25,7 @@ func TestNewSizeReaderAt(t *testing.T) { reader := strings.NewReader("abc") - n, err = sqlite3vfs.NewSizeReaderAt(reader).Size() + n, err = NewSizeReaderAt(reader).Size() if err != nil { t.Fatal(err) } @@ -150,7 +33,7 @@ func TestNewSizeReaderAt(t *testing.T) { t.Errorf("got %d", n) } - n, err = sqlite3vfs.NewSizeReaderAt(lener{reader, reader.Len()}).Size() + n, err = NewSizeReaderAt(readlener{reader, reader.Len()}).Size() if err != nil { t.Fatal(err) } @@ -158,7 +41,7 @@ func TestNewSizeReaderAt(t *testing.T) { t.Errorf("got %d", n) } - n, err = sqlite3vfs.NewSizeReaderAt(sizer{reader, reader.Size()}).Size() + n, err = NewSizeReaderAt(readsizer{reader, reader.Size()}).Size() if err != nil { t.Fatal(err) } @@ -166,7 +49,7 @@ func TestNewSizeReaderAt(t *testing.T) { t.Errorf("got %d", n) } - n, err = sqlite3vfs.NewSizeReaderAt(seeker{reader, reader}).Size() + n, err = NewSizeReaderAt(readseeker{reader, reader}).Size() if err != nil { t.Fatal(err) } @@ -174,27 +57,27 @@ func TestNewSizeReaderAt(t *testing.T) { t.Errorf("got %d", n) } - _, err = sqlite3vfs.NewSizeReaderAt(readerat{reader}).Size() + _, err = NewSizeReaderAt(readerat{reader}).Size() if err == nil { t.Error("want error") } } -type lener struct { +type readlener struct { io.ReaderAt len int } -func (l lener) Len() int { return l.len } +func (l readlener) Len() int { return l.len } -type sizer struct { +type readsizer struct { io.ReaderAt size int64 } -func (l sizer) Size() (int64, error) { return l.size, nil } +func (l readsizer) Size() (int64, error) { return l.size, nil } -type seeker struct { +type readseeker struct { io.ReaderAt io.Seeker } diff --git a/sqlite3vfs/vfs.go b/sqlite3vfs/vfs.go index 85d9fc3..fbd9f3a 100644 --- a/sqlite3vfs/vfs.go +++ b/sqlite3vfs/vfs.go @@ -206,7 +206,7 @@ func vfsOpen(ctx context.Context, mod api.Module, pVfs, zPath, pFile uint32, fla func vfsClose(ctx context.Context, mod api.Module, pFile uint32) _ErrorCode { err := vfsFileClose(ctx, mod, pFile) if err != nil { - return _IOERR_CLOSE + return vfsErrorCode(err, _IOERR_CLOSE) } return _OK } @@ -220,7 +220,7 @@ func vfsRead(ctx context.Context, mod api.Module, pFile, zBuf, iAmt uint32, iOfs return _OK } if n == 0 && err != io.EOF { - return _IOERR_READ + return vfsErrorCode(err, _IOERR_READ) } clear(buf[n:]) return _IOERR_SHORT_READ @@ -232,7 +232,7 @@ func vfsWrite(ctx context.Context, mod api.Module, pFile, zBuf, iAmt uint32, iOf _, err := file.WriteAt(buf, iOfst) if err != nil { - return _IOERR_WRITE + return vfsErrorCode(err, _IOERR_WRITE) } return _OK } diff --git a/tests/db_test.go b/tests/db_test.go index a511c94..f5bcb95 100644 --- a/tests/db_test.go +++ b/tests/db_test.go @@ -10,10 +10,12 @@ import ( ) func TestDB_memory(t *testing.T) { + t.Parallel() testDB(t, ":memory:") } func TestDB_file(t *testing.T) { + t.Parallel() testDB(t, filepath.Join(t.TempDir(), "test.db")) } @@ -21,12 +23,11 @@ func TestDB_VFS(t *testing.T) { sqlite3vfs.Register("memvfs", sqlite3vfs.MemoryVFS{ "test.db": &sqlite3vfs.MemoryDB{}, }) + defer sqlite3vfs.Unregister("memvfs") testDB(t, "file:test.db?vfs=memvfs&_pragma=journal_mode(memory)") } func testDB(t *testing.T, name string) { - t.Parallel() - db, err := sqlite3.Open(name) if err != nil { t.Fatal(err) diff --git a/tests/parallel/parallel_test.go b/tests/parallel/parallel_test.go index 7ca2cfe..21cfe4b 100644 --- a/tests/parallel/parallel_test.go +++ b/tests/parallel/parallel_test.go @@ -43,6 +43,7 @@ func TestMemory(t *testing.T) { sqlite3vfs.Register("memvfs", sqlite3vfs.MemoryVFS{ "test.db": &sqlite3vfs.MemoryDB{}, }) + defer sqlite3vfs.Unregister("memvfs") name := "file:test.db?vfs=memvfs" + "&_pragma=busy_timeout(10000)" + diff --git a/tests/testdata/test.db b/tests/testdata/test.db new file mode 100644 index 0000000000000000000000000000000000000000..48ea4e753d38dc9d6fa28a2db509ed892712c035 GIT binary patch literal 1024 zcmWFz^vNtqRY=P(%1ta$FlJz4U}WTRP*7lCU|<1aCLo3ZW*`G31_O*R1|FJ)K`&mG z7pQ=daXkYNjRML-z?qRvTwI(ny(BRyC$+RVwWt_OFgXXgI)=C^gg83+xGI2!6f`nZ z6g>SxbQJOub5j+<9D|%a9D_6r4Ky{8odAj(X2$CbjBgpA0;y3jJVJnnnVC_RsXQaG aB(*HHh>eMvQ@AQKot2T9Lohub*^2=AjwOHq literal 0 HcmV?d00001 diff --git a/tests/vfs_test.go b/tests/vfs_test.go new file mode 100644 index 0000000..ef45c80 --- /dev/null +++ b/tests/vfs_test.go @@ -0,0 +1,94 @@ +package tests + +import ( + "errors" + "strings" + "testing" + + _ "embed" + + "github.com/ncruces/go-sqlite3" + _ "github.com/ncruces/go-sqlite3/driver" + _ "github.com/ncruces/go-sqlite3/embed" + "github.com/ncruces/go-sqlite3/sqlite3vfs" +) + +//go:embed testdata/test.db +var testdata string + +func TestMemoryVFS_Open_notfound(t *testing.T) { + sqlite3vfs.Register("memory", sqlite3vfs.MemoryVFS{ + "test.db": &sqlite3vfs.MemoryDB{}, + }) + defer sqlite3vfs.Unregister("memory") + + _, err := sqlite3.Open("file:demo.db?vfs=memory&mode=ro") + if err == nil { + t.Error("want error") + } + if !errors.Is(err, sqlite3.CANTOPEN) { + t.Errorf("got %v, want sqlite3.CANTOPEN", err) + } +} + +func TestMemoryVFS_Open_journal(t *testing.T) { + sqlite3vfs.Register("memory", sqlite3vfs.MemoryVFS{ + "test.db": &sqlite3vfs.MemoryDB{}, + }) + defer sqlite3vfs.Unregister("memory") + + db, err := sqlite3.Open("file:test.db?vfs=memory") + if err != nil { + t.Fatal(err) + } + defer db.Close() + + err = db.Exec(`CREATE TABLE IF NOT EXISTS test (col)`) + if err == nil { + t.Error("want error") + } + if !errors.Is(err, sqlite3.CANTOPEN) { + t.Errorf("got %v, want sqlite3.CANTOPEN", err) + } +} + +func TestMemoryVFS_Open_errors(t *testing.T) { + sqlite3vfs.Register("memory", sqlite3vfs.MemoryVFS{ + "test.db": &sqlite3vfs.MemoryDB{MaxSize: 65536}, + }) + defer sqlite3vfs.Unregister("memory") + + db, err := sqlite3.Open("file:test.db?vfs=memory&_pragma=journal_mode(memory)") + if err != nil { + t.Fatal(err) + } + defer db.Close() + + err = db.Exec(`CREATE TABLE IF NOT EXISTS test (col)`) + if err != nil { + t.Fatal(err) + } + + err = db.Exec(`INSERT INTO test VALUES (zeroblob(65536))`) + if err == nil { + t.Error("want error") + } + if !errors.Is(err, sqlite3.FULL) { + t.Errorf("got %v, want sqlite3.FULL", err) + } +} + +func TestReaderVFS_Open_notfound(t *testing.T) { + sqlite3vfs.Register("reader", sqlite3vfs.ReaderVFS{ + "test.db": sqlite3vfs.NewSizeReaderAt(strings.NewReader(testdata)), + }) + defer sqlite3vfs.Unregister("reader") + + _, err := sqlite3.Open("file:demo.db?vfs=reader&mode=ro") + if err == nil { + t.Error("want error") + } + if !errors.Is(err, sqlite3.CANTOPEN) { + t.Errorf("got %v, want sqlite3.CANTOPEN", err) + } +}