Merge pull request #46 from ucan-wg/rework-policies

selector: rework to match the spec, cleanup lots of edge cases
This commit is contained in:
Michael Muré
2024-10-24 13:01:10 +02:00
committed by GitHub
7 changed files with 1021 additions and 949 deletions

View File

@@ -20,72 +20,47 @@ func (p Policy) Match(node datamodel.Node) bool {
return true
}
// Filter performs a recursive filtering of the Statement, and prunes what doesn't match the given path
func (p Policy) Filter(path ...string) Policy {
var filtered Policy
for _, stmt := range p {
newChild, remain := filter(stmt, path)
if newChild != nil && len(remain) == 0 {
filtered = append(filtered, newChild)
}
}
return filtered
}
func matchStatement(statement Statement, node ipld.Node) bool {
switch statement.Kind() {
case KindEqual:
if s, ok := statement.(equality); ok {
one, many, err := s.selector.Select(node)
res, err := s.selector.Select(node)
if err != nil {
return false
}
if one != nil {
return datamodel.DeepEqual(s.value, one)
}
if many != nil {
for _, n := range many {
if eq := datamodel.DeepEqual(s.value, n); eq {
return true
}
}
}
return false
return datamodel.DeepEqual(s.value, res)
}
case KindGreaterThan:
if s, ok := statement.(equality); ok {
one, _, err := s.selector.Select(node)
if err != nil || one == nil {
res, err := s.selector.Select(node)
if err != nil {
return false
}
return isOrdered(s.value, one, gt)
return isOrdered(s.value, res, gt)
}
case KindGreaterThanOrEqual:
if s, ok := statement.(equality); ok {
one, _, err := s.selector.Select(node)
if err != nil || one == nil {
res, err := s.selector.Select(node)
if err != nil {
return false
}
return isOrdered(s.value, one, gte)
return isOrdered(s.value, res, gte)
}
case KindLessThan:
if s, ok := statement.(equality); ok {
one, _, err := s.selector.Select(node)
if err != nil || one == nil {
res, err := s.selector.Select(node)
if err != nil {
return false
}
return isOrdered(s.value, one, lt)
return isOrdered(s.value, res, lt)
}
case KindLessThanOrEqual:
if s, ok := statement.(equality); ok {
one, _, err := s.selector.Select(node)
if err != nil || one == nil {
res, err := s.selector.Select(node)
if err != nil {
return false
}
return isOrdered(s.value, one, lte)
return isOrdered(s.value, res, lte)
}
case KindNot:
if s, ok := statement.(negation); ok {
@@ -116,24 +91,32 @@ func matchStatement(statement Statement, node ipld.Node) bool {
}
case KindLike:
if s, ok := statement.(wildcard); ok {
one, _, err := s.selector.Select(node)
if err != nil || one == nil {
return false
}
v, err := one.AsString()
res, err := s.selector.Select(node)
if err != nil {
return false
}
v, err := res.AsString()
if err != nil {
return false // not a string
}
return s.pattern.Match(v)
}
case KindAll:
if s, ok := statement.(quantifier); ok {
_, many, err := s.selector.Select(node)
if err != nil || many == nil {
res, err := s.selector.Select(node)
if err != nil {
return false
}
for _, n := range many {
ok := matchStatement(s.statement, n)
it := res.ListIterator()
if it == nil {
return false // not a list
}
for !it.Done() {
_, v, err := it.Next()
if err != nil {
return false
}
ok := matchStatement(s.statement, v)
if !ok {
return false
}
@@ -142,95 +125,30 @@ func matchStatement(statement Statement, node ipld.Node) bool {
}
case KindAny:
if s, ok := statement.(quantifier); ok {
one, many, err := s.selector.Select(node)
res, err := s.selector.Select(node)
if err != nil {
return false
}
if one != nil {
ok := matchStatement(s.statement, one)
it := res.ListIterator()
if it == nil {
return false // not a list
}
for !it.Done() {
_, v, err := it.Next()
if err != nil {
return false
}
ok := matchStatement(s.statement, v)
if ok {
return true
}
}
if many != nil {
for _, n := range many {
ok := matchStatement(s.statement, n)
if ok {
return true
}
}
}
return false
}
}
panic(fmt.Errorf("unimplemented statement kind: %s", statement.Kind()))
}
// filter performs a recursive filtering of the Statement, and prunes what doesn't match the given path
func filter(stmt Statement, path []string) (Statement, []string) {
// For each kind, we do some of the following if it applies:
// - test the path against the selector, consuming segments
// - for terminal statements (equality, wildcard), require all the segments to have been consumed
// - recursively filter child (negation, quantifier) or children (connective) statements with the remaining path
switch stmt.(type) {
case equality:
match, remain := stmt.(equality).selector.MatchPath(path...)
if match && len(remain) == 0 {
return stmt, remain
}
return nil, nil
case negation:
newChild, remain := filter(stmt.(negation).statement, path)
if newChild != nil && len(remain) == 0 {
return negation{
statement: newChild,
}, nil
}
return nil, nil
case connective:
var newChildren []Statement
for _, child := range stmt.(connective).statements {
newChild, remain := filter(child, path)
if newChild != nil && len(remain) == 0 {
newChildren = append(newChildren, newChild)
}
}
if len(newChildren) == 0 {
return nil, nil
}
return connective{
kind: stmt.(connective).kind,
statements: newChildren,
}, nil
case wildcard:
match, remain := stmt.(wildcard).selector.MatchPath(path...)
if match && len(remain) == 0 {
return stmt, remain
}
return nil, nil
case quantifier:
match, remain := stmt.(quantifier).selector.MatchPath(path...)
if match && len(remain) == 0 {
return stmt, remain
}
if !match {
return nil, nil
}
newChild, remain := filter(stmt.(quantifier).statement, remain)
if newChild != nil && len(remain) == 0 {
return quantifier{
kind: stmt.(quantifier).kind,
selector: stmt.(quantifier).selector,
statement: newChild,
}, nil
}
return nil, nil
default:
panic(fmt.Errorf("unimplemented statement kind: %s", stmt.Kind()))
}
}
func isOrdered(expected ipld.Node, actual ipld.Node, satisfies func(order int) bool) bool {
if expected.Kind() == ipld.Kind_Int && actual.Kind() == ipld.Kind_Int {
a := must.Int(actual)

View File

@@ -2,7 +2,6 @@ package policy
import (
"fmt"
"strings"
"testing"
"github.com/ipfs/go-cid"
@@ -513,103 +512,3 @@ func FuzzMatch(f *testing.F) {
policy.Match(dataNode)
})
}
func TestPolicyFilter(t *testing.T) {
pol := MustConstruct(
Any(".http", And(
Equal(".method", literal.String("GET")),
Equal(".path", literal.String("/foo")),
)),
Equal(".http", literal.String("foobar")),
All(".jsonrpc.foo", Or(
Not(Equal(".bar", literal.String("foo"))),
Equal(".", literal.String("foo")),
Like(".boo", "abcd"),
Like(".boo", "*bcd"),
)),
)
for _, tc := range []struct {
path string
expected Policy
}{
{
path: "http",
expected: MustConstruct(
Any(".http", And(
Equal(".method", literal.String("GET")),
Equal(".path", literal.String("/foo")),
)),
Equal(".http", literal.String("foobar")),
),
},
{
path: "http,method",
expected: MustConstruct(
Any(".http", And(
Equal(".method", literal.String("GET")),
)),
),
},
{
path: "http,path",
expected: MustConstruct(
Any(".http", And(
Equal(".path", literal.String("/foo")),
)),
),
},
{
path: "http,foo",
expected: Policy{},
},
{
path: "jsonrpc",
expected: MustConstruct(
All(".jsonrpc.foo", Or(
Not(Equal(".bar", literal.String("foo"))),
Equal(".", literal.String("foo")),
Like(".boo", "abcd"),
Like(".boo", "*bcd"),
)),
),
},
{
path: "jsonrpc,baz",
expected: Policy{},
},
{
path: "jsonrpc,foo",
expected: MustConstruct(
All(".jsonrpc.foo", Or(
Not(Equal(".bar", literal.String("foo"))),
Equal(".", literal.String("foo")),
Like(".boo", "abcd"),
Like(".boo", "*bcd"),
)),
),
},
{
path: "jsonrpc,foo,bar",
expected: MustConstruct(
All(".jsonrpc.foo", Or(
Not(Equal(".bar", literal.String("foo"))),
)),
),
},
{
path: "jsonrpc,foo,boo",
expected: MustConstruct(
All(".jsonrpc.foo", Or(
Like(".boo", "abcd"),
Like(".boo", "*bcd"),
)),
),
},
} {
t.Run(tc.path, func(t *testing.T) {
res := pol.Filter(strings.Split(tc.path, ",")...)
require.Equal(t, tc.expected.String(), res.String())
})
}
}

View File

@@ -2,6 +2,7 @@ package selector
import (
"fmt"
"math"
"regexp"
"strconv"
"strings"
@@ -24,6 +25,9 @@ func Parse(str string) (Selector, error) {
if str == "." {
return identity, nil
}
if str == ".?" {
return Selector{segment{str: ".?", identity: true, optional: true}}, nil
}
col := 0
var sel Selector
@@ -31,56 +35,70 @@ func Parse(str string) (Selector, error) {
seg := tok
opt := strings.HasSuffix(tok, "?")
if opt {
seg = tok[0 : len(tok)-1]
seg = strings.TrimRight(tok, "?")
}
switch seg {
case ".":
switch {
case seg == ".":
if len(sel) > 0 && sel[len(sel)-1].Identity() {
return nil, newParseError("selector contains unsupported recursive descent segment: '..'", str, col, tok)
}
sel = append(sel, segment{str: ".", identity: true})
case "[]":
sel = append(sel, segment{str: tok, optional: opt, iterator: true})
default:
if strings.HasPrefix(seg, "[") && strings.HasSuffix(seg, "]") {
lookup := seg[1 : len(seg)-1]
if indexRegex.MatchString(lookup) { // index
idx, err := strconv.Atoi(lookup)
if err != nil {
return nil, newParseError("invalid index", str, col, tok)
}
sel = append(sel, segment{str: tok, optional: opt, index: idx})
} else if strings.HasPrefix(lookup, "\"") && strings.HasSuffix(lookup, "\"") { // explicit field
sel = append(sel, segment{str: tok, optional: opt, field: lookup[1 : len(lookup)-1]})
} else if sliceRegex.MatchString(lookup) { // slice [3:5] or [:5] or [3:]
var rng []int
splt := strings.Split(lookup, ":")
if splt[0] == "" {
rng = append(rng, 0)
} else {
i, err := strconv.Atoi(splt[0])
if err != nil {
return nil, newParseError("invalid slice index", str, col, tok)
}
rng = append(rng, i)
}
if splt[1] != "" {
i, err := strconv.Atoi(splt[1])
if err != nil {
return nil, newParseError("invalid slice index", str, col, tok)
}
rng = append(rng, i)
}
sel = append(sel, segment{str: tok, optional: opt, slice: rng})
} else {
case seg == "[]":
sel = append(sel, segment{str: tok, optional: opt, iterator: true})
case strings.HasPrefix(seg, "[") && strings.HasSuffix(seg, "]"):
lookup := seg[1 : len(seg)-1]
switch {
// index, [123]
case indexRegex.MatchString(lookup):
idx, err := strconv.Atoi(lookup)
if err != nil {
return nil, newParseError("invalid index", str, col, tok)
}
sel = append(sel, segment{str: tok, optional: opt, index: idx})
// explicit field, ["abcd"]
case strings.HasPrefix(lookup, "\"") && strings.HasSuffix(lookup, "\""):
fieldName := lookup[1 : len(lookup)-1]
if strings.Contains(fieldName, ":") {
return nil, newParseError(fmt.Sprintf("invalid segment: %s", seg), str, col, tok)
}
} else if fieldRegex.MatchString(seg) {
sel = append(sel, segment{str: tok, optional: opt, field: seg[1:]})
} else {
sel = append(sel, segment{str: tok, optional: opt, field: fieldName})
// slice [3:5] or [:5] or [3:], also negative numbers
case sliceRegex.MatchString(lookup):
var rng [2]int64
splt := strings.Split(lookup, ":")
if splt[0] == "" {
rng[0] = math.MinInt
} else {
i, err := strconv.ParseInt(splt[0], 10, 0)
if err != nil {
return nil, newParseError("invalid slice index", str, col, tok)
}
rng[0] = i
}
if splt[1] == "" {
rng[1] = math.MaxInt
} else {
i, err := strconv.ParseInt(splt[1], 10, 0)
if err != nil {
return nil, newParseError("invalid slice index", str, col, tok)
}
rng[1] = i
}
sel = append(sel, segment{str: tok, optional: opt, slice: rng[:]})
default:
return nil, newParseError(fmt.Sprintf("invalid segment: %s", seg), str, col, tok)
}
case fieldRegex.MatchString(seg):
sel = append(sel, segment{str: tok, optional: opt, field: seg[1:]})
default:
return nil, newParseError(fmt.Sprintf("invalid segment: %s", seg), str, col, tok)
}
col += len(tok)
}

View File

@@ -0,0 +1,562 @@
package selector
import (
"fmt"
"math"
"testing"
"github.com/stretchr/testify/require"
)
func TestParse(t *testing.T) {
t.Run("identity", func(t *testing.T) {
sel, err := Parse(".")
require.NoError(t, err)
require.Equal(t, 1, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
})
t.Run("dotted field name", func(t *testing.T) {
sel, err := Parse(".foo")
require.NoError(t, err)
require.Equal(t, 1, len(sel))
require.False(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Equal(t, sel[0].Field(), "foo")
require.Empty(t, sel[0].Index())
})
t.Run("explicit field", func(t *testing.T) {
sel, err := Parse(`.["foo"]`)
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.False(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Empty(t, sel[1].Slice())
require.Equal(t, sel[1].Field(), "foo")
require.Empty(t, sel[1].Index())
})
t.Run("iterator, collection value", func(t *testing.T) {
sel, err := Parse(".[]")
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.False(t, sel[1].Optional())
require.True(t, sel[1].Iterator())
require.Empty(t, sel[1].Slice())
require.Empty(t, sel[1].Field())
require.Empty(t, sel[1].Index())
})
t.Run("index", func(t *testing.T) {
sel, err := Parse(".[138]")
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.False(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Empty(t, sel[1].Slice())
require.Empty(t, sel[1].Field())
require.Equal(t, sel[1].Index(), 138)
})
t.Run("negative index", func(t *testing.T) {
sel, err := Parse(".[-138]")
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.False(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Empty(t, sel[1].Slice())
require.Empty(t, sel[1].Field())
require.Equal(t, sel[1].Index(), -138)
})
t.Run("List slice", func(t *testing.T) {
sel, err := Parse(".[7:11]")
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.False(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Equal(t, sel[1].Slice(), []int64{7, 11})
require.Empty(t, sel[1].Field())
require.Empty(t, sel[1].Index())
sel, err = Parse(".[2:]")
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.False(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Equal(t, sel[1].Slice(), []int64{2, math.MaxInt})
require.Empty(t, sel[1].Field())
require.Empty(t, sel[1].Index())
sel, err = Parse(".[:42]")
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.False(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Equal(t, sel[1].Slice(), []int64{math.MinInt, 42})
require.Empty(t, sel[1].Field())
require.Empty(t, sel[1].Index())
sel, err = Parse(".[0:-2]")
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.False(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Equal(t, sel[1].Slice(), []int64{0, -2})
require.Empty(t, sel[1].Field())
require.Empty(t, sel[1].Index())
})
t.Run("optional identity", func(t *testing.T) {
sel, err := Parse(".?")
require.NoError(t, err)
require.Equal(t, 1, len(sel))
require.True(t, sel[0].Identity())
require.True(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
})
t.Run("optional dotted field name", func(t *testing.T) {
sel, err := Parse(".foo?")
require.NoError(t, err)
require.Equal(t, 1, len(sel))
require.False(t, sel[0].Identity())
require.True(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Equal(t, sel[0].Field(), "foo")
require.Empty(t, sel[0].Index())
})
t.Run("optional explicit field", func(t *testing.T) {
sel, err := Parse(`.["foo"]?`)
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.True(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Empty(t, sel[1].Slice())
require.Equal(t, sel[1].Field(), "foo")
require.Empty(t, sel[1].Index())
})
t.Run("optional iterator", func(t *testing.T) {
sel, err := Parse(".[]?")
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.True(t, sel[1].Optional())
require.True(t, sel[1].Iterator())
require.Empty(t, sel[1].Slice())
require.Empty(t, sel[1].Field())
require.Empty(t, sel[1].Index())
})
t.Run("optional index", func(t *testing.T) {
sel, err := Parse(".[138]?")
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.True(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Empty(t, sel[1].Slice())
require.Empty(t, sel[1].Field())
require.Equal(t, sel[1].Index(), 138)
})
t.Run("optional negative index", func(t *testing.T) {
sel, err := Parse(".[-138]?")
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.True(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Empty(t, sel[1].Slice())
require.Empty(t, sel[1].Field())
require.Equal(t, sel[1].Index(), -138)
})
t.Run("optional list slice", func(t *testing.T) {
sel, err := Parse(".[7:11]?")
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.True(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Equal(t, sel[1].Slice(), []int64{7, 11})
require.Empty(t, sel[1].Field())
require.Empty(t, sel[1].Index())
sel, err = Parse(".[2:]?")
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.True(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Equal(t, sel[1].Slice(), []int64{2, math.MaxInt})
require.Empty(t, sel[1].Field())
require.Empty(t, sel[1].Index())
sel, err = Parse(".[:42]?")
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.True(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Equal(t, sel[1].Slice(), []int64{math.MinInt, 42})
require.Empty(t, sel[1].Field())
require.Empty(t, sel[1].Index())
sel, err = Parse(".[0:-2]?")
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.True(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Equal(t, sel[1].Slice(), []int64{0, -2})
require.Empty(t, sel[1].Field())
require.Empty(t, sel[1].Index())
})
t.Run("idempotent optional", func(t *testing.T) {
sel, err := Parse(".foo???")
require.NoError(t, err)
require.Equal(t, 1, len(sel))
require.False(t, sel[0].Identity())
require.True(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Equal(t, sel[0].Field(), "foo")
require.Empty(t, sel[0].Index())
})
t.Run("deny multi dot", func(t *testing.T) {
_, err := Parse("..")
require.Error(t, err)
})
t.Run("nesting", func(t *testing.T) {
str := `.foo.["bar"].[138]?.baz[1:]`
sel, err := Parse(str)
require.NoError(t, err)
printSegments(sel)
require.Equal(t, str, sel.String())
require.Equal(t, 7, len(sel))
require.False(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Equal(t, sel[0].Field(), "foo")
require.Empty(t, sel[0].Index())
require.True(t, sel[1].Identity())
require.False(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Empty(t, sel[1].Slice())
require.Empty(t, sel[1].Field())
require.Empty(t, sel[1].Index())
require.False(t, sel[2].Identity())
require.False(t, sel[2].Optional())
require.False(t, sel[2].Iterator())
require.Empty(t, sel[2].Slice())
require.Equal(t, sel[2].Field(), "bar")
require.Empty(t, sel[2].Index())
require.True(t, sel[3].Identity())
require.False(t, sel[3].Optional())
require.False(t, sel[3].Iterator())
require.Empty(t, sel[3].Slice())
require.Empty(t, sel[3].Field())
require.Empty(t, sel[3].Index())
require.False(t, sel[4].Identity())
require.True(t, sel[4].Optional())
require.False(t, sel[4].Iterator())
require.Empty(t, sel[4].Slice())
require.Empty(t, sel[4].Field())
require.Equal(t, sel[4].Index(), 138)
require.False(t, sel[5].Identity())
require.False(t, sel[5].Optional())
require.False(t, sel[5].Iterator())
require.Empty(t, sel[5].Slice())
require.Equal(t, sel[5].Field(), "baz")
require.Empty(t, sel[5].Index())
require.False(t, sel[6].Identity())
require.False(t, sel[6].Optional())
require.False(t, sel[6].Iterator())
require.Equal(t, sel[6].Slice(), []int64{1, math.MaxInt})
require.Empty(t, sel[6].Field())
require.Empty(t, sel[6].Index())
})
t.Run("non dotted", func(t *testing.T) {
_, err := Parse("foo")
require.NotNil(t, err)
fmt.Println(err)
})
t.Run("non quoted", func(t *testing.T) {
_, err := Parse(".[foo]")
require.NotNil(t, err)
fmt.Println(err)
})
t.Run("slice with negative start and positive end", func(t *testing.T) {
sel, err := Parse(".[0:-2]")
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.False(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Equal(t, sel[1].Slice(), []int64{0, -2})
require.Empty(t, sel[1].Field())
require.Empty(t, sel[1].Index())
})
t.Run("slice with start greater than end", func(t *testing.T) {
sel, err := Parse(".[5:2]")
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.False(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Equal(t, sel[1].Slice(), []int64{5, 2})
require.Empty(t, sel[1].Field())
require.Empty(t, sel[1].Index())
})
t.Run("slice on string", func(t *testing.T) {
sel, err := Parse(`.["foo"].[1:3]`)
require.NoError(t, err)
require.Equal(t, 4, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.False(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Empty(t, sel[1].Slice())
require.Equal(t, sel[1].Field(), "foo")
require.Empty(t, sel[1].Index())
require.True(t, sel[2].Identity())
require.False(t, sel[2].Optional())
require.False(t, sel[2].Iterator())
require.Empty(t, sel[2].Slice())
require.Empty(t, sel[2].Field())
require.Empty(t, sel[2].Index())
require.False(t, sel[3].Identity())
require.False(t, sel[3].Optional())
require.False(t, sel[3].Iterator())
require.Equal(t, sel[3].Slice(), []int64{1, 3})
require.Empty(t, sel[3].Field())
require.Empty(t, sel[3].Index())
})
t.Run("slice on array", func(t *testing.T) {
sel, err := Parse(`.[1:3]`)
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.False(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Equal(t, sel[1].Slice(), []int64{1, 3})
require.Empty(t, sel[1].Field())
require.Empty(t, sel[1].Index())
})
t.Run("index on array", func(t *testing.T) {
sel, err := Parse(`.[1]`)
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.False(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Empty(t, sel[1].Slice())
require.Empty(t, sel[1].Field())
require.Equal(t, sel[1].Index(), 1)
})
t.Run("invalid slice on object", func(t *testing.T) {
_, err := Parse(`.["foo":"bar"]`)
require.Error(t, err)
require.Contains(t, err.Error(), "invalid segment")
})
t.Run("index on object", func(t *testing.T) {
sel, err := Parse(`.["foo"]`)
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.False(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Empty(t, sel[1].Slice())
require.Equal(t, sel[1].Field(), "foo")
require.Empty(t, sel[1].Index())
})
t.Run("slice with non-integer start", func(t *testing.T) {
_, err := Parse(".[foo:3]")
require.Error(t, err)
})
t.Run("slice with non-integer end", func(t *testing.T) {
_, err := Parse(".[1:bar]")
require.Error(t, err)
})
t.Run("index with non-integer", func(t *testing.T) {
_, err := Parse(".[foo]")
require.Error(t, err)
})
}
func printSegments(s Selector) {
for i, seg := range s {
fmt.Printf("%d: %s\n", i, seg.String())
}
}

View File

@@ -1,11 +1,15 @@
package selector
import (
"errors"
"fmt"
"math"
"strconv"
"strings"
"github.com/ipld/go-ipld-prime"
"github.com/ipld/go-ipld-prime/datamodel"
"github.com/ipld/go-ipld-prime/fluent/qp"
"github.com/ipld/go-ipld-prime/node/basicnode"
"github.com/ipld/go-ipld-prime/schema"
)
@@ -15,17 +19,14 @@ import (
type Selector []segment
// Select perform the selection described by the selector on the input IPLD DAG.
// If no error, Select returns either one ipld.Node or a []ipld.Node.
func (s Selector) Select(subject ipld.Node) (ipld.Node, []ipld.Node, error) {
// Select can return:
// - exactly one matched IPLD node
// - a resolutionerr error if not being able to resolve to a node
// - nil and no errors, if the selector couldn't match on an optional segment (with ?).
func (s Selector) Select(subject ipld.Node) (ipld.Node, error) {
return resolve(s, subject, nil)
}
// MatchPath tells if the selector operates on the given (string only) path segments.
// It returns the segments that didn't get consumed by the matching.
func (s Selector) MatchPath(pathSegment ...string) (bool, []string) {
return matchPath(s, pathSegment)
}
func (s Selector) String() string {
var res strings.Builder
for _, seg := range s {
@@ -39,7 +40,7 @@ type segment struct {
identity bool
optional bool
iterator bool
slice []int
slice []int64 // either 0-length or 2-length
field string
index int
}
@@ -65,7 +66,7 @@ func (s segment) Iterator() bool {
}
// Slice flags that this segment targets a range of a slice.
func (s segment) Slice() []int {
func (s segment) Slice() []int64 {
return s.slice
}
@@ -79,335 +80,168 @@ func (s segment) Index() int {
return s.index
}
func resolve(sel Selector, subject ipld.Node, at []string) (ipld.Node, []ipld.Node, error) {
cur := subject
for i, seg := range sel {
if seg.Identity() {
continue
}
// 1st level: handle the different segment types (iterator, field, slice, index)
// 2nd level: handle different node kinds (list, map, string, bytes)
switch {
case seg.Iterator():
if cur == nil || cur.Kind() == datamodel.Kind_Null {
if seg.Optional() {
// build empty list
nb := basicnode.Prototype.List.NewBuilder()
assembler, err := nb.BeginList(0)
if err != nil {
return nil, nil, err
}
if err = assembler.Finish(); err != nil {
return nil, nil, err
}
return nb.Build(), nil, nil
} else {
return nil, nil, newResolutionError(fmt.Sprintf("can not iterate over kind: %s", kindString(cur)), at)
}
} else {
var many []ipld.Node
switch cur.Kind() {
case datamodel.Kind_List:
it := cur.ListIterator()
for !it.Done() {
_, v, err := it.Next()
if err != nil {
return nil, nil, err
}
// check if there are more iterator segments
if len(sel) > i+1 && sel[i+1].Iterator() {
if v.Kind() == datamodel.Kind_List {
// recursively resolve the remaining selector segments
var o ipld.Node
var m []ipld.Node
o, m, err = resolve(sel[i+1:], v, at)
if err != nil {
// if the segment is optional and an error occurs, skip the current iteration.
if seg.Optional() {
continue
} else {
return nil, nil, err
}
}
if m != nil {
many = append(many, m...)
} else if o != nil {
many = append(many, o)
}
} else {
// if the current value is not a list and the next segment is optional, skip the current iteration
if sel[i+1].Optional() {
continue
} else {
return nil, nil, newResolutionError(fmt.Sprintf("can not iterate over kind: %s", kindString(v)), at)
}
}
} else {
// if there are no more iterator segments, append the current value to the result
many = append(many, v)
}
}
case datamodel.Kind_Map:
it := cur.MapIterator()
for !it.Done() {
_, v, err := it.Next()
if err != nil {
return nil, nil, err
}
if len(sel) > i+1 && sel[i+1].Iterator() {
if v.Kind() == datamodel.Kind_List {
var o ipld.Node
var m []ipld.Node
o, m, err = resolve(sel[i+1:], v, at)
if err != nil {
if seg.Optional() {
continue
} else {
return nil, nil, err
}
}
if m != nil {
many = append(many, m...)
} else if o != nil {
many = append(many, o)
}
} else {
if sel[i+1].Optional() {
continue
} else {
return nil, nil, newResolutionError(fmt.Sprintf("can not iterate over kind: %s", kindString(v)), at)
}
}
} else {
many = append(many, v)
}
}
default:
return nil, nil, newResolutionError(fmt.Sprintf("can not iterate over kind: %s", kindString(cur)), at)
}
return nil, many, nil
}
case seg.Field() != "":
at = append(at, seg.Field())
if cur == nil {
if seg.Optional() {
cur = nil
} else {
return nil, nil, newResolutionError(fmt.Sprintf("can not access field: %s on kind: %s", seg.Field(), kindString(cur)), at)
}
} else {
switch cur.Kind() {
case datamodel.Kind_Map:
n, err := cur.LookupByString(seg.Field())
if err != nil {
if isMissing(err) {
if seg.Optional() {
cur = nil
} else {
return nil, nil, newResolutionError(fmt.Sprintf("object has no field named: %s", seg.Field()), at)
}
} else {
return nil, nil, err
}
} else {
cur = n
}
case datamodel.Kind_List:
var many []ipld.Node
it := cur.ListIterator()
for !it.Done() {
_, v, err := it.Next()
if err != nil {
return nil, nil, err
}
if v.Kind() == datamodel.Kind_Map {
n, err := v.LookupByString(seg.Field())
if err == nil {
many = append(many, n)
}
}
}
if len(many) > 0 {
cur = nil
return nil, many, nil
} else if seg.Optional() {
cur = nil
} else {
return nil, nil, newResolutionError(fmt.Sprintf("no elements in list have field named: %s", seg.Field()), at)
}
default:
if seg.Optional() {
cur = nil
} else {
return nil, nil, newResolutionError(fmt.Sprintf("can not access field: %s on kind: %s", seg.Field(), kindString(cur)), at)
}
}
}
case seg.Slice() != nil:
if cur == nil {
if seg.Optional() {
cur = nil
} else {
return nil, nil, newResolutionError(fmt.Sprintf("can not slice on kind: %s", kindString(cur)), at)
}
} else {
slice := seg.Slice()
var start, end, length int64
switch cur.Kind() {
case datamodel.Kind_List:
length = cur.Length()
start, end = resolveSliceIndices(slice, length)
case datamodel.Kind_Bytes:
b, _ := cur.AsBytes()
length = int64(len(b))
start, end = resolveSliceIndices(slice, length)
case datamodel.Kind_String:
str, _ := cur.AsString()
length = int64(len(str))
start, end = resolveSliceIndices(slice, length)
default:
return nil, nil, newResolutionError(fmt.Sprintf("can not slice on kind: %s", kindString(cur)), at)
}
if start < 0 || end < start || end > length {
if seg.Optional() {
cur = nil
} else {
return nil, nil, newResolutionError(fmt.Sprintf("slice out of bounds: [%d:%d]", start, end), at)
}
} else {
switch cur.Kind() {
case datamodel.Kind_List:
if end > cur.Length() {
end = cur.Length()
}
nb := basicnode.Prototype.List.NewBuilder()
assembler, _ := nb.BeginList(int64(end - start))
for i := start; i < end; i++ {
item, _ := cur.LookupByIndex(int64(i))
assembler.AssembleValue().AssignNode(item)
}
assembler.Finish()
cur = nb.Build()
case datamodel.Kind_Bytes:
b, _ := cur.AsBytes()
l := int64(len(b))
if end > l {
end = l
}
cur = basicnode.NewBytes(b[start:end])
case datamodel.Kind_String:
str, _ := cur.AsString()
l := int64(len(str))
if end > l {
end = l
}
cur = basicnode.NewString(str[start:end])
}
}
}
default: // Index()
at = append(at, fmt.Sprintf("%d", seg.Index()))
if cur == nil {
if seg.Optional() {
cur = nil
} else {
return nil, nil, newResolutionError(fmt.Sprintf("can not access index: %d on kind: %s", seg.Index(), kindString(cur)), at)
}
} else {
idx := seg.Index()
switch cur.Kind() {
case datamodel.Kind_List:
if idx < 0 {
idx = int(cur.Length()) + idx
}
if idx < 0 || idx >= int(cur.Length()) {
if seg.Optional() {
cur = nil
} else {
return nil, nil, newResolutionError(fmt.Sprintf("index out of bounds: %d", seg.Index()), at)
}
} else {
cur, _ = cur.LookupByIndex(int64(idx))
}
case datamodel.Kind_String:
str, _ := cur.AsString()
if idx < 0 {
idx = len(str) + idx
}
if idx < 0 || idx >= len(str) {
if seg.Optional() {
cur = nil
} else {
return nil, nil, newResolutionError(fmt.Sprintf("index out of bounds: %d", seg.Index()), at)
}
} else {
cur = basicnode.NewString(string(str[idx]))
}
case datamodel.Kind_Bytes:
b, _ := cur.AsBytes()
if idx < 0 {
idx = len(b) + idx
}
if idx < 0 || idx >= len(b) {
if seg.Optional() {
cur = nil
} else {
return nil, nil, newResolutionError(fmt.Sprintf("index out of bounds: %d", seg.Index()), at)
}
} else {
cur = basicnode.NewInt(int64(b[idx]))
}
default:
return nil, nil, newResolutionError(fmt.Sprintf("can not access index: %d on kind: %s", seg.Index(), kindString(cur)), at)
}
}
func resolve(sel Selector, subject ipld.Node, at []string) (ipld.Node, error) {
errIfNotOptional := func(s segment, err error) error {
if !s.Optional() {
return err
}
return nil
}
return cur, nil, nil
}
func matchPath(sel Selector, path []string) (bool, []string) {
cur := subject
for _, seg := range sel {
if len(path) == 0 {
return true, path
}
// 1st level: handle the different segment types (iterator, field, slice, index)
// 2nd level: handle different node kinds (list, map, string, bytes)
switch {
case seg.Identity():
continue
case seg.Iterator():
// we have reached a [] iterator, it should have matched earlier
return false, nil
switch {
case cur == nil || cur.Kind() == datamodel.Kind_Null:
if seg.Optional() {
// build an empty list
n, _ := qp.BuildList(basicnode.Prototype.Any, 0, func(_ datamodel.ListAssembler) {})
return n, nil
}
return nil, newResolutionError(fmt.Sprintf("can not iterate over kind: %s", kindString(cur)), at)
case cur.Kind() == datamodel.Kind_List:
// iterators are no-op on list
continue
case cur.Kind() == datamodel.Kind_Map:
// iterators on maps collect the values
nd, err := qp.BuildList(basicnode.Prototype.Any, cur.Length(), func(l datamodel.ListAssembler) {
it := cur.MapIterator()
for !it.Done() {
_, v, err := it.Next()
if err != nil {
// recovered by BuildList
// Error is bubbled up, but should never occur as we already checked the type,
// and are using the iterator correctly.
// This is verified with fuzzing.
panic(err)
}
qp.ListEntry(l, qp.Node(v))
}
})
if err != nil {
panic("should never happen")
}
return nd, nil
default:
return nil, newResolutionError(fmt.Sprintf("can not iterate over kind: %s", kindString(cur)), at)
}
case seg.Field() != "":
// if exact match on the segment, we continue
if path[0] == seg.Field() {
path = path[1:]
continue
}
return false, nil
at = append(at, seg.Field())
switch {
case cur == nil:
err := newResolutionError(fmt.Sprintf("can not access field: %s on kind: %s", seg.Field(), kindString(cur)), at)
return nil, errIfNotOptional(seg, err)
case seg.Slice() != nil:
// we have reached a [<int>:<int>] slicing, it should have matched earlier
return false, nil
case cur.Kind() == datamodel.Kind_Map:
n, err := cur.LookupByString(seg.Field())
if err != nil {
// the only possible error is missing field as we already check the type
if seg.Optional() {
cur = nil
} else {
return nil, newResolutionError(fmt.Sprintf("object has no field named: %s", seg.Field()), at)
}
} else {
cur = n
}
default:
err := newResolutionError(fmt.Sprintf("can not access field: %s on kind: %s", seg.Field(), kindString(cur)), at)
return nil, errIfNotOptional(seg, err)
}
case len(seg.Slice()) > 0:
if cur == nil {
err := newResolutionError(fmt.Sprintf("can not slice on kind: %s", kindString(cur)), at)
return nil, errIfNotOptional(seg, err)
}
slice := seg.Slice()
switch cur.Kind() {
case datamodel.Kind_List:
start, end := resolveSliceIndices(slice, cur.Length())
sliced, err := qp.BuildList(basicnode.Prototype.Any, end-start, func(l datamodel.ListAssembler) {
for i := start; i < end; i++ {
item, err := cur.LookupByIndex(i)
if err != nil {
// recovered by BuildList
// Error is bubbled up, but should never occur as we already checked the type and boundaries
// This is verified with fuzzing.
panic(err)
}
qp.ListEntry(l, qp.Node(item))
}
})
if err != nil {
panic("should never happen")
}
cur = sliced
case datamodel.Kind_Bytes:
b, _ := cur.AsBytes()
start, end := resolveSliceIndices(slice, int64(len(b)))
cur = basicnode.NewBytes(b[start:end])
case datamodel.Kind_String:
str, _ := cur.AsString()
runes := []rune(str)
start, end := resolveSliceIndices(slice, int64(len(runes)))
cur = basicnode.NewString(string(runes[start:end]))
default:
return nil, newResolutionError(fmt.Sprintf("can not slice on kind: %s", kindString(cur)), at)
}
default: // Index()
// we have reached a [<int>] indexing, it should have matched earlier
return false, nil
at = append(at, strconv.Itoa(seg.Index()))
if cur == nil {
err := newResolutionError(fmt.Sprintf("can not access index: %d on kind: %s", seg.Index(), kindString(cur)), at)
return nil, errIfNotOptional(seg, err)
}
idx := seg.Index()
switch cur.Kind() {
case datamodel.Kind_List:
if idx < 0 {
idx = int(cur.Length()) + idx
}
if idx < 0 || idx >= int(cur.Length()) {
err := newResolutionError(fmt.Sprintf("index out of bounds: %d", seg.Index()), at)
return nil, errIfNotOptional(seg, err)
}
cur, _ = cur.LookupByIndex(int64(idx))
case datamodel.Kind_Bytes:
b, _ := cur.AsBytes()
if idx < 0 {
idx = len(b) + idx
}
if idx < 0 || idx >= len(b) {
err := newResolutionError(fmt.Sprintf("index %d out of bounds for bytes of length %d", seg.Index(), len(b)), at)
return nil, errIfNotOptional(seg, err)
}
cur = basicnode.NewInt(int64(b[idx]))
default:
return nil, newResolutionError(fmt.Sprintf("can not access index: %d on kind: %s", seg.Index(), kindString(cur)), at)
}
}
}
return true, path
// segment exhausted, we return where we are
return cur, nil
}
// resolveSliceIndices resolves the start and end indices for slicing a list or byte array.
@@ -421,26 +255,41 @@ func matchPath(sel Selector, path []string) (bool, []string) {
//
// Returns:
// - start: The resolved start index for slicing.
// - end: The resolved end index for slicing.
func resolveSliceIndices(slice []int, length int64) (int64, int64) {
start, end := int64(0), length
if len(slice) > 0 {
start = int64(slice[0])
if start < 0 {
start = length + start
if start < 0 {
start = 0
}
}
// - end: The resolved **excluded** end index for slicing.
func resolveSliceIndices(slice []int64, length int64) (start int64, end int64) {
if len(slice) != 2 {
panic("should always be 2-length")
}
if len(slice) > 1 {
end = int64(slice[1])
if end <= 0 {
end = length + end
if end < start {
end = start
}
}
start, end = slice[0], slice[1]
// adjust boundaries
switch {
case slice[0] == math.MinInt:
start = 0
case slice[0] < 0:
start = length + slice[0]
}
switch {
case slice[1] == math.MaxInt:
end = length
case slice[1] < 0:
end = length + slice[1]
}
// backward iteration is not allowed, shortcut to an empty result
if start >= end {
start, end = 0, 0
}
// clamp out of bound
if start < 0 {
start = 0
}
if start > length {
start = length
}
if end > length {
end = length
}
return start, end
@@ -466,6 +315,10 @@ func isMissing(err error) bool {
return false
}
func IsResolutionErr(err error) bool {
return errors.As(err, &resolutionerr{})
}
type resolutionerr struct {
msg string
at []string

View File

@@ -7,6 +7,8 @@ import (
"github.com/ipld/go-ipld-prime"
"github.com/ipld/go-ipld-prime/codec/dagjson"
"github.com/ipld/go-ipld-prime/datamodel"
"github.com/ipld/go-ipld-prime/fluent/qp"
"github.com/ipld/go-ipld-prime/must"
basicnode "github.com/ipld/go-ipld-prime/node/basic"
"github.com/ipld/go-ipld-prime/node/bindnode"
@@ -14,239 +16,6 @@ import (
"github.com/stretchr/testify/require"
)
func TestParse(t *testing.T) {
t.Run("identity", func(t *testing.T) {
sel, err := Parse(".")
require.NoError(t, err)
require.Equal(t, 1, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
})
t.Run("field", func(t *testing.T) {
sel, err := Parse(".foo")
require.NoError(t, err)
require.Equal(t, 1, len(sel))
require.False(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Equal(t, sel[0].Field(), "foo")
require.Empty(t, sel[0].Index())
})
t.Run("explicit field", func(t *testing.T) {
sel, err := Parse(`.["foo"]`)
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.False(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Empty(t, sel[1].Slice())
require.Equal(t, sel[1].Field(), "foo")
require.Empty(t, sel[1].Index())
})
t.Run("index", func(t *testing.T) {
sel, err := Parse(".[138]")
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.False(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Empty(t, sel[1].Slice())
require.Empty(t, sel[1].Field())
require.Equal(t, sel[1].Index(), 138)
})
t.Run("negative index", func(t *testing.T) {
sel, err := Parse(".[-138]")
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.False(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Empty(t, sel[1].Slice())
require.Empty(t, sel[1].Field())
require.Equal(t, sel[1].Index(), -138)
})
t.Run("iterator", func(t *testing.T) {
sel, err := Parse(".[]")
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.False(t, sel[1].Optional())
require.True(t, sel[1].Iterator())
require.Empty(t, sel[1].Slice())
require.Empty(t, sel[1].Field())
require.Empty(t, sel[1].Index())
})
t.Run("optional field", func(t *testing.T) {
sel, err := Parse(".foo?")
require.NoError(t, err)
require.Equal(t, 1, len(sel))
require.False(t, sel[0].Identity())
require.True(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Equal(t, sel[0].Field(), "foo")
require.Empty(t, sel[0].Index())
})
t.Run("optional explicit field", func(t *testing.T) {
sel, err := Parse(`.["foo"]?`)
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.True(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Empty(t, sel[1].Slice())
require.Equal(t, sel[1].Field(), "foo")
require.Empty(t, sel[1].Index())
})
t.Run("optional index", func(t *testing.T) {
sel, err := Parse(".[138]?")
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.True(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Empty(t, sel[1].Slice())
require.Empty(t, sel[1].Field())
require.Equal(t, sel[1].Index(), 138)
})
t.Run("optional iterator", func(t *testing.T) {
sel, err := Parse(".[]?")
require.NoError(t, err)
require.Equal(t, 2, len(sel))
require.True(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Empty(t, sel[0].Field())
require.Empty(t, sel[0].Index())
require.False(t, sel[1].Identity())
require.True(t, sel[1].Optional())
require.True(t, sel[1].Iterator())
require.Empty(t, sel[1].Slice())
require.Empty(t, sel[1].Field())
require.Empty(t, sel[1].Index())
})
t.Run("nesting", func(t *testing.T) {
str := `.foo.["bar"].[138]?.baz[1:]`
sel, err := Parse(str)
require.NoError(t, err)
printSegments(sel)
require.Equal(t, str, sel.String())
require.Equal(t, 7, len(sel))
require.False(t, sel[0].Identity())
require.False(t, sel[0].Optional())
require.False(t, sel[0].Iterator())
require.Empty(t, sel[0].Slice())
require.Equal(t, sel[0].Field(), "foo")
require.Empty(t, sel[0].Index())
require.True(t, sel[1].Identity())
require.False(t, sel[1].Optional())
require.False(t, sel[1].Iterator())
require.Empty(t, sel[1].Slice())
require.Empty(t, sel[1].Field())
require.Empty(t, sel[1].Index())
require.False(t, sel[2].Identity())
require.False(t, sel[2].Optional())
require.False(t, sel[2].Iterator())
require.Empty(t, sel[2].Slice())
require.Equal(t, sel[2].Field(), "bar")
require.Empty(t, sel[2].Index())
require.True(t, sel[3].Identity())
require.False(t, sel[3].Optional())
require.False(t, sel[3].Iterator())
require.Empty(t, sel[3].Slice())
require.Empty(t, sel[3].Field())
require.Empty(t, sel[3].Index())
require.False(t, sel[4].Identity())
require.True(t, sel[4].Optional())
require.False(t, sel[4].Iterator())
require.Empty(t, sel[4].Slice())
require.Empty(t, sel[4].Field())
require.Equal(t, sel[4].Index(), 138)
require.False(t, sel[5].Identity())
require.False(t, sel[5].Optional())
require.False(t, sel[5].Iterator())
require.Empty(t, sel[5].Slice())
require.Equal(t, sel[5].Field(), "baz")
require.Empty(t, sel[5].Index())
require.False(t, sel[6].Identity())
require.False(t, sel[6].Optional())
require.False(t, sel[6].Iterator())
require.Equal(t, sel[6].Slice(), []int{1})
require.Empty(t, sel[6].Field())
require.Empty(t, sel[6].Index())
})
t.Run("non dotted", func(t *testing.T) {
_, err := Parse("foo")
require.NotNil(t, err)
fmt.Println(err)
})
t.Run("non quoted", func(t *testing.T) {
_, err := Parse(".[foo]")
require.NotNil(t, err)
fmt.Println(err)
})
}
func printSegments(s Selector) {
for i, seg := range s {
fmt.Printf("%d: %s\n", i, seg.String())
}
}
func TestSelect(t *testing.T) {
type name struct {
First string
@@ -313,14 +82,13 @@ func TestSelect(t *testing.T) {
sel, err := Parse(".")
require.NoError(t, err)
one, many, err := sel.Select(anode)
res, err := sel.Select(anode)
require.NoError(t, err)
require.NotEmpty(t, one)
require.Empty(t, many)
require.NotEmpty(t, res)
fmt.Println(printer.Sprint(one))
fmt.Println(printer.Sprint(res))
age := must.Int(must.Node(one.LookupByString("age")))
age := must.Int(must.Node(res.LookupByString("age")))
require.Equal(t, int64(alice.Age), age)
})
@@ -328,24 +96,22 @@ func TestSelect(t *testing.T) {
sel, err := Parse(".name.first")
require.NoError(t, err)
one, many, err := sel.Select(anode)
res, err := sel.Select(anode)
require.NoError(t, err)
require.NotEmpty(t, one)
require.Empty(t, many)
require.NotEmpty(t, res)
fmt.Println(printer.Sprint(one))
fmt.Println(printer.Sprint(res))
name := must.String(one)
name := must.String(res)
require.Equal(t, alice.Name.First, name)
one, many, err = sel.Select(bnode)
res, err = sel.Select(bnode)
require.NoError(t, err)
require.NotEmpty(t, one)
require.Empty(t, many)
require.NotEmpty(t, res)
fmt.Println(printer.Sprint(one))
fmt.Println(printer.Sprint(res))
name = must.String(one)
name = must.String(res)
require.Equal(t, bob.Name.First, name)
})
@@ -353,108 +119,185 @@ func TestSelect(t *testing.T) {
sel, err := Parse(".name.middle?")
require.NoError(t, err)
one, many, err := sel.Select(anode)
res, err := sel.Select(anode)
require.NoError(t, err)
require.NotEmpty(t, one)
require.Empty(t, many)
require.NotEmpty(t, res)
fmt.Println(printer.Sprint(one))
fmt.Println(printer.Sprint(res))
name := must.String(one)
name := must.String(res)
require.Equal(t, *alice.Name.Middle, name)
one, many, err = sel.Select(bnode)
res, err = sel.Select(bnode)
require.NoError(t, err)
require.Empty(t, one)
require.Empty(t, many)
require.Empty(t, res)
})
t.Run("not exists", func(t *testing.T) {
sel, err := Parse(".name.foo")
require.NoError(t, err)
one, many, err := sel.Select(anode)
res, err := sel.Select(anode)
require.Error(t, err)
require.Empty(t, one)
require.Empty(t, many)
require.Empty(t, res)
fmt.Println(err)
require.ErrorAs(t, err, &resolutionerr{}, "error was not a resolution error")
require.ErrorAs(t, err, &resolutionerr{}, "error should be a resolution error")
require.True(t, IsResolutionErr(err))
})
t.Run("optional not exists", func(t *testing.T) {
sel, err := Parse(".name.foo?")
require.NoError(t, err)
one, many, err := sel.Select(anode)
one, err := sel.Select(anode)
require.NoError(t, err)
require.Empty(t, one)
require.Empty(t, many)
})
t.Run("iterator", func(t *testing.T) {
sel, err := Parse(".interests[]")
require.NoError(t, err)
one, many, err := sel.Select(anode)
res, err := sel.Select(anode)
require.NoError(t, err)
require.Empty(t, one)
require.NotEmpty(t, many)
require.NotEmpty(t, res)
for _, n := range many {
fmt.Println(printer.Sprint(n))
}
fmt.Println(printer.Sprint(res))
iname := must.String(must.Node(many[0].LookupByString("name")))
iname := must.String(must.Node(must.Node(res.LookupByIndex(0)).LookupByString("name")))
require.Equal(t, alice.Interests[0].Name, iname)
iname = must.String(must.Node(many[1].LookupByString("name")))
iname = must.String(must.Node(must.Node(res.LookupByIndex(1)).LookupByString("name")))
require.Equal(t, alice.Interests[1].Name, iname)
})
t.Run("map iterator", func(t *testing.T) {
sel, err := Parse(".interests[0][]")
t.Run("slice on string", func(t *testing.T) {
sel, err := Parse(`.[1:3]`)
require.NoError(t, err)
one, many, err := sel.Select(anode)
node := basicnode.NewString("hello")
res, err := sel.Select(node)
require.NoError(t, err)
require.Empty(t, one)
require.NotEmpty(t, many)
require.NotEmpty(t, res)
for _, n := range many {
fmt.Println(printer.Sprint(n))
}
require.Equal(t, alice.Interests[0].Name, must.String(many[0]))
require.Equal(t, alice.Interests[0].Experience, int(must.Int(many[2])))
str, err := res.AsString()
require.NoError(t, err)
require.Equal(t, "el", str) // assert sliced substring
})
}
func TestMatch(t *testing.T) {
for _, tc := range []struct {
sel string
path []string
want bool
remaining []string
}{
{sel: ".foo.bar", path: []string{"foo", "bar"}, want: true, remaining: []string{}},
{sel: ".foo.bar", path: []string{"foo"}, want: true, remaining: []string{}},
{sel: ".foo.bar", path: []string{"foo", "bar", "baz"}, want: true, remaining: []string{"baz"}},
{sel: ".foo.bar", path: []string{"foo", "faa"}, want: false},
{sel: ".foo.[]", path: []string{"foo", "faa"}, want: false},
{sel: ".foo.[]", path: []string{"foo"}, want: true, remaining: []string{}},
{sel: ".foo.bar?", path: []string{"foo"}, want: true, remaining: []string{}},
{sel: ".foo.bar?", path: []string{"foo", "bar"}, want: true, remaining: []string{}},
{sel: ".foo.bar?", path: []string{"foo", "baz"}, want: false},
} {
t.Run(tc.sel, func(t *testing.T) {
sel := MustParse(tc.sel)
res, remain := sel.MatchPath(tc.path...)
require.Equal(t, tc.want, res)
require.EqualValues(t, tc.remaining, remain)
t.Run("out of bounds slicing", func(t *testing.T) {
node, err := qp.BuildList(basicnode.Prototype.Any, 3, func(la datamodel.ListAssembler) {
qp.ListEntry(la, qp.Int(1))
qp.ListEntry(la, qp.Int(2))
qp.ListEntry(la, qp.Int(3))
})
}
require.NoError(t, err)
sel, err := Parse(`.[10:20]`)
require.NoError(t, err)
res, err := sel.Select(node)
require.NoError(t, err)
require.NotEmpty(t, res)
require.Equal(t, int64(0), res.Length())
_, err = res.LookupByIndex(0)
require.ErrorIs(t, err, datamodel.ErrNotExists{}) // assert empty result for out of bounds slice
})
t.Run("backward slicing", func(t *testing.T) {
node, err := qp.BuildList(basicnode.Prototype.Any, 3, func(la datamodel.ListAssembler) {
qp.ListEntry(la, qp.Int(1))
qp.ListEntry(la, qp.Int(2))
qp.ListEntry(la, qp.Int(3))
})
require.NoError(t, err)
sel, err := Parse(`.[5:2]`)
require.NoError(t, err)
res, err := sel.Select(node)
require.NoError(t, err)
require.NotEmpty(t, res)
require.Equal(t, int64(0), res.Length())
_, err = res.LookupByIndex(0)
require.ErrorIs(t, err, datamodel.ErrNotExists{}) // assert empty result for backward slice
})
t.Run("slice with negative index", func(t *testing.T) {
node, err := qp.BuildList(basicnode.Prototype.Any, 3, func(la datamodel.ListAssembler) {
qp.ListEntry(la, qp.Int(1))
qp.ListEntry(la, qp.Int(2))
qp.ListEntry(la, qp.Int(3))
})
require.NoError(t, err)
sel, err := Parse(`.[0:-1]`)
require.NoError(t, err)
res, err := sel.Select(node)
require.NoError(t, err)
require.NotEmpty(t, res)
val, err := res.LookupByIndex(1)
require.NoError(t, err)
require.Equal(t, 2, int(must.Int(val))) // Assert sliced value at index 1
})
t.Run("slice on bytes", func(t *testing.T) {
sel, err := Parse(`.[1:3]`)
require.NoError(t, err)
node := basicnode.NewBytes([]byte{0x01, 0x02, 0x03, 0x04, 0x05})
res, err := sel.Select(node)
require.NoError(t, err)
require.NotEmpty(t, res)
bytes, err := res.AsBytes()
require.NoError(t, err)
require.Equal(t, []byte{0x02, 0x03}, bytes) // assert sliced bytes
})
t.Run("index on bytes", func(t *testing.T) {
sel, err := Parse(`.[2]`)
require.NoError(t, err)
node := basicnode.NewBytes([]byte{0x01, 0x02, 0x03, 0x04, 0x05})
res, err := sel.Select(node)
require.NoError(t, err)
require.NotEmpty(t, res)
val, err := res.AsInt()
require.NoError(t, err)
require.Equal(t, int64(0x03), val) // assert indexed byte value
})
t.Run("out of bounds slicing on bytes", func(t *testing.T) {
sel, err := Parse(`.[10:20]`)
require.NoError(t, err)
node := basicnode.NewBytes([]byte{0x01, 0x02, 0x03})
res, err := sel.Select(node)
require.NoError(t, err)
require.NotNil(t, res)
bytes, err := res.AsBytes()
require.NoError(t, err)
require.Empty(t, bytes) // assert empty result for out of bounds slice
})
t.Run("out of bounds indexing on bytes", func(t *testing.T) {
sel, err := Parse(`.[10]`)
require.NoError(t, err)
node := basicnode.NewBytes([]byte{0x01, 0x02, 0x03})
_, err = sel.Select(node)
require.Error(t, err)
require.Contains(t, err.Error(), "can not resolve path: .10") // assert error for out of bounds index
})
}
func FuzzParse(f *testing.F) {
@@ -520,6 +363,10 @@ func FuzzParseAndSelect(f *testing.F) {
}
// look for panic()
_, _, _ = sel.Select(node)
_, err = sel.Select(node)
if err != nil && !IsResolutionErr(err) {
// not normal, we should only have resolution errors
t.Fatal(err)
}
})
}

View File

@@ -26,17 +26,15 @@ func TestSupportedForms(t *testing.T) {
Output string
}
// Pass
// Pass and return a node
for _, testcase := range []Testcase{
{Name: "Identity", Selector: `.`, Input: `{"x":1}`, Output: `{"x":1}`},
{Name: "Iterator", Selector: `.[]`, Input: `[1, 2]`, Output: `[1, 2]`},
{Name: "Optional Null Iterator", Selector: `.[]?`, Input: `null`, Output: `[]`},
{Name: "Optional Iterator", Selector: `.[][]?`, Input: `[[1], 2, [3]]`, Output: `[1, 3]`},
{Name: "Object Key", Selector: `.x`, Input: `{"x": 1 }`, Output: `1`},
{Name: "Quoted Key", Selector: `.["x"]`, Input: `{"x": 1}`, Output: `1`},
{Name: "Index", Selector: `.[0]`, Input: `[1, 2]`, Output: `1`},
{Name: "Negative Index", Selector: `.[-1]`, Input: `[1, 2]`, Output: `2`},
{Name: "String Index", Selector: `.[0]`, Input: `"Hi"`, Output: `"H"`},
{Name: "Bytes Index", Selector: `.[0]`, Input: `{"/":{"bytes":"AAE"}}`, Output: `0`},
{Name: "Array Slice", Selector: `.[0:2]`, Input: `[0, 1, 2]`, Output: `[0, 1]`},
{Name: "Array Slice", Selector: `.[1:]`, Input: `[0, 1, 2]`, Output: `[1, 2]`},
@@ -52,35 +50,16 @@ func TestSupportedForms(t *testing.T) {
require.NoError(t, err)
// attempt to select
node, nodes, err := sel.Select(makeNode(t, tc.Input))
res, err := sel.Select(makeNode(t, tc.Input))
require.NoError(t, err)
require.NotEqual(t, node != nil, len(nodes) > 0) // XOR (only one of node or nodes should be set)
// make an IPLD List node from a []datamodel.Node
if node == nil {
nb := basicnode.Prototype.List.NewBuilder()
la, err := nb.BeginList(int64(len(nodes)))
require.NoError(t, err)
for _, n := range nodes {
// TODO: This code is probably not needed if the Select operation properly prunes nil values - e.g.: Optional Iterator
if n == nil {
n = datamodel.Null
}
require.NoError(t, la.AssembleValue().AssignNode(n))
}
require.NoError(t, la.Finish())
node = nb.Build()
}
require.NotNil(t, res)
exp := makeNode(t, tc.Output)
equalIPLD(t, exp, node)
equalIPLD(t, exp, res)
})
}
// null
// No error and return null, as optional
for _, testcase := range []Testcase{
{Name: "Optional Missing Key", Selector: `.x?`, Input: `{}`},
{Name: "Optional Null Key", Selector: `.x?`, Input: `null`},
@@ -97,19 +76,15 @@ func TestSupportedForms(t *testing.T) {
require.NoError(t, err)
// attempt to select
node, nodes, err := sel.Select(makeNode(t, tc.Input))
res, err := sel.Select(makeNode(t, tc.Input))
require.NoError(t, err)
// TODO: should Select return a single node which is sometimes a list or null?
// require.Equal(t, datamodel.Null, node)
assert.Nil(t, node)
assert.Empty(t, nodes)
require.Nil(t, res)
})
}
// error
// fail to select and return an error
for _, testcase := range []Testcase{
{Name: "Null Iterator", Selector: `.[]`, Input: `null`},
{Name: "Nested Iterator", Selector: `.[][]`, Input: `[[1], 2, [3]]`},
{Name: "Missing Key", Selector: `.x`, Input: `{}`},
{Name: "Null Key", Selector: `.x`, Input: `null`},
{Name: "Array Key", Selector: `.x`, Input: `[]`},
@@ -124,10 +99,10 @@ func TestSupportedForms(t *testing.T) {
require.NoError(t, err)
// attempt to select
node, nodes, err := sel.Select(makeNode(t, tc.Input))
res, err := sel.Select(makeNode(t, tc.Input))
require.Error(t, err)
assert.Nil(t, node)
assert.Empty(t, nodes)
require.True(t, selector.IsResolutionErr(err))
require.Nil(t, res)
})
}
}