From 346efbd31db989746c378c9bfb1bf9ee6b2b1051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Mur=C3=A9?= Date: Thu, 19 Sep 2024 21:26:42 +0200 Subject: [PATCH] container: add cbor serialisation --- go.mod | 3 ++ go.sum | 3 +- pkg/container/car.go | 4 ++- pkg/container/container.go | 54 ++++++++++++++++++++++++++++++--- pkg/container/container_test.go | 10 +++++- 5 files changed, 66 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 7059979..ce6e58c 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,9 @@ module github.com/ucan-wg/go-ucan go 1.23 +// https://github.com/ipfs/go-ipld-cbor/pull/102 +replace github.com/ipfs/go-ipld-cbor => github.com/MichaelMure/go-ipld-cbor v0.0.0-20240918161052-74fa05e9e786 + require ( github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-ipld-cbor v0.1.0 diff --git a/go.sum b/go.sum index a2ea228..78f2e82 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/MichaelMure/go-ipld-cbor v0.0.0-20240918161052-74fa05e9e786 h1:zKLMs9f7nmgEhu/JgIvcQ4zDRKczbTj3KXrVfOIQFd0= +github.com/MichaelMure/go-ipld-cbor v0.0.0-20240918161052-74fa05e9e786/go.mod h1:Cp8T7w1NKcu4AQJLqK0tWpd1nkgTxEVB5C6kVpLW6/0= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -21,7 +23,6 @@ github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= -github.com/ipfs/go-ipld-cbor v0.1.0 h1:dx0nS0kILVivGhfWuB6dUpMa/LAwElHPw1yOGYopoYs= github.com/ipfs/go-ipld-cbor v0.1.0/go.mod h1:U2aYlmVrJr2wsUBU67K4KgepApSZddGRDWBYR0H4sCk= github.com/ipfs/go-ipld-format v0.5.0 h1:WyEle9K96MSrvr47zZHKKcDxJ/vlpET6PSiQsAFO+Ds= github.com/ipfs/go-ipld-format v0.5.0/go.mod h1:ImdZqJQaEouMjCvqCe0ORUS+uoBmf7Hf+EO/jh+nk3M= diff --git a/pkg/container/car.go b/pkg/container/car.go index 677fe7d..dcdd589 100644 --- a/pkg/container/car.go +++ b/pkg/container/car.go @@ -29,7 +29,9 @@ type carBlock struct { data []byte } -// writeCar writes a CARv1 file with no roots, containing the blocks from the iterator. +// writeCar writes a CARv1 file containing the blocks from the iterator. +// If no roots are provided, a single EmptyCid is used as root to make the file +// spec compliant. func writeCar(w io.Writer, roots []cid.Cid, blocks iter.Seq[carBlock]) error { if len(roots) == 0 { roots = []cid.Cid{EmptyCid} diff --git a/pkg/container/container.go b/pkg/container/container.go index 6632a85..58e3f87 100644 --- a/pkg/container/container.go +++ b/pkg/container/container.go @@ -5,6 +5,8 @@ import ( "io" "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" + "github.com/multiformats/go-multihash" ) // TODO: should the invocation being set as root in the car file? @@ -37,6 +39,43 @@ func FromCarBase64(r io.Reader) (Container, error) { return FromCar(base64.NewDecoder(base64.StdEncoding, r)) } +func FromCbor(r io.Reader) (Container, error) { + var raw [][]byte + err := cbor.DecodeReader(r, &raw) + if err != nil { + return nil, err + } + + // TODO: the CID computation will likely be handled in the envelope + // TODO: the envelope should likely be able to deserialize arbitrary types based on the tag value + // TODO: the container should likely expose the decoded token, and have search methods (simple, but also DAG reconstruction, graph path search) + var cidBuilder = cid.V1Builder{Codec: cid.DagCBOR, MhType: multihash.SHA2_256} + + ctn := make(Container, len(raw)) + for _, bytes := range raw { + c, err := cidBuilder.Sum(bytes) + if err != nil { + return nil, err + } + ctn[c] = bytes + } + + return ctn, nil +} + +func FromCborBase64(r io.Reader) (Container, error) { + return FromCbor(base64.NewDecoder(base64.StdEncoding, r)) +} + +func (ctn Container) AddBytes(cid cid.Cid, data []byte) { + ctn[cid] = data +} + +func (ctn Container) GetBytes(cid cid.Cid) ([]byte, bool) { + b, ok := ctn[cid] + return b, ok +} + func (ctn Container) ToCar(w io.Writer) error { return writeCar(w, nil, func(yield func(carBlock) bool) { for c, bytes := range ctn { @@ -53,11 +92,16 @@ func (ctn Container) ToCarBase64(w io.Writer) error { return ctn.ToCar(w2) } -func (ctn Container) AddBytes(cid cid.Cid, data []byte) { - ctn[cid] = data +func (ctn Container) ToCbor(w io.Writer) error { + raw := make([][]byte, 0, len(ctn)) + for _, bytes := range ctn { + raw = append(raw, bytes) + } + return cbor.EncodeWriter(raw, w) } -func (ctn Container) GetBytes(cid cid.Cid) ([]byte, bool) { - b, ok := ctn[cid] - return b, ok +func (ctn Container) ToCborBase64(w io.Writer) error { + w2 := base64.NewEncoder(base64.StdEncoding, w) + defer w2.Close() + return ctn.ToCbor(w2) } diff --git a/pkg/container/container_test.go b/pkg/container/container_test.go index dfa7282..b294b43 100644 --- a/pkg/container/container_test.go +++ b/pkg/container/container_test.go @@ -19,17 +19,22 @@ func TestContainerRoundTrip(t *testing.T) { }{ {"carBytes", Container.ToCar, FromCar}, {"carBase64", Container.ToCarBase64, FromCarBase64}, + {"cbor", Container.ToCbor, FromCbor}, + {"cborBase64", Container.ToCborBase64, FromCborBase64}, } { t.Run(tc.name, func(t *testing.T) { ctn := New() - builder := cid.V1Builder{Codec: cid.Raw, MhType: mh.SHA2_256} + builder := cid.V1Builder{Codec: cid.DagCBOR, MhType: mh.SHA2_256} + + var dataSize int for i := 0; i < 10; i++ { data := randBytes(32) c, err := builder.Sum(data) require.NoError(t, err) ctn.AddBytes(c, data) + dataSize += len(data) } buf := bytes.NewBuffer(nil) @@ -37,6 +42,9 @@ func TestContainerRoundTrip(t *testing.T) { err := tc.writer(ctn, buf) require.NoError(t, err) + t.Logf("data size %d", dataSize) + t.Logf("container overhead: %d%%, %d bytes", int(float32(buf.Len()-dataSize)/float32(dataSize)*100.0), buf.Len()-dataSize) + ctn2, err := tc.reader(bytes.NewReader(buf.Bytes())) require.NoError(t, err)