go/ucan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

container: streamed and non-streamed IO, documentation

Browse Source
This commit is contained in:
Michael Muré
2024-11-21 15:49:29 +01:00
parent ba0038b0ae
commit 820057e41e
4 changed files with 134 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
did/didtest/crypto.go
View File

@@ -85,7 +85,11 @@ func (p Persona) Name() string {
// PrivKey returns the Ed25519 private key for the Persona.
func (p Persona) PrivKey() crypto.PrivKey {
return privKeys[p]
res, ok := privKeys[p]
if !ok {
panic(fmt.Sprintf("Unknown persona: %v", p))
}
return res
}
// PubKey returns the Ed25519 public key for the Persona.

79
pkg/container/reader.go
View File

@@ -1,6 +1,7 @@
package container
import (
"bytes"
"encoding/base64"
"errors"
"fmt"
@@ -71,33 +72,13 @@ func (ctn Reader) GetInvocation() (*invocation.Token, error) {
return nil, ErrNotFound
}
func FromCar(r io.Reader) (Reader, error) {
_, it, err := readCar(r)
if err != nil {
return nil, err
}
ctn := make(Reader)
for block, err := range it {
if err != nil {
return nil, err
}
err = ctn.addToken(block.data)
if err != nil {
return nil, err
}
}
return ctn, nil
// FromCbor decodes a DAG-CBOR encoded container.
func FromCbor(data []byte) (Reader, error) {
return FromCborReader(bytes.NewReader(data))
}
func FromCarBase64(r io.Reader) (Reader, error) {
return FromCar(base64.NewDecoder(base64.StdEncoding, r))
}
func FromCbor(r io.Reader) (Reader, error) {
// FromCborReader is the same as FromCbor, but with an io.Reader.
func FromCborReader(r io.Reader) (Reader, error) {
n, err := ipld.DecodeStreaming(r, dagcbor.Decode)
if err != nil {
return nil, err
@@ -147,8 +128,52 @@ func FromCbor(r io.Reader) (Reader, error) {
return ctn, nil
}
func FromCborBase64(r io.Reader) (Reader, error) {
return FromCbor(base64.NewDecoder(base64.StdEncoding, r))
// FromCborBase64 decodes a base64 DAG-CBOR encoded container.
func FromCborBase64(data []byte) (Reader, error) {
return FromCborBase64Reader(bytes.NewReader(data))
}
// FromCborBase64Reader is the same as FromCborBase64, but with an io.Reader.
func FromCborBase64Reader(r io.Reader) (Reader, error) {
return FromCborReader(base64.NewDecoder(base64.StdEncoding, r))
}
// FromCar decodes a CAR file encoded container.
func FromCar(data []byte) (Reader, error) {
return FromCarReader(bytes.NewReader(data))
}
// FromCarReader is the same as FromCar, but with an io.Reader.
func FromCarReader(r io.Reader) (Reader, error) {
_, it, err := readCar(r)
if err != nil {
return nil, err
}
ctn := make(Reader)
for block, err := range it {
if err != nil {
return nil, err
}
err = ctn.addToken(block.data)
if err != nil {
return nil, err
}
}
return ctn, nil
}
// FromCarBase64 decodes a base64 CAR file encoded container.
func FromCarBase64(data []byte) (Reader, error) {
return FromCarReader(bytes.NewReader(data))
}
// FromCarBase64Reader is the same as FromCarBase64, but with an io.Reader.
func FromCarBase64Reader(r io.Reader) (Reader, error) {
return FromCarReader(base64.NewDecoder(base64.StdEncoding, r))
}
func (ctn Reader) addToken(data []byte) error {

23
pkg/container/serial_test.go
View File

@@ -26,10 +26,10 @@ func TestContainerRoundTrip(t *testing.T) {
writer func(ctn Writer, w io.Writer) error
reader func(io.Reader) (Reader, error)
}{
{"car", Writer.ToCar, FromCar},
{"carBase64", Writer.ToCarBase64, FromCarBase64},
{"cbor", Writer.ToCbor, FromCbor},
{"cborBase64", Writer.ToCborBase64, FromCborBase64},
{"car", Writer.ToCarWriter, FromCarReader},
{"carBase64", Writer.ToCarBase64Writer, FromCarBase64Reader},
{"cbor", Writer.ToCborWriter, FromCborReader},
{"cborBase64", Writer.ToCborBase64Writer, FromCborBase64Reader},
} {
t.Run(tc.name, func(t *testing.T) {
tokens := make(map[cid.Cid]*delegation.Token)
@@ -98,10 +98,10 @@ func BenchmarkContainerSerialisation(b *testing.B) {
writer func(ctn Writer, w io.Writer) error
reader func(io.Reader) (Reader, error)
}{
{"car", Writer.ToCar, FromCar},
{"carBase64", Writer.ToCarBase64, FromCarBase64},
{"cbor", Writer.ToCbor, FromCbor},
{"cborBase64", Writer.ToCborBase64, FromCborBase64},
{"car", Writer.ToCarWriter, FromCarReader},
{"carBase64", Writer.ToCarBase64Writer, FromCarBase64Reader},
{"cbor", Writer.ToCborWriter, FromCborReader},
{"cborBase64", Writer.ToCborBase64Writer, FromCborBase64Reader},
} {
writer := NewWriter()
@@ -185,18 +185,17 @@ func FuzzContainerRead(f *testing.F) {
_, c, data := randToken()
writer.AddSealed(c, data)
}
buf := bytes.NewBuffer(nil)
err := writer.ToCbor(buf)
data, err := writer.ToCbor()
require.NoError(f, err)
f.Add(buf.Bytes())
f.Add(data)
}
f.Fuzz(func(t *testing.T, data []byte) {
start := time.Now()
// search for panics
_, _ = FromCbor(bytes.NewReader(data))
_, _ = FromCbor(data)
if time.Since(start) > 100*time.Millisecond {
panic("too long")

89
pkg/container/writer.go
View File

@@ -1,6 +1,7 @@
package container
import (
"bytes"
"encoding/base64"
"io"
@@ -12,10 +13,6 @@ import (
"github.com/ipld/go-ipld-prime/node/basicnode"
)
// TODO: should we have a multibase to wrap the cbor? but there is no reader/write in go-multibase :-(
const currentContainerVersion = "ctn-v1"
// Writer is a token container writer. It provides a convenient way to aggregate and serialize tokens together.
type Writer map[cid.Cid][]byte
@@ -28,27 +25,24 @@ func (ctn Writer) AddSealed(cid cid.Cid, data []byte) {
ctn[cid] = data
}
func (ctn Writer) ToCar(w io.Writer) error {
return writeCar(w, nil, func(yield func(carBlock, error) bool) {
for c, bytes := range ctn {
if !yield(carBlock{c: c, data: bytes}, nil) {
return
}
}
})
const currentContainerVersion = "ctn-v1"
// ToCbor encode the container into a DAG-CBOR binary format.
func (ctn Writer) ToCbor() ([]byte, error) {
var buf bytes.Buffer
err := ctn.ToCborWriter(&buf)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func (ctn Writer) ToCarBase64(w io.Writer) error {
w2 := base64.NewEncoder(base64.StdEncoding, w)
defer w2.Close()
return ctn.ToCar(w2)
}
func (ctn Writer) ToCbor(w io.Writer) error {
// ToCborWriter is the same as ToCbor, but with an io.Writer.
func (ctn Writer) ToCborWriter(w io.Writer) error {
node, err := qp.BuildMap(basicnode.Prototype.Any, 1, func(ma datamodel.MapAssembler) {
qp.MapEntry(ma, currentContainerVersion, qp.List(int64(len(ctn)), func(la datamodel.ListAssembler) {
for _, bytes := range ctn {
qp.ListEntry(la, qp.Bytes(bytes))
for _, data := range ctn {
qp.ListEntry(la, qp.Bytes(data))
}
}))
})
@@ -58,8 +52,57 @@ func (ctn Writer) ToCbor(w io.Writer) error {
return ipld.EncodeStreaming(w, node, dagcbor.Encode)
}
func (ctn Writer) ToCborBase64(w io.Writer) error {
// ToCborBase64 encode the container into a base64 encoded DAG-CBOR binary format.
func (ctn Writer) ToCborBase64() ([]byte, error) {
var buf bytes.Buffer
err := ctn.ToCborBase64Writer(&buf)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// ToCborBase64Writer is the same as ToCborBase64, but with an io.Writer.
func (ctn Writer) ToCborBase64Writer(w io.Writer) error {
w2 := base64.NewEncoder(base64.StdEncoding, w)
defer w2.Close()
return ctn.ToCbor(w2)
return ctn.ToCborWriter(w2)
}
// ToCar encode the container into a CAR file.
func (ctn Writer) ToCar() ([]byte, error) {
var buf bytes.Buffer
err := ctn.ToCarWriter(&buf)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// ToCarWriter is the same as ToCar, but with an io.Writer.
func (ctn Writer) ToCarWriter(w io.Writer) error {
return writeCar(w, nil, func(yield func(carBlock, error) bool) {
for c, data := range ctn {
if !yield(carBlock{c: c, data: data}, nil) {
return
}
}
})
}
// ToCarBase64 encode the container into a base64 encoded CAR file.
func (ctn Writer) ToCarBase64() ([]byte, error) {
var buf bytes.Buffer
err := ctn.ToCarBase64Writer(&buf)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// ToCarBase64Writer is the same as ToCarBase64, but with an io.Writer.
func (ctn Writer) ToCarBase64Writer(w io.Writer) error {
w2 := base64.NewEncoder(base64.StdEncoding, w)
defer w2.Close()
return ctn.ToCarWriter(w2)
}