diff --git a/go.mod b/go.mod index ce6e58c..28cd9c6 100644 --- a/go.mod +++ b/go.mod @@ -2,12 +2,8 @@ module github.com/ucan-wg/go-ucan go 1.23 -// https://github.com/ipfs/go-ipld-cbor/pull/102 -replace github.com/ipfs/go-ipld-cbor => github.com/MichaelMure/go-ipld-cbor v0.0.0-20240918161052-74fa05e9e786 - require ( github.com/ipfs/go-cid v0.4.1 - github.com/ipfs/go-ipld-cbor v0.1.0 github.com/ipld/go-ipld-prime v0.21.0 github.com/libp2p/go-libp2p v0.36.3 github.com/multiformats/go-multibase v0.2.0 @@ -22,9 +18,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect github.com/google/go-cmp v0.5.9 // indirect - github.com/ipfs/go-block-format v0.1.2 // indirect - github.com/ipfs/go-ipfs-util v0.0.2 // indirect - github.com/ipfs/go-ipld-format v0.5.0 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mr-tron/base58 v1.2.0 // indirect @@ -33,10 +26,8 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.89.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25 // indirect golang.org/x/crypto v0.25.0 // indirect golang.org/x/sys v0.22.0 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 78f2e82..1d69a3a 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,4 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/MichaelMure/go-ipld-cbor v0.0.0-20240918161052-74fa05e9e786 h1:zKLMs9f7nmgEhu/JgIvcQ4zDRKczbTj3KXrVfOIQFd0= -github.com/MichaelMure/go-ipld-cbor v0.0.0-20240918161052-74fa05e9e786/go.mod h1:Cp8T7w1NKcu4AQJLqK0tWpd1nkgTxEVB5C6kVpLW6/0= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -11,21 +9,12 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+DyPUaJo= -github.com/ipfs/go-block-format v0.1.2/go.mod h1:mACVcrxarQKstUU3Yf/RdwbC4DzPV6++rO2a3d+a/KE= -github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= -github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= -github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= -github.com/ipfs/go-ipld-cbor v0.1.0/go.mod h1:U2aYlmVrJr2wsUBU67K4KgepApSZddGRDWBYR0H4sCk= -github.com/ipfs/go-ipld-format v0.5.0 h1:WyEle9K96MSrvr47zZHKKcDxJ/vlpET6PSiQsAFO+Ds= -github.com/ipfs/go-ipld-format v0.5.0/go.mod h1:ImdZqJQaEouMjCvqCe0ORUS+uoBmf7Hf+EO/jh+nk3M= github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E= github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -40,31 +29,22 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-libp2p v0.36.3 h1:NHz30+G7D8Y8YmznrVZZla0ofVANrvBl2c+oARfMeDQ= github.com/libp2p/go-libp2p v0.36.3/go.mod h1:4Y5vFyCUiJuluEPmpnKYf6WFx5ViKPUYs/ixe9ANFZ8= -github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= -github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= -github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= -github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= -github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= github.com/multiformats/go-multiaddr v0.13.0 h1:BCBzs61E3AGHcYYTv8dqRH43ZfyrqM8RXVPT8t13tLQ= github.com/multiformats/go-multiaddr v0.13.0/go.mod h1:sBXrNzucqkFJhvKOiwwLyqamGa/P5EIXNPLovyhQCII= -github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg= github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= -github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= -github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -86,26 +66,18 @@ github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8 github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= -github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25 h1:yVYDLoN2gmB3OdBXFW8e1UwgVbmCvNlnAKhvHPaNARI= -github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/container/Readme.md b/pkg/container/Readme.md new file mode 100644 index 0000000..176358b --- /dev/null +++ b/pkg/container/Readme.md @@ -0,0 +1,86 @@ +# Token container + +## Why do I need that? + +Some common situation asks to package multiple tokens together: +- calling a service requires sending an invocation, alongside the matching delegations +- sending a series of revocations +- \ + +The UCAN specification defines how a single token is serialized (envelope with signature, IPLD encoded as Dag-cbor), but it's entirely left open how to package multiple tokens together. To be clear, this is a correct thing to do for a specification, as different ways equally valid to solve that problem exists and can coexist. Any wire format holding a list of bytes would do (cbor, json, csv ...). + +**go-ucan** however, provide an opinionated implementation, which may or may not work in your situation. + +Some experiment has been done over which format is appropriate, and two have been selected: +- **DAG-CBOR** of a list of bytes, as a low overhead option +- **CAR** file, as a somewhat common ways to cary arbitrary blocks of data + +Notably, **compression is not included**, even though it does work reasonably well. This is because your transport medium might already do it, or should. + +## Wire format consideration + +Several possible formats have been explored: +- CAR files (binary or base64) +- DAG-CBOR (binary or base64) + +Additionally, gzip and deflate compression has been experimented with. + +Below are the results in terms of storage used, as percentage and byte overhead over the raw tokens: + +| Token count | car | carBase64 | carGzip | carGzipBase64 | cbor | cborBase64 | cborGzip | cborGzipBase64 | cborFlate | cborFlateBase64 | +|-------------|-----|-----------|---------|---------------|------|------------|----------|----------------|-----------|-----------------| +| 1 | 15 | 54 | 7 | 42 | 0 | 35 | \-8 | 22 | \-12 | 16 | +| 2 | 12 | 49 | \-12 | 15 | 0 | 34 | \-25 | 0 | \-28 | \-3 | +| 3 | 11 | 48 | \-21 | 4 | 0 | 34 | \-32 | \-10 | \-34 | \-11 | +| 4 | 10 | 47 | \-26 | \-1 | 0 | 34 | \-36 | \-15 | \-37 | \-17 | +| 5 | 10 | 47 | \-28 | \-4 | 0 | 34 | \-38 | \-18 | \-40 | \-20 | +| 6 | 10 | 47 | \-30 | \-7 | 0 | 34 | \-40 | \-20 | \-40 | \-20 | +| 7 | 10 | 46 | \-31 | \-8 | 0 | 34 | \-41 | \-21 | \-42 | \-22 | +| 8 | 9 | 46 | \-32 | \-10 | 0 | 34 | \-42 | \-22 | \-42 | \-23 | +| 9 | 9 | 46 | \-33 | \-11 | 0 | 34 | \-43 | \-23 | \-43 | \-24 | +| 10 | 9 | 46 | \-34 | \-12 | 0 | 34 | \-43 | \-25 | \-44 | \-25 | + +![Overhead %](img/overhead_percent.png) + +| Token count | car | carBase64 | carGzip | carGzipBase64 | cbor | cborBase64 | cborGzip | cborGzipBase64 | cborFlate | cborFlateBase64 | +|-------------|-----|-----------|---------|---------------|------|------------|----------|----------------|-----------|-----------------| +| 1 | 64 | 226 | 29 | 178 | 4 | 146 | \-35 | 94 | \-52 | 70 | +| 2 | 102 | 412 | \-107 | 128 | 7 | 288 | \-211 | 0 | \-234 | \-32 | +| 3 | 140 | 602 | \-270 | 58 | 10 | 430 | \-405 | \-126 | \-429 | \-146 | +| 4 | 178 | 792 | \-432 | \-28 | 13 | 572 | \-602 | \-252 | \-617 | \-288 | +| 5 | 216 | 978 | \-582 | \-94 | 16 | 714 | \-805 | \-386 | \-839 | \-418 | +| 6 | 254 | 1168 | \-759 | \-176 | 19 | 856 | \-1001 | \-508 | \-1018 | \-520 | +| 7 | 292 | 1358 | \-908 | \-246 | 22 | 998 | \-1204 | \-634 | \-1229 | \-650 | +| 8 | 330 | 1544 | \-1085 | \-332 | 25 | 1140 | \-1398 | \-756 | \-1423 | \-792 | +| 9 | 368 | 1734 | \-1257 | \-414 | 28 | 1282 | \-1614 | \-894 | \-1625 | \-930 | +| 10 | 406 | 1924 | \-1408 | \-508 | 31 | 1424 | \-1804 | \-1040 | \-1826 | \-1060 | + +![img.png](img/overhead_bytes.png) + +Following is the performance aspect, with CPU usage and memory allocation: + +| | Write ns/op | Read ns/op | Write B/op | Read B/op | Write allocs/op | Read allocs/op | +|-----------------|-------------|------------|------------|-----------|-----------------|----------------| +| car | 8451 | 1474630 | 17928 | 149437 | 59 | 2631 | +| carBase64 | 16750 | 1437678 | 24232 | 151502 | 61 | 2633 | +| carGzip | 320253 | 1581412 | 823887 | 192272 | 76 | 2665 | +| carGzipBase64 | 343305 | 1486269 | 828782 | 198543 | 77 | 2669 | +| cbor | 6419 | 1301554 | 16368 | 138891 | 25 | 2534 | +| cborBase64 | 12860 | 1386728 | 20720 | 140962 | 26 | 2536 | +| cborGzip | 310106 | 1379146 | 822742 | 182003 | 42 | 2585 | +| cborGzipBase64 | 317001 | 1462548 | 827640 | 189283 | 43 | 2594 | +| cborFlate | 327112 | 1555007 | 822473 | 181537 | 40 | 2591 | +| cborFlateBase64 | 311276 | 1456562 | 826042 | 188665 | 41 | 2596 | + +(BEWARE: logarithmic scale) + +![img.png](img/cpu.png) +![img_1.png](img/alloc_byte.png) +![img_2.png](img/alloc_count.png) + +Conclusion: +- CAR files are heavy for this usage, notably because they carry the CIDs of the tokens +- compression works quite well and warrants its usage even with a single token +- DAG-CBOR outperform CAR files everywhere, and comes with a tiny ~3 bytes per token overhead. + +**Formats beside DAG-CBOR and CAR, with or without base64, have been removed. They are in the git history though.** \ No newline at end of file diff --git a/pkg/container/car.go b/pkg/container/car.go index dcdd589..ee6d18c 100644 --- a/pkg/container/car.go +++ b/pkg/container/car.go @@ -9,7 +9,12 @@ import ( "iter" "github.com/ipfs/go-cid" - cbor "github.com/ipfs/go-ipld-cbor" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/codec/dagcbor" + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/fluent/qp" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/node/basicnode" ) /* @@ -40,7 +45,7 @@ func writeCar(w io.Writer, roots []cid.Cid, blocks iter.Seq[carBlock]) error { Roots: roots, Version: 1, } - hb, err := cbor.DumpObject(h) + hb, err := h.Write() if err != nil { return err } @@ -67,11 +72,10 @@ func readCar(r io.Reader) (roots []cid.Cid, blocks iter.Seq2[carBlock, error], e if err != nil { return nil, nil, err } - var h carHeader - if err := cbor.DecodeInto(hb, &h); err != nil { - return nil, nil, fmt.Errorf("invalid header: %v", err) + h, err := readHeader(hb) + if err != nil { + return nil, nil, err } - if h.Version != 1 { return nil, nil, fmt.Errorf("invalid car version: %d", h.Version) } @@ -183,6 +187,67 @@ type carHeader struct { Version uint64 } -func init() { - cbor.RegisterCborType(carHeader{}) +const rootsKey = "roots" +const versionKey = "version" + +func readHeader(data []byte) (*carHeader, error) { + var header carHeader + + nd, err := ipld.Decode(data, dagcbor.Decode) + if err != nil { + return nil, err + } + if nd.Length() != 2 { + return nil, fmt.Errorf("malformed car header") + } + rootsNd, err := nd.LookupByString(rootsKey) + if err != nil { + return nil, fmt.Errorf("malformed car header") + } + it := rootsNd.ListIterator() + if it == nil { + return nil, fmt.Errorf("malformed car header") + } + header.Roots = make([]cid.Cid, 0, rootsNd.Length()) + for !it.Done() { + _, nd, err := it.Next() + if err != nil { + return nil, err + } + lk, err := nd.AsLink() + if err != nil { + return nil, err + } + switch lk := lk.(type) { + case cidlink.Link: + header.Roots = append(header.Roots, lk.Cid) + default: + return nil, fmt.Errorf("malformed car header") + } + } + versionNd, err := nd.LookupByString(versionKey) + if err != nil { + return nil, fmt.Errorf("malformed car header") + } + version, err := versionNd.AsInt() + if err != nil { + return nil, fmt.Errorf("malformed car header") + } + header.Version = uint64(version) + return &header, nil +} + +func (ch *carHeader) Write() ([]byte, error) { + nd, err := qp.BuildMap(basicnode.Prototype.Any, 2, func(ma datamodel.MapAssembler) { + qp.MapEntry(ma, rootsKey, qp.List(int64(len(ch.Roots)), func(la datamodel.ListAssembler) { + for _, root := range ch.Roots { + qp.ListEntry(la, qp.Link(cidlink.Link{Cid: root})) + } + })) + qp.MapEntry(ma, versionKey, qp.Int(1)) + }) + if err != nil { + return nil, err + } + return ipld.Encode(nd, dagcbor.Encode) } diff --git a/pkg/container/car_test.go b/pkg/container/car_test.go index 711fc86..9b0ef31 100644 --- a/pkg/container/car_test.go +++ b/pkg/container/car_test.go @@ -38,3 +38,15 @@ func TestCarRoundTrip(t *testing.T) { // Bytes equal after the round-trip require.Equal(t, original, buf.Bytes()) } + +func FuzzCarRead(f *testing.F) { + example, err := os.ReadFile("testdata/sample-v1.car") + require.NoError(f, err) + + f.Add(example) + + f.Fuzz(func(t *testing.T, data []byte) { + _, _, _ = readCar(bytes.NewReader(data)) + // only looking for panics + }) +} diff --git a/pkg/container/img/alloc_byte.png b/pkg/container/img/alloc_byte.png new file mode 100644 index 0000000..ad4b54d Binary files /dev/null and b/pkg/container/img/alloc_byte.png differ diff --git a/pkg/container/img/alloc_count.png b/pkg/container/img/alloc_count.png new file mode 100644 index 0000000..5f4e1b1 Binary files /dev/null and b/pkg/container/img/alloc_count.png differ diff --git a/pkg/container/img/cpu.png b/pkg/container/img/cpu.png new file mode 100644 index 0000000..4b2c293 Binary files /dev/null and b/pkg/container/img/cpu.png differ diff --git a/pkg/container/img/overhead_bytes.png b/pkg/container/img/overhead_bytes.png new file mode 100644 index 0000000..c849787 Binary files /dev/null and b/pkg/container/img/overhead_bytes.png differ diff --git a/pkg/container/img/overhead_percent.png b/pkg/container/img/overhead_percent.png new file mode 100644 index 0000000..fa9dfc4 Binary files /dev/null and b/pkg/container/img/overhead_percent.png differ diff --git a/pkg/container/reader.go b/pkg/container/reader.go index 15f47b5..61402e4 100644 --- a/pkg/container/reader.go +++ b/pkg/container/reader.go @@ -1,14 +1,11 @@ package container import ( - "compress/flate" - "compress/gzip" "encoding/base64" "fmt" "io" "github.com/ipfs/go-cid" - cbor "github.com/ipfs/go-ipld-cbor" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/datamodel" @@ -20,8 +17,11 @@ import ( var ErrNotFound = fmt.Errorf("not found") +// Reader is a token container reader. It exposes the tokens conveniently decoded. type Reader map[cid.Cid]token.Token +// GetToken returns an arbitrary decoded token, from its CID. +// If not found, ErrNotFound is returned. func (ctn Reader) GetToken(cid cid.Cid) (token.Token, error) { tkn, ok := ctn[cid] if !ok { @@ -30,6 +30,7 @@ func (ctn Reader) GetToken(cid cid.Cid) (token.Token, error) { return tkn, nil } +// GetDelegation is the same as GetToken but only return a delegation.Token, with the right type. func (ctn Reader) GetDelegation(cid cid.Cid) (*delegation.Token, error) { tkn, err := ctn.GetToken(cid) if err != nil { @@ -41,6 +42,8 @@ func (ctn Reader) GetDelegation(cid cid.Cid) (*delegation.Token, error) { return nil, fmt.Errorf("not a delegation token") } +// GetInvocation returns the first found invocation.Token. +// If none are found, ErrNotFound is returned. func (ctn Reader) GetInvocation() (*invocation.Token, error) { for _, t := range ctn { if inv, ok := t.(*invocation.Token); ok { @@ -76,38 +79,7 @@ func FromCarBase64(r io.Reader) (Reader, error) { return FromCar(base64.NewDecoder(base64.StdEncoding, r)) } -func FromCarGzip(r io.Reader) (Reader, error) { - r2, err := gzip.NewReader(r) - if err != nil { - return nil, err - } - defer r2.Close() - return FromCar(r2) -} - -func FromCarGzipBase64(r io.Reader) (Reader, error) { - return FromCarGzip(base64.NewDecoder(base64.StdEncoding, r)) -} - func FromCbor(r io.Reader) (Reader, error) { - var raw [][]byte - err := cbor.DecodeReader(r, &raw) - if err != nil { - return nil, err - } - - ctn := make(Reader, len(raw)) - for _, data := range raw { - err = ctn.addToken(data) - if err != nil { - return nil, err - } - } - - return ctn, nil -} - -func FromCbor2(r io.Reader) (Reader, error) { n, err := ipld.DecodeStreaming(r, dagcbor.Decode) if err != nil { return nil, err @@ -140,29 +112,6 @@ func FromCborBase64(r io.Reader) (Reader, error) { return FromCbor(base64.NewDecoder(base64.StdEncoding, r)) } -func FromCborGzip(r io.Reader) (Reader, error) { - r2, err := gzip.NewReader(r) - if err != nil { - return nil, err - } - defer r2.Close() - return FromCbor(r2) -} - -func FromCborGzipBase64(r io.Reader) (Reader, error) { - return FromCborGzip(base64.NewDecoder(base64.StdEncoding, r)) -} - -func FromCborFlate(r io.Reader) (Reader, error) { - r2 := flate.NewReader(r) - defer r2.Close() - return FromCbor(r2) -} - -func FromCborFlateBase64(r io.Reader) (Reader, error) { - return FromCborFlate(base64.NewDecoder(base64.StdEncoding, r)) -} - func (ctn Reader) addToken(data []byte) error { tkn, c, err := token.FromSealed(data) if err != nil { diff --git a/pkg/container/serial_test.go b/pkg/container/serial_test.go index 76e8d74..6552f88 100644 --- a/pkg/container/serial_test.go +++ b/pkg/container/serial_test.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "fmt" "io" + "strings" "testing" "time" @@ -28,15 +29,8 @@ func TestContainerRoundTrip(t *testing.T) { }{ {"car", Writer.ToCar, FromCar}, {"carBase64", Writer.ToCarBase64, FromCarBase64}, - {"carGzip", Writer.ToCarGzip, FromCarGzip}, - {"carGzipBase64", Writer.ToCarGzipBase64, FromCarGzipBase64}, {"cbor", Writer.ToCbor, FromCbor}, {"cborBase64", Writer.ToCborBase64, FromCborBase64}, - {"cborGzip", Writer.ToCborGzip, FromCborGzip}, - {"cborGzipBase64", Writer.ToCborGzipBase64, FromCborGzipBase64}, - {"cborFlate", Writer.ToCborFlate, FromCborFlate}, - {"cborFlateBase64", Writer.ToCborFlateBase64, FromCborFlateBase64}, - {"cbor2", Writer.ToCbor2, FromCbor2}, } { t.Run(tc.name, func(t *testing.T) { tokens := make(map[cid.Cid]*delegation.Token) @@ -92,6 +86,14 @@ func TestContainerRoundTrip(t *testing.T) { } func BenchmarkContainerSerialisation(b *testing.B) { + var duration strings.Builder + var allocByte strings.Builder + var allocCount strings.Builder + + for _, builder := range []strings.Builder{duration, allocByte, allocCount} { + builder.WriteString("car\tcarBase64\tcarGzip\tcarGzipBase64\tcbor\tcborBase64\tcborGzip\tcborGzipBase64\tcborFlate\tcborFlateBase64\n") + } + for _, tc := range []struct { name string writer func(ctn Writer, w io.Writer) error @@ -99,15 +101,8 @@ func BenchmarkContainerSerialisation(b *testing.B) { }{ {"car", Writer.ToCar, FromCar}, {"carBase64", Writer.ToCarBase64, FromCarBase64}, - {"carGzip", Writer.ToCarGzip, FromCarGzip}, - {"carGzipBase64", Writer.ToCarGzipBase64, FromCarGzipBase64}, {"cbor", Writer.ToCbor, FromCbor}, {"cborBase64", Writer.ToCborBase64, FromCborBase64}, - {"cborGzip", Writer.ToCborGzip, FromCborGzip}, - {"cborGzipBase64", Writer.ToCborGzipBase64, FromCborGzipBase64}, - {"cborFlate", Writer.ToCborFlate, FromCborFlate}, - {"cborFlateBase64", Writer.ToCborFlateBase64, FromCborFlateBase64}, - {"cbor2", Writer.ToCbor2, FromCbor2}, } { writer := NewWriter() diff --git a/pkg/container/writer.go b/pkg/container/writer.go index 28a9dc1..cec6675 100644 --- a/pkg/container/writer.go +++ b/pkg/container/writer.go @@ -1,13 +1,10 @@ package container import ( - "compress/flate" - "compress/gzip" "encoding/base64" "io" "github.com/ipfs/go-cid" - cbor "github.com/ipfs/go-ipld-cbor" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/datamodel" @@ -15,12 +12,16 @@ import ( "github.com/ipld/go-ipld-prime/node/basicnode" ) +// TODO: should we have a multibase to wrap the cbor? but there is no reader/write in go-multibase :-( + +// Writer is a token container writer. It provides a convenient way to aggregate and serialize tokens together. type Writer map[cid.Cid][]byte func NewWriter() Writer { return make(Writer) } +// AddSealed includes a "sealed" token (serialized with a ToSealed* function) in the container. func (ctn Writer) AddSealed(cid cid.Cid, data []byte) { ctn[cid] = data } @@ -41,19 +42,7 @@ func (ctn Writer) ToCarBase64(w io.Writer) error { return ctn.ToCar(w2) } -func (ctn Writer) ToCarGzip(w io.Writer) error { - w2 := gzip.NewWriter(w) - defer w2.Close() - return ctn.ToCar(w2) -} - -func (ctn Writer) ToCarGzipBase64(w io.Writer) error { - w2 := base64.NewEncoder(base64.StdEncoding, w) - defer w2.Close() - return ctn.ToCarGzip(w2) -} - -func (ctn Writer) ToCbor2(w io.Writer) error { +func (ctn Writer) ToCbor(w io.Writer) error { node, err := qp.BuildList(basicnode.Prototype.Any, int64(len(ctn)), func(la datamodel.ListAssembler) { for _, bytes := range ctn { qp.ListEntry(la, qp.Bytes(bytes)) @@ -65,43 +54,8 @@ func (ctn Writer) ToCbor2(w io.Writer) error { return ipld.EncodeStreaming(w, node, dagcbor.Encode) } -func (ctn Writer) ToCbor(w io.Writer) error { - raw := make([][]byte, 0, len(ctn)) - for _, bytes := range ctn { - raw = append(raw, bytes) - } - return cbor.EncodeWriter(raw, w) -} - func (ctn Writer) ToCborBase64(w io.Writer) error { w2 := base64.NewEncoder(base64.StdEncoding, w) defer w2.Close() return ctn.ToCbor(w2) } - -func (ctn Writer) ToCborGzip(w io.Writer) error { - w2 := gzip.NewWriter(w) - defer w2.Close() - return ctn.ToCbor(w2) -} - -func (ctn Writer) ToCborGzipBase64(w io.Writer) error { - w2 := base64.NewEncoder(base64.StdEncoding, w) - defer w2.Close() - return ctn.ToCborGzip(w2) -} - -func (ctn Writer) ToCborFlate(w io.Writer) error { - w2, err := flate.NewWriter(w, flate.DefaultCompression) - if err != nil { - return err - } - defer w2.Close() - return ctn.ToCbor(w2) -} - -func (ctn Writer) ToCborFlateBase64(w io.Writer) error { - w2 := base64.NewEncoder(base64.StdEncoding, w) - defer w2.Close() - return ctn.ToCborFlate(w2) -}