container: add readme, remove extra formats, remove go-ipld-cbor dependency
This commit is contained in:
Michael Muré
9
go.mod
9
go.mod
@@ -2,12 +2,8 @@ module github.com/ucan-wg/go-ucan
|
||||
|
||||
go 1.23
|
||||
|
||||
// https://github.com/ipfs/go-ipld-cbor/pull/102
|
||||
replace github.com/ipfs/go-ipld-cbor => github.com/MichaelMure/go-ipld-cbor v0.0.0-20240918161052-74fa05e9e786
|
||||
|
||||
require (
|
||||
github.com/ipfs/go-cid v0.4.1
|
||||
github.com/ipfs/go-ipld-cbor v0.1.0
|
||||
github.com/ipld/go-ipld-prime v0.21.0
|
||||
github.com/libp2p/go-libp2p v0.36.3
|
||||
github.com/multiformats/go-multibase v0.2.0
|
||||
@@ -22,9 +18,6 @@ require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
|
||||
github.com/google/go-cmp v0.5.9 // indirect
|
||||
github.com/ipfs/go-block-format v0.1.2 // indirect
|
||||
github.com/ipfs/go-ipfs-util v0.0.2 // indirect
|
||||
github.com/ipfs/go-ipld-format v0.5.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
|
||||
github.com/minio/sha256-simd v1.0.1 // indirect
|
||||
github.com/mr-tron/base58 v1.2.0 // indirect
|
||||
@@ -33,10 +26,8 @@ require (
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/polydawn/refmt v0.89.0 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25 // indirect
|
||||
golang.org/x/crypto v0.25.0 // indirect
|
||||
golang.org/x/sys v0.22.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
||||
google.golang.org/protobuf v1.34.2 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
|
||||
28
go.sum
28
go.sum
@@ -1,6 +1,4 @@
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/MichaelMure/go-ipld-cbor v0.0.0-20240918161052-74fa05e9e786 h1:zKLMs9f7nmgEhu/JgIvcQ4zDRKczbTj3KXrVfOIQFd0=
|
||||
github.com/MichaelMure/go-ipld-cbor v0.0.0-20240918161052-74fa05e9e786/go.mod h1:Cp8T7w1NKcu4AQJLqK0tWpd1nkgTxEVB5C6kVpLW6/0=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@@ -11,21 +9,12 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+DyPUaJo=
|
||||
github.com/ipfs/go-block-format v0.1.2/go.mod h1:mACVcrxarQKstUU3Yf/RdwbC4DzPV6++rO2a3d+a/KE=
|
||||
github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I=
|
||||
github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s=
|
||||
github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk=
|
||||
github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8=
|
||||
github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ=
|
||||
github.com/ipfs/go-ipld-cbor v0.1.0/go.mod h1:U2aYlmVrJr2wsUBU67K4KgepApSZddGRDWBYR0H4sCk=
|
||||
github.com/ipfs/go-ipld-format v0.5.0 h1:WyEle9K96MSrvr47zZHKKcDxJ/vlpET6PSiQsAFO+Ds=
|
||||
github.com/ipfs/go-ipld-format v0.5.0/go.mod h1:ImdZqJQaEouMjCvqCe0ORUS+uoBmf7Hf+EO/jh+nk3M=
|
||||
github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E=
|
||||
github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ=
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
@@ -40,31 +29,22 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6
|
||||
github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
|
||||
github.com/libp2p/go-libp2p v0.36.3 h1:NHz30+G7D8Y8YmznrVZZla0ofVANrvBl2c+oARfMeDQ=
|
||||
github.com/libp2p/go-libp2p v0.36.3/go.mod h1:4Y5vFyCUiJuluEPmpnKYf6WFx5ViKPUYs/ixe9ANFZ8=
|
||||
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
|
||||
github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
|
||||
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
|
||||
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
|
||||
github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
|
||||
github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
|
||||
github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
|
||||
github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
|
||||
github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
|
||||
github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE=
|
||||
github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI=
|
||||
github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM=
|
||||
github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0=
|
||||
github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4=
|
||||
github.com/multiformats/go-multiaddr v0.13.0 h1:BCBzs61E3AGHcYYTv8dqRH43ZfyrqM8RXVPT8t13tLQ=
|
||||
github.com/multiformats/go-multiaddr v0.13.0/go.mod h1:sBXrNzucqkFJhvKOiwwLyqamGa/P5EIXNPLovyhQCII=
|
||||
github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc=
|
||||
github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g=
|
||||
github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk=
|
||||
github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg=
|
||||
github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k=
|
||||
github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
|
||||
github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U=
|
||||
github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM=
|
||||
github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
|
||||
github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8=
|
||||
github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
@@ -86,26 +66,18 @@ github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8
|
||||
github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||
github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ=
|
||||
github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
|
||||
github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25 h1:yVYDLoN2gmB3OdBXFW8e1UwgVbmCvNlnAKhvHPaNARI=
|
||||
github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
|
||||
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
|
||||
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
|
||||
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
|
||||
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
|
||||
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
|
||||
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
|
||||
86
pkg/container/Readme.md
Normal file
86
pkg/container/Readme.md
Normal file
@@ -0,0 +1,86 @@
|
||||
# Token container
|
||||
|
||||
## Why do I need that?
|
||||
|
||||
Some common situation asks to package multiple tokens together:
|
||||
- calling a service requires sending an invocation, alongside the matching delegations
|
||||
- sending a series of revocations
|
||||
- \<insert your application specific scenario here>
|
||||
|
||||
The UCAN specification defines how a single token is serialized (envelope with signature, IPLD encoded as Dag-cbor), but it's entirely left open how to package multiple tokens together. To be clear, this is a correct thing to do for a specification, as different ways equally valid to solve that problem exists and can coexist. Any wire format holding a list of bytes would do (cbor, json, csv ...).
|
||||
|
||||
**go-ucan** however, provide an opinionated implementation, which may or may not work in your situation.
|
||||
|
||||
Some experiment has been done over which format is appropriate, and two have been selected:
|
||||
- **DAG-CBOR** of a list of bytes, as a low overhead option
|
||||
- **CAR** file, as a somewhat common ways to cary arbitrary blocks of data
|
||||
|
||||
Notably, **compression is not included**, even though it does work reasonably well. This is because your transport medium might already do it, or should.
|
||||
|
||||
## Wire format consideration
|
||||
|
||||
Several possible formats have been explored:
|
||||
- CAR files (binary or base64)
|
||||
- DAG-CBOR (binary or base64)
|
||||
|
||||
Additionally, gzip and deflate compression has been experimented with.
|
||||
|
||||
Below are the results in terms of storage used, as percentage and byte overhead over the raw tokens:
|
||||
|
||||
| Token count | car | carBase64 | carGzip | carGzipBase64 | cbor | cborBase64 | cborGzip | cborGzipBase64 | cborFlate | cborFlateBase64 |
|
||||
|-------------|-----|-----------|---------|---------------|------|------------|----------|----------------|-----------|-----------------|
|
||||
| 1 | 15 | 54 | 7 | 42 | 0 | 35 | \-8 | 22 | \-12 | 16 |
|
||||
| 2 | 12 | 49 | \-12 | 15 | 0 | 34 | \-25 | 0 | \-28 | \-3 |
|
||||
| 3 | 11 | 48 | \-21 | 4 | 0 | 34 | \-32 | \-10 | \-34 | \-11 |
|
||||
| 4 | 10 | 47 | \-26 | \-1 | 0 | 34 | \-36 | \-15 | \-37 | \-17 |
|
||||
| 5 | 10 | 47 | \-28 | \-4 | 0 | 34 | \-38 | \-18 | \-40 | \-20 |
|
||||
| 6 | 10 | 47 | \-30 | \-7 | 0 | 34 | \-40 | \-20 | \-40 | \-20 |
|
||||
| 7 | 10 | 46 | \-31 | \-8 | 0 | 34 | \-41 | \-21 | \-42 | \-22 |
|
||||
| 8 | 9 | 46 | \-32 | \-10 | 0 | 34 | \-42 | \-22 | \-42 | \-23 |
|
||||
| 9 | 9 | 46 | \-33 | \-11 | 0 | 34 | \-43 | \-23 | \-43 | \-24 |
|
||||
| 10 | 9 | 46 | \-34 | \-12 | 0 | 34 | \-43 | \-25 | \-44 | \-25 |
|
||||
|
||||
|
||||
|
||||
| Token count | car | carBase64 | carGzip | carGzipBase64 | cbor | cborBase64 | cborGzip | cborGzipBase64 | cborFlate | cborFlateBase64 |
|
||||
|-------------|-----|-----------|---------|---------------|------|------------|----------|----------------|-----------|-----------------|
|
||||
| 1 | 64 | 226 | 29 | 178 | 4 | 146 | \-35 | 94 | \-52 | 70 |
|
||||
| 2 | 102 | 412 | \-107 | 128 | 7 | 288 | \-211 | 0 | \-234 | \-32 |
|
||||
| 3 | 140 | 602 | \-270 | 58 | 10 | 430 | \-405 | \-126 | \-429 | \-146 |
|
||||
| 4 | 178 | 792 | \-432 | \-28 | 13 | 572 | \-602 | \-252 | \-617 | \-288 |
|
||||
| 5 | 216 | 978 | \-582 | \-94 | 16 | 714 | \-805 | \-386 | \-839 | \-418 |
|
||||
| 6 | 254 | 1168 | \-759 | \-176 | 19 | 856 | \-1001 | \-508 | \-1018 | \-520 |
|
||||
| 7 | 292 | 1358 | \-908 | \-246 | 22 | 998 | \-1204 | \-634 | \-1229 | \-650 |
|
||||
| 8 | 330 | 1544 | \-1085 | \-332 | 25 | 1140 | \-1398 | \-756 | \-1423 | \-792 |
|
||||
| 9 | 368 | 1734 | \-1257 | \-414 | 28 | 1282 | \-1614 | \-894 | \-1625 | \-930 |
|
||||
| 10 | 406 | 1924 | \-1408 | \-508 | 31 | 1424 | \-1804 | \-1040 | \-1826 | \-1060 |
|
||||
|
||||
|
||||
|
||||
Following is the performance aspect, with CPU usage and memory allocation:
|
||||
|
||||
| | Write ns/op | Read ns/op | Write B/op | Read B/op | Write allocs/op | Read allocs/op |
|
||||
|-----------------|-------------|------------|------------|-----------|-----------------|----------------|
|
||||
| car | 8451 | 1474630 | 17928 | 149437 | 59 | 2631 |
|
||||
| carBase64 | 16750 | 1437678 | 24232 | 151502 | 61 | 2633 |
|
||||
| carGzip | 320253 | 1581412 | 823887 | 192272 | 76 | 2665 |
|
||||
| carGzipBase64 | 343305 | 1486269 | 828782 | 198543 | 77 | 2669 |
|
||||
| cbor | 6419 | 1301554 | 16368 | 138891 | 25 | 2534 |
|
||||
| cborBase64 | 12860 | 1386728 | 20720 | 140962 | 26 | 2536 |
|
||||
| cborGzip | 310106 | 1379146 | 822742 | 182003 | 42 | 2585 |
|
||||
| cborGzipBase64 | 317001 | 1462548 | 827640 | 189283 | 43 | 2594 |
|
||||
| cborFlate | 327112 | 1555007 | 822473 | 181537 | 40 | 2591 |
|
||||
| cborFlateBase64 | 311276 | 1456562 | 826042 | 188665 | 41 | 2596 |
|
||||
|
||||
(BEWARE: logarithmic scale)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Conclusion:
|
||||
- CAR files are heavy for this usage, notably because they carry the CIDs of the tokens
|
||||
- compression works quite well and warrants its usage even with a single token
|
||||
- DAG-CBOR outperform CAR files everywhere, and comes with a tiny ~3 bytes per token overhead.
|
||||
|
||||
**Formats beside DAG-CBOR and CAR, with or without base64, have been removed. They are in the git history though.**
|
||||
@@ -9,7 +9,12 @@ import (
|
||||
"iter"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
cbor "github.com/ipfs/go-ipld-cbor"
|
||||
"github.com/ipld/go-ipld-prime"
|
||||
"github.com/ipld/go-ipld-prime/codec/dagcbor"
|
||||
"github.com/ipld/go-ipld-prime/datamodel"
|
||||
"github.com/ipld/go-ipld-prime/fluent/qp"
|
||||
cidlink "github.com/ipld/go-ipld-prime/linking/cid"
|
||||
"github.com/ipld/go-ipld-prime/node/basicnode"
|
||||
)
|
||||
|
||||
/*
|
||||
@@ -40,7 +45,7 @@ func writeCar(w io.Writer, roots []cid.Cid, blocks iter.Seq[carBlock]) error {
|
||||
Roots: roots,
|
||||
Version: 1,
|
||||
}
|
||||
hb, err := cbor.DumpObject(h)
|
||||
hb, err := h.Write()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -67,11 +72,10 @@ func readCar(r io.Reader) (roots []cid.Cid, blocks iter.Seq2[carBlock, error], e
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var h carHeader
|
||||
if err := cbor.DecodeInto(hb, &h); err != nil {
|
||||
return nil, nil, fmt.Errorf("invalid header: %v", err)
|
||||
h, err := readHeader(hb)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if h.Version != 1 {
|
||||
return nil, nil, fmt.Errorf("invalid car version: %d", h.Version)
|
||||
}
|
||||
@@ -183,6 +187,67 @@ type carHeader struct {
|
||||
Version uint64
|
||||
}
|
||||
|
||||
func init() {
|
||||
cbor.RegisterCborType(carHeader{})
|
||||
const rootsKey = "roots"
|
||||
const versionKey = "version"
|
||||
|
||||
func readHeader(data []byte) (*carHeader, error) {
|
||||
var header carHeader
|
||||
|
||||
nd, err := ipld.Decode(data, dagcbor.Decode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if nd.Length() != 2 {
|
||||
return nil, fmt.Errorf("malformed car header")
|
||||
}
|
||||
rootsNd, err := nd.LookupByString(rootsKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("malformed car header")
|
||||
}
|
||||
it := rootsNd.ListIterator()
|
||||
if it == nil {
|
||||
return nil, fmt.Errorf("malformed car header")
|
||||
}
|
||||
header.Roots = make([]cid.Cid, 0, rootsNd.Length())
|
||||
for !it.Done() {
|
||||
_, nd, err := it.Next()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lk, err := nd.AsLink()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch lk := lk.(type) {
|
||||
case cidlink.Link:
|
||||
header.Roots = append(header.Roots, lk.Cid)
|
||||
default:
|
||||
return nil, fmt.Errorf("malformed car header")
|
||||
}
|
||||
}
|
||||
versionNd, err := nd.LookupByString(versionKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("malformed car header")
|
||||
}
|
||||
version, err := versionNd.AsInt()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("malformed car header")
|
||||
}
|
||||
header.Version = uint64(version)
|
||||
return &header, nil
|
||||
}
|
||||
|
||||
func (ch *carHeader) Write() ([]byte, error) {
|
||||
nd, err := qp.BuildMap(basicnode.Prototype.Any, 2, func(ma datamodel.MapAssembler) {
|
||||
qp.MapEntry(ma, rootsKey, qp.List(int64(len(ch.Roots)), func(la datamodel.ListAssembler) {
|
||||
for _, root := range ch.Roots {
|
||||
qp.ListEntry(la, qp.Link(cidlink.Link{Cid: root}))
|
||||
}
|
||||
}))
|
||||
qp.MapEntry(ma, versionKey, qp.Int(1))
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ipld.Encode(nd, dagcbor.Encode)
|
||||
}
|
||||
|
||||
@@ -38,3 +38,15 @@ func TestCarRoundTrip(t *testing.T) {
|
||||
// Bytes equal after the round-trip
|
||||
require.Equal(t, original, buf.Bytes())
|
||||
}
|
||||
|
||||
func FuzzCarRead(f *testing.F) {
|
||||
example, err := os.ReadFile("testdata/sample-v1.car")
|
||||
require.NoError(f, err)
|
||||
|
||||
f.Add(example)
|
||||
|
||||
f.Fuzz(func(t *testing.T, data []byte) {
|
||||
_, _, _ = readCar(bytes.NewReader(data))
|
||||
// only looking for panics
|
||||
})
|
||||
}
|
||||
|
||||
BIN
pkg/container/img/alloc_byte.png
Normal file
BIN
pkg/container/img/alloc_byte.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 25 KiB |
BIN
pkg/container/img/alloc_count.png
Normal file
BIN
pkg/container/img/alloc_count.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 23 KiB |
BIN
pkg/container/img/cpu.png
Normal file
BIN
pkg/container/img/cpu.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 25 KiB |
BIN
pkg/container/img/overhead_bytes.png
Normal file
BIN
pkg/container/img/overhead_bytes.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 31 KiB |
BIN
pkg/container/img/overhead_percent.png
Normal file
BIN
pkg/container/img/overhead_percent.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 29 KiB |
@@ -1,14 +1,11 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"compress/flate"
|
||||
"compress/gzip"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
cbor "github.com/ipfs/go-ipld-cbor"
|
||||
"github.com/ipld/go-ipld-prime"
|
||||
"github.com/ipld/go-ipld-prime/codec/dagcbor"
|
||||
"github.com/ipld/go-ipld-prime/datamodel"
|
||||
@@ -20,8 +17,11 @@ import (
|
||||
|
||||
var ErrNotFound = fmt.Errorf("not found")
|
||||
|
||||
// Reader is a token container reader. It exposes the tokens conveniently decoded.
|
||||
type Reader map[cid.Cid]token.Token
|
||||
|
||||
// GetToken returns an arbitrary decoded token, from its CID.
|
||||
// If not found, ErrNotFound is returned.
|
||||
func (ctn Reader) GetToken(cid cid.Cid) (token.Token, error) {
|
||||
tkn, ok := ctn[cid]
|
||||
if !ok {
|
||||
@@ -30,6 +30,7 @@ func (ctn Reader) GetToken(cid cid.Cid) (token.Token, error) {
|
||||
return tkn, nil
|
||||
}
|
||||
|
||||
// GetDelegation is the same as GetToken but only return a delegation.Token, with the right type.
|
||||
func (ctn Reader) GetDelegation(cid cid.Cid) (*delegation.Token, error) {
|
||||
tkn, err := ctn.GetToken(cid)
|
||||
if err != nil {
|
||||
@@ -41,6 +42,8 @@ func (ctn Reader) GetDelegation(cid cid.Cid) (*delegation.Token, error) {
|
||||
return nil, fmt.Errorf("not a delegation token")
|
||||
}
|
||||
|
||||
// GetInvocation returns the first found invocation.Token.
|
||||
// If none are found, ErrNotFound is returned.
|
||||
func (ctn Reader) GetInvocation() (*invocation.Token, error) {
|
||||
for _, t := range ctn {
|
||||
if inv, ok := t.(*invocation.Token); ok {
|
||||
@@ -76,38 +79,7 @@ func FromCarBase64(r io.Reader) (Reader, error) {
|
||||
return FromCar(base64.NewDecoder(base64.StdEncoding, r))
|
||||
}
|
||||
|
||||
func FromCarGzip(r io.Reader) (Reader, error) {
|
||||
r2, err := gzip.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer r2.Close()
|
||||
return FromCar(r2)
|
||||
}
|
||||
|
||||
func FromCarGzipBase64(r io.Reader) (Reader, error) {
|
||||
return FromCarGzip(base64.NewDecoder(base64.StdEncoding, r))
|
||||
}
|
||||
|
||||
func FromCbor(r io.Reader) (Reader, error) {
|
||||
var raw [][]byte
|
||||
err := cbor.DecodeReader(r, &raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ctn := make(Reader, len(raw))
|
||||
for _, data := range raw {
|
||||
err = ctn.addToken(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return ctn, nil
|
||||
}
|
||||
|
||||
func FromCbor2(r io.Reader) (Reader, error) {
|
||||
n, err := ipld.DecodeStreaming(r, dagcbor.Decode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -140,29 +112,6 @@ func FromCborBase64(r io.Reader) (Reader, error) {
|
||||
return FromCbor(base64.NewDecoder(base64.StdEncoding, r))
|
||||
}
|
||||
|
||||
func FromCborGzip(r io.Reader) (Reader, error) {
|
||||
r2, err := gzip.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer r2.Close()
|
||||
return FromCbor(r2)
|
||||
}
|
||||
|
||||
func FromCborGzipBase64(r io.Reader) (Reader, error) {
|
||||
return FromCborGzip(base64.NewDecoder(base64.StdEncoding, r))
|
||||
}
|
||||
|
||||
func FromCborFlate(r io.Reader) (Reader, error) {
|
||||
r2 := flate.NewReader(r)
|
||||
defer r2.Close()
|
||||
return FromCbor(r2)
|
||||
}
|
||||
|
||||
func FromCborFlateBase64(r io.Reader) (Reader, error) {
|
||||
return FromCborFlate(base64.NewDecoder(base64.StdEncoding, r))
|
||||
}
|
||||
|
||||
func (ctn Reader) addToken(data []byte) error {
|
||||
tkn, c, err := token.FromSealed(data)
|
||||
if err != nil {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -28,15 +29,8 @@ func TestContainerRoundTrip(t *testing.T) {
|
||||
}{
|
||||
{"car", Writer.ToCar, FromCar},
|
||||
{"carBase64", Writer.ToCarBase64, FromCarBase64},
|
||||
{"carGzip", Writer.ToCarGzip, FromCarGzip},
|
||||
{"carGzipBase64", Writer.ToCarGzipBase64, FromCarGzipBase64},
|
||||
{"cbor", Writer.ToCbor, FromCbor},
|
||||
{"cborBase64", Writer.ToCborBase64, FromCborBase64},
|
||||
{"cborGzip", Writer.ToCborGzip, FromCborGzip},
|
||||
{"cborGzipBase64", Writer.ToCborGzipBase64, FromCborGzipBase64},
|
||||
{"cborFlate", Writer.ToCborFlate, FromCborFlate},
|
||||
{"cborFlateBase64", Writer.ToCborFlateBase64, FromCborFlateBase64},
|
||||
{"cbor2", Writer.ToCbor2, FromCbor2},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
tokens := make(map[cid.Cid]*delegation.Token)
|
||||
@@ -92,6 +86,14 @@ func TestContainerRoundTrip(t *testing.T) {
|
||||
}
|
||||
|
||||
func BenchmarkContainerSerialisation(b *testing.B) {
|
||||
var duration strings.Builder
|
||||
var allocByte strings.Builder
|
||||
var allocCount strings.Builder
|
||||
|
||||
for _, builder := range []strings.Builder{duration, allocByte, allocCount} {
|
||||
builder.WriteString("car\tcarBase64\tcarGzip\tcarGzipBase64\tcbor\tcborBase64\tcborGzip\tcborGzipBase64\tcborFlate\tcborFlateBase64\n")
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
writer func(ctn Writer, w io.Writer) error
|
||||
@@ -99,15 +101,8 @@ func BenchmarkContainerSerialisation(b *testing.B) {
|
||||
}{
|
||||
{"car", Writer.ToCar, FromCar},
|
||||
{"carBase64", Writer.ToCarBase64, FromCarBase64},
|
||||
{"carGzip", Writer.ToCarGzip, FromCarGzip},
|
||||
{"carGzipBase64", Writer.ToCarGzipBase64, FromCarGzipBase64},
|
||||
{"cbor", Writer.ToCbor, FromCbor},
|
||||
{"cborBase64", Writer.ToCborBase64, FromCborBase64},
|
||||
{"cborGzip", Writer.ToCborGzip, FromCborGzip},
|
||||
{"cborGzipBase64", Writer.ToCborGzipBase64, FromCborGzipBase64},
|
||||
{"cborFlate", Writer.ToCborFlate, FromCborFlate},
|
||||
{"cborFlateBase64", Writer.ToCborFlateBase64, FromCborFlateBase64},
|
||||
{"cbor2", Writer.ToCbor2, FromCbor2},
|
||||
} {
|
||||
writer := NewWriter()
|
||||
|
||||
|
||||
@@ -1,13 +1,10 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"compress/flate"
|
||||
"compress/gzip"
|
||||
"encoding/base64"
|
||||
"io"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
cbor "github.com/ipfs/go-ipld-cbor"
|
||||
"github.com/ipld/go-ipld-prime"
|
||||
"github.com/ipld/go-ipld-prime/codec/dagcbor"
|
||||
"github.com/ipld/go-ipld-prime/datamodel"
|
||||
@@ -15,12 +12,16 @@ import (
|
||||
"github.com/ipld/go-ipld-prime/node/basicnode"
|
||||
)
|
||||
|
||||
// TODO: should we have a multibase to wrap the cbor? but there is no reader/write in go-multibase :-(
|
||||
|
||||
// Writer is a token container writer. It provides a convenient way to aggregate and serialize tokens together.
|
||||
type Writer map[cid.Cid][]byte
|
||||
|
||||
func NewWriter() Writer {
|
||||
return make(Writer)
|
||||
}
|
||||
|
||||
// AddSealed includes a "sealed" token (serialized with a ToSealed* function) in the container.
|
||||
func (ctn Writer) AddSealed(cid cid.Cid, data []byte) {
|
||||
ctn[cid] = data
|
||||
}
|
||||
@@ -41,19 +42,7 @@ func (ctn Writer) ToCarBase64(w io.Writer) error {
|
||||
return ctn.ToCar(w2)
|
||||
}
|
||||
|
||||
func (ctn Writer) ToCarGzip(w io.Writer) error {
|
||||
w2 := gzip.NewWriter(w)
|
||||
defer w2.Close()
|
||||
return ctn.ToCar(w2)
|
||||
}
|
||||
|
||||
func (ctn Writer) ToCarGzipBase64(w io.Writer) error {
|
||||
w2 := base64.NewEncoder(base64.StdEncoding, w)
|
||||
defer w2.Close()
|
||||
return ctn.ToCarGzip(w2)
|
||||
}
|
||||
|
||||
func (ctn Writer) ToCbor2(w io.Writer) error {
|
||||
func (ctn Writer) ToCbor(w io.Writer) error {
|
||||
node, err := qp.BuildList(basicnode.Prototype.Any, int64(len(ctn)), func(la datamodel.ListAssembler) {
|
||||
for _, bytes := range ctn {
|
||||
qp.ListEntry(la, qp.Bytes(bytes))
|
||||
@@ -65,43 +54,8 @@ func (ctn Writer) ToCbor2(w io.Writer) error {
|
||||
return ipld.EncodeStreaming(w, node, dagcbor.Encode)
|
||||
}
|
||||
|
||||
func (ctn Writer) ToCbor(w io.Writer) error {
|
||||
raw := make([][]byte, 0, len(ctn))
|
||||
for _, bytes := range ctn {
|
||||
raw = append(raw, bytes)
|
||||
}
|
||||
return cbor.EncodeWriter(raw, w)
|
||||
}
|
||||
|
||||
func (ctn Writer) ToCborBase64(w io.Writer) error {
|
||||
w2 := base64.NewEncoder(base64.StdEncoding, w)
|
||||
defer w2.Close()
|
||||
return ctn.ToCbor(w2)
|
||||
}
|
||||
|
||||
func (ctn Writer) ToCborGzip(w io.Writer) error {
|
||||
w2 := gzip.NewWriter(w)
|
||||
defer w2.Close()
|
||||
return ctn.ToCbor(w2)
|
||||
}
|
||||
|
||||
func (ctn Writer) ToCborGzipBase64(w io.Writer) error {
|
||||
w2 := base64.NewEncoder(base64.StdEncoding, w)
|
||||
defer w2.Close()
|
||||
return ctn.ToCborGzip(w2)
|
||||
}
|
||||
|
||||
func (ctn Writer) ToCborFlate(w io.Writer) error {
|
||||
w2, err := flate.NewWriter(w, flate.DefaultCompression)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer w2.Close()
|
||||
return ctn.ToCbor(w2)
|
||||
}
|
||||
|
||||
func (ctn Writer) ToCborFlateBase64(w io.Writer) error {
|
||||
w2 := base64.NewEncoder(base64.StdEncoding, w)
|
||||
defer w2.Close()
|
||||
return ctn.ToCborFlate(w2)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user