add a token container with serialization as CARv1 file
This commit is contained in:
Michael Muré
11
go.mod
11
go.mod
@@ -1,11 +1,10 @@
|
||||
module github.com/ucan-wg/go-ucan
|
||||
|
||||
go 1.22
|
||||
|
||||
toolchain go1.22.4
|
||||
go 1.23
|
||||
|
||||
require (
|
||||
github.com/ipfs/go-cid v0.4.1
|
||||
github.com/ipfs/go-ipld-cbor v0.1.0
|
||||
github.com/ipld/go-ipld-prime v0.21.0
|
||||
github.com/libp2p/go-libp2p v0.36.3
|
||||
github.com/multiformats/go-multibase v0.2.0
|
||||
@@ -20,6 +19,9 @@ require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
|
||||
github.com/google/go-cmp v0.5.9 // indirect
|
||||
github.com/ipfs/go-block-format v0.1.2 // indirect
|
||||
github.com/ipfs/go-ipfs-util v0.0.2 // indirect
|
||||
github.com/ipfs/go-ipld-format v0.5.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
|
||||
github.com/minio/sha256-simd v1.0.1 // indirect
|
||||
github.com/mr-tron/base58 v1.2.0 // indirect
|
||||
@@ -28,9 +30,12 @@ require (
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/polydawn/refmt v0.89.0 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25 // indirect
|
||||
golang.org/x/crypto v0.25.0 // indirect
|
||||
golang.org/x/sys v0.22.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
||||
google.golang.org/protobuf v1.34.2 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
lukechampine.com/blake3 v1.3.0 // indirect
|
||||
)
|
||||
|
||||
30
go.sum
30
go.sum
@@ -9,12 +9,22 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+DyPUaJo=
|
||||
github.com/ipfs/go-block-format v0.1.2/go.mod h1:mACVcrxarQKstUU3Yf/RdwbC4DzPV6++rO2a3d+a/KE=
|
||||
github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I=
|
||||
github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s=
|
||||
github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk=
|
||||
github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8=
|
||||
github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ=
|
||||
github.com/ipfs/go-ipld-cbor v0.1.0 h1:dx0nS0kILVivGhfWuB6dUpMa/LAwElHPw1yOGYopoYs=
|
||||
github.com/ipfs/go-ipld-cbor v0.1.0/go.mod h1:U2aYlmVrJr2wsUBU67K4KgepApSZddGRDWBYR0H4sCk=
|
||||
github.com/ipfs/go-ipld-format v0.5.0 h1:WyEle9K96MSrvr47zZHKKcDxJ/vlpET6PSiQsAFO+Ds=
|
||||
github.com/ipfs/go-ipld-format v0.5.0/go.mod h1:ImdZqJQaEouMjCvqCe0ORUS+uoBmf7Hf+EO/jh+nk3M=
|
||||
github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E=
|
||||
github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ=
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
@@ -29,22 +39,31 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6
|
||||
github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
|
||||
github.com/libp2p/go-libp2p v0.36.3 h1:NHz30+G7D8Y8YmznrVZZla0ofVANrvBl2c+oARfMeDQ=
|
||||
github.com/libp2p/go-libp2p v0.36.3/go.mod h1:4Y5vFyCUiJuluEPmpnKYf6WFx5ViKPUYs/ixe9ANFZ8=
|
||||
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
|
||||
github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
|
||||
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
|
||||
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
|
||||
github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
|
||||
github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
|
||||
github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
|
||||
github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
|
||||
github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
|
||||
github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE=
|
||||
github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI=
|
||||
github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM=
|
||||
github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0=
|
||||
github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4=
|
||||
github.com/multiformats/go-multiaddr v0.13.0 h1:BCBzs61E3AGHcYYTv8dqRH43ZfyrqM8RXVPT8t13tLQ=
|
||||
github.com/multiformats/go-multiaddr v0.13.0/go.mod h1:sBXrNzucqkFJhvKOiwwLyqamGa/P5EIXNPLovyhQCII=
|
||||
github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc=
|
||||
github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g=
|
||||
github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk=
|
||||
github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg=
|
||||
github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k=
|
||||
github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
|
||||
github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U=
|
||||
github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM=
|
||||
github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
|
||||
github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8=
|
||||
github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
@@ -66,22 +85,31 @@ github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8
|
||||
github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||
github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ=
|
||||
github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
|
||||
github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25 h1:yVYDLoN2gmB3OdBXFW8e1UwgVbmCvNlnAKhvHPaNARI=
|
||||
github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
|
||||
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
|
||||
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
|
||||
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
|
||||
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
|
||||
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
|
||||
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
|
||||
186
pkg/container/car.go
Normal file
186
pkg/container/car.go
Normal file
@@ -0,0 +1,186 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"iter"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
cbor "github.com/ipfs/go-ipld-cbor"
|
||||
)
|
||||
|
||||
/*
|
||||
Note: below is essentially a re-implementation of the CAR file v1 read and write.
|
||||
This exists here for two reasons:
|
||||
- go-car's API forces to go through an IPLD getter or through a blockstore API
|
||||
- generally, go-car is a very complex and large dependency
|
||||
*/
|
||||
|
||||
// EmptyCid is a "zero" Cid: zero-length "identity" multihash with "raw" codec
|
||||
// It can be used to have at least one root in a CARv1 file (making it legal), yet
|
||||
// denote that it can be ignored.
|
||||
var EmptyCid = cid.MustParse([]byte{01, 55, 00, 00})
|
||||
|
||||
type carBlock struct {
|
||||
c cid.Cid
|
||||
data []byte
|
||||
}
|
||||
|
||||
// writeCar writes a CARv1 file with no roots, containing the blocks from the iterator.
|
||||
func writeCar(w io.Writer, roots []cid.Cid, blocks iter.Seq[carBlock]) error {
|
||||
if len(roots) == 0 {
|
||||
roots = []cid.Cid{EmptyCid}
|
||||
}
|
||||
h := carHeader{
|
||||
Roots: roots,
|
||||
Version: 1,
|
||||
}
|
||||
hb, err := cbor.DumpObject(h)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = ldWrite(w, hb)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for block := range blocks {
|
||||
err = ldWrite(w, block.c.Bytes(), block.data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// readCar reads a CARv1 file from the reader, and return a block iterator.
|
||||
// Roots are ignored.
|
||||
func readCar(r io.Reader) (roots []cid.Cid, blocks iter.Seq2[carBlock, error], err error) {
|
||||
br := bufio.NewReader(r)
|
||||
|
||||
hb, err := ldRead(br)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var h carHeader
|
||||
if err := cbor.DecodeInto(hb, &h); err != nil {
|
||||
return nil, nil, fmt.Errorf("invalid header: %v", err)
|
||||
}
|
||||
|
||||
if h.Version != 1 {
|
||||
return nil, nil, fmt.Errorf("invalid car version: %d", h.Version)
|
||||
}
|
||||
|
||||
return h.Roots, func(yield func(block carBlock, err error) bool) {
|
||||
for {
|
||||
block, err := readBlock(br)
|
||||
if err == io.EOF {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
if !yield(carBlock{}, err) {
|
||||
return
|
||||
}
|
||||
}
|
||||
if !yield(block, nil) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
// readBlock reads a section from the reader and decode a (cid+data) block.
|
||||
func readBlock(r *bufio.Reader) (carBlock, error) {
|
||||
raw, err := ldRead(r)
|
||||
if err != nil {
|
||||
return carBlock{}, err
|
||||
}
|
||||
|
||||
n, c, err := cid.CidFromReader(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
return carBlock{}, err
|
||||
}
|
||||
data := raw[n:]
|
||||
|
||||
// integrity check
|
||||
hashed, err := c.Prefix().Sum(data)
|
||||
if err != nil {
|
||||
return carBlock{}, err
|
||||
}
|
||||
|
||||
if !hashed.Equals(c) {
|
||||
return carBlock{}, fmt.Errorf("mismatch in content integrity, name: %s, data: %s", c, hashed)
|
||||
}
|
||||
|
||||
return carBlock{c: c, data: data}, nil
|
||||
}
|
||||
|
||||
// maxAllowedSectionSize dictates the maximum number of bytes that a CARv1 header
|
||||
// or section is allowed to occupy without causing a decode to error.
|
||||
// This cannot be supplied as an option, only adjusted as a global. You should
|
||||
// use v2#NewReader instead since it allows for options to be passed in.
|
||||
var maxAllowedSectionSize uint = 32 << 20 // 32MiB
|
||||
|
||||
// ldRead performs a length-delimited read of a section from the reader.
|
||||
// A section is composed of an uint length followed by the data.
|
||||
func ldRead(r *bufio.Reader) ([]byte, error) {
|
||||
if _, err := r.Peek(1); err != nil { // no more blocks, likely clean io.EOF
|
||||
return nil, err
|
||||
}
|
||||
|
||||
l, err := binary.ReadUvarint(r)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
return nil, io.ErrUnexpectedEOF // don't silently pretend this is a clean EOF
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if l > uint64(maxAllowedSectionSize) { // Don't OOM
|
||||
return nil, fmt.Errorf("malformed car; header is bigger than MaxAllowedSectionSize")
|
||||
}
|
||||
|
||||
buf := make([]byte, l)
|
||||
if _, err := io.ReadFull(r, buf); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
// ldWrite performs a length-delimited write of a section on the writer.
|
||||
// A section is composed of an uint length followed by the data.
|
||||
func ldWrite(w io.Writer, d ...[]byte) error {
|
||||
var sum uint64
|
||||
for _, s := range d {
|
||||
sum += uint64(len(s))
|
||||
}
|
||||
|
||||
buf := make([]byte, 8)
|
||||
n := binary.PutUvarint(buf, sum)
|
||||
_, err := w.Write(buf[:n])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, s := range d {
|
||||
_, err = w.Write(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type carHeader struct {
|
||||
Roots []cid.Cid
|
||||
Version uint64
|
||||
}
|
||||
|
||||
func init() {
|
||||
cbor.RegisterCborType(carHeader{})
|
||||
}
|
||||
40
pkg/container/car_test.go
Normal file
40
pkg/container/car_test.go
Normal file
@@ -0,0 +1,40 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCarRoundTrip(t *testing.T) {
|
||||
// this car file is a complex and legal CARv1 file
|
||||
original, err := os.ReadFile("testdata/sample-v1.car")
|
||||
require.NoError(t, err)
|
||||
|
||||
roots, it, err := readCar(bytes.NewReader(original))
|
||||
require.NoError(t, err)
|
||||
|
||||
var blks []carBlock
|
||||
for blk, err := range it {
|
||||
require.NoError(t, err)
|
||||
blks = append(blks, blk)
|
||||
}
|
||||
|
||||
require.Len(t, blks, 1049)
|
||||
|
||||
buf := bytes.NewBuffer(nil)
|
||||
|
||||
err = writeCar(buf, roots, func(yield func(carBlock) bool) {
|
||||
for _, blk := range blks {
|
||||
if !yield(blk) {
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Bytes equal after the round-trip
|
||||
require.Equal(t, original, buf.Bytes())
|
||||
}
|
||||
63
pkg/container/container.go
Normal file
63
pkg/container/container.go
Normal file
@@ -0,0 +1,63 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"io"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
// TODO: should the invocation being set as root in the car file?
|
||||
|
||||
type Container map[cid.Cid][]byte
|
||||
|
||||
func New() Container {
|
||||
return make(Container)
|
||||
}
|
||||
|
||||
func FromCar(r io.Reader) (Container, error) {
|
||||
_, it, err := readCar(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
c := New()
|
||||
|
||||
for block, err := range it {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c[block.c] = block.data
|
||||
}
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func FromCarBase64(r io.Reader) (Container, error) {
|
||||
return FromCar(base64.NewDecoder(base64.StdEncoding, r))
|
||||
}
|
||||
|
||||
func (ctn Container) ToCar(w io.Writer) error {
|
||||
return writeCar(w, nil, func(yield func(carBlock) bool) {
|
||||
for c, bytes := range ctn {
|
||||
if !yield(carBlock{c: c, data: bytes}) {
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (ctn Container) ToCarBase64(w io.Writer) error {
|
||||
w2 := base64.NewEncoder(base64.StdEncoding, w)
|
||||
defer w2.Close()
|
||||
return ctn.ToCar(w2)
|
||||
}
|
||||
|
||||
func (ctn Container) AddBytes(cid cid.Cid, data []byte) {
|
||||
ctn[cid] = data
|
||||
}
|
||||
|
||||
func (ctn Container) GetBytes(cid cid.Cid) ([]byte, bool) {
|
||||
b, ok := ctn[cid]
|
||||
return b, ok
|
||||
}
|
||||
52
pkg/container/container_test.go
Normal file
52
pkg/container/container_test.go
Normal file
@@ -0,0 +1,52 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
mh "github.com/multiformats/go-multihash"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestContainerRoundTrip(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
writer func(ctn Container, w io.Writer) error
|
||||
reader func(io.Reader) (Container, error)
|
||||
}{
|
||||
{"carBytes", Container.ToCar, FromCar},
|
||||
{"carBase64", Container.ToCarBase64, FromCarBase64},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ctn := New()
|
||||
|
||||
builder := cid.V1Builder{Codec: cid.Raw, MhType: mh.SHA2_256}
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
data := randBytes(32)
|
||||
c, err := builder.Sum(data)
|
||||
require.NoError(t, err)
|
||||
ctn.AddBytes(c, data)
|
||||
}
|
||||
|
||||
buf := bytes.NewBuffer(nil)
|
||||
|
||||
err := tc.writer(ctn, buf)
|
||||
require.NoError(t, err)
|
||||
|
||||
ctn2, err := tc.reader(bytes.NewReader(buf.Bytes()))
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, ctn, ctn2)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func randBytes(n int) []byte {
|
||||
b := make([]byte, n)
|
||||
_, _ = rand.Read(b)
|
||||
return b
|
||||
}
|
||||
BIN
pkg/container/testdata/sample-v1.car
vendored
Normal file
BIN
pkg/container/testdata/sample-v1.car
vendored
Normal file
Binary file not shown.
Reference in New Issue
Block a user