feat: add non-dag codec variants (json/cbor)

We now CBOR, at least, in Filecoin when we want structured data that
can't have links.

.github/dependabot.yml

@@ -0,0 +1,8 @@
+version: 2
+updates:
+- package-ecosystem: gomod
+  directory: "/"
+  schedule:
+    interval: weekly
+    time: "11:00"
+  open-pull-requests-limit: 10

cid.go

@@ -81,6 +81,10 @@ const (
 	DagJSON     = 0x0129 // https://ipld.io/docs/codecs/known/dag-json/
 	Libp2pKey   = 0x72   // https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#peer-ids
 
+	// Non "dag" variants of common codecs
+	CBOR = 0x51   // Regular (arbitrary) CBOR.
+	JSON = 0x0200 // Regular (arbitrary) JSON.
+
 	// other
 	GitRaw                = 0x78
 	DagJOSE               = 0x85 // https://ipld.io/specs/codecs/dag-jose/spec/
@@ -456,7 +460,7 @@ func (c Cid) Equals(o Cid) bool {
 // UnmarshalJSON parses the JSON representation of a Cid.
 func (c *Cid) UnmarshalJSON(b []byte) error {
 	if len(b) < 2 {
-		return fmt.Errorf("invalid cid json blob")
+		return ErrInvalidCid{fmt.Errorf("invalid cid json blob")}
 	}
 	obj := struct {
 		CidTarget string `json:"/"`
@@ -464,7 +468,7 @@ func (c *Cid) UnmarshalJSON(b []byte) error {
 	objptr := &obj
 	err := json.Unmarshal(b, &objptr)
 	if err != nil {
-		return err
+		return ErrInvalidCid{err}
 	}
 	if objptr == nil {
 		*c = Cid{}
@@ -472,12 +476,12 @@ func (c *Cid) UnmarshalJSON(b []byte) error {
 	}
 
 	if obj.CidTarget == "" {
-		return fmt.Errorf("cid was incorrectly formatted")
+		return ErrInvalidCid{fmt.Errorf("cid was incorrectly formatted")}
 	}
 
 	out, err := Decode(obj.CidTarget)
 	if err != nil {
-		return err
+		return ErrInvalidCid{err}
 	}
 
 	*c = out
@@ -564,12 +568,12 @@ func (p Prefix) Sum(data []byte) (Cid, error) {
 	if p.Version == 0 && (p.MhType != mh.SHA2_256 ||
 		(p.MhLength != 32 && p.MhLength != -1)) {
 
-		return Undef, fmt.Errorf("invalid v0 prefix")
+		return Undef, ErrInvalidCid{fmt.Errorf("invalid v0 prefix")}
 	}
 
 	hash, err := mh.Sum(data, p.MhType, length)
 	if err != nil {
-		return Undef, err
+		return Undef, ErrInvalidCid{err}
 	}
 
 	switch p.Version {
@@ -578,7 +582,7 @@ func (p Prefix) Sum(data []byte) (Cid, error) {
 	case 1:
 		return NewCidV1(p.Codec, hash), nil
 	default:
-		return Undef, fmt.Errorf("invalid cid version")
+		return Undef, ErrInvalidCid{fmt.Errorf("invalid cid version")}
 	}
 }
 
@@ -608,22 +612,22 @@ func PrefixFromBytes(buf []byte) (Prefix, error) {
 	r := bytes.NewReader(buf)
 	vers, err := varint.ReadUvarint(r)
 	if err != nil {
-		return Prefix{}, err
+		return Prefix{}, ErrInvalidCid{err}
 	}
 
 	codec, err := varint.ReadUvarint(r)
 	if err != nil {
-		return Prefix{}, err
+		return Prefix{}, ErrInvalidCid{err}
 	}
 
 	mhtype, err := varint.ReadUvarint(r)
 	if err != nil {
-		return Prefix{}, err
+		return Prefix{}, ErrInvalidCid{err}
 	}
 
 	mhlen, err := varint.ReadUvarint(r)
 	if err != nil {
-		return Prefix{}, err
+		return Prefix{}, ErrInvalidCid{err}
 	}
 
 	return Prefix{
@@ -717,6 +721,9 @@ func (r *bufByteReader) ReadByte() (byte, error) {
 // It's recommended to supply a reader that buffers and implements io.ByteReader,
 // as CidFromReader has to do many single-byte reads to decode varints.
 // If the argument only implements io.Reader, single-byte Read calls are used instead.
+//
+// If the Reader is found to yield zero bytes, an io.EOF error is returned directly, in all
+// other error cases, an ErrInvalidCid, wrapping the original error, is returned.
 func CidFromReader(r io.Reader) (int, Cid, error) {
 	// 64 bytes is enough for any CIDv0,
 	// and it's enough for most CIDv1s in practice.
@@ -727,6 +734,11 @@ func CidFromReader(r io.Reader) (int, Cid, error) {
 	// The varint package wants a io.ByteReader, so we must wrap our io.Reader.
 	vers, err := varint.ReadUvarint(br)
 	if err != nil {
+		if err == io.EOF {
+			// First-byte read in ReadUvarint errors with io.EOF, so reader has no data.
+			// Subsequent reads with an EOF will return io.ErrUnexpectedEOF and be wrapped here.
+			return 0, Undef, err
+		}
 		return len(br.dst), Undef, ErrInvalidCid{err}
 	}
 

cid_test.go

@@ -163,6 +163,9 @@ func TestBasesMarshaling(t *testing.T) {
 	if err == nil {
 		t.Fatal("expected too-short error from ExtractEncoding")
 	}
+	if !errors.Is(err, ErrInvalidCid{}) {
+		t.Fatal("expected error to be ErrInvalidCid")
+	}
 	if ee != -1 {
 		t.Fatal("expected -1 from too-short ExtractEncoding")
 	}
@@ -379,6 +382,9 @@ func TestInvalidV0Prefix(t *testing.T) {
 		if err == nil {
 			t.Fatalf("should error (index %d)", i)
 		}
+		if !errors.Is(err, ErrInvalidCid{}) {
+			t.Fatal("expected error to be ErrInvalidCid")
+		}
 	}
 }
 
@@ -388,6 +394,9 @@ func TestBadPrefix(t *testing.T) {
 	if err == nil {
 		t.Fatalf("expected error on v3 prefix Sum")
 	}
+	if !errors.Is(err, ErrInvalidCid{}) {
+		t.Fatal("expected error to be ErrInvalidCid")
+	}
 }
 
 func TestPrefixRoundtrip(t *testing.T) {
@@ -424,18 +433,30 @@ func TestBadPrefixFromBytes(t *testing.T) {
 	if err == nil {
 		t.Fatal("expected error for bad byte 0")
 	}
+	if !errors.Is(err, ErrInvalidCid{}) {
+		t.Fatal("expected error to be ErrInvalidCid")
+	}
 	_, err = PrefixFromBytes([]byte{0x01, 0x80})
 	if err == nil {
 		t.Fatal("expected error for bad byte 1")
 	}
+	if !errors.Is(err, ErrInvalidCid{}) {
+		t.Fatal("expected error to be ErrInvalidCid")
+	}
 	_, err = PrefixFromBytes([]byte{0x01, 0x01, 0x80})
 	if err == nil {
 		t.Fatal("expected error for bad byte 2")
 	}
+	if !errors.Is(err, ErrInvalidCid{}) {
+		t.Fatal("expected error to be ErrInvalidCid")
+	}
 	_, err = PrefixFromBytes([]byte{0x01, 0x01, 0x01, 0x80})
 	if err == nil {
 		t.Fatal("expected error for bad byte 3")
 	}
+	if !errors.Is(err, ErrInvalidCid{}) {
+		t.Fatal("expected error to be ErrInvalidCid")
+	}
 }
 
 func Test16BytesVarint(t *testing.T) {
@@ -582,17 +603,29 @@ func TestJsonRoundTrip(t *testing.T) {
 		t.Fatal("cids not equal for Cid")
 	}
 
-	if err = actual2.UnmarshalJSON([]byte("1")); err == nil {
+	err = actual2.UnmarshalJSON([]byte("1"))
+	if err == nil {
 		t.Fatal("expected error for too-short JSON")
 	}
-
-	if err = actual2.UnmarshalJSON([]byte(`{"nope":"nope"}`)); err == nil {
-		t.Fatal("expected error for bad CID JSON")
+	if !errors.Is(err, ErrInvalidCid{}) {
+		t.Fatal("expected error to be ErrInvalidCid")
 	}
 
-	if err = actual2.UnmarshalJSON([]byte(`bad "" json!`)); err == nil {
+	err = actual2.UnmarshalJSON([]byte(`{"nope":"nope"}`))
+	if err == nil {
+		t.Fatal("expected error for bad CID JSON")
+	}
+	if !errors.Is(err, ErrInvalidCid{}) {
+		t.Fatal("expected error to be ErrInvalidCid")
+	}
+
+	err = actual2.UnmarshalJSON([]byte(`bad "" json!`))
+	if err == nil {
 		t.Fatal("expected error for bad JSON")
 	}
+	if !errors.Is(err, ErrInvalidCid{}) {
+		t.Fatal("expected error to be ErrInvalidCid")
+	}
 
 	var actual3 Cid
 	enc, err = actual3.MarshalJSON()
@@ -750,6 +783,30 @@ func TestBadCidInput(t *testing.T) {
 	}
 }
 
+func TestFromReaderNoData(t *testing.T) {
+	// Reading no data from io.Reader should return io.EOF, not ErrInvalidCid.
+	n, cid, err := CidFromReader(bytes.NewReader(nil))
+	if err != io.EOF {
+		t.Fatal("Expected io.EOF error")
+	}
+	if cid != Undef {
+		t.Fatal("Expected Undef CID")
+	}
+	if n != 0 {
+		t.Fatal("Expected 0 data")
+	}
+
+	// Read byte indicatiing more data to and check error is ErrInvalidCid.
+	_, _, err = CidFromReader(bytes.NewReader([]byte{0x80}))
+	if !errors.Is(err, ErrInvalidCid{}) {
+		t.Fatal("Expected ErrInvalidCid error")
+	}
+	// Check for expected wrapped error.
+	if !errors.Is(err, io.ErrUnexpectedEOF) {
+		t.Fatal("Expected error", io.ErrUnexpectedEOF)
+	}
+}
+
 func TestBadParse(t *testing.T) {
 	hash, err := mh.Sum([]byte("foobar"), mh.SHA3_256, -1)
 	if err != nil {
@@ -777,14 +834,79 @@ func TestLoggable(t *testing.T) {
 	}
 }
 
-func TestErrInvalidCid(t *testing.T) {
-	_, err := Decode("not-a-cid")
-	if err == nil {
-		t.Fatal("expected error")
-	}
-
-	is := errors.Is(err, ErrInvalidCid{})
-	if !is {
-		t.Fatal("expected error to be ErrInvalidCid")
+func TestErrInvalidCidIs(t *testing.T) {
+	for i, test := range []struct {
+		err    error
+		target error
+	}{
+		{&ErrInvalidCid{}, ErrInvalidCid{}},
+		{ErrInvalidCid{}, &ErrInvalidCid{}},
+		{ErrInvalidCid{}, ErrInvalidCid{}},
+		{&ErrInvalidCid{}, &ErrInvalidCid{}},
+	} {
+		if !errors.Is(test.err, test.target) {
+			t.Fatalf("expected error to be ErrInvalidCid, case %d", i)
+		}
 	}
 }
+
+func TestErrInvalidCid(t *testing.T) {
+	run := func(err error) {
+		if err == nil {
+			t.Fatal("expected error")
+		}
+
+		if !strings.HasPrefix(err.Error(), "invalid cid: ") {
+			t.Fatal(`expected error message to contain "invalid cid: "`)
+		}
+
+		is := errors.Is(err, ErrInvalidCid{})
+		if !is {
+			t.Fatal("expected error to be ErrInvalidCid")
+		}
+
+		if !errors.Is(err, &ErrInvalidCid{}) {
+			t.Fatal("expected error to be &ErrInvalidCid")
+		}
+	}
+
+	_, err := Decode("")
+	run(err)
+
+	_, err = Decode("not-a-cid")
+	run(err)
+
+	_, err = Decode("bafyInvalid")
+	run(err)
+
+	_, err = Decode("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zIII")
+	run(err)
+
+	_, err = Cast([]byte("invalid"))
+	run(err)
+
+	_, err = Parse("not-a-cid")
+	run(err)
+
+	_, err = Parse("bafyInvalid")
+	run(err)
+
+	_, err = Parse("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zIII")
+	run(err)
+
+	_, err = Parse(123)
+	run(err)
+
+	_, _, err = CidFromBytes([]byte("invalid"))
+	run(err)
+
+	_, err = Prefix{}.Sum([]byte("data"))
+	run(err)
+
+	_, err = PrefixFromBytes([]byte{0x80})
+	run(err)
+
+	_, err = ExtractEncoding("invalid ")
+	run(err)
+
+}

go.mod

@@ -1,20 +1,21 @@
 module github.com/ipfs/go-cid
 
 require (
-	github.com/multiformats/go-multibase v0.0.3
-	github.com/multiformats/go-multihash v0.0.15
-	github.com/multiformats/go-varint v0.0.6
+	github.com/multiformats/go-multibase v0.2.0
+	github.com/multiformats/go-multihash v0.2.1
+	github.com/multiformats/go-varint v0.0.7
 )
 
 require (
-	github.com/klauspost/cpuid/v2 v2.0.4 // indirect
-	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
+	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
 	github.com/minio/sha256-simd v1.0.0 // indirect
 	github.com/mr-tron/base58 v1.2.0 // indirect
 	github.com/multiformats/go-base32 v0.0.3 // indirect
 	github.com/multiformats/go-base36 v0.1.0 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	golang.org/x/crypto v0.1.0 // indirect
 	golang.org/x/sys v0.1.0 // indirect
+	lukechampine.com/blake3 v1.1.6 // indirect
 )
 
 go 1.19

go.sum

@@ -1,31 +1,25 @@
-github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw=
 github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
-github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
+github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
+github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
 github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
-github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
 github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
 github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
 github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI=
 github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
 github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4=
 github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM=
-github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk=
-github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc=
-github.com/multiformats/go-multihash v0.0.15 h1:hWOPdrNqDjwHDx82vsYGSDZNyktOJJ2dzZJzFkOV1jM=
-github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg=
-github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY=
-github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g=
+github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk=
+github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108=
+github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc=
+github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8=
+github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
+github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU=
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+lukechampine.com/blake3 v1.1.6 h1:H3cROdztr7RCfoaTpGZFQsrqvweFLrqS73j7L7cmR5c=
+lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA=

version.json

@@ -1,3 +1,3 @@
 {
-  "version": "v0.3.2"
+  "version": "v0.4.1"
 }