Add SumStream to read data from io.Reader

This provides a way to create a CID without having to read all data into memory before passing it into Sum.  Instead, call SumStream passing in an io.Reader.

README.md

@@ -88,7 +88,7 @@ if c1.Equals(c2) {
 ```go
 // To check if some data matches a given cid, 
 // Get your CIDs prefix, and use that to sum the data in question:
-other, err := c.Prefix().Sum(mydata)
+other, err := c.Prefix().SumStream(myFile)
 if err != nil {...}
 
 if !c.Equals(other) {

builder.go

@@ -1,6 +1,8 @@
 package cid
 
 import (
+	"io"
+
 	mh "github.com/multiformats/go-multihash"
 )
 
@@ -41,6 +43,14 @@ func (p V0Builder) Sum(data []byte) (Cid, error) {
 	return Cid{string(hash)}, nil
 }
 
+func (p V0Builder) SumStream(r io.Reader) (Cid, error) {
+	hash, err := mh.SumStream(r, mh.SHA2_256, -1)
+	if err != nil {
+		return Undef, err
+	}
+	return Cid{string(hash)}, nil
+}
+
 func (p V0Builder) GetCodec() uint64 {
 	return DagProtobuf
 }
@@ -64,6 +74,18 @@ func (p V1Builder) Sum(data []byte) (Cid, error) {
 	return NewCidV1(p.Codec, hash), nil
 }
 
+func (p V1Builder) SumStream(r io.Reader) (Cid, error) {
+	mhLen := p.MhLength
+	if mhLen <= 0 {
+		mhLen = -1
+	}
+	hash, err := mh.SumStream(r, p.MhType, mhLen)
+	if err != nil {
+		return Undef, err
+	}
+	return NewCidV1(p.Codec, hash), nil
+}
+
 func (p V1Builder) GetCodec() uint64 {
 	return p.Codec
 }

builder_test.go

@@ -1,6 +1,7 @@
 package cid
 
 import (
+	"bytes"
 	"testing"
 
 	mh "github.com/multiformats/go-multihash"
@@ -16,6 +17,15 @@ func TestV0Builder(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	reader := bytes.NewReader(data)
+	c1a, err := format.SumStream(reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !c1a.Equals(c1) {
+		t.Fatal("Sum and SumStream create different cids")
+	}
+
 	// Construct c2
 	hash, err := mh.Sum(data, mh.SHA2_256, -1)
 	if err != nil {
@@ -29,6 +39,16 @@ func TestV0Builder(t *testing.T) {
 	if c1.Prefix() != c2.Prefix() {
 		t.Fatal("prefixes mismatch")
 	}
+
+	reader.Seek(0, 0)
+	hash, err = mh.SumStream(reader, mh.SHA2_256, -1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	c2 = NewCidV0(hash)
+	if !c1.Equals(c2) {
+		t.Fatal("Sum and SumStream create different cids")
+	}
 }
 
 func TestV1Builder(t *testing.T) {
@@ -41,6 +61,15 @@ func TestV1Builder(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	reader := bytes.NewReader(data)
+	c1a, err := format.SumStream(reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !c1a.Equals(c1) {
+		t.Fatal("Sum and SumStream create different cids")
+	}
+
 	// Construct c2
 	hash, err := mh.Sum(data, mh.SHA2_256, -1)
 	if err != nil {

cid.go

@@ -589,6 +589,36 @@ func (p Prefix) Sum(data []byte) (Cid, error) {
 	}
 }
 
+// SumStream uses the information in a prefix, and data read from a io.Reader,
+// to perform a multihash.SumStream() and return a newly constructed Cid with
+// the resulting multihash.
+func (p Prefix) SumStream(r io.Reader) (Cid, error) {
+	length := p.MhLength
+	if p.MhType == mh.ID {
+		length = -1
+	}
+
+	if p.Version == 0 && (p.MhType != mh.SHA2_256 ||
+		(p.MhLength != 32 && p.MhLength != -1)) {
+
+		return Undef, fmt.Errorf("invalid v0 prefix")
+	}
+
+	hash, err := mh.SumStream(r, p.MhType, length)
+	if err != nil {
+		return Undef, err
+	}
+
+	switch p.Version {
+	case 0:
+		return NewCidV0(hash), nil
+	case 1:
+		return NewCidV1(p.Codec, hash), nil
+	default:
+		return Undef, fmt.Errorf("invalid cid version")
+	}
+}
+
 // Bytes returns a byte representation of a Prefix. It looks like:
 //
 //     <version><codec><mh-type><mh-length>

go.mod

@@ -2,8 +2,8 @@ module github.com/ipfs/go-cid
 
 require (
 	github.com/multiformats/go-multibase v0.0.3
-	github.com/multiformats/go-multihash v0.0.14
+	github.com/multiformats/go-multihash v0.0.16-0.20210406191223-75ae3688857d
 	github.com/multiformats/go-varint v0.0.6
 )
 
-go 1.13
+go 1.15

go.sum

@@ -1,30 +1,29 @@
+github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw=
+github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
-github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo=
-github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
-github.com/mr-tron/base58 v1.1.0 h1:Y51FGVJ91WBqCEabAi5OPUz38eAx8DakuAm5svLcsfQ=
+github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
+github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
 github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
-github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc=
-github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
+github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
+github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
 github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI=
 github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
 github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4=
 github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM=
 github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk=
 github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc=
-github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc=
-github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
-github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
-github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg=
-github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
+github.com/multiformats/go-multihash v0.0.16-0.20210406191223-75ae3688857d h1:xUl/JcLVtafS0n5X8guAExxDS/mJUGoL31p/FlNtC8o=
+github.com/multiformats/go-multihash v0.0.16-0.20210406191223-75ae3688857d/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg=
+github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY=
 github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
-github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
-github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY=
+golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=