Add SumStream to read data from io.Reader

This provides a way to create a CID without having to read all data into memory before passing it into Sum.  Instead, call SumStream passing in an io.Reader.

.github/workflows/automerge.yml

@@ -1,27 +0,0 @@
-# File managed by web3-bot. DO NOT EDIT.
-# See https://github.com/protocol/.github/ for details.
-
-# Automatically merge pull requests opened by web3-bot, as soon as (and only if) all tests pass.
-# This reduces the friction associated with updating with our workflows.
-
-on: [ pull_request ]
-
-jobs:
-  automerge:
-    if: github.event.pull_request.user.login == 'web3-bot'
-    runs-on: ubuntu-latest
-    steps:
-    - name: Wait on tests
-      uses: lewagon/wait-on-check-action@bafe56a6863672c681c3cf671f5e10b20abf2eaa # v0.2
-      with:
-        ref: ${{ github.event.pull_request.head.sha }}
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        wait-interval: 10
-        running-workflow-name: 'automerge' # the name of this job
-    - name: Merge PR
-      uses: pascalgn/automerge-action@741c311a47881be9625932b0a0de1b0937aab1ae # v0.13.1
-      env:
-        GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-        MERGE_LABELS: ""
-        MERGE_METHOD: "squash"
-        MERGE_DELETE_BRANCH: true

.github/workflows/go-check.yml

@@ -1,41 +0,0 @@
-# File managed by web3-bot. DO NOT EDIT.
-# See https://github.com/protocol/.github/ for details.
-
-on: [push, pull_request]
-
-jobs:
-  unit:
-    runs-on: ubuntu-latest
-    name: Go checks
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
-        with:
-          go-version: "1.16.x"
-      - name: Install staticcheck
-        run: go install honnef.co/go/tools/cmd/staticcheck@be534f007836a777104a15f2456cd1fffd3ddee8 # v2020.2.2
-      - name: Check that go.mod is tidy
-        run: |
-          go mod tidy
-          if [[ -n $(git ls-files --other --exclude-standard --directory -- go.sum) ]]; then
-            echo "go.sum was added by go mod tidy"
-            exit 1
-          fi
-          git diff --exit-code -- go.sum go.mod
-      - name: gofmt
-        if: ${{ success() || failure() }} # run this step even if the previous one failed
-        run: |
-          out=$(gofmt -s -l .)
-          if [[ -n "$out" ]]; then
-            echo $out | awk '{print "::error file=" $0 ",line=0,col=0::File is not gofmt-ed."}'
-            exit 1
-          fi
-      - name: go vet
-        if: ${{ success() || failure() }} # run this step even if the previous one failed
-        run: go vet ./...
-      - name: staticcheck
-        if: ${{ success() || failure() }} # run this step even if the previous one failed
-        run: |
-          set -o pipefail
-          staticcheck ./... | sed -e 's@\(.*\)\.go@./\1.go@g'
-

.github/workflows/go-test.yml

@@ -1,38 +0,0 @@
-# File managed by web3-bot. DO NOT EDIT.
-# See https://github.com/protocol/.github/ for details.
-
-on: [push, pull_request]
-
-jobs:
-  unit:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ "ubuntu", "windows", "macos" ]
-        go: [ "1.15.x", "1.16.x" ]
-    runs-on: ${{ matrix.os }}-latest
-    name: Unit tests (${{ matrix.os}}, Go ${{ matrix.go }})
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
-        with:
-          go-version: ${{ matrix.go }}
-      - name: Go information
-        run: |
-          go version
-          go env
-      - name: Run tests
-        run: go test -v -coverprofile coverage.txt ./...
-      - name: Run tests (32 bit)
-        if: ${{ matrix.os != 'macos' }} # can't run 32 bit tests on OSX.
-        env:
-          GOARCH: 386
-        run: go test -v ./...
-      - name: Run tests with race detector
-        if: ${{ matrix.os == 'ubuntu' }} # speed things up. Windows and OSX VMs are slow
-        run: go test -v -race ./...
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@967e2b38a85a62bd61be5529ada27ebc109948c2 # v1.4.1
-        with:
-          file: coverage.txt
-          env_vars: OS=${{ matrix.os }}, GO=${{ matrix.go }}

.travis.yml

@@ -0,0 +1,30 @@
+os:
+  - linux
+
+language: go
+
+go:
+  - 1.11.x
+
+env:
+  global:
+    - GOTFLAGS="-race"
+  matrix:
+    - BUILD_DEPTYPE=gomod
+
+
+# disable travis install
+install:
+  - true
+
+script:
+  - bash <(curl -s https://raw.githubusercontent.com/ipfs/ci-helpers/master/travis-ci/run-standard-tests.sh)
+
+
+cache:
+  directories:
+    - $GOPATH/pkg/mod
+    - $HOME/.cache/go-build
+
+notifications:
+  email: false

README.md

@@ -88,7 +88,7 @@ if c1.Equals(c2) {
 ```go
 // To check if some data matches a given cid, 
 // Get your CIDs prefix, and use that to sum the data in question:
-other, err := c.Prefix().Sum(mydata)
+other, err := c.Prefix().SumStream(myFile)
 if err != nil {...}
 
 if !c.Equals(other) {

builder.go

@@ -1,6 +1,8 @@
 package cid
 
 import (
+	"io"
+
 	mh "github.com/multiformats/go-multihash"
 )
 
@@ -41,6 +43,14 @@ func (p V0Builder) Sum(data []byte) (Cid, error) {
 	return Cid{string(hash)}, nil
 }
 
+func (p V0Builder) SumStream(r io.Reader) (Cid, error) {
+	hash, err := mh.SumStream(r, mh.SHA2_256, -1)
+	if err != nil {
+		return Undef, err
+	}
+	return Cid{string(hash)}, nil
+}
+
 func (p V0Builder) GetCodec() uint64 {
 	return DagProtobuf
 }
@@ -64,6 +74,18 @@ func (p V1Builder) Sum(data []byte) (Cid, error) {
 	return NewCidV1(p.Codec, hash), nil
 }
 
+func (p V1Builder) SumStream(r io.Reader) (Cid, error) {
+	mhLen := p.MhLength
+	if mhLen <= 0 {
+		mhLen = -1
+	}
+	hash, err := mh.SumStream(r, p.MhType, mhLen)
+	if err != nil {
+		return Undef, err
+	}
+	return NewCidV1(p.Codec, hash), nil
+}
+
 func (p V1Builder) GetCodec() uint64 {
 	return p.Codec
 }

builder_test.go

@@ -1,6 +1,7 @@
 package cid
 
 import (
+	"bytes"
 	"testing"
 
 	mh "github.com/multiformats/go-multihash"
@@ -16,6 +17,15 @@ func TestV0Builder(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	reader := bytes.NewReader(data)
+	c1a, err := format.SumStream(reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !c1a.Equals(c1) {
+		t.Fatal("Sum and SumStream create different cids")
+	}
+
 	// Construct c2
 	hash, err := mh.Sum(data, mh.SHA2_256, -1)
 	if err != nil {
@@ -29,6 +39,16 @@ func TestV0Builder(t *testing.T) {
 	if c1.Prefix() != c2.Prefix() {
 		t.Fatal("prefixes mismatch")
 	}
+
+	reader.Seek(0, 0)
+	hash, err = mh.SumStream(reader, mh.SHA2_256, -1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	c2 = NewCidV0(hash)
+	if !c1.Equals(c2) {
+		t.Fatal("Sum and SumStream create different cids")
+	}
 }
 
 func TestV1Builder(t *testing.T) {
@@ -41,6 +61,15 @@ func TestV1Builder(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	reader := bytes.NewReader(data)
+	c1a, err := format.SumStream(reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !c1a.Equals(c1) {
+		t.Fatal("Sum and SumStream create different cids")
+	}
+
 	// Construct c2
 	hash, err := mh.Sum(data, mh.SHA2_256, -1)
 	if err != nil {

cid.go

@@ -185,12 +185,10 @@ func NewCidV1(codecType uint64, mhash mh.Multihash) Cid {
 	return Cid{string(buf[:n+hashlen])}
 }
 
-var (
-	_ encoding.BinaryMarshaler   = Cid{}
-	_ encoding.BinaryUnmarshaler = (*Cid)(nil)
-	_ encoding.TextMarshaler     = Cid{}
-	_ encoding.TextUnmarshaler   = (*Cid)(nil)
-)
+var _ encoding.BinaryMarshaler = Cid{}
+var _ encoding.BinaryUnmarshaler = (*Cid)(nil)
+var _ encoding.TextMarshaler = Cid{}
+var _ encoding.TextUnmarshaler = (*Cid)(nil)
 
 // Cid represents a self-describing content addressed
 // identifier. It is formed by a Version, a Codec (which indicates
@@ -566,7 +564,7 @@ type Prefix struct {
 // and return a newly constructed Cid with the resulting multihash.
 func (p Prefix) Sum(data []byte) (Cid, error) {
 	length := p.MhLength
-	if p.MhType == mh.IDENTITY {
+	if p.MhType == mh.ID {
 		length = -1
 	}
 
@@ -591,6 +589,36 @@ func (p Prefix) Sum(data []byte) (Cid, error) {
 	}
 }
 
+// SumStream uses the information in a prefix, and data read from a io.Reader,
+// to perform a multihash.SumStream() and return a newly constructed Cid with
+// the resulting multihash.
+func (p Prefix) SumStream(r io.Reader) (Cid, error) {
+	length := p.MhLength
+	if p.MhType == mh.ID {
+		length = -1
+	}
+
+	if p.Version == 0 && (p.MhType != mh.SHA2_256 ||
+		(p.MhLength != 32 && p.MhLength != -1)) {
+
+		return Undef, fmt.Errorf("invalid v0 prefix")
+	}
+
+	hash, err := mh.SumStream(r, p.MhType, length)
+	if err != nil {
+		return Undef, err
+	}
+
+	switch p.Version {
+	case 0:
+		return NewCidV0(hash), nil
+	case 1:
+		return NewCidV1(p.Codec, hash), nil
+	default:
+		return Undef, fmt.Errorf("invalid cid version")
+	}
+}
+
 // Bytes returns a byte representation of a Prefix. It looks like:
 //
 //     <version><codec><mh-type><mh-length>

cid_fuzz.go

@@ -4,6 +4,7 @@ package cid
 
 func Fuzz(data []byte) int {
 	cid, err := Cast(data)
+
 	if err != nil {
 		return 0
 	}

cid_test.go

@@ -82,7 +82,7 @@ func TestPrefixSum(t *testing.T) {
 	// Test creating CIDs both manually and with Prefix.
 	// Tests: https://github.com/ipfs/go-cid/issues/83
 	for _, hashfun := range []uint64{
-		mh.IDENTITY, mh.SHA3, mh.SHA2_256,
+		mh.ID, mh.SHA3, mh.SHA2_256,
 	} {
 		h1, err := mh.Sum([]byte("TEST"), hashfun, -1)
 		if err != nil {
@@ -388,6 +388,7 @@ func TestNewPrefixV0(t *testing.T) {
 	if c1.Prefix() != c2.Prefix() {
 		t.Fatal("prefixes mismatch")
 	}
+
 }
 
 func TestInvalidV0Prefix(t *testing.T) {
@@ -511,13 +512,13 @@ func TestParse(t *testing.T) {
 	}
 
 	assertions := [][]interface{}{
-		{NewCidV0(h), theHash},
-		{NewCidV0(h).Bytes(), theHash},
-		{h, theHash},
-		{theHash, theHash},
-		{"/ipfs/" + theHash, theHash},
-		{"https://ipfs.io/ipfs/" + theHash, theHash},
-		{"http://localhost:8080/ipfs/" + theHash, theHash},
+		[]interface{}{NewCidV0(h), theHash},
+		[]interface{}{NewCidV0(h).Bytes(), theHash},
+		[]interface{}{h, theHash},
+		[]interface{}{theHash, theHash},
+		[]interface{}{"/ipfs/" + theHash, theHash},
+		[]interface{}{"https://ipfs.io/ipfs/" + theHash, theHash},
+		[]interface{}{"http://localhost:8080/ipfs/" + theHash, theHash},
 	}
 
 	assert := func(arg interface{}, expected string) error {

go.mod

@@ -2,9 +2,8 @@ module github.com/ipfs/go-cid
 
 require (
 	github.com/multiformats/go-multibase v0.0.3
-	github.com/multiformats/go-multihash v0.0.14
+	github.com/multiformats/go-multihash v0.0.16-0.20210406191223-75ae3688857d
 	github.com/multiformats/go-varint v0.0.6
-	golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf // indirect
 )
 
 go 1.15

go.sum

@@ -1,34 +1,29 @@
+github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw=
+github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
-github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo=
-github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
+github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
+github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
 github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
-github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc=
-github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
+github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
+github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
 github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI=
 github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
 github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4=
 github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM=
 github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk=
 github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc=
-github.com/multiformats/go-multihash v0.0.14 h1:QoBceQYQQtNUuf6s7wHxnE2c8bhbMqhfGzNI032se/I=
-github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
-github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
+github.com/multiformats/go-multihash v0.0.16-0.20210406191223-75ae3688857d h1:xUl/JcLVtafS0n5X8guAExxDS/mJUGoL31p/FlNtC8o=
+github.com/multiformats/go-multihash v0.0.16-0.20210406191223-75ae3688857d/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg=
 github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY=
 github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
-github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
-github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf h1:B2n+Zi5QeYRDAEodEu72OS36gmTWjgpXr2+cWcBW90o=
-golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY=
+golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=