Upgrade module requirements

* add version.json file

* update .github/workflows/go-test.yml

* update .github/workflows/go-check.yml

* add .github/workflows/releaser.yml

* add .github/workflows/release-check.yml

* add .github/workflows/tagpush.yml

Co-authored-by: web3-bot <web3-bot@users.noreply.github.com>

.github/workflows/automerge.yml

@@ -33,7 +33,9 @@ jobs:
   automerge:
     needs: automerge-check
     runs-on: ubuntu-latest
-    if: ${{ needs.automerge-check.outputs.status == 'true' }}
+    # The check for the user is redundant here, as this job depends on the automerge-check job,
+    # but it prevents this job from spinning up, just to be skipped shortly after.
+    if: github.event.pull_request.user.login == 'web3-bot' && needs.automerge-check.outputs.status == 'true'
     steps:
     - name: Wait on tests
       uses: lewagon/wait-on-check-action@bafe56a6863672c681c3cf671f5e10b20abf2eaa # v0.2

.github/workflows/go-check.yml

@@ -8,17 +8,28 @@ jobs:
   unit:
     runs-on: ubuntu-latest
     name: All
+    env:
+      RUNGOGENERATE: false
     steps:
       - uses: actions/checkout@v2
         with:
           submodules: recursive
       - uses: actions/setup-go@v2
         with:
-          go-version: "1.16.x"
+          go-version: "1.17.x"
+      - name: Run repo-specific setup
+        uses: ./.github/actions/go-check-setup
+        if: hashFiles('./.github/actions/go-check-setup') != ''
+      - name: Read config
+        if: hashFiles('./.github/workflows/go-check-config.json') != ''
+        run: |
+          if jq -re .gogenerate ./.github/workflows/go-check-config.json; then
+            echo "RUNGOGENERATE=true" >> $GITHUB_ENV
+          fi
       - name: Install staticcheck
-        run: go install honnef.co/go/tools/cmd/staticcheck@434f5f3816b358fe468fa83dcba62d794e7fe04b # 2021.1 (v0.2.0)
+        run: go install honnef.co/go/tools/cmd/staticcheck@c8caa92bad8c27ae734c6725b8a04932d54a147b # 2021.1.2 (v0.2.2)
       - name: Check that go.mod is tidy
-        uses: protocol/multiple-go-modules@v1.0
+        uses: protocol/multiple-go-modules@v1.2
         with:
           run: |
             go mod tidy
@@ -37,14 +48,27 @@ jobs:
           fi
       - name: go vet
         if: ${{ success() || failure() }} # run this step even if the previous one failed
-        uses: protocol/multiple-go-modules@v1.0
+        uses: protocol/multiple-go-modules@v1.2
         with:
           run: go vet ./...
       - name: staticcheck
         if: ${{ success() || failure() }} # run this step even if the previous one failed
-        uses: protocol/multiple-go-modules@v1.0
+        uses: protocol/multiple-go-modules@v1.2
         with:
           run: |
             set -o pipefail
             staticcheck ./... | sed -e 's@\(.*\)\.go@./\1.go@g'
+      - name: go generate
+        uses: protocol/multiple-go-modules@v1.2
+        if: (success() || failure()) && env.RUNGOGENERATE == 'true'
+        with:
+          run: |
+            git clean -fd # make sure there aren't untracked files / directories
+            go generate ./...
+            # check if go generate modified or added any files
+            if ! $(git add . && git diff-index HEAD --exit-code --quiet); then
+              echo "go generated caused changes to the repository:"
+              git status --short
+              exit 1
+            fi
 

.github/workflows/go-test.yml

@@ -10,7 +10,9 @@ jobs:
       fail-fast: false
       matrix:
         os: [ "ubuntu", "windows", "macos" ]
-        go: [ "1.15.x", "1.16.x" ]
+        go: [ "1.16.x", "1.17.x" ]
+    env:
+      COVERAGES: ""
     runs-on: ${{ matrix.os }}-latest
     name: ${{ matrix.os}} (go ${{ matrix.go }})
     steps:
@@ -24,24 +26,41 @@ jobs:
         run: |
           go version
           go env
+      - name: Use msys2 on windows
+        if: ${{ matrix.os == 'windows' }}
+        shell: bash
+        # The executable for msys2 is also called bash.cmd
+        #   https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md#shells
+        # If we prepend its location to the PATH
+        #   subsequent 'shell: bash' steps will use msys2 instead of gitbash
+        run: echo "C:/msys64/usr/bin" >> $GITHUB_PATH
+      - name: Run repo-specific setup
+        uses: ./.github/actions/go-test-setup
+        if: hashFiles('./.github/actions/go-test-setup') != ''
       - name: Run tests
-        uses: protocol/multiple-go-modules@v1.0
+        uses: protocol/multiple-go-modules@v1.2
         with:
-          run: go test -v -coverprofile coverage.txt ./...
+          # Use -coverpkg=./..., so that we include cross-package coverage.
+          # If package ./A imports ./B, and ./A's tests also cover ./B,
+          # this means ./B's coverage will be significantly higher than 0%.
+          run: go test -v -coverprofile=module-coverage.txt -coverpkg=./... ./...
       - name: Run tests (32 bit)
         if: ${{ matrix.os != 'macos' }} # can't run 32 bit tests on OSX.
-        uses: protocol/multiple-go-modules@v1.0
+        uses: protocol/multiple-go-modules@v1.2
         env:
           GOARCH: 386
         with:
           run: go test -v ./...
       - name: Run tests with race detector
         if: ${{ matrix.os == 'ubuntu' }} # speed things up. Windows and OSX VMs are slow
-        uses: protocol/multiple-go-modules@v1.0
+        uses: protocol/multiple-go-modules@v1.2
         with:
           run: go test -v -race ./...
+      - name: Collect coverage files
+        shell: bash
+        run: echo "COVERAGES=$(find . -type f -name 'module-coverage.txt' | tr -s '\n' ',' | sed 's/,$//')" >> $GITHUB_ENV
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@a1ed4b322b4b38cb846afb5a0ebfa17086917d27 # v1.5.0
+        uses: codecov/codecov-action@f32b3a3741e1053eb607407145bc9619351dc93b # v2.1.0
         with:
-          file: coverage.txt
+          files: '${{ env.COVERAGES }}'
           env_vars: OS=${{ matrix.os }}, GO=${{ matrix.go }}

.github/workflows/release-check.yml

@@ -0,0 +1,11 @@
+# File managed by web3-bot. DO NOT EDIT.
+# See https://github.com/protocol/.github/ for details.
+
+name: Release Checker
+on:
+  pull_request:
+    paths: [ 'version.json' ]
+
+jobs:
+  release-check:
+    uses: protocol/.github/.github/workflows/release-check.yml@master

.github/workflows/releaser.yml

@@ -0,0 +1,11 @@
+# File managed by web3-bot. DO NOT EDIT.
+# See https://github.com/protocol/.github/ for details.
+
+name: Releaser
+on:
+  push:
+    paths: [ 'version.json' ]
+
+jobs:
+  releaser:
+    uses: protocol/.github/.github/workflows/releaser.yml@master

.github/workflows/tagpush.yml

@@ -0,0 +1,12 @@
+# File managed by web3-bot. DO NOT EDIT.
+# See https://github.com/protocol/.github/ for details.
+
+name: Tag Push Checker
+on:
+  push:
+    tags:
+      - v*
+
+jobs:
+  releaser:
+    uses: protocol/.github/.github/workflows/tagpush.yml@master

benchmark_test.go

@@ -0,0 +1,61 @@
+package cid_test
+
+import (
+	"math/rand"
+	"testing"
+
+	"github.com/ipfs/go-cid"
+	"github.com/multiformats/go-multihash"
+)
+
+// BenchmarkIdentityCheck benchmarks two ways of checking whether a CIDv1 has multihash.IDENTITY
+// code:
+//  1. Cid.Prefix(), and
+//  2. decoding the Cid.Hash().
+//
+// This benchmark illustrates that using Cid.Prefix is more efficient than multihash.Decode.
+// Users wishing to perform such a check should use Cid.Prefix.
+//
+// Consider that `Cid.Prefix` is already efficient enough and introducing a dedicated API for
+// performing this check will likely result in small gains.
+func BenchmarkIdentityCheck(b *testing.B) {
+	rng := rand.New(rand.NewSource(1413))
+
+	data := make([]byte, rng.Intn(100)+1024)
+	if _, err := rng.Read(data); err != nil {
+		b.Fatal(err)
+	}
+	mh, err := multihash.Sum(data, multihash.IDENTITY, -1)
+	if err != nil {
+		b.Fatal(err)
+	}
+	cv1 := cid.NewCidV1(cid.Raw, mh)
+
+	b.SetBytes(int64(cv1.ByteLen()))
+	b.ReportAllocs()
+	b.ResetTimer()
+
+	b.Run("Prefix", func(b *testing.B) {
+		b.RunParallel(func(pb *testing.PB) {
+			for pb.Next() {
+				if cv1.Prefix().MhType != multihash.IDENTITY {
+					b.Fatal("expected IDENTITY CID")
+				}
+			}
+		})
+	})
+
+	b.Run("MultihashDecode", func(b *testing.B) {
+		b.RunParallel(func(pb *testing.PB) {
+			for pb.Next() {
+				dmh, err := multihash.Decode(cv1.Hash())
+				if err != nil {
+					b.Fatal(err)
+				}
+				if dmh.Code != multihash.IDENTITY {
+					b.Fatal("expected IDENTITY CID")
+				}
+			}
+		})
+	})
+}

cid.go

@@ -22,6 +22,7 @@ package cid
 import (
 	"bytes"
 	"encoding"
+	"encoding/binary"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -173,16 +174,24 @@ func NewCidV0(mhash mh.Multihash) Cid {
 // Panics if the multihash is invalid.
 func NewCidV1(codecType uint64, mhash mh.Multihash) Cid {
 	hashlen := len(mhash)
-	// two 8 bytes (max) numbers plus hash
-	buf := make([]byte, 1+varint.UvarintSize(codecType)+hashlen)
-	n := varint.PutUvarint(buf, 1)
-	n += varint.PutUvarint(buf[n:], codecType)
-	cn := copy(buf[n:], mhash)
+
+	// Two 8 bytes (max) numbers plus hash.
+	// We use strings.Builder to only allocate once.
+	var b strings.Builder
+	b.Grow(1 + varint.UvarintSize(codecType) + hashlen)
+
+	b.WriteByte(1)
+
+	var buf [binary.MaxVarintLen64]byte
+	n := varint.PutUvarint(buf[:], codecType)
+	b.Write(buf[:n])
+
+	cn, _ := b.Write(mhash)
 	if cn != hashlen {
 		panic("copy hash length is inconsistent")
 	}
 
-	return Cid{string(buf[:n+hashlen])}
+	return Cid{b.String()}
 }
 
 var (
@@ -694,19 +703,10 @@ type bufByteReader struct {
 	direct   io.ByteReader
 	fallback io.Reader
 
-	consumed int
-	dst      []byte
+	dst []byte
 }
 
 func (r *bufByteReader) ReadByte() (byte, error) {
-	// We still have some of the initial bytes to use.
-	if r.consumed < len(r.dst) {
-		b := r.dst[r.consumed]
-		r.consumed++
-		return b, nil
-	}
-	r.consumed++
-
 	// The underlying reader has ReadByte; use it.
 	if br := r.direct; br != nil {
 		b, err := br.ReadByte()
@@ -718,6 +718,8 @@ func (r *bufByteReader) ReadByte() (byte, error) {
 	}
 
 	// Fall back to a one-byte Read.
+	// TODO: consider reading straight into dst,
+	// once we have benchmarks and if they prove that to be faster.
 	var p [1]byte
 	if _, err := io.ReadFull(r.fallback, p[:]); err != nil {
 		return 0, err
@@ -737,38 +739,30 @@ func CidFromReader(r io.Reader) (int, Cid, error) {
 	// 64 bytes is enough for any CIDv0,
 	// and it's enough for most CIDv1s in practice.
 	// If the digest is too long, we'll allocate more.
-	buf := make([]byte, 0, 64)
-
-	// We read two bytes, to tell if this is a CIDv0 or a CIDv1.
-	if n, err := io.ReadFull(r, buf[:2]); err != nil {
-		return n, Undef, err
-	}
-	buf = buf[:2]
-
-	// If we have a CIDv0, read the rest of the bytes and cast the buffer.
-	if buf[0] == mh.SHA2_256 && buf[1] == 32 {
-		if n, err := io.ReadFull(r, buf[2:34]); err != nil {
-			return len(buf) + n, Undef, err
-		}
-
-		buf = buf[:34]
-		h, err := mh.Cast(buf)
-		if err != nil {
-			return len(buf), Undef, err
-		}
-
-		return len(buf), Cid{string(h)}, nil
-	}
+	br := toBufByteReader(r, make([]byte, 0, 64))
 
+	// We read the first varint, to tell if this is a CIDv0 or a CIDv1.
 	// The varint package wants a io.ByteReader, so we must wrap our io.Reader.
-	// Note that we already read two bytes, so bufByteReader uses those first.
-	// After those two bytes, bufByteReader appends the read bytes to br.dst.
-	br := toBufByteReader(r, buf[:2])
 	vers, err := varint.ReadUvarint(br)
 	if err != nil {
 		return len(br.dst), Undef, err
 	}
 
+	// If we have a CIDv0, read the rest of the bytes and cast the buffer.
+	if vers == mh.SHA2_256 {
+		if n, err := io.ReadFull(r, br.dst[1:34]); err != nil {
+			return len(br.dst) + n, Undef, err
+		}
+
+		br.dst = br.dst[:34]
+		h, err := mh.Cast(br.dst)
+		if err != nil {
+			return len(br.dst), Undef, err
+		}
+
+		return len(br.dst), Cid{string(h)}, nil
+	}
+
 	if vers != 1 {
 		return len(br.dst), Undef, fmt.Errorf("expected 1 as the cid version number, got: %d", vers)
 	}
@@ -796,29 +790,38 @@ func CidFromReader(r io.Reader) (int, Cid, error) {
 		return len(br.dst), Undef, err
 	}
 
-	// Update buf's length.
-	// We're not reading single bytes beyond this point.
-	buf = br.dst
-	br = nil
-
-	// Multihash digest; might be too long, so allocate.
 	// Refuse to make large allocations to prevent OOMs due to bugs.
-	// TODO: reuse buf if it has enough space
 	const maxDigestAlloc = 32 << 20 // 32MiB
 	if mhl > maxDigestAlloc {
-		return len(buf), Undef, fmt.Errorf("refusing to allocate %d bytes for a digest", mhl)
+		return len(br.dst), Undef, fmt.Errorf("refusing to allocate %d bytes for a digest", mhl)
 	}
-	digest := make([]byte, int(mhl))
-	if n, err := io.ReadFull(r, digest); err != nil {
-		return len(buf) + n, Undef, err
+
+	// Fine to convert mhl to int, given maxDigestAlloc.
+	prefixLength := len(br.dst)
+	cidLength := prefixLength + int(mhl)
+	if cidLength > cap(br.dst) {
+		// If the multihash digest doesn't fit in our initial 64 bytes,
+		// efficiently extend the slice via append+make.
+		br.dst = append(br.dst, make([]byte, cidLength-len(br.dst))...)
+	} else {
+		// The multihash digest fits inside our buffer,
+		// so just extend its capacity.
+		br.dst = br.dst[:cidLength]
+	}
+
+	if n, err := io.ReadFull(r, br.dst[prefixLength:cidLength]); err != nil {
+		// We can't use len(br.dst) here,
+		// as we've only read n bytes past prefixLength.
+		return prefixLength + n, Undef, err
 	}
-	buf = append(buf, digest...)
 
 	// This simply ensures the multihash is valid.
-	_, _, err = mh.MHFromBytes(buf[mhStart:])
+	// TODO: consider removing this bit, as it's probably redundant;
+	// for now, it helps ensure consistency with CidFromBytes.
+	_, _, err = mh.MHFromBytes(br.dst[mhStart:])
 	if err != nil {
-		return len(buf), Undef, err
+		return len(br.dst), Undef, err
 	}
 
-	return len(buf), Cid{string(buf)}, nil
+	return len(br.dst), Cid{string(br.dst)}, nil
 }

cid_fuzz.go

@@ -1,3 +1,4 @@
+//go:build gofuzz
 // +build gofuzz
 
 package cid

cid_test.go

@@ -667,6 +667,7 @@ func TestReadCidsFromBuffer(t *testing.T) {
 		"k2cwueckqkibutvhkr4p2ln2pjcaxaakpd9db0e7j7ax1lxhhxy3ekpv",
 		"Qmf5Qzp6nGBku7CEn2UQx4mgN8TW69YUok36DrGa6NN893",
 		"zb2rhZi1JR4eNc2jBGaRYJKYM8JEB4ovenym8L1CmFsRAytkz",
+		"bafkqarjpmzuwyzltorxxezjpkvcfgqkfjfbfcvslivje2vchkzdu6rckjjcfgtkolaze6mssjqzeyn2ekrcfatkjku2vowseky3fswkfkm2deqkrju3e2",
 	}
 
 	var cids []Cid
@@ -721,49 +722,71 @@ func TestReadCidsFromBuffer(t *testing.T) {
 	}
 }
 
-func TestBadCidFromBytes(t *testing.T) {
-	l, c, err := CidFromBytes([]byte{mh.SHA2_256, 32, 0x00})
-	if err == nil {
-		t.Fatal("expected not-enough-bytes for V0 CidFromBytes")
-	}
-	if l != 0 {
-		t.Fatal("expected length=0 from bad CidFromBytes")
-	}
-	if c != Undef {
-		t.Fatal("expected Undef CID from bad CidFromBytes")
-	}
+func TestBadCidInput(t *testing.T) {
+	for _, name := range []string{
+		"FromBytes",
+		"FromReader",
+	} {
+		t.Run(name, func(t *testing.T) {
+			usingReader := name == "FromReader"
 
-	c, err = Decode("bafkreie5qrjvaw64n4tjm6hbnm7fnqvcssfed4whsjqxzslbd3jwhsk3mm")
-	if err != nil {
-		t.Fatal(err)
-	}
-	byts := make([]byte, c.ByteLen())
-	copy(byts, c.Bytes())
-	byts[1] = 0x80 // bad codec varint
-	byts[2] = 0x00
-	l, c, err = CidFromBytes(byts)
-	if err == nil {
-		t.Fatal("expected not-enough-bytes for V1 CidFromBytes")
-	}
-	if l != 0 {
-		t.Fatal("expected length=0 from bad CidFromBytes")
-	}
-	if c != Undef {
-		t.Fatal("expected Undef CID from bad CidFromBytes")
-	}
+			fromBytes := CidFromBytes
+			if usingReader {
+				fromBytes = func(data []byte) (int, Cid, error) {
+					return CidFromReader(bytes.NewReader(data))
+				}
+			}
 
-	copy(byts, c.Bytes())
-	byts[2] = 0x80 // bad multihash varint
-	byts[3] = 0x00
-	l, c, err = CidFromBytes(byts)
-	if err == nil {
-		t.Fatal("expected not-enough-bytes for V1 CidFromBytes")
-	}
-	if l != 0 {
-		t.Fatal("expected length=0 from bad CidFromBytes")
-	}
-	if c != Undef {
-		t.Fatal("expected Undef CID from bad CidFromBytes")
+			l, c, err := fromBytes([]byte{mh.SHA2_256, 32, 0x00})
+			if err == nil {
+				t.Fatal("expected not-enough-bytes for V0 CID")
+			}
+			if !usingReader && l != 0 {
+				t.Fatal("expected length==0 from bad CID")
+			} else if usingReader && l == 0 {
+				t.Fatal("expected length!=0 from bad CID")
+			}
+			if c != Undef {
+				t.Fatal("expected Undef CID from bad CID")
+			}
+
+			c, err = Decode("bafkreie5qrjvaw64n4tjm6hbnm7fnqvcssfed4whsjqxzslbd3jwhsk3mm")
+			if err != nil {
+				t.Fatal(err)
+			}
+			byts := make([]byte, c.ByteLen())
+			copy(byts, c.Bytes())
+			byts[1] = 0x80 // bad codec varint
+			byts[2] = 0x00
+			l, c, err = fromBytes(byts)
+			if err == nil {
+				t.Fatal("expected not-enough-bytes for V1 CID")
+			}
+			if !usingReader && l != 0 {
+				t.Fatal("expected length==0 from bad CID")
+			} else if usingReader && l == 0 {
+				t.Fatal("expected length!=0 from bad CID")
+			}
+			if c != Undef {
+				t.Fatal("expected Undef CID from bad CID")
+			}
+
+			copy(byts, c.Bytes())
+			byts[2] = 0x80 // bad multihash varint
+			byts[3] = 0x00
+			l, c, err = fromBytes(byts)
+			if err == nil {
+				t.Fatal("expected not-enough-bytes for V1 CID")
+			}
+			if !usingReader && l != 0 {
+				t.Fatal("expected length==0 from bad CID")
+			} else if usingReader && l == 0 {
+				t.Fatal("expected length!=0 from bad CID")
+			}
+			if c != Undef {
+				t.Fatal("expected Undef CID from bad CidFromBytes")
+			}
+		})
 	}
 }
 

go.mod

@@ -2,9 +2,9 @@ module github.com/ipfs/go-cid
 
 require (
 	github.com/multiformats/go-multibase v0.0.3
-	github.com/multiformats/go-multihash v0.0.15
+	github.com/multiformats/go-multihash v0.1.0
 	github.com/multiformats/go-varint v0.0.6
 	golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf // indirect
 )
 
-go 1.15
+go 1.16

version.json

@@ -0,0 +1,3 @@
+{
+  "version": "v0.1.0"
+}