Compare commits

...

11 Commits

Author SHA1 Message Date
Alex
32aa7389ca Upgrade module requirements 2022-01-05 10:40:05 +11:00
web3-bot
dc3bb41324 sync: update CI config files (#136)
* add version.json file

* update .github/workflows/go-test.yml

* update .github/workflows/go-check.yml

* add .github/workflows/releaser.yml

* add .github/workflows/release-check.yml

* add .github/workflows/tagpush.yml

Co-authored-by: web3-bot <web3-bot@users.noreply.github.com>
2021-12-13 12:26:14 +01:00
Masih H. Derkani
28f4a5eab6 Benchmark existing ways to check for IDENTITY CIDs
Add benchmarks that compare two ways of checking for
`multihash.IDENTITY` code:
1. `Cid.Prefix().MhType`
2. Decode of `Cid.Hash()`

This benchmark illustrates that using Cid.Prefix is more efficient than
`multihash.Decode`. Users wishing to perform such a check should use
`Cid.Prefix`.

Consider that `Cid.Prefix` is already efficient enough and introducing a
dedicated API for performing this check will likely result in small
gains.

Relates to #133
2021-11-08 14:52:45 +00:00
Daniel Martí
5640b0169f avoid double alloc in NewCidV1
We allocate once via "make([]byte, len)",
and again when that buffer is converted to a string.

Thankfully, since Go 1.10 we have strings.Builder,
designed specifically for this use case.

In a downstream benchmark in go-car,
which needs to reconstruct many CID values,
we see small but nice gains:

    name           old time/op    new time/op    delta
    ReadBlocks-16    1.09ms ± 4%    1.06ms ± 5%   -3.33%  (p=0.007 n=11+11)

    name           old speed      new speed      delta
    ReadBlocks-16   478MB/s ± 4%   494MB/s ± 5%   +3.46%  (p=0.007 n=11+11)

    name           old alloc/op   new alloc/op   delta
    ReadBlocks-16    1.30MB ± 0%    1.25MB ± 0%   -3.86%  (p=0.000 n=12+12)

    name           old allocs/op  new allocs/op  delta
    ReadBlocks-16     9.50k ± 0%     8.45k ± 0%  -11.05%  (p=0.000 n=12+12)
2021-09-12 14:08:59 +02:00
Steven Allen
cf76220258 Merge pull request #131 from ipfs/web3-bot/sync
sync: update CI config files
2021-08-30 08:37:22 -07:00
web3-bot
9e2855d9ff update .github/workflows/go-check.yml 2021-08-17 13:32:06 +00:00
web3-bot
b8eba8ea35 update .github/workflows/go-test.yml 2021-08-17 13:32:06 +00:00
web3-bot
2628583977 update .github/workflows/automerge.yml 2021-08-17 13:32:06 +00:00
web3-bot
6e96c56557 run gofmt -s 2021-08-17 13:32:05 +00:00
web3-bot
44cccd62db bump go.mod to Go 1.16 and run go fix 2021-08-17 13:32:05 +00:00
Daniel Martí
de6c03deae amend the CidFromReader slice extension math
The append+make slice extension idiom works, but note that append uses
the slice's length as its base. We need to append the number of bytes
required for length to reach cidLength, not the capacity.

The added test case panicked before this change, and works now:

	--- FAIL: TestReadCidsFromBuffer (0.00s)
	panic: runtime error: slice bounds out of range [:73] with capacity 64 [recovered]
		panic: runtime error: slice bounds out of range [:73] with capacity 64

	goroutine 37 [running]:
	testing.tRunner.func1.2({0x570d60, 0xc000016438})
		testing/testing.go:1203 +0x24e
	testing.tRunner.func1()
		testing/testing.go:1206 +0x218
	panic({0x570d60, 0xc000016438})
		runtime/panic.go:1038 +0x215
	github.com/ipfs/go-cid.CidFromReader({0x5b0e20, 0xc000010900})
		github.com/ipfs/go-cid/cid.go:803 +0x75f
	github.com/ipfs/go-cid.TestReadCidsFromBuffer(0xc00014ba00)
		github.com/ipfs/go-cid/cid_test.go:710 +0x625
	testing.tRunner(0xc00014ba00, 0x58af38)
		testing/testing.go:1253 +0x102
	created by testing.(*T).Run
		testing/testing.go:1300 +0x35a
	exit status 2
	FAIL	github.com/ipfs/go-cid	0.004s
2021-07-16 10:10:50 +01:00
12 changed files with 176 additions and 22 deletions

View File

@@ -33,7 +33,9 @@ jobs:
automerge: automerge:
needs: automerge-check needs: automerge-check
runs-on: ubuntu-latest runs-on: ubuntu-latest
if: ${{ needs.automerge-check.outputs.status == 'true' }} # The check for the user is redundant here, as this job depends on the automerge-check job,
# but it prevents this job from spinning up, just to be skipped shortly after.
if: github.event.pull_request.user.login == 'web3-bot' && needs.automerge-check.outputs.status == 'true'
steps: steps:
- name: Wait on tests - name: Wait on tests
uses: lewagon/wait-on-check-action@bafe56a6863672c681c3cf671f5e10b20abf2eaa # v0.2 uses: lewagon/wait-on-check-action@bafe56a6863672c681c3cf671f5e10b20abf2eaa # v0.2

View File

@@ -8,17 +8,28 @@ jobs:
unit: unit:
runs-on: ubuntu-latest runs-on: ubuntu-latest
name: All name: All
env:
RUNGOGENERATE: false
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
with: with:
submodules: recursive submodules: recursive
- uses: actions/setup-go@v2 - uses: actions/setup-go@v2
with: with:
go-version: "1.16.x" go-version: "1.17.x"
- name: Run repo-specific setup
uses: ./.github/actions/go-check-setup
if: hashFiles('./.github/actions/go-check-setup') != ''
- name: Read config
if: hashFiles('./.github/workflows/go-check-config.json') != ''
run: |
if jq -re .gogenerate ./.github/workflows/go-check-config.json; then
echo "RUNGOGENERATE=true" >> $GITHUB_ENV
fi
- name: Install staticcheck - name: Install staticcheck
run: go install honnef.co/go/tools/cmd/staticcheck@434f5f3816b358fe468fa83dcba62d794e7fe04b # 2021.1 (v0.2.0) run: go install honnef.co/go/tools/cmd/staticcheck@c8caa92bad8c27ae734c6725b8a04932d54a147b # 2021.1.2 (v0.2.2)
- name: Check that go.mod is tidy - name: Check that go.mod is tidy
uses: protocol/multiple-go-modules@v1.0 uses: protocol/multiple-go-modules@v1.2
with: with:
run: | run: |
go mod tidy go mod tidy
@@ -37,14 +48,27 @@ jobs:
fi fi
- name: go vet - name: go vet
if: ${{ success() || failure() }} # run this step even if the previous one failed if: ${{ success() || failure() }} # run this step even if the previous one failed
uses: protocol/multiple-go-modules@v1.0 uses: protocol/multiple-go-modules@v1.2
with: with:
run: go vet ./... run: go vet ./...
- name: staticcheck - name: staticcheck
if: ${{ success() || failure() }} # run this step even if the previous one failed if: ${{ success() || failure() }} # run this step even if the previous one failed
uses: protocol/multiple-go-modules@v1.0 uses: protocol/multiple-go-modules@v1.2
with: with:
run: | run: |
set -o pipefail set -o pipefail
staticcheck ./... | sed -e 's@\(.*\)\.go@./\1.go@g' staticcheck ./... | sed -e 's@\(.*\)\.go@./\1.go@g'
- name: go generate
uses: protocol/multiple-go-modules@v1.2
if: (success() || failure()) && env.RUNGOGENERATE == 'true'
with:
run: |
git clean -fd # make sure there aren't untracked files / directories
go generate ./...
# check if go generate modified or added any files
if ! $(git add . && git diff-index HEAD --exit-code --quiet); then
echo "go generated caused changes to the repository:"
git status --short
exit 1
fi

View File

@@ -10,7 +10,9 @@ jobs:
fail-fast: false fail-fast: false
matrix: matrix:
os: [ "ubuntu", "windows", "macos" ] os: [ "ubuntu", "windows", "macos" ]
go: [ "1.15.x", "1.16.x" ] go: [ "1.16.x", "1.17.x" ]
env:
COVERAGES: ""
runs-on: ${{ matrix.os }}-latest runs-on: ${{ matrix.os }}-latest
name: ${{ matrix.os}} (go ${{ matrix.go }}) name: ${{ matrix.os}} (go ${{ matrix.go }})
steps: steps:
@@ -24,24 +26,41 @@ jobs:
run: | run: |
go version go version
go env go env
- name: Use msys2 on windows
if: ${{ matrix.os == 'windows' }}
shell: bash
# The executable for msys2 is also called bash.cmd
# https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md#shells
# If we prepend its location to the PATH
# subsequent 'shell: bash' steps will use msys2 instead of gitbash
run: echo "C:/msys64/usr/bin" >> $GITHUB_PATH
- name: Run repo-specific setup
uses: ./.github/actions/go-test-setup
if: hashFiles('./.github/actions/go-test-setup') != ''
- name: Run tests - name: Run tests
uses: protocol/multiple-go-modules@v1.0 uses: protocol/multiple-go-modules@v1.2
with: with:
run: go test -v -coverprofile coverage.txt ./... # Use -coverpkg=./..., so that we include cross-package coverage.
# If package ./A imports ./B, and ./A's tests also cover ./B,
# this means ./B's coverage will be significantly higher than 0%.
run: go test -v -coverprofile=module-coverage.txt -coverpkg=./... ./...
- name: Run tests (32 bit) - name: Run tests (32 bit)
if: ${{ matrix.os != 'macos' }} # can't run 32 bit tests on OSX. if: ${{ matrix.os != 'macos' }} # can't run 32 bit tests on OSX.
uses: protocol/multiple-go-modules@v1.0 uses: protocol/multiple-go-modules@v1.2
env: env:
GOARCH: 386 GOARCH: 386
with: with:
run: go test -v ./... run: go test -v ./...
- name: Run tests with race detector - name: Run tests with race detector
if: ${{ matrix.os == 'ubuntu' }} # speed things up. Windows and OSX VMs are slow if: ${{ matrix.os == 'ubuntu' }} # speed things up. Windows and OSX VMs are slow
uses: protocol/multiple-go-modules@v1.0 uses: protocol/multiple-go-modules@v1.2
with: with:
run: go test -v -race ./... run: go test -v -race ./...
- name: Collect coverage files
shell: bash
run: echo "COVERAGES=$(find . -type f -name 'module-coverage.txt' | tr -s '\n' ',' | sed 's/,$//')" >> $GITHUB_ENV
- name: Upload coverage to Codecov - name: Upload coverage to Codecov
uses: codecov/codecov-action@a1ed4b322b4b38cb846afb5a0ebfa17086917d27 # v1.5.0 uses: codecov/codecov-action@f32b3a3741e1053eb607407145bc9619351dc93b # v2.1.0
with: with:
file: coverage.txt files: '${{ env.COVERAGES }}'
env_vars: OS=${{ matrix.os }}, GO=${{ matrix.go }} env_vars: OS=${{ matrix.os }}, GO=${{ matrix.go }}

11
.github/workflows/release-check.yml vendored Normal file
View File

@@ -0,0 +1,11 @@
# File managed by web3-bot. DO NOT EDIT.
# See https://github.com/protocol/.github/ for details.
name: Release Checker
on:
pull_request:
paths: [ 'version.json' ]
jobs:
release-check:
uses: protocol/.github/.github/workflows/release-check.yml@master

11
.github/workflows/releaser.yml vendored Normal file
View File

@@ -0,0 +1,11 @@
# File managed by web3-bot. DO NOT EDIT.
# See https://github.com/protocol/.github/ for details.
name: Releaser
on:
push:
paths: [ 'version.json' ]
jobs:
releaser:
uses: protocol/.github/.github/workflows/releaser.yml@master

12
.github/workflows/tagpush.yml vendored Normal file
View File

@@ -0,0 +1,12 @@
# File managed by web3-bot. DO NOT EDIT.
# See https://github.com/protocol/.github/ for details.
name: Tag Push Checker
on:
push:
tags:
- v*
jobs:
releaser:
uses: protocol/.github/.github/workflows/tagpush.yml@master

61
benchmark_test.go Normal file
View File

@@ -0,0 +1,61 @@
package cid_test
import (
"math/rand"
"testing"
"github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash"
)
// BenchmarkIdentityCheck benchmarks two ways of checking whether a CIDv1 has multihash.IDENTITY
// code:
// 1. Cid.Prefix(), and
// 2. decoding the Cid.Hash().
//
// This benchmark illustrates that using Cid.Prefix is more efficient than multihash.Decode.
// Users wishing to perform such a check should use Cid.Prefix.
//
// Consider that `Cid.Prefix` is already efficient enough and introducing a dedicated API for
// performing this check will likely result in small gains.
func BenchmarkIdentityCheck(b *testing.B) {
rng := rand.New(rand.NewSource(1413))
data := make([]byte, rng.Intn(100)+1024)
if _, err := rng.Read(data); err != nil {
b.Fatal(err)
}
mh, err := multihash.Sum(data, multihash.IDENTITY, -1)
if err != nil {
b.Fatal(err)
}
cv1 := cid.NewCidV1(cid.Raw, mh)
b.SetBytes(int64(cv1.ByteLen()))
b.ReportAllocs()
b.ResetTimer()
b.Run("Prefix", func(b *testing.B) {
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
if cv1.Prefix().MhType != multihash.IDENTITY {
b.Fatal("expected IDENTITY CID")
}
}
})
})
b.Run("MultihashDecode", func(b *testing.B) {
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
dmh, err := multihash.Decode(cv1.Hash())
if err != nil {
b.Fatal(err)
}
if dmh.Code != multihash.IDENTITY {
b.Fatal("expected IDENTITY CID")
}
}
})
})
}

23
cid.go
View File

@@ -22,6 +22,7 @@ package cid
import ( import (
"bytes" "bytes"
"encoding" "encoding"
"encoding/binary"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
@@ -173,16 +174,24 @@ func NewCidV0(mhash mh.Multihash) Cid {
// Panics if the multihash is invalid. // Panics if the multihash is invalid.
func NewCidV1(codecType uint64, mhash mh.Multihash) Cid { func NewCidV1(codecType uint64, mhash mh.Multihash) Cid {
hashlen := len(mhash) hashlen := len(mhash)
// two 8 bytes (max) numbers plus hash
buf := make([]byte, 1+varint.UvarintSize(codecType)+hashlen) // Two 8 bytes (max) numbers plus hash.
n := varint.PutUvarint(buf, 1) // We use strings.Builder to only allocate once.
n += varint.PutUvarint(buf[n:], codecType) var b strings.Builder
cn := copy(buf[n:], mhash) b.Grow(1 + varint.UvarintSize(codecType) + hashlen)
b.WriteByte(1)
var buf [binary.MaxVarintLen64]byte
n := varint.PutUvarint(buf[:], codecType)
b.Write(buf[:n])
cn, _ := b.Write(mhash)
if cn != hashlen { if cn != hashlen {
panic("copy hash length is inconsistent") panic("copy hash length is inconsistent")
} }
return Cid{string(buf[:n+hashlen])} return Cid{b.String()}
} }
var ( var (
@@ -793,7 +802,7 @@ func CidFromReader(r io.Reader) (int, Cid, error) {
if cidLength > cap(br.dst) { if cidLength > cap(br.dst) {
// If the multihash digest doesn't fit in our initial 64 bytes, // If the multihash digest doesn't fit in our initial 64 bytes,
// efficiently extend the slice via append+make. // efficiently extend the slice via append+make.
br.dst = append(br.dst, make([]byte, cidLength-cap(br.dst))...) br.dst = append(br.dst, make([]byte, cidLength-len(br.dst))...)
} else { } else {
// The multihash digest fits inside our buffer, // The multihash digest fits inside our buffer,
// so just extend its capacity. // so just extend its capacity.

View File

@@ -1,3 +1,4 @@
//go:build gofuzz
// +build gofuzz // +build gofuzz
package cid package cid

View File

@@ -667,6 +667,7 @@ func TestReadCidsFromBuffer(t *testing.T) {
"k2cwueckqkibutvhkr4p2ln2pjcaxaakpd9db0e7j7ax1lxhhxy3ekpv", "k2cwueckqkibutvhkr4p2ln2pjcaxaakpd9db0e7j7ax1lxhhxy3ekpv",
"Qmf5Qzp6nGBku7CEn2UQx4mgN8TW69YUok36DrGa6NN893", "Qmf5Qzp6nGBku7CEn2UQx4mgN8TW69YUok36DrGa6NN893",
"zb2rhZi1JR4eNc2jBGaRYJKYM8JEB4ovenym8L1CmFsRAytkz", "zb2rhZi1JR4eNc2jBGaRYJKYM8JEB4ovenym8L1CmFsRAytkz",
"bafkqarjpmzuwyzltorxxezjpkvcfgqkfjfbfcvslivje2vchkzdu6rckjjcfgtkolaze6mssjqzeyn2ekrcfatkjku2vowseky3fswkfkm2deqkrju3e2",
} }
var cids []Cid var cids []Cid

4
go.mod
View File

@@ -2,9 +2,9 @@ module github.com/ipfs/go-cid
require ( require (
github.com/multiformats/go-multibase v0.0.3 github.com/multiformats/go-multibase v0.0.3
github.com/multiformats/go-multihash v0.0.15 github.com/multiformats/go-multihash v0.1.0
github.com/multiformats/go-varint v0.0.6 github.com/multiformats/go-varint v0.0.6
golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf // indirect golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf // indirect
) )
go 1.15 go 1.16

3
version.json Normal file
View File

@@ -0,0 +1,3 @@
{
"version": "v0.1.0"
}