gx publish 0.8.0

Extract non-core functionality from go-cid into go-cidutil

.gx/lastpubver

@@ -1 +1 @@
-0.7.13: QmZe3S3sXMWar3oPmrd8jHV8NoAWfPfLGFD8PsK3YrYpqv
+0.8.0: QmZFbDTY9jfSBms2MchvYM9oYRbAF19K7Pby47yDBfpPrb

.travis.yml

@@ -1,24 +1,24 @@
-os:
-  - linux
-  - osx
+sudo: false
+
 
 language: go
-
 go:
-    - 1.7
-
-install: true
-
-before_install:
-  - make deps
-
+  - 'tip'
+  
+install:
+  - go get github.com/whyrusleeping/gx
+  - go get github.com/whyrusleeping/gx-go
+  - gx install --global
 script:
-  - go vet
-  - $GOPATH/bin/goveralls -service="travis-ci"
+  - gx test -v -race -coverprofile=coverage.txt -covermode=atomic .
+  
+after_success:
+  - bash <(curl -s https://codecov.io/bash)
 
 cache:
-    directories:
-        - $GOPATH/src/gx
+  directories:
+    - $GOPATH/src/gx
 
 notifications:
-  email: false
+email: false
+  

builder.go

@@ -0,0 +1,74 @@
+package cid
+
+import (
+	mh "github.com/multiformats/go-multihash"
+)
+
+type Builder interface {
+	Sum(data []byte) (*Cid, error)
+	GetCodec() uint64
+	WithCodec(uint64) Builder
+}
+
+type V0Builder struct{}
+
+type V1Builder struct {
+	Codec    uint64
+	MhType   uint64
+	MhLength int // MhLength <= 0 means the default length
+}
+
+func (p Prefix) GetCodec() uint64 {
+	return p.Codec
+}
+
+func (p Prefix) WithCodec(c uint64) Builder {
+	if c == p.Codec {
+		return p
+	}
+	p.Codec = c
+	if c != DagProtobuf {
+		p.Version = 1
+	}
+	return p
+}
+
+func (p V0Builder) Sum(data []byte) (*Cid, error) {
+	hash, err := mh.Sum(data, mh.SHA2_256, -1)
+	if err != nil {
+		return nil, err
+	}
+	return NewCidV0(hash), nil
+}
+
+func (p V0Builder) GetCodec() uint64 {
+	return DagProtobuf
+}
+
+func (p V0Builder) WithCodec(c uint64) Builder {
+	if c == DagProtobuf {
+		return p
+	}
+	return V1Builder{Codec: c, MhType: mh.SHA2_256}
+}
+
+func (p V1Builder) Sum(data []byte) (*Cid, error) {
+	mhLen := p.MhLength
+	if mhLen <= 0 {
+		mhLen = -1
+	}
+	hash, err := mh.Sum(data, p.MhType, mhLen)
+	if err != nil {
+		return nil, err
+	}
+	return NewCidV1(p.Codec, hash), nil
+}
+
+func (p V1Builder) GetCodec() uint64 {
+	return p.Codec
+}
+
+func (p V1Builder) WithCodec(c uint64) Builder {
+	p.Codec = c
+	return p
+}

builder_test.go

@@ -0,0 +1,92 @@
+package cid
+
+import (
+	"testing"
+
+	mh "github.com/multiformats/go-multihash"
+)
+
+func TestV0Builder(t *testing.T) {
+	data := []byte("this is some test content")
+
+	// Construct c1
+	format := V0Builder{}
+	c1, err := format.Sum(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Construct c2
+	hash, err := mh.Sum(data, mh.SHA2_256, -1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	c2 := NewCidV0(hash)
+
+	if !c1.Equals(c2) {
+		t.Fatal("cids mismatch")
+	}
+	if c1.Prefix() != c2.Prefix() {
+		t.Fatal("prefixes mismatch")
+	}
+}
+
+func TestV1Builder(t *testing.T) {
+	data := []byte("this is some test content")
+
+	// Construct c1
+	format := V1Builder{Codec: DagCBOR, MhType: mh.SHA2_256}
+	c1, err := format.Sum(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Construct c2
+	hash, err := mh.Sum(data, mh.SHA2_256, -1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	c2 := NewCidV1(DagCBOR, hash)
+
+	if !c1.Equals(c2) {
+		t.Fatal("cids mismatch")
+	}
+	if c1.Prefix() != c2.Prefix() {
+		t.Fatal("prefixes mismatch")
+	}
+}
+
+func TestCodecChange(t *testing.T) {
+	t.Run("Prefix-CidV0", func(t *testing.T) {
+		p := Prefix{Version: 0, Codec: DagProtobuf, MhType: mh.SHA2_256, MhLength: mh.DefaultLengths[mh.SHA2_256]}
+		testCodecChange(t, p)
+	})
+	t.Run("Prefix-CidV1", func(t *testing.T) {
+		p := Prefix{Version: 1, Codec: DagProtobuf, MhType: mh.SHA2_256, MhLength: mh.DefaultLengths[mh.SHA2_256]}
+		testCodecChange(t, p)
+	})
+	t.Run("V0Builder", func(t *testing.T) {
+		testCodecChange(t, V0Builder{})
+	})
+	t.Run("V1Builder", func(t *testing.T) {
+		testCodecChange(t, V1Builder{Codec: DagProtobuf, MhType: mh.SHA2_256})
+	})
+}
+
+func testCodecChange(t *testing.T, b Builder) {
+	data := []byte("this is some test content")
+
+	if b.GetCodec() != DagProtobuf {
+		t.Fatal("original builder not using Protobuf codec")
+	}
+
+	b = b.WithCodec(Raw)
+	c, err := b.Sum(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if c.Type() != Raw {
+		t.Fatal("new cid codec did not change to Raw")
+	}
+}

cid.go

@@ -47,6 +47,10 @@ var (
 	// ErrCidTooShort means that the cid passed to decode was not long
 	// enough to be a valid Cid
 	ErrCidTooShort = errors.New("cid too short")
+
+	// ErrInvalidEncoding means that selected encoding is not supported
+	// by this Cid version
+	ErrInvalidEncoding = errors.New("invalid base encoding")
 )
 
 // These are multicodec-packed content types. The should match
@@ -58,6 +62,8 @@ const (
 	DagProtobuf = 0x70
 	DagCBOR     = 0x71
 
+	GitRaw = 0x78
+
 	EthBlock           = 0x90
 	EthBlockList       = 0x91
 	EthTxTrie          = 0x92
@@ -71,8 +77,57 @@ const (
 	BitcoinTx          = 0xb1
 	ZcashBlock         = 0xc0
 	ZcashTx            = 0xc1
+	DecredBlock        = 0xe0
+	DecredTx           = 0xe1
 )
 
+// Codecs maps the name of a codec to its type
+var Codecs = map[string]uint64{
+	"v0":                   DagProtobuf,
+	"raw":                  Raw,
+	"protobuf":             DagProtobuf,
+	"cbor":                 DagCBOR,
+	"git-raw":              GitRaw,
+	"eth-block":            EthBlock,
+	"eth-block-list":       EthBlockList,
+	"eth-tx-trie":          EthTxTrie,
+	"eth-tx":               EthTx,
+	"eth-tx-receipt-trie":  EthTxReceiptTrie,
+	"eth-tx-receipt":       EthTxReceipt,
+	"eth-state-trie":       EthStateTrie,
+	"eth-account-snapshot": EthAccountSnapshot,
+	"eth-storage-trie":     EthStorageTrie,
+	"bitcoin-block":        BitcoinBlock,
+	"bitcoin-tx":           BitcoinTx,
+	"zcash-block":          ZcashBlock,
+	"zcash-tx":             ZcashTx,
+	"decred-block":         DecredBlock,
+	"decred-tx":            DecredTx,
+}
+
+// CodecToStr maps the numeric codec to its name
+var CodecToStr = map[uint64]string{
+	Raw:                "raw",
+	DagProtobuf:        "protobuf",
+	DagCBOR:            "cbor",
+	GitRaw:             "git-raw",
+	EthBlock:           "eth-block",
+	EthBlockList:       "eth-block-list",
+	EthTxTrie:          "eth-tx-trie",
+	EthTx:              "eth-tx",
+	EthTxReceiptTrie:   "eth-tx-receipt-trie",
+	EthTxReceipt:       "eth-tx-receipt",
+	EthStateTrie:       "eth-state-trie",
+	EthAccountSnapshot: "eth-account-snapshot",
+	EthStorageTrie:     "eth-storage-trie",
+	BitcoinBlock:       "bitcoin-block",
+	BitcoinTx:          "bitcoin-tx",
+	ZcashBlock:         "zcash-block",
+	ZcashTx:            "zcash-tx",
+	DecredBlock:        "decred-block",
+	DecredTx:           "decred-tx",
+}
+
 // NewCidV0 returns a Cid-wrapped multihash.
 // They exist to allow IPFS to work with Cids while keeping
 // compatibility with the plain-multihash format used used in IPFS.
@@ -158,6 +213,28 @@ func Decode(v string) (*Cid, error) {
 	return Cast(data)
 }
 
+// Extract the encoding from a Cid.  If Decode on the same string did
+// not return an error neither will this function.
+func ExtractEncoding(v string) (mbase.Encoding, error) {
+	if len(v) < 2 {
+		return -1, ErrCidTooShort
+	}
+
+	if len(v) == 46 && v[:2] == "Qm" {
+		return mbase.Base58BTC, nil
+	}
+
+	encoding := mbase.Encoding(v[0])
+
+	// check encoding is valid
+	_, err := mbase.NewEncoder(encoding)
+	if err != nil {
+		return -1, err
+	}
+
+	return encoding, nil
+}
+
 func uvError(read int) error {
 	switch {
 	case read == 0:
@@ -245,6 +322,22 @@ func (c *Cid) String() string {
 	}
 }
 
+// String returns the string representation of a Cid
+// encoded is selected base
+func (c *Cid) StringOfBase(base mbase.Encoding) (string, error) {
+	switch c.version {
+	case 0:
+		if base != mbase.Base58BTC {
+			return "", ErrInvalidEncoding
+		}
+		return c.hash.B58String(), nil
+	case 1:
+		return mbase.Encode(base, c.bytesV1())
+	default:
+		panic("not possible to reach this point")
+	}
+}
+
 // Hash returns the multihash contained by a Cid.
 func (c *Cid) Hash() mh.Multihash {
 	return c.hash
@@ -324,7 +417,7 @@ func (c *Cid) UnmarshalJSON(b []byte) error {
 //
 // Note that this formatting comes from the IPLD specification
 // (https://github.com/ipld/specs/tree/master/ipld)
-func (c *Cid) MarshalJSON() ([]byte, error) {
+func (c Cid) MarshalJSON() ([]byte, error) {
 	return []byte(fmt.Sprintf("{\"/\":\"%s\"}", c.String())), nil
 }
 
@@ -356,6 +449,8 @@ func (c *Cid) Prefix() Prefix {
 // that is, the Version, the Codec, the Multihash type
 // and the Multihash length. It does not contains
 // any actual content information.
+// NOTE: The use -1 in MhLength to mean default length is deprecated,
+//   use the V0Builder or V1Builder structures instead
 type Prefix struct {
 	Version  uint64
 	Codec    uint64

cid_test.go

@@ -8,9 +8,35 @@ import (
 	"strings"
 	"testing"
 
+	mbase "github.com/multiformats/go-multibase"
 	mh "github.com/multiformats/go-multihash"
 )
 
+// Copying the "silly test" idea from
+// https://github.com/multiformats/go-multihash/blob/7aa9f26a231c6f34f4e9fad52bf580fd36627285/multihash_test.go#L13
+// Makes it so changing the table accidentally has to happen twice.
+var tCodecs = map[uint64]string{
+	Raw:                "raw",
+	DagProtobuf:        "protobuf",
+	DagCBOR:            "cbor",
+	GitRaw:             "git-raw",
+	EthBlock:           "eth-block",
+	EthBlockList:       "eth-block-list",
+	EthTxTrie:          "eth-tx-trie",
+	EthTx:              "eth-tx",
+	EthTxReceiptTrie:   "eth-tx-receipt-trie",
+	EthTxReceipt:       "eth-tx-receipt",
+	EthStateTrie:       "eth-state-trie",
+	EthAccountSnapshot: "eth-account-snapshot",
+	EthStorageTrie:     "eth-storage-trie",
+	BitcoinBlock:       "bitcoin-block",
+	BitcoinTx:          "bitcoin-tx",
+	ZcashBlock:         "zcash-block",
+	ZcashTx:            "zcash-tx",
+	DecredBlock:        "decred-block",
+	DecredTx:           "decred-tx",
+}
+
 func assertEqual(t *testing.T, a, b *Cid) {
 	if a.codec != b.codec {
 		t.Fatal("mismatch on type")
@@ -25,6 +51,26 @@ func assertEqual(t *testing.T, a, b *Cid) {
 	}
 }
 
+func TestTable(t *testing.T) {
+	if len(tCodecs) != len(Codecs)-1 {
+		t.Errorf("Item count mismatch in the Table of Codec. Should be %d, got %d", len(tCodecs)+1, len(Codecs))
+	}
+
+	for k, v := range tCodecs {
+		if Codecs[v] != k {
+			t.Errorf("Table mismatch: 0x%x %s", k, v)
+		}
+	}
+}
+
+// The table returns cid.DagProtobuf for "v0"
+// so we test it apart
+func TestTableForV0(t *testing.T) {
+	if Codecs["v0"] != DagProtobuf {
+		t.Error("Table mismatch: Codecs[\"v0\"] should resolve to DagProtobuf (0x70)")
+	}
+}
+
 func TestBasicMarshaling(t *testing.T) {
 	h, err := mh.Sum([]byte("TEST"), mh.SHA3, 4)
 	if err != nil {
@@ -55,6 +101,60 @@ func TestBasicMarshaling(t *testing.T) {
 	assertEqual(t, cid, out2)
 }
 
+func TestBasesMarshaling(t *testing.T) {
+	h, err := mh.Sum([]byte("TEST"), mh.SHA3, 4)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cid := &Cid{
+		codec:   7,
+		version: 1,
+		hash:    h,
+	}
+
+	data := cid.Bytes()
+
+	out, err := Cast(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assertEqual(t, cid, out)
+
+	testBases := []mbase.Encoding{
+		mbase.Base16,
+		mbase.Base32,
+		mbase.Base32hex,
+		mbase.Base32pad,
+		mbase.Base32hexPad,
+		mbase.Base58BTC,
+		mbase.Base58Flickr,
+		mbase.Base64pad,
+		mbase.Base64urlPad,
+		mbase.Base64url,
+		mbase.Base64,
+	}
+
+	for _, b := range testBases {
+		s, err := cid.StringOfBase(b)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if s[0] != byte(b) {
+			t.Fatal("Invalid multibase header")
+		}
+
+		out2, err := Decode(s)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		assertEqual(t, cid, out2)
+	}
+}
+
 func TestEmptyString(t *testing.T) {
 	_, err := Decode("")
 	if err == nil {
@@ -91,6 +191,64 @@ func TestV0ErrorCases(t *testing.T) {
 	}
 }
 
+func TestNewPrefixV1(t *testing.T) {
+	data := []byte("this is some test content")
+
+	// Construct c1
+	prefix := NewPrefixV1(DagCBOR, mh.SHA2_256)
+	c1, err := prefix.Sum(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if c1.Prefix() != prefix {
+		t.Fatal("prefix not preserved")
+	}
+
+	// Construct c2
+	hash, err := mh.Sum(data, mh.SHA2_256, -1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	c2 := NewCidV1(DagCBOR, hash)
+
+	if !c1.Equals(c2) {
+		t.Fatal("cids mismatch")
+	}
+	if c1.Prefix() != c2.Prefix() {
+		t.Fatal("prefixes mismatch")
+	}
+}
+
+func TestNewPrefixV0(t *testing.T) {
+	data := []byte("this is some test content")
+
+	// Construct c1
+	prefix := NewPrefixV0(mh.SHA2_256)
+	c1, err := prefix.Sum(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if c1.Prefix() != prefix {
+		t.Fatal("prefix not preserved")
+	}
+
+	// Construct c2
+	hash, err := mh.Sum(data, mh.SHA2_256, -1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	c2 := NewCidV0(hash)
+
+	if !c1.Equals(c2) {
+		t.Fatal("cids mismatch")
+	}
+	if c1.Prefix() != c2.Prefix() {
+		t.Fatal("prefixes mismatch")
+	}
+}
+
 func TestPrefixRoundtrip(t *testing.T) {
 	data := []byte("this is some test content")
 	hash, _ := mh.Sum(data, mh.SHA2_256, -1)
@@ -227,3 +385,46 @@ func TestFromJson(t *testing.T) {
 		t.Fatal("json parsing failed")
 	}
 }
+
+func TestJsonRoundTrip(t *testing.T) {
+	exp, err := Decode("zb2rhhFAEMepUBbGyP1k8tGfz7BSciKXP6GHuUeUsJBaK6cqG")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify it works for a *Cid.
+	enc, err := json.Marshal(exp)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var actual Cid
+	err = json.Unmarshal(enc, &actual)
+	if !exp.Equals(&actual) {
+		t.Fatal("cids not equal for *Cid")
+	}
+
+	// Verify it works for a Cid.
+	enc, err = json.Marshal(*exp)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var actual2 Cid
+	err = json.Unmarshal(enc, &actual2)
+	if !exp.Equals(&actual2) {
+		t.Fatal("cids not equal for Cid")
+	}
+}
+
+func BenchmarkStringV1(b *testing.B) {
+	data := []byte("this is some test content")
+	hash, _ := mh.Sum(data, mh.SHA2_256, -1)
+	cid := NewCidV1(Raw, hash)
+	b.ResetTimer()
+	count := 0
+	for i := 0; i < b.N; i++ {
+		count += len(cid.String())
+	}
+	if count != 49*b.N {
+		b.FailNow()
+	}
+}

codecov.yml

@@ -0,0 +1,3 @@
+coverage:
+  range: "50...100"
+comment: off

deprecated.go

@@ -0,0 +1,28 @@
+package cid
+
+import (
+	mh "github.com/multiformats/go-multihash"
+)
+
+// NewPrefixV0 returns a CIDv0 prefix with the specified multihash type.
+// DEPRECATED: Use V0Builder
+func NewPrefixV0(mhType uint64) Prefix {
+	return Prefix{
+		MhType:   mhType,
+		MhLength: mh.DefaultLengths[mhType],
+		Version:  0,
+		Codec:    DagProtobuf,
+	}
+}
+
+// NewPrefixV1 returns a CIDv1 prefix with the specified codec and multihash
+// type.
+// DEPRECATED: Use V1Builder
+func NewPrefixV1(codecType uint64, mhType uint64) Prefix {
+	return Prefix{
+		MhType:   mhType,
+		MhLength: mh.DefaultLengths[mhType],
+		Version:  1,
+		Codec:    codecType,
+	}
+}

package.json

@@ -9,15 +9,15 @@
   "gxDependencies": [
     {
       "author": "whyrusleeping",
-      "hash": "QmVGtdTZdTFaLsaj2RwdVG8jcjNNcp1DE914DKZ2kHmXHw",
+      "hash": "QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8",
       "name": "go-multihash",
-      "version": "1.0.4"
+      "version": "1.0.8"
     },
     {
       "author": "whyrusleeping",
-      "hash": "Qme4T6BE4sQxg7ZouamF5M7Tx1ZFTqzcns7BkyQPXpoT99",
+      "hash": "QmSbvata2WqNkqGtZNg8MR3SKwnB8iQ7vTPJgWqB8bC5kR",
       "name": "go-multibase",
-      "version": "0.2.4"
+      "version": "0.2.7"
     }
   ],
   "gxVersion": "0.8.0",
@@ -25,6 +25,6 @@
   "license": "MIT",
   "name": "go-cid",
   "releaseCmd": "git commit -a -m \"gx publish $VERSION\"",
-  "version": "0.7.13"
+  "version": "0.8.0"
 }
 

set.go

@@ -65,3 +65,4 @@ func (s *Set) ForEach(f func(c *Cid) error) error {
 	}
 	return nil
 }
+

set_test.go

@@ -0,0 +1,92 @@
+package cid
+
+import (
+	"crypto/rand"
+	"errors"
+	"testing"
+
+	mh "github.com/multiformats/go-multihash"
+)
+
+func makeRandomCid(t *testing.T) *Cid {
+	p := make([]byte, 256)
+	_, err := rand.Read(p)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	h, err := mh.Sum(p, mh.SHA3, 4)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cid := &Cid{
+		codec:   7,
+		version: 1,
+		hash:    h,
+	}
+
+	return cid
+}
+
+func TestSet(t *testing.T) {
+	cid := makeRandomCid(t)
+	cid2 := makeRandomCid(t)
+	s := NewSet()
+
+	s.Add(cid)
+
+	if !s.Has(cid) {
+		t.Error("should have the CID")
+	}
+
+	if s.Len() != 1 {
+		t.Error("should report 1 element")
+	}
+
+	keys := s.Keys()
+
+	if len(keys) != 1 || !keys[0].Equals(cid) {
+		t.Error("key should correspond to Cid")
+	}
+
+	if s.Visit(cid) {
+		t.Error("visit should return false")
+	}
+
+	foreach := []*Cid{}
+	foreachF := func(c *Cid) error {
+		foreach = append(foreach, c)
+		return nil
+	}
+
+	if err := s.ForEach(foreachF); err != nil {
+		t.Error(err)
+	}
+
+	if len(foreach) != 1 {
+		t.Error("ForEach should have visited 1 element")
+	}
+
+	foreachErr := func(c *Cid) error {
+		return errors.New("test")
+	}
+
+	if err := s.ForEach(foreachErr); err == nil {
+		t.Error("Should have returned an error")
+	}
+
+	if !s.Visit(cid2) {
+		t.Error("should have visited a new Cid")
+	}
+
+	if s.Len() != 2 {
+		t.Error("len should be 2 now")
+	}
+
+	s.Remove(cid2)
+
+	if s.Len() != 1 {
+		t.Error("len should be 1 now")
+	}
+}