Merge pull request #36 from multiformats/feat/base36

Base36 implementation

.gx/lastpubver

@@ -1 +0,0 @@
-0.3.0: QmekxXDhCxCJRNuzmHreuaT3BsuJcsjcXWNrtV9C8DRHtd

.gxignore

@@ -1,2 +0,0 @@
-/spec/
-*_test.go

.travis.yml

@@ -10,7 +10,6 @@ env:
   global:
     - GOTFLAGS="-race"
   matrix:
-    - BUILD_DEPTYPE=gx
     - BUILD_DEPTYPE=gomod
 
 
@@ -24,7 +23,6 @@ script:
 
 cache:
   directories:
-    - $GOPATH/src/gx
     - $GOPATH/pkg/mod
     - /home/travis/.cache/go-build
 

Makefile

@@ -1,13 +1,7 @@
 test: deps
-	go test -race -v ./...
+	go test -count=1 -race -v ./...
 
 export IPFS_API ?= v04x.ipfs.io
 
-gx:
-	go get -u github.com/whyrusleeping/gx
-	go get -u github.com/whyrusleeping/gx-go
-
-deps: gx
-	gx --verbose install --global
-	gx-go rewrite
+deps:
 	go get -t ./...

README.md

@@ -18,24 +18,6 @@
 go get github.com/multiformats/go-multibase
 ```
 
-Note that `go-multibase` is packaged with Gx, so it is recommended to use Gx to install and use it (see Usage section).
-
-## Usage
-
-This module is packaged with [Gx](https://github.com/whyrusleeping/gx). In order to use it in your own project it is recommended that you:
-
-```sh
-go get -u github.com/whyrusleeping/gx
-go get -u github.com/whyrusleeping/gx-go
-cd <your-project-repository>
-gx init
-gx import github.com/multiformats/go-multibase
-gx install --global
-gx-go --rewrite
-```
-
-Please check [Gx](https://github.com/whyrusleeping/gx) and [Gx-go](https://github.com/whyrusleeping/gx-go) documentation for more information.
-
 ## Contribute
 
 Contributions welcome. Please check out [the issues](https://github.com/multiformats/go-multibase/issues).

go.mod

@@ -1,6 +1,9 @@
 module github.com/multiformats/go-multibase
 
+go 1.11
+
 require (
 	github.com/mr-tron/base58 v1.1.0
 	github.com/multiformats/go-base32 v0.0.3
+	github.com/multiformats/go-base36 v0.1.0
 )

go.sum

@@ -2,3 +2,5 @@ github.com/mr-tron/base58 v1.1.0 h1:Y51FGVJ91WBqCEabAi5OPUz38eAx8DakuAm5svLcsfQ=
 github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
 github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI=
 github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
+github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4=
+github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM=

multibase.go

@@ -7,6 +7,7 @@ import (
 
 	b58 "github.com/mr-tron/base58/base58"
 	b32 "github.com/multiformats/go-base32"
+	b36 "github.com/multiformats/go-base36"
 )
 
 // Encoding identifies the type of base-encoding that a multibase is carrying.
@@ -29,37 +30,18 @@ const (
 	Base32hexUpper    = 'V'
 	Base32hexPad      = 't'
 	Base32hexPadUpper = 'T'
-	Base58Flickr      = 'Z'
+	Base36            = 'k'
+	Base36Upper       = 'K'
 	Base58BTC         = 'z'
+	Base58Flickr      = 'Z'
 	Base64            = 'm'
 	Base64url         = 'u'
 	Base64pad         = 'M'
 	Base64urlPad      = 'U'
 )
 
-// Encodings is a map of the supported encoding, unsupported encoding
+// EncodingToStr is a map of the supported encoding, unsupported encoding
 // specified in standard are left out
-var Encodings = map[string]Encoding{
-	"identity":          0x00,
-	"base2":             '0',
-	"base16":            'f',
-	"base16upper":       'F',
-	"base32":            'b',
-	"base32upper":       'B',
-	"base32pad":         'c',
-	"base32padupper":    'C',
-	"base32hex":         'v',
-	"base32hexupper":    'V',
-	"base32hexpad":      't',
-	"base32hexpadupper": 'T',
-	"base58flickr":      'Z',
-	"base58btc":         'z',
-	"base64":            'm',
-	"base64url":         'u',
-	"base64pad":         'M',
-	"base64urlpad":      'U',
-}
-
 var EncodingToStr = map[Encoding]string{
 	0x00: "identity",
 	'0':  "base2",
@@ -73,14 +55,24 @@ var EncodingToStr = map[Encoding]string{
 	'V':  "base32hexupper",
 	't':  "base32hexpad",
 	'T':  "base32hexpadupper",
-	'Z':  "base58flickr",
+	'k':  "base36",
+	'K':  "base36upper",
 	'z':  "base58btc",
+	'Z':  "base58flickr",
 	'm':  "base64",
 	'u':  "base64url",
 	'M':  "base64pad",
 	'U':  "base64urlpad",
 }
 
+var Encodings = map[string]Encoding{}
+
+func init() {
+	for e, n := range EncodingToStr {
+		Encodings[n] = e
+	}
+}
+
 // ErrUnsupportedEncoding is returned when the selected encoding is not known or
 // implemented.
 var ErrUnsupportedEncoding = fmt.Errorf("selected encoding not supported")
@@ -115,6 +107,10 @@ func Encode(base Encoding, data []byte) (string, error) {
 		return string(Base32hexPad) + base32HexLowerPad.EncodeToString(data), nil
 	case Base32hexPadUpper:
 		return string(Base32hexPadUpper) + base32HexUpperPad.EncodeToString(data), nil
+	case Base36:
+		return string(Base36) + b36.EncodeToStringLc(data), nil
+	case Base36Upper:
+		return string(Base36Upper) + b36.EncodeToStringUc(data), nil
 	case Base58BTC:
 		return string(Base58BTC) + b58.EncodeAlphabet(data, b58.BTCAlphabet), nil
 	case Base58Flickr:
@@ -162,6 +158,9 @@ func Decode(data string) (Encoding, []byte, error) {
 	case Base32hexPad, Base32hexPadUpper:
 		bytes, err := b32.HexEncoding.DecodeString(data[1:])
 		return enc, bytes, err
+	case Base36, Base36Upper:
+		bytes, err := b36.DecodeString(data[1:])
+		return enc, bytes, err
 	case Base58BTC:
 		bytes, err := b58.DecodeAlphabet(data[1:], b58.BTCAlphabet)
 		return Base58BTC, bytes, err

multibase_test.go

@@ -3,6 +3,7 @@ package multibase
 import (
 	"bytes"
 	"math/rand"
+	"sort"
 	"testing"
 )
 
@@ -35,7 +36,10 @@ var encodedSamples = map[Encoding]string{
 	Base32hexUpper:    "V8HIM6PBEEHP62R39F9II0PBMCLP7IT38D5N6E89144",
 	Base32hexPad:      "t8him6pbeehp62r39f9ii0pbmclp7it38d5n6e89144======",
 	Base32hexPadUpper: "T8HIM6PBEEHP62R39F9II0PBMCLP7IT38D5N6E89144======",
+	Base36:            "km552ng4dabi4neu1oo8l4i5mndwmpc3mkukwtxy9",
+	Base36Upper:       "KM552NG4DABI4NEU1OO8L4I5MNDWMPC3MKUKWTXY9",
 	Base58BTC:         "z36UQrhJq9fNDS7DiAHM9YXqDHMPfr4EMArvt",
+	Base58Flickr:      "Z36tpRGiQ9Endr7dHahm9xwQdhmoER4emaRVT",
 	Base64:            "mRGVjZW50cmFsaXplIGV2ZXJ5dGhpbmchISE",
 	Base64url:         "uRGVjZW50cmFsaXplIGV2ZXJ5dGhpbmchISE",
 	Base64pad:         "MRGVjZW50cmFsaXplIGV2ZXJ5dGhpbmchISE=",
@@ -49,7 +53,7 @@ func testEncode(t *testing.T, encoding Encoding, bytes []byte, expected string)
 		return
 	}
 	if actual != expected {
-		t.Errorf("encoding failed for %c (%d), expected: %s, got: %s", encoding, encoding, expected, actual)
+		t.Errorf("encoding failed for %c (%d / %s), expected: %s, got: %s", encoding, encoding, EncodingToStr[encoding], expected, actual)
 	}
 }
 
@@ -68,79 +72,178 @@ func testDecode(t *testing.T, expectedEncoding Encoding, expectedBytes []byte, d
 }
 
 func TestEncode(t *testing.T) {
-	for encoding, data := range encodedSamples {
-		testEncode(t, encoding, sampleBytes, data)
+	for encoding := range EncodingToStr {
+		testEncode(t, encoding, sampleBytes, encodedSamples[encoding])
 	}
 }
 
 func TestDecode(t *testing.T) {
-	for encoding, data := range encodedSamples {
-		testDecode(t, encoding, sampleBytes, data)
+	for encoding := range EncodingToStr {
+		testDecode(t, encoding, sampleBytes, encodedSamples[encoding])
 	}
 }
 
 func TestRoundTrip(t *testing.T) {
-	buf := make([]byte, 17)
-	rand.Read(buf)
 
-	baseList := []Encoding{Identity, Base16, Base32, Base32hex, Base32pad, Base32hexPad, Base58BTC, Base58Flickr, Base64pad, Base64urlPad}
-
-	for _, base := range baseList {
-		enc, err := Encode(base, buf)
-		if err != nil {
-			t.Fatal(err)
+	for base := range EncodingToStr {
+		if int(base) == 0 {
+			// skip identity: any byte goes there
+			continue
 		}
 
-		e, out, err := Decode(enc)
-		if err != nil {
-			t.Fatal(err)
+		_, _, err := Decode(string(base) + "\u00A0")
+		if err == nil {
+			t.Fatal(EncodingToStr[base] + " decode should fail on low-unicode")
 		}
 
-		if e != base {
-			t.Fatal("got wrong encoding out")
+		_, _, err = Decode(string(base) + "\u1F4A8")
+		if err == nil {
+			t.Fatal(EncodingToStr[base] + " decode should fail on emoji")
 		}
 
-		if !bytes.Equal(buf, out) {
-			t.Fatal("input wasnt the same as output", buf, out)
+		_, _, err = Decode(string(base) + "!")
+		if err == nil {
+			t.Fatal(EncodingToStr[base] + " decode should fail on punctuation")
+		}
+
+		_, _, err = Decode(string(base) + "\xA0")
+		if err == nil {
+			t.Fatal(EncodingToStr[base] + " decode should fail on high-latin1")
+		}
+	}
+
+	buf := make([]byte, 137+16) // sufficiently large prime number of bytes + another 16 to test leading 0s
+	rand.Read(buf[16:])
+
+	for base := range EncodingToStr {
+
+		// test roundtrip from the full zero-prefixed buffer down to a single byte
+		for i := 0; i < len(buf); i++ {
+
+			// use a copy to verify we are not overwriting the supplied buffer
+			newBuf := make([]byte, len(buf)-i)
+			copy(newBuf, buf[i:])
+
+			enc, err := Encode(base, newBuf)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			e, out, err := Decode(enc)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if e != base {
+				t.Fatal("got wrong encoding out")
+			}
+
+			if !bytes.Equal(newBuf, buf[i:]) {
+				t.Fatal("the provided buffer was modified", buf[i:], out)
+			}
+
+			if !bytes.Equal(buf[i:], out) {
+				t.Fatal("input wasnt the same as output", buf[i:], out)
+			}
+
+			// When we have 3 leading zeroes, do a few extra tests
+			// ( choice of leading zeroes is arbitrary - just cutting down on test permutations )
+
+			if i == 13 {
+
+				// if this is a case-insensitive codec semi-randomly swap case in enc and try again
+				name := EncodingToStr[base]
+				if name[len(name)-5:] == "upper" || Encodings[name+"upper"] > 0 {
+					caseTamperedEnc := []byte(enc)
+
+					for _, j := range []int{3, 5, 8, 13, 21, 23, 29, 47, 52} {
+						if caseTamperedEnc[j] >= 65 && caseTamperedEnc[j] <= 90 {
+							caseTamperedEnc[j] += 32
+						} else if caseTamperedEnc[j] >= 97 && caseTamperedEnc[j] <= 122 {
+							caseTamperedEnc[j] -= 32
+						}
+					}
+
+					e, out, err := Decode(string(caseTamperedEnc))
+					if err != nil {
+						t.Fatal(err)
+					}
+
+					if e != base {
+						t.Fatal("got wrong encoding out")
+					}
+					if !bytes.Equal(buf[i:], out) {
+						t.Fatal("input wasn't the same as output", buf[i:], out)
+					}
+				}
+			}
+		}
+	}
+
+	// Test that nothing overflows
+	maxValueBuf := make([]byte, 131)
+	for i := 0; i < len(maxValueBuf); i++ {
+		maxValueBuf[i] = 0xFF
+	}
+
+	for base := range EncodingToStr {
+
+		// test roundtrip from the complete buffer down to a single byte
+		for i := 0; i < len(maxValueBuf); i++ {
+
+			enc, err := Encode(base, maxValueBuf[i:])
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			e, out, err := Decode(enc)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if e != base {
+				t.Fatal("got wrong encoding out")
+			}
+
+			if !bytes.Equal(maxValueBuf[i:], out) {
+				t.Fatal("input wasn't the same as output", maxValueBuf[i:], out)
+			}
 		}
 	}
 
 	_, _, err := Decode("")
 	if err == nil {
-		t.Fatal("shouldnt be able to decode empty string")
+		t.Fatal("shouldn't be able to decode empty string")
 	}
 }
 
+var benchmarkBuf [36]byte // typical CID size
+var benchmarkCodecs []string
+
+func init() {
+	rand.Read(benchmarkBuf[:])
+
+	benchmarkCodecs = make([]string, 0, len(Encodings))
+	for n := range Encodings {
+
+		// // Only bench b36 and b58
+		// if len(n) < 6 || (n[4:6] != "36" && n[4:6] != "58") {
+		// 	continue
+		// }
+
+		benchmarkCodecs = append(benchmarkCodecs, n)
+	}
+	sort.Strings(benchmarkCodecs)
+}
+
 func BenchmarkRoundTrip(b *testing.B) {
-	buf := make([]byte, 32)
-	rand.Read(buf)
 	b.ResetTimer()
 
-	bases := map[string]Encoding{
-		"Identity":          Identity,
-		"Base2":             Base2,
-		"Base16":            Base16,
-		"Base16Upper":       Base16Upper,
-		"Base32":            Base32,
-		"Base32Upper":       Base32Upper,
-		"Base32pad":         Base32pad,
-		"Base32padUpper":    Base32padUpper,
-		"Base32hex":         Base32hex,
-		"Base32hexUpper":    Base32hexUpper,
-		"Base32hexPad":      Base32hexPad,
-		"Base32hexPadUpper": Base32hexPadUpper,
-		"Base58Flickr":      Base58Flickr,
-		"Base58BTC":         Base58BTC,
-		"Base64":            Base64,
-		"Base64url":         Base64url,
-		"Base64pad":         Base64pad,
-		"Base64urlPad":      Base64urlPad,
-	}
-
-	for name, base := range bases {
+	for _, name := range benchmarkCodecs {
 		b.Run(name, func(b *testing.B) {
+			base := Encodings[name]
 			for i := 0; i < b.N; i++ {
-				enc, err := Encode(base, buf)
+				enc, err := Encode(base, benchmarkBuf[:])
 				if err != nil {
 					b.Fatal(err)
 				}
@@ -154,8 +257,40 @@ func BenchmarkRoundTrip(b *testing.B) {
 					b.Fatal("got wrong encoding out")
 				}
 
-				if !bytes.Equal(buf, out) {
-					b.Fatal("input wasnt the same as output", buf, out)
+				if !bytes.Equal(benchmarkBuf[:], out) {
+					b.Fatal("input wasnt the same as output", benchmarkBuf, out)
+				}
+			}
+		})
+	}
+}
+
+func BenchmarkEncode(b *testing.B) {
+	b.ResetTimer()
+
+	for _, name := range benchmarkCodecs {
+		b.Run(name, func(b *testing.B) {
+			base := Encodings[name]
+			for i := 0; i < b.N; i++ {
+				_, err := Encode(base, benchmarkBuf[:])
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}
+
+func BenchmarkDecode(b *testing.B) {
+	b.ResetTimer()
+
+	for _, name := range benchmarkCodecs {
+		b.Run(name, func(b *testing.B) {
+			enc, _ := Encode(Encodings[name], benchmarkBuf[:])
+			for i := 0; i < b.N; i++ {
+				_, _, err := Decode(enc)
+				if err != nil {
+					b.Fatal(err)
 				}
 			}
 		})

package.json

@@ -3,28 +3,8 @@
   "bugs": {
     "url": "https://github.com/multiformats/go-multibase"
   },
-  "gx": {
-    "dvcsimport": "github.com/multiformats/go-multibase"
-  },
-  "gxDependencies": [
-    {
-      "author": "mr-tron",
-      "hash": "QmWFAMPqsEyUX7gDUsRVmMWz59FxSpJ1b2v6bJ1yYzo7jY",
-      "name": "go-base58-fast",
-      "version": "0.1.1"
-    },
-    {
-      "author": "Golang",
-      "hash": "QmPbbYin7KBd1Y1BfUe15vHzwJiioyi3wtKQTtXWWf8SC5",
-      "name": "base32",
-      "version": "0.0.3"
-    }
-  ],
-  "gxVersion": "0.8.0",
   "language": "go",
   "license": "",
   "name": "go-multibase",
-  "releaseCmd": "git commit -a -m \"gx publish $VERSION\"",
   "version": "0.3.0"
 }
-

spec_test.go

@@ -19,7 +19,7 @@ func TestSpec(t *testing.T) {
 
 	reader := csv.NewReader(file)
 	reader.LazyQuotes = false
-	reader.FieldsPerRecord = 3
+	reader.FieldsPerRecord = 4
 	reader.TrimLeadingSpace = true
 
 	values, err := reader.ReadAll()