From 64d960ce3d3d0a655570f5e0a9bab42054a3467f Mon Sep 17 00:00:00 2001 From: Erin Swenson-Healey Date: Sat, 7 Apr 2018 15:17:53 -0700 Subject: [PATCH] add audit proof --- README.md | 74 ++++++++++++- hashing.go | 14 +++ main.go | 136 ----------------------- main_test.go | 137 ----------------------- merkletree.go | 266 +++++++++++++++++++++++++++++++++++++++++++++ merkletree_test.go | 182 +++++++++++++++++++++++++++++++ 6 files changed, 535 insertions(+), 274 deletions(-) create mode 100644 hashing.go delete mode 100644 main.go delete mode 100644 main_test.go create mode 100644 merkletree.go create mode 100644 merkletree_test.go diff --git a/README.md b/README.md index f8b62f0..cc047ac 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,74 @@ # go-merkle-tree -A Merkle tree, implemented in Golang + +> A Merkle Tree, implemented in Golang + +Many people have written many things about Merkle Trees. For a good overview (uses, characteristics, etc.), read Marc +Clifton's [_Understanding Merkle Trees - Why use them, who uses them, and how to use them_][1]. + +## Warning + +*Warning: This is alpha software.* + +## Usage + +### Construction + +```go +data := [][]byte{[]byte("alpha"), []byte("beta"), []byte("kappa")} + +tree := CreateTree(Sha256DoubleHash, data) + +fmt.Println(tree.AsString(hex.EncodeToString, 0)) + +/* + +output: + +(B root: 1add1cfdf5df28414b715199a740f80b7f559bd558a3f0c0186e60149ee86620 + (B root: 65492c0681df09eb403160136bb648de17f67bd8efc441467c0fc23b8d2950e9 + (L root: aa86be763e41db7eaae266afc79ab46d02343c5d3b05da171d351afbd25c1525) + (L root: 05e3bc756e005c1bc5e4daf8a3da95d435af52476b0a0e6d52e719a2b1e3434a)) + (B root: 3ae3330dcf932104d42b75b4da386896a628926f411737b34430fa65e526824d + (L root: 4cc9e99389b5f729cbef6fe79e97a6f562841a2852e25e508e3bd06ce0de9c26) + (L root: 4cc9e99389b5f729cbef6fe79e97a6f562841a2852e25e508e3bd06ce0de9c26))) + +*/ +``` + +### Audit Proof + +```go + +blocks := [][]byte{ + []byte("alpha"), + []byte("beta"), + []byte("kappa"), + []byte("gamma"), + []byte("epsilon"), + []byte("omega"), + []byte("mu"), + []byte("zeta"), +} + +tree := NewTree(IdentityHashForTest, blocks) +checksum := treeA.checksumFunc([]byte("omega")) + +// for printing checksums +f := func(xs []byte) string { + return string(xs) +} + +fmt.Println(tree.GetProofString(tree.root.GetChecksum(), checksum, f)) + +/* + +output: + +epsilon + omega = epsilonomega +epsilonomega + muzeta = epsilonomegamuzeta +alphabetakappagamma + epsilonomegamuzeta = alphabetakappagammaepsilonomegamuzeta + +*/ +``` + +[1]: https://www.codeproject.com/Articles/1176140/Understanding-Merkle-Trees-Why-use-them-who-uses-t \ No newline at end of file diff --git a/hashing.go b/hashing.go new file mode 100644 index 0000000..843cc96 --- /dev/null +++ b/hashing.go @@ -0,0 +1,14 @@ +package merkletree + +import "crypto/sha256" + +func Sha256DoubleHash(data []byte) []byte { + first := sha256.Sum256(data) + secnd := sha256.Sum256(first[:]) + + return secnd[:] +} + +func IdentityHashForTest(strbytes []byte) []byte { + return strbytes +} diff --git a/main.go b/main.go deleted file mode 100644 index b97c92f..0000000 --- a/main.go +++ /dev/null @@ -1,136 +0,0 @@ -package main - -import ( - "crypto/sha256" - "encoding/hex" - "fmt" - "math" -) - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// TYPES -//////// - -type Node interface { - Checksum() []byte - AsString(func([]byte) string, int) string -} - -type Branch struct { - checksum []byte - left Node - right Node -} - -type Leaf struct { - checksum []byte - block []byte -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// CONSTRUCTORS -/////////////// - -func NewLeaf(sum func([]byte) []byte, block []byte) *Leaf { - return &Leaf{ - checksum: sum(block), - block: block, - } -} - -func NewBranch(sum func([]byte) []byte, left Node, right Node) *Branch { - return &Branch{ - checksum: sum(append(left.Checksum(), right.Checksum()...)), - left: left, - right: right, - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// METHODS -////////// - -func (m *Branch) Checksum() []byte { - return m.checksum -} - -func (m *Branch) AsString(checksumToString func([]byte) string, n int) string { - c := checksumToString(m.checksum) - l := m.left.AsString(checksumToString, n+2) - r := m.right.AsString(checksumToString, n+2) - - return fmt.Sprintf("\n"+indent(n, "(B root: %s %s %s)"), c, l, r) -} - -func (m *Leaf) Checksum() []byte { - return m.checksum -} - -func (m *Leaf) AsString(f func([]byte) string, n int) string { - return fmt.Sprintf("\n"+indent(n, "(L root: %s)"), f(m.checksum)) -} - -func CreateTree(sum func([]byte) []byte, blocks [][]byte) Node { - levels := int(math.Ceil(math.Log2(float64(len(blocks)+len(blocks)%2))) + 1) - - // represents each row in the tree, where rows[0] is the base and rows[len(rows)-1] is the root - rows := make([][]Node, levels) - - // build our base of leaves - for i := 0; i < len(blocks); i++ { - rows[0] = append(rows[0], NewLeaf(sum, blocks[i])) - } - - // build upwards until we hit the root - for i := 1; i < levels; i++ { - prev := rows[i-1] - - // each iteration creates a branch from a pair of values originating from the previous level - for j := 0; j < len(prev); j = j + 2 { - var l, r Node - - // if we don't have enough to make a pair, duplicate the left - if j+1 >= len(prev) { - l = prev[j] - r = l - } else { - l = prev[j] - r = prev[j+1] - } - - rows[i] = append(rows[i], NewBranch(sum, l, r)) - } - } - - root := rows[len(rows)-1][0] - - return root -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// UTILITIES -//////////// - -func indent(spaces int, orig string) string { - str := "" - for i := 0; i < spaces; i++ { - str += " " - } - - return str + orig -} - -func main() { - stuff := [][]byte{[]byte("alpha"), []byte("beta"), []byte("kappa"), []byte("gamma"), []byte("epsilon"), []byte("omega")} - - doubleSha256 := func(data []byte) []byte { - first := sha256.Sum256(data) - secnd := sha256.Sum256(first[:]) - - return secnd[:] - } - - tree := CreateTree(doubleSha256, stuff) - - fmt.Println(tree.AsString(hex.EncodeToString, 0)) -} diff --git a/main_test.go b/main_test.go deleted file mode 100644 index 0a1a99e..0000000 --- a/main_test.go +++ /dev/null @@ -1,137 +0,0 @@ -package main - -import ( - "fmt" - "strings" - "testing" -) - -func trimNewlines(str string) string { - return strings.Trim(str, "\n") -} - -func expectEqual(t *testing.T, actual string, expected string) { - if trimNewlines(actual) != expected { - fmt.Println(fmt.Sprintf("=====ACTUAL======\n\n%s\n\n=====EXPECTED======\n\n%s\n", actual, expected)) - t.Fail() - } -} - -func id(strbytes []byte) []byte { - return strbytes -} - -func str(strbytes []byte) string { - return string(strbytes) -} - -var givenOneBlock = trimNewlines(` -(B root: alphaalpha - (L root: alpha) - (L root: alpha)) -`) - -var givenFourBlocks = trimNewlines(` -(B root: alphabetakappagamma - (B root: alphabeta - (L root: alpha) - (L root: beta)) - (B root: kappagamma - (L root: kappa) - (L root: gamma))) -`) - -var givenTwoBlocks = trimNewlines(` -(B root: alphabeta - (L root: alpha) - (L root: beta)) -`) - -var givenThreeBlocks = trimNewlines(` -(B root: alphabetakappakappa - (B root: alphabeta - (L root: alpha) - (L root: beta)) - (B root: kappakappa - (L root: kappa) - (L root: kappa))) -`) - -var givenSixBlocks = trimNewlines(` -(B root: alphabetakappagammaepsilonomegaepsilonomega - (B root: alphabetakappagamma - (B root: alphabeta - (L root: alpha) - (L root: beta)) - (B root: kappagamma - (L root: kappa) - (L root: gamma))) - (B root: epsilonomegaepsilonomega - (B root: epsilonomega - (L root: epsilon) - (L root: omega)) - (B root: epsilonomega - (L root: epsilon) - (L root: omega)))) -`) - -func TestMerkleTree(t *testing.T) { - t.Run("easy tree - just one level (the root) of nodes", func(t *testing.T) { - stuff := [][]byte{[]byte("alpha"), []byte("beta")} - tree := CreateTree(id, stuff) - - expectEqual(t, tree.AsString(str, 0), givenTwoBlocks) - }) - - t.Run("two levels of nodes", func(t *testing.T) { - stuff := [][]byte{[]byte("alpha"), []byte("beta"), []byte("kappa"), []byte("gamma")} - tree := CreateTree(id, stuff) - - expectEqual(t, tree.AsString(str, 0), givenFourBlocks) - }) - - t.Run("one block - one level", func(t *testing.T) { - stuff := [][]byte{[]byte("alpha")} - tree := CreateTree(id, stuff) - - expectEqual(t, tree.AsString(str, 0), givenOneBlock) - }) - - /* - - duplicate a leaf - - 123{3} - / \ - 12 3{3} - / \ / \ - 1 2 3 {3} - - */ - t.Run("duplicate a leaf to keep the binary tree balanced", func(t *testing.T) { - stuff := [][]byte{[]byte("alpha"), []byte("beta"), []byte("kappa")} - tree := CreateTree(id, stuff) - - expectEqual(t, tree.AsString(str, 0), givenThreeBlocks) - }) - - /* - - duplicate a node - - 123456{56} - / \ - 1234 56{56} - / \ / \ - 12 34 56 {56} - / \ / \ / \ / \ - 1 2 3 4 5 6 {5} {6} - - */ - t.Run("duplicate a branch to keep the tree balanced", func(t *testing.T) { - stuff := [][]byte{[]byte("alpha"), []byte("beta"), []byte("kappa"), []byte("gamma"), []byte("epsilon"), []byte("omega")} - tree := CreateTree(id, stuff) - - expectEqual(t, tree.AsString(str, 0), givenSixBlocks) - }) -} diff --git a/merkletree.go b/merkletree.go new file mode 100644 index 0000000..74ba33e --- /dev/null +++ b/merkletree.go @@ -0,0 +1,266 @@ +package merkletree + +import ( + "bytes" + "fmt" + "math" + "strings" +) + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TYPES +//////// + +type Tree struct { + root Node + leaves []*Leaf + checksumFunc func([]byte) []byte +} + +type Node interface { + GetChecksum() []byte + ToString(func([]byte) string, int) string + SetParent(*Branch) + GetParent() *Branch +} + +type Branch struct { + checksum []byte + left Node + right Node + parent *Branch +} + +type Leaf struct { + checksum []byte + block []byte + parent *Branch +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// CONSTRUCTORS +/////////////// + +func NewLeaf(sumFunc func([]byte) []byte, block []byte, parent *Branch) *Leaf { + return &Leaf{ + checksum: sumFunc(block), + block: block, + parent: parent, + } +} + +func NewBranch(sumFunc func([]byte) []byte, left Node, right Node, parent *Branch) *Branch { + return &Branch{ + checksum: sumFunc(append(left.GetChecksum(), right.GetChecksum()...)), + left: left, + right: right, + parent: parent, + } +} + +func NewTree(sumFunc func([]byte) []byte, blocks [][]byte) *Tree { + levels := int(math.Ceil(math.Log2(float64(len(blocks)+len(blocks)%2))) + 1) + + // represents each row in the tree, where rows[0] is the base and rows[len(rows)-1] is the root + rows := make([][]Node, levels) + + // build our base of leaves + for i := 0; i < len(blocks); i++ { + rows[0] = append(rows[0], NewLeaf(sumFunc, blocks[i], nil)) + } + + // build upwards until we hit the root + for i := 1; i < levels; i++ { + prev := rows[i-1] + + // each iteration creates a branch from a pair of values originating from the previous level + for j := 0; j < len(prev); j = j + 2 { + var l, r Node + + // if we don't have enough to make a pair, duplicate the left + if j+1 >= len(prev) { + l = prev[j] + r = l + } else { + l = prev[j] + r = prev[j+1] + } + + // yuck + b := NewBranch(sumFunc, l, r, nil) + l.SetParent(b) + r.SetParent(b) + + rows[i] = append(rows[i], b) + } + } + + leaves := make([]*Leaf, len(rows[0])) + for i := 0; i < len(leaves); i++ { + leaves[i] = rows[0][i].(*Leaf) + } + + return &Tree{ + checksumFunc: sumFunc, + leaves: leaves, + root: rows[len(rows)-1][0], + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// METHODS +////////// + +func (m *Branch) SetParent(parent *Branch) { + m.parent = parent +} + +func (m *Branch) GetParent() *Branch { + return m.parent +} + +func (m *Branch) GetChecksum() []byte { + return m.checksum +} + +func (m *Branch) ToString(checksumToString func([]byte) string, n int) string { + c := checksumToString(m.checksum) + l := m.left.ToString(checksumToString, n+2) + r := m.right.ToString(checksumToString, n+2) + + return fmt.Sprintf("\n"+indent(n, "(B root: %s %s %s)"), c, l, r) +} + +func (m *Leaf) SetParent(parent *Branch) { + m.parent = parent +} + +func (m *Leaf) GetParent() *Branch { + return m.parent +} + +func (m *Leaf) GetChecksum() []byte { + return m.checksum +} + +func (m *Leaf) ToString(f func([]byte) string, n int) string { + return fmt.Sprintf("\n"+indent(n, "(L root: %s)"), f(m.checksum)) +} + +func (t *Tree) Verify(rootChecksum []byte, leafChecksum []byte) bool { + if !bytes.Equal(rootChecksum, t.root.GetChecksum()) { + return false + } + + var found Node = nil + for i := 0; i < len(t.leaves); i++ { + if bytes.Equal(leafChecksum, t.leaves[i].GetChecksum()) { + found = t.leaves[i] + } + } + + if found == nil { + return false + } + + return true +} + +type PathPart struct { + left []byte + right []byte +} + +func (t *Tree) GetProofString(rootChecksum []byte, leafChecksum []byte, f func([]byte) string) string { + var lines []string + + parts := t.AuditPath(rootChecksum, leafChecksum) + if len(parts) == 0 { + return "" + } + + for _, part := range parts { + lines = append(lines, fmt.Sprintf("%s + %s = %s", f(part.left), f(part.right), f(t.checksumFunc(append(part.left, part.right...))))) + } + + return strings.Join(lines, "\n") +} + +// AuditPath returns the path from leaf to root. +func (t *Tree) AuditPath(rootChecksum []byte, leafChecksum []byte) []PathPart { + var pathParts []PathPart + + if !bytes.Equal(rootChecksum, t.root.GetChecksum()) { + return pathParts + } + + found := t.getLeafByChecksum(leafChecksum) + if found == nil { + return pathParts + } + + // start with the immediate parent of the target checksum + fparent := found.GetParent + if bytes.Equal(leafChecksum, fparent().left.GetChecksum()) { + pathParts = append(pathParts, PathPart{ + left: leafChecksum, + right: fparent().right.GetChecksum(), + }) + } else if bytes.Equal(leafChecksum, fparent().right.GetChecksum()) { + pathParts = append(pathParts, PathPart{ + left: fparent().left.GetChecksum(), + right: leafChecksum, + }) + } + + // once we've computed the checksum of the target + its sibling, work our way up towards the root + gparent := fparent().GetParent() + for gparent != nil { + h := t.checksumFunc(append(pathParts[len(pathParts)-1].left, pathParts[len(pathParts)-1].right...)) + + if bytes.Equal(h, gparent.left.GetChecksum()) { + pathParts = append(pathParts, PathPart{ + left: h, + right: gparent.right.GetChecksum(), + }) + } else if bytes.Equal(h, gparent.right.GetChecksum()) { + pathParts = append(pathParts, PathPart{ + left: gparent.left.GetChecksum(), + right: h, + }) + } + + gparent = gparent.parent + } + + return pathParts +} + +func (t *Tree) ToString(f func([]byte) string, n int) string { + return t.root.ToString(f, n) +} + +func (t *Tree) getLeafByChecksum(checksum []byte) *Leaf { + var found *Leaf = nil + + for i := 0; i < len(t.leaves); i++ { + if bytes.Equal(checksum, t.leaves[i].GetChecksum()) { + found = t.leaves[i] + } + } + + return found +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// HELPERS +////////// + +func indent(spaces int, orig string) string { + str := "" + for i := 0; i < spaces; i++ { + str += " " + } + + return str + orig +} diff --git a/merkletree_test.go b/merkletree_test.go new file mode 100644 index 0000000..fc854c6 --- /dev/null +++ b/merkletree_test.go @@ -0,0 +1,182 @@ +package merkletree + +import ( + "fmt" + "strings" + "testing" +) + +func trimNewlines(str string) string { + return strings.Trim(str, "\n") +} + +func expectStrEqual(t *testing.T, actual string, expected string) { + if trimNewlines(actual) != expected { + fmt.Println(fmt.Sprintf("=====ACTUAL======\n\n%s\n\n=====EXPECTED======\n\n%s\n", actual, expected)) + t.Fail() + } +} + +func str(strbytes []byte) string { + return string(strbytes) +} + +var givenOneBlock = trimNewlines(` +(B root: alphaalpha + (L root: alpha) + (L root: alpha)) +`) + +var givenFourBlocks = trimNewlines(` +(B root: alphabetakappagamma + (B root: alphabeta + (L root: alpha) + (L root: beta)) + (B root: kappagamma + (L root: kappa) + (L root: gamma))) +`) + +var givenTwoBlocks = trimNewlines(` +(B root: alphabeta + (L root: alpha) + (L root: beta)) +`) + +var givenThreeBlocks = trimNewlines(` +(B root: alphabetakappakappa + (B root: alphabeta + (L root: alpha) + (L root: beta)) + (B root: kappakappa + (L root: kappa) + (L root: kappa))) +`) + +var givenSixBlocks = trimNewlines(` +(B root: alphabetakappagammaepsilonomegaepsilonomega + (B root: alphabetakappagamma + (B root: alphabeta + (L root: alpha) + (L root: beta)) + (B root: kappagamma + (L root: kappa) + (L root: gamma))) + (B root: epsilonomegaepsilonomega + (B root: epsilonomega + (L root: epsilon) + (L root: omega)) + (B root: epsilonomega + (L root: epsilon) + (L root: omega)))) +`) + +var proofA = trimNewlines(` +epsilon + omega = epsilonomega +epsilonomega + muzeta = epsilonomegamuzeta +alphabetakappagamma + epsilonomegamuzeta = alphabetakappagammaepsilonomegamuzeta +`) + +func TestCreateMerkleTree(t *testing.T) { + t.Run("easy tree - just one level (the root) of nodes", func(t *testing.T) { + blocks := [][]byte{[]byte("alpha"), []byte("beta")} + tree := NewTree(IdentityHashForTest, blocks) + + expectStrEqual(t, tree.ToString(str, 0), givenTwoBlocks) + }) + + t.Run("two levels of nodes", func(t *testing.T) { + blocks := [][]byte{[]byte("alpha"), []byte("beta"), []byte("kappa"), []byte("gamma")} + tree := NewTree(IdentityHashForTest, blocks) + + expectStrEqual(t, tree.ToString(str, 0), givenFourBlocks) + }) + + t.Run("one block - one level", func(t *testing.T) { + blocks := [][]byte{[]byte("alpha")} + tree := NewTree(IdentityHashForTest, blocks) + + expectStrEqual(t, tree.ToString(str, 0), givenOneBlock) + }) + + /* + + duplicate a leaf + + 123{3} + / \ + 12 3{3} + / \ / \ + 1 2 3 {3} + + */ + t.Run("duplicate a leaf to keep the binary tree balanced", func(t *testing.T) { + blocks := [][]byte{[]byte("alpha"), []byte("beta"), []byte("kappa")} + tree := NewTree(IdentityHashForTest, blocks) + + expectStrEqual(t, tree.ToString(str, 0), givenThreeBlocks) + }) + + /* + + duplicate a node + + 123456{56} + / \ + 1234 56{56} + / \ / \ + 12 34 56 {56} + / \ / \ / \ / \ + 1 2 3 4 5 6 {5} {6} + + */ + t.Run("duplicate a branch to keep the tree balanced", func(t *testing.T) { + blocks := [][]byte{[]byte("alpha"), []byte("beta"), []byte("kappa"), []byte("gamma"), []byte("epsilon"), []byte("omega")} + tree := NewTree(IdentityHashForTest, blocks) + + expectStrEqual(t, tree.ToString(str, 0), givenSixBlocks) + }) +} + +func TestConsistencyProof(t *testing.T) { + blocksA := [][]byte{[]byte("alpha"), []byte("beta"), []byte("kappa")} + treeA := NewTree(IdentityHashForTest, blocksA) + + blocksB := [][]byte{[]byte("alpha"), []byte("beta")} + treeB := NewTree(IdentityHashForTest, blocksB) + + t.Run("Tree#Verify presented Merkle root (checksum) does not match receiving tree", func(t *testing.T) { + if treeB.Verify(treeA.root.GetChecksum(), []byte("beta")) { + s := "should have failed (different checksums) - treeA: %s treeB: %s" + t.Fatalf(s, treeA.root.GetChecksum(), treeB.root.GetChecksum()) + } + }) + + t.Run("Tree#Verify presented leaf checksum does not exist in receiving tree", func(t *testing.T) { + if treeB.Verify(treeB.root.GetChecksum(), []byte("kappa")) { + t.Fatal("should have failed (leaf shouldn't exist in receiver's cache") + } + }) + + t.Run("Tree#GetProofString", func(t *testing.T) { + blocks := [][]byte{ + []byte("alpha"), + []byte("beta"), + []byte("kappa"), + []byte("gamma"), + []byte("epsilon"), + []byte("omega"), + []byte("mu"), + []byte("zeta"), + } + + tree := NewTree(IdentityHashForTest, blocks) + checksum := treeA.checksumFunc([]byte("omega")) + + f := func(xs []byte) string { + return string(xs) + } + + expectStrEqual(t, tree.GetProofString(tree.root.GetChecksum(), checksum, f), proofA) + }) +}