Merge pull request #4 from transmute-industries/updates

Refactored to align with transparency dev / go interfaces

.gitignore

@@ -5,4 +5,6 @@ dist
 
 
 
-!./examples/transparency-dev/test-package/node_modules
+!./examples/transparency-dev/test-package/node_modules
+
+*.db

.vscode/settings.json

@@ -1,3 +1,6 @@
 {
-  "editor.formatOnSave": true
+  "editor.formatOnSave": true,
+  "go.testEnvVars": {
+    "CGO_ENABLED": "0"
+  },
 }

README.md

@@ -20,40 +20,105 @@ npm install '@transmute/rfc9162'
 ```
 
 ```ts
-import RFC9162 from '@transmute/rfc9162';
+import RFC9162 from "@transmute/rfc9162";
 ```
 
 ```js
-const RFC9162 = require('@transmute/rfc9162');
+const RFC9162 = require("@transmute/rfc9162");
 ```
 
 ### Usage
 
 ```ts
-import RFC9162 from '@transmute/rfc9162'
-
-const entries: Uint8Array[] = []
-  for (let i = 0; i < 10; i++) {
-    entries.push(RFC9162.strToBin(`${String.fromCharCode(65 + i)}`))
-  }
-  const root = RFC9162.treeHead(entries)
-  const inclusionProof = RFC9162.inclusionProof(entries[2], entries)
-  const leaf = RFC9162.leaf(entries[2])
-  const verifiedInclusionProof = RFC9162.verifyInclusionProof(
-    root,
-    leaf,
-    inclusionProof,
-  )
-  // expect(verifiedInclusionProof).toBe(true)
-  entries.push(RFC9162.strToBin('Spicy update 🔥'))
-  const root2 = RFC9162.treeHead(entries)
-  const consistencyProof = RFC9162.consistencyProof(inclusionProof, entries)
-  const verifiedConsistencyProof = RFC9162.verifyConsistencyProof(
-    root,
-    root2,
-    consistencyProof,
-  )
-  // expect(verifiedConsistencyProof).toBe(true)
+import crypto from "crypto";
+import sqlite from "better-sqlite3";
+import { TileLog, to_hex } from "@transmute/rfc9162";
+
+const db = new sqlite("./test/transparency.db");
+db.prepare(
+  `CREATE TABLE IF NOT EXISTS tiles (id TEXT PRIMARY KEY, data BLOB);`
+).run();
+const hash_size = 32;
+const tile_height = 2;
+const log = new TileLog({
+  tile_height,
+  hash_size,
+  hash_function: (data: Uint8Array) => {
+    return new Uint8Array(crypto.createHash("sha256").update(data).digest());
+  },
+  read_tile: (tile: string): Uint8Array => {
+    const [base_tile] = tile.split(".");
+    // look for completed tiles first
+    for (let i = 4; i > 0; i--) {
+      const tile_path = base_tile + "." + i;
+      const rows = db
+        .prepare(
+          `
+          SELECT * FROM tiles
+          WHERE id = '${tile_path}'
+                  `
+        )
+        .all();
+      if (rows.length) {
+        const [row] = rows;
+        return row.data;
+      }
+    }
+    return new Uint8Array(32);
+  },
+  update_tiles: function (
+    tile_path: string,
+    start: number,
+    end: number,
+    stored_hash: Uint8Array
+  ) {
+    if (end - start !== 32) {
+      // this hash is an intermediate of the tile
+      // so it will never be persisted
+      return null;
+    }
+    let tile_data = this.read_tile(tile_path);
+    if (tile_data.length < end) {
+      const expanded_tile_data = new Uint8Array(tile_data.length + 32);
+      expanded_tile_data.set(tile_data);
+      tile_data = expanded_tile_data;
+    }
+    tile_data.set(stored_hash, start);
+    try {
+      db.prepare(
+        `
+    INSERT INTO tiles (id, data)
+    VALUES( '${tile_path}',	x'${Buffer.from(tile_data).toString("hex")}');
+            `
+      ).run();
+    } catch (e) {
+      // ignore errors
+    }
+    return tile_data;
+  },
+});
+const encoder = new TextEncoder();
+
+for (let i = 0; i < 26; i++) {
+  const record = encoder.encode(`entry-${i}`);
+  log.write_record(record);
+}
+
+// prove 17 was in log at tree size 20
+const inclusion_proof = log.inclusion_proof(20, 17);
+const root_from_inclusion_proof = log.root_from_inclusion_proof(
+  inclusion_proof,
+  log.record_hash(encoder.encode(`entry-${17}`))
+);
+// console.log(to_hex(root_from_inclusion_proof) === to_hex(log.root_at(20))) // true
+
+// prove log is append only from root at 20 to current log size
+const consistency_proof = log.consistency_proof(20, log.size());
+const root_from_consistency_proof = log.root_from_consistency_proof(
+  root_from_inclusion_proof,
+  consistency_proof
+);
+// console.log(to_hex(root_from_consistency_proof) === to_hex(log.root())) // true
 ```
 
 ## Develop
@@ -63,4 +128,4 @@ npm i
 npm t
 npm run lint
 npm run build
-```
+```

examples/transparency-dev/node_test.go

@@ -0,0 +1,137 @@
+package proof
+
+import (
+	"math/bits"
+	"testing"
+
+	"github.com/transparency-dev/merkle/compact"
+	// "github.com/transparency-dev/merkle/compact"
+)
+
+// Nodes contains information on how to construct a log Merkle tree proof. It
+// supports any proof that has at most one ephemeral node, such as inclusion
+// and consistency proofs defined in RFC 6962.
+type Nodes struct {
+	// IDs contains the IDs of non-ephemeral nodes sufficient to build the proof.
+	// If an ephemeral node is needed for a proof, it can be recomputed based on
+	// a subset of nodes in this list.
+	IDs []compact.NodeID
+	// begin is the beginning index (inclusive) into the IDs[begin:end] subslice
+	// of the nodes which will be used to re-create the ephemeral node.
+	begin int
+	// end is the ending (exclusive) index into the IDs[begin:end] subslice of
+	// the nodes which will be used to re-create the ephemeral node.
+	end int
+	// ephem is the ID of the ephemeral node in the proof. This node is a common
+	// ancestor of all nodes in IDs[begin:end]. It is the node that otherwise
+	// would have been used in the proof if the tree was perfect.
+	ephem compact.NodeID
+}
+
+func (n Nodes) skipFirst() Nodes {
+	n.IDs = n.IDs[1:]
+	// Fixup the indices into the IDs slice.
+	if n.begin < n.end {
+		n.begin--
+		n.end--
+	}
+	return n
+}
+
+func reverse(ids []compact.NodeID) {
+	for i, j := 0, len(ids)-1; i < j; i, j = i+1, j-1 {
+		ids[i], ids[j] = ids[j], ids[i]
+	}
+}
+
+// nodes returns the node IDs necessary to prove that the (level, index) node
+// is included in the Merkle tree of the given size.
+func nodes(index uint64, level uint, size uint64) Nodes {
+	// Compute the `fork` node, where the path from root to (level, index) node
+	// diverges from the path to (0, size).
+	//
+	// The sibling of this node is the ephemeral node which represents a subtree
+	// that is not complete in the tree of the given size. To compute the hash
+	// of the ephemeral node, we need all the non-ephemeral nodes that cover the
+	// same range of leaves.
+	//
+	// The `inner` variable is how many layers up from (level, index) the `fork`
+	// and the ephemeral nodes are.
+	inner := bits.Len64(index^(size>>level)) - 1
+	fork := compact.NewNodeID(level+uint(inner), index>>inner)
+
+	begin, end := fork.Coverage()
+	left := compact.RangeSize(0, begin)
+	right := compact.RangeSize(end, size)
+
+	node := compact.NewNodeID(level, index)
+	// Pre-allocate the exact number of nodes for the proof, in order:
+	// - The seed node for which we are building the proof.
+	// - The `inner` nodes at each level up to the fork node.
+	// - The `right` nodes, comprising the ephemeral node.
+	// - The `left` nodes, completing the coverage of the whole [0, size) range.
+	allocation := make([]compact.NodeID, 0, 1+inner+right+left)
+	// fmt.Println(allocation)
+	nodes := append(allocation, node)
+
+	// The first portion of the proof consists of the siblings for nodes of the
+	// path going up to the level at which the ephemeral node appears.
+	for ; node.Level < fork.Level; node = node.Parent() {
+		nodes = append(nodes, node.Sibling())
+	}
+	// This portion of the proof covers the range [begin, end) under it. The
+	// ranges to the left and to the right from it remain to be covered.
+
+	// Add all the nodes (potentially none) that cover the right range, and
+	// represent the ephemeral node. Reverse them so that the rehash method can
+	// process hashes in the convenient order, from lower to upper levels.
+	len1 := len(nodes)
+	nodes = compact.RangeNodes(end, size, nodes)
+	reverse(nodes[len(nodes)-right:])
+	len2 := len(nodes)
+	// Add the nodes that cover the left range, ordered increasingly by level.
+	nodes = compact.RangeNodes(0, begin, nodes)
+	reverse(nodes[len(nodes)-left:])
+
+	// nodes[len1:len2] contains the nodes representing the ephemeral node. If
+	// it's empty, make it zero. Note that it can also contain a single node.
+	// Depending on the preference of the layer above, it may or may not be
+	// considered ephemeral.
+	if len1 >= len2 {
+		len1, len2 = 0, 0
+	}
+
+	return Nodes{IDs: nodes, begin: len1, end: len2, ephem: fork.Sibling()}
+}
+
+func TestNewNodeID(t *testing.T) {
+	// n0 := compact.NewNodeID(uint(23), uint64(42))
+	// n1 := n0.Parent()
+	// n2 := n0.Sibling()
+	// r0, r1 := n0.Coverage()
+	// fmt.Println(n0, n1, n2, r0, r1)
+	//
+	// fmt.Println(bits.Len64(255))
+	// fmt.Println(bits.Len32(256))
+
+	// fmt.Println(bits.trailing_zeros_64(uint64(875214)))
+	// fmt.Println(compact.RangeNodes(0, 15, []compact.NodeID{}))
+
+	// fmt.Println(bits.ones_count_64(123))
+
+	// fmt.Println(compact.Decompose(13, 27))
+	// fmt.Println(compact.RangeSize(13, 27))
+
+	// n0 := nodes(1, 2, 3)
+	// fmt.Println(n0.IDs)
+
+	// p0, _ := proof.inclusion(15, 35)
+
+	// fmt.Println(p0.IDs)
+	// fmt.Println(p0.Ephem())
+
+	// p0, _ := proof.consistency(15, 35)
+
+	// fmt.Println(p0.IDs)
+
+}

examples/transparency-dev/receipt.json

@@ -1,16 +1 @@
-{
-  "leaf": 29,
-  "size": 279,
-  "root": "2na8Vb6XCgktxT8kQTUGjg+NYUX+NlAAlPXdDwac9Q8=",
-  "proof": [
-    "Xyl4xA1LaTywlVFWsJ1GAxKYRW7I1wKDDUX83iHwrls=",
-    "VWfoRshIIEZdrDrdSYlB4Fzh9hmRFA7aA+J7vqEXccc=",
-    "asE7wKBBZ8PwePo/oXV/UIcA/7U6coUuE10RFakxtnU=",
-    "AhzQ81CuMOKOuf1eFOy5ro6d2FAemvGl0ATk1Fk6Czc=",
-    "CVbFOaRJcim2eVQDYHKM/YXqK58tDmZ9n9CuHr5ouqU=",
-    "HsgyHgCSowc8xgwj81yMz40N8h1O3d/bVM3eCmM2BE4=",
-    "pghmE0W5XoBLin78y9gWiuBr1Qg3op2uT0xZAwq0zZ8=",
-    "6Y1A7WwsED21P/rss9GXvYPBnmn0VFTEvyw9btK5Qps=",
-    "0qI3hI6Y3U8/HyxY9j52hPMnZsM9nBzrXlOTZrxu//A="
-  ]
-}
+{"leaf":29,"size":279,"root":"e/6XZiL6T1cpy91yt4ZooplD9PxgLnsOOjcq0KZgFEU=","proof":["Xyl4xA1LaTywlVFWsJ1GAxKYRW7I1wKDDUX83iHwrls=","VWfoRshIIEZdrDrdSYlB4Fzh9hmRFA7aA+J7vqEXccc=","asE7wKBBZ8PwePo/oXV/UIcA/7U6coUuE10RFakxtnU=","AhzQ81CuMOKOuf1eFOy5ro6d2FAemvGl0ATk1Fk6Czc=","WccyUzxaE6CTBeuNKdwKRMQYXGGNd9rShjSdkNfv4EM=","HsgyHgCSowc8xgwj81yMz40N8h1O3d/bVM3eCmM2BE4=","pghmE0W5XoBLin78y9gWiuBr1Qg3op2uT0xZAwq0zZ8=","6Y1A7WwsED21P/rss9GXvYPBnmn0VFTEvyw9btK5Qps=","0qI3hI6Y3U8/HyxY9j52hPMnZsM9nBzrXlOTZrxu//A="]}