diff --git a/rvgo/cmd/run.go b/rvgo/cmd/run.go index c9962a63..7e240531 100644 --- a/rvgo/cmd/run.go +++ b/rvgo/cmd/run.go @@ -259,6 +259,8 @@ func Run(ctx *cli.Context) error { if err := jsonutil.WriteJSON(ctx.Path(cannon.RunOutputFlag.Name), state, OutFilePerm); err != nil { return fmt.Errorf("failed to write state output: %w", err) } + + state.Close() return nil } diff --git a/rvgo/fast/instrumented.go b/rvgo/fast/instrumented.go index 8898c4d0..bf2d2574 100644 --- a/rvgo/fast/instrumented.go +++ b/rvgo/fast/instrumented.go @@ -3,6 +3,7 @@ package fast import ( "encoding/binary" "fmt" + "github.com/google/uuid" "io" ) @@ -42,6 +43,13 @@ func NewInstrumentedState(state *VMState, po PreimageOracle, stdOut, stdErr io.W } } +func (m *InstrumentedState) Close() { + // When done, close the Memory instance + id := uuid.New() + m.state.Memory.statsFile = id.String() + m.state.Close() +} + func (m *InstrumentedState) Step(proof bool) (wit *StepWitness, err error) { m.memProofEnabled = proof m.memAccess = m.memAccess[:0] diff --git a/rvgo/fast/memory.go b/rvgo/fast/memory.go index 462ab4e9..6e85d7ee 100644 --- a/rvgo/fast/memory.go +++ b/rvgo/fast/memory.go @@ -4,6 +4,9 @@ import ( "encoding/json" "fmt" "io" + "log" + "net/http" + _ "net/http/pprof" "sort" "github.com/ethereum/go-ethereum/crypto" @@ -45,6 +48,9 @@ type Memory struct { radix *L1 branchFactors [10]uint64 + stats *Stats // Reference to your Stats struct + statsFile string // Filename for the CSV output + // Note: since we don't de-alloc pages, we don't do ref-counting. // Once a page exists, it doesn't leave memory @@ -56,14 +62,34 @@ type Memory struct { func NewMemory() *Memory { node := &L1{} + go func() { + log.Println(http.ListenAndServe("localhost:6060", nil)) + }() return &Memory{ radix: node, + stats: NewStats(), pages: make(map[uint64]*CachedPage), branchFactors: [10]uint64{4, 4, 4, 4, 4, 4, 4, 8, 8, 8}, lastPageKeys: [2]uint64{^uint64(0), ^uint64(0)}, // default to invalid keys, to not match any pages } } +func (m *Memory) Close() error { + // Perform any necessary cleanup here + + // Write the statistics to the CSV file + if m.stats != nil && m.statsFile != "" { + err := m.stats.WriteToCSV(m.statsFile) + if err != nil { + return fmt.Errorf("failed to write stats to CSV: %w", err) + } + } + + // If you have any other resources to clean up, do it here + + return nil +} + func (m *Memory) PageCount() int { return len(m.pages) } @@ -202,6 +228,7 @@ func (m *Memory) UnmarshalJSON(data []byte) error { m.branchFactors = [10]uint64{4, 4, 4, 4, 4, 4, 4, 8, 8, 8} m.radix = &L1{} + m.stats = NewStats() m.pages = make(map[uint64]*CachedPage) m.lastPageKeys = [2]uint64{^uint64(0), ^uint64(0)} m.lastPage = [2]*CachedPage{nil, nil} diff --git a/rvgo/fast/memory_test.go b/rvgo/fast/memory_test.go index d57e4f7f..184f05ba 100644 --- a/rvgo/fast/memory_test.go +++ b/rvgo/fast/memory_test.go @@ -423,6 +423,7 @@ func BenchmarkMemoryOperations(b *testing.B) { {"MerkleProofGeneration_Large", benchMerkleProofGeneration(largeDataset)}, {"MerkleRootCalculation_Small", benchMerkleRootCalculation(smallDataset)}, {"MerkleRootCalculation_Large", benchMerkleRootCalculation(largeDataset)}, + {"CacheInvalidation", benchCacheInvalidation}, } for _, bm := range benchmarks { @@ -522,3 +523,17 @@ func benchMerkleRootCalculation(size int) func(b *testing.B, m *Memory) { } } } + +func benchCacheInvalidation(b *testing.B, m *Memory) { + data := make([]byte, 8) + addresses := make([]uint64, b.N) + for i := range addresses { + addresses[i] = mathrand.Uint64() + m.SetUnaligned(addresses[i], data) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + m.Invalidate(addresses[i%len(addresses)]) + } +} diff --git a/rvgo/fast/radix.go b/rvgo/fast/radix.go index 8ebcb1ac..ae4617db 100644 --- a/rvgo/fast/radix.go +++ b/rvgo/fast/radix.go @@ -1,7 +1,11 @@ package fast import ( + "encoding/csv" "math/bits" + "os" + "strconv" + "sync" ) // RadixNode is an interface defining the operations for a node in a radix trie. @@ -14,21 +18,27 @@ type RadixNode interface { MerkleizeNode(addr, gindex uint64) [32]byte } +const SmallRadixSize = 4 + // SmallRadixNode is a radix trie node with a branching factor of 4 bits. type SmallRadixNode[C RadixNode] struct { - Children [1 << 4]*C // Array of child nodes, indexed by 4-bit keys. - Hashes [1 << 4][32]byte // Cached hashes for each child node. - ChildExists uint16 // Bitmask indicating which children exist (1 bit per child). - HashValid uint16 // Bitmask indicating which hashes are valid (1 bit per child). - Depth uint64 // The depth of this node in the trie (number of bits from the root). + Stats *Stats + Children [1 << SmallRadixSize]*C // Array of child nodes, indexed by 4-bit keys. + Hashes [1 << SmallRadixSize][32]byte // Cached hashes for each child node. + ChildExists uint16 // Bitmask indicating which children exist (1 bit per child). + HashValid uint16 // Bitmask indicating which hashes are valid (1 bit per child). + Depth uint64 // The depth of this node in the trie (number of bits from the root). } +const LargeRadixSize = 8 + // LargeRadixNode is a radix trie node with a branching factor of 8 bits. type LargeRadixNode[C RadixNode] struct { - Children [1 << 8]*C // Array of child nodes, indexed by 8-bit keys. - Hashes [1 << 8][32]byte - ChildExists [(1 << 8) / 64]uint64 - HashValid [(1 << 8) / 64]uint64 + Stats *Stats + Children [1 << LargeRadixSize]*C // Array of child nodes, indexed by 8-bit keys. + Hashes [1 << LargeRadixSize][32]byte + ChildExists [(1 << LargeRadixSize) / 64]uint64 + HashValid [(1 << LargeRadixSize) / 64]uint64 Depth uint64 } @@ -51,9 +61,9 @@ type L11 = *Memory // InvalidateNode invalidates the hash cache along the path to the specified address. // It marks the necessary child hashes as invalid, forcing them to be recomputed when needed. func (n *SmallRadixNode[C]) InvalidateNode(addr uint64) { - childIdx := addressToRadixPath(addr, n.Depth, 4) // Get the 4-bit child index at the current depth. + childIdx := addressToRadixPath(addr, n.Depth, SmallRadixSize) // Get the 4-bit child index at the current depth. - branchIdx := (childIdx + 1<<4) / 2 // Compute the index for the hash tree traversal. + branchIdx := (childIdx + 1< 0; index >>= 1 { @@ -64,9 +74,9 @@ func (n *SmallRadixNode[C]) InvalidateNode(addr uint64) { } func (n *LargeRadixNode[C]) InvalidateNode(addr uint64) { - childIdx := addressToRadixPath(addr, n.Depth, 8) + childIdx := addressToRadixPath(addr, n.Depth, LargeRadixSize) - branchIdx := (childIdx + 1<<8) / 2 + branchIdx := (childIdx + 1< 0; index >>= 1 { hashIndex := index >> 6 @@ -86,18 +96,18 @@ func (m *Memory) InvalidateNode(addr uint64) { // It collects the necessary sibling hashes along the path to reconstruct the Merkle proof. func (n *SmallRadixNode[C]) GenerateProof(addr uint64) [][32]byte { var proofs [][32]byte - path := addressToRadixPath(addr, n.Depth, 4) + path := addressToRadixPath(addr, n.Depth, SmallRadixSize) if n.Children[path] == nil { // When no child exists at this path, the rest of the proofs are zero hashes. - proofs = zeroHashRange(0, 60-n.Depth-4) + proofs = zeroHashRange(0, 60-n.Depth-SmallRadixSize) } else { // Recursively generate proofs from the child node. proofs = (*n.Children[path]).GenerateProof(addr) } // Collect sibling hashes along the path for the proof. - for idx := path + 1<<4; idx > 1; idx >>= 1 { + for idx := path + 1< 1; idx >>= 1 { sibling := idx ^ 1 // Get the sibling index. proofs = append(proofs, n.MerkleizeNode(addr>>(64-n.Depth), sibling)) } @@ -107,15 +117,15 @@ func (n *SmallRadixNode[C]) GenerateProof(addr uint64) [][32]byte { func (n *LargeRadixNode[C]) GenerateProof(addr uint64) [][32]byte { var proofs [][32]byte - path := addressToRadixPath(addr, n.Depth, 8) + path := addressToRadixPath(addr, n.Depth, LargeRadixSize) if n.Children[path] == nil { - proofs = zeroHashRange(0, 60-n.Depth-8) + proofs = zeroHashRange(0, 60-n.Depth-LargeRadixSize) } else { proofs = (*n.Children[path]).GenerateProof(addr) } - for idx := path + 1<<8; idx > 1; idx >>= 1 { + for idx := path + 1< 1; idx >>= 1 { sibling := idx ^ 1 proofs = append(proofs, n.MerkleizeNode(addr>>(64-n.Depth), sibling)) } @@ -138,7 +148,7 @@ func (m *Memory) GenerateProof(addr uint64) [][32]byte { func (n *SmallRadixNode[C]) MerkleizeNode(addr, gindex uint64) [32]byte { depth := uint64(bits.Len64(gindex)) // Get the depth of the current gindex. - if depth <= 4 { + if depth <= SmallRadixSize { hashBit := gindex & 15 if (n.ChildExists & (1 << hashBit)) != 0 { @@ -165,7 +175,7 @@ func (n *SmallRadixNode[C]) MerkleizeNode(addr, gindex uint64) [32]byte { panic("gindex too deep") } - childIndex := gindex - 1<<4 + childIndex := gindex - 1<> 6 hashBit := gindex & 63 if (n.ChildExists[hashIndex] & (1 << hashBit)) != 0 { @@ -206,12 +216,12 @@ func (n *LargeRadixNode[C]) MerkleizeNode(addr, gindex uint64) [32]byte { panic("gindex too deep") } - childIndex := gindex - 1<<8 + childIndex := gindex - 1<