From aae204b2865f8591f529b7912ee7e5333f4137e9 Mon Sep 17 00:00:00 2001 From: Ryan Staudt Date: Tue, 15 Sep 2020 14:28:38 -0500 Subject: [PATCH] txscript: Proactively evict SigCache entries. This adds functionality to proactively evict SigCache entries when they are nearly guaranteed to no longer be useful. It accomplishes this by evicting entries related to transactions in the block that is 2 levels deep from a newly processed block. Proactively evicting entries reduces the likelihood of the SigCache reaching maximum capacity quickly and then relying on random eviction, which may randomly evict entries that are still useful. --- blockmanager.go | 21 +++++++ server.go | 1 + txscript/data/block432100.bz2 | Bin 0 -> 2358 bytes txscript/sigcache.go | 61 ++++++++++++++++++++ txscript/sigcache_test.go | 102 +++++++++++++++++++++++++++------- 5 files changed, 165 insertions(+), 20 deletions(-) create mode 100644 txscript/data/block432100.bz2 diff --git a/blockmanager.go b/blockmanager.go index f1414865a2..bfa4394b1f 100644 --- a/blockmanager.go +++ b/blockmanager.go @@ -27,6 +27,7 @@ import ( "github.com/decred/dcrd/internal/mining" "github.com/decred/dcrd/internal/rpcserver" peerpkg "github.com/decred/dcrd/peer/v2" + "github.com/decred/dcrd/txscript/v3" "github.com/decred/dcrd/wire" ) @@ -269,6 +270,9 @@ type blockManagerConfig struct { ChainParams *chaincfg.Params SubsidyCache *standalone.SubsidyCache + // SigCache defines the signature cache to use. + SigCache *txscript.SigCache + // The following fields provide access to the fee estimator, mempool and // the background block template generator. FeeEstimator *fees.Estimator @@ -1130,6 +1134,9 @@ func (b *blockManager) handleBlockMsg(bmsg *blockMsg) { // Clear the rejected transactions. b.rejectedTxns = make(map[chainhash.Hash]struct{}) + + // Proactively evict SigCache entries. + b.proactivelyEvictSigCacheEntries(best.Height) } } @@ -1199,6 +1206,20 @@ func (b *blockManager) handleBlockMsg(bmsg *blockMsg) { } } +// proactivelyEvictSigCacheEntries fetches the block that is +// txscript.ProactiveEvictionDepth levels deep from bestHeight and passes it to +// SigCache to evict the entries associated with the transactions in that block. +func (b *blockManager) proactivelyEvictSigCacheEntries(bestHeight int64) { + evictHeight := bestHeight - txscript.ProactiveEvictionDepth + block, err := b.cfg.Chain.BlockByHeight(evictHeight) + if err == nil { + b.cfg.SigCache.EvictEntries(block.MsgBlock()) + } else { + bmgrLog.Warnf("Failed to retrieve the block at height %d: %v", + evictHeight, err) + } +} + // fetchHeaderBlocks creates and sends a request to the syncPeer for the next // list of blocks to be downloaded based on the current list of headers. func (b *blockManager) fetchHeaderBlocks() { diff --git a/server.go b/server.go index 1b6b9794fb..6c4f151475 100644 --- a/server.go +++ b/server.go @@ -3122,6 +3122,7 @@ func newServer(ctx context.Context, listenAddrs []string, db database.DB, chainP PeerNotifier: &s, Chain: s.chain, ChainParams: s.chainParams, + SigCache: s.sigCache, SubsidyCache: s.subsidyCache, TimeSource: s.timeSource, FeeEstimator: s.feeEstimator, diff --git a/txscript/data/block432100.bz2 b/txscript/data/block432100.bz2 new file mode 100644 index 0000000000000000000000000000000000000000..b37cdf45d204954ceba103b50bbaa51eefba32cd GIT binary patch literal 2358 zcmV-63CZ?CT4*^jL0KkKS@>pi!2kg$fB*mg|NsC0|NsC0|NsC0|NsC0|NsC0|NsC0 z|NsC0|Nqbho!~7v^OBouCask`B9KJMrUcW}(-YKbiRx`8M%2V)4=Lp|WW>qnOlmY_ zJf=+s8jTuhrkOn;(@g^t)Xhvy21c4R$)~9GGeTrDAZlSSk5d5+l+1!M0TiZ2k*3sS zW~Y>To}g*unq&<S)pF9)$FoW>C}AX_M5+lTSt>FeVIy zW}pU#qYswwG|2Gq?)n@}?;kQ+$aPf*yWsp=0=wGT#9%6cc1 zZAYZ?o8UPvq1Rp1fH4ZS+b5u`^})v!q_fTB4-3Z%t#Ob7fz zNuE@Is6b!{KtfOe6d^5@MFCifh)Ga@Ni?K2LtIOsaLFWzLBeq5fLRbjyD2Y-^SFFN zIcrYr1ZyMX!jrnbzei-9xcBEi_wDDMp)`t8Iw%Ki2$w#nj(TqA1GJ~`S~pv;cdxs^ zN?LnRC1$T^x_V(`AuCk|!@I#SQLMcq+5sevgkeyaj#M)Mi$Ws6O)KdV5YPXJxWF97 zPMW|#ItPRr5HM}Opf$*=Og*u{9wfV<6bOjnQ-WiG+MR(T>$2moF8_{^u&_!|(?62= zfIy8gOqC_=!qc1D$|3uWCMgBp>T|r_K!A&)kw?&A+KChb!fFHM4m7)E0zQi6Oo_E7 z(O8HhE)*cZ?;{oh9je@4+N}hINmb;v)yuIUK689nf0@E&E&5G-EYbwCRx_mwm8Z9Drx8|mZ6ct%w`Cj7z~*7pB0Br zh!Itj)iem~r6y1+R%mjw^Bk=kO=tsvA(C&1ynfr-az8NYAc=tI?OVXH_;}=5o3%i^ ztT2RHX66lxN%99V7W;@+GP9MbpzD)Tk?;XuTiV#LatGx2#SPgMpR1L6wyP3wL;-aw&lBJu%>LF=LW^pACzskr6Gp?q~O%&4!lKYGf|OBR}og;%QTCX;-%Nc z!%x|LC#y`|V$N_09<}B|w6F*TPLvh>Db<7oTalPDHVl9U^oj!#01<=A7A8rg!CHue zO9H+1QV@)0+eAXC*Rmd{St=6> z*ulihQE}`v;B3D?u+|}g0l-k9DJ1?EPAkl?Qi7jK*seotvi8hvAk($x?mJD;5HKlG zHbRgP81Cjx6R)@FL6a{9{It@KbQ_D~qCSHc-jI<*j6@i|^68cr3P}n`Pq_?Z&yM5=hL&wHFeF$#N3IeNhMOOY5%Zsk1yn&hF}ak5KGpZ>$s6%a5S z6Xo>xO&|*wEr*&oU=T%rs~%I)@A1mYWC{rg6r74b6o!SZzY&I?*w7#c zmfI{~fV!-1I0A*P*g%+tgDMaMz6_{d)Sa^r@shZhpPh=pf>7go%5I5Ct7}}cN#PGI z!Q4(5oRHHOGU5QIKM8ZG!U{fqAmAc8;>oLJUZV};uQj$Y=vp&oD3$zJWqdCf8}=`b zD!>3vJ@-I5M0>^qT1_T_v3CH0`-Z#1gqQ8{X#q`Orz2`Y0+MCox#5W%b77%+net|{ zw6~SQ_y?w4GU zu)ZAqbXe90VV+$x?UP(hy0t|<<;=7pgOf99p5vL!3cim`jqRMJD=?{NnRIqWTtHuP z$g5*lakKBB2}FieEykY;t8AVnDp!ArJti|@(8I~bdV^VgM$3A?0$NTYwvqBO)e8q_ z2$q@4h-Am`I6dm8H}4fO&|w(QZ0^hK(sxt8klhlmAF*070}wk77qEkZ9w=;(1zsif zO446FglZ*<9JEuYR3|opl2hD&5I`2Ts(IY}Mhw}f%#)N8{kj^mk`>V@QB<43KHV=Y z-bJ77#J1p6SQ$}$=R=zQI#TDIcy;--8F}04B#gKFmzTqxnw2`IyU>3Xhg%v_vLH~m zG_96uCu;SDXw>&9WM(d{%MEpdU}|0K2xrzjR2(8C3?6_F?2Gqy7R*FWZWIzVmJUxl c%-{k>2#UZIV`2eR1^i_@% literal 0 HcmV?d00001 diff --git a/txscript/sigcache.go b/txscript/sigcache.go index fca2933a33..ec683749d8 100644 --- a/txscript/sigcache.go +++ b/txscript/sigcache.go @@ -14,6 +14,11 @@ import ( "github.com/decred/dcrd/wire" ) +// ProactiveEvictionDepth is the depth of the block at which the signatures for +// the transactions within the block are nearly guaranteed to no longer be +// useful. +const ProactiveEvictionDepth = 2 + // sigCacheEntry represents an entry in the SigCache. Entries within the // SigCache are keyed according to the sigHash of the signature. In the // scenario of a cache-hit (according to the sigHash), an additional comparison @@ -106,3 +111,59 @@ func (s *SigCache) Add(sigHash chainhash.Hash, sig *ecdsa.Signature, pubKey *sec } s.validSigs[sigHash] = sigCacheEntry{sig, pubKey, tx.ShortTxHash(s.shortTxHashKey)} } + +// EvictEntries removes all entries from the SigCache that correspond to the +// transactions in the given block. The block that is passed must be +// ProactiveEvictionDepth blocks deep, which is the depth at which the +// signatures for the transactions within the block are nearly guaranteed to no +// longer be useful. +// +// EvictEntries wraps the unexported evictEntries method, which is run from a +// goroutine. evictEntries is only invoked if validSigs is not empty. This +// avoids starting a new goroutine when there is nothing to evict, such as when +// syncing is ongoing. +func (s *SigCache) EvictEntries(block *wire.MsgBlock) { + s.RLock() + if len(s.validSigs) == 0 { + return + } + s.RUnlock() + + go s.evictEntries(block) +} + +// evictEntries removes all entries from the SigCache that correspond to the +// transactions in the given block. The block that is passed must be +// ProactiveEvictionDepth blocks deep, which is the depth at which the +// signatures for the transactions within the block are nearly guaranteed to no +// longer be useful. +// +// Proactively evicting entries reduces the likelihood of the SigCache reaching +// maximum capacity quickly and then relying on random eviction, which may +// randomly evict entries that are still useful. +// +// This method must be run from a goroutine and should not be run during block +// validation. +func (s *SigCache) evictEntries(block *wire.MsgBlock) { + // Create a set consisting of the short tx hashes that are in the block. + numTxns := len(block.Transactions) + len(block.STransactions) + shortTxHashSet := make(map[uint64]struct{}, numTxns) + for _, tx := range block.Transactions { + shortTxHashSet[tx.ShortTxHash(s.shortTxHashKey)] = struct{}{} + } + for _, stx := range block.STransactions { + shortTxHashSet[stx.ShortTxHash(s.shortTxHashKey)] = struct{}{} + } + + // Iterate through the entries in validSigs and remove any that are associated + // with a transaction in the block. This is done by iterating through every + // entry in validSigs, since the alternative of also keying the map by the + // shortTxHash would take extra space. + s.Lock() + for sigHash, sigEntry := range s.validSigs { + if _, ok := shortTxHashSet[sigEntry.shortTxHash]; ok { + delete(s.validSigs, sigHash) + } + } + s.Unlock() +} diff --git a/txscript/sigcache_test.go b/txscript/sigcache_test.go index a43abd51b8..a2eeadf117 100644 --- a/txscript/sigcache_test.go +++ b/txscript/sigcache_test.go @@ -6,7 +6,10 @@ package txscript import ( + "compress/bzip2" "crypto/rand" + "os" + "path/filepath" "testing" "github.com/decred/dcrd/chaincfg/chainhash" @@ -15,6 +18,9 @@ import ( "github.com/decred/dcrd/wire" ) +// testDataPath is the path where txscript test fixtures reside. +const testDataPath = "data" + // shortTxHashKey defines a short transaction hash key to be used throughout // the tests. var shortTxHashKey = func() [wire.ShortTxHashKeySize]byte { @@ -64,23 +70,43 @@ var msgTx113875_1 = func() *wire.MsgTx { return msgTx }() +// block432100 mocks block 432,100 of the block chain. It is loaded and +// deserialized immediately here and then can be used throughout the tests. +var block432100 = func() wire.MsgBlock { + // Load and deserialize the test block. + blockDataFile := filepath.Join(testDataPath, "block432100.bz2") + fi, err := os.Open(blockDataFile) + if err != nil { + panic(err) + } + defer fi.Close() + var block wire.MsgBlock + err = block.Deserialize(bzip2.NewReader(fi)) + if err != nil { + panic(err) + } + return block +}() + // genRandomSig returns a random message, a signature of the message under the // public key and the public key. This function is used to generate randomized // test data. -func genRandomSig() (*chainhash.Hash, *ecdsa.Signature, *secp256k1.PublicKey, error) { +func genRandomSig(t *testing.T) (*chainhash.Hash, *ecdsa.Signature, *secp256k1.PublicKey) { + t.Helper() + privKey, err := secp256k1.GeneratePrivateKey() if err != nil { - return nil, nil, nil, err + t.Fatalf("error generating private key: %v", err) } pub := privKey.PubKey() var msgHash chainhash.Hash if _, err := rand.Read(msgHash[:]); err != nil { - return nil, nil, nil, err + t.Fatalf("error reading random hash: %v", err) } sig := ecdsa.Sign(privKey, msgHash[:]) - return &msgHash, sig, pub, nil + return &msgHash, sig, pub } // TestSigCacheAddExists tests the ability to add, and later check the @@ -89,10 +115,7 @@ func TestSigCacheAddExists(t *testing.T) { sigCache := NewSigCache(200, shortTxHashKey) // Generate a random sigCache entry triplet. - msg1, sig1, key1, err := genRandomSig() - if err != nil { - t.Errorf("unable to generate random signature test data") - } + msg1, sig1, key1 := genRandomSig(t) // Add the triplet to the signature cache. sigCache.Add(*msg1, sig1, key1, msgTx113875_1) @@ -116,10 +139,7 @@ func TestSigCacheAddEvictEntry(t *testing.T) { // Fill the sigcache up with some random sig triplets. for i := uint(0); i < sigCacheSize; i++ { - msg, sig, key, err := genRandomSig() - if err != nil { - t.Fatalf("unable to generate random signature test data") - } + msg, sig, key := genRandomSig(t) sigCache.Add(*msg, sig, key, msgTx113875_1) sigCopy, _ := ecdsa.ParseDERSignature(sig.Serialize()) @@ -138,10 +158,7 @@ func TestSigCacheAddEvictEntry(t *testing.T) { // Add a new entry, this should cause eviction of a randomly chosen // previous entry. - msgNew, sigNew, keyNew, err := genRandomSig() - if err != nil { - t.Fatalf("unable to generate random signature test data") - } + msgNew, sigNew, keyNew := genRandomSig(t) sigCache.Add(*msgNew, sigNew, keyNew, msgTx113875_1) // The sigcache should still have sigCache entries. @@ -165,10 +182,7 @@ func TestSigCacheAddMaxEntriesZeroOrNegative(t *testing.T) { sigCache := NewSigCache(0, shortTxHashKey) // Generate a random sigCache entry triplet. - msg1, sig1, key1, err := genRandomSig() - if err != nil { - t.Errorf("unable to generate random signature test data") - } + msg1, sig1, key1 := genRandomSig(t) // Add the triplet to the signature cache. sigCache.Add(*msg1, sig1, key1, msgTx113875_1) @@ -187,3 +201,51 @@ func TestSigCacheAddMaxEntriesZeroOrNegative(t *testing.T) { "been added", len(sigCache.validSigs)) } } + +// TestEvictEntries tests that evictEntries properly removes all SigCache +// entries related to the given block. +func TestEvictEntries(t *testing.T) { + sigCache := NewSigCache(8, shortTxHashKey) + + // Add random signatures to the SigCache for each transaction in block432100. + numTxns := len(block432100.Transactions) + len(block432100.STransactions) + for _, tx := range block432100.Transactions { + msg, sig, key := genRandomSig(t) + sigCache.Add(*msg, sig, key, tx) + } + for _, stx := range block432100.STransactions { + msg, sig, key := genRandomSig(t) + sigCache.Add(*msg, sig, key, stx) + } + + // Add another random signature that is not related to a transaction in + // block432100. + msg, sig, key := genRandomSig(t) + sigCache.Add(*msg, sig, key, msgTx113875_1) + + // Validate the number of entries that should exist in the SigCache before + // eviction. + wantLength := numTxns + 1 + gotLength := len(sigCache.validSigs) + if gotLength != wantLength { + t.Fatalf("Incorrect number of entries before eviction: "+ + "gotLength: %d, wantLength: %d", gotLength, wantLength) + } + + // Evict entries for block432100. + sigCache.evictEntries(&block432100) + + // Validate that entries related to block432100 have been removed and that + // entries unrelated to block432100 have not been removed. + wantLength = 1 + gotLength = len(sigCache.validSigs) + if gotLength != wantLength { + t.Errorf("Incorrect number of entries after eviction: "+ + "gotLength: %d, wantLength: %d", gotLength, wantLength) + } + sigCopy, _ := ecdsa.ParseDERSignature(sig.Serialize()) + keyCopy, _ := secp256k1.ParsePubKey(key.SerializeCompressed()) + if !sigCache.Exists(*msg, sigCopy, keyCopy) { + t.Errorf("previously added item not found in signature cache") + } +}