From df141bbff34cb11c1383e611c3a7c1dced50ab0b Mon Sep 17 00:00:00 2001 From: Ryan Staudt Date: Sat, 19 Sep 2020 06:58:11 -0500 Subject: [PATCH] txscript: Proactively evict SigCache entries. This adds functionality to proactively evict SigCache entries when they are nearly guaranteed to no longer be useful. It accomplishes this by evicting entries related to transactions in the block that is 2 levels deep from a newly processed block. Proactively evicting entries reduces the likelihood of the SigCache reaching maximum capacity quickly and then relying on random eviction, which may randomly evict entries that are still useful. --- blockmanager.go | 27 +++++++++ server.go | 1 + txscript/data/block432100.bz2 | Bin 0 -> 2358 bytes txscript/sigcache.go | 62 ++++++++++++++++++++ txscript/sigcache_test.go | 106 +++++++++++++++++++++++++++------- 5 files changed, 176 insertions(+), 20 deletions(-) create mode 100644 txscript/data/block432100.bz2 diff --git a/blockmanager.go b/blockmanager.go index 49ec7764fe..017ac44b6b 100644 --- a/blockmanager.go +++ b/blockmanager.go @@ -27,6 +27,7 @@ import ( "github.com/decred/dcrd/internal/mining" "github.com/decred/dcrd/internal/rpcserver" peerpkg "github.com/decred/dcrd/peer/v2" + "github.com/decred/dcrd/txscript/v3" "github.com/decred/dcrd/wire" ) @@ -269,6 +270,9 @@ type blockManagerConfig struct { ChainParams *chaincfg.Params SubsidyCache *standalone.SubsidyCache + // SigCache defines the signature cache to use. + SigCache *txscript.SigCache + // The following fields provide access to the fee estimator, mempool and // the background block template generator. FeeEstimator *fees.Estimator @@ -1163,6 +1167,9 @@ func (b *blockManager) handleBlockMsg(bmsg *blockMsg) { // Clear the rejected transactions. b.rejectedTxns = make(map[chainhash.Hash]struct{}) + + // Proactively evict SigCache entries. + b.proactivelyEvictSigCacheEntries(best.Height) } } @@ -1232,6 +1239,26 @@ func (b *blockManager) handleBlockMsg(bmsg *blockMsg) { } } +// proactivelyEvictSigCacheEntries fetches the block that is +// txscript.ProactiveEvictionDepth levels deep from bestHeight and passes it to +// SigCache to evict the entries associated with the transactions in that block. +func (b *blockManager) proactivelyEvictSigCacheEntries(bestHeight int64) { + // Nothing to do before the eviction depth is reached. + if bestHeight <= txscript.ProactiveEvictionDepth { + return + } + + evictHeight := bestHeight - txscript.ProactiveEvictionDepth + block, err := b.cfg.Chain.BlockByHeight(evictHeight) + if err != nil { + bmgrLog.Warnf("Failed to retrieve the block at height %d: %v", + evictHeight, err) + return + } + + b.cfg.SigCache.EvictEntries(block.MsgBlock()) +} + // fetchHeaderBlocks creates and sends a request to the syncPeer for the next // list of blocks to be downloaded based on the current list of headers. func (b *blockManager) fetchHeaderBlocks() { diff --git a/server.go b/server.go index 5a76b43ba7..8e153c5a2d 100644 --- a/server.go +++ b/server.go @@ -3251,6 +3251,7 @@ func newServer(ctx context.Context, listenAddrs []string, db database.DB, chainP PeerNotifier: &s, Chain: s.chain, ChainParams: s.chainParams, + SigCache: s.sigCache, SubsidyCache: s.subsidyCache, TimeSource: s.timeSource, FeeEstimator: s.feeEstimator, diff --git a/txscript/data/block432100.bz2 b/txscript/data/block432100.bz2 new file mode 100644 index 0000000000000000000000000000000000000000..b37cdf45d204954ceba103b50bbaa51eefba32cd GIT binary patch literal 2358 zcmV-63CZ?CT4*^jL0KkKS@>pi!2kg$fB*mg|NsC0|NsC0|NsC0|NsC0|NsC0|NsC0 z|NsC0|Nqbho!~7v^OBouCask`B9KJMrUcW}(-YKbiRx`8M%2V)4=Lp|WW>qnOlmY_ zJf=+s8jTuhrkOn;(@g^t)Xhvy21c4R$)~9GGeTrDAZlSSk5d5+l+1!M0TiZ2k*3sS zW~Y>To}g*unq&<S)pF9)$FoW>C}AX_M5+lTSt>FeVIy zW}pU#qYswwG|2Gq?)n@}?;kQ+$aPf*yWsp=0=wGT#9%6cc1 zZAYZ?o8UPvq1Rp1fH4ZS+b5u`^})v!q_fTB4-3Z%t#Ob7fz zNuE@Is6b!{KtfOe6d^5@MFCifh)Ga@Ni?K2LtIOsaLFWzLBeq5fLRbjyD2Y-^SFFN zIcrYr1ZyMX!jrnbzei-9xcBEi_wDDMp)`t8Iw%Ki2$w#nj(TqA1GJ~`S~pv;cdxs^ zN?LnRC1$T^x_V(`AuCk|!@I#SQLMcq+5sevgkeyaj#M)Mi$Ws6O)KdV5YPXJxWF97 zPMW|#ItPRr5HM}Opf$*=Og*u{9wfV<6bOjnQ-WiG+MR(T>$2moF8_{^u&_!|(?62= zfIy8gOqC_=!qc1D$|3uWCMgBp>T|r_K!A&)kw?&A+KChb!fFHM4m7)E0zQi6Oo_E7 z(O8HhE)*cZ?;{oh9je@4+N}hINmb;v)yuIUK689nf0@E&E&5G-EYbwCRx_mwm8Z9Drx8|mZ6ct%w`Cj7z~*7pB0Br zh!Itj)iem~r6y1+R%mjw^Bk=kO=tsvA(C&1ynfr-az8NYAc=tI?OVXH_;}=5o3%i^ ztT2RHX66lxN%99V7W;@+GP9MbpzD)Tk?;XuTiV#LatGx2#SPgMpR1L6wyP3wL;-aw&lBJu%>LF=LW^pACzskr6Gp?q~O%&4!lKYGf|OBR}og;%QTCX;-%Nc z!%x|LC#y`|V$N_09<}B|w6F*TPLvh>Db<7oTalPDHVl9U^oj!#01<=A7A8rg!CHue zO9H+1QV@)0+eAXC*Rmd{St=6> z*ulihQE}`v;B3D?u+|}g0l-k9DJ1?EPAkl?Qi7jK*seotvi8hvAk($x?mJD;5HKlG zHbRgP81Cjx6R)@FL6a{9{It@KbQ_D~qCSHc-jI<*j6@i|^68cr3P}n`Pq_?Z&yM5=hL&wHFeF$#N3IeNhMOOY5%Zsk1yn&hF}ak5KGpZ>$s6%a5S z6Xo>xO&|*wEr*&oU=T%rs~%I)@A1mYWC{rg6r74b6o!SZzY&I?*w7#c zmfI{~fV!-1I0A*P*g%+tgDMaMz6_{d)Sa^r@shZhpPh=pf>7go%5I5Ct7}}cN#PGI z!Q4(5oRHHOGU5QIKM8ZG!U{fqAmAc8;>oLJUZV};uQj$Y=vp&oD3$zJWqdCf8}=`b zD!>3vJ@-I5M0>^qT1_T_v3CH0`-Z#1gqQ8{X#q`Orz2`Y0+MCox#5W%b77%+net|{ zw6~SQ_y?w4GU zu)ZAqbXe90VV+$x?UP(hy0t|<<;=7pgOf99p5vL!3cim`jqRMJD=?{NnRIqWTtHuP z$g5*lakKBB2}FieEykY;t8AVnDp!ArJti|@(8I~bdV^VgM$3A?0$NTYwvqBO)e8q_ z2$q@4h-Am`I6dm8H}4fO&|w(QZ0^hK(sxt8klhlmAF*070}wk77qEkZ9w=;(1zsif zO446FglZ*<9JEuYR3|opl2hD&5I`2Ts(IY}Mhw}f%#)N8{kj^mk`>V@QB<43KHV=Y z-bJ77#J1p6SQ$}$=R=zQI#TDIcy;--8F}04B#gKFmzTqxnw2`IyU>3Xhg%v_vLH~m zG_96uCu;SDXw>&9WM(d{%MEpdU}|0K2xrzjR2(8C3?6_F?2Gqy7R*FWZWIzVmJUxl c%-{k>2#UZIV`2eR1^i_@% literal 0 HcmV?d00001 diff --git a/txscript/sigcache.go b/txscript/sigcache.go index 98e3c7e388..8381b8eb66 100644 --- a/txscript/sigcache.go +++ b/txscript/sigcache.go @@ -17,6 +17,11 @@ import ( "github.com/decred/dcrd/wire" ) +// ProactiveEvictionDepth is the depth of the block at which the signatures for +// the transactions within the block are nearly guaranteed to no longer be +// useful. +const ProactiveEvictionDepth = 2 + // shortTxHashKeySize is the size of the byte array required for key material // for the SipHash keyed shortTxHash function. const shortTxHashKeySize = 16 @@ -139,3 +144,60 @@ func shortTxHash(msg *wire.MsgTx, key [shortTxHashKeySize]byte) uint64 { txHash := msg.TxHash() return siphash.Hash(k0, k1, txHash[:]) } + +// EvictEntries removes all entries from the SigCache that correspond to the +// transactions in the given block. The block that is passed should be +// ProactiveEvictionDepth blocks deep, which is the depth at which the +// signatures for the transactions within the block are nearly guaranteed to no +// longer be useful. +// +// EvictEntries wraps the unexported evictEntries method, which is run from a +// goroutine. evictEntries is only invoked if validSigs is not empty. This +// avoids starting a new goroutine when there is nothing to evict, such as when +// syncing is ongoing. +func (s *SigCache) EvictEntries(block *wire.MsgBlock) { + s.RLock() + if len(s.validSigs) == 0 { + s.RUnlock() + return + } + s.RUnlock() + + go s.evictEntries(block) +} + +// evictEntries removes all entries from the SigCache that correspond to the +// transactions in the given block. The block that is passed should be +// ProactiveEvictionDepth blocks deep, which is the depth at which the +// signatures for the transactions within the block are nearly guaranteed to no +// longer be useful. +// +// Proactively evicting entries reduces the likelihood of the SigCache reaching +// maximum capacity quickly and then relying on random eviction, which may +// randomly evict entries that are still useful. +// +// This method must be run from a goroutine and should not be run during block +// validation. +func (s *SigCache) evictEntries(block *wire.MsgBlock) { + // Create a set consisting of the short tx hashes that are in the block. + numTxns := len(block.Transactions) + len(block.STransactions) + shortTxHashSet := make(map[uint64]struct{}, numTxns) + for _, tx := range block.Transactions { + shortTxHashSet[shortTxHash(tx, s.shortTxHashKey)] = struct{}{} + } + for _, stx := range block.STransactions { + shortTxHashSet[shortTxHash(stx, s.shortTxHashKey)] = struct{}{} + } + + // Iterate through the entries in validSigs and remove any that are associated + // with a transaction in the block. This is done by iterating through every + // entry in validSigs, since the alternative of also keying the map by the + // shortTxHash would take extra space. + s.Lock() + for sigHash, sigEntry := range s.validSigs { + if _, ok := shortTxHashSet[sigEntry.shortTxHash]; ok { + delete(s.validSigs, sigHash) + } + } + s.Unlock() +} diff --git a/txscript/sigcache_test.go b/txscript/sigcache_test.go index 771dfb8011..8095736fa3 100644 --- a/txscript/sigcache_test.go +++ b/txscript/sigcache_test.go @@ -6,7 +6,10 @@ package txscript import ( + "compress/bzip2" "crypto/rand" + "os" + "path/filepath" "testing" "github.com/decred/dcrd/chaincfg/chainhash" @@ -15,6 +18,27 @@ import ( "github.com/decred/dcrd/wire" ) +// testDataPath is the path where txscript test fixtures reside. +const testDataPath = "data" + +// block432100 mocks block 432,100 of the block chain. It is loaded and +// deserialized immediately here and then can be used throughout the tests. +var block432100 = func() wire.MsgBlock { + // Load and deserialize the test block. + blockDataFile := filepath.Join(testDataPath, "block432100.bz2") + fi, err := os.Open(blockDataFile) + if err != nil { + panic(err) + } + defer fi.Close() + var block wire.MsgBlock + err = block.Deserialize(bzip2.NewReader(fi)) + if err != nil { + panic(err) + } + return block +}() + // msgTx113875_1 mocks the first transaction from block 113875. func msgTx113875_1() *wire.MsgTx { msgTx := wire.NewMsgTx() @@ -47,20 +71,22 @@ func msgTx113875_1() *wire.MsgTx { // genRandomSig returns a random message, a signature of the message under the // public key and the public key. This function is used to generate randomized // test data. -func genRandomSig() (*chainhash.Hash, *ecdsa.Signature, *secp256k1.PublicKey, error) { +func genRandomSig(t *testing.T) (*chainhash.Hash, *ecdsa.Signature, *secp256k1.PublicKey) { + t.Helper() + privKey, err := secp256k1.GeneratePrivateKey() if err != nil { - return nil, nil, nil, err + t.Fatalf("error generating private key: %v", err) } pub := privKey.PubKey() var msgHash chainhash.Hash if _, err := rand.Read(msgHash[:]); err != nil { - return nil, nil, nil, err + t.Fatalf("error reading random hash: %v", err) } sig := ecdsa.Sign(privKey, msgHash[:]) - return &msgHash, sig, pub, nil + return &msgHash, sig, pub } // TestSigCacheAddExists tests the ability to add, and later check the @@ -72,10 +98,7 @@ func TestSigCacheAddExists(t *testing.T) { } // Generate a random sigCache entry triplet. - msg1, sig1, key1, err := genRandomSig() - if err != nil { - t.Errorf("unable to generate random signature test data") - } + msg1, sig1, key1 := genRandomSig(t) // Add the triplet to the signature cache. sigCache.Add(*msg1, sig1, key1, msgTx113875_1()) @@ -104,10 +127,7 @@ func TestSigCacheAddEvictEntry(t *testing.T) { // Fill the sigcache up with some random sig triplets. for i := uint(0); i < sigCacheSize; i++ { - msg, sig, key, err := genRandomSig() - if err != nil { - t.Fatalf("unable to generate random signature test data") - } + msg, sig, key := genRandomSig(t) sigCache.Add(*msg, sig, key, tx) sigCopy, _ := ecdsa.ParseDERSignature(sig.Serialize()) @@ -126,10 +146,7 @@ func TestSigCacheAddEvictEntry(t *testing.T) { // Add a new entry, this should cause eviction of a randomly chosen // previous entry. - msgNew, sigNew, keyNew, err := genRandomSig() - if err != nil { - t.Fatalf("unable to generate random signature test data") - } + msgNew, sigNew, keyNew := genRandomSig(t) sigCache.Add(*msgNew, sigNew, keyNew, tx) // The sigcache should still have sigCache entries. @@ -156,10 +173,7 @@ func TestSigCacheAddMaxEntriesZeroOrNegative(t *testing.T) { } // Generate a random sigCache entry triplet. - msg1, sig1, key1, err := genRandomSig() - if err != nil { - t.Errorf("unable to generate random signature test data") - } + msg1, sig1, key1 := genRandomSig(t) // Add the triplet to the signature cache. sigCache.Add(*msg1, sig1, key1, msgTx113875_1()) @@ -208,3 +222,55 @@ func TestShortTxHash(t *testing.T) { t.Errorf("shortTxHash: wanted different hash, but got same hash %d", got) } } + +// TestEvictEntries tests that evictEntries properly removes all SigCache +// entries related to the given block. +func TestEvictEntries(t *testing.T) { + // Create a SigCache instance. + numTxns := len(block432100.Transactions) + len(block432100.STransactions) + sigCache, err := NewSigCache(uint(numTxns + 1)) + if err != nil { + t.Fatalf("error creating NewSigCache: %v", err) + } + + // Add random signatures to the SigCache for each transaction in block432100. + for _, tx := range block432100.Transactions { + msg, sig, key := genRandomSig(t) + sigCache.Add(*msg, sig, key, tx) + } + for _, stx := range block432100.STransactions { + msg, sig, key := genRandomSig(t) + sigCache.Add(*msg, sig, key, stx) + } + + // Add another random signature that is not related to a transaction in + // block432100. + msg, sig, key := genRandomSig(t) + sigCache.Add(*msg, sig, key, msgTx113875_1()) + + // Validate the number of entries that should exist in the SigCache before + // eviction. + wantLength := numTxns + 1 + gotLength := len(sigCache.validSigs) + if gotLength != wantLength { + t.Fatalf("Incorrect number of entries before eviction: "+ + "gotLength: %d, wantLength: %d", gotLength, wantLength) + } + + // Evict entries for block432100. + sigCache.evictEntries(&block432100) + + // Validate that entries related to block432100 have been removed and that + // entries unrelated to block432100 have not been removed. + wantLength = 1 + gotLength = len(sigCache.validSigs) + if gotLength != wantLength { + t.Errorf("Incorrect number of entries after eviction: "+ + "gotLength: %d, wantLength: %d", gotLength, wantLength) + } + sigCopy, _ := ecdsa.ParseDERSignature(sig.Serialize()) + keyCopy, _ := secp256k1.ParsePubKey(key.SerializeCompressed()) + if !sigCache.Exists(*msg, sigCopy, keyCopy) { + t.Errorf("previously added item not found in signature cache") + } +}