Fix pokt-network#1457; Fix pokt-network#1484;

Fix a high memory consumption that also is part of the issue pokt-network#1457.
Under high load of requests (1000/rps or more) the RAM got crazy and scale up to 40GB or close to that.
Now after the fix of pokt-network#1457 with the worker pool, the node remains under 14gb of ram in my local tests.

app/common_test.go

@@ -105,6 +105,7 @@ func NewInMemoryTendermintNodeAminoWithValidators(t *testing.T, genesisState []b
 			panic(err)
 		}
 		pocketTypes.CleanPocketNodes()
+		pocketTypes.StopEvidenceWorker()
 		PCA = nil
 		inMemKB = nil
 		err := inMemDB.Close()
@@ -170,6 +171,7 @@ func NewInMemoryTendermintNodeProtoWithValidators(t *testing.T, genesisState []b
 		}
 
 		pocketTypes.CleanPocketNodes()
+		pocketTypes.StopEvidenceWorker()
 
 		PCA = nil
 		inMemKB = nil

app/config.go

@@ -472,6 +472,7 @@ func InitPocketCoreConfig(chains *types.HostedBlockchains, logger log.Logger) {
 }
 
 func ShutdownPocketCore() {
+	types.StopEvidenceWorker()
 	types.FlushSessionCache()
 	types.StopServiceMetrics()
 }

app/query_test.go

@@ -791,6 +791,7 @@ func TestQueryRelay(t *testing.T) {
 			_, stopCli, evtChan := subscribeTo(t, tmTypes.EventNewBlock)
 			select {
 			case <-evtChan:
+				assert.Equal(t, uint64(1), types.GlobalEvidenceWorker.SuccessfulTasks())
 				inv, err := types.GetEvidence(types.SessionHeader{
 					ApplicationPubKey:  aat.ApplicationPublicKey,
 					Chain:              relay.Proof.Blockchain,
@@ -858,16 +859,18 @@ func TestQueryRelayMultipleNodes(t *testing.T) {
 			}
 			_, stopCli, evtChan := subscribeTo(t, tmTypes.EventNewBlock)
 			<-evtChan // Wait for block
+			chain := sdk.PlaceholderHash
+			sessionBlockHeight := int64(1)
 			// setup relay
 			for _, v := range validators {
 				relay := types.Relay{
 					Payload: payload,
 					Meta:    types.RelayMeta{BlockHeight: 5}, // todo race condition here
 					Proof: types.RelayProof{
 						Entropy:            32598345349034509,
-						SessionBlockHeight: 1,
+						SessionBlockHeight: sessionBlockHeight,
 						ServicerPubKey:     v.PublicKey().RawString(),
-						Blockchain:         sdk.PlaceholderHash,
+						Blockchain:         chain,
 						Token:              aat,
 						Signature:          "",
 					},
@@ -886,23 +889,32 @@ func TestQueryRelayMultipleNodes(t *testing.T) {
 					BodyString(expectedRequest).
 					Reply(200).
 					BodyString(expectedResponse)
+			}
 
-				validatorAddress := sdk.GetAddress(v.PublicKey())
-				node, nodeErr := types.GetPocketNodeByAddress(&validatorAddress)
+			select {
+			case <-evtChan:
+				// verify that each store task was successful
+				assert.Equal(t, uint64(len(validators)), types.GlobalEvidenceWorker.SuccessfulTasks())
+				// check the evidence store of each node.
+				for _, v := range validators {
+					validatorAddress := sdk.GetAddress(v.PublicKey())
+					_node, nodeErr := types.GetPocketNodeByAddress(&validatorAddress)
+
+					assert.Nil(t, nodeErr)
+					inv, err := types.GetEvidence(types.SessionHeader{
+						ApplicationPubKey:  aat.ApplicationPublicKey,
+						Chain:              chain,
+						SessionBlockHeight: sessionBlockHeight,
+					}, types.RelayEvidence, sdk.NewInt(10000), _node.EvidenceStore)
+					assert.Nil(t, err)
+					assert.NotNil(t, inv)
+					assert.Equal(t, int64(1), inv.NumOfProofs)
+				}
 
-				assert.Nil(t, nodeErr)
-				inv, err := types.GetEvidence(types.SessionHeader{
-					ApplicationPubKey:  aat.ApplicationPublicKey,
-					Chain:              relay.Proof.Blockchain,
-					SessionBlockHeight: relay.Proof.SessionBlockHeight,
-				}, types.RelayEvidence, sdk.NewInt(10000), node.EvidenceStore)
-				assert.Nil(t, err)
-				assert.NotNil(t, inv)
-				assert.Equal(t, inv.NumOfProofs, int64(1))
+				cleanup()
+				stopCli()
+				gock.Off()
 			}
-			cleanup()
-			stopCli()
-			gock.Off()
 			return
 		})
 	}

go.mod

@@ -7,7 +7,8 @@ replace github.com/tendermint/tendermint => github.com/pokt-network/tendermint v
 replace github.com/tendermint/tm-db => github.com/pokt-network/tm-db v0.5.2-0.20220118210553-9b2300f289ba
 
 require (
-	github.com/cucumber/godog v0.12.5
+	github.com/alitto/pond v1.8.1
+	github.com/cucumber/godog v0.12.6
 	github.com/go-kit/kit v0.12.0
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang/protobuf v1.5.2
@@ -18,7 +19,7 @@ require (
 	github.com/prometheus/client_golang v1.11.0
 	github.com/regen-network/cosmos-proto v0.3.0
 	github.com/spf13/cobra v1.4.0
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/testify v1.7.1
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 	github.com/tendermint/go-amino v0.15.1
 	github.com/tendermint/tendermint v0.33.7
@@ -42,7 +43,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-kit/log v0.2.0 // indirect
 	github.com/go-logfmt/logfmt v0.5.1 // indirect
-	github.com/gofrs/uuid v4.0.0+incompatible // indirect
+	github.com/gofrs/uuid v4.2.0+incompatible // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.0.0 // indirect
 	github.com/google/go-cmp v0.5.6 // indirect
@@ -51,7 +52,7 @@ require (
 	github.com/gtank/ristretto255 v0.1.2 // indirect
 	github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
-	github.com/hashicorp/go-memdb v1.3.0 // indirect
+	github.com/hashicorp/go-memdb v1.3.2 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jmhodges/levigo v1.0.0 // indirect
 	github.com/kr/text v0.2.0 // indirect