From b04528a205fe7da3659a3eda6fa0575a56ffe975 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Mon, 22 Apr 2024 16:04:50 +0200 Subject: [PATCH] centralised ephemeral keys on testnet check, fixed keeper smoke tests, added root key buffer check --- .../actions/automationv2/actions.go | 15 +- integration-tests/actions/seth/actions.go | 38 ++++- .../actions/seth/automation_ocr_helpers.go | 6 +- .../actions/seth/keeper_helpers.go | 56 ++++++- integration-tests/benchmark/keeper_test.go | 7 +- .../chaos/automation_chaos_test.go | 13 +- .../docker/test_env/test_env_builder.go | 4 +- integration-tests/go.mod | 2 +- integration-tests/go.sum | 4 +- .../automationv2_1/automationv2_1_test.go | 10 +- .../load/automationv2_1/helpers.go | 7 +- integration-tests/load/go.mod | 2 +- integration-tests/load/go.sum | 4 +- integration-tests/smoke/keeper_test.go | 147 ++++++++++-------- integration-tests/testconfig/testconfig.go | 55 +++++++ .../testsetups/keeper_benchmark.go | 17 -- .../universal/log_poller/helpers.go | 4 - integration-tests/utils/seth.go | 24 --- 18 files changed, 261 insertions(+), 154 deletions(-) diff --git a/integration-tests/actions/automationv2/actions.go b/integration-tests/actions/automationv2/actions.go index aad163032a9..7a1d0dbc2d4 100644 --- a/integration-tests/actions/automationv2/actions.go +++ b/integration-tests/actions/automationv2/actions.go @@ -29,6 +29,7 @@ import ( ocr2keepers20config "github.com/smartcontractkit/chainlink-automation/pkg/v2/config" ocr2keepers30config "github.com/smartcontractkit/chainlink-automation/pkg/v3/config" + actions_seth "github.com/smartcontractkit/chainlink/integration-tests/actions/seth" "github.com/smartcontractkit/chainlink/integration-tests/docker/test_env" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/automation_registrar_wrapper2_1" @@ -583,7 +584,10 @@ func calculateOCR3ConfigArgs(a *AutomationTest, S []int, oracleIdentities []conf func (a *AutomationTest) RegisterUpkeeps(upkeepConfigs []UpkeepConfig) ([]common.Hash, error) { registrationTxHashes := make([]common.Hash, 0) - concurrency := a.GetConcurrency() + concurrency, err := actions_seth.GetAndAssertCorrectConcurrency(a.ChainClient, 1) + if err != nil { + return nil, err + } type result struct { txHash common.Hash @@ -721,7 +725,10 @@ func (a *AutomationTest) RegisterUpkeeps(upkeepConfigs []UpkeepConfig) ([]common func (a *AutomationTest) ConfirmUpkeepsRegistered(registrationTxHashes []common.Hash) ([]*big.Int, error) { upkeepIds := make([]*big.Int, 0) - concurrency := a.GetConcurrency() + concurrency, err := actions_seth.GetAndAssertCorrectConcurrency(a.ChainClient, 1) + if err != nil { + return nil, err + } type result struct { upkeepID *big.Int @@ -904,7 +911,3 @@ func (a *AutomationTest) LoadAutomationDeployment(t *testing.T, linkTokenAddress a.AddJobsAndSetConfig(t) } - -func (a *AutomationTest) GetConcurrency() int { - return int(*a.ChainClient.Cfg.EphemeralAddrs) -} diff --git a/integration-tests/actions/seth/actions.go b/integration-tests/actions/seth/actions.go index c0aa33c30b9..1386892f373 100644 --- a/integration-tests/actions/seth/actions.go +++ b/integration-tests/actions/seth/actions.go @@ -887,10 +887,6 @@ func SendLinkFundsToDeploymentAddresses( return data, nil } - if *chainClient.Cfg.EphemeralAddrs == 0 { - return nil - } - toTransferToMultiCallContract := big.NewInt(0).Mul(linkAmountPerUpkeep, big.NewInt(int64(totalUpkeeps+concurrency))) toTransferPerClient := big.NewInt(0).Mul(linkAmountPerUpkeep, big.NewInt(int64(operationsPerAddress+1))) err := linkToken.Transfer(multicallAddress.Hex(), toTransferToMultiCallContract) @@ -951,6 +947,40 @@ var noOpSethConfigFn = func(cfg *seth.Config) error { return nil } type SethConfigFunction = func(*seth.Config) error +// OneEphemeralKeysLiveTestnetCheckFn checks whether there's at least one ephemeral key on a simulated network or at least one static key on a live network, +// and that there are no epehemeral keys on a live network. Root key is excluded from the check. +var OneEphemeralKeysLiveTestnetCheckFn = func(sethCfg *seth.Config) error { + concurrency := sethCfg.GetMaxConcurrency() + + if sethCfg.IsSimulatedNetwork() { + if concurrency < 1 { + return fmt.Errorf(INSUFFICIENT_EPHEMERAL_KEYS, 0) + } + + return nil + } + + if sethCfg.EphemeralAddrs != nil && int(*sethCfg.EphemeralAddrs) > 0 { + ephMsg := ` + Error: Ephemeral Addresses Detected on Live Network + + Ephemeral addresses are currently set for use on a live network, which is not permitted. The number of ephemeral addresses set is %d. Please make the following update to your TOML configuration file to correct this: + '[Seth] ephemeral_addresses_number = 0' + + Additionally, ensure the following requirements are met to run this test on a live network: + 1. Use more than one private key in your network configuration. + ` + + return errors.New(ephMsg) + } + + if concurrency < 1 { + return fmt.Errorf(INSUFFICIENT_STATIC_KEYS, len(sethCfg.Network.PrivateKeys)) + } + + return nil +} + // GetChainClient returns a seth client for the given network after validating the config func GetChainClient(config tc.SethConfig, network blockchain.EVMNetwork) (*seth.Client, error) { return GetChainClientWithConfigFunction(config, network, noOpSethConfigFn) diff --git a/integration-tests/actions/seth/automation_ocr_helpers.go b/integration-tests/actions/seth/automation_ocr_helpers.go index 88a41982f39..283ddfc026f 100644 --- a/integration-tests/actions/seth/automation_ocr_helpers.go +++ b/integration-tests/actions/seth/automation_ocr_helpers.go @@ -111,7 +111,11 @@ func DeployMultiCallAndFundDeploymentAddresses( numberOfUpkeeps int, linkFundsForEachUpkeep *big.Int, ) error { - concurrency := int(*chainClient.Cfg.EphemeralAddrs) + concurrency, err := GetAndAssertCorrectConcurrency(chainClient, 1) + if err != nil { + return err + } + operationsPerAddress := numberOfUpkeeps / concurrency multicallAddress, err := contracts.DeployMultiCallContract(chainClient) diff --git a/integration-tests/actions/seth/keeper_helpers.go b/integration-tests/actions/seth/keeper_helpers.go index 0056f743196..9309522228d 100644 --- a/integration-tests/actions/seth/keeper_helpers.go +++ b/integration-tests/actions/seth/keeper_helpers.go @@ -230,8 +230,8 @@ func RegisterUpkeepContractsWithCheckData(t *testing.T, client *seth.Client, lin registrationTxHashes := make([]common.Hash, 0) upkeepIds := make([]*big.Int, 0) - concurrency := int(*client.Cfg.EphemeralAddrs) - require.GreaterOrEqual(t, concurrency, 1, "You need at least 1 ephemeral address to deploy consumers. Please set them in TOML config. Example: `[Seth] ephemeral_addresses_number = 10`") + concurrency, err := GetAndAssertCorrectConcurrency(client, 1) + require.NoError(t, err, "Insufficient concurrency to execute action") type config struct { address string @@ -374,8 +374,8 @@ func DeployKeeperConsumers(t *testing.T, client *seth.Client, numberOfContracts l := logging.GetTestLogger(t) keeperConsumerContracts := make([]contracts.KeeperConsumer, 0) - concurrency := int(*client.Cfg.EphemeralAddrs) - require.GreaterOrEqual(t, concurrency, 1, "You need at least 1 ephemeral address to deploy consumers. Please set them in TOML config. Example: `[Seth] ephemeral_addresses_number = 10`") + concurrency, err := GetAndAssertCorrectConcurrency(client, 1) + require.NoError(t, err, "Insufficient concurrency to execute action") type result struct { contract contracts.KeeperConsumer @@ -601,7 +601,9 @@ func RegisterNewUpkeeps( addressesOfNewUpkeeps = append(addressesOfNewUpkeeps, upkeep.Address()) } - concurrency := int(*chainClient.Cfg.EphemeralAddrs) + concurrency, err := GetAndAssertCorrectConcurrency(chainClient, 1) + require.NoError(t, err, "Insufficient concurrency to execute action") + operationsPerAddress := numberOfNewUpkeeps / concurrency multicallAddress, err := contracts.DeployMultiCallContract(chainClient) @@ -616,3 +618,47 @@ func RegisterNewUpkeeps( return newlyDeployedUpkeeps, newUpkeepIDs } + +var INSUFFICIENT_EPHEMERAL_KEYS = ` +Error: Insufficient Ephemeral Addresses for Simulated Network + +To operate on a simulated network, you must configure at least one ephemeral address. Currently, %d ephemeral address(es) are set. Please update your TOML configuration file as follows to meet this requirement: +[Seth] ephemeral_addresses_number = 1 + +This adjustment ensures that your setup is minimaly viable. Although it is highly recommended to use at least 20 ephemeral addresses. +` + +var INSUFFICIENT_STATIC_KEYS = ` +Error: Insufficient Private Keys for Live Network + +To run this test on a live network, you must either: +1. Set at least two private keys in the '[Network.WalletKeys]' section of your TOML configuration file. Example format: + [Network.WalletKeys] + NETWORK_NAME=["PRIVATE_KEY_1", "PRIVATE_KEY_2"] +2. Set at least two private keys in the '[Network.EVMNetworks.NETWORK_NAME] section of your TOML configuration file. Example format: + evm_keys=["PRIVATE_KEY_1", "PRIVATE_KEY_2"] + +Currently, only %d private key/s is/are set. + +Recommended Action: +Distribute your funds across multiple private keys and update your configuration accordingly. Even though 1 private key is sufficient for testing, it is highly recommended to use at least 10 private keys. +` + +// GetAndAssertCorrectConcurrency checks Seth configuration for the number of ephemeral keys or static keys (depending on Seth configuration) and makes sure that +// the number is at least minConcurrency. If the number is less than minConcurrency, it returns an error. Root key is always excuded from the count. +func GetAndAssertCorrectConcurrency(client *seth.Client, minConcurrency int) (int, error) { + concurrency := client.Cfg.GetMaxConcurrency() + + var msg string + if client.Cfg.IsSimulatedNetwork() { + msg = fmt.Sprintf(INSUFFICIENT_EPHEMERAL_KEYS, concurrency) + } else { + msg = fmt.Sprintf(INSUFFICIENT_STATIC_KEYS, concurrency) + } + + if concurrency < minConcurrency { + return 0, fmt.Errorf(msg) + } + + return concurrency, nil +} diff --git a/integration-tests/benchmark/keeper_test.go b/integration-tests/benchmark/keeper_test.go index f334ddb5e1c..54dd1e71276 100644 --- a/integration-tests/benchmark/keeper_test.go +++ b/integration-tests/benchmark/keeper_test.go @@ -143,10 +143,9 @@ func TestAutomationBenchmark(t *testing.T) { benchmarkTestNetwork := getNetworkConfig(&config) l.Info().Str("Namespace", testEnvironment.Cfg.Namespace).Msg("Connected to Keepers Benchmark Environment") - testNetwork := utils.MustReplaceSimulatedNetworkUrlWithK8(l, benchmarkNetwork, *testEnvironment) - chainClient, err := actions_seth.GetChainClient(&config, testNetwork) + chainClient, err := actions_seth.GetChainClientWithConfigFunction(&config, testNetwork, actions_seth.OneEphemeralKeysLiveTestnetCheckFn) require.NoError(t, err, "Error getting Seth client") registryVersions := addRegistry(&config) @@ -331,6 +330,10 @@ func SetupAutomationBenchmarkEnv(t *testing.T, keeperTestConfig types.KeeperBenc numberOfNodes++ } + networkName := strings.ReplaceAll(testNetwork.Name, " ", "-") + networkName = strings.ReplaceAll(networkName, "_", "-") + testNetwork.Name = networkName + testEnvironment := environment.New(&environment.Config{ TTL: time.Hour * 720, // 30 days, NamespacePrefix: fmt.Sprintf( diff --git a/integration-tests/chaos/automation_chaos_test.go b/integration-tests/chaos/automation_chaos_test.go index 31504375546..605f7f083f4 100644 --- a/integration-tests/chaos/automation_chaos_test.go +++ b/integration-tests/chaos/automation_chaos_test.go @@ -7,7 +7,6 @@ import ( "time" "github.com/onsi/gomega" - "github.com/smartcontractkit/seth" "github.com/stretchr/testify/require" "go.uber.org/zap/zapcore" @@ -247,17 +246,7 @@ func TestAutomationChaos(t *testing.T) { network = utils.MustReplaceSimulatedNetworkUrlWithK8(l, network, *testEnvironment) - sethConfigFn := func(sethCfg *seth.Config) error { - if sethCfg.IsSimulatedNetwork() { - require.Equal(t, int(*sethCfg.EphemeralAddrs), 0, "You must not use ephemeral addresses on a simulated network. Please update '[Seth] ephemeral_addresses_number = 0' field in the TOML config file", *sethCfg.EphemeralAddrs) - // take only the first key, all others are not funded in genesis and will crash the test ¯\_(ツ)_/¯ - sethCfg.Network.PrivateKeys = sethCfg.Network.PrivateKeys[0:1] - } - - return nil - } - - chainClient, err := actions_seth.GetChainClientWithConfigFunction(&config, network, sethConfigFn) + chainClient, err := actions_seth.GetChainClientWithConfigFunction(&config, network, actions_seth.OneEphemeralKeysLiveTestnetCheckFn) require.NoError(t, err, "Error creating seth client") // Register cleanup for any test diff --git a/integration-tests/docker/test_env/test_env_builder.go b/integration-tests/docker/test_env/test_env_builder.go index 4a926bb2c65..0e18f0c22b7 100644 --- a/integration-tests/docker/test_env/test_env_builder.go +++ b/integration-tests/docker/test_env/test_env_builder.go @@ -323,7 +323,7 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) { } if b.hasSeth { - seth, err := actions_seth.GetChainClient(b.testConfig, networkConfig) + seth, err := actions_seth.GetChainClientWithConfigFunction(b.testConfig, networkConfig, actions_seth.OneEphemeralKeysLiveTestnetCheckFn) if err != nil { return nil, err } @@ -416,7 +416,7 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) { if b.hasSeth { b.te.sethClients = make(map[int64]*seth.Client) - seth, err := actions_seth.GetChainClient(b.testConfig, networkConfig) + seth, err := actions_seth.GetChainClientWithConfigFunction(b.testConfig, networkConfig, actions_seth.OneEphemeralKeysLiveTestnetCheckFn) if err != nil { return nil, err } diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 5311727f0f5..d1a2b62c8c5 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -29,7 +29,7 @@ require ( github.com/smartcontractkit/chainlink-vrf v0.0.0-20240222010609-cd67d123c772 github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 github.com/smartcontractkit/libocr v0.0.0-20240326191951-2bbe9382d052 - github.com/smartcontractkit/seth v0.1.6-0.20240416163802-874941ddafe0 + github.com/smartcontractkit/seth v0.1.6-0.20240422133624-326a80f0e828 github.com/smartcontractkit/wasp v0.4.7 github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.9.0 diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 93c46fe9998..d300fca267c 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1539,8 +1539,8 @@ github.com/smartcontractkit/grpc-proxy v0.0.0-20230731113816-f1be6620749f h1:hgJ github.com/smartcontractkit/grpc-proxy v0.0.0-20230731113816-f1be6620749f/go.mod h1:MvMXoufZAtqExNexqi4cjrNYE9MefKddKylxjS+//n0= github.com/smartcontractkit/libocr v0.0.0-20240326191951-2bbe9382d052 h1:1WFjrrVrWoQ9UpVMh7Mx4jDpzhmo1h8hFUKd9awIhIU= github.com/smartcontractkit/libocr v0.0.0-20240326191951-2bbe9382d052/go.mod h1:SJEZCHgMCAzzBvo9vMV2DQ9onfEcIJCYSViyP4JI6c4= -github.com/smartcontractkit/seth v0.1.6-0.20240416163802-874941ddafe0 h1:pmszbHy+Bi3WE9QguhsiSVwFHwHWCBZMldUxEBYjPH0= -github.com/smartcontractkit/seth v0.1.6-0.20240416163802-874941ddafe0/go.mod h1:2TMOZQ8WTAw7rR1YBbXpnad6VmT/+xDd/nXLmB7Eero= +github.com/smartcontractkit/seth v0.1.6-0.20240422133624-326a80f0e828 h1:fJv2+66PvfpASpt1Fq/R9BhYMHgD5I/SRp95IozqNj0= +github.com/smartcontractkit/seth v0.1.6-0.20240422133624-326a80f0e828/go.mod h1:2TMOZQ8WTAw7rR1YBbXpnad6VmT/+xDd/nXLmB7Eero= github.com/smartcontractkit/tdh2/go/ocr2/decryptionplugin v0.0.0-20230906073235-9e478e5e19f1 h1:yiKnypAqP8l0OX0P3klzZ7SCcBUxy5KqTAKZmQOvSQE= github.com/smartcontractkit/tdh2/go/ocr2/decryptionplugin v0.0.0-20230906073235-9e478e5e19f1/go.mod h1:q6f4fe39oZPdsh1i57WznEZgxd8siidMaSFq3wdPmVg= github.com/smartcontractkit/tdh2/go/tdh2 v0.0.0-20230906073235-9e478e5e19f1 h1:Dai1bn+Q5cpeGMQwRdjOdVjG8mmFFROVkSKuUgBErRQ= diff --git a/integration-tests/load/automationv2_1/automationv2_1_test.go b/integration-tests/load/automationv2_1/automationv2_1_test.go index a43a2c0f7e7..6ab73ead0f2 100644 --- a/integration-tests/load/automationv2_1/automationv2_1_test.go +++ b/integration-tests/load/automationv2_1/automationv2_1_test.go @@ -15,8 +15,6 @@ import ( "github.com/pkg/errors" - "github.com/smartcontractkit/seth" - geth "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -322,11 +320,7 @@ Load Config: testNetwork = utils.MustReplaceSimulatedNetworkUrlWithK8(l, testNetwork, *testEnvironment) - sethConfigFn := func(sethCfg *seth.Config) error { - return utils.ValidateAddressesTypeAndNumber(sethCfg, 50) - } - - chainClient, err := actions_seth.GetChainClientWithConfigFunction(loadedTestConfig, testNetwork, sethConfigFn) + chainClient, err := actions_seth.GetChainClientWithConfigFunction(loadedTestConfig, testNetwork, actions_seth.OneEphemeralKeysLiveTestnetCheckFn) require.NoError(t, err, "Error creating seth client") chainlinkNodes, err := client.ConnectChainlinkNodes(testEnvironment) @@ -430,7 +424,7 @@ Load Config: } for _, u := range loadedTestConfig.Automation.Load { - deploymentData, err := deployConsumerAndTriggerContracts(l, u, a.ChainClient, a.GetConcurrency(), multicallAddress, automationDefaultLinkFunds, a.LinkToken) + deploymentData, err := deployConsumerAndTriggerContracts(l, u, a.ChainClient, multicallAddress, automationDefaultLinkFunds, a.LinkToken) require.NoError(t, err, "Error deploying consumer and trigger contracts") consumerContracts = append(consumerContracts, deploymentData.ConsumerContracts...) diff --git a/integration-tests/load/automationv2_1/helpers.go b/integration-tests/load/automationv2_1/helpers.go index b5a5f4f2762..2086d02b863 100644 --- a/integration-tests/load/automationv2_1/helpers.go +++ b/integration-tests/load/automationv2_1/helpers.go @@ -80,9 +80,14 @@ type DeploymentData struct { LoadConfigs []aconfig.Load } -func deployConsumerAndTriggerContracts(l zerolog.Logger, loadConfig aconfig.Load, chainClient *seth.Client, concurrency int, multicallAddress common.Address, automationDefaultLinkFunds *big.Int, linkToken contracts.LinkToken) (DeploymentData, error) { +func deployConsumerAndTriggerContracts(l zerolog.Logger, loadConfig aconfig.Load, chainClient *seth.Client, multicallAddress common.Address, automationDefaultLinkFunds *big.Int, linkToken contracts.LinkToken) (DeploymentData, error) { data := DeploymentData{} + concurrency, err := actions_seth.GetAndAssertCorrectConcurrency(chainClient, 1) + if err != nil { + return DeploymentData{}, err + } + type deployedContractData struct { consumerContract contracts.KeeperConsumer triggerContract contracts.LogEmitter diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 2ece8122263..4d2d451cc30 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -21,7 +21,7 @@ require ( github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8 github.com/smartcontractkit/libocr v0.0.0-20240326191951-2bbe9382d052 - github.com/smartcontractkit/seth v0.1.6-0.20240416163802-874941ddafe0 + github.com/smartcontractkit/seth v0.1.6-0.20240422133624-326a80f0e828 github.com/smartcontractkit/tdh2/go/tdh2 v0.0.0-20230906073235-9e478e5e19f1 github.com/smartcontractkit/wasp v0.4.7 github.com/stretchr/testify v1.9.0 diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index d9744212a8f..ed16a966422 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1530,8 +1530,8 @@ github.com/smartcontractkit/grpc-proxy v0.0.0-20230731113816-f1be6620749f h1:hgJ github.com/smartcontractkit/grpc-proxy v0.0.0-20230731113816-f1be6620749f/go.mod h1:MvMXoufZAtqExNexqi4cjrNYE9MefKddKylxjS+//n0= github.com/smartcontractkit/libocr v0.0.0-20240326191951-2bbe9382d052 h1:1WFjrrVrWoQ9UpVMh7Mx4jDpzhmo1h8hFUKd9awIhIU= github.com/smartcontractkit/libocr v0.0.0-20240326191951-2bbe9382d052/go.mod h1:SJEZCHgMCAzzBvo9vMV2DQ9onfEcIJCYSViyP4JI6c4= -github.com/smartcontractkit/seth v0.1.6-0.20240416163802-874941ddafe0 h1:pmszbHy+Bi3WE9QguhsiSVwFHwHWCBZMldUxEBYjPH0= -github.com/smartcontractkit/seth v0.1.6-0.20240416163802-874941ddafe0/go.mod h1:2TMOZQ8WTAw7rR1YBbXpnad6VmT/+xDd/nXLmB7Eero= +github.com/smartcontractkit/seth v0.1.6-0.20240422133624-326a80f0e828 h1:fJv2+66PvfpASpt1Fq/R9BhYMHgD5I/SRp95IozqNj0= +github.com/smartcontractkit/seth v0.1.6-0.20240422133624-326a80f0e828/go.mod h1:2TMOZQ8WTAw7rR1YBbXpnad6VmT/+xDd/nXLmB7Eero= github.com/smartcontractkit/tdh2/go/ocr2/decryptionplugin v0.0.0-20230906073235-9e478e5e19f1 h1:yiKnypAqP8l0OX0P3klzZ7SCcBUxy5KqTAKZmQOvSQE= github.com/smartcontractkit/tdh2/go/ocr2/decryptionplugin v0.0.0-20230906073235-9e478e5e19f1/go.mod h1:q6f4fe39oZPdsh1i57WznEZgxd8siidMaSFq3wdPmVg= github.com/smartcontractkit/tdh2/go/tdh2 v0.0.0-20230906073235-9e478e5e19f1 h1:Dai1bn+Q5cpeGMQwRdjOdVjG8mmFFROVkSKuUgBErRQ= diff --git a/integration-tests/smoke/keeper_test.go b/integration-tests/smoke/keeper_test.go index b28219a18ef..f9c02e5f627 100644 --- a/integration-tests/smoke/keeper_test.go +++ b/integration-tests/smoke/keeper_test.go @@ -33,7 +33,7 @@ import ( const ( keeperDefaultUpkeepGasLimit = uint32(2500000) keeperDefaultLinkFunds = int64(9e18) - keeperDefaultUpkeepsToDeploy = 10 + keeperDefaultUpkeepsToDeploy = 2 numUpkeepsAllowedForStragglingTxs = 6 keeperExpectedData = "abcdef" ) @@ -221,85 +221,108 @@ func TestKeeperBlockCountPerTurn(t *testing.T) { } }) - keepersPerformed := make(map[*big.Int][]string, 0) + keepersPerformedLowFreq := map[*big.Int][]string{} - gom := gomega.NewGomegaWithT(t) - // Wait for upkeep to be performed twice by different keepers (buddies) - l.Info().Msg("Waiting for 1m for upkeeps to be performed by different keepers") - gom.Eventually(func(g gomega.Gomega) { - for i := 0; i < len(upkeepIDs); i++ { - counter, err := consumers[i].Counter(testcontext.Get(t)) - g.Expect(err).ShouldNot(gomega.HaveOccurred(), "Calling consumer's counter shouldn't fail") - l.Info().Int64("Upkeep counter", counter.Int64()).Msg("Number of upkeeps performed") + // gom := gomega.NewGomegaWithT(t) + // Wait for upkeep to be performed by two different keepers that alternate (buddies) + l.Info().Msg("Waiting for 2m for upkeeps to be performed by different keepers") + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + stop := time.After(2 * time.Minute) + + LOW_LOOP: + for { + select { + case <-ticker.C: + for i := 0; i < len(upkeepIDs); i++ { + counter, err := consumers[i].Counter(testcontext.Get(t)) + require.NoError(t, err, "Calling consumer's counter shouldn't fail") + l.Info().Str("UpkeepId", upkeepIDs[i].String()).Int64("Upkeep counter", counter.Int64()).Msg("Number of upkeeps performed") - upkeepInfo, err := registry.GetUpkeepInfo(testcontext.Get(t), upkeepIDs[i]) - g.Expect(err).ShouldNot(gomega.HaveOccurred(), "Registry's getUpkeep shouldn't fail") + upkeepInfo, err := registry.GetUpkeepInfo(testcontext.Get(t), upkeepIDs[i]) + require.NoError(t, err, "Registry's getUpkeep shouldn't fail") - latestKeeper := upkeepInfo.LastKeeper - previouslyPerformed := keepersPerformed[upkeepIDs[i]] - l.Info().Str("keeper", latestKeeper).Msg("last keeper to perform upkeep") - g.Expect(latestKeeper).ShouldNot(gomega.Equal(actions.ZeroAddress.String()), "Last keeper should be non zero") - g.Expect(latestKeeper).ShouldNot(gomega.BeElementOf(previouslyPerformed), "A new keeper node should perform this upkeep") + latestKeeper := upkeepInfo.LastKeeper + if latestKeeper == actions.ZeroAddress.String() { + continue + } - l.Info().Str("keeper", latestKeeper).Msg("New keeper performed upkeep") - keepersPerformed[upkeepIDs[i]] = append(keepersPerformed[upkeepIDs[i]], latestKeeper) + keepersPerformedLowFreq[upkeepIDs[i]] = append(keepersPerformedLowFreq[upkeepIDs[i]], latestKeeper) + } + case <-stop: + ticker.Stop() + break LOW_LOOP } - }, "1m", "1s").Should(gomega.Succeed()) + } - l.Info().Msg("Waiting again for 1m for upkeeps to be performed by different keepers") - gom.Eventually(func(g gomega.Gomega) { - for i := 0; i < len(upkeepIDs); i++ { - upkeepInfo, err := registry.GetUpkeepInfo(testcontext.Get(t), upkeepIDs[i]) - g.Expect(err).ShouldNot(gomega.HaveOccurred(), "Registry's getUpkeep shouldn't fail") + require.GreaterOrEqual(t, 2, len(keepersPerformedLowFreq), "At least 2 different keepers should have been performing upkeeps") + + // Now set BCPT to be low, so keepers change turn frequently + err = registry.SetConfig(lowBCPTRegistryConfig, contracts.OCRv2Config{}) + require.NoError(t, err, "Error setting registry config") + + keepersPerformedHigherFreq := map[*big.Int][]string{} + + ticker = time.NewTicker(1 * time.Second) + defer ticker.Stop() + + stop = time.After(2 * time.Minute) - latestKeeper := upkeepInfo.LastKeeper - previouslyPerformed := keepersPerformed[upkeepIDs[i]] - g.Expect(latestKeeper).ShouldNot(gomega.Equal(actions.ZeroAddress.String()), "Last keeper should be non zero") - g.Expect(latestKeeper).ShouldNot(gomega.BeElementOf(previouslyPerformed), "A new keeper node should perform this upkeep") + HIGH_LOOP: + for { + select { + case <-ticker.C: + for i := 0; i < len(upkeepIDs); i++ { + counter, err := consumers[i].Counter(testcontext.Get(t)) + require.NoError(t, err, "Calling consumer's counter shouldn't fail") + l.Info().Str("UpkeepId", upkeepIDs[i].String()).Int64("Upkeep counter", counter.Int64()).Msg("Number of upkeeps performed") - l.Info().Str("Keeper", latestKeeper).Msg("New keeper performed upkeep") - keepersPerformed[upkeepIDs[i]] = append(keepersPerformed[upkeepIDs[i]], latestKeeper) + upkeepInfo, err := registry.GetUpkeepInfo(testcontext.Get(t), upkeepIDs[i]) + require.NoError(t, err, "Registry's getUpkeep shouldn't fail") + + latestKeeper := upkeepInfo.LastKeeper + if latestKeeper == actions.ZeroAddress.String() { + continue + } + + keepersPerformedHigherFreq[upkeepIDs[i]] = append(keepersPerformedHigherFreq[upkeepIDs[i]], latestKeeper) + } + case <-stop: + ticker.Stop() + break HIGH_LOOP } - }, "1m", "1s").Should(gomega.Succeed()) + } - // Expect no new keepers to perform for a while - l.Info().Msg("Waiting for 1m for to check whether no upkeeps are performed") - gom.Consistently(func(g gomega.Gomega) { - for i := 0; i < len(upkeepIDs); i++ { - upkeepInfo, err := registry.GetUpkeepInfo(testcontext.Get(t), upkeepIDs[i]) - g.Expect(err).ShouldNot(gomega.HaveOccurred(), "Registry's getUpkeep shouldn't fail") + require.GreaterOrEqual(t, 3, len(keepersPerformedHigherFreq), "At least 3 different keepers should have been performing upkeeps after BCPT change") - latestKeeper := upkeepInfo.LastKeeper - previouslyPerformed := keepersPerformed[upkeepIDs[i]] - g.Expect(latestKeeper).ShouldNot(gomega.Equal(actions.ZeroAddress.String()), "Last keeper should be non zero") - g.Expect(latestKeeper).Should(gomega.BeElementOf(previouslyPerformed), "Existing keepers should alternate turns within BCPT") + var countFreq = func(keepers []string, freqMap map[string]int) { + for _, keeper := range keepers { + freqMap[keeper]++ } - }, "1m", "1s").Should(gomega.Succeed()) + } - // Now set BCPT to be low, so keepers change turn frequently - err = registry.SetConfig(lowBCPTRegistryConfig, contracts.OCRv2Config{}) - require.NoError(t, err, "Error setting registry config") + for i := 0; i < len(upkeepIDs); i++ { + lowFreqMap := make(map[string]int) + highFreqMap := make(map[string]int) - // Expect a new keeper to perform - l.Info().Msg("Waiting for 1m for upkeeps to be performed by different keepers after BCPT change") - gom.Eventually(func(g gomega.Gomega) { - for i := 0; i < len(upkeepIDs); i++ { - counter, err := consumers[i].Counter(testcontext.Get(t)) - g.Expect(err).ShouldNot(gomega.HaveOccurred(), "Calling consumer's counter shouldn't fail") - l.Info().Int64("Upkeep counter", counter.Int64()).Msg("Num upkeeps performed") + countFreq(keepersPerformedLowFreq[upkeepIDs[i]], lowFreqMap) + countFreq(keepersPerformedHigherFreq[upkeepIDs[i]], highFreqMap) - upkeepInfo, err := registry.GetUpkeepInfo(testcontext.Get(t), upkeepIDs[i]) - g.Expect(err).ShouldNot(gomega.HaveOccurred(), "Registry's getUpkeep shouldn't fail") + require.Greater(t, len(highFreqMap), len(lowFreqMap), "High frequency map should have more keepers than low frequency map") - latestKeeper := upkeepInfo.LastKeeper - previouslyPerformed := keepersPerformed[upkeepIDs[i]] - l.Info().Str("keeper", latestKeeper).Msg("last keeper to perform upkeep") - g.Expect(latestKeeper).ShouldNot(gomega.Equal(actions.ZeroAddress.String()), "Last keeper should be non zero") - g.Expect(latestKeeper).ShouldNot(gomega.BeElementOf(previouslyPerformed), "A new keeper node should perform this upkeep") + l.Info().Interface("Low BCPT", lowFreqMap).Interface("High BCPT", highFreqMap).Str("UpkeepID", upkeepIDs[i].String()).Msg("Keeper frequency map") - l.Info().Str("keeper", latestKeeper).Msg("New keeper performed upkeep") + for lowKeeper, lowFreq := range lowFreqMap { + highFreq, ok := highFreqMap[lowKeeper] + // it might happen due to fluke that a keeper is not found in high frequency map + if !ok { + continue + } + // require.True(t, ok, "Keeper %s not found in high frequency map. This should not happen", lowKeeper) + require.GreaterOrEqual(t, lowFreq, highFreq, "Keeper %s should have performed less times with high BCPT than with low BCPT", lowKeeper) } - }, "1m", "1s").Should(gomega.Succeed()) + } }) } } diff --git a/integration-tests/testconfig/testconfig.go b/integration-tests/testconfig/testconfig.go index 30a795e1881..1f76e65f531 100644 --- a/integration-tests/testconfig/testconfig.go +++ b/integration-tests/testconfig/testconfig.go @@ -4,6 +4,7 @@ import ( "embed" "encoding/base64" "fmt" + "math/big" "os" "slices" "strings" @@ -23,6 +24,7 @@ import ( ctf_test_env "github.com/smartcontractkit/chainlink-testing-framework/docker/test_env" k8s_config "github.com/smartcontractkit/chainlink-testing-framework/k8s/config" "github.com/smartcontractkit/chainlink-testing-framework/logging" + "github.com/smartcontractkit/chainlink-testing-framework/utils/conversions" "github.com/smartcontractkit/chainlink-testing-framework/utils/osutil" a_config "github.com/smartcontractkit/chainlink/integration-tests/testconfig/automation" f_config "github.com/smartcontractkit/chainlink/integration-tests/testconfig/functions" @@ -396,6 +398,59 @@ func GetConfig(configurationName string, product Product) (TestConfig, error) { testConfig.Common = &Common{} } + isAnySimulated := false + for _, network := range testConfig.Network.SelectedNetworks { + if strings.Contains(strings.ToUpper(network), "SIMULATED") { + isAnySimulated = true + break + } + } + + if testConfig.Seth != nil && !isAnySimulated && (testConfig.Seth.EphemeralAddrs != nil && *testConfig.Seth.EphemeralAddrs != 0) { + msg := ` + Error: Ephemeral Addresses Misconfigured + + Your configuration contains %d ephemeral address(es), which are only permissible for use with simulated networks. Using them on live networks poses a risk of losing your funds. Please adjust your TOML configuration file to ensure proper settings: + [Seth] + ephemeral_addresses_number = 0 +` + return TestConfig{}, fmt.Errorf(msg, *testConfig.Seth.EphemeralAddrs) + } + + if testConfig.Seth != nil && (testConfig.Seth.EphemeralAddrs != nil && *testConfig.Seth.EphemeralAddrs != 0) { + rootBuffer := testConfig.Seth.RootKeyFundsBuffer + if rootBuffer == nil { + rootBuffer = big.NewInt(0) + } + clNodeFunding := testConfig.Common.ChainlinkNodeFunding + if clNodeFunding == nil { + zero := 0.0 + clNodeFunding = &zero + } + minRequiredFunds := big.NewFloat(0).Mul(big.NewFloat(*clNodeFunding), big.NewFloat(6.0)) + + //add buffer to the minimum required funds, this isn't even a rough estimate, because we don't know how many contracts will be deployed from root key, but it's here to let you know that you should have some buffer + minRequiredFundsBuffered := big.NewFloat(0).Mul(minRequiredFunds, big.NewFloat(1.2)) + minRequiredFundsBufferedInt, _ := minRequiredFundsBuffered.Int(nil) + + rootBuffer64, _ := rootBuffer.Float64() + + if big.NewFloat(rootBuffer64).Cmp(minRequiredFundsBuffered) <= 0 { + msg := ` +The funds allocated to the root key buffer are below the minimum requirement, which could lead to insufficient funds for performing contract deployments. Please review and adjust your TOML configuration file to ensure that the root key buffer has adequate funds. Increase the fund settings as necessary to meet this requirement. + +Example: +[Seth] +root_key_funds_buffer = 1_000 +` + + logger.Warn(). + Str("Root key buffer (wei/ether)", fmt.Sprintf("%s/%s", rootBuffer.String(), conversions.WeiToEther(rootBuffer).Text('f', -1))). + Str("Minimum required funds (wei/ether)", fmt.Sprintf("%s/%s", minRequiredFundsBuffered.String(), conversions.WeiToEther(minRequiredFundsBufferedInt).Text('f', -1))). + Msg(msg) + } + } + logger.Debug().Msg("Correct test config constructed successfully") return testConfig, nil } diff --git a/integration-tests/testsetups/keeper_benchmark.go b/integration-tests/testsetups/keeper_benchmark.go index fd45d1d6093..c2c24fac1e3 100644 --- a/integration-tests/testsetups/keeper_benchmark.go +++ b/integration-tests/testsetups/keeper_benchmark.go @@ -330,23 +330,6 @@ func (k *KeeperBenchmarkTest) Run() { } } - // Progress log for visibility - go func() { - ticker := time.NewTicker(10 * time.Second) - defer ticker.Stop() - for { - select { - case <-ticker.C: - k.log.Warn().Str("Done/Total", fmt.Sprintf("%d/%d", finishedObservations.Load(), startedObservations.Load())).Msg("Upkeep observation progress") - if finishedObservations.Load() == startedObservations.Load() { - return - } - case <-ctx.Done(): - return - } - } - }() - if err := errgroup.Wait(); err != nil { k.t.Fatalf("errored when waiting for upkeeps: %v", err) } diff --git a/integration-tests/universal/log_poller/helpers.go b/integration-tests/universal/log_poller/helpers.go index efa3eb9abc3..0da2c3eb49f 100644 --- a/integration-tests/universal/log_poller/helpers.go +++ b/integration-tests/universal/log_poller/helpers.go @@ -41,7 +41,6 @@ import ( tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig" lp_config "github.com/smartcontractkit/chainlink/integration-tests/testconfig/log_poller" "github.com/smartcontractkit/chainlink/integration-tests/types/config/node" - "github.com/smartcontractkit/chainlink/integration-tests/utils" evmcfg "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/toml" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/logpoller" cltypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" @@ -1189,9 +1188,6 @@ func SetupLogPollerTestDocker( chainClient, err := env.GetSethClient(selectedNetwork.ChainID) require.NoError(t, err, "Error getting seth client") - err = utils.ValidateAddressesTypeAndNumber(chainClient.Cfg, *testConfig.LogPoller.General.Contracts) - require.NoError(t, err, "Error validating Seth addresses types and number") - nodeClients := env.ClCluster.NodeAPIs() workerNodes := nodeClients[1:] diff --git a/integration-tests/utils/seth.go b/integration-tests/utils/seth.go index 4fab11cc728..ef9b331a447 100644 --- a/integration-tests/utils/seth.go +++ b/integration-tests/utils/seth.go @@ -120,27 +120,3 @@ func ValidateSethNetworkConfig(cfg *seth.Network) error { return nil } - -// ValidateAddressesTypeAndNumber makes sure that minKeyNumber of epehemeral addresses are used on a simulated network and that no ephemeral addresses are used on a live network. Also, it makes sure that at least minKeyNumber of private keys are used on a live network. -func ValidateAddressesTypeAndNumber(sethCfg *seth.Config, minKeyNumber int) error { - if sethCfg.IsSimulatedNetwork() { - if int(*sethCfg.EphemeralAddrs) < minKeyNumber { - return fmt.Errorf("You need to use at least %d ephemeral addresses on a simulated network, but %d was used. Please update '[Seth] ephemeral_addresses_number' field in the TOML config file", minKeyNumber, int(*sethCfg.EphemeralAddrs)) - } - // take only the first key, all others are not funded in genesis and will crash the test ¯\_(ツ)_/¯ - sethCfg.Network.PrivateKeys = sethCfg.Network.PrivateKeys[0:1] - } else { - if int(*sethCfg.EphemeralAddrs) > 0 { - msg := `You must not use any ephemeral addresses on a live network, but %d were set. All funds send to them will be lost. You should prefund some addresses instead manually (or using Seth CLI) and pass their private keys in network configuration or - TOML keyfile. But if you really know what you are doing remove this check from the test.` - return fmt.Errorf(msg, int(*sethCfg.EphemeralAddrs)) - } - if len(sethCfg.Network.PrivateKeys) < minKeyNumber { - msg := `You need to use at least %d addresses on a live network, but %d were passed. With current setting either concurrent deployment or load generation will surely fail. Please update '[Network.NETWORK_NAME] private_keys_secret=[]' field in the TOML config file - or set SETH_KEYFILE_PATH env var with path to keyfile.toml generated by Seth key CLI.` - return fmt.Errorf(msg, minKeyNumber, len(sethCfg.Network.PrivateKeys)) - } - } - - return nil -}