Add lane test enable (#14704)

* enable test

* test

* changes

* enabling another lane

* half enable

* reset

* changes

* fix lint

* fix

* review comments

* lint

* fix test

integration-tests/deployment/ccip/add_lane_test.go

@@ -2,31 +2,46 @@ package ccipdeployment
 
 import (
 	"testing"
+	"time"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/stretchr/testify/require"
 
 	"github.com/smartcontractkit/chainlink-testing-framework/lib/utils/testcontext"
+
 	"github.com/smartcontractkit/chainlink/integration-tests/deployment"
+	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/offramp"
 	"github.com/smartcontractkit/chainlink/v2/core/logger"
 )
 
-// TestAddLane covers the workflow of adding a lane
-// between existing supported chains in CCIP.
+// TestAddLane covers the workflow of adding a lane between two chains and enabling it.
+// It also covers the case where the onRamp is disabled on the OffRamp contract initially and then enabled.
 func TestAddLane(t *testing.T) {
-	// TODO: The offchain code doesn't yet support partial lane
-	// enablement, need to address then re-enable this test.
-	t.Skip()
-	e := NewMemoryEnvironmentWithJobs(t, logger.TestLogger(t), 3, 4)
+	// We add more chains to the chainlink nodes than the number of chains where CCIP is deployed.
+	e := NewMemoryEnvironmentWithJobs(t, logger.TestLogger(t), 4, 4)
 	// Here we have CR + nodes set up, but no CCIP contracts deployed.
 	state, err := LoadOnchainState(e.Env, e.Ab)
 	require.NoError(t, err)
+
+	selectors := e.Env.AllChainSelectors()
+	// deploy CCIP contracts on two chains
+	chain1, chain2 := selectors[0], selectors[1]
+
+	feeds := state.Chains[e.FeedChainSel].USDFeeds
+	tokenConfig := NewTestTokenConfig(feeds)
+
+	feeTokenContracts := make(map[uint64]FeeTokenContracts)
+	for _, sel := range []uint64{chain1, chain2} {
+		feeTokenContracts[sel] = e.FeeTokenContracts[sel]
+	}
 	// Set up CCIP contracts and a DON per chain.
 	err = DeployCCIPContracts(e.Env, e.Ab, DeployCCIPContractConfig{
 		HomeChainSel:       e.HomeChainSel,
 		FeedChainSel:       e.FeedChainSel,
-		TokenConfig:        NewTokenConfig(),
+		TokenConfig:        tokenConfig,
 		MCMSConfig:         NewTestMCMSConfig(t, e.Env),
-		FeeTokenContracts:  e.FeeTokenContracts,
+		FeeTokenContracts:  feeTokenContracts,
+		ChainsToDeploy:     []uint64{chain1, chain2},
 		CapabilityRegistry: state.Chains[e.HomeChainSel].CapabilityRegistry.Address(),
 		OCRSecrets:         deployment.XXXGenerateTestOCRSecrets(),
 	})
@@ -35,32 +50,90 @@ func TestAddLane(t *testing.T) {
 	// We expect no lanes available on any chain.
 	state, err = LoadOnchainState(e.Env, e.Ab)
 	require.NoError(t, err)
-	for _, chain := range state.Chains {
+	for _, sel := range []uint64{chain1, chain2} {
+		chain := state.Chains[sel]
 		offRamps, err := chain.Router.GetOffRamps(nil)
 		require.NoError(t, err)
 		require.Len(t, offRamps, 0)
 	}
 
-	// Add one lane and send traffic.
-	from, to := e.Env.AllChainSelectors()[0], e.Env.AllChainSelectors()[1]
-	require.NoError(t, AddLane(e.Env, state, from, to))
+	replayBlocks, err := LatestBlocksByChain(testcontext.Get(t), e.Env.Chains)
+	require.NoError(t, err)
+
+	// Add one lane from chain1 to chain 2 and send traffic.
+	require.NoError(t, AddLane(e.Env, state, chain1, chain2))
+
+	ReplayLogs(t, e.Env.Offchain, replayBlocks)
+	time.Sleep(30 * time.Second)
+	// disable the onRamp initially on OffRamp
+	disableRampTx, err := state.Chains[chain2].OffRamp.ApplySourceChainConfigUpdates(e.Env.Chains[chain2].DeployerKey, []offramp.OffRampSourceChainConfigArgs{
+		{
+			Router:              state.Chains[chain2].Router.Address(),
+			SourceChainSelector: chain1,
+			IsEnabled:           false,
+			OnRamp:              common.LeftPadBytes(state.Chains[chain1].OnRamp.Address().Bytes(), 32),
+		},
+	})
+	_, err = deployment.ConfirmIfNoError(e.Env.Chains[chain2], disableRampTx, err)
+	require.NoError(t, err)
 
-	for sel, chain := range state.Chains {
+	for _, sel := range []uint64{chain1, chain2} {
+		chain := state.Chains[sel]
 		offRamps, err := chain.Router.GetOffRamps(nil)
 		require.NoError(t, err)
-		if sel == to {
+		if sel == chain2 {
 			require.Len(t, offRamps, 1)
+			srcCfg, err := chain.OffRamp.GetSourceChainConfig(nil, chain1)
+			require.NoError(t, err)
+			require.Equal(t, common.LeftPadBytes(state.Chains[chain1].OnRamp.Address().Bytes(), 32), srcCfg.OnRamp)
+			require.False(t, srcCfg.IsEnabled)
 		} else {
 			require.Len(t, offRamps, 0)
 		}
 	}
-	latesthdr, err := e.Env.Chains[to].Client.HeaderByNumber(testcontext.Get(t), nil)
+
+	latesthdr, err := e.Env.Chains[chain2].Client.HeaderByNumber(testcontext.Get(t), nil)
 	require.NoError(t, err)
 	startBlock := latesthdr.Number.Uint64()
-	seqNum := SendRequest(t, e.Env, state, from, to, false)
-	require.Equal(t, uint64(1), seqNum)
-	require.NoError(t, ConfirmExecWithSeqNr(t, e.Env.Chains[from], e.Env.Chains[to], state.Chains[to].OffRamp, &startBlock, seqNum))
+	// Send traffic on the first lane and it should not be processed by the plugin as onRamp is disabled
+	// we will check this by confirming that the message is not executed by the end of the test
+	seqNum1 := TestSendRequest(t, e.Env, state, chain1, chain2, false)
+	require.Equal(t, uint64(1), seqNum1)
+
+	// Add another lane
+	require.NoError(t, AddLane(e.Env, state, chain2, chain1))
+
+	// Send traffic on the second lane and it should succeed
+	latesthdr, err = e.Env.Chains[chain1].Client.HeaderByNumber(testcontext.Get(t), nil)
+	require.NoError(t, err)
+	startBlock2 := latesthdr.Number.Uint64()
+	seqNum2 := TestSendRequest(t, e.Env, state, chain2, chain1, false)
+	require.Equal(t, uint64(1), seqNum2)
+	require.NoError(t, ConfirmExecWithSeqNr(t, e.Env.Chains[chain2], e.Env.Chains[chain1], state.Chains[chain1].OffRamp, &startBlock2, seqNum2))
+
+	// now check for the previous message from chain 1 to chain 2 that it has not been executed till now as the onRamp was disabled
+	ConfirmNoExecConsistentlyWithSeqNr(t, e.Env.Chains[chain1], e.Env.Chains[chain2], state.Chains[chain2].OffRamp, seqNum1, 30*time.Second)
+
+	// enable the onRamp on OffRamp
+	enableRampTx, err := state.Chains[chain2].OffRamp.ApplySourceChainConfigUpdates(e.Env.Chains[chain2].DeployerKey, []offramp.OffRampSourceChainConfigArgs{
+		{
+			Router:              state.Chains[chain2].Router.Address(),
+			SourceChainSelector: chain1,
+			IsEnabled:           true,
+			OnRamp:              common.LeftPadBytes(state.Chains[chain1].OnRamp.Address().Bytes(), 32),
+		},
+	})
+	_, err = deployment.ConfirmIfNoError(e.Env.Chains[chain2], enableRampTx, err)
+	require.NoError(t, err)
+
+	srcCfg, err := state.Chains[chain2].OffRamp.GetSourceChainConfig(nil, chain1)
+	require.NoError(t, err)
+	require.Equal(t, common.LeftPadBytes(state.Chains[chain1].OnRamp.Address().Bytes(), 32), srcCfg.OnRamp)
+	require.True(t, srcCfg.IsEnabled)
 
-	// TODO: Add a second lane, then disable the first and
-	// ensure we can send on the second but not the first.
+	// we need the replay here otherwise plugin is not able to locate the message
+	ReplayLogs(t, e.Env.Offchain, replayBlocks)
+	time.Sleep(30 * time.Second)
+	// Now that the onRamp is enabled, the request should be processed
+	require.NoError(t, ConfirmExecWithSeqNr(t, e.Env.Chains[chain1], e.Env.Chains[chain2], state.Chains[chain2].OffRamp, &startBlock, seqNum1))
 }

integration-tests/deployment/ccip/changeset/active_candidate_test.go

@@ -7,15 +7,18 @@ import (
 	"github.com/smartcontractkit/ccip-owner-contracts/pkg/proposal/timelock"
 
 	"github.com/smartcontractkit/chainlink-testing-framework/lib/utils/testcontext"
+
 	cctypes "github.com/smartcontractkit/chainlink/v2/core/capabilities/ccip/types"
 
 	cciptypes "github.com/smartcontractkit/chainlink-ccip/pkg/types/ccipocr3"
 	"github.com/smartcontractkit/chainlink-ccip/pluginconfig"
+
 	"github.com/smartcontractkit/chainlink/integration-tests/deployment"
 
 	"github.com/stretchr/testify/require"
 
 	jobv1 "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/job"
+
 	ccdeploy "github.com/smartcontractkit/chainlink/integration-tests/deployment/ccip"
 
 	"github.com/smartcontractkit/chainlink/v2/core/logger"
@@ -98,7 +101,7 @@ func TestActiveCandidate(t *testing.T) {
 			require.NoError(t, err)
 			block := latesthdr.Number.Uint64()
 			startBlocks[dest] = &block
-			seqNum := ccdeploy.SendRequest(t, e, state, src, dest, false)
+			seqNum := ccdeploy.TestSendRequest(t, e, state, src, dest, false)
 			expectedSeqNum[dest] = seqNum
 		}
 	}

integration-tests/deployment/ccip/changeset/add_chain_test.go

@@ -12,12 +12,11 @@ import (
 
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/rmn_home"
 
-	cciptypes "github.com/smartcontractkit/chainlink-ccip/pkg/types/ccipocr3"
-	"github.com/smartcontractkit/chainlink-ccip/pluginconfig"
-
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	cciptypes "github.com/smartcontractkit/chainlink-ccip/pkg/types/ccipocr3"
+
 	"github.com/smartcontractkit/chainlink-testing-framework/lib/utils/testcontext"
 
 	"github.com/smartcontractkit/chainlink/integration-tests/deployment"
@@ -37,22 +36,7 @@ func TestAddChainInbound(t *testing.T) {
 	// We deploy to the rest.
 	initialDeploy := e.Env.AllChainSelectorsExcluding([]uint64{newChain})
 
-	feeds := state.Chains[e.FeedChainSel].USDFeeds
-	tokenConfig := ccipdeployment.NewTokenConfig()
-	tokenConfig.UpsertTokenInfo(ccipdeployment.LinkSymbol,
-		pluginconfig.TokenInfo{
-			AggregatorAddress: cciptypes.UnknownEncodedAddress(feeds[ccipdeployment.LinkSymbol].Address().String()),
-			Decimals:          ccipdeployment.LinkDecimals,
-			DeviationPPB:      cciptypes.NewBigIntFromInt64(1e9),
-		},
-	)
-	tokenConfig.UpsertTokenInfo(ccipdeployment.WethSymbol,
-		pluginconfig.TokenInfo{
-			AggregatorAddress: cciptypes.UnknownEncodedAddress(feeds[ccipdeployment.WethSymbol].Address().String()),
-			Decimals:          ccipdeployment.WethDecimals,
-			DeviationPPB:      cciptypes.NewBigIntFromInt64(1e9),
-		},
-	)
+	tokenConfig := ccipdeployment.NewTestTokenConfig(state.Chains[e.FeedChainSel].USDFeeds)
 	err = ccipdeployment.DeployCCIPContracts(e.Env, e.Ab, ccipdeployment.DeployCCIPContractConfig{
 		HomeChainSel:       e.HomeChainSel,
 		FeedChainSel:       e.FeedChainSel,
@@ -149,7 +133,7 @@ func TestAddChainInbound(t *testing.T) {
 	}
 	// TODO This currently is not working - Able to send the request here but request gets stuck in execution
 	// Send a new message and expect that this is delivered once the chain is completely set up as inbound
-	//SendRequest(t, e.Env, state, initialDeploy[0], newChain, true)
+	//TestSendRequest(t, e.Env, state, initialDeploy[0], newChain, true)
 
 	t.Logf("Executing add don and set candidate proposal for commit plugin on chain %d", newChain)
 	addDonProp, err := AddDonAndSetCandidateProposal(state, e.Env, nodes, deployment.XXXGenerateTestOCRSecrets(), e.HomeChainSel, e.FeedChainSel, newChain, tokenConfig, types.PluginTypeCCIPCommit)
@@ -226,7 +210,7 @@ func TestAddChainInbound(t *testing.T) {
 	latesthdr, err := e.Env.Chains[newChain].Client.HeaderByNumber(testcontext.Get(t), nil)
 	require.NoError(t, err)
 	startBlock := latesthdr.Number.Uint64()
-	seqNr := ccipdeployment.SendRequest(t, e.Env, state, initialDeploy[0], newChain, true)
+	seqNr := ccipdeployment.TestSendRequest(t, e.Env, state, initialDeploy[0], newChain, true)
 	require.NoError(t,
 		ccipdeployment.ConfirmCommitWithExpectedSeqNumRange(t, e.Env.Chains[initialDeploy[0]], e.Env.Chains[newChain], state.Chains[newChain].OffRamp, &startBlock, cciptypes.SeqNumRange{
 			cciptypes.SeqNum(1),

integration-tests/deployment/ccip/changeset/initial_deploy_test.go

@@ -3,9 +3,6 @@ package changeset
 import (
 	"testing"
 
-	cciptypes "github.com/smartcontractkit/chainlink-ccip/pkg/types/ccipocr3"
-	"github.com/smartcontractkit/chainlink-ccip/pluginconfig"
-
 	"github.com/smartcontractkit/chainlink/integration-tests/deployment"
 	ccdeploy "github.com/smartcontractkit/chainlink/integration-tests/deployment/ccip"
 
@@ -28,28 +25,11 @@ func TestInitialDeploy(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, state.Chains[tenv.HomeChainSel].LinkToken)
 
-	feeds := state.Chains[tenv.FeedChainSel].USDFeeds
-	tokenConfig := ccdeploy.NewTokenConfig()
-	tokenConfig.UpsertTokenInfo(ccdeploy.LinkSymbol,
-		pluginconfig.TokenInfo{
-			AggregatorAddress: cciptypes.UnknownEncodedAddress(feeds[ccdeploy.LinkSymbol].Address().String()),
-			Decimals:          ccdeploy.LinkDecimals,
-			DeviationPPB:      cciptypes.NewBigIntFromInt64(1e9),
-		},
-	)
-	tokenConfig.UpsertTokenInfo(ccdeploy.WethSymbol,
-		pluginconfig.TokenInfo{
-			AggregatorAddress: cciptypes.UnknownEncodedAddress(feeds[ccdeploy.WethSymbol].Address().String()),
-			Decimals:          ccdeploy.WethDecimals,
-			DeviationPPB:      cciptypes.NewBigIntFromInt64(1e9),
-		},
-	)
-
 	output, err := InitialDeployChangeSet(tenv.Ab, tenv.Env, ccdeploy.DeployCCIPContractConfig{
 		HomeChainSel:       tenv.HomeChainSel,
 		FeedChainSel:       tenv.FeedChainSel,
 		ChainsToDeploy:     tenv.Env.AllChainSelectors(),
-		TokenConfig:        tokenConfig,
+		TokenConfig:        ccdeploy.NewTestTokenConfig(state.Chains[tenv.FeedChainSel].USDFeeds),
 		MCMSConfig:         ccdeploy.NewTestMCMSConfig(t, e),
 		CapabilityRegistry: state.Chains[tenv.HomeChainSel].CapabilityRegistry.Address(),
 		FeeTokenContracts:  tenv.FeeTokenContracts,
@@ -92,7 +72,7 @@ func TestInitialDeploy(t *testing.T) {
 			require.NoError(t, err)
 			block := latesthdr.Number.Uint64()
 			startBlocks[dest] = &block
-			seqNum := ccdeploy.SendRequest(t, e, state, src, dest, false)
+			seqNum := ccdeploy.TestSendRequest(t, e, state, src, dest, false)
 			expectedSeqNum[dest] = seqNum
 		}
 	}

integration-tests/deployment/ccip/deploy_home_chain.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"math/big"
 	"os"
@@ -27,6 +28,8 @@ import (
 	"github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3confighelper"
 
 	"github.com/smartcontractkit/chainlink/integration-tests/deployment"
+	p2ptypes "github.com/smartcontractkit/chainlink/v2/core/services/p2p/types"
+
 	cctypes "github.com/smartcontractkit/chainlink/v2/core/capabilities/ccip/types"
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ccip_home"
@@ -201,13 +204,38 @@ func DeployCapReg(lggr logger.Logger, ab deployment.AddressBook, chain deploymen
 	return capReg, nil
 }
 
+func isEqualCapabilitiesRegistryNodeParams(a, b capabilities_registry.CapabilitiesRegistryNodeParams) (bool, error) {
+	aBytes, err := json.Marshal(a)
+	if err != nil {
+		return false, err
+	}
+	bBytes, err := json.Marshal(b)
+	if err != nil {
+		return false, err
+	}
+	return bytes.Equal(aBytes, bBytes), nil
+}
+
 func AddNodes(
 	lggr logger.Logger,
 	capReg *capabilities_registry.CapabilitiesRegistry,
 	chain deployment.Chain,
 	p2pIDs [][32]byte,
 ) error {
 	var nodeParams []capabilities_registry.CapabilitiesRegistryNodeParams
+	nodes, err := capReg.GetNodes(nil)
+	if err != nil {
+		return err
+	}
+	existingNodeParams := make(map[p2ptypes.PeerID]capabilities_registry.CapabilitiesRegistryNodeParams)
+	for _, node := range nodes {
+		existingNodeParams[node.P2pId] = capabilities_registry.CapabilitiesRegistryNodeParams{
+			NodeOperatorId:      node.NodeOperatorId,
+			Signer:              node.Signer,
+			P2pId:               node.P2pId,
+			HashedCapabilityIds: node.HashedCapabilityIds,
+		}
+	}
 	for _, p2pID := range p2pIDs {
 		// if any p2pIDs are empty throw error
 		if bytes.Equal(p2pID[:], make([]byte, 32)) {
@@ -220,8 +248,19 @@ func AddNodes(
 			EncryptionPublicKey: p2pID, // Not used in tests
 			HashedCapabilityIds: [][32]byte{CCIPCapabilityID},
 		}
+		if existing, ok := existingNodeParams[p2pID]; ok {
+			if isEqual, err := isEqualCapabilitiesRegistryNodeParams(existing, nodeParam); err != nil && isEqual {
+				lggr.Infow("Node already exists", "p2pID", p2pID)
+				continue
+			}
+		}
+
 		nodeParams = append(nodeParams, nodeParam)
 	}
+	if len(nodeParams) == 0 {
+		lggr.Infow("No new nodes to add")
+		return nil
+	}
 	tx, err := capReg.AddNodes(chain.DeployerKey, nodeParams)
 	if err != nil {
 		lggr.Errorw("Failed to add nodes", "err", deployment.MaybeDataErr(err))

integration-tests/deployment/ccip/rmn_test.go

@@ -12,6 +12,7 @@ import (
 
 	jobv1 "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/job"
 	"github.com/smartcontractkit/chainlink-testing-framework/lib/utils/testcontext"
+
 	"github.com/smartcontractkit/chainlink/integration-tests/deployment"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/rmn_home"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/rmn_remote"
@@ -181,7 +182,7 @@ func TestRMN(t *testing.T) {
 	require.NoError(t, err)
 	block := latesthdr.Number.Uint64()
 	startBlocks[dstChain.Selector] = &block
-	seqNum := SendRequest(t, envWithRMN.Env, onChainState, srcChain.Selector, dstChain.Selector, false)
+	seqNum := TestSendRequest(t, envWithRMN.Env, onChainState, srcChain.Selector, dstChain.Selector, false)
 	t.Logf("expected seqNum: %d", seqNum)
 
 	expectedSeqNum := make(map[uint64]uint64)