Merge branch 'develop' into refactor-status-message

.github/workflows/e2e.yml

@@ -23,7 +23,7 @@ on:
         type: boolean
         default: false
 
-concurrency: 
+concurrency:
   group: e2e-${{ github.head_ref || github.sha }}
   cancel-in-progress: true
 
@@ -115,6 +115,7 @@ jobs:
       UPGRADE_LIGHT_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_LIGHT_TESTS }}
       UPGRADE_IMPORT_MAINNET_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_IMPORT_MAINNET_TESTS }}
       ADMIN_TESTS: ${{ steps.matrix-conditionals.outputs.ADMIN_TESTS }}
+      ADMIN_UPGRADE_TESTS: ${{ steps.matrix-conditionals.outputs.ADMIN_UPGRADE_TESTS }}
       PERFORMANCE_TESTS: ${{ steps.matrix-conditionals.outputs.PERFORMANCE_TESTS }}
       STATEFUL_DATA_TESTS: ${{ steps.matrix-conditionals.outputs.STATEFUL_DATA_TESTS }}
       TSS_MIGRATION_TESTS: ${{ steps.matrix-conditionals.outputs.TSS_MIGRATION_TESTS }}
@@ -147,6 +148,7 @@ jobs:
               core.setOutput('UPGRADE_LIGHT_TESTS', labels.includes('UPGRADE_LIGHT_TESTS'));
               core.setOutput('UPGRADE_IMPORT_MAINNET_TESTS', labels.includes('UPGRADE_IMPORT_MAINNET_TESTS'));
               core.setOutput('ADMIN_TESTS', labels.includes('ADMIN_TESTS'));
+              core.setOutput('ADMIN_UPGRADE_TESTS', labels.includes('ADMIN_UPGRADE_TESTS'));
               core.setOutput('PERFORMANCE_TESTS', labels.includes('PERFORMANCE_TESTS'));
               core.setOutput('STATEFUL_DATA_TESTS', labels.includes('STATEFUL_DATA_TESTS'));
               core.setOutput('TSS_MIGRATION_TESTS', labels.includes('TSS_MIGRATION_TESTS'));
@@ -157,7 +159,8 @@ jobs:
             } else if (context.eventName === 'merge_group') {
               // default mergequeue tests
               core.setOutput('DEFAULT_TESTS', true);
-              core.setOutput('UPGRADE_LIGHT_TESTS', true);
+              core.setOutput('UPGRADE_TESTS', true);
+              core.setOutput('ADMIN_UPGRADE_TESTS', true);
 
               // conditional tests based on PR labels
               const commit_message = context.payload.merge_group.head_commit.message;
@@ -177,6 +180,7 @@ jobs:
               core.setOutput('UPGRADE_LIGHT_TESTS', true);
               core.setOutput('UPGRADE_IMPORT_MAINNET_TESTS', true);
               core.setOutput('ADMIN_TESTS', true);
+              core.setOutput('ADMIN_UPGRADE_TESTS', true);
               core.setOutput('PERFORMANCE_TESTS', true);
               core.setOutput('STATEFUL_DATA_TESTS', true);
               core.setOutput('SOLANA_TESTS', true);
@@ -188,6 +192,7 @@ jobs:
               core.setOutput('UPGRADE_LIGHT_TESTS', true);
               core.setOutput('UPGRADE_IMPORT_MAINNET_TESTS', true);
               core.setOutput('ADMIN_TESTS', true);
+              core.setOutput('ADMIN_UPGRADE_TESTS', true);
               core.setOutput('PERFORMANCE_TESTS', true);
               core.setOutput('STATEFUL_DATA_TESTS', true);
               core.setOutput('TSS_MIGRATION_TESTS', true);
@@ -201,6 +206,7 @@ jobs:
               core.setOutput('UPGRADE_LIGHT_TESTS', makeTargets.includes('upgrade-test-light'));
               core.setOutput('UPGRADE_IMPORT_MAINNET_TESTS', makeTargets.includes('upgrade-import-mainnet-test'));
               core.setOutput('ADMIN_TESTS', makeTargets.includes('admin-test'));
+              core.setOutput('ADMIN_UPGRADE_TESTS', makeTargets.includes('admin-upgrade-test'));
               core.setOutput('PERFORMANCE_TESTS', makeTargets.includes('performance-test'));
               core.setOutput('STATEFUL_DATA_TESTS', makeTargets.includes('import-mainnet-test'));
               core.setOutput('TSS_MIGRATION_TESTS', makeTargets.includes('tss-migration-test'));
@@ -238,6 +244,10 @@ jobs:
           - make-target: "start-e2e-admin-test"
             runs-on: ubuntu-20.04
             run: ${{ needs.matrix-conditionals.outputs.ADMIN_TESTS == 'true' }}
+          - make-target: "start-upgrade-test-admin"
+            runs-on: ubuntu-20.04
+            run: ${{ needs.matrix-conditionals.outputs.ADMIN_UPGRADE_TESTS == 'true' }}
+            timeout-minutes: 40
           - make-target: "start-e2e-performance-test"
             runs-on: buildjet-4vcpu-ubuntu-2204
             run: ${{ needs.matrix-conditionals.outputs.PERFORMANCE_TESTS == 'true' }}

cmd/zetae2e/balances.go

@@ -46,7 +46,7 @@ func runBalances(cmd *cobra.Command, args []string) error {
 	logger := runner.NewLogger(false, color.FgHiCyan, "")
 
 	// initialize context
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancelCause(context.Background())
 
 	// initialize deployer runner with config
 	r, err := zetae2econfig.RunnerFromConfig(
@@ -58,13 +58,13 @@ func runBalances(cmd *cobra.Command, args []string) error {
 		logger,
 	)
 	if err != nil {
-		cancel()
+		cancel(err)
 		return err
 	}
 
 	balances, err := r.GetAccountBalances(skipBTC)
 	if err != nil {
-		cancel()
+		cancel(err)
 		return err
 	}
 	r.PrintAccountBalances(balances)

cmd/zetae2e/bitcoin_address.go

@@ -46,7 +46,7 @@ func runBitcoinAddress(cmd *cobra.Command, args []string) error {
 	logger := runner.NewLogger(false, color.FgHiYellow, "")
 
 	// initialize context
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancelCause(context.Background())
 
 	// initialize deployer runner with config
 	r, err := zetae2econfig.RunnerFromConfig(
@@ -58,7 +58,7 @@ func runBitcoinAddress(cmd *cobra.Command, args []string) error {
 		logger,
 	)
 	if err != nil {
-		cancel()
+		cancel(err)
 		return err
 	}
 

cmd/zetae2e/config/config.go

@@ -12,7 +12,7 @@ import (
 func RunnerFromConfig(
 	ctx context.Context,
 	name string,
-	ctxCancel context.CancelFunc,
+	ctxCancel context.CancelCauseFunc,
 	conf config.Config,
 	account config.Account,
 	logger *runner.Logger,

cmd/zetae2e/local/local.go

@@ -3,8 +3,11 @@ package local
 import (
 	"context"
 	"errors"
+	"fmt"
 	"os"
+	"os/signal"
 	"path/filepath"
+	"syscall"
 	"time"
 
 	"github.com/fatih/color"
@@ -47,7 +50,8 @@ const (
 )
 
 var (
-	TestTimeout = 20 * time.Minute
+	TestTimeout        = 20 * time.Minute
+	ErrTopLevelTimeout = errors.New("top level test timeout")
 )
 
 var noError = testutil.NoError
@@ -61,7 +65,7 @@ func NewLocalCmd() *cobra.Command {
 		Run:   localE2ETest,
 	}
 	cmd.Flags().Bool(flagContractsDeployed, false, "set to to true if running tests again with existing state")
-	cmd.Flags().Int64(flagWaitForHeight, 0, "block height for tests to begin, ex. --wait-for 100")
+	cmd.Flags().Int64(flagWaitForHeight, 1, "block height for tests to begin, ex. --wait-for 100")
 	cmd.Flags().String(FlagConfigFile, "", "config file to use for the tests")
 	cmd.Flags().Bool(flagVerbose, false, "set to true to enable verbose logging")
 	cmd.Flags().Bool(flagTestAdmin, false, "set to true to run admin tests")
@@ -130,13 +134,6 @@ func localE2ETest(cmd *cobra.Command, _ []string) {
 		skipPrecompiles = true
 	}
 
-	// start timer
-	go func() {
-		time.Sleep(TestTimeout)
-		logger.Error("Test timed out after %s", TestTimeout.String())
-		os.Exit(1)
-	}()
-
 	// initialize tests config
 	conf, err := GetConfig(cmd)
 	noError(err)
@@ -147,12 +144,22 @@ func localE2ETest(cmd *cobra.Command, _ []string) {
 	}
 
 	// initialize context
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, timeoutCancel := context.WithTimeoutCause(context.Background(), TestTimeout, ErrTopLevelTimeout)
+	defer timeoutCancel()
+	ctx, cancel := context.WithCancelCause(ctx)
+
+	// route os signals to context cancellation.
+	// using NotifyContext will ensure that the second signal
+	// will not be handled and should kill the process.
+	go func() {
+		notifyCtx, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM)
+		<-notifyCtx.Done()
+		cancel(fmt.Errorf("notify context: %w", notifyCtx.Err()))
+		stop()
+	}()
 
 	// wait for a specific height on ZetaChain
-	if waitForHeight != 0 {
-		noError(utils.WaitForBlockHeight(ctx, waitForHeight, conf.RPCs.ZetaCoreRPC, logger))
-	}
+	noError(utils.WaitForBlockHeight(ctx, waitForHeight, conf.RPCs.ZetaCoreRPC, logger))
 
 	zetaTxServer, err := txserver.NewZetaTxServer(
 		conf.RPCs.ZetaCoreRPC,
@@ -185,9 +192,7 @@ func localE2ETest(cmd *cobra.Command, _ []string) {
 	noError(err)
 
 	// monitor block production to ensure we fail fast if there are consensus failures
-	// this is not run in an errgroup since only returning an error will not exit immediately
-	// this needs to be early to quickly detect consensus failure during genesis
-	go monitorBlockProductionExit(ctx, conf)
+	go monitorBlockProductionCancel(ctx, cancel, conf)
 
 	// set the authority client to the zeta tx server to be able to query message permissions
 	deployerRunner.ZetaTxServer.SetAuthorityClient(deployerRunner.AuthorityClient)
@@ -506,7 +511,7 @@ func localE2ETest(cmd *cobra.Command, _ []string) {
 	go monitorTxPriorityInBlocks(ctx, conf, txPriorityErrCh)
 
 	if err := eg.Wait(); err != nil {
-		deployerRunner.CtxCancel()
+		deployerRunner.CtxCancel(err)
 		monitorPriorityCancel()
 		logger.Print("❌ %v", err)
 		logger.Print("❌ e2e tests failed after %s", time.Since(testStartTime).String())

cmd/zetae2e/local/monitor_block_production.go

@@ -3,7 +3,6 @@ package local
 import (
 	"context"
 	"fmt"
-	"os"
 	"time"
 
 	rpchttp "github.com/cometbft/cometbft/rpc/client/http"
@@ -12,12 +11,11 @@ import (
 	"github.com/zeta-chain/node/e2e/config"
 )
 
-// monitorBlockProductionExit calls monitorBlockProduction and exits upon any error
-func monitorBlockProductionExit(ctx context.Context, conf config.Config) {
+// monitorBlockProductionCancel calls monitorBlockProduction and exits upon any error
+func monitorBlockProductionCancel(ctx context.Context, cancel context.CancelCauseFunc, conf config.Config) {
 	err := monitorBlockProduction(ctx, conf)
 	if err != nil {
-		fmt.Printf("❌ block monitor: %v\n", err)
-		os.Exit(2)
+		cancel(err)
 	}
 }
 

cmd/zetae2e/run.go

@@ -99,8 +99,8 @@ func runE2ETest(cmd *cobra.Command, args []string) error {
 	}
 
 	// initialize context
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
+	ctx, cancel := context.WithCancelCause(context.Background())
+	defer cancel(nil)
 
 	var runnerOpts []runner.E2ERunnerOption
 

cmd/zetae2e/setup_bitcoin.go

@@ -34,7 +34,7 @@ func runSetupBitcoin(_ *cobra.Command, args []string) error {
 	logger := runner.NewLogger(false, color.FgHiYellow, "")
 
 	// initialize context
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancelCause(context.Background())
 
 	// initialize deployer runner with config
 	r, err := zetae2econfig.RunnerFromConfig(
@@ -46,7 +46,7 @@ func runSetupBitcoin(_ *cobra.Command, args []string) error {
 		logger,
 	)
 	if err != nil {
-		cancel()
+		cancel(err)
 		return err
 	}
 

cmd/zetae2e/show_tss.go

@@ -34,7 +34,7 @@ func runShowTSS(_ *cobra.Command, args []string) error {
 	logger := runner.NewLogger(true, color.FgHiCyan, "")
 
 	// initialize context
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancelCause(context.Background())
 
 	// initialize deployer runner with config
 	testRunner, err := zetae2econfig.RunnerFromConfig(
@@ -46,7 +46,7 @@ func runShowTSS(_ *cobra.Command, args []string) error {
 		logger,
 	)
 	if err != nil {
-		cancel()
+		cancel(err)
 		return err
 	}
 

cmd/zetae2e/stress.go

@@ -112,7 +112,7 @@ func StressTest(cmd *cobra.Command, _ []string) {
 	}
 
 	// initialize context
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancelCause(context.Background())
 
 	verbose := must(cmd.Flags().GetBool(flagVerbose))
 	logger := runner.NewLogger(verbose, color.FgWhite, "setup")

contrib/localnet/orchestrator/start-zetae2e.sh

@@ -7,6 +7,9 @@
 # A second optional argument can be passed and can have the following value:
 # upgrade: run the local e2e once, then restart zetaclientd at upgrade height and run the local e2e again
 
+# Trap signals and forward to children
+trap 'kill -- -$$' SIGINT SIGTERM
+
 get_zetacored_version() {
   retries=10
   node_info=""

contrib/localnet/scripts/start-zetaclientd.sh

@@ -41,10 +41,10 @@ while [ ! -f ~/.ssh/authorized_keys ]; do
     sleep 1
 done
 
-# need to wait for zetacore0 to be up
-while ! curl -s -o /dev/null zetacore0:26657/status ; do
-    echo "Waiting for zetacore0 rpc"
-    sleep 20
+# need to wait for zetacore0 block 1 to be available
+while curl -s zetacore0:26657/block?height=1 | jq -e '.error != null' > /dev/null; do
+    echo "Waiting for zetacore0 block 1"
+    sleep 5
 done
 
 # read HOTKEY_BACKEND env var for hotkey keyring backend and set default to test

e2e/e2etests/e2etests.go

@@ -304,6 +304,7 @@ var AllE2ETests = []runner.E2ETest{
 			{Description: "amount in wei", DefaultValue: "10000000000000000"},
 		},
 		TestETHWithdrawAndCallRevertWithWithdraw,
+		runner.WithMinimumVersion("v26.0.0"),
 	),
 	runner.NewE2ETest(
 		TestDepositAndCallOutOfGasName,
@@ -662,6 +663,7 @@ var AllE2ETests = []runner.E2ETest{
 		"deposit Bitcoin into ZEVM that aborts due to insufficient deposit fee",
 		[]runner.ArgDefinition{},
 		TestBitcoinDepositAndAbortWithLowDepositFee,
+		runner.WithMinimumVersion("v27.0.0"),
 	),
 	runner.NewE2ETest(
 		TestBitcoinWithdrawSegWitName,