ci: reuse workflows

Signed-off-by: Artur Troian <[email protected]>
akash-network · Oct 9, 2024 · b372f18 · b372f18
1 parent d580137
commit b372f18
Show file tree

Hide file tree

Showing 4 changed files with 193 additions and 25 deletions.
diff --git a/.github/actions/setup-ubuntu/action.yaml b/.github/actions/setup-ubuntu/action.yaml
@@ -0,0 +1,29 @@
+---
+name: setup-ubuntu
+runs:
+  using: 'composite'
+  steps:
+    - name: Install dependencies
+      # Shell must explicitly specify the shell for each step. https://github.com/orgs/community/discussions/18597
+      shell: bash
+      run: sudo apt install -y make direnv unzip lz4 wget curl npm jq pv coreutils
+    - name: Install git-cliff
+      shell: bash
+      uses: baptiste0928/cargo-install@v3
+      with:
+        crate: git-cliff
+    - name: Detect required Go version
+      shell: bash
+      run: |
+        toolchain=$(./script/tools.sh gotoolchain | sed 's/go*//')
+        echo "GOVERSION=${toolchain}" >> $GITHUB_ENV
+    - uses: actions/setup-go@v5
+      shell: bash
+      with:
+        go-version: "${{ env.GOVERSION }}"
+        check-latest: true
+    - name: set environment
+      shell: bash
+      uses: HatsuneMiku3939/direnv-action@v1
+      with:
+        masks: ''
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -22,10 +22,6 @@ jobs:
         run: |
           brew install bash direnv pv lz4 git-cliff
           sudo chsh -s /usr/local/bin/bash
-      - name: Install git-cliff
-        uses: baptiste0928/cargo-install@v3
-        with:
-          crate: git-cliff
       - name: Hook direnv to bash
         run: echo 'eval "$(direnv hook bash)"' >> $HOME/.bashrc
       - uses: actions/checkout@v4
@@ -43,6 +39,7 @@ jobs:
           direnv allow
           direnv export gha >> "$GITHUB_ENV"
       - run: make bins
+
   build-bins:
     runs-on: ubuntu-latest
     steps:
@@ -64,8 +61,6 @@ jobs:
           check-latest: true
       - name: set environment
         uses: HatsuneMiku3939/direnv-action@v1
-        with:
-          masks: ''
       - run: make bins
       - run: make docker-image
 

diff --git a/script/tools.sh b/script/tools.sh
@@ -5,6 +5,30 @@ SEMVER=$SCRIPT_DIR/semver.sh
 
 gomod="$SCRIPT_DIR/../go.mod"
 
+macos_deps=(
+	"bash"
+	"direnv"
+	"pv"
+	"lz4"
+	"git-cliff"
+)
+
+debian_deps=(
+	"make"
+	"build-essentials"
+	"direnv"
+	"unzip"
+	"wget"
+	"curl"
+	"npm"
+	"jq"
+	"coreutils"
+)
+
+is_command() {
+	command -v "$1" >/dev/null
+}
+
 function get_gotoolchain() {
 	local gotoolchain
 	local goversion
@@ -117,6 +141,80 @@ function replace_import_path() {
 	replace_paths "./go/go.mod" "$curr_module_name" "$new_module_name"
 }
 
+function install_gha_deps() {
+	if [[ "$OSTYPE" == "darwin"* ]]; then
+		echo "Detected Darwin based system"
+
+		if ! is_command brew; then
+			echo "homebrew is not installed. visit https://brew.sh"
+			exit 1
+		fi
+
+		local tools
+
+		if ! is_command make || [[ $(make --version | head -1 | cut -d" " -f3 | cut -d"." -f1) -lt 4 ]]; then
+			tools="$tools make"
+		fi
+
+		# shellcheck disable=SC2068
+		for dep in ${macos_deps[@]}; do
+			echo -n "detecting $dep ..."
+			status="(installed)"
+			if ! brew list "$dep" >/dev/null 2>&1; then
+				tools="$tools $dep"
+				status="(not installed)"
+			fi
+
+			echo " $status"
+		done
+
+		if [[ "$tools" != "" ]]; then
+			# don't put quotes around $tools!
+			# shellcheck disable=SC2086
+			brew install $tools
+		else
+			echo "All requirements already met. Nothing to install"
+		fi
+	elif [[ "$OSTYPE" == "linux-gnu"* ]]; then
+		if is_command dpkg; then
+			echo "Detected Debian based system"
+			local tools
+
+			# shellcheck disable=SC2068
+			for dep in ${debian_deps[@]}; do
+				echo -n "detecting $dep ..."
+				status="(installed)"
+				if ! dpkg -l "$dep"; then
+					tools="$tools $dep"
+					status="(not installed)"
+				fi
+				echo " $status"
+			done
+
+			cmd="apt-get"
+
+			if is_command sudo; then
+				cmd="sudo $cmd"
+			fi
+
+			if [[ "$tools" != "" ]]; then
+				$cmd update
+				# don't put quotes around $tools!
+				# shellcheck disable=SC2086
+				(
+					set -x
+					$cmd install -y $tools
+				)
+			else
+				echo "All requirements already met. Nothing to install"
+			fi
+		fi
+	else
+		echo "Unsupported OS $OSTYPE"
+		exit 1
+	fi
+}
+
 case "$1" in
 	gotoolchain)
 		get_gotoolchain

diff --git a/tests/upgrade/upgrade_test.go b/tests/upgrade/upgrade_test.go
@@ -35,7 +35,7 @@ import (
 )
 
 const (
-	blockTimeWindow = 7 * time.Second
+	blockTimeWindow = 60 * time.Second
 )
 
 type nodeEvent int
@@ -65,7 +65,8 @@ const (
 const (
 	nodeTestStagePreUpgrade nodeTestStage = iota
 	nodeTestStageUpgrade
-	nodeTestStagePostUpgrade
+	nodeTestStagePostUpgrade1
+	nodeTestStagePostUpgrade2
 )
 
 const (
@@ -86,9 +87,10 @@ type publisher interface {
 
 var (
 	nodeTestStageMapStr = map[nodeTestStage]string{
-		nodeTestStagePreUpgrade:  "preupgrade",
-		nodeTestStageUpgrade:     "upgrade",
-		nodeTestStagePostUpgrade: "postupgrade",
+		nodeTestStagePreUpgrade:   "preupgrade",
+		nodeTestStageUpgrade:      "upgrade",
+		nodeTestStagePostUpgrade1: "postupgrade1",
+		nodeTestStagePostUpgrade2: "postupgrade2",
 	}
 
 	testModuleStatusMapStr = map[testModuleStatus]string{
@@ -439,6 +441,8 @@ func TestUpgrade(t *testing.T) {
 		}
 	}
 
+	listenAddr := "127.0.0.1"
+
 	for name, params := range initParams {
 		var unconditionalPeerIDs string
 		var persistentPeers string
@@ -449,7 +453,7 @@ func TestUpgrade(t *testing.T) {
 			}
 
 			unconditionalPeerIDs += params1.nodeID + ","
-			persistentPeers += fmt.Sprintf("%s@127.0.0.1:%d,", params1.nodeID, params1.p2pPort)
+			persistentPeers += fmt.Sprintf("%s@%s:%d,", params1.nodeID, listenAddr, params1.p2pPort)
 		}
 
 		validatorsParams[name] = validatorParams{
@@ -478,12 +482,12 @@ func TestUpgrade(t *testing.T) {
 				fmt.Sprintf("AKASH_P2P_SEEDS=%s", strings.TrimSuffix(persistentPeers, ",")),
 				fmt.Sprintf("AKASH_P2P_PERSISTENT_PEERS=%s", strings.TrimSuffix(persistentPeers, ",")),
 				fmt.Sprintf("AKASH_P2P_UNCONDITIONAL_PEER_IDS=%s", strings.TrimSuffix(unconditionalPeerIDs, ",")),
-				fmt.Sprintf("AKASH_P2P_LADDR=tcp://127.0.0.1:%d", params.p2pPort),
-				fmt.Sprintf("AKASH_RPC_LADDR=tcp://127.0.0.1:%d", params.rpc.port),
-				fmt.Sprintf("AKASH_RPC_GRPC_LADDR=tcp://127.0.0.1:%d", params.rpc.grpc),
-				fmt.Sprintf("AKASH_RPC_PPROF_LADDR=localhost:%d", params.pprofPort),
-				fmt.Sprintf("AKASH_GRPC_ADDRESS=tcp://127.0.0.1:%d", params.grpcPort),
-				fmt.Sprintf("AKASH_GRPC_WEB_ADDRESS=tcp://127.0.0.1:%d", params.grpcWebPort),
+				fmt.Sprintf("AKASH_P2P_LADDR=tcp://%s:%d", listenAddr, params.p2pPort),
+				fmt.Sprintf("AKASH_RPC_LADDR=tcp://%s:%d", listenAddr, params.rpc.port),
+				fmt.Sprintf("AKASH_RPC_GRPC_LADDR=tcp://%s:%d", listenAddr, params.rpc.grpc),
+				fmt.Sprintf("AKASH_RPC_PPROF_LADDR=%s:%d", listenAddr, params.pprofPort),
+				fmt.Sprintf("AKASH_GRPC_ADDRESS=tcp://%s:%d", listenAddr, params.grpcPort),
+				fmt.Sprintf("AKASH_GRPC_WEB_ADDRESS=tcp://%s:%d", listenAddr, params.grpcWebPort),
 				"AKASH_P2P_PEX=true",
 				"AKASH_P2P_ADDR_BOOK_STRICT=false",
 				"AKASH_P2P_ALLOW_DUPLICATE_IP=true",
@@ -531,7 +535,9 @@ func TestUpgrade(t *testing.T) {
 		}(name)
 	}
 
+	t.Logf("waiting for validator(s) to complete tasks")
 	err = group.Wait()
+	t.Logf("all validators finished")
 	assert.NoError(t, err)
 
 	fail := false
@@ -548,6 +554,8 @@ func TestUpgrade(t *testing.T) {
 		}
 	}
 
+	bus.Close()
+
 	if fail {
 		t.Fail()
 	}
@@ -611,8 +619,10 @@ loop:
 						})
 
 						if !result {
-							l.t.Error("post upgrade test handler failed")
+							l.t.Error("post upgrade test FAIL")
 							return fmt.Errorf("post-upgrade check failed")
+						} else {
+							l.t.Log("post upgrade test PASS")
 						}
 
 						return nil
@@ -832,6 +842,9 @@ func (l *validator) run() error {
 	}
 
 	l.group.Go(func() error {
+		defer l.t.Logf("[%s] log scanner finished", l.params.name)
+		l.t.Logf("[%s] log scanner started", l.params.name)
+
 		defer func() {
 			if r := recover(); r != nil {
 				l.t.Fatal(r)
@@ -842,6 +855,9 @@ func (l *validator) run() error {
 	})
 
 	l.group.Go(func() error {
+		defer l.t.Logf("[%s] test case watcher finished", l.params.name)
+		l.t.Logf("[%s] test case watcher started", l.params.name)
+
 		defer func() {
 			if r := recover(); r != nil {
 				l.t.Fatal(r)
@@ -857,6 +873,9 @@ func (l *validator) run() error {
 	})
 
 	l.group.Go(func() error {
+		defer l.t.Logf("[%s] stdout reader finished", l.params.name)
+		l.t.Logf("[%s] stdout reader started", l.params.name)
+
 		defer func() {
 			if r := recover(); r != nil {
 				l.t.Fatal(r)
@@ -871,6 +890,9 @@ func (l *validator) run() error {
 	})
 
 	l.group.Go(func() error {
+		defer l.t.Logf("[%s] blocks watchdog finished", l.params.name)
+		l.t.Logf("[%s] blocks watchdog started", l.params.name)
+
 		defer func() {
 			if r := recover(); r != nil {
 				l.t.Fatal(r)
@@ -887,6 +909,9 @@ func (l *validator) run() error {
 
 	// state machine
 	l.group.Go(func() error {
+		defer l.t.Logf("[%s] state machine finished", l.params.name)
+		l.t.Logf("[%s] state machine started", l.params.name)
+
 		defer func() {
 			if r := recover(); r != nil {
 				l.t.Fatal(r)
@@ -906,6 +931,7 @@ func (l *validator) run() error {
 	select {
 	case <-l.upgradeSuccessful:
 		err = nil
+		l.t.Logf("[%s] all workers finished", l.params.name)
 	default:
 		l.t.Logf("[%s] cosmovisor finished with error. check %s", l.params.name, lStderr.Name())
 	}
@@ -955,7 +981,7 @@ loop:
 				case nodeEventStart:
 					l.t.Logf("[%s][%s]: node started", l.params.name, nodeTestStageMapStr[stage])
 					if stage == nodeTestStageUpgrade {
-						stage = nodeTestStagePostUpgrade
+						stage = nodeTestStagePostUpgrade1
 						blocksCount = 0
 						replayDone = false
 					}
@@ -982,7 +1008,8 @@ loop:
 						_ = l.params.pub.Publish(nodePreUpgradeReady{
 							name: l.params.name,
 						})
-					} else if stage == nodeTestStagePostUpgrade && blocksCount == 10 {
+					} else if stage == nodeTestStagePostUpgrade1 && blocksCount >= 10 {
+						stage = nodeTestStagePostUpgrade2
 						l.t.Logf("[%s][%s]: counted 10 blocks. signaling has performed upgrade", l.params.name, nodeTestStageMapStr[stage])
 						l.upgradeSuccessful <- struct{}{}
 
@@ -1136,14 +1163,23 @@ func (l *validator) blocksWatchdog(ctx context.Context, sub pubsub.Subscriber) e
 	defer func() {
 		if err != nil {
 			l.t.Logf("blocksWatchdog finished with error: %s", err.Error())
+		} else {
+			l.t.Logf("blocksWatchdog finished")
 		}
 	}()
 
+	// first few blocks may take a while to produce.
+	// give a dog generous timeout on them
+
+	blockWindow := 20 * time.Minute
+
+	blocksTm := time.NewTicker(blockWindow)
+	blocksTm.Stop()
+
+	blocks := 0
+
 loop:
 	for {
-		blocksTm := time.NewTicker(blockTimeWindow)
-		blocksTm.Stop()
-
 		select {
 		case <-ctx.Done():
 			break loop
@@ -1159,11 +1195,21 @@ loop:
 				case watchdogCtrlStart:
 					fallthrough
 				case watchdogCtrlBlock:
-					blocksTm.Reset(blockTimeWindow)
+					blocks++
+
+					if blocks > 3 {
+						blockWindow = blockTimeWindow
+					}
+
+					blocksTm.Reset(blockWindow)
 				case watchdogCtrlPause:
 					blocksTm.Stop()
+					blocks = 0
+					blockWindow = 20 * time.Minute
 				case watchdogCtrlStop:
+					blocks = 0
 					blocksTm.Stop()
+					blockWindow = 20 * time.Minute
 					break loop
 				}
 			}