Skip to content

Commit

Permalink
ci(.github): enable self hosted runners for AMD64 E2E tasks (#10745)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jijiechen @kumahq
jijiechen authored and kumahq[bot] committed Jul 3, 2024
1 parent e28b733 commit 06e8e60
Show file tree
Hide file tree
Showing 2 changed files with 173 additions and 0 deletions.
164 changes: 164 additions & 0 deletions .github/workflows/_test.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
on:
workflow_call:
inputs:
FULL_MATRIX:
required: true
type: string
permissions:
contents: read
env:
CI_TOOLS_DIR: "/home/runner/work/kuma/kuma/.ci_tools"
# This is automatically managed by CI
K8S_MIN_VERSION: v1.23.17-k3s1
K8S_MAX_VERSION: v1.30.0-k3s1
jobs:
test_unit:
timeout-minutes: 20
if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci/skip-test') }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
fetch-depth: 0
- uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
with:
go-version-file: go.mod
- uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
with:
path: |
${{ env.CI_TOOLS_DIR }}
key: ${{ runner.os }}-${{ runner.arch }}-devtools-${{ hashFiles('mk/dependencies/deps.lock') }}
restore-keys: |
${{ runner.os }}-${{ runner.arch }}-devtools
- run: |
make dev/tools
- run: |
make test
gen_e2e_matrix:
timeout-minutes: 2
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.generate-matrix.outputs.matrix }}
runners: ${{ steps.assign-e2e-runners.outputs.runners }}
steps:
- id: generate-matrix
name: Generate matrix
env:
BASE_MATRIX: |-
{
"test_e2e": {
"target": [""],
"k8sVersion": ["kindIpv6", "${{ env.K8S_MIN_VERSION }}", "${{ env.K8S_MAX_VERSION }}"],
"arch": ["amd64"],
"parallelism": [4],
"cniNetworkPlugin": ["flannel"],
"sidecarContainers": [""]
},
"test_e2e_env": {
"target": ["kubernetes", "universal", "multizone"],
"k8sVersion": ["kind", "kindIpv6", "${{ env.K8S_MIN_VERSION }}", "${{ env.K8S_MAX_VERSION }}"],
"arch": ["amd64"],
"parallelism": [1],
"cniNetworkPlugin": ["flannel"],
"sidecarContainers": [""],
"exclude":[
{"target": "kubernetes", "k8sVersion":"kind"},
{"target": "multizone", "k8sVersion":"kind"},
{"target":"universal", "k8sVersion":"${{ env.K8S_MIN_VERSION }}"},
{"target":"universal", "k8sVersion":"${{ env.K8S_MAX_VERSION }}"}
],
"include":[
{"sidecarContainers": "sidecarContainers", "k8sVersion": "${{ env.K8S_MAX_VERSION }}", "target": "kubernetes", "arch": "amd64"},
{"k8sVersion": "${{ env.K8S_MAX_VERSION }}", "target": "multizone", "arch": "arm64"},
{"k8sVersion": "${{ env.K8S_MAX_VERSION }}", "target": "kubernetes", "arch": "arm64"},
{"k8sVersion": "${{ env.K8S_MAX_VERSION }}", "target": "universal", "arch": "arm64"},
{"k8sVersion": "${{ env.K8S_MAX_VERSION }}", "target": "gatewayapi", "arch": "amd64"},
{"cniNetworkPlugin": "calico", "k8sVersion": "${{ env.K8S_MAX_VERSION }}", "target": "multizone", "arch": "amd64"}
]
}
}
# You can modify the include to run one of test suites on PRs (though you'd need to then remove it)
OVERRIDE_JQ_CMD: |-
.test_e2e = false
| .test_e2e_env.include = []
| .test_e2e_env.exclude += [{"arch": "arm64"}, {"k8sVersion": "kindIpv6"}, {"k8sVersion": "${{ inputs.K8S_MIN_VERSION}}"}]
run: |-
BASE_MATRIX_ALL='${{ env.BASE_MATRIX }}'
if [[ "${{ inputs.FULL_MATRIX }}" != "true" ]]; then
BASE_MATRIX_ALL=$(echo $BASE_MATRIX_ALL | jq -r '${{ env.OVERRIDE_JQ_CMD }}')
fi
# Skip e2e tests if the PR has the label "ci/skip-e2e-test" or "ci/skip-test"
if [[ "${{ contains(github.event.pull_request.labels.*.name, 'ci/skip-e2e-test') || contains(github.event.pull_request.labels.*.name, 'ci/skip-test') }}" == "true" ]]; then
BASE_MATRIX_ALL='{}'
fi
echo "final matrix: $BASE_MATRIX_ALL"
echo "matrix<<EOF" >> $GITHUB_OUTPUT
echo "$BASE_MATRIX_ALL" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
- id: assign-e2e-runners
name: Assign runners for E2E tests
env:
IS_TRUSTED_BUILD: ${{ github.event_name == 'push' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }}
ALL_RUNNERS: |-
{
"trusted_builds": {
"amd64": [
"ubuntu-latest-kong"
],
"arm64": [
"ubuntu-latest-arm64-kong"
]
},
"untrusted_builds": {
"amd64": [
"ubuntu-latest"
],
"arm64": []
}
}
run: |-
ALL_RUNNERS='${{ env.ALL_RUNNERS }}'
if [[ "${{ env.IS_TRUSTED_BUILD }}" == "true" ]]; then
ALL_RUNNERS=$(echo $ALL_RUNNERS | jq -r '.trusted_builds')
else
ALL_RUNNERS=$(echo $ALL_RUNNERS | jq -r '.untrusted_builds')
fi
for ARCH in amd64 arm64; do
COUNT=$(echo $ALL_RUNNERS | jq -r ".$ARCH | length")
if [[ $COUNT -eq 0 ]]; then
ALL_RUNNERS=$(echo $ALL_RUNNERS | jq -r ".$ARCH=\"\"")
else
ALL_RUNNERS=$(echo $ALL_RUNNERS | jq -r ".$ARCH=.$ARCH[0]")
fi
done
echo "final runners: $ALL_RUNNERS"
echo "runners<<EOF" >> $GITHUB_OUTPUT
echo "$ALL_RUNNERS" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
test_e2e:
needs: ["gen_e2e_matrix"]
if: fromJSON(needs.gen_e2e_matrix.outputs.matrix).test_e2e
strategy:
max-parallel: 5
matrix: ${{ fromJSON(needs.gen_e2e_matrix.outputs.matrix).test_e2e }}
fail-fast: false
uses: ./.github/workflows/_e2e.yaml
with:
matrix: ${{ toJSON(matrix) }}
runnersByArch: ${{ needs.gen_e2e_matrix.outputs.runners }}
secrets: inherit
test_e2e_env:
needs: ["gen_e2e_matrix"]
if: fromJSON(needs.gen_e2e_matrix.outputs.matrix).test_e2e_env
strategy:
max-parallel: 5
matrix: ${{ fromJSON(needs.gen_e2e_matrix.outputs.matrix).test_e2e_env }}
fail-fast: false
uses: ./.github/workflows/_e2e.yaml
with:
matrix: ${{ toJSON(matrix) }}
runnersByArch: ${{ needs.gen_e2e_matrix.outputs.runners }}
secrets: inherit
9 changes: 9 additions & 0 deletions .github/workflows/e2e.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,14 @@ on:
runnersByArch:
type: string
required: false
<<<<<<< HEAD:.github/workflows/e2e.yaml
default: '{"amd64": "ubuntu-latest", "arm64": "circleci"}'
secrets:
circleCIToken:
required: true
=======
default: ''
>>>>>>> 9fbead7e2 (ci(.github): enable self hosted runners for AMD64 E2E tasks (#10745)):.github/workflows/_e2e.yaml
permissions:
contents: read
env:
Expand All @@ -23,8 +27,13 @@ env:
CI_TOOLS_DIR: /home/runner/work/kuma/kuma/.ci_tools
jobs:
e2e:
<<<<<<< HEAD:.github/workflows/e2e.yaml
# use the runner from the map, if the runner is circleci or '' then use ubuntu-latest
runs-on: ${{ contains(fromJSON('["circleci", ""]'), fromJSON(inputs.runnersByArch)[fromJSON(inputs.matrix).arch]) && 'ubuntu-latest' || fromJSON(inputs.runnersByArch)[fromJSON(inputs.matrix).arch]}}
=======
timeout-minutes: 60
runs-on: ${{ fromJSON(inputs.runnersByArch)[fromJSON(inputs.matrix).arch] }}
>>>>>>> 9fbead7e2 (ci(.github): enable self hosted runners for AMD64 E2E tasks (#10745)):.github/workflows/_e2e.yaml
strategy:
fail-fast: false
matrix:
Expand Down

0 comments on commit 06e8e60

Please sign in to comment.