Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/main' into pause
Browse files Browse the repository at this point in the history
  • Loading branch information
ndbaker1 committed Oct 21, 2024
2 parents 6cc6cb9 + a3f8fd4 commit 6b4e4d9
Show file tree
Hide file tree
Showing 16 changed files with 237 additions and 66 deletions.
25 changes: 25 additions & 0 deletions .github/workflows/ci-canary.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
name: '[CI] Canary'
on:
schedule:
# daily at midnight
- cron: "0 0 * * *"

jobs:
setup:
runs-on: ubuntu-latest
outputs:
timestamp: ${{ steps.variables.outputs.timestamp }}
steps:
- id: variables
run: |
echo "timestamp=$(date +%s)" >> $GITHUB_OUTPUT
ci:
needs:
- setup
uses: ./.github/workflows/ci.yaml
secrets: inherit
with:
git_sha: "main"
run_name: "ci(canary): ${{ needs.setup.outputs.timestamp }}"
resource_id: "ci-canary-${{ needs.setup.outputs.timestamp }}"
75 changes: 20 additions & 55 deletions .github/workflows/ci-manual.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -45,19 +45,11 @@ jobs:
outputs:
git_sha_short: ${{ steps.variables.outputs.git_sha_short }}
workflow_run_url: ${{ steps.variables.outputs.workflow_run_url }}
kubernetes_versions: ${{ steps.variables.outputs.kubernetes_versions }}
build_id: ${{ steps.variables.outputs.build_id }}
ci_step_name_prefix: ${{ steps.variables.outputs.ci_step_name_prefix }}
os_distros: ${{ steps.variables.outputs.os_distros }}
steps:
- id: variables
run: |
echo "git_sha_short=$(echo ${{ inputs.git_sha }} | rev | cut -c-7 | rev)" >> $GITHUB_OUTPUT
echo "workflow_run_url=https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_OUTPUT
echo "build_id=ci-${{ inputs.pr_number }}-${{ needs.setup.outputs.git_sha_short }}-${{ inputs.uuid }}" >> $GITHUB_OUTPUT
echo 'ci_step_name_prefix=CI:' >> $GITHUB_OUTPUT
echo "kubernetes_versions=$(jq -Rn --arg input '${{ inputs.k8s_versions }}' '($input | split(","))' | jq -c .)" >> $GITHUB_OUTPUT
echo "os_distros=$(jq -Rn --arg input '${{ inputs.os_distros }}' '($input | split(","))' | jq -c .)" >> $GITHUB_OUTPUT
notify-start:
runs-on: ubuntu-latest
Expand All @@ -73,57 +65,27 @@ jobs:
issue_number: ${{ inputs.pr_number }},
body: `@${{ inputs.requester }} roger [that](${{ inputs.comment_url }})! I've dispatched a [workflow](${{ needs.setup.outputs.workflow_run_url }}). 👍`
});
kubernetes-versions:
runs-on: ubuntu-latest
name: ${{ matrix.k8s_version }} / ${{ matrix.os_distro }}
ci:
uses: ./.github/workflows/ci.yaml
needs:
- setup
- notify-start
permissions:
id-token: write
contents: read
strategy:
# don't bail out of all sub-tasks if one fails
fail-fast: false
matrix:
k8s_version: ${{ fromJson(needs.setup.outputs.kubernetes_versions) }}
os_distro: ${{ fromJson(needs.setup.outputs.os_distros) }}
steps:
- uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # 4.2.1
with:
ref: 'main'
- uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # 4.0.2
with:
aws-region: ${{ secrets.AWS_REGION }}
role-to-assume: ${{ secrets.AWS_ROLE_ARN_CI }}
# 2.5 hours (job usually completes within 2 hours)
role-duration-seconds: 9000
- name: "${{ needs.setup.outputs.ci_step_name_prefix }} Build"
id: build
uses: ./.github/actions/ci/build
with:
git_sha: ${{ inputs.git_sha }}
k8s_version: ${{ matrix.k8s_version }}
os_distro: ${{ matrix.os_distro }}
build_id: ${{ needs.setup.outputs.build_id }}
additional_arguments: ${{ inputs.build_arguments }}
- if: ${{ inputs.goal == 'test' }}
name: "${{ needs.setup.outputs.ci_step_name_prefix }} Test"
id: test
uses: ./.github/actions/ci/kubetest2
with:
ami_id: ${{ steps.build.outputs.ami_id }}
k8s_version: ${{ matrix.k8s_version }}
os_distro: ${{ matrix.os_distro }}
build_id: ${{ needs.setup.outputs.build_id }}
aws_region: ${{ secrets.AWS_REGION }}
log_bucket: ${{ secrets.CI_LOG_BUCKET }}
secrets: inherit
with:
k8s_versions: ${{ inputs.k8s_versions }}
os_distros: ${{ inputs.os_distros }}
git_sha: ${{ inputs.git_sha }}
build_arguments: ${{ inputs.build-arguments }}
resource_id: "ci-${{ inputs.pr_number }}-${{ needs.setup.outputs.git_sha_short }}-${{ inputs.uuid }}"
run_name: "ci(#${{ inputs.pr_number }}@${{ needs.setup.outputs.git_sha_short }}): ${{ inputs.uuid }}"
goal: "${{ inputs.goal }}"

notify-outcome:
if: ${{ always() }}
runs-on: ubuntu-latest
needs:
- setup
- kubernetes-versions
- ci
steps:
- uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # 7.0.1
with:
Expand All @@ -141,16 +103,19 @@ jobs:
};
const uniqueStepNames = new Set();
const stepConclusionsByJob = new Map();
const ciStepNamePrefix = "${{ needs.setup.outputs.ci_step_name_prefix }}";
const ciStepNamePrefix = "${{ needs.ci.outputs.ci_step_name_prefix }}";
for (const job of data.jobs) {
if (/^\d+\.\d+/.test(job.name)) {
// look for jobs named "ci / k8sVersion / osDistro"
if (/^ci \/ \d+\.\d+/.test(job.name)) {
// remove the "ci /" prefix
const jobName = job.name.substring("ci /".length).trim();
for (const step of job.steps) {
if (step.name.startsWith(ciStepNamePrefix)) {
const stepName = step.name.substring(ciStepNamePrefix.length).trim();
let stepConclusions = stepConclusionsByJob.get(job.name);
let stepConclusions = stepConclusionsByJob.get(jobName);
if (!stepConclusions) {
stepConclusions = new Map();
stepConclusionsByJob.set(job.name, stepConclusions);
stepConclusionsByJob.set(jobName, stepConclusions);
}
stepConclusions.set(stepName, step.conclusion);
uniqueStepNames.add(stepName);
Expand Down
93 changes: 93 additions & 0 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
name: '[CI]'
run-name: "${{ inputs.run_name }}"
on:
workflow_call:
inputs:
git_sha:
required: true
type: string
run_name:
required: true
type: string
resource_id:
required: true
type: string
goal:
required: false
type: string
default: "test"
os_distros:
description: 'Operating System Distributions (comma-separated, e.g., al2,al2023)'
default: "al2,al2023"
required: false
type: string
k8s_versions:
description: 'Kubernetes Versions (comma-separated, e.g., 1.29,1.30)'
default: "1.24,1.25,1.26,1.27,1.28,1.29,1.30,1.31"
required: false
type: string
build_arguments:
required: false
type: string
outputs:
ci_step_name_prefix:
description: "Prefix of job steps containing CI activities"
value: ${{ jobs.setup.outputs.ci_step_name_prefix }}

jobs:
setup:
runs-on: ubuntu-latest
outputs:
kubernetes_versions: ${{ steps.variables.outputs.kubernetes_versions }}
ci_step_name_prefix: ${{ steps.variables.outputs.ci_step_name_prefix }}
os_distros: ${{ steps.variables.outputs.os_distros }}
steps:
- id: variables
run: |
echo 'ci_step_name_prefix=CI:' >> $GITHUB_OUTPUT
echo "kubernetes_versions=$(jq -Rn --arg input '${{ inputs.k8s_versions }}' '($input | split(","))' | jq -c .)" >> $GITHUB_OUTPUT
echo "os_distros=$(jq -Rn --arg input '${{ inputs.os_distros }}' '($input | split(","))' | jq -c .)" >> $GITHUB_OUTPUT
kubernetes-versions:
runs-on: ubuntu-latest
name: ${{ matrix.k8s_version }} / ${{ matrix.os_distro }}
needs:
- setup
permissions:
id-token: write
contents: read
strategy:
# don't bail out of all sub-tasks if one fails
fail-fast: false
matrix:
k8s_version: ${{ fromJson(needs.setup.outputs.kubernetes_versions) }}
os_distro: ${{ fromJson(needs.setup.outputs.os_distros) }}
steps:
- uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # 4.2.1
with:
ref: 'main'
- uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # 4.0.2
with:
aws-region: ${{ secrets.AWS_REGION }}
role-to-assume: ${{ secrets.AWS_ROLE_ARN_CI }}
# 2.5 hours (job usually completes within 2 hours)
role-duration-seconds: 9000
- name: "${{ needs.setup.outputs.ci_step_name_prefix }} Build"
id: build
uses: ./.github/actions/ci/build
with:
git_sha: ${{ inputs.git_sha }}
k8s_version: ${{ matrix.k8s_version }}
os_distro: ${{ matrix.os_distro }}
build_id: ${{ inputs.resource_id }}
additional_arguments: ${{ inputs.build_arguments }}
- if: ${{ inputs.goal == 'test' }}
name: "${{ needs.setup.outputs.ci_step_name_prefix }} Test"
id: test
uses: ./.github/actions/ci/kubetest2
with:
ami_id: ${{ steps.build.outputs.ami_id }}
k8s_version: ${{ matrix.k8s_version }}
os_distro: ${{ matrix.os_distro }}
build_id: ${{ inputs.resource_id }}
aws_region: ${{ secrets.AWS_REGION }}
log_bucket: ${{ secrets.CI_LOG_BUCKET }}
19 changes: 19 additions & 0 deletions log-collector-script/linux/eks-log-collector.sh
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,20 @@ get_iptables_info() {
ip6tables-save > "${COLLECT_DIR}"/networking/ip6tables-save.txt
fi

if ! command -v ipvsadm && command -v ipset > /dev/null 2>&1; then
echo "IPVS Linux kernel module not installed" | tee ipvsadm.txt ipset.txt
else
# check that ip_vs module is loaded in get_modinfo()
try "collect ipvs information"
ipvsadm --save | tee "${COLLECT_DIR}"/networking/ipvsadm.txt && sed -i '1s/^/add:service/server \tprotocol \tvirtual-server \tscheduler algorithm \treal-server \n/' "${COLLECT_DIR}"/networking/ipvsadm.txt
ipvsadm --list --numeric --rate | tee -a "${COLLECT_DIR}"/networking/ipvsadm.txt
ok -e "\n" | tee -a "${COLLECT_DIR}"/networking/ipvsadm.txt
ipvsadm --list --numeric --stats --exact | tee -a "${COLLECT_DIR}"/networking/ipvsadm.txt
ipset --list | tee "${COLLECT_DIR}"/networking/ipset.txt
ok -e "\n" | tee -a "${COLLECT_DIR}"/networking/ipset.txt
ipset --save | tee -a "${COLLECT_DIR}"/networking/ipset.txt
fi

ok
}

Expand Down Expand Up @@ -412,6 +426,7 @@ get_kernel_info() {
get_modinfo() {
try "collect modinfo"
modinfo lustre > "${COLLECT_DIR}/modinfo/lustre"
lsmod | grep -e ip_vs -e nf_conntrack > "${COLLECT_DIR}/modinfo/ip_vs"
}

get_docker_logs() {
Expand Down Expand Up @@ -692,6 +707,10 @@ get_system_services() {
;;
esac

if [ "${INIT_TYPE}" = "systemd" ]; then
systemd-analyze plot > "${COLLECT_DIR}/system/systemd-analyze.svg" 2>&1
fi

timeout 75 top -b -n 1 > "${COLLECT_DIR}"/system/top.txt 2>&1
timeout 75 ps fauxwww --headers > "${COLLECT_DIR}"/system/ps.txt 2>&1
timeout 75 ps -eTF --headers > "${COLLECT_DIR}"/system/ps-threads.txt 2>&1
Expand Down
4 changes: 2 additions & 2 deletions nodeadm/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ module github.com/awslabs/amazon-eks-ami/nodeadm
go 1.22.5

require (
github.com/aws/aws-sdk-go-v2/config v1.27.43
github.com/aws/aws-sdk-go-v2/config v1.28.0
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17
github.com/aws/aws-sdk-go-v2/service/ec2 v1.182.0
github.com/aws/aws-sdk-go-v2/service/ec2 v1.183.0
github.com/aws/smithy-go v1.22.0
github.com/coreos/go-systemd/v22 v22.5.0
github.com/integrii/flaggy v1.5.2
Expand Down
8 changes: 4 additions & 4 deletions nodeadm/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s=
dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
github.com/aws/aws-sdk-go-v2 v1.32.2 h1:AkNLZEyYMLnx/Q/mSKkcMqwNFXMAvFto9bNsHqcTduI=
github.com/aws/aws-sdk-go-v2 v1.32.2/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo=
github.com/aws/aws-sdk-go-v2/config v1.27.43 h1:p33fDDihFC390dhhuv8nOmX419wjOSDQRb+USt20RrU=
github.com/aws/aws-sdk-go-v2/config v1.27.43/go.mod h1:pYhbtvg1siOOg8h5an77rXle9tVG8T+BWLWAo7cOukc=
github.com/aws/aws-sdk-go-v2/config v1.28.0 h1:FosVYWcqEtWNxHn8gB/Vs6jOlNwSoyOCA/g/sxyySOQ=
github.com/aws/aws-sdk-go-v2/config v1.28.0/go.mod h1:pYhbtvg1siOOg8h5an77rXle9tVG8T+BWLWAo7cOukc=
github.com/aws/aws-sdk-go-v2/credentials v1.17.41 h1:7gXo+Axmp+R4Z+AK8YFQO0ZV3L0gizGINCOWxSLY9W8=
github.com/aws/aws-sdk-go-v2/credentials v1.17.41/go.mod h1:u4Eb8d3394YLubphT4jLEwN1rLNq2wFOlT6OuxFwPzU=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 h1:TMH3f/SCAWdNtXXVPPu5D6wrr4G5hI1rAxbcocKfC7Q=
Expand All @@ -14,8 +14,8 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 h1:6jZVETqmYCadGFvrYE
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21/go.mod h1:1SR0GbLlnN3QUmYaflZNiH1ql+1qrSiB2vwcJ+4UM60=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
github.com/aws/aws-sdk-go-v2/service/ec2 v1.182.0 h1:LaeziEhHZ/SJZYBK223QVzl3ucHvA9IP4tQMcxGrc9I=
github.com/aws/aws-sdk-go-v2/service/ec2 v1.182.0/go.mod h1:kYXaB4FzyhEJjvrJ84oPnMElLiEAjGxxUunVW2tBSng=
github.com/aws/aws-sdk-go-v2/service/ec2 v1.183.0 h1:LgwYvo4kycfT/UD7vjQhSVZSatxHAI41/54q9O6jljI=
github.com/aws/aws-sdk-go-v2/service/ec2 v1.183.0/go.mod h1:kYXaB4FzyhEJjvrJ84oPnMElLiEAjGxxUunVW2tBSng=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 h1:s7NA1SOw8q/5c0wr8477yOPp0z+uBaXBnLE0XYb0POA=
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

33 changes: 33 additions & 0 deletions nodeadm/vendor/github.com/aws/aws-sdk-go-v2/config/load_options.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 6b4e4d9

Please sign in to comment.