From f8669351bb8f1238223ccab2a80dbd618c0faa9f Mon Sep 17 00:00:00 2001 From: Jason Deal Date: Wed, 13 Dec 2023 11:52:48 -0800 Subject: [PATCH] update e2e workflow --- .github/actions/e2e/setup-cluster/action.yaml | 26 +++++++++++---- .github/workflows/e2e-matrix.yaml | 33 ++++++++++++------- .github/workflows/e2e.yaml | 12 ++++--- 3 files changed, 48 insertions(+), 23 deletions(-) diff --git a/.github/actions/e2e/setup-cluster/action.yaml b/.github/actions/e2e/setup-cluster/action.yaml index 332468205437..9ba803bd31ab 100644 --- a/.github/actions/e2e/setup-cluster/action.yaml +++ b/.github/actions/e2e/setup-cluster/action.yaml @@ -40,6 +40,10 @@ inputs: git_ref: description: "The git commit, tag, or branch to check out. Requires a corresponding Karpenter snapshot release" required: false + enable_local_zones: + description: "Whether to include local zones in the VPC created for the cluster." + required: false + default: 'false' runs: using: "composite" steps: @@ -81,14 +85,15 @@ runs: IP_FAMILY: ${{ inputs.ip_family }} PRIVATE_CLUSTER: ${{ inputs.private_cluster }} GIT_REF: ${{ inputs.git_ref }} + ENABLE_LOCAL_ZONES: ${{ inputs.enable_local_zones }} run: | - if [[ "$GIT_REF" == '' ]]; then + if [[ "$GIT_REF" == '' ]]; then GIT_REF=$(git rev-parse HEAD) fi - # Disable Pod Identity for Karpenter on K8s 1.23. Pod Identity is not supported on K8s 1.23 + # Disable Pod Identity for Karpenter on K8s 1.23. Pod Identity is not supported on K8s 1.23 # https://docs.aws.amazon.com/eks/latest/userguide/pod-identities.html#pod-id-considerations - if [[ "$K8S_VERSION" == '1.23' ]]; then + if [[ "$K8S_VERSION" == '1.23' ]]; then KARPENTER_IAM=""" - metadata: name: karpenter @@ -104,14 +109,14 @@ runs: serviceAccountName: karpenter roleName: karpenter-irsa-${{ inputs.cluster_name }} permissionsBoundaryARN: "arn:aws:iam::${{ inputs.account_id }}:policy/GithubActionsPermissionsBoundary" - permissionPolicyARNs: + permissionPolicyARNs: - "arn:aws:iam::${{ inputs.account_id }}:policy/KarpenterControllerPolicy-${{ inputs.cluster_name }}"""" POD_IDENTITY="""- name: eks-pod-identity-agent permissionsBoundary: "arn:aws:iam::$ACCOUNT_ID:policy/GithubActionsPermissionsBoundary" configurationValues: | tolerations: - operator: Exists""" - fi + fi # Create or Upgrade the cluster based on whether the cluster already exists cmd="create" @@ -178,6 +183,13 @@ runs: $POD_IDENTITY EOF + if [[ $ENABLE_LOCAL_ZONES == "true" ]]; then + local_zones=$(AWS_REGION=$REGION aws ec2 describe-availability-zones | yq '.AvailabilityZones | filter(.ZoneType == "local-zone") | [.[].ZoneName] | join(" ")') + for zone in $local_zones; do + yq -i ".localZones += [\"$zone\"]" clusterconfig.yaml + done + fi + if [[ $PRIVATE_CLUSTER == 'true' ]]; then yq -i '.privateCluster.enabled=true' clusterconfig.yaml yq -i '.managedNodeGroups[0].privateNetworking=true' clusterconfig.yaml @@ -211,9 +223,9 @@ runs: CLUSTER_NAME: ${{ inputs.cluster_name }} GIT_REF: ${{ inputs.git_ref }} run: | - if [[ "$GIT_REF" == '' ]]; then + if [[ "$GIT_REF" == '' ]]; then GIT_REF=$(git rev-parse HEAD) - fi + fi oidc_id=$(aws eks describe-cluster --name "$CLUSTER_NAME" --query "cluster.identity.oidc.issuer" --output text | cut -d '/' -f 3,4,5) arn="arn:aws:iam::$ACCOUNT_ID:oidc-provider/${oidc_id}" aws iam tag-open-id-connect-provider --open-id-connect-provider-arn $arn \ diff --git a/.github/workflows/e2e-matrix.yaml b/.github/workflows/e2e-matrix.yaml index e3dc749bbe41..12f3d3e37869 100644 --- a/.github/workflows/e2e-matrix.yaml +++ b/.github/workflows/e2e-matrix.yaml @@ -59,19 +59,30 @@ jobs: max-parallel: ${{ inputs.parallelism || 100 }} matrix: suite: - - Integration - - NodeClaim - - Consolidation - - Interruption - - Drift - - Expiration - - Chaos - - IPv6 + - name: Integration + region: ${{ inputs.region }} + - name: NodeClaim + region: ${{ inputs.region }} + - name: Consolidation + region: ${{ inputs.region }} + - name: Interruption + region: ${{ inputs.region }} + - name: Drift + region: ${{ inputs.region }} + - name: Expiration + region: ${{ inputs.region }} + - name: Chaos + region: ${{ inputs.region }} + - name: IPv6 + region: ${{ inputs.region }} + - name: LocalZone + # LAX is the only local zone available in the CI account, therefore only use us-west-2 + region: us-west-2 uses: ./.github/workflows/e2e.yaml with: - suite: ${{ matrix.suite }} + suite: ${{ matrix.suite.name }} git_ref: ${{ inputs.git_ref }} - region: ${{ inputs.region }} + region: ${{ matrix.suite.region }} k8s_version: ${{ inputs.k8s_version }} cleanup: ${{ inputs.cleanup }} workflow_trigger: ${{ inputs.workflow_trigger }} @@ -83,7 +94,7 @@ jobs: statuses: write # ./.github/actions/commit-status/start uses: ./.github/workflows/e2e-upgrade.yaml with: - # This version matches the version switch between IRSA -> Pod Identity + # This version matches the version switch between IRSA -> Pod Identity # https://github.com/aws/karpenter-provider-aws/pull/5262 from_git_ref: 8f500c23be18aa5cb8089a83c43e763303faa9ac to_git_ref: ${{ inputs.git_ref }} diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 4663ff732335..130f37ce2c32 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -26,6 +26,7 @@ on: - IPv6 - Scale - PrivateCluster + - LocalZone k8s_version: type: choice options: @@ -108,12 +109,12 @@ jobs: CLUSTER_NAME: ${{ inputs.cluster_name }} WORKFLOW_TRIGGER: ${{ inputs.workflow_trigger }} run: | - if [[ "$CLUSTER_NAME" == '' ]]; then - if [[ "$WORKFLOW_TRIGGER" == 'soak' ]]; then + if [[ "$CLUSTER_NAME" == '' ]]; then + if [[ "$WORKFLOW_TRIGGER" == 'soak' ]]; then CLUSTER_NAME=$(echo "soak-periodic-$RANDOM$RANDOM" | awk '{print tolower($0)}' | tr / -) - else + else CLUSTER_NAME=$(echo "$SUITE-$RANDOM$RANDOM" | awk '{print tolower($0)}' | tr / -) - fi + fi fi echo "Using cluster name \"$CLUSTER_NAME\"" echo CLUSTER_NAME="$CLUSTER_NAME" >> "$GITHUB_OUTPUT" @@ -134,6 +135,7 @@ jobs: ecr_region: ${{ vars.SNAPSHOT_REGION }} prometheus_workspace_id: ${{ vars.WORKSPACE_ID }} prometheus_region: ${{ vars.PROMETHEUS_REGION }} + enable_local_zones: ${{ inputs.suite == 'LocalZone' }} - name: run the ${{ inputs.suite }} test suite env: SUITE: ${{ inputs.suite }} @@ -148,7 +150,7 @@ jobs: kubectl delete nodepool --all kubectl delete ec2nodeclass --all kubectl delete deployment --all - + TEST_SUITE="$SUITE" ENABLE_METRICS=$ENABLE_METRICS METRICS_REGION=${{ vars.TIMESTREAM_REGION }} GIT_REF="$(git rev-parse HEAD)" \ CLUSTER_NAME="${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }}" CLUSTER_ENDPOINT="$(aws eks describe-cluster --name ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }} --query "cluster.endpoint" --output text)" \ INTERRUPTION_QUEUE="${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }}" make e2etests