feat: adding stateless CNI pipeline test

Azure · Aug 13, 2024 · a00c923 · a00c923
1 parent fc4da42
commit a00c923
Show file tree

Hide file tree

Showing 9 changed files with 826 additions and 5 deletions.
diff --git a/.pipelines/cni/pipeline.yaml b/.pipelines/cni/pipeline.yaml
@@ -224,6 +224,21 @@ stages:
       osSkuWin: 'Windows2022'
       scaleup: ${SCALEUP_WIN}
       iterations: ${ITERATIONS_WIN}
+  - template: singletenancy/cniv2-stateless-template.yaml
+    parameters:
+      name: windows_overlay_stateless
+      clusterType: overlay-byocni-up
+      clusterName: w22-over
+      nodeCount: ${NODE_COUNT_WINCLUSTER_SYSTEMPOOL}
+      nodeCountWin: ${NODE_COUNT_WIN}
+      vmSize: ${VM_SIZE_WINCLUSTER_SYSTEMPOOL}
+      vmSizeWin: ${VM_SIZE_WIN}
+      arch: amd64
+      os: windows
+      os_version: 'ltsc2022'
+      osSkuWin: 'Windows2022'
+      scaleup: ${SCALEUP_WIN}
+      iterations: ${ITERATIONS_WIN}
 
   - template: singletenancy/cniv2-template.yaml
     parameters:

diff --git a/.pipelines/cni/singletenancy/cniv2-stateless-template.yaml b/.pipelines/cni/singletenancy/cniv2-stateless-template.yaml
diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml
@@ -417,6 +417,17 @@ stages:
         vmSize: Standard_B2ms
         k8sVersion: ""
         dependsOn: "containerize"
+    # Azure Overlay E2E Stateless CNI tests
+    - template: singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-job-template.yaml
+      parameters:
+        name: "azure_overlay_stateless_e2e"
+        displayName: Azure Stateless CNI Overlay
+        os: linux
+        clusterType: overlay-byocni-up
+        clusterName: "azovere2estateless"
+        vmSize: Standard_B2ms
+        k8sVersion: ""
+        dependsOn: "containerize"
 
     # AKS Swift E2E tests
     - template: singletenancy/aks-swift/e2e-job-template.yaml

diff --git a/...ngletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-job-template.yaml b/...ngletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-job-template.yaml
@@ -0,0 +1,126 @@
+parameters:
+  name: ""
+  displayName: ""
+  clusterType: ""
+  clusterName: ""
+  vmSize: ""
+  k8sVersion: ""
+  dependsOn: ""
+
+stages:
+  - stage: ${{ parameters.clusterName }}
+    displayName: Create Cluster - ${{ parameters.displayName }}
+    dependsOn:
+      - ${{ parameters.dependsOn }}
+      - setup
+    pool:
+      name: $(BUILD_POOL_NAME_DEFAULT)
+    variables:
+      commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
+    jobs:
+      - template: ../../templates/create-cluster.yaml
+        parameters:
+          name: ${{ parameters.name }}
+          displayName: ${{ parameters.displayName }}
+          clusterType: ${{ parameters.clusterType }}
+          clusterName: ${{ parameters.clusterName }}-$(commitID)
+          vmSize: ${{ parameters.vmSize }}
+          k8sVersion: ${{ parameters.k8sVersion }}
+          dependsOn: ${{ parameters.dependsOn }}
+          region: $(REGION_AKS_CLUSTER_TEST)
+
+  - stage: ${{ parameters.name }}
+    displayName: E2E - ${{ parameters.displayName }}
+    dependsOn:
+    - setup
+    - publish
+    - ${{ parameters.clusterName }}
+    variables:
+      commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
+      GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
+      GOBIN: "$(GOPATH)/bin" # Go binaries path
+      modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
+    pool:
+      name: $(BUILD_POOL_NAME_DEFAULT)
+    jobs:
+      - job: ${{ parameters.name }}_linux
+        displayName: Azure CNI Overlay Test Suite | Linux - (${{ parameters.name }})
+        pool:
+          name: $(BUILD_POOL_NAME_DEFAULT)
+          demands:
+          - agent.os -equals Linux
+          - Role -equals $(CUSTOM_E2E_ROLE)
+        steps:
+          - template: azure-cni-overlay-stateless-e2e-step-template.yaml
+            parameters:
+              name: ${{ parameters.name }}
+              clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: linux
+
+      - job: ${{ parameters.name }}_windows
+        displayName: Azure CNI Overlay Test Suite | Windows - (${{ parameters.name }})
+        dependsOn: ${{ parameters.name }}_linux
+        pool:
+          name: $(BUILD_POOL_NAME_DEFAULT)
+          demands:
+          - agent.os -equals Linux
+          - Role -equals $(CUSTOM_E2E_ROLE)
+        steps:
+          - template: azure-cni-overlay-stateless-e2e-step-template.yaml
+            parameters:
+              name: ${{ parameters.name }}
+              clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: windows
+              vmSizeWin: ${{ parameters.vmSize }} # Matching linux vmSize
+
+      - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
+        parameters:
+          sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+          clusterName: ${{ parameters.clusterName }}-$(commitID)
+          os: ${{ parameters.os }}
+          dependsOn: ${{ parameters.name }}_windows
+          datapath: true
+          dns: true
+          portforward: true
+          hostport: true
+          service: true
+
+      - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
+        parameters:
+          sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+          clusterName: ${{ parameters.clusterName }}-$(commitID)
+          os: windows
+          dependsOn: cni_${{ parameters.os }}
+          datapath: true
+          dns: true
+          portforward: true
+          hostport: true
+          service: true
+          hybridWin: true
+
+      - job: failedE2ELogs_linux
+        displayName: "Linux Failure Logs"
+        dependsOn:
+          - ${{ parameters.name }}_linux
+          - cni_linux
+        condition: in(dependencies.${{ parameters.name }}_linux.result, 'Failed')
+        steps:
+          - template: ../../templates/log-template.yaml
+            parameters:
+              clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: linux
+              cni: cniv2
+
+      - job: failedE2ELogs_windows
+        displayName: "Windows Failure Logs"
+        dependsOn:
+          - ${{ parameters.name }}_windows
+          - cni_windows
+        condition: in(dependencies.${{ parameters.name }}_windows.result, 'Failed')
+        steps:
+          - template: ../../templates/log-template.yaml
+            parameters:
+              clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: windows
+              cni: cniv2
+
diff --git a/...gletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-step-template.yaml b/...gletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-step-template.yaml
@@ -0,0 +1,134 @@
+parameters:
+  name: ""
+  clusterName: ""
+  os: ""
+
+steps:
+  - bash: |
+      go version
+      go env
+      mkdir -p '$(GOBIN)'
+      mkdir -p '$(GOPATH)/pkg'
+      mkdir -p '$(modulePath)'
+      echo '##vso[task.prependpath]$(GOBIN)'
+      echo '##vso[task.prependpath]$(GOROOT)/bin'
+    name: "GoEnv"
+    displayName: "Set up the Go environment"
+
+  - task: KubectlInstaller@0
+    inputs:
+      kubectlVersion: latest
+
+  - task: AzureCLI@1
+    inputs:
+      azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+      scriptLocation: "inlineScript"
+      scriptType: "bash"
+      addSpnToEnvironment: true
+      inlineScript: |
+        set -e
+        make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}
+    name: "kubeconfig"
+    displayName: "Set Kubeconfig"
+
+  - ${{ if eq(parameters.os, 'linux') }}:
+    - script: |
+        echo "Start Integration Tests on Overlay Cluster"
+        kubectl get po -owide -A
+        sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=statelessCNI VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_AZURE_VNET_STATELESS=true VALIDATE_V4OVERLAY=true AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) CLEANUP=true
+      retryCountOnTaskFailure: 2
+      name: "integrationTest"
+      displayName: "Run CNS Integration Tests on AKS Overlay"
+
+    - task: AzureCLI@1
+      inputs:
+        azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+        scriptLocation: "inlineScript"
+        scriptType: "bash"
+        addSpnToEnvironment: true
+        inlineScript: |
+          set -e
+          kubectl get po -owide -A
+          clusterName=${{ parameters.clusterName }}
+          echo "Restarting nodes"
+          for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
+            make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
+          done
+      displayName: "Restart Nodes"
+
+    - task: AzureCLI@1
+      inputs:
+        azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+        scriptLocation: "inlineScript"
+        scriptType: "bash"
+        addSpnToEnvironment: true
+        inlineScript: |
+          cd test/integration/load
+          clusterName=${{ parameters.clusterName }}
+          make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName}
+          make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)
+          kubectl get pods -owide -A
+          echo "Validating Node Restart"
+          CNI_TYPE=statelessCNI RESTART_CASE=true go test -timeout 30m -tags load -run ^TestValidateState$
+      displayName: "Validate Node Restart"
+      retryCountOnTaskFailure: 3
+
+    - script: |
+        echo "validate pod IP assignment before CNS restart"
+        kubectl get pod -owide -A
+        echo "validate pod state before CNS restarts"
+        cd test/integration/load
+        CNI_TYPE=statelessCNI go test -timeout 30m -tags load -run ^TestValidateState$
+        kubectl rollout restart ds azure-cns -n kube-system
+        kubectl rollout status ds azure-cns -n kube-system
+        kubectl get pod -owide -A
+        echo "validate pods after CNS restart"
+        CNI_TYPE=statelessCNI go test -timeout 30m -tags load -run ^TestValidateState$
+      name: "restartCNS_ValidatePodState"
+      displayName: "Restart CNS and validate pod state"
+      retryCountOnTaskFailure: 3
+
+    - script: |
+        echo "Run wireserver and metadata connectivity Tests"
+        bash test/network/wireserver_metadata_test.sh
+      retryCountOnTaskFailure: 3
+      name: "WireserverMetadataConnectivityTests"
+      displayName: "Run Wireserver and Metadata Connectivity Tests"
+
+  - ${{ if eq(parameters.os, 'windows') }}:
+    - task: AzureCLI@2
+      inputs:
+        azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+        scriptLocation: "inlineScript"
+        scriptType: "bash"
+        addSpnToEnvironment: true
+        inlineScript: |
+          set -e
+          make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} VM_SIZE_WIN=${{ parameters.vmSizeWin }}
+          echo "Windows node are successfully added to v4 Overlay Cluster"
+          kubectl cluster-info
+          kubectl get node -owide
+          kubectl get po -owide -A
+      name: "Add_Windows_Node"
+      displayName: "Add windows node on v4 overlay cluster"
+
+    - script: |
+        nodeList=`kubectl get node -owide | grep Windows | awk '{print $1}'`
+        for node in $nodeList; do
+            taint=`kubectl describe node $node | grep Taints | awk '{print $2}'`
+            if [ $taint == "node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule" ]; then
+                kubectl taint nodes $node node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule-
+            fi
+        done
+        sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=windows CNI_TYPE=statelessCNI VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_AZURE_VNET_STATELESS=true VALIDATE_V4OVERLAY=true CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) CLEANUP=true
+      name: "WindowsOverlayControlPlaneScaleTests"
+      displayName: "Windows v4Overlay ControlPlane Scale Tests"
+      retryCountOnTaskFailure: 2
+
+    - script: |
+        echo "IPv4 Overlay DataPath Test"
+        cd test/integration/datapath
+        sudo -E env "PATH=$PATH" go test -count=1 datapath_windows_test.go -timeout 3m -tags connection -restartKubeproxy true -run ^TestDatapathWin$
+      name: "WindowsV4OverlayDatapathTests"
+      displayName: "Windows v4Overlay Datapath Tests"
+      retryCountOnTaskFailure: 3
diff --git a/test/integration/manifests/cnsconfig/azurestatelesscnioverlaywindowsconfigmap.yaml b/test/integration/manifests/cnsconfig/azurestatelesscnioverlaywindowsconfigmap.yaml
@@ -0,0 +1,35 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cns-win-config
+  namespace: kube-system
+data:
+  cns_config.json: |
+    {
+      "TelemetrySettings": {
+          "TelemetryBatchSizeBytes": 16384,
+          "TelemetryBatchIntervalInSecs": 15,
+          "RefreshIntervalInSecs": 15,
+          "DisableAll": false,
+          "HeartBeatIntervalInMins": 30,
+          "DebugMode": false,
+          "SnapshotIntervalInMins": 60
+      },
+      "ManagedSettings": {
+          "PrivateEndpoint": "",
+          "InfrastructureNetworkID": "",
+          "NodeID": "",
+          "NodeSyncIntervalInSeconds": 30
+      },
+      "EnableSubnetScarcity": false,
+      "ChannelMode": "CRD",
+      "InitializeFromCNI": true,
+      "ManageEndpointState": true,
+      "ProgramSNATIPTables" : false,
+      "MetricsBindAddress": ":10092",
+      "EnableCNIConflistGeneration": false,
+      "CNIConflistFilepath": "C:\\k\\azurecni\\netconf\\10-azure.conflist",
+      "CNIConflistScenario": "v4overlay",
+      "EnableAsyncPodDelete": true,
+      "AsyncPodDeletePath": "/var/run/azure-vnet/deleteIDs"
+    }