Skip to content

Commit

Permalink
merge with master
Browse files Browse the repository at this point in the history
  • Loading branch information
r2k1 committed Oct 11, 2024
2 parents 0ea56f9 + 22fa404 commit b5ddb7c
Show file tree
Hide file tree
Showing 426 changed files with 65,791 additions and 21,319 deletions.
14 changes: 0 additions & 14 deletions .pipelines/.vsts-vhd-builder.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -104,18 +104,4 @@ stages:
LOCATION: $(PACKER_BUILD_LOCATION)
jobs:
- template: ./templates/e2e-template.yaml
- stage: Generate and Publish Testdata
condition: succeeded()
jobs:
- job: Generate_Publish_Testdata
timeoutInMinutes: 10
steps:
- bash: |
make generate
displayName: Generate Testdata
- task: 1ES.PublishPipelineArtifact@1
inputs:
targetPath: $(Pipeline.Workspace)/pkg/agent/testdata
artifact: testdata
publishLocation: pipeline

38 changes: 38 additions & 0 deletions .pipelines/.vsts-vhd-refresh-testdata.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
name: $(Date:yyyyMMdd)$(Rev:.r)_$(Build.SourceBranchName)_$(BuildID)
trigger:
branches:
include:
- master
paths:
include:
- schemas
- vhdbuilder/packer
- vhdbuilder/scripts/linux
- .pipelines/.vsts-vhd-refresh-testdata.yaml
- parts/linux/*
- packer.mk
exclude:
- vhdbuilder/packer/*.ps1
- vhdbuilder/packer/**/*.ps1
- vhdbuilder/packer/*windows*
- vhdbuilder/packer/**/*windows*

pool:
name: $(POOL_NAME)

stages:
- stage: Generate_and_Publish_Testdata
condition: succeeded()
jobs:
- job: Generate_Publish_Testdata
timeoutInMinutes: 10
steps:
- bash: |
make generate
displayName: Generate Testdata
- task: PublishPipelineArtifact@1
inputs:
targetPath: $(Pipeline.Workspace)/s/pkg/agent/testdata
artifact: testdata
publishLocation: pipeline

2 changes: 1 addition & 1 deletion .pipelines/scripts/verify_shell.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ else
echo "shellcheck installed"
fi

filesToCheck=$(find . -type f -name "*.sh" -not -path './parts/linux/cloud-init/artifacts/*' -not -path './pkg/agent/testdata/*' -not -path './vendor/*' -not -path './hack/tools/vendor/*' -not -path './.git/*' -not -path './self-contained/*' -not -path './hack/tools/bin/shellspecsrc/*')
filesToCheck=$(find . -type f -name "*.sh" -not -path './parts/linux/cloud-init/artifacts/*' -not -path './pkg/agent/testdata/*' -not -path './vendor/*' -not -path './hack/tools/vendor/*' -not -path './.git/*' -not -path './hack/tools/bin/shellspecsrc/*')

# also shell-check generated test data
generatedTestData=$(find ./pkg/agent/testdata -type f -name "*.sh" )
Expand Down
23 changes: 7 additions & 16 deletions .pipelines/templates/.builder-release-template-windows.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -69,21 +69,6 @@ steps:
echo "##vso[task.setvariable variable=BUILD_DATE]$BUILD_DATE"
displayName: Get Build Mode
- bash: |
if [[ ${{ parameters.artifactName }} =~ "2019" ]]; then
WINDOWS_VERSION="$(cat vhdbuilder/packer/windows-image.env | grep -a "WINDOWS_2019_BASE_IMAGE_VERSION" | cut -d "=" -f 2 | cut -d "." -f 1,2)"
elif [[ ${{ parameters.artifactName }} =~ "2022" ]]; then
WINDOWS_VERSION="$(cat vhdbuilder/packer/windows-image.env | grep -a "WINDOWS_2022_BASE_IMAGE_VERSION" | cut -d "=" -f 2 | cut -d "." -f 1,2)"
elif [[ ${{ parameters.artifactName }} =~ "23H2" ]]; then
WINDOWS_VERSION="$(cat vhdbuilder/packer/windows-image.env | grep -a "WINDOWS_23H2_BASE_IMAGE_VERSION" | cut -d "=" -f 2 | cut -d "." -f 1,2)"
else
echo "Current distro is not supported to get image version. You need to update related code."
exit 1
fi
AKS_WINDOWS_IMAGE_VERSION="$WINDOWS_VERSION.${BUILD_DATE}"
echo "##vso[task.setvariable variable=AKS_WINDOWS_IMAGE_VERSION]$AKS_WINDOWS_IMAGE_VERSION"
displayName: Get Image Version
- script: |
branch=$(Build.SourceBranch)
branch=$(echo "${branch}" | sed 's/refs\/heads\///g')
Expand Down Expand Up @@ -116,7 +101,7 @@ steps:
-e WINDOWS_CORE_IMAGE_URL=${WINDOWS_CORE_IMAGE_URL} \
-e WINDOWS_PRIVATE_PACKAGES_URL=${WINDOWS_PRIVATE_PACKAGES_URL} \
-e AZURE_MSI_RESOURCE_STRING=${AZURE_MSI_RESOURCE_STRING} \
-e AKS_WINDOWS_IMAGE_VERSION=${AKS_WINDOWS_IMAGE_VERSION} \
-e BUILD_DATE=${BUILD_DATE} \
${AZURE_CONTAINER_IMAGE} make -f packer.mk run-packer-windows
displayName: Building windows VHD
Expand Down Expand Up @@ -156,6 +141,12 @@ steps:
- bash: |
sudo chmod 777 image-bom.json
jq . image-bom.json > tmp.json
echo "Reading image version from image-bom.json"
AKS_WINDOWS_IMAGE_VERSION=$(cat image-bom.json | jq -r '.imageVersion')
echo "##vso[task.setvariable variable=AKS_WINDOWS_IMAGE_VERSION]$AKS_WINDOWS_IMAGE_VERSION"
echo "Image version: $AKS_WINDOWS_IMAGE_VERSION"
mv tmp.json ${AKS_WINDOWS_IMAGE_VERSION}-image-list.json
cp release-notes.txt ${AKS_WINDOWS_IMAGE_VERSION}.txt
displayName: Reformat image-bom.json and rename release-notes.txt
Expand Down
21 changes: 18 additions & 3 deletions .pipelines/templates/.builder-release-template.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ steps:
echo "##vso[task.setvariable variable=OS_DISK_URI]$(cat packer-output | grep "OSDiskUri:" | cut -d " " -f 2)" && \
echo "##vso[task.setvariable variable=MANAGED_SIG_ID]$(cat packer-output | grep "ManagedImageSharedImageGalleryId:" | cut -d " " -f 2)" && \
echo "##vso[task.setvariable variable=SIG_GALLERY_NAME]$(cat vhdbuilder/packer/settings.json | grep "sig_gallery_name" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
echo "##vso[task.setvariable variable=BUILD_PERF_DATA_FILE]vhd-build-performance-data.json" && \
echo "##vso[task.setvariable variable=PERFORMANCE_DATA_FILE]vhd-build-performance-data.json" && \
echo "##vso[task.setvariable variable=PKR_RG_NAME]$(cat packer-output | grep "ResourceGroupName" | cut -d "'" -f 2 | head -1)" && \
echo "##vso[task.setvariable variable=IS_NOT_1804]$( [[ "${OS_VERSION}" != "18.04" ]] && echo true || echo false )" && \
echo "##vso[task.setvariable variable=OS_NAME]Linux" && \
Expand All @@ -163,9 +163,20 @@ steps:
SYSTEM_TEAMPROJECT: $(System.TeamProject)
BUILD_RUN_NUMBER: $(Build.BuildNumber)

- bash: make -f packer.mk evaluate-build-performance
- task: PublishPipelineArtifact@0
condition: always()
displayName: Check Build Performance
displayName: Publish BCC Tools Installation Log
inputs:
artifactName: 'bcc-tools-installation-log-${{ parameters.artifactName }}'
targetPath: 'bcc-tools-installation.log'

- task: CopyFiles@2
condition: always()
displayName: Copy BCC Tools Installation Log
inputs:
SourceFolder: '$(System.DefaultWorkingDirectory)'
Contents: 'bcc-tools-installation.log'
TargetFolder: '$(Build.ArtifactStagingDirectory)'

- task: CopyFiles@2
condition: eq(variables['IS_NOT_1804'], 'true')
Expand Down Expand Up @@ -216,6 +227,10 @@ steps:
env:
RESOURCE_GROUP_NAME: $(AZURE_RESOURCE_GROUP_NAME)

- bash: make -f packer.mk evaluate-build-performance
condition: always()
displayName: Check Build Performance

- bash: make -f packer.mk generate-sas
displayName: Getting Shared Access Signature URI
condition: and(succeeded(), eq(variables.DRY_RUN, 'False'))
Expand Down
65 changes: 65 additions & 0 deletions parts/linux/cloud-init/artifacts/cse_benchmark_functions.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
#!/bin/bash

script_start_stopwatch=$(date +%s)
section_start_stopwatch=$(date +%s)
SCRIPT_NAME=$(basename $0 .sh)
SCRIPT_NAME="${SCRIPT_NAME//-/_}"
declare -A benchmarks=()
declare -a benchmarks_order=()

check_array_size() {
declare -n array_name=$1
local array_size=${#array_name[@]}
if [[ ${array_size} -gt 0 ]]; then
last_index=$(( ${#array_name[@]} - 1 ))
else
return 1
fi
}

capture_benchmark() {
set +x
local title="$1"
title="${title//[[:space:]]/_}"
title="${title//-/_}"
local is_final_section=${2:-false}

local current_time
current_time=$(date +%s)
if [[ "$is_final_section" == true ]]; then
local start_time=$script_start_stopwatch
else
local start_time=$section_start_stopwatch
fi

local total_time_elapsed
total_time_elapsed=$(date -d@$((current_time - start_time)) -u +%H:%M:%S)
benchmarks[$title]=${total_time_elapsed}
benchmarks_order+=($title) # use this array to maintain order of benchmarks

# reset timers for next section
section_start_stopwatch=$(date +%s)
}

process_benchmarks() {
set +x

if [ ! -f ${PERFORMANCE_DATA_FILE} ]; then
echo '{}' > ${PERFORMANCE_DATA_FILE}
fi

check_array_size benchmarks || { echo "Benchmarks array is empty"; return; }
# create script object, then append each section object to it in the for loop
script_object=$(jq -n --arg script_name "${SCRIPT_NAME}" '{($script_name): {}}')

for ((i=0; i<${#benchmarks_order[@]}; i+=1)); do
section_name=${benchmarks_order[i]}
section_object=$(jq -n --arg section_name "${section_name}" --arg total_time_elapsed "${benchmarks[${section_name}]}" \
'{($section_name): $total_time_elapsed'})
script_object=$(jq -n --argjson script_object "$script_object" --argjson section_object "$section_object" --arg script_name "${SCRIPT_NAME}" \
'$script_object | .[$script_name] += $section_object')
done

jq ". += $script_object" ${PERFORMANCE_DATA_FILE} > temp-perf-file.json && mv temp-perf-file.json ${PERFORMANCE_DATA_FILE}
chmod 755 ${PERFORMANCE_DATA_FILE}
}
70 changes: 10 additions & 60 deletions parts/linux/cloud-init/artifacts/cse_helpers.sh
Original file line number Diff line number Diff line change
Expand Up @@ -482,6 +482,16 @@ isMarinerOrAzureLinux() {
return 1
}

evalPackageDownloadURL() {
local url=${1:-}
if [[ -n "$url" ]]; then
eval "result=${url}"
echo $result
return
fi
echo ""
}

installJq() {
# jq is not available until downloaded in install-dependencies.sh with the installDeps function
# but it is needed earlier to call the capture_benchmarks function in pre-install-dependencies.sh
Expand All @@ -497,66 +507,6 @@ installJq() {
fi
}

check_array_size() {
declare -n array_name=$1
local array_size=${#array_name[@]}
if [[ ${array_size} -gt 0 ]]; then
last_index=$(( ${#array_name[@]} - 1 ))
else
return 1
fi
}

capture_benchmark() {
set +x
local title="$1"
title="${title//[[:space:]]/_}"
title="${title//-/_}"
local is_final_section=${2:-false}

local current_time=$(date +%s)
if [[ "$is_final_section" == true ]]; then
local start_time=$script_start_stopwatch
else
local start_time=$section_start_stopwatch
fi

total_time_elapsed=$(date -d@$((current_time - start_time)) -u +%H:%M:%S)
benchmarks[$title]=${total_time_elapsed}
benchmarks_order+=($title) # use this array to maintain order of benchmarks

# reset timers for next section
section_start_stopwatch=$(date +%s)
}

process_benchmarks() {
set +x
check_array_size benchmarks || { echo "Benchmarks array is empty"; return; }
# create script object, then append each section object to it in the for loop
script_object=$(jq -n --arg script_name "${SCRIPT_NAME}" '{($script_name): {}}')

for ((i=0; i<${#benchmarks_order[@]}; i+=1)); do
section_name=${benchmarks_order[i]}
section_object=$(jq -n --arg section_name "${section_name}" --arg total_time_elapsed "${benchmarks[${section_name}]}" \
'{($section_name): $total_time_elapsed'})
script_object=$(jq -n --argjson script_object "$script_object" --argjson section_object "$section_object" --arg script_name "${SCRIPT_NAME}" \
'$script_object | .[$script_name] += $section_object')
done

jq ". += $script_object" ${VHD_BUILD_PERF_DATA} > temp-build-perf-file.json && mv temp-build-perf-file.json ${VHD_BUILD_PERF_DATA}
chmod 755 ${VHD_BUILD_PERF_DATA}
}

evalPackageDownloadURL() {
local url=${1:-}
if [[ -n "$url" ]]; then
eval "result=${url}"
echo $result
return
fi
echo ""
}

# sets RELEASE to proper release metadata for the package based on the os and osVersion
# e.g., For os UBUNTU 18.04, if there is a release "r1804" defined in components.json, then set RELEASE to "r1804".
# Otherwise set RELEASE to "current"
Expand Down
32 changes: 2 additions & 30 deletions pkg/agent/baker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -238,21 +238,7 @@ var _ = Describe("Assert generated customData and cseCmd", func() {

agentPool := cs.Properties.AgentPoolProfiles[0]

fullK8sComponentsMap := K8sComponentsByVersionMap[cs.Properties.OrchestratorProfile.OrchestratorVersion]
pauseImage := cs.Properties.OrchestratorProfile.KubernetesConfig.MCRKubernetesImageBase + fullK8sComponentsMap["pause"]

hyperkubeImageBase := cs.Properties.OrchestratorProfile.KubernetesConfig.KubernetesImageBase
hyperkubeImage := hyperkubeImageBase + fullK8sComponentsMap["hyperkube"]
if cs.Properties.OrchestratorProfile.KubernetesConfig.CustomHyperkubeImage != "" {
hyperkubeImage = cs.Properties.OrchestratorProfile.KubernetesConfig.CustomHyperkubeImage
}

windowsPackage := datamodel.AzurePublicCloudSpecForTest.KubernetesSpecConfig.KubeBinariesSASURLBase + fullK8sComponentsMap["windowszip"]
k8sComponents := &datamodel.K8sComponents{
PodInfraContainerImageURL: pauseImage,
HyperkubeImageURL: hyperkubeImage,
WindowsPackageURL: windowsPackage,
}
k8sComponents := &datamodel.K8sComponents{}

if IsKubernetesVersionGe(k8sVersion, "1.29.0") {
k8sComponents.WindowsCredentialProviderURL = fmt.Sprintf("https://acs-mirror.azureedge.net/cloud-provider-azure/v%s/binaries/azure-acr-credential-provider-windows-amd64-v%s.tar.gz", k8sVersion, k8sVersion) //nolint:lll
Expand Down Expand Up @@ -1706,21 +1692,7 @@ var _ = Describe("Assert generated customData and cseCmd for Windows", func() {

agentPool := cs.Properties.AgentPoolProfiles[0]

fullK8sComponentsMap := K8sComponentsByVersionMap[cs.Properties.OrchestratorProfile.OrchestratorVersion]
pauseImage := cs.Properties.OrchestratorProfile.KubernetesConfig.MCRKubernetesImageBase + fullK8sComponentsMap["pause"]

hyperkubeImageBase := cs.Properties.OrchestratorProfile.KubernetesConfig.KubernetesImageBase
hyperkubeImage := hyperkubeImageBase + fullK8sComponentsMap["hyperkube"]
if cs.Properties.OrchestratorProfile.KubernetesConfig.CustomHyperkubeImage != "" {
hyperkubeImage = cs.Properties.OrchestratorProfile.KubernetesConfig.CustomHyperkubeImage
}

windowsPackage := datamodel.AzurePublicCloudSpecForTest.KubernetesSpecConfig.KubeBinariesSASURLBase + fullK8sComponentsMap["windowszip"]
k8sComponents := &datamodel.K8sComponents{
PodInfraContainerImageURL: pauseImage,
HyperkubeImageURL: hyperkubeImage,
WindowsPackageURL: windowsPackage,
}
k8sComponents := &datamodel.K8sComponents{}

if IsKubernetesVersionGe(k8sVersion, "1.29.0") {
// This is test only, credential provider version does not align with k8s version
Expand Down
16 changes: 1 addition & 15 deletions pkg/agent/bakerapi_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,21 +68,7 @@ var _ = Describe("AgentBaker API implementation tests", func() {

agentPool := cs.Properties.AgentPoolProfiles[0]

fullK8sComponentsMap := K8sComponentsByVersionMap[cs.Properties.OrchestratorProfile.OrchestratorVersion]
pauseImage := cs.Properties.OrchestratorProfile.KubernetesConfig.MCRKubernetesImageBase + fullK8sComponentsMap["pause"]

hyperkubeImageBase := cs.Properties.OrchestratorProfile.KubernetesConfig.KubernetesImageBase
hyperkubeImage := hyperkubeImageBase + fullK8sComponentsMap["hyperkube"]
if cs.Properties.OrchestratorProfile.KubernetesConfig.CustomHyperkubeImage != "" {
hyperkubeImage = cs.Properties.OrchestratorProfile.KubernetesConfig.CustomHyperkubeImage
}

windowsPackage := datamodel.AzurePublicCloudSpecForTest.KubernetesSpecConfig.KubeBinariesSASURLBase + fullK8sComponentsMap["windowszip"]
k8sComponents := &datamodel.K8sComponents{
PodInfraContainerImageURL: pauseImage,
HyperkubeImageURL: hyperkubeImage,
WindowsPackageURL: windowsPackage,
}
k8sComponents := &datamodel.K8sComponents{}

kubeletConfig := map[string]string{
"--address": "0.0.0.0",
Expand Down
2 changes: 1 addition & 1 deletion pkg/agent/datamodel/linux_sig_version.json
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{
"ostype": "LinuxSIGImageVersion",
"version": "202409.30.0"
"version": "202410.09.0"
}
Loading

0 comments on commit b5ddb7c

Please sign in to comment.