From 27b94b5712e9136201243023870e379aa1b2c6ab Mon Sep 17 00:00:00 2001 From: Abhay Krishna Date: Tue, 25 Jun 2024 14:59:43 -0700 Subject: [PATCH] Backport upgrader changes to release-0.19 branch (#3383) * Automate Bottlerocket release version updates (#2959) * Automate Cilium and EKS Distro releases upgrade (#2973) * Define Go version files for etcdadm bootstrap provider and controller (#2977) * Extract only the Go binary from GitHub release tarball (#2978) * Allow different tags for required images in Helm charts (#2983) * Remove rolesanywhere-credential-helper project from upgrade buildspec (#2987) * Attempt to apply patches and generate checksums/attribution if successful (#2992) * Remove newlines from kubeVersion field (#3002) * Fix checksums and attribution generation during upgrade (#3009) * Fix condition for patches warning comment on upgrade PRs (#3038) * Fix attribution and checksum generation for successful patch application (#3060) * Fix Bottlerocket host container metadata files (#3075) * Support upgrading release-branched projects (#3066) * Filter out pre-release tags in upgrader flow (#3089) * Handle cases where GitHub release does not exist for tag (#3116) * Fix Go mod location for cluster-autoscaler (#3127) * Allow upgrading projects tracked with commits (#3136) * Fix pre-release detection logic when fetching latest GitHub revision (#3140) * Fix cert-manager GitHub release tarball name (#3142) * Handle patches application for release-branched projects (#3150) * Allow projects to selectively upgrade to pre-release tags (#3164) * Remove dependencies for EKS Distro version upgrade step (#3166) * Refactor latest release logic for EKS Distro upgrades (#3174) * Fix the issue of patch repo in upgrade cmd (#3220) * Make image-builder upgrade flow non-release-branched (#3234) * Use latest release branch by default if not defined, fall back to Github file for Go versions (#3244) * Avoid printing directory name for Make command (#3249) * Use go.mod file to retrieve cert-manager Go version (#3273) * Exclude GitHub Helm chart release tags (#3320) * Allow project upgrades on release branch (#3275) * Use Go 1.22 to build version tracker binary (#3081) --------- Co-authored-by: Xu Deng --- Common.mk | 5 +- Makefile | 6 +- build/lib/generate_staging_buildspec.sh | 11 +- build/lib/helm_replace.sh | 2 +- build/lib/helm_require.sh | 108 +-- buildspecs/upgrade-eks-distro-buildspec.yml | 14 + .../aws/image-builder/builder/artifacts.go | 3 +- projects/aws/image-builder/builder/builder.go | 4 +- projects/aws/image-builder/builder/utils.go | 2 +- .../aws/image-builder/builder/utils_test.go | 2 +- .../image-builder/cmd/downloadartifacts.go | 3 +- .../rolesanywhere-credential-helper/Makefile | 2 + projects/cilium/cilium/Makefile | 1 - projects/kubernetes/autoscaler/Makefile | 1 - .../kubernetes/cloud-provider-aws/Makefile | 1 - .../cloud-provider-vsphere/Makefile | 1 - projects/prometheus/prometheus/Makefile | 3 + tools/version-tracker/Makefile | 7 +- tools/version-tracker/SKIPPED_PROJECTS | 1 - tools/version-tracker/buildspecs/upgrade.yml | 168 +++- tools/version-tracker/go.mod | 28 +- tools/version-tracker/go.sum | 79 +- .../pkg/commands/display/display.go | 78 +- .../pkg/commands/listprojects/listprojects.go | 16 +- .../pkg/commands/upgrade/upgrade.go | 802 +++++++++++++++--- .../pkg/constants/constants.go | 143 +++- .../pkg/ecrpublic/ecrpublic.go | 57 ++ tools/version-tracker/pkg/git/git.go | 27 +- tools/version-tracker/pkg/github/github.go | 400 +++++---- tools/version-tracker/pkg/types/types.go | 16 +- tools/version-tracker/pkg/util/file/file.go | 15 + .../version-tracker/pkg/util/slices/slices.go | 10 - tools/version-tracker/pkg/util/tar/tar.go | 18 +- .../pkg/util/version/version.go | 3 +- 34 files changed, 1542 insertions(+), 495 deletions(-) create mode 100644 buildspecs/upgrade-eks-distro-buildspec.yml create mode 100644 tools/version-tracker/pkg/ecrpublic/ecrpublic.go delete mode 100644 tools/version-tracker/pkg/util/slices/slices.go diff --git a/Common.mk b/Common.mk index 4661d2015d..3d2356a8e8 100644 --- a/Common.mk +++ b/Common.mk @@ -103,7 +103,7 @@ SUPPORTED_K8S_VERSIONS=$(shell cat $(BASE_DIRECTORY)/release/SUPPORTED_RELEASE_B SKIPPED_K8S_VERSIONS?= BINARIES_ARE_RELEASE_BRANCHED?=true IS_RELEASE_BRANCH_BUILD=$(filter true,$(HAS_RELEASE_BRANCHES)) -UNRELEASE_BRANCH_BINARY_TARGETS=binaries attribution checksums +UNRELEASE_BRANCH_BINARY_TARGETS=patch-repo binaries attribution checksums IS_UNRELEASE_BRANCH_TARGET=$(and $(filter false,$(BINARIES_ARE_RELEASE_BRANCHED)),$(filter $(UNRELEASE_BRANCH_BINARY_TARGETS) $(foreach target,$(UNRELEASE_BRANCH_BINARY_TARGETS),run-$(target)-in-docker run-in-docker/$(target)),$(MAKECMDGOALS))) TARGETS_ALLOWED_WITH_NO_RELEASE_BRANCH?= TARGETS_ALLOWED_WITH_NO_RELEASE_BRANCH+=build release clean clean-extra clean-go-cache help start-docker-builder stop-docker-builder create-ecr-repos all-attributions all-checksums all-attributions-checksums update-patch-numbers check-for-release-branch-skip run-buildkit-and-registry $(if $(filter false, $(HAS_LICENSES)),attribution,) $(if $(filter true, $(HAS_HELM_CHART)),,helm/push) @@ -213,6 +213,7 @@ HELM_USE_UPSTREAM_IMAGE?=false HELM_DIRECTORY?=. HELM_DESTINATION_REPOSITORY?=$(HELM_CHART_NAME) HELM_IMAGE_LIST?=$(IMAGE_COMPONENT) +HELM_IMAGE_TAG_LIST?=$(foreach _,$(HELM_IMAGE_LIST),$(IMAGE_TAG)) HELM_GIT_CHECKOUT_TARGET?=$(HELM_SOURCE_REPOSITORY)/eks-anywhere-checkout-$(HELM_GIT_TAG) HELM_GIT_PATCH_TARGET?=$(HELM_SOURCE_REPOSITORY)/eks-anywhere-helm-patched PACKAGE_DEPENDENCIES?= @@ -856,7 +857,7 @@ $(call FULL_CHART_TARGETS,copy) : %/helm/copy: checkout-repo checkout-helm-repo @$(BUILD_LIB)/helm_copy.sh $(HELM_SOURCE_REPOSITORY) $(HELM_DESTINATION_REPOSITORY) $(HELM_DIRECTORY) $(OUTPUT_DIR) $(call FULL_CHART_TARGETS,require) : %/helm/require: %/helm/copy | $$(ENABLE_LOGGING) - @$(BUILD_LIB)/helm_require.sh $(HELM_SOURCE_IMAGE_REPO) $(HELM_DESTINATION_REPOSITORY) $(OUTPUT_DIR) $(IMAGE_TAG) $(HELM_TAG) $(PROJECT_ROOT) $(LATEST) $(HELM_USE_UPSTREAM_IMAGE) "$(PACKAGE_DEPENDENCIES)" "$(FORCE_JSON_SCHEMA_FILE)" $(HELM_IMAGE_LIST) + @$(BUILD_LIB)/helm_require.sh $(HELM_SOURCE_IMAGE_REPO) $(HELM_DESTINATION_REPOSITORY) $(OUTPUT_DIR) $(IMAGE_TAG) $(HELM_TAG) $(PROJECT_ROOT) $(LATEST) $(HELM_USE_UPSTREAM_IMAGE) "$(PACKAGE_DEPENDENCIES)" "$(FORCE_JSON_SCHEMA_FILE)" "$(HELM_IMAGE_LIST)" "$(HELM_IMAGE_TAG_LIST)" $(call FULL_CHART_TARGETS,replace) : %/helm/replace: %/helm/require | $$(ENABLE_LOGGING) @$(BUILD_LIB)/helm_replace.sh $(HELM_DESTINATION_REPOSITORY) $(OUTPUT_DIR) diff --git a/Makefile b/Makefile index 78f828122d..877ba952dc 100644 --- a/Makefile +++ b/Makefile @@ -207,7 +207,7 @@ generate-staging-buildspec: | ensure-locale build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "emissary-ingress_emissary" "$(BASE_DIRECTORY)/projects/emissary-ingress/emissary/buildspecs/batch-build.yml" "$(BASE_DIRECTORY)/buildspec.yml" true "DO_NOT_EXCLUDE_FROM_BUILDSPEC" build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "goharbor_harbor" "$(BASE_DIRECTORY)/projects/goharbor/harbor/buildspecs/batch-build.yml" "$(BASE_DIRECTORY)/buildspec.yml" true "DO_NOT_EXCLUDE_FROM_BUILDSPEC" build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "aws_upgrader" "$(BASE_DIRECTORY)/projects/aws/upgrader/buildspecs/batch-build.yml" "$(BASE_DIRECTORY)/buildspec.yml" true "DO_NOT_EXCLUDE_FROM_BUILDSPEC" - build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "$(ALL_PROJECTS)" "$(BASE_DIRECTORY)/tools/version-tracker/buildspecs/upgrade.yml" "$(BASE_DIRECTORY)/buildspecs/upgrade-buildspec.yml" true EXCLUDE_FROM_UPGRADE_BUILDSPEC UPGRADE_BUILDSPECS false + build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "$(ALL_PROJECTS)" "$(BASE_DIRECTORY)/tools/version-tracker/buildspecs/upgrade.yml" "$(BASE_DIRECTORY)/buildspecs/upgrade-buildspec.yml" true EXCLUDE_FROM_UPGRADE_BUILDSPEC UPGRADE_BUILDSPECS false buildspecs/upgrade-eks-distro-buildspec.yml true .PHONY: generate generate: generate-project-list generate-staging-buildspec @@ -254,3 +254,7 @@ ensure-locale: exporting LANG=C.UTF-8 to generate files instead.; \ fi; \ fi + +.PHONY: get-default-release-branch +get-default-release-branch: + @echo $(LATEST_EKSD_RELEASE) diff --git a/build/lib/generate_staging_buildspec.sh b/build/lib/generate_staging_buildspec.sh index 8404d7f53c..38417042e8 100755 --- a/build/lib/generate_staging_buildspec.sh +++ b/build/lib/generate_staging_buildspec.sh @@ -26,6 +26,7 @@ EXCLUDE_VAR="${6:-EXCLUDE_FROM_STAGING_BUILDSPEC}" BUILDSPECS_VAR="${7:-BUILDSPECS}" FAST_FAIL="${8:-true}" ADD_FINAL_STAGE="${9:-}" +NO_DEPS_FOR_FINAL_STAGE="${10:-false}" SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" source "${SCRIPT_ROOT}/common.sh" @@ -164,6 +165,10 @@ for project in "${PROJECTS[@]}"; do BUILDSPEC_VARS_KEYS="" fi + if [[ "$BUILDSPECS_VAR" == "UPGRADE_BUILDSPECS" ]] && [[ "${IDENTIFIER}" = "kubernetes_sigs_image_builder" ]]; then + BUILDSPEC_VARS_KEYS="" + fi + if [[ -n "$BUILDSPEC_VARS_KEYS" ]]; then KEYS=(${BUILDSPEC_VARS_KEYS// / }) @@ -241,8 +246,12 @@ done if [ -n "${ADD_FINAL_STAGE}" ]; then ARCH_TYPE="\"type\":\"ARM_CONTAINER\",\"compute-type\":\"BUILD_GENERAL1_SMALL\"" + DEPEND_ON=",\"depend-on\":[${ALL_PROJECT_IDS%?}]" + if [ "$NO_DEPS_FOR_FINAL_STAGE" = "true" ]; then + DEPEND_ON="" + fi yq eval -i -P \ - ".batch.build-graph += [{\"identifier\":\"final_stage\",\"buildspec\":\"$ADD_FINAL_STAGE\",\"env\":{$ARCH_TYPE},\"depend-on\":[${ALL_PROJECT_IDS%?}]}]" \ + ".batch.build-graph += [{\"identifier\":\"final_stage\",\"buildspec\":\"$ADD_FINAL_STAGE\",\"env\":{$ARCH_TYPE}$DEPEND_ON}]" \ $STAGING_BUILDSPEC_FILE fi diff --git a/build/lib/helm_replace.sh b/build/lib/helm_replace.sh index 0299c66853..ea1374ec05 100755 --- a/build/lib/helm_replace.sh +++ b/build/lib/helm_replace.sh @@ -43,4 +43,4 @@ if [ -d ${OUTPUT_DIR}/helm/${CHART_NAME}/crds ]; then do build::common::echo_and_run $SED -f ${SEDFILE} -i ${DEST_DIR}/${file} done -fi \ No newline at end of file +fi diff --git a/build/lib/helm_require.sh b/build/lib/helm_require.sh index 289c793020..a9a5a30b35 100755 --- a/build/lib/helm_require.sh +++ b/build/lib/helm_require.sh @@ -28,7 +28,8 @@ LATEST="${7?Seventh argument is latest tag}" HELM_USE_UPSTREAM_IMAGE="${8?Eigth argument is bool determining whether to use cached upstream images}" PACKAGE_DEPENDENCIES="${9?Ninth argument is optional project dependencies}" FORCE_JSON_SCHEMA_FILE="${10?Tenth argument is optional schema file}" -HELM_IMAGE_LIST="${@:11}" +HELM_IMAGE_LIST="${11}" +HELM_IMAGE_TAG_LIST="${12}" CHART_NAME=$(basename ${HELM_DESTINATION_REPOSITORY}) DEST_DIR=${OUTPUT_DIR}/helm/${CHART_NAME} @@ -78,7 +79,7 @@ function get_image_tag_not_latest() { # to find another tag associated with this image we have to use the aws cli # the following only works for ecr repos if [ "${JOB_TYPE:-}" = "presubmit" ] || [[ "${IMAGE_REGISTRY}" != *"ecr"* ]]; then - echo ${tag} + echo ${tag} else if ! aws sts get-caller-identity &> /dev/null; then echo "The AWS cli is used to find the ECR registries and repos for the current AWS account please login!" @@ -87,63 +88,70 @@ function get_image_tag_not_latest() { local service="ecr" if [[ "${IMAGE_REGISTRY}" = *"public.ecr"* ]]; then - service="--region us-east-1 ecr-public" + service="--region us-east-1 ecr-public" fi - build::common::echo_and_run aws ${service} describe-images --repository-name ${image} --image-id imageDigest=${shasum} --query 'imageDetails[0].imageTags' --output yaml | grep -v ${LATEST} | head -1| sed -e 's/- //' + build::common::echo_and_run aws ${service} describe-images --repository-name ${image} --image-id imageDigest=${shasum} --query 'imageDetails[0].imageTags' --output yaml | grep -v ${LATEST} | head -1| sed -e 's/- //' fi } -for IMAGE in ${HELM_IMAGE_LIST:-}; do - # the image_list will include images built by the current project and potentially images built from - # other projects, ex: prometheus chart includes the node_exporter which is built seperately - # since each project is built independently and is tagged with the current HEAD commit hash - # images built via this current build may not be tagged exactly the same as images from other builds - # this code will first try to pull the image by the IMAGE_TAG and if that is not available - # it will fallback to the LATEST tag which follows the same pattern we use for artifacts on s3 - # in the event that the LATEST tag is used, the ecr api will be used to get a different tag, which - # should be the tag in the format -, this tag will be used in the requires.yaml - IMAGE_SHASUM=$(get_image_shasum ${IMAGE} ${IMAGE_TAG}) - - if [[ -z ${IMAGE_SHASUM} ]]; then - IMAGE_SHASUM=$(get_image_shasum ${IMAGE} ${LATEST}) - fi - if [[ -z ${IMAGE_SHASUM} ]]; then - echo "Neither ${IMAGE}@${IMAGE_TAG} nor ${IMAGE}@${LATEST} exists!" - exit 1 - fi - - echo "s,{{${IMAGE}}},${IMAGE_SHASUM},g" >>${SEDFILE} - if [ "${IMAGE_TAG}" = "${LATEST}" ]; then - # if finding an image from another project using the `latest` tag, find the image and a different tag associated with that image - USE_TAG=$(get_image_tag_not_latest ${IMAGE} ${IMAGE_SHASUM}) - if [[ -z ${USE_TAG} ]]; then - echo "non-${LATEST} tag does not exist for ${IMAGE}@${IMAGE_SHASUM}!" +if [ -n "$HELM_IMAGE_LIST" ]; then + HELM_IMAGE_ARR=($HELM_IMAGE_LIST) + HELM_IMAGE_TAG_ARR=($HELM_IMAGE_TAG_LIST) + for i in "${!HELM_IMAGE_ARR[@]}"; do + IMAGE="${HELM_IMAGE_ARR[$i]}" + TAG="${HELM_IMAGE_TAG_ARR[$i]}" + # the image_list will include images built by the current project and potentially images built from + # other projects, ex: prometheus chart includes the node_exporter which is built seperately + # since each project is built independently and is tagged with the current HEAD commit hash + # images built via this current build may not be tagged exactly the same as images from other builds + # this code will first try to pull the image by the IMAGE_TAG and if that is not available + # it will fallback to the LATEST tag which follows the same pattern we use for artifacts on s3 + # in the event that the LATEST tag is used, the ecr api will be used to get a different tag, which + # should be the tag in the format -, this tag will be used in the requires.yaml + IMAGE_SHASUM=$(get_image_shasum ${IMAGE} ${TAG}) + + if [[ -z ${IMAGE_SHASUM} ]]; then + IMAGE_SHASUM=$(get_image_shasum ${IMAGE} ${LATEST}) + fi + + if [[ -z ${IMAGE_SHASUM} ]]; then + echo "Neither ${IMAGE}@${TAG} nor ${IMAGE}@${LATEST} exists!" exit 1 - fi - else - USE_TAG=$IMAGE_TAG - fi - - # If HELM_USE_UPSTREAM_IMAGE is true, we are using images from upstream. - # Though we pull images directly from upstream for build tooling checks (i.e. - # get images shasums), we will use cached images in the helm charts. Cached - # images follow the convention of ${PROJECT_NAME}/${UPSTREAM_IMAGE_NAME}. - if [ "${HELM_USE_UPSTREAM_IMAGE}" == true ]; then - PROJECT_NAME=$(echo "$HELM_DESTINATION_REPOSITORY" | awk -F "/" '{print $1}') - IMAGE_REPO="${PROJECT_NAME}/${IMAGE}" - else - IMAGE_REPO="${IMAGE}" - fi + fi - cat >>${REQUIRES_FILE} <>${SEDFILE} + if [ "${TAG}" = "${LATEST}" ]; then + # if finding an image from another project using the `latest` tag, find the image and a different tag associated with that image + USE_TAG=$(get_image_tag_not_latest ${IMAGE} ${IMAGE_SHASUM}) + if [[ -z ${USE_TAG} ]]; then + echo "non-${LATEST} tag does not exist for ${IMAGE}@${IMAGE_SHASUM}!" + exit 1 + fi + else + USE_TAG=$TAG + fi + + # If HELM_USE_UPSTREAM_IMAGE is true, we are using images from upstream. + # Though we pull images directly from upstream for build tooling checks (i.e. + # get images shasums), we will use cached images in the helm charts. Cached + # images follow the convention of ${PROJECT_NAME}/${UPSTREAM_IMAGE_NAME}. + if [ "${HELM_USE_UPSTREAM_IMAGE}" == true ]; then + PROJECT_NAME=$(echo "$HELM_DESTINATION_REPOSITORY" | awk -F "/" '{print $1}') + IMAGE_REPO="${PROJECT_NAME}/${IMAGE}" + else + IMAGE_REPO="${IMAGE}" + fi + + cat >>${REQUIRES_FILE} < 1 { + releaseBranched = true + } + if releaseBranched { + supportedReleaseBranches, err := getSupportedReleaseBranches(buildToolingRepoPath) + if err != nil { + return fmt.Errorf("getting supported EKS Distro release branches: %v", err) + } + releaseBranch := os.Getenv(constants.ReleaseBranchEnvvar) + releaseBranchIndex := slices.Index(supportedReleaseBranches, releaseBranch) + currentVersion = repo.Versions[releaseBranchIndex] + } else { + currentVersion = repo.Versions[0] + } + var currentRevision string + var isTrackedByCommitHash bool if currentVersion.Tag != "" { currentRevision = currentVersion.Tag } else if currentVersion.Commit != "" { currentRevision = currentVersion.Commit - } - fullRepoName := fmt.Sprintf("%s/%s", org, repoName) - if displayOptions.ProjectName != "" && displayOptions.ProjectName != fullRepoName { - continue + isTrackedByCommitHash = true } // Get latest revision for the project from GitHub. - latestRevision, _, err := github.GetLatestRevision(client, org, repoName, currentRevision) + latestRevision, _, err := github.GetLatestRevision(client, org, repoName, currentRevision, branchName, isTrackedByCommitHash, releaseBranched) if err != nil { return fmt.Errorf("getting latest revision from GitHub: %v", err) } @@ -107,3 +157,15 @@ func Run(displayOptions *types.DisplayOptions) error { return nil } + +func getSupportedReleaseBranches(buildToolingRepoPath string) ([]string, error) { + supportedReleaseBranchesFilepath := filepath.Join(buildToolingRepoPath, constants.SupportedReleaseBranchesFile) + + supportedReleaseBranchesFileContents, err := os.ReadFile(supportedReleaseBranchesFilepath) + if err != nil { + return nil, fmt.Errorf("reading supported release branches file: %v", err) + } + supportedK8sVersions := strings.Split(strings.TrimRight(string(supportedReleaseBranchesFileContents), "\n"), "\n") + + return supportedK8sVersions, nil +} diff --git a/tools/version-tracker/pkg/commands/listprojects/listprojects.go b/tools/version-tracker/pkg/commands/listprojects/listprojects.go index d90e2f9c7d..4d494969c3 100644 --- a/tools/version-tracker/pkg/commands/listprojects/listprojects.go +++ b/tools/version-tracker/pkg/commands/listprojects/listprojects.go @@ -21,9 +21,21 @@ func Run() error { return fmt.Errorf("retrieving current working directory: %v", err) } + // Check if branch name environment variable has been set. + branchName, ok := os.LookupEnv(constants.BranchNameEnvVar) + if !ok { + branchName = constants.MainBranchName + } + + // Get base repository owner environment variable if set. + baseRepoOwner := os.Getenv(constants.BaseRepoOwnerEnvvar) + if baseRepoOwner == "" { + baseRepoOwner = constants.DefaultBaseRepoOwner + } + // Clone the eks-anywhere-build-tooling repository. buildToolingRepoPath := filepath.Join(cwd, constants.BuildToolingRepoName) - _, _, err = git.CloneRepo(constants.BuildToolingRepoURL, buildToolingRepoPath, "") + _, _, err = git.CloneRepo(fmt.Sprintf(constants.BuildToolingRepoURL, baseRepoOwner), buildToolingRepoPath, "", branchName) if err != nil { return fmt.Errorf("cloning build-tooling repo: %v", err) } @@ -39,7 +51,7 @@ func Run() error { var projectsList types.ProjectsList err = yaml.Unmarshal(contents, &projectsList) if err != nil { - return fmt.Errorf("unmarshaling upstream projects tracker file to YAML: %v", err) + return fmt.Errorf("unmarshalling upstream projects tracker file: %v", err) } var projectVersionInfoList []types.ProjectVersionInfo diff --git a/tools/version-tracker/pkg/commands/upgrade/upgrade.go b/tools/version-tracker/pkg/commands/upgrade/upgrade.go index d221e6ba24..e666c5e2fb 100644 --- a/tools/version-tracker/pkg/commands/upgrade/upgrade.go +++ b/tools/version-tracker/pkg/commands/upgrade/upgrade.go @@ -4,27 +4,53 @@ import ( "bufio" "bytes" "context" + "encoding/json" "fmt" "os" "os/exec" "path/filepath" + "regexp" + "slices" "strconv" "strings" + eksdistrorelease "github.com/aws/eks-distro-build-tooling/release/api/v1alpha1" + "github.com/ghodss/yaml" gogithub "github.com/google/go-github/v53/github" - "gopkg.in/yaml.v3" + "github.com/pelletier/go-toml/v2" + goyamlv3 "gopkg.in/yaml.v3" + sigsyaml "sigs.k8s.io/yaml" "github.com/aws/eks-anywhere-build-tooling/tools/version-tracker/pkg/constants" + "github.com/aws/eks-anywhere-build-tooling/tools/version-tracker/pkg/ecrpublic" "github.com/aws/eks-anywhere-build-tooling/tools/version-tracker/pkg/git" "github.com/aws/eks-anywhere-build-tooling/tools/version-tracker/pkg/github" "github.com/aws/eks-anywhere-build-tooling/tools/version-tracker/pkg/types" "github.com/aws/eks-anywhere-build-tooling/tools/version-tracker/pkg/util/command" + "github.com/aws/eks-anywhere-build-tooling/tools/version-tracker/pkg/util/file" "github.com/aws/eks-anywhere-build-tooling/tools/version-tracker/pkg/util/logger" - "github.com/aws/eks-anywhere-build-tooling/tools/version-tracker/pkg/util/slices" ) // Run contains the business logic to execute the `upgrade` subcommand. func Run(upgradeOptions *types.UpgradeOptions) error { + var currentRevision, latestRevision string + var isTrackedByCommitHash, patchApplySucceeded, addPatchWarningComment bool + var totalPatchCount int + var updatedFiles []string + patchesWarningComment := constants.PatchesCommentBody + + projectName := upgradeOptions.ProjectName + + // Get org and repository name from project name. + projectOrg := strings.Split(projectName, "/")[0] + projectRepo := strings.Split(projectName, "/")[1] + + // Check if branch name environment variable has been set. + branchName, ok := os.LookupEnv(constants.BranchNameEnvVar) + if !ok { + branchName = constants.MainBranchName + } + // Check if base repository owner environment variable has been set. baseRepoOwner, ok := os.LookupEnv(constants.BaseRepoOwnerEnvvar) if !ok { @@ -55,14 +81,14 @@ func Run(upgradeOptions *types.UpgradeOptions) error { return fmt.Errorf("reading skipped projects file: %v", err) } skippedProjects := strings.Split(string(contents), "\n") - if slices.Contains(skippedProjects, upgradeOptions.ProjectName) { + if slices.Contains(skippedProjects, projectName) { logger.Info("Project is in SKIPPED_PROJECTS list. Skipping upgrade") return nil } // Clone the eks-anywhere-build-tooling repository. buildToolingRepoPath := filepath.Join(cwd, constants.BuildToolingRepoName) - repo, headCommit, err := git.CloneRepo(constants.BuildToolingRepoURL, buildToolingRepoPath, headRepoOwner) + repo, headCommit, err := git.CloneRepo(fmt.Sprintf(constants.BuildToolingRepoURL, baseRepoOwner), buildToolingRepoPath, headRepoOwner, branchName) if err != nil { return fmt.Errorf("cloning build-tooling repo: %v", err) } @@ -73,167 +99,314 @@ func Run(upgradeOptions *types.UpgradeOptions) error { return fmt.Errorf("getting repo's current worktree: %v", err) } - // Validate if the project name provided exists in the repository. - projectPath := filepath.Join("projects", upgradeOptions.ProjectName) - projectRootFilepath := filepath.Join(buildToolingRepoPath, projectPath) - if _, err := os.Stat(projectRootFilepath); os.IsNotExist(err) { - return fmt.Errorf("invalid project name %s", upgradeOptions.ProjectName) - } - - // Check if project to be upgraded has patches - projectHasPatches := false - if _, err := os.Stat(filepath.Join(projectRootFilepath, constants.PatchesDirectory)); err == nil { - projectHasPatches = true - } - - // Get org and repository name from project name. - projectOrg := strings.Split(upgradeOptions.ProjectName, "/")[0] - projectRepo := strings.Split(upgradeOptions.ProjectName, "/")[1] - - // Load upstream projects tracker file. - upstreamProjectsTrackerFilePath := filepath.Join(buildToolingRepoPath, constants.UpstreamProjectsTrackerFile) - _, targetRepo, err := loadUpstreamProjectsTrackerFile(upstreamProjectsTrackerFilePath, projectOrg, projectRepo) + // Checkout the eks-anywhere-build-tooling repo at the provided branch name. + createBranch := (branchName != constants.MainBranchName) + err = git.Checkout(worktree, branchName, createBranch) if err != nil { - return fmt.Errorf("loading upstream projects tracker file: %v", err) - } - - // Validate whether the given project is release-branched. - if len(targetRepo.Versions) > 1 { - return fmt.Errorf("release-branched projects not supported at this time") - } - - currentVersion := targetRepo.Versions[0] - // Validate whether the project builds off a commit hash instead of a tag. - if currentVersion.Tag == "" { - return fmt.Errorf("projects tracked with commit hashes not supported at this time") + return fmt.Errorf("checking out worktree at branch %s: %v", branchName, err) } - currentRevision := currentVersion.Tag - // Get latest revision for the project from GitHub. - latestRevision, needsUpgrade, err := github.GetLatestRevision(client, projectOrg, projectRepo, currentRevision) + // Reset current worktree to get a clean index. + err = git.ResetToHEAD(worktree, headCommit) if err != nil { - return fmt.Errorf("getting latest revision from GitHub: %v", err) + return fmt.Errorf("resetting new branch to [origin/%s] HEAD: %v", branchName, err) } - // Upgrade project if latest commit was made after current commit and the semver of the latest revision is - // greater than the semver of the current version. - if needsUpgrade { - logger.Info("Project is out of date.", "Current version", currentRevision, "Latest version", latestRevision) - - var updatedFiles []string - headBranchName := fmt.Sprintf("update-%s-%s", projectOrg, projectRepo) - baseBranchName := constants.MainBranchName + var headBranchName, baseBranchName, commitMessage, pullRequestBody string + if isEKSDistroUpgrade(projectName) { + headBranchName = fmt.Sprintf("update-eks-distro-latest-releases-%s", branchName) + baseBranchName = branchName + commitMessage = "Bump EKS Distro releases to latest" + pullRequestBody = constants.EKSDistroUpgradePullRequestBody // Checkout a new branch to keep track of version upgrade chaneges. - err = git.Checkout(worktree, headBranchName) + err = git.Checkout(worktree, headBranchName, true) if err != nil { - return fmt.Errorf("getting repo's current worktree: %v", err) + return fmt.Errorf("checking out worktree at branch %s: %v", headBranchName, err) } // Reset current worktree to get a clean index. - err = git.ResetToMain(worktree, headCommit) + err = git.ResetToHEAD(worktree, headCommit) if err != nil { - return fmt.Errorf("resetting new branch to [origin/main] HEAD: %v", err) + return fmt.Errorf("resetting new branch to [origin/%s] HEAD: %v", branchName, err) } - // Reload upstream projects tracker file to get its original value instead of - // the updated one from another project's previous upgrade - projectsList, targetRepo, err := loadUpstreamProjectsTrackerFile(upstreamProjectsTrackerFilePath, projectOrg, projectRepo) + isUpdated, err := updateEKSDistroReleasesFile(client, buildToolingRepoPath) if err != nil { - return fmt.Errorf("reloading upstream projects tracker file: %v", err) + return fmt.Errorf("updating EKS Distro releases file: %v", err) + } + if isUpdated { + updatedFiles = append(updatedFiles, constants.EKSDistroLatestReleasesFile) + } + } else { + // Validate if the project name provided exists in the repository. + projectPath := filepath.Join("projects", projectName) + projectRootFilepath := filepath.Join(buildToolingRepoPath, projectPath) + if _, err := os.Stat(projectRootFilepath); os.IsNotExist(err) { + return fmt.Errorf("invalid project name %s", projectName) } - targetRepo.Versions[0].Tag = latestRevision - // Update the Git tag file corresponding to the project - logger.Info("Updating Git tag file corresponding to the project") - projectGitTagRelativePath, err := updateProjectVersionFile(buildToolingRepoPath, constants.GitTagFile, upgradeOptions.ProjectName, latestRevision) + // Load upstream projects tracker file. + upstreamProjectsTrackerFilePath := filepath.Join(buildToolingRepoPath, constants.UpstreamProjectsTrackerFile) + _, targetRepo, err := loadUpstreamProjectsTrackerFile(upstreamProjectsTrackerFilePath, projectOrg, projectRepo) if err != nil { - return fmt.Errorf("updating project GIT_TAG file: %v", err) + return fmt.Errorf("loading upstream projects tracker file: %v", err) } - updatedFiles = append(updatedFiles, projectGitTagRelativePath) - var latestGoVersion string - if currentVersion.GoVersion != "N/A" { - currentGoVersion := currentVersion.GoVersion - // Get Go version corresponding to the latest revision of the project. - latestGoVersion, err := github.GetGoVersionForLatestRevision(client, projectOrg, projectRepo, latestRevision) + // Validate whether the given project is release-branched. + var isReleaseBranched bool + var currentVersion types.Version + var versionIndex int + if len(targetRepo.Versions) > 1 { + isReleaseBranched = true + } + releaseBranch := os.Getenv(constants.ReleaseBranchEnvvar) + if releaseBranch == "" { + releaseBranch, err = getDefaultReleaseBranch(buildToolingRepoPath) if err != nil { - return fmt.Errorf("getting latest Go version for release %s: %v", latestRevision, err) + return fmt.Errorf("getting default EKS Distro release branch: %v", err) } - - // Get the minor version for the current revision's Go version. - currentGoMinorVersion, err := strconv.Atoi(strings.Split(currentGoVersion, ".")[1]) + } + if isReleaseBranched { + supportedReleaseBranches, err := getSupportedReleaseBranches(buildToolingRepoPath) if err != nil { - return fmt.Errorf("getting current Go minor version: %v", err) + return fmt.Errorf("getting supported EKS Distro release branches: %v", err) } - // Get the major version for the latest revision's Go version. - latestGoMinorVersion, err := strconv.Atoi(strings.Split(latestGoVersion, ".")[1]) + versionIndex = slices.Index(supportedReleaseBranches, releaseBranch) + } else { + versionIndex = 0 + } + currentVersion = targetRepo.Versions[versionIndex] + + if currentVersion.Tag != "" { + currentRevision = currentVersion.Tag + } else if currentVersion.Commit != "" { + currentRevision = currentVersion.Commit + isTrackedByCommitHash = true + } + + // Check if project to be upgraded has patches + projectHasPatches := false + patchesDirectory := filepath.Join(projectRootFilepath, constants.PatchesDirectory) + if isReleaseBranched { + patchesDirectory = filepath.Join(projectRootFilepath, releaseBranch, constants.PatchesDirectory) + } + if _, err := os.Stat(patchesDirectory); err == nil { + projectHasPatches = true + patchFiles, err := os.ReadDir(patchesDirectory) if err != nil { - return fmt.Errorf("getting latest Go minor version: %v", err) + return fmt.Errorf("reading patches directory: %v", err) } + totalPatchCount = len(patchFiles) + } - // If the Go version has been updated in the latest revision, then update the Go version file corresponding to the project. - if latestGoMinorVersion > currentGoMinorVersion { - logger.Info("Project Go version needs to be updated.", "Current Go version", currentGoVersion, "Latest Go version", latestGoVersion) - targetRepo.Versions[0].GoVersion = latestGoVersion + headBranchName = fmt.Sprintf("update-%s-%s-%s", projectOrg, projectRepo, branchName) + baseBranchName = branchName + commitMessage = fmt.Sprintf("Bump %s to latest release", projectName) + if isReleaseBranched { + headBranchName = fmt.Sprintf("update-%s-%s-%s-%s", projectOrg, projectRepo, releaseBranch, branchName) + commitMessage = fmt.Sprintf("Bump %s %s release branch to latest release", projectName, releaseBranch) + } - logger.Info("Updating Go version file corresponding to the project") - projectGoVersionRelativePath, err := updateProjectVersionFile(buildToolingRepoPath, constants.GoVersionFile, upgradeOptions.ProjectName, latestGoVersion) - if err != nil { - return fmt.Errorf("updating project GOLANG_VERSION file: %v", err) - } - updatedFiles = append(updatedFiles, projectGoVersionRelativePath) + var latestRevision string + var needsUpgrade bool + if projectName == "cilium/cilium" { + latestRevision, needsUpgrade, err = ecrpublic.GetLatestRevision(constants.CiliumImageRepository, currentRevision) + if err != nil { + return fmt.Errorf("getting latest revision from ECR Public: %v", err) } } else { - latestGoVersion = "N/A" - targetRepo.Versions[0].GoVersion = latestGoVersion + // Get latest revision for the project from GitHub. + latestRevision, needsUpgrade, err = github.GetLatestRevision(client, projectOrg, projectRepo, currentRevision, branchName, isTrackedByCommitHash, isReleaseBranched) + if err != nil { + return fmt.Errorf("getting latest revision from GitHub: %v", err) + } } - // Update the tag and Go version in the section of the upstream projects tracker file corresponding to the given project. - logger.Info("Updating Git tag and Go version in upstream projects tracker file") - err = updateUpstreamProjectsTrackerFile(&projectsList, targetRepo, buildToolingRepoPath, upstreamProjectsTrackerFilePath, latestRevision, latestGoVersion) - if err != nil { - return fmt.Errorf("updating upstream projects tracker file: %v", err) - } - updatedFiles = append(updatedFiles, constants.UpstreamProjectsTrackerFile) + pullRequestBody = fmt.Sprintf(constants.DefaultUpgradePullRequestBody, projectOrg, projectRepo, currentRevision, latestRevision) - // Update the version in the project's README file. - logger.Info("Updating project README file") - projectReadmePath := filepath.Join(projectPath, constants.ReadmeFile) - err = updateProjectReadmeVersion(buildToolingRepoPath, projectOrg, projectRepo) - if err != nil { - return fmt.Errorf("updating version in project README: %v", err) - } - updatedFiles = append(updatedFiles, projectReadmePath) + // Upgrade project if latest commit was made after current commit and the semver of the latest revision is + // greater than the semver of the current version. + if needsUpgrade || slices.Contains(constants.ProjectsWithUnconventionalUpgradeFlows, projectName) { + // Checkout a new branch to keep track of version upgrade chaneges. + err = git.Checkout(worktree, headBranchName, true) + if err != nil { + return fmt.Errorf("checking out worktree at branch %s: %v", headBranchName, err) + } + + // Reset current worktree to get a clean index. + err = git.ResetToHEAD(worktree, headCommit) + if err != nil { + return fmt.Errorf("resetting new branch to [origin/%s] HEAD: %v", branchName, err) + } + + if needsUpgrade { + logger.Info("Project is out of date.", "Current version", currentRevision, "Latest version", latestRevision) + + // Reload upstream projects tracker file to get its original value instead of + // the updated one from another project's previous upgrade + projectsList, targetRepo, err := loadUpstreamProjectsTrackerFile(upstreamProjectsTrackerFilePath, projectOrg, projectRepo) + if err != nil { + return fmt.Errorf("reloading upstream projects tracker file: %v", err) + } + if isTrackedByCommitHash { + targetRepo.Versions[versionIndex].Commit = latestRevision + } else { + targetRepo.Versions[versionIndex].Tag = latestRevision + } + + // Update the Git tag file corresponding to the project + logger.Info("Updating Git tag file corresponding to the project") + projectGitTagRelativePath, err := updateProjectVersionFile(buildToolingRepoPath, constants.GitTagFile, projectName, latestRevision, releaseBranch, isReleaseBranched) + if err != nil { + return fmt.Errorf("updating project GIT_TAG file: %v", err) + } + updatedFiles = append(updatedFiles, projectGitTagRelativePath) + + var latestGoVersion string + if currentVersion.GoVersion != "N/A" { + currentGoVersion := currentVersion.GoVersion + // Get Go version corresponding to the latest revision of the project. + latestGoVersion, err := github.GetGoVersionForLatestRevision(client, projectOrg, projectRepo, latestRevision) + if err != nil { + return fmt.Errorf("getting latest Go version for release %s: %v", latestRevision, err) + } + + // Get the minor version for the current revision's Go version. + currentGoMinorVersion, err := strconv.Atoi(strings.Split(currentGoVersion, ".")[1]) + if err != nil { + return fmt.Errorf("getting current Go minor version: %v", err) + } + + // Get the major version for the latest revision's Go version. + latestGoMinorVersion, err := strconv.Atoi(strings.Split(latestGoVersion, ".")[1]) + if err != nil { + return fmt.Errorf("getting latest Go minor version: %v", err) + } + + // If the Go version has been updated in the latest revision, then update the Go version file corresponding to the project. + if latestGoMinorVersion > currentGoMinorVersion { + logger.Info("Project Go version needs to be updated.", "Current Go version", currentGoVersion, "Latest Go version", latestGoVersion) + targetRepo.Versions[versionIndex].GoVersion = latestGoVersion + + logger.Info("Updating Go version file corresponding to the project") + projectGoVersionRelativePath, err := updateProjectVersionFile(buildToolingRepoPath, constants.GoVersionFile, projectName, latestGoVersion, releaseBranch, isReleaseBranched) + if err != nil { + return fmt.Errorf("updating project GOLANG_VERSION file: %v", err) + } + updatedFiles = append(updatedFiles, projectGoVersionRelativePath) + } + } else { + latestGoVersion = "N/A" + targetRepo.Versions[versionIndex].GoVersion = latestGoVersion + } + + // Update the tag and Go version in the section of the upstream projects tracker file corresponding to the given project. + logger.Info("Updating Git tag and Go version in upstream projects tracker file") + err = updateUpstreamProjectsTrackerFile(&projectsList, targetRepo, buildToolingRepoPath, upstreamProjectsTrackerFilePath, latestRevision, latestGoVersion) + if err != nil { + return fmt.Errorf("updating upstream projects tracker file: %v", err) + } + updatedFiles = append(updatedFiles, constants.UpstreamProjectsTrackerFile) - // Update the checksums file and attribution file(s) corresponding to the project. - if !projectHasPatches { - if _, err := os.Stat(filepath.Join(projectRootFilepath, constants.ChecksumsFile)); err == nil { - logger.Info("Updating project checksums and attribution files") - projectChecksumsFileRelativePath := filepath.Join(projectPath, constants.ChecksumsFile) - err = updateChecksumsAttributionFiles(projectRootFilepath) + // Update the version in the project's README file. + logger.Info("Updating project README file") + projectReadmePath := filepath.Join(projectPath, constants.ReadmeFile) + err = updateProjectReadmeVersion(buildToolingRepoPath, projectOrg, projectRepo) if err != nil { - return fmt.Errorf("updating project checksums and attribution files: %v", err) + return fmt.Errorf("updating version in project README: %v", err) + } + updatedFiles = append(updatedFiles, projectReadmePath) + + // If project has patches, attempt to apply them. Track failed patches and files that failed to apply, if any. + if projectHasPatches { + appliedPatchesCount, failedPatch, applyFailedFiles, err := applyPatchesToRepo(projectRootFilepath, projectRepo, releaseBranch, totalPatchCount) + if appliedPatchesCount == totalPatchCount { + patchApplySucceeded = true + } + if err != nil { + return fmt.Errorf("applying patches to repository: %v", err) + } + if !patchApplySucceeded { + addPatchWarningComment = true + patchesWarningComment = fmt.Sprintf(constants.PatchesCommentBody, appliedPatchesCount, totalPatchCount, failedPatch, applyFailedFiles) + } + } + + // If project doesn't have patches, or it does and they were applied successfully, then update the checksums file + // and attribution file(s) corresponding to the project. + if !projectHasPatches || patchApplySucceeded { + projectChecksumsFile := filepath.Join(projectRootFilepath, constants.ChecksumsFile) + projectChecksumsFileRelativePath := filepath.Join(projectPath, constants.ChecksumsFile) + projectAttributionFileGlob := filepath.Join(projectRootFilepath, constants.AttributionsFilePattern) + if isReleaseBranched { + projectChecksumsFile = filepath.Join(projectRootFilepath, releaseBranch, constants.ChecksumsFile) + projectChecksumsFileRelativePath = filepath.Join(projectPath, releaseBranch, constants.ChecksumsFile) + projectAttributionFileGlob = filepath.Join(projectRootFilepath, releaseBranch, constants.AttributionsFilePattern) + } + if _, err := os.Stat(projectChecksumsFile); err == nil { + logger.Info("Updating project checksums and attribution files") + err = updateChecksumsAttributionFiles(projectRootFilepath) + if err != nil { + return fmt.Errorf("updating project checksums and attribution files: %v", err) + } + updatedFiles = append(updatedFiles, projectChecksumsFileRelativePath) + + // Attribution files can have a binary name prefix so we use a common prefix regular expression + // and glob them to cover all possibilities. + projectAttributionFileGlob, err := filepath.Glob(projectAttributionFileGlob) + if err != nil { + return fmt.Errorf("finding filenames matching attribution file pattern [%s]: %v", constants.AttributionsFilePattern, err) + } + for _, attributionFile := range projectAttributionFileGlob { + attributionFileRelativePath, err := filepath.Rel(buildToolingRepoPath, attributionFile) + if err != nil { + return fmt.Errorf("getting relative path for attribution file: %v", err) + } + updatedFiles = append(updatedFiles, attributionFileRelativePath) + } + } + } + + if projectName == "cilium/cilium" { + updatedCiliumImageDigestFiles, err := updateCiliumImageDigestFiles(projectRootFilepath, projectPath) + if err != nil { + return fmt.Errorf("updating Cilium image digest files: %v", err) + } + updatedFiles = append(updatedFiles, updatedCiliumImageDigestFiles...) } - updatedFiles = append(updatedFiles, projectChecksumsFileRelativePath) + } - // Attribution files can have a binary name prefix so we use a common prefix regular expression - // and glob them to cover all possibilities. - projectAttributionFileGlob, err := filepath.Glob(filepath.Join(projectRootFilepath, constants.AttributionsFilePattern)) + if projectName == "kubernetes-sigs/image-builder" { + currentBottlerocketVersion, latestBottlerocketVersion, updatedBRFiles, err := updateBottlerocketVersionFiles(client, projectRootFilepath, projectPath, branchName) if err != nil { - return fmt.Errorf("finding filenames matching attribution file pattern [%s]: %v", constants.AttributionsFilePattern, err) + return fmt.Errorf("updating Bottlerocket version and metadata files: %v", err) } - for _, attributionFile := range projectAttributionFileGlob { - attributionFileRelativePath, err := filepath.Rel(buildToolingRepoPath, attributionFile) + if len(updatedBRFiles) > 0 { + updatedFiles = append(updatedFiles, updatedBRFiles...) + if len(updatedFiles) == len(updatedBRFiles) { + headBranchName = "update-bottlerocket-releases" + commitMessage = "Bump Bottlerocket versions to latest release" + pullRequestBody = fmt.Sprintf(constants.BottlerocketUpgradePullRequestBody, currentBottlerocketVersion, latestBottlerocketVersion) + } else { + headBranchName = fmt.Sprintf("update-%s-%s-and-bottlerocket", projectOrg, projectRepo) + commitMessage = fmt.Sprintf("Bump %s and Bottlerocket versions to latest release", projectName) + pullRequestBody = fmt.Sprintf(constants.CombinedImageBuilderBottlerocketUpgradePullRequestBody, currentRevision, latestRevision, currentBottlerocketVersion, latestBottlerocketVersion) + } + + err = git.Checkout(worktree, headBranchName, true) if err != nil { - return fmt.Errorf("getting relative path for attribution file: %v", err) + return fmt.Errorf("checking out worktree at branch %s: %v", headBranchName, err) } - updatedFiles = append(updatedFiles, attributionFileRelativePath) } } + } else if latestRevision == currentRevision { + logger.Info("Project is at the latest available version.", "Current version", currentRevision, "Latest version", latestRevision) } + } + if len(updatedFiles) > 0 { // Add all the updated files to the index. err = git.Add(worktree, updatedFiles) if err != nil { @@ -241,38 +414,137 @@ func Run(upgradeOptions *types.UpgradeOptions) error { } // Create a new commit including the updated files, with an appropriate commit message. - err = git.Commit(worktree, fmt.Sprintf("Bump %s to latest release", upgradeOptions.ProjectName)) + err = git.Commit(worktree, commitMessage) if err != nil { - return fmt.Errorf("committing updated project version files for [%s] project: %v", upgradeOptions.ProjectName, err) + return fmt.Errorf("committing updated project version files for [%s] project: %v", projectName, err) } if upgradeOptions.DryRun { - logger.Info(fmt.Sprintf("Completed dry run of upgrade for project %s", upgradeOptions.ProjectName)) + logger.Info(fmt.Sprintf("Completed dry run of upgrade for project %s", projectName)) return nil } // Push the changes to the target branch in the head repository. err = git.Push(repo, headRepoOwner, headBranchName, githubToken) if err != nil { - return fmt.Errorf("pushing updated project version files for [%s] project: %v", upgradeOptions.ProjectName, err) + return fmt.Errorf("pushing updated project version files for [%s] project: %v", projectName, err) } // Create a pull request from the bramch in the head repository to the target branch in the aws/eks-anywhere-build-tooling repository. logger.Info("Creating pull request with updated files") - err = github.CreatePullRequest(client, projectOrg, projectRepo, baseRepoOwner, baseBranchName, headRepoOwner, headBranchName, currentRevision, latestRevision, projectHasPatches) + err = github.CreatePullRequest(client, projectOrg, projectRepo, commitMessage, pullRequestBody, baseRepoOwner, baseBranchName, headRepoOwner, headBranchName, currentRevision, latestRevision, addPatchWarningComment, patchesWarningComment) if err != nil { return fmt.Errorf("creating pull request to %s repository: %v", constants.BuildToolingRepoName, err) } - } else if latestRevision == currentRevision { - logger.Info("Project is at the latest available version.", "Current version", currentRevision, "Latest version", latestRevision) } return nil } +func updateEKSDistroReleasesFile(client *gogithub.Client, buildToolingRepoPath string) (bool, error) { + var isUpdated bool + eksDistroReleasesFilepath := filepath.Join(buildToolingRepoPath, constants.EKSDistroLatestReleasesFile) + + eksDistroReleasesFileContents, err := os.ReadFile(eksDistroReleasesFilepath) + if err != nil { + return false, fmt.Errorf("reading EKS Distro latest releases file: %v", err) + } + + var eksDistroLatestReleases types.EKSDistroLatestReleases + err = yaml.Unmarshal(eksDistroReleasesFileContents, &eksDistroLatestReleases) + if err != nil { + return false, fmt.Errorf("unmarshalling EKS Distro latest releases file: %v", err) + } + + supportedReleaseBranches, err := getSupportedReleaseBranches(buildToolingRepoPath) + if err != nil { + return false, fmt.Errorf("getting supported EKS Distro release branches: %v", err) + } + + for i := range eksDistroLatestReleases.Releases { + if slices.Contains(supportedReleaseBranches, eksDistroLatestReleases.Releases[i].Branch) { + number, kubeVersion, err := getLatestEKSDistroRelease(eksDistroLatestReleases.Releases[i].Branch) + if err != nil { + return false, fmt.Errorf("getting latest EKS Distro release for %s branch: %v", eksDistroLatestReleases.Releases[i].Branch, err) + } + if eksDistroLatestReleases.Releases[i].Number != number || eksDistroLatestReleases.Releases[i].KubeVersion != kubeVersion { + isUpdated = true + eksDistroLatestReleases.Releases[i].Number = number + eksDistroLatestReleases.Releases[i].KubeVersion = kubeVersion + } + } + } + + if isUpdated { + updatedEKSDistroReleasesFileContents, err := yaml.Marshal(eksDistroLatestReleases) + if err != nil { + return false, fmt.Errorf("marshalling EKS Distro latest releases: %v", err) + } + + err = os.WriteFile(eksDistroReleasesFilepath, updatedEKSDistroReleasesFileContents, 0o644) + if err != nil { + return false, fmt.Errorf("writing EKS Distro latest releases file: %v", err) + } + } + + return isUpdated, nil +} + +func getSupportedReleaseBranches(buildToolingRepoPath string) ([]string, error) { + supportedReleaseBranchesFilepath := filepath.Join(buildToolingRepoPath, constants.SupportedReleaseBranchesFile) + + supportedReleaseBranchesFileContents, err := os.ReadFile(supportedReleaseBranchesFilepath) + if err != nil { + return nil, fmt.Errorf("reading supported release branches file: %v", err) + } + supportedK8sVersions := strings.Split(strings.TrimRight(string(supportedReleaseBranchesFileContents), "\n"), "\n") + + return supportedK8sVersions, nil +} + +func getLatestEKSDistroRelease(branch string) (int, string, error) { + var eksDistroReleaseChannel eksdistrorelease.ReleaseChannel + var eksDistroRelease eksdistrorelease.Release + var kubeVersion string + + eksDistroReleaseChannelsFileURL := fmt.Sprintf(constants.EKSDistroReleaseChannelsFileURLFormat, branch) + eksDistroReleaseChannelsFileContents, err := file.ReadURL(eksDistroReleaseChannelsFileURL) + if err != nil { + return 0, "", fmt.Errorf("reading EKS Distro ReleaseChannels file URL: %v", err) + } + + err = sigsyaml.Unmarshal(eksDistroReleaseChannelsFileContents, &eksDistroReleaseChannel) + if err != nil { + return 0, "", fmt.Errorf("unmarshalling EKS Distro ReleaseChannels file: %v", err) + } + releaseNumber := eksDistroReleaseChannel.Status.LatestRelease + + eksDistroReleaseManifestURL := fmt.Sprintf(constants.EKSDistroReleaseManifestURLFormat, branch, releaseNumber) + eksDistroReleaseManifestContents, err := file.ReadURL(eksDistroReleaseManifestURL) + if err != nil { + return 0, "", fmt.Errorf("reading EKS Distro release manifest URL: %v", err) + } + + err = sigsyaml.Unmarshal(eksDistroReleaseManifestContents, &eksDistroRelease) + if err != nil { + return 0, "", fmt.Errorf("unmarshalling EKS Distro release manifest: %v", err) + } + for _, component := range eksDistroRelease.Status.Components { + if component.Name == "kubernetes" { + kubeVersion = component.GitTag + break + } + } + + return releaseNumber, kubeVersion, nil +} + // updateProjectVersionFile updates the version information stored in a specific file. -func updateProjectVersionFile(buildToolingRepoPath, filename, projectName, value string) (string, error) { +func updateProjectVersionFile(buildToolingRepoPath, filename, projectName, value, releaseBranch string, isReleaseBranched bool) (string, error) { fileRelativepath := filepath.Join("projects", projectName, filename) + if isReleaseBranched { + fileRelativepath = filepath.Join("projects", projectName, releaseBranch, filename) + } fileAbsolutepath := filepath.Join(buildToolingRepoPath, fileRelativepath) fileAbsolutePathStat, err := os.Stat(fileAbsolutepath) if err != nil { @@ -294,9 +566,9 @@ func loadUpstreamProjectsTrackerFile(upstreamProjectsTrackerFilePath, org, repos } var projectsList types.ProjectsList - err = yaml.Unmarshal(contents, &projectsList) + err = goyamlv3.Unmarshal(contents, &projectsList) if err != nil { - return types.ProjectsList{}, types.Repo{}, fmt.Errorf("unmarshaling upstream projects tracker file to YAML: %v", err) + return types.ProjectsList{}, types.Repo{}, fmt.Errorf("unmarshalling upstream projects tracker file: %v", err) } var targetRepo types.Repo @@ -340,7 +612,7 @@ func updateUpstreamProjectsTrackerFile(projectsList *types.ProjectsList, targetR b.Write([]byte("\n")) // Create a new YAML encoder with an appropriate indentation value and encode the project list into a byte buufer - yamlEncoder := yaml.NewEncoder(&b) + yamlEncoder := goyamlv3.NewEncoder(&b) yamlEncoder.SetIndent(2) yamlEncoder.Encode(&projectsList) @@ -352,6 +624,57 @@ func updateUpstreamProjectsTrackerFile(projectsList *types.ProjectsList, targetR return nil } +// applyPatchesToRepo runs a Make command to apply patches to the cloned repository of the project +// being upgraded. +func applyPatchesToRepo(projectRootFilepath, projectRepo, releaseBranch string, totalPatchCount int) (int, string, string, error) { + var patchesApplied int + var failedPatch, failedFilesInPatch string + patchApplySucceeded := true + + applyPatchesCommandSequence := fmt.Sprintf("RELEASE_BRANCH=%s make -C %s patch-repo", releaseBranch, projectRootFilepath) + applyPatchesCmd := exec.Command("bash", "-c", applyPatchesCommandSequence) + applyPatchesOutput, err := command.ExecCommand(applyPatchesCmd) + if err != nil { + if strings.Contains(applyPatchesOutput, constants.FailedPatchApplyMarker) { + patchApplySucceeded = false + } else { + return 0, "", "", fmt.Errorf("running patch-repo Make command: %v", err) + } + } + + if patchApplySucceeded { + patchesApplied = totalPatchCount + } else { + failedFiles := []string{} + gitDescribeRegex := regexp.MustCompile(`v?\d+\.\d+\.\d+(-([0-9]+)-g.*)?`) + gitDescribeCmd := exec.Command("git", "-C", filepath.Join(projectRootFilepath, projectRepo), "describe", "--tag") + gitDescribeOutput, err := command.ExecCommand(gitDescribeCmd) + if err != nil { + return 0, "", "", fmt.Errorf("running git describe command: %v", err) + } + gitDescribeMatches := gitDescribeRegex.FindStringSubmatch(gitDescribeOutput) + if gitDescribeMatches[1] != "" { + patchesApplied, err = strconv.Atoi(gitDescribeMatches[2]) + if err != nil { + return 0, "", "", fmt.Errorf("converting patch count to integer %v", err) + } + } + + failedPatchRegex := regexp.MustCompile(constants.FailedPatchApplyRegex) + failedPatch = failedPatchRegex.FindString(applyPatchesOutput) + + failedPatchFileRegex := regexp.MustCompile(constants.FailedPatchFilesRegex) + applyFailedFiles := failedPatchFileRegex.FindAllStringSubmatch(applyPatchesOutput, -1) + for _, files := range applyFailedFiles { + failedFiles = append(failedFiles, fmt.Sprintf("`%s`", files[1])) + } + + failedFilesInPatch = strings.Join(failedFiles, ",") + } + + return patchesApplied, failedPatch, failedFilesInPatch, nil +} + // updateChecksumsAttributionFiles runs a Make command to update the checksums and attribution files // corresponding to the project being upgraded. func updateChecksumsAttributionFiles(projectRootFilepath string) error { @@ -365,7 +688,7 @@ func updateChecksumsAttributionFiles(projectRootFilepath string) error { return nil } -// updateChecksumsAttributionFiles runs a script to update the version in the README file corresponding +// updateProjectReadmeVersion runs a script to update the version in the README file corresponding // to the project being upgraded. func updateProjectReadmeVersion(buildToolingRepoPath, projectOrg, projectRepo string) error { readmeUpdateScriptFilepath := filepath.Join(buildToolingRepoPath, constants.ReadmeUpdateScriptFile) @@ -378,3 +701,222 @@ func updateProjectReadmeVersion(buildToolingRepoPath, projectOrg, projectRepo st return nil } + +func updateCiliumImageDigestFiles(projectRootFilepath, projectPath string) ([]string, error) { + updateCiliumFiles := []string{} + updateDigestsCommandSequence := fmt.Sprintf("make -C %s update-digests", projectRootFilepath) + updateDigestsCmd := exec.Command("bash", "-c", updateDigestsCommandSequence) + _, err := command.ExecCommand(updateDigestsCmd) + if err != nil { + return nil, fmt.Errorf("running update-digests Make command: %v", err) + } + + for _, directory := range constants.CiliumImageDirectories { + updateCiliumFiles = append(updateCiliumFiles, filepath.Join(projectPath, "images", directory, "IMAGE_DIGEST")) + } + return updateCiliumFiles, nil +} + +func updateBottlerocketVersionFiles(client *gogithub.Client, projectRootFilepath, projectPath, branchName string) (string, string, []string, error) { + updatedBRFiles := []string{} + var bottlerocketReleaseMap map[string]interface{} + bottlerocketReleasesFilePath := filepath.Join(projectRootFilepath, constants.BottlerocketReleasesFile) + bottlerocketReleasesRelativeFilePath := filepath.Join(projectPath, constants.BottlerocketReleasesFile) + bottlerocketReleasesFileContents, err := os.ReadFile(bottlerocketReleasesFilePath) + if err != nil { + return "", "", nil, fmt.Errorf("reading Bottlerocket releases file: %v", err) + } + + err = yaml.Unmarshal(bottlerocketReleasesFileContents, &bottlerocketReleaseMap) + if err != nil { + return "", "", nil, fmt.Errorf("unmarshalling Bottlerocket releases file: %v", err) + } + + var currentBottlerocketVersion string + for channel := range bottlerocketReleaseMap { + for _, format := range constants.BottlerocketImageFormats { + releaseVersionByFormat := bottlerocketReleaseMap[channel].(map[string]interface{})[fmt.Sprintf("%s-release-version", format)] + if releaseVersionByFormat != nil { + currentBottlerocketVersion = releaseVersionByFormat.(string) + break + } + } + if currentBottlerocketVersion != "" { + break + } + } + + latestBottlerocketVersion, needsUpgrade, err := github.GetLatestRevision(client, "bottlerocket-os", "bottlerocket", currentBottlerocketVersion, branchName, false, false) + if err != nil { + return "", "", nil, fmt.Errorf("getting latest Bottlerocket version from GitHub: %v", err) + } + + if needsUpgrade { + logger.Info("Bottlerocket version is out of date.", "Current version", currentBottlerocketVersion, "Latest version", latestBottlerocketVersion) + + err = updateBottlerocketReleasesFile(bottlerocketReleaseMap, bottlerocketReleasesFilePath, latestBottlerocketVersion) + if err != nil { + return "", "", nil, fmt.Errorf("updating Bottlerocket releases file: %v", err) + } + updatedBRFiles = append(updatedBRFiles, bottlerocketReleasesRelativeFilePath) + + updatedHostContainerFiles, err := updateBottlerocketHostContainerMetadata(client, projectRootFilepath, projectPath, latestBottlerocketVersion) + if err != nil { + return "", "", nil, fmt.Errorf("updating Bottlerocket host containers metadata files: %v", err) + } + updatedBRFiles = append(updatedBRFiles, updatedHostContainerFiles...) + } + + return currentBottlerocketVersion, latestBottlerocketVersion, updatedBRFiles, nil +} + +func updateBottlerocketReleasesFile(bottlerocketReleaseMap map[string]interface{}, bottlerocketReleasesFilePath, latestBottlerocketVersion string) error { + for channel := range bottlerocketReleaseMap { + for _, format := range constants.BottlerocketImageFormats { + releaseVersionByFormat := bottlerocketReleaseMap[channel].(map[string]interface{})[fmt.Sprintf("%s-release-version", format)] + if releaseVersionByFormat != nil { + imageExists, err := verifyBRImageExists(channel, format, latestBottlerocketVersion) + if err != nil { + return fmt.Errorf("checking if Bottlerocket %s image exists for %s release branch: %v", format, channel, err) + } + + if imageExists { + bottlerocketReleaseMap[channel].(map[string]interface{})[fmt.Sprintf("%s-release-version", format)] = latestBottlerocketVersion + } + } + } + } + updatedBottlerocketReleases, err := yaml.Marshal(bottlerocketReleaseMap) + if err != nil { + return fmt.Errorf("marshalling Bottlerocket releases: %v", err) + } + + err = os.WriteFile(bottlerocketReleasesFilePath, updatedBottlerocketReleases, 0o644) + if err != nil { + return fmt.Errorf("writing Bottlerocket releases file: %v", err) + } + + return nil +} + +func verifyBRImageExists(channel, format, bottlerocketVersion string) (bool, error) { + kubeVersion := strings.ReplaceAll(channel, "-", ".") + var variant, imageTarget string + switch format { + case "ami": + variant = "aws" + imageTarget = fmt.Sprintf("bottlerocket-%s-k8s-%s-x86_64-%s.img.lz4", variant, kubeVersion, bottlerocketVersion) + case "ova": + variant = "vmware" + imageTarget = fmt.Sprintf("bottlerocket-%s-k8s-%s-x86_64-%s.ova", variant, kubeVersion, bottlerocketVersion) + case "raw": + variant = "metal" + imageTarget = fmt.Sprintf("bottlerocket-%s-k8s-%s-x86_64-%s.img.lz4", variant, kubeVersion, bottlerocketVersion) + } + + timestampURL := fmt.Sprintf("https://updates.bottlerocket.aws/2020-07-07/%s-k8s-%s/x86_64/timestamp.json", variant, kubeVersion) + timestampManifest, err := file.ReadURL(timestampURL) + if err != nil { + return false, fmt.Errorf("reading Bottlerocket timestamp URL: %v", err) + } + + var timestampData interface{} + err = json.Unmarshal(timestampManifest, ×tampData) + if err != nil { + return false, fmt.Errorf("unmarshalling Bottlerocket timestamp manifest: %v", err) + } + + version := timestampData.(map[string]interface{})["signed"].(map[string]interface{})["version"].(float64) + versionString := fmt.Sprintf("%.0f", version) + + targetsURL := fmt.Sprintf("https://updates.bottlerocket.aws/2020-07-07/%s-k8s-%s/x86_64/%s.targets.json", variant, kubeVersion, versionString) + targetsManifest, err := file.ReadURL(targetsURL) + if err != nil { + return false, fmt.Errorf("reading Bottlerocket targets URL: %v", err) + } + + var targetsData interface{} + err = json.Unmarshal(targetsManifest, &targetsData) + if err != nil { + return false, fmt.Errorf("unmarshalling Bottlerocket targets manifest: %v", err) + } + + targets := targetsData.(map[string]interface{})["signed"].(map[string]interface{})["targets"].(map[string]interface{}) + for target := range targets { + if target == imageTarget { + return true, nil + } + } + + return false, nil +} + +func updateBottlerocketHostContainerMetadata(client *gogithub.Client, projectRootFilepath, projectPath, latestBottlerocketVersion string) ([]string, error) { + updatedHostContainerFiles := []string{} + hostContainersTOMLContents, err := github.GetFileContents(client, "bottlerocket-os", "bottlerocket", constants.BottlerocketHostContainersTOMLFile, latestBottlerocketVersion) + if err != nil { + return nil, fmt.Errorf("getting contents of Bottlerocket host containers file: %v", err) + } + + var hostContainersTOMLMap interface{} + err = toml.Unmarshal(hostContainersTOMLContents, &hostContainersTOMLMap) + if err != nil { + return nil, fmt.Errorf("unmarshalling Bottlerocket host containers file: %v", err) + } + + for _, container := range constants.BottlerocketHostContainers { + var hostContainerImageMetadata types.ImageMetadata + hostContainerMetadataFilePath := filepath.Join(projectRootFilepath, fmt.Sprintf(constants.BottlerocketContainerMetadataFileFormat, strings.ToUpper(container))) + hostContainerMetadataRelativeFilePath := filepath.Join(projectPath, fmt.Sprintf(constants.BottlerocketContainerMetadataFileFormat, strings.ToUpper(container))) + hostContainerMetadataFileContents, err := os.ReadFile(hostContainerMetadataFilePath) + if err != nil { + return nil, fmt.Errorf("reading Bottlerocket %s container metadata file: %v", container, err) + } + err = yaml.Unmarshal(hostContainerMetadataFileContents, &hostContainerImageMetadata) + if err != nil { + return nil, fmt.Errorf("unmarshalling Bottlerocket %s container metadata file: %v", container, err) + } + + hostContainerSourceImage := hostContainersTOMLMap.(map[string]interface{})["settings"].(map[string]interface{})["host-containers"].(map[string]interface{})[container].(map[string]interface{})["source"].(string) + hostContainerSourceImageTag := strings.Split(hostContainerSourceImage, ":")[1] + + if hostContainerImageMetadata.Tag != hostContainerSourceImageTag { + hostContainerImageMetadata.Tag = hostContainerSourceImageTag + skopeoInspectCmd := exec.Command("skopeo", "inspect", fmt.Sprintf("docker://%s", hostContainerSourceImage), "--override-os", "linux", "--override-arch", "amd64", "--format", "{{.Digest}}") + stdout, err := command.ExecCommand(skopeoInspectCmd) + if err != nil { + return nil, fmt.Errorf("running skopeo inspect command: %v", err) + } + hostContainerImageMetadata.ImageDigest = stdout + + updatedHostContainerMetadataFileContents, err := yaml.Marshal(hostContainerImageMetadata) + if err != nil { + return nil, fmt.Errorf("marshalling updated Bottlerocket %s container: %v", container, err) + } + + err = os.WriteFile(hostContainerMetadataFilePath, updatedHostContainerMetadataFileContents, 0o644) + if err != nil { + return nil, fmt.Errorf("writing Bottlerocket releases file: %v", err) + } + + updatedHostContainerFiles = append(updatedHostContainerFiles, hostContainerMetadataRelativeFilePath) + } + } + + return updatedHostContainerFiles, nil +} + +func isEKSDistroUpgrade(projectName string) bool { + return projectName == "aws/eks-distro" +} + +func getDefaultReleaseBranch(buildToolingRepoPath string) (string, error) { + defaultReleaseBranchCommandSequence := fmt.Sprintf("make --no-print-directory -C %s get-default-release-branch", buildToolingRepoPath) + defaultReleaseBranchCmd := exec.Command("bash", "-c", defaultReleaseBranchCommandSequence) + defaultReleaseBranch, err := command.ExecCommand(defaultReleaseBranchCmd) + if err != nil { + return "", fmt.Errorf("running get-default-release-branch Make command: %v", err) + } + + return defaultReleaseBranch, nil +} diff --git a/tools/version-tracker/pkg/constants/constants.go b/tools/version-tracker/pkg/constants/constants.go index 01dd4132fb..d9a8a272e6 100644 --- a/tools/version-tracker/pkg/constants/constants.go +++ b/tools/version-tracker/pkg/constants/constants.go @@ -6,30 +6,51 @@ import ( // Constants used across the version-tracker source code. const ( - BaseRepoOwnerEnvvar = "BASE_REPO_OWNER" - HeadRepoOwnerEnvvar = "HEAD_REPO_OWNER" - GitHubTokenEnvvar = "GITHUB_TOKEN" - CommitAuthorNameEnvvar = "COMMIT_AUTHOR_NAME" - CommitAuthorEmailEnvvar = "COMMIT_AUTHOR_EMAIL" - DefaultCommitAuthorName = "EKS Distro PR Bot" - DefaultCommitAuthorEmail = "aws-model-rocket-bots+eksdistroprbot@amazon.com" - BuildToolingRepoName = "eks-anywhere-build-tooling" - BuildToolingRepoURL = "https://github.com/aws/eks-anywhere-build-tooling" - ReadmeFile = "README.md" - ReadmeUpdateScriptFile = "build/lib/readme_check.sh" - LicenseBoilerplateFile = "hack/boilerplate.yq.txt" - SkippedProjectsFile = "SKIPPED_PROJECTS" - UpstreamProjectsTrackerFile = "UPSTREAM_PROJECTS.yaml" - GitTagFile = "GIT_TAG" - GoVersionFile = "GOLANG_VERSION" - ChecksumsFile = "CHECKSUMS" - AttributionsFilePattern = "*ATTRIBUTION.txt" - PatchesDirectory = "patches" - GithubPerPage = 100 - datetimeFormat = "%Y-%m-%dT%H:%M:%SZ" - MainBranchName = "main" - BaseRepoHeadRevision = "refs/remotes/origin/main" - PullRequestBody = `This PR bumps %[1]s/%[2]s to the latest Git revision, along with other updates such as Go version, checksums and attribution files. + BranchNameEnvVar = "BRANCH_NAME" + BaseRepoOwnerEnvvar = "BASE_REPO_OWNER" + HeadRepoOwnerEnvvar = "HEAD_REPO_OWNER" + GitHubTokenEnvvar = "GITHUB_TOKEN" + CommitAuthorNameEnvvar = "COMMIT_AUTHOR_NAME" + CommitAuthorEmailEnvvar = "COMMIT_AUTHOR_EMAIL" + ReleaseBranchEnvvar = "RELEASE_BRANCH" + DefaultCommitAuthorName = "EKS Distro PR Bot" + DefaultCommitAuthorEmail = "aws-model-rocket-bots+eksdistroprbot@amazon.com" + BuildToolingRepoName = "eks-anywhere-build-tooling" + DefaultBaseRepoOwner = "aws" + BuildToolingRepoURL = "https://github.com/%s/eks-anywhere-build-tooling" + ReadmeFile = "README.md" + ReadmeUpdateScriptFile = "build/lib/readme_check.sh" + LicenseBoilerplateFile = "hack/boilerplate.yq.txt" + EKSDistroLatestReleasesFile = "EKSD_LATEST_RELEASES" + EKSDistroReleaseChannelsFileURLFormat = "https://distro.eks.amazonaws.com/releasechannels/%s.yaml" + EKSDistroReleaseManifestURLFormat = "https://distro.eks.amazonaws.com/kubernetes-%[1]s/kubernetes-%[1]s-eks-%d.yaml" + SkippedProjectsFile = "SKIPPED_PROJECTS" + UpstreamProjectsTrackerFile = "UPSTREAM_PROJECTS.yaml" + SupportedReleaseBranchesFile = "release/SUPPORTED_RELEASE_BRANCHES" + GitTagFile = "GIT_TAG" + GoVersionFile = "GOLANG_VERSION" + ChecksumsFile = "CHECKSUMS" + AttributionsFilePattern = "*ATTRIBUTION.txt" + PatchesDirectory = "patches" + FailedPatchApplyMarker = "patch does not apply" + SemverRegex = `v?(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?` + FailedPatchApplyRegex = "Patch failed at .*" + FailedPatchFilesRegex = "error: (.*): patch does not apply" + BottlerocketReleasesFile = "BOTTLEROCKET_RELEASES" + BottlerocketContainerMetadataFileFormat = "BOTTLEROCKET_%s_CONTAINER_METADATA" + BottlerocketHostContainersTOMLFile = "sources/models/shared-defaults/public-host-containers.toml" + CiliumImageRepository = "public.ecr.aws/isovalent/cilium" + GithubPerPage = 100 + datetimeFormat = "%Y-%m-%dT%H:%M:%SZ" + MainBranchName = "main" + BaseRepoHeadRevisionPattern = "refs/remotes/origin/%s" + EKSDistroUpgradePullRequestBody = `This PR bumps EKS Distro releases to the latest available release versions. + +/hold +/area dependencies + +By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.` + DefaultUpgradePullRequestBody = `This PR bumps %[1]s/%[2]s to the latest Git revision. [Compare changes](https://github.com/%[1]s/%[2]s/compare/%[3]s...%[4]s) [Release notes](https://github.com/%[1]s/%[2]s/releases/%[4]s) @@ -37,8 +58,36 @@ const ( /hold /area dependencies +By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.` + BottlerocketUpgradePullRequestBody = `This PR bumps Bottlerocket releases to the latest Git revision. + +[Compare changes](https://github.com/bottlerocket-os/bottlerocket/compare/%[1]s...%[2]s) +[Release notes](https://github.com/bottlerocket-os/bottlerocket/releases/%[2]s) + +/hold +/area dependencies + +By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.` + + CombinedImageBuilderBottlerocketUpgradePullRequestBody = `This PR bumps kubernetes-sigs/image-builder and Bottlerocket releases to the latest Git revision. + +[Compare changes for image-builder](https://github.com/kubernetes-sigs/image-builder/compare/%[1]s...%[2]s) +[Release notes for image-builder](https://github.com/kubernetes-sigs/image-builder/releases/%[2]s) + +[Compare changes for Bottlerocket](https://github.com/bottlerocket-os/bottlerocket/compare/%[3]s...%[4]s) +[Release notes for Bottlerocket](https://github.com/bottlerocket-os/bottlerocket/releases/%[4]s) + +/hold +/area dependencies + By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.` PatchesCommentBody = `# This pull request is incomplete! +## Failed patch details +**Only %d/%d patches were applied!** +%s +The following files in the above patch did not apply successfully: +%s + The project being upgraded in this pull request needs changes to patches that cannot be handled automatically. A developer will need to regenerate the patches locally and update the pull request. In addition to patches, the checksums and attribution file(s) corresponding to the project will need to be updated.` ) @@ -69,11 +118,6 @@ var ( Extract: false, TrimLeadingVersionPrefix: true, }, - "cert-manager/cert-manager": { - AssetName: "cmctl-linux-amd64.tar.gz", - BinaryName: "cmctl", - Extract: true, - }, "containerd/containerd": { AssetName: "containerd-%s-linux-amd64.tar.gz", BinaryName: "bin/containerd", @@ -149,10 +193,22 @@ var ( // ProjectGoVersionSourceOfTruth is the mapping of project name to Go version source of truth files configuration. ProjectGoVersionSourceOfTruth = map[string]types.GoVersionSourceOfTruth{ + "aws/etcdadm-bootstrap-provider": { + SourceOfTruthFile: "go.mod", + GoVersionSearchString: `go (1\.\d\d)`, + }, + "aws/etcdadm-controller": { + SourceOfTruthFile: "go.mod", + GoVersionSearchString: `go (1\.\d\d)`, + }, "brancz/kube-rbac-proxy": { SourceOfTruthFile: ".github/workflows/build.yml", GoVersionSearchString: `go-version: '(1\.\d\d)\.\d+'`, }, + "cert-manager/cert-manager": { + SourceOfTruthFile: "go.mod", + GoVersionSearchString: `go (1\.\d\d)`, + }, "emissary-ingress/emissary": { SourceOfTruthFile: "go.mod", GoVersionSearchString: `go (1\.\d\d)`, @@ -181,6 +237,18 @@ var ( SourceOfTruthFile: "Dockerfile", GoVersionSearchString: `golang:(1\.\d\d)`, }, + "kubernetes/autoscaler": { + SourceOfTruthFile: "cluster-autoscaler/go.mod", + GoVersionSearchString: `go (1\.\d\d)`, + }, + "kubernetes/cloud-provider-aws": { + SourceOfTruthFile: "go.mod", + GoVersionSearchString: `go (1\.\d\d)`, + }, + "kubernetes/cloud-provider-vsphere": { + SourceOfTruthFile: "go.mod", + GoVersionSearchString: `go (1\.\d\d)`, + }, "kubernetes-sigs/cluster-api-provider-cloudstack": { SourceOfTruthFile: "go.mod", GoVersionSearchString: `go (1\.\d\d)`, @@ -201,6 +269,10 @@ var ( SourceOfTruthFile: "go.mod", GoVersionSearchString: `go (1\.\d\d)`, }, + "prometheus/prometheus": { + SourceOfTruthFile: ".promu.yml", + GoVersionSearchString: `version: (1\.\d\d)`, + }, "tinkerbell/boots": { SourceOfTruthFile: "go.mod", GoVersionSearchString: `go (1\.\d\d)`, @@ -218,4 +290,17 @@ var ( GoVersionSearchString: `GO_VERSION: "(1\.\d\d)"`, }, } + + ProjectsWithUnconventionalUpgradeFlows = []string{ + "cilium/cilium", + "kubernetes-sigs/image-builder", + } + + BottlerocketImageFormats = []string{"ami", "ova", "raw"} + + BottlerocketHostContainers = []string{"admin", "control"} + + CiliumImageDirectories = []string{"cilium", "operator-generic", "cilium-chart"} + + ProjectsSupportingPrereleaseTags = []string{"kubernetes-sigs/cluster-api-provider-cloudstack"} ) diff --git a/tools/version-tracker/pkg/ecrpublic/ecrpublic.go b/tools/version-tracker/pkg/ecrpublic/ecrpublic.go new file mode 100644 index 0000000000..8ebb54e444 --- /dev/null +++ b/tools/version-tracker/pkg/ecrpublic/ecrpublic.go @@ -0,0 +1,57 @@ +package ecrpublic + +import ( + "encoding/json" + "fmt" + "os/exec" + "strings" + + "github.com/aws/eks-anywhere/pkg/semver" + + "github.com/aws/eks-anywhere-build-tooling/tools/version-tracker/pkg/util/command" +) + +func GetLatestRevision(imageRepository, currentRevision string) (string, bool, error) { + var latestRevision string + currentRevisionSemver, err := semver.New(currentRevision) + if err != nil { + return "", false, fmt.Errorf("getting semver for current version: %v", err) + } + + skopeoListTagsCmd := exec.Command("skopeo", "list-tags", fmt.Sprintf("docker://%s", imageRepository)) + listTagsOutput, err := command.ExecCommand(skopeoListTagsCmd) + if err != nil { + return "", false, fmt.Errorf("running Go version command: %v", err) + } + + var tagsList interface{} + err = json.Unmarshal([]byte(listTagsOutput), &tagsList) + if err != nil { + return "", false, fmt.Errorf("unmarshalling output of Skopeo list-tags command: %v", err) + } + + ciliumTags := tagsList.(map[string]interface{})["Tags"].([]interface{}) + + latestRevisionSemver := currentRevisionSemver + for _, tag := range ciliumTags { + tag := tag.(string) + if !strings.HasPrefix(tag, "v") { + continue + } + + tagSemver, err := semver.New(tag) + if err != nil { + return "", false, fmt.Errorf("getting semver for Cilium tag [%s]: %v", tag, err) + } + + if tagSemver.GreaterThan(latestRevisionSemver) { + latestRevisionSemver = tagSemver + latestRevision = tag + } + } + if latestRevision == "" { + return "", false, nil + } + + return latestRevision, true, nil +} diff --git a/tools/version-tracker/pkg/git/git.go b/tools/version-tracker/pkg/git/git.go index c7ad7df625..8a7270e674 100644 --- a/tools/version-tracker/pkg/git/git.go +++ b/tools/version-tracker/pkg/git/git.go @@ -5,6 +5,7 @@ import ( "io" "os" "strings" + "time" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" @@ -17,8 +18,8 @@ import ( ) // CloneRepo clones the remote repository to a destination folder and creates a Git remote. -func CloneRepo(cloneURL, destination, headRepoOwner string) (*git.Repository, string, error) { - logger.V(6).Info(fmt.Sprintf("Cloning repository [%s] to %s directory\n", cloneURL, destination)) +func CloneRepo(cloneURL, destination, headRepoOwner, branch string) (*git.Repository, string, error) { + logger.V(6).Info(fmt.Sprintf("Cloning repository [%s] to %s directory", cloneURL, destination)) progress := io.Discard if logger.Verbosity >= 6 { progress = os.Stdout @@ -29,16 +30,19 @@ func CloneRepo(cloneURL, destination, headRepoOwner string) (*git.Repository, st }) if err != nil { if err == git.ErrRepositoryAlreadyExists { - logger.V(6).Info(fmt.Sprintf("Repo already exists at %s\n", destination)) + logger.V(6).Info(fmt.Sprintf("Repo already exists at %s", destination)) repo, err = git.PlainOpen(destination) + if err != nil { + return nil, "", fmt.Errorf("opening repo from %s directory: %v", destination, err) + } } else { return nil, "", fmt.Errorf("cloning repo %s to %s directory: %v", cloneURL, destination, err) } } - repoHeadCommit, err := repo.ResolveRevision(plumbing.Revision(constants.BaseRepoHeadRevision)) + repoHeadCommit, err := repo.ResolveRevision(plumbing.Revision(fmt.Sprintf(constants.BaseRepoHeadRevisionPattern, branch))) if err != nil { - return nil, "", fmt.Errorf("resolving revision [%s] to commit hash: %v", constants.BaseRepoHeadRevision, err) + return nil, "", fmt.Errorf("resolving revision [%s] to commit hash: %v", fmt.Sprintf(constants.BaseRepoHeadRevisionPattern, branch), err) } repoHeadCommitHash := strings.Split(repoHeadCommit.String(), " ")[0] @@ -59,8 +63,8 @@ func CloneRepo(cloneURL, destination, headRepoOwner string) (*git.Repository, st return repo, repoHeadCommitHash, nil } -// ResetToMain hard-resets the current working tree to point to the HEAD commit of the base repository. -func ResetToMain(worktree *git.Worktree, baseRepoHeadCommit string) error { +// ResetToHEAD hard-resets the current working tree to point to the HEAD commit of the base repository. +func ResetToHEAD(worktree *git.Worktree, baseRepoHeadCommit string) error { err := worktree.Reset(&git.ResetOptions{ Commit: plumbing.NewHash(baseRepoHeadCommit), Mode: git.HardReset, @@ -73,13 +77,13 @@ func ResetToMain(worktree *git.Worktree, baseRepoHeadCommit string) error { } // Checkout checks out the working tree at the given branch, creating a new branch if necessary. -func Checkout(worktree *git.Worktree, branch string) error { - logger.V(6).Info(fmt.Sprintf("Checking out branch [%s] in local worktree\n", branch)) +func Checkout(worktree *git.Worktree, branch string, create bool) error { + logger.V(6).Info(fmt.Sprintf("Checking out branch [%s] in local worktree", branch)) err := worktree.Checkout(&git.CheckoutOptions{ Branch: plumbing.NewBranchReferenceName(branch), - Force: true, - Create: true, + Keep: true, + Create: create, }) if err != nil { return fmt.Errorf("checking out branch [%s]: %v", branch, err) @@ -117,6 +121,7 @@ func Commit(worktree *git.Worktree, commitMessage string) error { Author: &object.Signature{ Name: commitAuthorName, Email: commitAuthorEmail, + When: time.Now(), }, }) if err != nil { diff --git a/tools/version-tracker/pkg/github/github.go b/tools/version-tracker/pkg/github/github.go index dea52bc745..0e30bf319e 100644 --- a/tools/version-tracker/pkg/github/github.go +++ b/tools/version-tracker/pkg/github/github.go @@ -4,9 +4,11 @@ import ( "context" "encoding/base64" "fmt" + "net/http" "os" "path/filepath" "regexp" + "slices" "strings" "github.com/aws/eks-anywhere/pkg/semver" @@ -19,36 +21,9 @@ import ( "github.com/aws/eks-anywhere-build-tooling/tools/version-tracker/pkg/util/version" ) -// getReleasesForRepo retrieves the list of releases for the given GitHub repository. -func getReleasesForRepo(client *github.Client, org, repo string) ([]*github.RepositoryRelease, error) { - logger.V(6).Info(fmt.Sprintf("Getting releases for [%s/%s] repository\n", org, repo)) - var allReleases []*github.RepositoryRelease - listReleasesOptions := &github.ListOptions{ - PerPage: constants.GithubPerPage, - } - - for { - releases, resp, err := client.Repositories.ListReleases(context.Background(), org, repo, listReleasesOptions) - if err != nil { - return nil, fmt.Errorf("calling ListReleases API for [%s/%s] repository: %v", org, repo, err) - } - for _, release := range releases { - if !*release.Prerelease { - allReleases = append(allReleases, release) - } - } - if resp.NextPage == 0 { - break - } - listReleasesOptions.Page = resp.NextPage - } - - return allReleases, nil -} - // getTagsForRepo retrieves the list of tags for the given GitHub repository. func getTagsForRepo(client *github.Client, org, repo string) ([]*github.RepositoryTag, error) { - logger.V(6).Info(fmt.Sprintf("Getting tags for [%s/%s] repository\n", org, repo)) + logger.V(6).Info(fmt.Sprintf("Getting tags for [%s/%s] repository", org, repo)) var allTags []*github.RepositoryTag listTagOptions := &github.ListOptions{ PerPage: constants.GithubPerPage, @@ -72,7 +47,7 @@ func getTagsForRepo(client *github.Client, org, repo string) ([]*github.Reposito // getCommitsForRepo retrieves the list of commits for the given GitHub repository. func getCommitsForRepo(client *github.Client, org, repo string) ([]*github.RepositoryCommit, error) { - logger.V(6).Info(fmt.Sprintf("Getting commits for [%s/%s] repository\n", org, repo)) + logger.V(6).Info(fmt.Sprintf("Getting commits for [%s/%s] repository", org, repo)) var allCommits []*github.RepositoryCommit listCommitOptions := &github.CommitsListOptions{ ListOptions: github.ListOptions{ @@ -83,7 +58,7 @@ func getCommitsForRepo(client *github.Client, org, repo string) ([]*github.Repos for { commits, resp, err := client.Repositories.ListCommits(context.Background(), org, repo, listCommitOptions) if err != nil { - return nil, fmt.Errorf("calling ListCommits for [%s/%s] repository: %v", org, repo, err) + return nil, fmt.Errorf("calling ListCommits API for [%s/%s] repository: %v", org, repo, err) } allCommits = append(allCommits, commits...) @@ -98,7 +73,7 @@ func getCommitsForRepo(client *github.Client, org, repo string) ([]*github.Repos // getCommitDateEpoch gets the Unix epoch time equivalent of a given Github commit's date. func getCommitDateEpoch(client *github.Client, org, repo, commitSHA string) (int64, error) { - logger.V(6).Info(fmt.Sprintf("Getting date for commit %s in [%s/%s] repository\n", commitSHA, org, repo)) + logger.V(6).Info(fmt.Sprintf("Getting date for commit %s in [%s/%s] repository", commitSHA, org, repo)) commit, _, err := client.Repositories.GetCommit(context.Background(), org, repo, commitSHA, nil) if err != nil { @@ -108,18 +83,25 @@ func getCommitDateEpoch(client *github.Client, org, repo, commitSHA string) (int return (*commit.Commit.Author.Date).Unix(), nil } -// GetLatestRevision returns the latest revision (GitHub release or tag) for a given GitHub repository. -func GetLatestRevision(client *github.Client, org, repo, currentRevision string) (string, bool, error) { - logger.V(6).Info(fmt.Sprintf("Getting latest revision for [%s/%s] repository\n", org, repo)) - var latestRevision string - needsUpgrade := false - - // Get all GitHub releases for this project. - allReleases, err := getReleasesForRepo(client, org, repo) +func GetFileContents(client *github.Client, org, repo, filePath, ref string) ([]byte, error) { + contents, _, _, err := client.Repositories.GetContents(context.Background(), org, repo, filePath, &github.RepositoryContentGetOptions{Ref: ref}) + if err != nil { + return nil, fmt.Errorf("getting contents of file [%s]: %v", filePath, err) + } + contentsDecoded, err := base64.StdEncoding.DecodeString(*contents.Content) if err != nil { - return "", false, fmt.Errorf("getting all releases for [%s/%s] repository: %v", org, repo, err) + return nil, fmt.Errorf("decoding contents of file [%s]: %v", filePath, err) } + return contentsDecoded, nil +} + +// GetLatestRevision returns the latest revision (GitHub release or tag) for a given GitHub repository. +func GetLatestRevision(client *github.Client, org, repo, currentRevision, branchName string, isTrackedUsingCommitHash, releaseBranched bool) (string, bool, error) { + logger.V(6).Info(fmt.Sprintf("Getting latest revision for [%s/%s] repository", org, repo)) + var currentRevisionCommit, latestRevision string + needsUpgrade := false + // Get all GitHub tags for this project. allTags, err := getTagsForRepo(client, org, repo) if err != nil { @@ -127,7 +109,11 @@ func GetLatestRevision(client *github.Client, org, repo, currentRevision string) } // Get commit hash corresponding to current revision tag. - currentRevisionCommit := getCommitForTag(allTags, currentRevision) + if isTrackedUsingCommitHash { + currentRevisionCommit = currentRevision + } else { + currentRevisionCommit = getCommitForTag(allTags, currentRevision) + } // Get Unix timestamp for current revision's commit. currentRevisionCommitEpoch, err := getCommitDateEpoch(client, org, repo, currentRevisionCommit) @@ -135,16 +121,64 @@ func GetLatestRevision(client *github.Client, org, repo, currentRevision string) return "", false, fmt.Errorf("getting epoch time corresponding to current revision commit: %v", err) } - // Get SemVer construct corresponding to the current revision tag. - currentRevisionSemver, err := semver.New(currentRevision) - if err != nil { - return "", false, fmt.Errorf("getting semver for current version: %v", err) - } + // If the project is tracked using a commit hash, upgrade to the latest commit. + if isTrackedUsingCommitHash { + // If the project does not have Github tags, pick the latest commit. + allCommits, err := getCommitsForRepo(client, org, repo) + if err != nil { + return "", false, fmt.Errorf("getting all commits for [%s/%s] repository: %v", org, repo, err) + } + latestRevision = *allCommits[0].SHA + needsUpgrade = true + } else { + semverRegex := regexp.MustCompile(constants.SemverRegex) + currentRevisionForSemver := semverRegex.FindString(currentRevision) + + // Get SemVer construct corresponding to the current revision tag. + currentRevisionSemver, err := semver.New(currentRevisionForSemver) + if err != nil { + return "", false, fmt.Errorf("getting semver for current version: %v", err) + } - // If the project has GitHub releases, determine the latest from among them. - if len(allReleases) > 0 { - for _, release := range allReleases { - latestRevision = *release.TagName + for _, tag := range allTags { + tagName := *tag.Name + if strings.Contains(tagName, "chart") || strings.Contains(tagName, "helm") { + continue + } + if org == "kubernetes" && repo == "autoscaler" { + if !strings.HasPrefix(tagName, "cluster-autoscaler-") { + continue + } + } + tagNameForSemver := semverRegex.FindString(tagName) + if tagNameForSemver == "" { + continue + } + + if releaseBranched { + releaseBranch := os.Getenv(constants.ReleaseBranchEnvvar) + releaseNumber := strings.Split(releaseBranch, "-")[1] + tagRegex := regexp.MustCompile(fmt.Sprintf(`^v?1.%s.\d+$`, releaseNumber)) + if !tagRegex.MatchString(tagNameForSemver) { + continue + } + } + if branchName != constants.MainBranchName { + tagRegex := regexp.MustCompile(fmt.Sprintf(`^v%d.%d.\d+`, currentRevisionSemver.Major, currentRevisionSemver.Minor)) + if !tagRegex.MatchString(tagNameForSemver) { + continue + } + } + + revisionSemver, err := semver.New(tagNameForSemver) + if err != nil { + return "", false, fmt.Errorf("getting semver for the version under consideration: %v", err) + } + if !slices.Contains(constants.ProjectsSupportingPrereleaseTags, fmt.Sprintf("%s/%s", org, repo)) && revisionSemver.Prerelease != "" { + continue + } + + latestRevision = tagName // Determine if upgrade is required based on current and latest revisions upgradeRequired, shouldBreak, err := isUpgradeRequired(client, org, repo, latestRevision, currentRevisionCommitEpoch, currentRevisionSemver, allTags) @@ -156,31 +190,6 @@ func GetLatestRevision(client *github.Client, org, repo, currentRevision string) break } } - } else { - // If the project doesn't have GitHub releases but has tags on GitHub, determine the latest from among them. - if len(allTags) > 0 { - for _, tag := range allTags { - latestRevision = *tag.Name - - // Determine if upgrade is required based on current and latest revisions - upgradeRequired, shouldBreak, err := isUpgradeRequired(client, org, repo, latestRevision, currentRevisionCommitEpoch, currentRevisionSemver, allTags) - if err != nil { - return "", false, fmt.Errorf("determining if upgrade is required for project: %v", err) - } - if shouldBreak { - needsUpgrade = upgradeRequired - break - } - } - } else { - // If the project has neither Github releases nor tags, pick the latest commit. - allCommits, err := getCommitsForRepo(client, org, repo) - if err != nil { - return "", false, fmt.Errorf("getting all commits for [%s/%s] repository: %v", org, repo, err) - } - latestRevision = *allCommits[0].SHA - needsUpgrade = true - } } return latestRevision, needsUpgrade, nil @@ -203,15 +212,18 @@ func isUpgradeRequired(client *github.Client, org, repo, latestRevision string, return false, false, fmt.Errorf("getting epoch time corresponding to latest revision commit: %v", err) } + semverRegex := regexp.MustCompile(constants.SemverRegex) + latestRevisionForSemver := semverRegex.FindString(latestRevision) + // Get SemVer construct corresponding to the latest revision tag. - latestRevisionSemver, err := semver.New(latestRevision) + latestRevisionSemver, err := semver.New(latestRevisionForSemver) if err != nil { return false, false, fmt.Errorf("getting semver for latest version: %v", err) } // If the latest revision comes after the current revision both chronologically and semantically, then declare that // an upgrade is required - if latestRevisionCommitEpoch > currentRevisionCommitEpoch && latestRevisionSemver.GreaterThan(currentRevisionSemver) { + if latestRevisionSemver.GreaterThan(currentRevisionSemver) || latestRevisionCommitEpoch > currentRevisionCommitEpoch { needsUpgrade = true shouldBreak = true } else if latestRevisionSemver.Equal(currentRevisionSemver) { @@ -234,140 +246,194 @@ func getCommitForTag(allTags []*github.RepositoryTag, searchTag string) string { // GetGoVersionForLatestRevision gets the Go version used to build the latest revision of the project. func GetGoVersionForLatestRevision(client *github.Client, org, repo, latestRevision string) (string, error) { - logger.V(6).Info(fmt.Sprintf("Getting Go version corresponding to latest revision %s for [%s/%s] repository\n", latestRevision, org, repo)) - cwd, err := os.Getwd() - if err != nil { - return "", fmt.Errorf("retrieving current working directory: %v", err) - } + logger.V(6).Info(fmt.Sprintf("Getting Go version corresponding to latest revision %s for [%s/%s] repository", latestRevision, org, repo)) var goVersion string + var err error projectFullName := fmt.Sprintf("%s/%s", org, repo) if _, ok := constants.ProjectReleaseAssets[projectFullName]; ok { - release, _, err := client.Repositories.GetReleaseByTag(context.Background(), org, repo, latestRevision) + release, response, err := client.Repositories.GetReleaseByTag(context.Background(), org, repo, latestRevision) if err != nil { - return "", fmt.Errorf("calling GetReleaseByTag for tag %s in [%s/%s] repository: %v", latestRevision, org, repo, err) - } - var tarballName, tarballUrl string - projectReleaseAsset := constants.ProjectReleaseAssets[projectFullName] - searchAssetName := projectReleaseAsset.AssetName - assetVersionReplacement := latestRevision - if constants.ProjectReleaseAssets[projectFullName].TrimLeadingVersionPrefix { - assetVersionReplacement = latestRevision[1:] - } - if strings.Count(searchAssetName, "%s") > 0 { - searchAssetName = fmt.Sprintf(searchAssetName, assetVersionReplacement) - } - if projectReleaseAsset.OverrideAssetURL != "" { - tarballName = searchAssetName - tarballUrl = projectReleaseAsset.OverrideAssetURL - if strings.Count(tarballUrl, "%s") > 0 { - tarballUrl = fmt.Sprintf(tarballUrl, assetVersionReplacement) + if response.StatusCode == http.StatusNotFound { + logger.V(6).Info(fmt.Sprintf("GitHub release for tag %s not found. Falling back to GitHub source of truth file for Go version", latestRevision)) + goVersion, err = getGoVersionFromGitHubFile(client, org, repo, projectFullName, latestRevision) + if err != nil { + return "", fmt.Errorf("getting Go version from GitHub source of truth file: %v", err) + } + } else { + return "", fmt.Errorf("calling GetReleaseByTag API for tag %s in [%s/%s] repository: %v", latestRevision, org, repo, err) } } else { - for _, asset := range release.Assets { - if *asset.Name == searchAssetName { - tarballName = *asset.Name - tarballUrl = *asset.BrowserDownloadURL - break - } + goVersion, err = getGoVersionFromGitHubRelease(release, projectFullName, latestRevision) + if err != nil { + return "", fmt.Errorf("getting Go version from GitHub release assets: %v", err) } } - - tarballDownloadPath := filepath.Join(cwd, "github-release-downloads") - err = os.MkdirAll(tarballDownloadPath, 0o755) + } else if _, ok := constants.ProjectGoVersionSourceOfTruth[projectFullName]; ok { + goVersion, err = getGoVersionFromGitHubFile(client, org, repo, projectFullName, latestRevision) if err != nil { - return "", fmt.Errorf("failed to create GitHub release downloads folder: %v", err) + return "", fmt.Errorf("getting Go version from GitHub source of truth file: %v", err) } - tarballFilePath := filepath.Join(tarballDownloadPath, tarballName) + } + + return goVersion, nil +} + +// CreatePullRequest creates a pull request from the head branch to the base branch on the base repository. +func CreatePullRequest(client *github.Client, org, repo, title, body, baseRepoOwner, baseBranch, headRepoOwner, headBranch, currentRevision, latestRevision string, addPatchWarningComment bool, patchesWarningComment string) error { + var pullRequest *github.PullRequest + var patchWarningCommentExists bool + + // Check if there is already a pull request from the head branch to the base branch. + pullRequests, _, err := client.PullRequests.List(context.Background(), baseRepoOwner, constants.BuildToolingRepoName, &github.PullRequestListOptions{ + Base: baseBranch, + Head: fmt.Sprintf("%s:%s", headRepoOwner, headBranch), + }) + if err != nil { + return fmt.Errorf("listing pull requests from %s:%s -> %s:%s: %v", headRepoOwner, headBranch, baseRepoOwner, baseBranch, err) + } + + if len(pullRequests) > 0 { + pullRequest = pullRequests[0] + logger.Info(fmt.Sprintf("A pull request already exists for %s:%s", headRepoOwner, headBranch), "Pull request", *pullRequest.HTMLURL) - err = file.Download(tarballUrl, tarballFilePath) + pullRequest.Body = github.String(body) + pullRequest, _, err = client.PullRequests.Edit(context.Background(), baseRepoOwner, constants.BuildToolingRepoName, *pullRequest.Number, pullRequest) if err != nil { - return "", fmt.Errorf("downloading release tarball from URL [%s]: %v", tarballUrl, err) + return fmt.Errorf("editing existing pull request [%s]: %v", *pullRequest.HTMLURL, err) } - if projectReleaseAsset.Extract { - tarballFile, err := os.Open(tarballFilePath) + // If patches to the project failed to apply, check if the PR already has a comment warning about + // the incomplete PR and patches needing to be regenerated. + if addPatchWarningComment { + pullRequestComments, _, err := client.Issues.ListComments(context.Background(), baseRepoOwner, constants.BuildToolingRepoName, *pullRequest.Number, nil) if err != nil { - return "", fmt.Errorf("opening tarball filepath: %v", err) + return fmt.Errorf("listing comments on pull request [%s]: %v", *pullRequest.HTMLURL, err) } - err = tar.ExtractTarGz(tarballDownloadPath, tarballFile) - if err != nil { - return "", fmt.Errorf("extracting tarball file: %v", err) + for _, comment := range pullRequestComments { + if comment.Body == github.String(patchesWarningComment) { + patchWarningCommentExists = true + } } } - - binaryName := projectReleaseAsset.BinaryName - if strings.Count(binaryName, "%s") > 0 { - binaryName = fmt.Sprintf(binaryName, assetVersionReplacement) + } else { + logger.V(6).Info(fmt.Sprintf("Creating pull request with updated versions for [%s/%s] repository", org, repo)) + + newPR := &github.NewPullRequest{ + Title: github.String(title), + Head: github.String(fmt.Sprintf("%s:%s", headRepoOwner, headBranch)), + Base: github.String(baseBranch), + Body: github.String(body), + MaintainerCanModify: github.Bool(true), } - binaryFilePath := filepath.Join(tarballDownloadPath, binaryName) - goVersion, err = version.GetGoVersion(binaryFilePath) + pullRequest, _, err = client.PullRequests.Create(context.Background(), baseRepoOwner, constants.BuildToolingRepoName, newPR) if err != nil { - return "", fmt.Errorf("getting Go version embedded in binary [%s]: %v", binaryFilePath, err) + return fmt.Errorf("creating pull request with updated versions from %s to %s: %v", headBranch, baseBranch, err) } - err = os.RemoveAll(tarballDownloadPath) - if err != nil { - return "", fmt.Errorf("removing tarball download path: %v", err) - } - } else if _, ok := constants.ProjectGoVersionSourceOfTruth[projectFullName]; ok { - projectGoVersionSourceOfTruthFile := constants.ProjectGoVersionSourceOfTruth[projectFullName].SourceOfTruthFile - workflowContents, _, _, err := client.Repositories.GetContents(context.Background(), org, repo, projectGoVersionSourceOfTruthFile, &github.RepositoryContentGetOptions{Ref: latestRevision}) - if err != nil { - return "", fmt.Errorf("getting contents of file [%s]: %v", projectGoVersionSourceOfTruthFile, err) + logger.Info(fmt.Sprintf("Created pull request: %s", *pullRequest.HTMLURL)) + } + + // If patches failed to apply and no patch warning comment exists (always the case for a new PR), then add a comment with the + // warning. + if addPatchWarningComment && !patchWarningCommentExists { + patchWarningComment := &github.IssueComment{ + Body: github.String(patchesWarningComment), } - workflowContentsDecoded, err := base64.StdEncoding.DecodeString(*workflowContents.Content) + + _, _, err = client.Issues.CreateComment(context.Background(), baseRepoOwner, constants.BuildToolingRepoName, *pullRequest.Number, patchWarningComment) if err != nil { - return "", fmt.Errorf("decoding contents of file [%s]: %v", projectGoVersionSourceOfTruthFile, err) + return fmt.Errorf("commenting failed patch apply warning on pull request [%s]: %v", *pullRequest.HTMLURL, err) } - pattern := regexp.MustCompile(constants.ProjectGoVersionSourceOfTruth[projectFullName].GoVersionSearchString) - matches := pattern.FindStringSubmatch(string(workflowContentsDecoded)) - - goVersion = matches[1] } - return goVersion, nil + return nil } -// CreatePullRequest creates a pull request from the head branch to the base branch on the base repository. -func CreatePullRequest(client *github.Client, org, repo, baseRepoOwner, baseBranch, headRepoOwner, headBranch, currentRevision, latestRevision string, projectHasPatches bool) error { - logger.V(6).Info(fmt.Sprintf("Creating pull request with updated versions for [%s/%s] repository\n", org, repo)) +func getGoVersionFromGitHubRelease(release *github.RepositoryRelease, projectFullName, latestRevision string) (string, error) { + var tarballName, tarballUrl string + projectReleaseAsset := constants.ProjectReleaseAssets[projectFullName] + searchAssetName := projectReleaseAsset.AssetName + assetVersionReplacement := latestRevision + if constants.ProjectReleaseAssets[projectFullName].TrimLeadingVersionPrefix { + assetVersionReplacement = latestRevision[1:] + } + if strings.Count(searchAssetName, "%s") > 0 { + searchAssetName = fmt.Sprintf(searchAssetName, assetVersionReplacement) + } + if projectReleaseAsset.OverrideAssetURL != "" { + tarballName = searchAssetName + tarballUrl = projectReleaseAsset.OverrideAssetURL + if strings.Count(tarballUrl, "%s") > 0 { + tarballUrl = fmt.Sprintf(tarballUrl, assetVersionReplacement) + } + } else { + for _, asset := range release.Assets { + if *asset.Name == searchAssetName { + tarballName = *asset.Name + tarballUrl = *asset.BrowserDownloadURL + break + } + } + } - pullRequests, _, err := client.PullRequests.List(context.Background(), baseRepoOwner, constants.BuildToolingRepoName, &github.PullRequestListOptions{ - Head: fmt.Sprintf("%s:%s", headRepoOwner, headBranch), - }) + cwd, err := os.Getwd() if err != nil { - return fmt.Errorf("listing pull requests with %s:%s as head branch: %v", headRepoOwner, headBranch, err) - } - if len(pullRequests) > 0 { - logger.Info(fmt.Sprintf("A pull request already exists for %s:%s\n", headRepoOwner, headBranch), "Pull request", *pullRequests[0].HTMLURL) - return nil + return "", fmt.Errorf("retrieving current working directory: %v", err) } - newPR := &github.NewPullRequest{ - Title: github.String(fmt.Sprintf("Bump %s/%s to latest release", org, repo)), - Head: github.String(fmt.Sprintf("%s:%s", headRepoOwner, headBranch)), - Base: github.String(baseBranch), - Body: github.String(fmt.Sprintf(constants.PullRequestBody, org, repo, currentRevision, latestRevision)), - MaintainerCanModify: github.Bool(true), + tarballDownloadPath := filepath.Join(cwd, "github-release-downloads") + err = os.MkdirAll(tarballDownloadPath, 0o755) + if err != nil { + return "", fmt.Errorf("failed to create GitHub release downloads folder: %v", err) } + tarballFilePath := filepath.Join(tarballDownloadPath, tarballName) - pullRequest, _, err := client.PullRequests.Create(context.Background(), baseRepoOwner, constants.BuildToolingRepoName, newPR) + err = file.Download(tarballUrl, tarballFilePath) if err != nil { - return fmt.Errorf("creating pull request with updated versions from %s to %s: %v", headBranch, baseBranch, err) + return "", fmt.Errorf("downloading release tarball from URL [%s]: %v", tarballUrl, err) } - if projectHasPatches { - newComment := &github.IssueComment{ - Body: github.String(constants.PatchesCommentBody), + binaryName := projectReleaseAsset.BinaryName + if strings.Count(binaryName, "%s") > 0 { + binaryName = fmt.Sprintf(binaryName, assetVersionReplacement) + } + if projectReleaseAsset.Extract { + tarballFile, err := os.Open(tarballFilePath) + if err != nil { + return "", fmt.Errorf("opening tarball filepath: %v", err) } - _, _, err = client.Issues.CreateComment(context.Background(), baseRepoOwner, constants.BuildToolingRepoName, *pullRequest.Number, newComment) + err = tar.ExtractFileFromTarball(tarballDownloadPath, tarballFile, binaryName) if err != nil { - return fmt.Errorf("commenting patch warning on pull request [%s]: %v", *pullRequest.HTMLURL, err) + return "", fmt.Errorf("extracting tarball file: %v", err) } } - return nil + binaryFilePath := filepath.Join(tarballDownloadPath, binaryName) + goVersion, err := version.GetGoVersion(binaryFilePath) + if err != nil { + return "", fmt.Errorf("getting Go version embedded in binary [%s]: %v", binaryFilePath, err) + } + + err = os.RemoveAll(tarballDownloadPath) + if err != nil { + return "", fmt.Errorf("removing tarball download path: %v", err) + } + + return goVersion, nil +} + +func getGoVersionFromGitHubFile(client *github.Client, org, repo, projectFullName, latestRevision string) (string, error) { + projectGoVersionSourceOfTruthFile := constants.ProjectGoVersionSourceOfTruth[projectFullName].SourceOfTruthFile + workflowContents, err := GetFileContents(client, org, repo, projectGoVersionSourceOfTruthFile, latestRevision) + if err != nil { + return "", fmt.Errorf("getting contents of file [%s]: %v", projectGoVersionSourceOfTruthFile, err) + } + + pattern := regexp.MustCompile(constants.ProjectGoVersionSourceOfTruth[projectFullName].GoVersionSearchString) + matches := pattern.FindStringSubmatch(string(workflowContents)) + + return matches[1], nil } diff --git a/tools/version-tracker/pkg/types/types.go b/tools/version-tracker/pkg/types/types.go index 7135fb20d6..1eb8933f66 100644 --- a/tools/version-tracker/pkg/types/types.go +++ b/tools/version-tracker/pkg/types/types.go @@ -62,6 +62,18 @@ type GoVersionSourceOfTruth struct { } type ImageMetadata struct { - Tag string `yaml:"tag,omitempty"` - ImageDigest string `yaml:"imageDigest,omitempty"` + Tag string `json:"tag,omitempty"` + ImageDigest string `json:"imageDigest,omitempty"` +} + +type EKSDistroRelease struct { + Branch string `json:"branch"` + KubeVersion string `json:"kubeVersion"` + Number int `json:"number"` + Dev *bool `json:"dev,omitempty"` +} + +type EKSDistroLatestReleases struct { + Releases []EKSDistroRelease `json:"releases"` + Latest string `json:"latest"` } diff --git a/tools/version-tracker/pkg/util/file/file.go b/tools/version-tracker/pkg/util/file/file.go index 946adbc49f..1ad387ad67 100644 --- a/tools/version-tracker/pkg/util/file/file.go +++ b/tools/version-tracker/pkg/util/file/file.go @@ -23,3 +23,18 @@ func Download(url, filepath string) error { _, err = io.Copy(out, resp.Body) return err } + +func ReadURL(url string) ([]byte, error) { + resp, err := http.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return data, nil +} diff --git a/tools/version-tracker/pkg/util/slices/slices.go b/tools/version-tracker/pkg/util/slices/slices.go deleted file mode 100644 index 4e965987af..0000000000 --- a/tools/version-tracker/pkg/util/slices/slices.go +++ /dev/null @@ -1,10 +0,0 @@ -package slices - -func Contains(s []string, str string) bool { - for _, elem := range s { - if elem == str { - return true - } - } - return false -} diff --git a/tools/version-tracker/pkg/util/tar/tar.go b/tools/version-tracker/pkg/util/tar/tar.go index b007f99790..c37ced3865 100644 --- a/tools/version-tracker/pkg/util/tar/tar.go +++ b/tools/version-tracker/pkg/util/tar/tar.go @@ -7,10 +7,11 @@ import ( "io" "os" "path/filepath" + "strings" ) -// ExtractTarGz extracts the contents of the given tarball. -func ExtractTarGz(tarballDownloadPath string, gzipStream io.Reader) error { +// ExtractFileFromTarball extracts the specified file from the given tarball. +func ExtractFileFromTarball(tarballDownloadPath string, gzipStream io.Reader, targetFile string) error { uncompressedStream, err := gzip.NewReader(gzipStream) if err != nil { return err @@ -19,12 +20,13 @@ func ExtractTarGz(tarballDownloadPath string, gzipStream io.Reader) error { tarReader := tar.NewReader(uncompressedStream) var header *tar.Header for header, err = tarReader.Next(); err == nil; header, err = tarReader.Next() { - switch header.Typeflag { - case tar.TypeDir: - if err := os.Mkdir(filepath.Join(tarballDownloadPath, header.Name), 0o755); err != nil { - return fmt.Errorf("creating directory from archive: %v", err) + if header.Name == targetFile { + if strings.Contains(header.Name, "/") { + err = os.MkdirAll(filepath.Join(tarballDownloadPath, filepath.Dir(header.Name)), 0o755) + if err != nil { + return fmt.Errorf("creating parent directory for archive contents: %v", err) + } } - case tar.TypeReg: outFile, err := os.Create(filepath.Join(tarballDownloadPath, header.Name)) if err != nil { return fmt.Errorf("creating file from archive: %v", err) @@ -37,8 +39,6 @@ func ExtractTarGz(tarballDownloadPath string, gzipStream io.Reader) error { if err := outFile.Close(); err != nil { return fmt.Errorf("closing output destination file descriptor: %v", err) } - default: - return fmt.Errorf("unknown type in tar header: %b in %s", header.Typeflag, header.Name) } } if err != io.EOF { diff --git a/tools/version-tracker/pkg/util/version/version.go b/tools/version-tracker/pkg/util/version/version.go index fcfd170d54..10b9bd75ed 100644 --- a/tools/version-tracker/pkg/util/version/version.go +++ b/tools/version-tracker/pkg/util/version/version.go @@ -16,7 +16,8 @@ func GetGoVersion(goBinaryLocation string) (string, error) { if err != nil { return "", fmt.Errorf("running Go version command: %v", err) } - pattern := regexp.MustCompile(fmt.Sprintf(`^%s: go(.*)\.\d+\n`, goBinaryLocation)) + // The first line could be a warning, so no ^ in the sprintf below + pattern := regexp.MustCompile(fmt.Sprintf(`%s: go(.*)\.\d+`, goBinaryLocation)) matches := pattern.FindStringSubmatch(commandOutput) return matches[1], nil