Update build an CI tooling to allow arbitrary container image registr…

…ies (#1516) Summary: This updates our config flags and ci scripts to allow our various container artifacts to be pushed to arbitrary image registies via a config flag. This doesn't yet support arbitrary registries since most registries don't allow you to treat image paths like nested dirs, so `$(IMAGE_PREFIX)/cloud/plugin/load_db` for example won't work. However this is a stepping stone along the way to supporting other registries without changing where we currently push images. Type of change: /kind cleanup Test Plan: Ran builds locally to test that image paths were sane. More testing will require creating RCs and deploying those. Signed-off-by: Vihang Mehta <[email protected]>
pixie-io · Aug 9, 2023 · b76a352 · b76a352
1 parent 1c328aa
commit b76a352
Show file tree

Hide file tree

Showing 40 changed files with 148 additions and 177 deletions.
diff --git a/bazel/images.bzl b/bazel/images.bzl
@@ -16,16 +16,12 @@
 
 load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo")
 
-PROPRIETARY_PREFIX = "gcr.io/pl-dev-infra/"
-PUBLIC_PREFIX = "gcr.io/pixie-oss/pixie-prod/"
-DEV_PREFIX = "gcr.io/pixie-oss/pixie-dev/"
-
-def image_replacements(image_map, existing_prefix, new_prefix):
+def image_replacements(image_map):
     replacements = {}
 
     for image in image_map.keys():
-        image = image.removeprefix("$(IMAGE_PREFIX)").removesuffix(":$(BUNDLE_VERSION)")
-        replacements[existing_prefix + image] = new_prefix + image + ":{BUNDLE_VERSION}"
+        image = image.removeprefix("$(IMAGE_PREFIX)/").removesuffix(":$(BUNDLE_VERSION)")
+        replacements[image] = "{IMAGE_PREFIX}/" + image + ":{BUNDLE_VERSION}"
 
     return replacements
 
@@ -46,14 +42,14 @@ bundle_version_provider = rule(
 def _image_prefix_provider_impl(ctx):
     return [
         platform_common.TemplateVariableInfo({
-            "IMAGE_PREFIX": ctx.attr.image_prefix,
+            "IMAGE_PREFIX": ctx.attr._image_prefix[BuildSettingInfo].value,
         }),
     ]
 
 image_prefix_provider = rule(
     implementation = _image_prefix_provider_impl,
     attrs = {
-        "image_prefix": attr.string(mandatory = True),
+        "_image_prefix": attr.label(default = "//k8s:image_repository"),
     },
 )
 

diff --git a/ci/cloud_build_release.sh b/ci/cloud_build_release.sh
@@ -34,16 +34,23 @@ if [[ "${release_tag}" == *"-"* ]]; then
 fi
 
 echo "The image tag is: ${release_tag}"
+image_repo="gcr.io/pixie-oss/pixie-prod"
 
-bazel run --config=stamp -c opt --action_env=GOOGLE_APPLICATION_CREDENTIALS --//k8s:image_version="${release_tag}" \
-    --//k8s:build_type=public //k8s/cloud:cloud_images_push
+bazel run -c opt \
+  --config=stamp \
+  --action_env=GOOGLE_APPLICATION_CREDENTIALS \
+  --//k8s:image_repository="${image_repo}" \
+  --//k8s:image_version="${release_tag}" \
+  //k8s/cloud:cloud_images_push
 
 while read -r image;
 do
   image_digest=$(crane digest "${image}")
   cosign sign --key env://COSIGN_PRIVATE_KEY --yes -r "${image}@${image_digest}"
-done < <(bazel run --config=stamp -c opt --action_env=GOOGLE_APPLICATION_CREDENTIALS --//k8s:image_version="${release_tag}" \
-    --//k8s:build_type=public //k8s/cloud:list_image_bundle)
+done < <(bazel run -c opt \
+  --//k8s:image_repository="${image_repo}" \
+  --//k8s:image_version="${release_tag}" \
+  //k8s/cloud:list_image_bundle)
 
 all_licenses_opts=("//tools/licenses:all_licenses" "--action_env=GOOGLE_APPLICATION_CREDENTIALS" "--remote_download_outputs=toplevel")
 all_licenses_path="$(bazel cquery "${all_licenses_opts[@]}"  --output starlark --starlark:expr "target.files.to_list()[0].path" 2> /dev/null)"

diff --git a/ci/image_utils.sh b/ci/image_utils.sh
@@ -27,11 +27,14 @@ push_images_for_arch() {
   arch="$1"
   image_rule="$2"
   release_tag="$3"
-  build_type="$4"
+  image_repo="$4"
 
-  bazel run --config=stamp -c opt --//k8s:image_version="${release_tag}-${arch}" \
+  bazel run -c opt \
+    --config=stamp \
     --config="${arch}_sysroot" \
-    --config=stamp "${build_type}" "${image_rule}" > /dev/null
+    --//k8s:image_repository="${image_repo}" \
+    --//k8s:image_version="${release_tag}-${arch}" \
+    "${image_rule}" > /dev/null
 }
 
 push_multiarch_image() {
@@ -54,14 +57,17 @@ push_all_multiarch_images() {
   image_rule="$1"
   image_list_rule="$2"
   release_tag="$3"
-  build_type="$4"
+  image_repo="$4"
 
-  push_images_for_arch "x86_64" "${image_rule}" "${release_tag}" "${build_type}"
-  push_images_for_arch "aarch64" "${image_rule}" "${release_tag}" "${build_type}"
+  push_images_for_arch "x86_64" "${image_rule}" "${release_tag}" "${image_repo}"
+  push_images_for_arch "aarch64" "${image_rule}" "${release_tag}" "${image_repo}"
 
   while read -r image;
   do
     push_multiarch_image "${image}"
-  done < <(bazel run --config=stamp -c opt --//k8s:image_version="${release_tag}" \
-          --config=stamp "${build_type}" "${image_list_rule}")
+  done < <(bazel run -c opt \
+    --config=stamp \
+    --//k8s:image_repository="${image_repo}" \
+    --//k8s:image_version="${release_tag}" \
+    "${image_list_rule}")
 }
diff --git a/ci/operator_build_release.sh b/ci/operator_build_release.sh
@@ -37,28 +37,32 @@ bazel run -c opt //src/utils/artifacts/versions_gen:versions_gen -- \
 tags=$(git for-each-ref --sort='-*authordate' --format '%(refname:short)' refs/tags \
     | grep "release/operator" | grep -v "\-")
 
-build_type="--//k8s:build_type=public"
-image_path="gcr.io/pixie-oss/pixie-prod/operator/operator_image:${release_tag}"
-deleter_image_path="gcr.io/pixie-oss/pixie-prod/operator/vizier_deleter:${release_tag}"
+image_repo="gcr.io/pixie-oss/pixie-prod"
+image_paths=$(bazel cquery //k8s/operator:image_bundle \
+  --//k8s:image_repository="${image_repo}" \
+  --//k8s:image_version="${release_tag}" \
+  --output=starlark \
+  --starlark:expr="'\n'.join(providers(target)['@io_bazel_rules_docker//container:providers.bzl%BundleInfo'].container_images.keys())")
+image_path=$(echo "${image_paths}" | grep -v deleter)
+deleter_image_path=$(echo "${image_paths}" | grep deleter)
+
+bucket="pixie-dev-public"
+
 channel="stable"
 channels="stable,dev"
-bucket="pixie-dev-public"
 # The previous version should be the 2nd item in the tags. Since this is a release build,
 # the first item in the tag is the current release.
 prev_tag=$(echo "$tags" | sed -n '2 p')
+
 if [[ $release_tag == *"-"* ]]; then
-  build_type="--//k8s:build_type=dev"
-  image_path="gcr.io/pixie-oss/pixie-dev/operator/operator_image:${release_tag}"
-  deleter_image_path="gcr.io/pixie-oss/pixie-dev/operator/vizier_deleter:${release_tag}"
   channel="dev"
   channels="dev"
-  # Use the same bucket as above.
   # The previous version should be the 1st item in the tags. Since this is a non-release build,
   # the first item in the tags is the previous release.
   prev_tag=$(echo "$tags" | sed -n '1 p')
 fi
 
-push_all_multiarch_images "//k8s/operator:operator_images_push" "//k8s/operator:list_image_bundle" "${release_tag}" "${build_type}"
+push_all_multiarch_images "//k8s/operator:operator_images_push" "//k8s/operator:list_image_bundle" "${release_tag}" "${image_repo}"
 
 # Build operator bundle for OLM.
 tmp_dir="$(mktemp -d)"

diff --git a/ci/vizier_build_release.sh b/ci/vizier_build_release.sh
@@ -35,15 +35,15 @@ echo "The release tag is: ${release_tag}"
 bazel run -c opt //src/utils/artifacts/versions_gen:versions_gen -- \
       --repo_path "${repo_path}" --artifact_name vizier --versions_file "${versions_file}"
 
-build_type="--//k8s:build_type=public"
-if [[ $release_tag == *"-"* ]]; then
-  build_type="--//k8s:build_type=dev"
-fi
+image_repo="gcr.io/pixie-oss/pixie-prod"
 
-push_all_multiarch_images "//k8s/vizier:vizier_images_push" "//k8s/vizier:list_image_bundle" "${release_tag}" "${build_type}"
+push_all_multiarch_images "//k8s/vizier:vizier_images_push" "//k8s/vizier:list_image_bundle" "${release_tag}" "${image_repo}"
 
-bazel build --config=stamp -c opt --//k8s:image_version="${release_tag}" \
-    --config=stamp "${build_type}" //k8s/vizier:vizier_yamls
+bazel build -c opt \
+  --config=stamp \
+  --//k8s:image_repository="${image_repo}" \
+  --//k8s:image_version="${release_tag}" \
+  //k8s/vizier:vizier_yamls
 
 yamls_tar="${repo_path}/bazel-bin/k8s/vizier/vizier_yamls.tar"
 

diff --git a/k8s/BUILD.bazel b/k8s/BUILD.bazel
@@ -15,7 +15,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 load("@bazel_skylib//rules:common_settings.bzl", "string_flag")
-load("//bazel:images.bzl", "DEV_PREFIX", "PROPRIETARY_PREFIX", "PUBLIC_PREFIX", "bundle_version_provider", "image_prefix_provider")
+load("//bazel:images.bzl", "bundle_version_provider", "image_prefix_provider")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -24,47 +24,15 @@ string_flag(
     build_setting_default = "latest",
 )
 
-string_flag(
-    name = "build_type",
-    build_setting_default = "dev",
-    values = [
-        "public",
-        "proprietary",
-        "dev",
-    ],
-)
-
-config_setting(
-    name = "dev",
-    flag_values = {
-        ":build_type": "dev",
-    },
-)
-
-config_setting(
-    name = "public",
-    flag_values = {
-        ":build_type": "public",
-    },
-)
-
-config_setting(
-    name = "proprietary",
-    flag_values = {
-        ":build_type": "proprietary",
-    },
-)
-
 bundle_version_provider(
     name = "bundle_version",
 )
 
+string_flag(
+    name = "image_repository",
+    build_setting_default = "localhost:5000",
+)
+
 image_prefix_provider(
     name = "image_prefix",
-    image_prefix = select({
-        ":dev": DEV_PREFIX,
-        ":proprietary": PROPRIETARY_PREFIX,
-        ":public": PUBLIC_PREFIX,
-        "//conditions:default": DEV_PREFIX,
-    }),
 )
diff --git a/k8s/cloud/BUILD.bazel b/k8s/cloud/BUILD.bazel
@@ -16,27 +16,27 @@
 
 load("@io_bazel_rules_docker//container:container.bzl", "container_bundle")
 load("@io_bazel_rules_docker//contrib:push-all.bzl", "container_push")
-load("//bazel:images.bzl", "DEV_PREFIX", "PROPRIETARY_PREFIX", "image_replacements", "list_image_bundle")
+load("//bazel:images.bzl", "image_replacements", "list_image_bundle")
 load("//bazel:kustomize.bzl", "kustomize_build")
 
 package(default_visibility = ["//visibility:public"])
 
 CLOUD_IMAGE_TO_LABEL = {
-    "$(IMAGE_PREFIX)cloud/api_server_image:$(BUNDLE_VERSION)": "//src/cloud/api:api_server_image",
-    "$(IMAGE_PREFIX)cloud/artifact_tracker_server_image:$(BUNDLE_VERSION)": "//src/cloud/artifact_tracker:artifact_tracker_server_image",
-    "$(IMAGE_PREFIX)cloud/auth_server_image:$(BUNDLE_VERSION)": "//src/cloud/auth:auth_server_image",
-    "$(IMAGE_PREFIX)cloud/config_manager_server_image:$(BUNDLE_VERSION)": "//src/cloud/config_manager:config_manager_server_image",
-    "$(IMAGE_PREFIX)cloud/cron_script_server_image:$(BUNDLE_VERSION)": "//src/cloud/cron_script:cron_script_server_image",
-    "$(IMAGE_PREFIX)cloud/indexer_server_image:$(BUNDLE_VERSION)": "//src/cloud/indexer:indexer_server_image",
-    "$(IMAGE_PREFIX)cloud/metrics_server_image:$(BUNDLE_VERSION)": "//src/cloud/metrics:metrics_server_image",
-    "$(IMAGE_PREFIX)cloud/plugin/load_db:$(BUNDLE_VERSION)": "//src/cloud/plugin/load_db:plugin_db_updater_image",
-    "$(IMAGE_PREFIX)cloud/plugin_server_image:$(BUNDLE_VERSION)": "//src/cloud/plugin:plugin_server_image",
-    "$(IMAGE_PREFIX)cloud/profile_server_image:$(BUNDLE_VERSION)": "//src/cloud/profile:profile_server_image",
-    "$(IMAGE_PREFIX)cloud/project_manager_server_image:$(BUNDLE_VERSION)": "//src/cloud/project_manager:project_manager_server_image",
-    "$(IMAGE_PREFIX)cloud/proxy_server_image:$(BUNDLE_VERSION)": "//src/cloud/proxy:proxy_server_image",
-    "$(IMAGE_PREFIX)cloud/scriptmgr_server_image:$(BUNDLE_VERSION)": "//src/cloud/scriptmgr:scriptmgr_server_image",
-    "$(IMAGE_PREFIX)cloud/vzconn_server_image:$(BUNDLE_VERSION)": "//src/cloud/vzconn:vzconn_server_image",
-    "$(IMAGE_PREFIX)cloud/vzmgr_server_image:$(BUNDLE_VERSION)": "//src/cloud/vzmgr:vzmgr_server_image",
+    "$(IMAGE_PREFIX)/cloud/api_server_image:$(BUNDLE_VERSION)": "//src/cloud/api:api_server_image",
+    "$(IMAGE_PREFIX)/cloud/artifact_tracker_server_image:$(BUNDLE_VERSION)": "//src/cloud/artifact_tracker:artifact_tracker_server_image",
+    "$(IMAGE_PREFIX)/cloud/auth_server_image:$(BUNDLE_VERSION)": "//src/cloud/auth:auth_server_image",
+    "$(IMAGE_PREFIX)/cloud/config_manager_server_image:$(BUNDLE_VERSION)": "//src/cloud/config_manager:config_manager_server_image",
+    "$(IMAGE_PREFIX)/cloud/cron_script_server_image:$(BUNDLE_VERSION)": "//src/cloud/cron_script:cron_script_server_image",
+    "$(IMAGE_PREFIX)/cloud/indexer_server_image:$(BUNDLE_VERSION)": "//src/cloud/indexer:indexer_server_image",
+    "$(IMAGE_PREFIX)/cloud/metrics_server_image:$(BUNDLE_VERSION)": "//src/cloud/metrics:metrics_server_image",
+    "$(IMAGE_PREFIX)/cloud/plugin/load_db:$(BUNDLE_VERSION)": "//src/cloud/plugin/load_db:plugin_db_updater_image",
+    "$(IMAGE_PREFIX)/cloud/plugin_server_image:$(BUNDLE_VERSION)": "//src/cloud/plugin:plugin_server_image",
+    "$(IMAGE_PREFIX)/cloud/profile_server_image:$(BUNDLE_VERSION)": "//src/cloud/profile:profile_server_image",
+    "$(IMAGE_PREFIX)/cloud/project_manager_server_image:$(BUNDLE_VERSION)": "//src/cloud/project_manager:project_manager_server_image",
+    "$(IMAGE_PREFIX)/cloud/proxy_server_image:$(BUNDLE_VERSION)": "//src/cloud/proxy:proxy_server_image",
+    "$(IMAGE_PREFIX)/cloud/scriptmgr_server_image:$(BUNDLE_VERSION)": "//src/cloud/scriptmgr:scriptmgr_server_image",
+    "$(IMAGE_PREFIX)/cloud/vzconn_server_image:$(BUNDLE_VERSION)": "//src/cloud/vzconn:vzconn_server_image",
+    "$(IMAGE_PREFIX)/cloud/vzmgr_server_image:$(BUNDLE_VERSION)": "//src/cloud/vzmgr:vzmgr_server_image",
 }
 
 kustomize_build(
@@ -51,11 +51,10 @@ kustomize_build(
     ),
     kustomization = "staging/kustomization.yaml",
     replacements = image_replacements(
-        existing_prefix = DEV_PREFIX,
         image_map = CLOUD_IMAGE_TO_LABEL,
-        new_prefix = PROPRIETARY_PREFIX,
     ),
     toolchains = [
+        "//k8s:image_prefix",
         "//k8s:bundle_version",
     ],
 )
@@ -72,11 +71,10 @@ kustomize_build(
     ),
     kustomization = "prod/kustomization.yaml",
     replacements = image_replacements(
-        existing_prefix = DEV_PREFIX,
         image_map = CLOUD_IMAGE_TO_LABEL,
-        new_prefix = PROPRIETARY_PREFIX,
     ),
     toolchains = [
+        "//k8s:image_prefix",
         "//k8s:bundle_version",
     ],
 )

diff --git a/k8s/cloud/base/api_deployment.yaml b/k8s/cloud/base/api_deployment.yaml
@@ -15,7 +15,7 @@ spec:
       containers:
       - name: api-server
         imagePullPolicy: IfNotPresent
-        image: gcr.io/pixie-oss/pixie-dev/cloud/api_server_image
+        image: cloud/api_server_image
         ports:
         - containerPort: 51200
           name: http2

diff --git a/k8s/cloud/base/artifact_tracker_deployment.yaml b/k8s/cloud/base/artifact_tracker_deployment.yaml
@@ -17,7 +17,7 @@ spec:
       containers:
       - name: artifact-tracker-server
         imagePullPolicy: IfNotPresent
-        image: gcr.io/pixie-oss/pixie-dev/cloud/artifact_tracker_server_image
+        image: cloud/artifact_tracker_server_image
         ports:
         - containerPort: 50750
           name: http2

diff --git a/k8s/cloud/base/auth_deployment.yaml b/k8s/cloud/base/auth_deployment.yaml
@@ -17,7 +17,7 @@ spec:
       containers:
       - name: auth-server
         imagePullPolicy: IfNotPresent
-        image: gcr.io/pixie-oss/pixie-dev/cloud/auth_server_image
+        image: cloud/auth_server_image
         ports:
         - containerPort: 50100
           name: http2

diff --git a/k8s/cloud/base/config_manager_deployment.yaml b/k8s/cloud/base/config_manager_deployment.yaml
@@ -15,7 +15,7 @@ spec:
       containers:
       - name: config-manager-server
         imagePullPolicy: IfNotPresent
-        image: gcr.io/pixie-oss/pixie-dev/cloud/config_manager_server_image
+        image: cloud/config_manager_server_image
         ports:
         - containerPort: 50500
           name: http2

diff --git a/k8s/cloud/base/cron_script_deployment.yaml b/k8s/cloud/base/cron_script_deployment.yaml
@@ -17,7 +17,7 @@ spec:
       containers:
       - name: cron-script-server
         imagePullPolicy: IfNotPresent
-        image: gcr.io/pixie-oss/pixie-dev/cloud/cron_script_server_image
+        image: cloud/cron_script_server_image
         ports:
         - containerPort: 50700
           name: http2

diff --git a/k8s/cloud/base/indexer_deployment.yaml b/k8s/cloud/base/indexer_deployment.yaml
@@ -22,7 +22,7 @@ spec:
       containers:
       - name: indexer-server
         imagePullPolicy: IfNotPresent
-        image: gcr.io/pixie-oss/pixie-dev/cloud/indexer_server_image
+        image: cloud/indexer_server_image
         ports:
         - containerPort: 51800
           name: http2

diff --git a/k8s/cloud/base/metrics_deployment.yaml b/k8s/cloud/base/metrics_deployment.yaml
@@ -20,7 +20,7 @@ spec:
       containers:
       - name: metrics-server
         imagePullPolicy: IfNotPresent
-        image: gcr.io/pixie-oss/pixie-dev/cloud/metrics_server_image
+        image: cloud/metrics_server_image
         ports:
         - containerPort: 50800
           name: http2

diff --git a/k8s/cloud/base/plugin_deployment.yaml b/k8s/cloud/base/plugin_deployment.yaml
@@ -17,7 +17,7 @@ spec:
       containers:
       - name: plugin-server
         imagePullPolicy: IfNotPresent
-        image: gcr.io/pixie-oss/pixie-dev/cloud/plugin_server_image
+        image: cloud/plugin_server_image
         ports:
         - containerPort: 50600
           name: http2

diff --git a/k8s/cloud/base/profile_deployment.yaml b/k8s/cloud/base/profile_deployment.yaml
@@ -17,7 +17,7 @@ spec:
       containers:
       - name: profile-server
         imagePullPolicy: IfNotPresent
-        image: gcr.io/pixie-oss/pixie-dev/cloud/profile_server_image
+        image: cloud/profile_server_image
         ports:
         - containerPort: 51500
           name: http2

diff --git a/k8s/cloud/base/project_manager_deployment.yaml b/k8s/cloud/base/project_manager_deployment.yaml
@@ -17,7 +17,7 @@ spec:
       containers:
       - name: project-manager-server
         imagePullPolicy: IfNotPresent
-        image: gcr.io/pixie-oss/pixie-dev/cloud/project_manager_server_image
+        image: cloud/project_manager_server_image
         ports:
         - containerPort: 50300
           name: http2