Merge commit 'ece844a103eb0561a2bd1bf91129c12915a28ea8' into jackies/…

…upgrade-bazel-buildfarm-to-v2.10.2

.bazelci/cache_test.sh

@@ -5,7 +5,7 @@
 # We ensure that the system can build a set of bazel targets.
 
 # Run redis container
-docker run -d --name buildfarm-redis --network host redis:5.0.9 --bind localhost
+docker run -d --name buildfarm-redis --network host redis:7.2.4 --bind localhost
 
 # Build a container for buildfarm services
 cp `which bazel` bazel

.bazelci/docker_unit_test.sh

@@ -4,4 +4,4 @@
 # Build a container for unit tests and run them
 cp `which bazel` bazel
 docker build -t buildfarm .
-docker run buildfarm /bin/bash -c "cd buildfarm; ./bazel test --build_tests_only --test_tag_filters=-integration,-redis ..."
+docker run buildfarm /bin/bash -c "cd buildfarm; ./bazel test --build_tests_only --test_tag_filters=-container,-integration,-redis ..."

.bazelci/format.sh

@@ -5,11 +5,9 @@
 
 FORMAT_JAVA=true
 REMOVE_NEWLINES_AFTER_START_BRACKET=true
-JAVA_FORMATTER_URL=https://github.com/google/google-java-format/releases/download/google-java-format-1.7/google-java-format-1.7-all-deps.jar
+JAVA_FORMATTER_URL=https://github.com/google/google-java-format/releases/download/v1.20.0/google-java-format-1.20.0-all-deps.jar
 LOCAL_FORMATTER="java_formatter.jar"
 
-FORMAT_PROTO=true
-CLANG_FORMAT=@llvm_toolchain//:clang-format
 if [ -z "$BAZEL" ]; then
   BAZEL=bazel
 fi
@@ -71,20 +69,6 @@ run_java_formatter () {
     java -jar $LOCAL_FORMATTER -i $files
 }
 
-run_proto_formatter () {
-    # Check whether any formatting changes need to be made.
-    # This is intended to be done by the CI.
-    if [[ "$@" == "--check" ]]
-    then
-        find $PWD -name '*.proto' -exec $BAZEL run $CLANG_FORMAT -- -i --dry-run --Werror {} +
-        handle_format_error_check
-        return
-    fi
-
-    # Fixes formatting issues
-    find $PWD -name '*.proto' -exec $BAZEL run $CLANG_FORMAT -- -i {} +
-}
-
 run_buildifier () {
     $BAZEL run $BUILDIFIER -- -r > /dev/null 2>&1
 }
@@ -93,10 +77,6 @@ if [ "${FORMAT_JAVA:-false}" = true ]; then
     run_java_formatter "$@"
 fi;
 
-if [ "${FORMAT_PROTO:-false}" = true ]; then
-    run_proto_formatter "$@"
-fi;
-
 if [ "${FORMAT_BUILD:-false}" = true ]; then
     run_buildifier "$@"
 fi;

.bazelci/integration_test.sh

@@ -5,7 +5,7 @@
 # We ensure that the system can build a set of bazel targets.
 
 # Run redis container
-docker run -d --name buildfarm-redis --network host redis:5.0.9 --bind localhost
+docker run -d --name buildfarm-redis --network host redis:7.2.4 --bind localhost
 
 # Build a container for buildfarm services
 cp `which bazel` bazel

.bazelci/presubmit.yml

@@ -1,5 +1,5 @@
 ---
-buildifier: 4.2.5
+buildifier: 6.4.0
 tasks:
 
   # Linting jobs
@@ -41,6 +41,8 @@ tasks:
     name: "Unit Tests"
     build_targets:
     - "..."
+    build_flags:
+    - "--build_tag_filters=-container"
     test_flags:
     - "--test_tag_filters=-integration,-redis"
     test_targets:
@@ -49,28 +51,34 @@ tasks:
     name: "Unit Tests"
     build_targets:
     - "..."
+    build_flags:
+    - "--build_tag_filters=-container"
     test_flags:
     - "--test_tag_filters=-integration,-redis"
     test_targets:
     - "..."
   macos:
     name: "Unit Tests"
+    environment:
+      USE_BAZEL_VERSION: 17be878292730359c9c90efdceabed26126df7ae
     build_flags:
+    - "--cxxopt=-std=c++14"
     - "--build_tag_filters=-container"
     build_targets:
     - "..."
     test_flags:
-    - "--test_tag_filters=-integration,-redis"
+    - "--test_tag_filters=-container,-integration,-redis"
     test_targets:
     - "..."
   windows:
     name: "Unit Tests"
     build_flags:
-    - "--build_tag_filters=-container,-audit"
+    - "--build_tag_filters=-container"
     build_targets:
     - "..."
     test_flags:
-    - "--test_tag_filters=-integration,-redis"
+    - "--@rules_jvm_external//settings:stamp_manifest=False"
+    - "--test_tag_filters=-container,-integration,-redis"
     test_targets:
     - "..."
   rpm_builds:

.bazelci/redis_unit_tests.sh

@@ -3,7 +3,9 @@
 # However this runs unit tests that interact directly with redis.
 
 # Run redis container
-docker run -d --rm --name buildfarm-redis --network host redis:5.0.9 --bind localhost
+docker run -d --rm --name buildfarm-redis --network host redis:7.2.4 --bind localhost
 
 # Run tests that rely on redis
 bazel test --build_tests_only --test_tag_filters=redis src/test/java/...
+
+docker stop buildfarm-redis

.bazelci/run_server_test.sh

@@ -1,18 +1,18 @@
 #!/bin/bash
 
 # Start redis container
-docker run -d --rm --name buildfarm-redis --network host redis:5.0.9 --bind localhost
+docker run -d --rm --name buildfarm-redis --network host redis:7.2.4 --bind localhost
 
 # Build worker and server targets
 bazel build //src/main/java/build/buildfarm:buildfarm-shard-worker
 bazel build //src/main/java/build/buildfarm:buildfarm-server
 
 # Start a single worker
-bazel run //src/main/java/build/buildfarm:buildfarm-shard-worker $(pwd)/examples/config.minimal.yml > server.log 2>&1 &
+bazel run //src/main/java/build/buildfarm:buildfarm-shard-worker $(pwd)/examples/config.minimal.yml > worker.log 2>&1 &
 echo "Started buildfarm-shard-worker..."
 
 # Start a single server
-bazel run //src/main/java/build/buildfarm:buildfarm-server $(pwd)/examples/config.minimal.yml > worker.log 2>&1 &
+bazel run //src/main/java/build/buildfarm:buildfarm-server $(pwd)/examples/config.minimal.yml > server.log 2>&1 &
 echo "Started buildfarm-server..."
 
 echo "Wait for startup to finish..."

.bazelrc

@@ -1,3 +1,10 @@
+build --java_language_version=17
+build --java_runtime_version=remotejdk_17
+
+build --tool_java_language_version=17
+build --tool_java_runtime_version=remotejdk_17
+
+
 common --enable_platform_specific_config
 
 build:fuse --define=fuse=true
@@ -14,3 +21,11 @@ test --test_tag_filters=-redis,-integration
 # Ensure buildfarm is compatible with future versions of bazel.
 # https://buildkite.com/bazel/bazelisk-plus-incompatible-flags
 common --incompatible_disallow_empty_glob
+
+common --enable_bzlmod
+
+# See also https://bazel.build/external/lockfile.
+common --lockfile_mode=off
+# It's off because we have mac/windows/linux developers who may not have access
+# to all three to update the platform-specific bits of the lockfile.
+

.bazelversion

@@ -1 +1 @@
-6.1.2
+7.1.1

.github/workflows/buildfarm-helm-chart-lint.yml

@@ -0,0 +1,24 @@
+---
+name: Lint Helm Chart
+
+on:
+  push:
+    paths:
+      - kubernetes/helm-charts/buildfarm/**
+
+env:
+  CHART_ROOT: kubernetes/helm-charts/buildfarm
+
+jobs:
+  lint:
+    name: Lint Helm Chart
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - id: helm-lint
+        name: Lint Helm Chart
+        run: |-
+          set -ex
+          helm dep up "${CHART_ROOT}"
+          helm lint "${CHART_ROOT}"

.github/workflows/buildfarm-helm-chart-publish.yml

@@ -0,0 +1,58 @@
+---
+name: Package and Publish Helm Chart
+
+on:
+  push:
+    tags:
+      - 'helm/*'
+
+env:
+  CHART_NAME: buildfarm
+  CHART_ROOT: ${{ github.workspace }}/kubernetes/helm-charts/buildfarm
+  GHCR_REPO: ghcr.io/${{ github.repository_owner }}
+
+jobs:
+  build:
+    name: Lint, Package, and Release BuildFarm Helm Chart
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - id: get-chart-ver
+        name: Extracting Helm Chart Version from Tag
+        run: |
+          set -ex
+          echo "chart_ver=$(echo $GITHUB_REF | cut -d / -f 4)" >> $GITHUB_OUTPUT
+      - id: set-chart-yaml-version
+        name: Etching Helm Chart Version into Chart.yaml for Packaging
+        run: |
+          set -ex
+          echo setting Chart version to \
+            "${{ steps.get-chart-ver.outputs.chart_ver }}" \
+            in ${CHART_ROOT}/Chart.yaml
+          yq -i \
+            '.version |= "${{ steps.get-chart-ver.outputs.chart_ver }}"' \
+            ${CHART_ROOT}/Chart.yaml
+      - id: helm-lint
+        name: Helm Chart Lint
+        run: |-
+          set -ex
+          helm dep up "${CHART_ROOT}"
+          helm lint "${CHART_ROOT}"
+      - id: helm-bundle-push
+        name: Helm Chart Bundle and Push
+        run: |-
+          set -e
+          echo ${{ secrets.GITHUB_TOKEN }} | \
+            helm registry \
+              login "${GHCR_REPO}" \
+              --username "${{ github.repository_owner }}" \
+              --password-stdin
+          set -ex
+          helm dep up "${CHART_ROOT}"
+          helm package "${CHART_ROOT}"
+          export CHART_BUNDLE="${CHART_NAME}-${{ steps.get-chart-ver.outputs.chart_ver }}.tgz"
+          ls -l "${CHART_BUNDLE}"
+          helm push \
+            "${CHART_BUNDLE}" \
+            "oci://${GHCR_REPO}"

.github/workflows/buildfarm-images-build-and-deploy.yml

@@ -0,0 +1,31 @@
+name: Build and Push Latest Buildfarm Images
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    if: github.repository == 'bazelbuild/bazel-buildfarm'
+    name: Build Buildfarm Images
+    runs-on: ubuntu-latest
+    steps:
+      - uses: bazelbuild/setup-bazelisk@v2
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Login to Bazelbuild Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.BAZELBUILD_DOCKERHUB_USERNAME }}
+          password: ${{ secrets.BAZELBUILD_DOCKERHUB_TOKEN }}
+
+      - name: Build Server Image
+        id: buildAndPushServerImage
+        run: bazel run public_push_buildfarm-server -- --tag latest
+
+      - name: Build Worker Image
+        id: buildAndPushWorkerImage
+        run: bazel run public_push_buildfarm-worker -- --tag latest

.github/workflows/buildfarm-release-build-and-deploy.yml

@@ -0,0 +1,30 @@
+name: Build and Push Buildfarm Releases
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build:
+    if: github.repository == 'bazelbuild/bazel-buildfarm'
+    name: Build Buildfarm Images
+    runs-on: ubuntu-latest
+    steps:
+      - uses: bazelbuild/setup-bazelisk@v2
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Login to Bazelbuild Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.BAZELBUILD_DOCKERHUB_USERNAME }}
+          password: ${{ secrets.BAZELBUILD_DOCKERHUB_TOKEN }}
+
+      - name: Build Server Image
+        id: buildAndPushServerImage
+        run: bazel run public_push_buildfarm-server -- --tag ${{ github.event.release.tag_name }}
+
+      - name: Build Worker Image
+        id: buildAndPushWorkerImage
+        run: bazel run public_push_buildfarm-worker -- --tag ${{ github.event.release.tag_name }}

.github/workflows/buildfarm-worker-base-build-and-deploy.yml

@@ -0,0 +1,39 @@
+name: Build and Push Base Buildfarm Worker Images
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - ci/base-worker-image/jammy/Dockerfile
+      - ci/base-worker-image/mantic/Dockerfile
+jobs:
+  build:
+    if: github.repository == 'bazelbuild/bazel-buildfarm'
+    name: Build Base Buildfarm Worker Image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Login to Bazelbuild Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.BAZELBUILD_DOCKERHUB_USERNAME }}
+          password: ${{ secrets.BAZELBUILD_DOCKERHUB_TOKEN }}
+
+      - name: Build Jammy Docker image
+        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        with:
+          context: .
+          file: ./ci/base-worker-image/jammy/Dockerfile
+          push: true
+          tags: bazelbuild/buildfarm-worker-base:jammy
+
+      - name: Build Mantic Docker image
+        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        with:
+          context: .
+          file: ./ci/base-worker-image/mantic/Dockerfile
+          push: true
+          tags: bazelbuild/buildfarm-worker-base:mantic