Merge branch 'envoyproxy:main' into master

.azure-pipelines/bazel.yml

@@ -7,12 +7,33 @@ parameters:
   displayName: "Suffix of artifact"
   type: string
   default: ""
+
+# caching
+- name: cacheKeyDocker
+  type: string
+  default: ".devcontainer/Dockerfile"
+- name: cacheKeyDockerVersion
+  type: string
+  default: $(cacheKeyBuildImage)
+- name: cacheKeyDockerName
+  type: string
+  default: envoy_build_image
+- name: cacheKeyDockerPath
+  type: string
+  default: /mnt/docker
+- name: cacheKeyDockerTmpDir
+  type: string
+  default: /mnt/docker_cache
+- name: cacheKeyDockerNoTmpfs
+  type: string
+  default: ''
 - name: cacheKey
   type: string
   default: $(cacheKeyBazelFiles)
 - name: cacheVersion
   type: string
   default: $(cacheKeyBazel)
+
 - name: rbe
   displayName: "Enable RBE"
   type: boolean
@@ -58,6 +79,9 @@ parameters:
   type: stepList
   default: []
 
+- name: env
+  type: object
+  default: {}
 
 steps:
 - checkout: self
@@ -125,7 +149,12 @@ steps:
   condition: and(not(canceled()), eq(variables.BAZEL_CACHE_RESTORED, 'true'))
 - template: cached.yml
   parameters:
-    version: "$(cacheKeyBuildImage)"
+    key: "${{ parameters.cacheKeyDocker }}"
+    version: "${{ parameters.cacheKeyDockerVersion }}"
+    name: "${{ parameters.cacheKeyDockerName }}"
+    path: "${{ parameters.cacheKeyDockerPath }}"
+    tmpDirectory: "${{ parameters.cacheKeyDockerTmpDir }}"
+    tmpNoTmpfs: "${{ parameters.cacheKeyDockerNoTmpfs }}"
     arch: "${{ parameters.artifactSuffix }}"
 
 - ${{ each step in parameters.stepsPre }}:
@@ -179,6 +208,8 @@ steps:
         BAZEL_REMOTE_INSTANCE_BRANCH: "$(System.PullRequest.TargetBranch)"
       ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
         BAZEL_REMOTE_INSTANCE_BRANCH: "$(Build.SourceBranchName)"
+    ${{ each var in parameters.env }}:
+      ${{ var.key }}: ${{ var.value }}
   displayName: "Run CI script ${{ parameters.ciTarget }}"
 
 - bash: |

.azure-pipelines/cached.yml

@@ -15,6 +15,9 @@ parameters:
 - name: tmpDirectory
   type: string
   default: /mnt/docker_cache
+- name: tmpNoTmpfs
+  type: string
+  default:
 - name: path
   type: string
   default: /mnt/docker
@@ -27,7 +30,7 @@ parameters:
 
 
 steps:
-- script: sudo .azure-pipelines/docker/prepare_cache.sh "${{ parameters.tmpDirectory }}"
+- script: sudo .azure-pipelines/docker/prepare_cache.sh "${{ parameters.tmpDirectory }}" "${{ parameters.tmpNoTmpfs }}"
   displayName: "Cache/prepare (${{ parameters.name }})"
 - task: Cache@2
   env:

.azure-pipelines/docker/load_cache.sh

@@ -43,9 +43,13 @@ fi
 echo "Starting Docker daemon ..."
 systemctl start docker
 
-echo "Unmount cache tmp ${DOCKER_CACHE_PATH} ..."
-umount "${DOCKER_CACHE_PATH}"
-
+if mountpoint -q "${DOCKER_CACHE_PATH}"; then
+    echo "Unmount cache tmp ${DOCKER_CACHE_PATH} ..."
+    umount "${DOCKER_CACHE_PATH}"
+else
+    echo "Remove cache tmp ${DOCKER_CACHE_PATH} ..."
+    rm -rf "${DOCKER_CACHE_PATH}"
+fi
 docker images
 df -h
 

.azure-pipelines/docker/prepare_cache.sh

@@ -1,6 +1,7 @@
 #!/bin/bash -e
 
 DOCKER_CACHE_PATH="$1"
+NO_MOUNT_TMPFS="${2:-}"
 DOCKER_CACHE_OWNERSHIP="vsts:vsts"
 
 
@@ -13,7 +14,10 @@ if ! id -u vsts &> /dev/null; then
     DOCKER_CACHE_OWNERSHIP=azure-pipelines
 fi
 
-echo "Mounting tmpfs cache directory (${DOCKER_CACHE_PATH}) ..."
+echo "Creating cache directory (${DOCKER_CACHE_PATH}) ..."
 mkdir -p "${DOCKER_CACHE_PATH}"
-mount -t tmpfs none "${DOCKER_CACHE_PATH}"
+if [[ -z "$NO_MOUNT_TMPFS" ]]; then
+    echo "Mount tmpfs directory: ${DOCKER_CACHE_PATH}"
+    mount -t tmpfs none "$DOCKER_CACHE_PATH"
+fi
 chown -R "$DOCKER_CACHE_OWNERSHIP" "${DOCKER_CACHE_PATH}"

.azure-pipelines/docker/save_cache.sh

@@ -1,6 +1,7 @@
 #!/bin/bash -e
 
 DOCKER_CACHE_PATH="$1"
+NO_MOUNT_TMPFS="${2:-}"
 
 if [[ -z "$DOCKER_CACHE_PATH" ]]; then
     echo "prime_docker_cache called without path arg" >&2
@@ -19,9 +20,13 @@ docker images
 echo "Stopping Docker ..."
 systemctl stop docker
 
-echo "Creating tmpfs directory to save tarball: ${DOCKER_CACHE_PATH}"
+echo "Creating directory to save tarball: ${DOCKER_CACHE_PATH}"
 mkdir -p "$DOCKER_CACHE_PATH"
-mount -t tmpfs none "$DOCKER_CACHE_PATH"
+
+if [[ -z "$NO_MOUNT_TMPFS" ]]; then
+    echo "Mount tmpfs directory: ${DOCKER_CACHE_PATH}"
+    mount -t tmpfs none "$DOCKER_CACHE_PATH"
+fi
 
 echo "Creating tarball: /var/lib/docker -> ${DOCKER_CACHE_TARBALL}"
 tar cf - -C /var/lib/docker . | zstd - -T0 -o "$DOCKER_CACHE_TARBALL"

.azure-pipelines/env.yml

@@ -122,6 +122,8 @@ jobs:
   - bash: |
       set -e
       VERSION_DEV="$(cat VERSION.txt | cut -d- -f2)"
+      VERSION_PATCH="$(cat VERSION.txt | cut -d- -f1 | rev | cut -d. -f1 | rev)"
+      echo "##vso[task.setvariable variable=versionPatch;isoutput=true]$VERSION_PATCH"
       if [[ $VERSION_DEV == "dev" ]]; then
           echo "##vso[task.setvariable variable=isDev;isoutput=true]true"
       else
@@ -180,16 +182,48 @@ jobs:
       # TODO(phlax): move this to a script to ensure proper linting etc
       set -e
 
-      PUBLISH_GITHUB_RELEASE=false
-      PUBLISH_NETLIFY=false
-      if [[ -n "$POSTSUBMIT" && "$NOSYNC" != true ]]; then
-          PUBLISH_NETLIFY=true
+      PUBLISH_GITHUB_RELEASE=$(run.packaging)
+      # NB: leave this empty as it is checked `-n`
+      PUBLISH_GITHUB_RELEASE_DRY_RUN=
+      PUBLISH_DOCKERHUB=false
+      PUBLISH_DOCS=false
+      PUBLISH_DOCS_LATEST=false
+      PUBLISH_DOCS_RELEASE=false
+
+      if [[ "$ISSTABLEBRANCH" == True && -n "$POSTSUBMIT" && "$NOSYNC" != true ]]; then
+          # Build docs for publishing either latest or a release build
+          PUBLISH_DOCS=true
+          # main
+          if [[ "$ISMAIN" == True ]]; then
+              # Update the Dockerhub README
+              PUBLISH_DOCKERHUB=true
+              if [[ "$(state.isDev)" == true ]]; then
+                  # Postsubmit on `main` trigger rebuild of latest docs
+                  PUBLISH_DOCS_LATEST=true
+              fi
+          # Not main, and not -dev
+          elif [[ "$(state.isDev)" == false ]]; then
+              if [[ "$(state.versionPatch)" -eq 0 ]]; then
+                  # A just-forked branch
+                  PUBLISH_GITHUB_RELEASE=false
+              fi
+              # A stable release, publish docs to the release
+              PUBLISH_DOCS_RELEASE=true
+          else
+              # Postsubmit for non-main/release, skip publishing docs in this case
+              PUBLISH_DOCS=false
+          fi
       fi
-      if [[ -n "$POSTSUBMIT" && "$NOSYNC" != true && "$ISSTABLEBRANCH" == True && "$(state.isdev)" == false ]]; then
-          PUBLISH_GITHUB_RELEASE=true
+      if [[ -z "$POSTSUBMIT" ]]; then
+          PUBLISH_GITHUB_RELEASE_DRY_RUN=true
       fi
+
       echo "##vso[task.setvariable variable=githubRelease;isoutput=true]${PUBLISH_GITHUB_RELEASE}"
-      echo "##vso[task.setvariable variable=netlify;isoutput=true]${PUBLISH_NETLIFY}"
+      echo "##vso[task.setvariable variable=githubReleaseDryRun;isoutput=true]${PUBLISH_GITHUB_RELEASE_DRY_RUN}"
+      echo "##vso[task.setvariable variable=dockerhub;isoutput=true]${PUBLISH_DOCKERHUB}"
+      echo "##vso[task.setvariable variable=docs;isoutput=true]${PUBLISH_DOCS}"
+      echo "##vso[task.setvariable variable=docsLatest;isoutput=true]${PUBLISH_DOCS_LATEST}"
+      echo "##vso[task.setvariable variable=docsRelease;isoutput=true]${PUBLISH_DOCS_RELEASE}"
 
     displayName: "Decide what to publish"
     workingDirectory: $(Build.SourcesDirectory)
@@ -205,12 +239,18 @@ jobs:
       echo "env.outputs['changed.requirements']: $(changed.requirements)"
       echo
       echo "env.outputs['state.isDev']: $(state.isDev)"
+      echo "env.outputs['state.versionPatch']: $(state.versionPatch)"
       echo
       echo "env.outputs['run.build']: $(run.build)"
       echo "env.outputs['run.checks']: $(run.checks)"
+      echo "env.outputs['run.packaging']: $(run.packaging)"
       echo
       echo "env.outputs['publish.githubRelease']: $(publish.githubRelease)"
-      echo "env.outputs['publish.netlify']: $(publish.netlify)"
+      echo "env.outputs['publish.githubReleaseDryRun']: $(publish.githubReleaseDryRun)"
+      echo "env.outputs['publish.dockerhub]: $(publish.dockerhub)"
+      echo "env.outputs['publish.docs]: $(publish.docs)"
+      echo "env.outputs['publish.docsLatest]: $(publish.docsLatest)"
+      echo "env.outputs['publish.docsRelease]: $(publish.docsRelease)"
 
     displayName: "Print build environment"